From 116d16828609f61df0b50a431494d98aaef04eb6 Mon Sep 17 00:00:00 2001 From: Alistair Miles Date: Wed, 6 Dec 2017 09:15:49 +0000 Subject: [PATCH 01/15] bump numcodecs --- requirements_dev.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements_dev.txt b/requirements_dev.txt index 5435d1a777..dc7b7d10b1 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -15,7 +15,7 @@ mccabe==0.6.1 monotonic==1.3 msgpack-python==0.4.8 nose==1.3.7 -numcodecs==0.4.1 +numcodecs==0.5.0 numpy==1.13.3 packaging==16.8 pkginfo==1.4.1 diff --git a/setup.py b/setup.py index 4f615d5144..da0cff8042 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ 'asciitree', 'numpy>=1.7', 'fasteners', - 'numcodecs>=0.4.1', + 'numcodecs>=0.5.0', ], package_dir={'': '.'}, packages=['zarr', 'zarr.tests'], From e584f686936ffd2f075cef42ead83da6ab60914e Mon Sep 17 00:00:00 2001 From: Alistair Miles Date: Wed, 6 Dec 2017 09:29:08 +0000 Subject: [PATCH 02/15] add vlen tests --- zarr/tests/test_core.py | 52 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 48 insertions(+), 4 deletions(-) diff --git a/zarr/tests/test_core.py b/zarr/tests/test_core.py index 47c3f3e3c8..4b4260a0d3 100644 --- a/zarr/tests/test_core.py +++ b/zarr/tests/test_core.py @@ -24,7 +24,8 @@ from zarr.compat import PY2 from zarr.util import buffer_size from numcodecs import (Delta, FixedScaleOffset, Zlib, Blosc, BZ2, MsgPack, Pickle, - Categorize, JSON) + Categorize, JSON, VLenUTF8, VLenBytes, VLenArray) +from numcodecs.tests.common import greetings # needed for PY2/PY3 consistent behaviour @@ -932,11 +933,14 @@ def test_object_arrays(self): a = z[:] assert a.dtype == object - def test_object_arrays_text(self): + def test_object_arrays_vlen_text(self): - from numcodecs.tests.common import greetings data = np.array(greetings * 1000, dtype=object) + z = self.create_array(shape=data.shape, dtype=object, object_codec=VLenUTF8()) + z[:] = data + assert_array_equal(data, z[:]) + z = self.create_array(shape=data.shape, dtype=object, object_codec=MsgPack()) z[:] = data assert_array_equal(data, z[:]) @@ -954,6 +958,38 @@ def test_object_arrays_text(self): z[:] = data assert_array_equal(data, z[:]) + def test_object_arrays_vlen_bytes(self): + + greetings_bytes = [g.encode('utf8') for g in greetings] + data = np.array(greetings_bytes * 1000, dtype=object) + + z = self.create_array(shape=data.shape, dtype=object, object_codec=VLenBytes()) + z[:] = data + assert_array_equal(data, z[:]) + + z = self.create_array(shape=data.shape, dtype=object, object_codec=Pickle()) + z[:] = data + assert_array_equal(data, z[:]) + + def test_object_arrays_vlen_array(self): + + data = np.array([np.array([1, 3, 7]), + np.array([5]), + np.array([2, 8, 12])] * 1000, dtype=object) + + codecs = VLenArray(int), VLenArray(' Date: Wed, 6 Dec 2017 12:21:00 +0000 Subject: [PATCH 03/15] add object dtype convenience API --- zarr/storage.py | 8 +++----- zarr/tests/test_core.py | 43 ++++++++++++++++++++++++++++++++--------- zarr/util.py | 43 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 80 insertions(+), 14 deletions(-) diff --git a/zarr/storage.py b/zarr/storage.py index bf821f82ab..4d25e6a634 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -25,7 +25,8 @@ from zarr.util import (normalize_shape, normalize_chunks, normalize_order, - normalize_storage_path, buffer_size, normalize_fill_value, nolock) + normalize_storage_path, buffer_size, + normalize_fill_value, nolock, normalize_dtype) from zarr.meta import encode_array_metadata, encode_group_metadata from zarr.compat import PY2, binary_type from numcodecs.registry import codec_registry @@ -308,10 +309,7 @@ def _init_array_metadata(store, shape, chunks=None, dtype=None, compressor='defa # normalize metadata shape = normalize_shape(shape) - dtype = np.dtype(dtype) - if dtype.kind in 'mM': - raise ValueError('datetime64 and timedelta64 dtypes are not currently supported; ' - 'please store the data using int64 instead') + dtype, object_codec = normalize_dtype(dtype, object_codec) chunks = normalize_chunks(chunks, shape, dtype.itemsize) order = normalize_order(order) fill_value = normalize_fill_value(fill_value, dtype) diff --git a/zarr/tests/test_core.py b/zarr/tests/test_core.py index 4b4260a0d3..cba492ccb8 100644 --- a/zarr/tests/test_core.py +++ b/zarr/tests/test_core.py @@ -21,7 +21,7 @@ DBMStore, LMDBStore, atexit_rmtree, atexit_rmglob) from zarr.core import Array from zarr.errors import PermissionError -from zarr.compat import PY2 +from zarr.compat import PY2, text_type, binary_type from zarr.util import buffer_size from numcodecs import (Delta, FixedScaleOffset, Zlib, Blosc, BZ2, MsgPack, Pickle, Categorize, JSON, VLenUTF8, VLenBytes, VLenArray) @@ -941,6 +941,13 @@ def test_object_arrays_vlen_text(self): z[:] = data assert_array_equal(data, z[:]) + # convenience API + z = self.create_array(shape=data.shape, dtype=text_type) + assert z.dtype == object + assert isinstance(z.filters[0], VLenUTF8) + z[:] = data + assert_array_equal(data, z[:]) + z = self.create_array(shape=data.shape, dtype=object, object_codec=MsgPack()) z[:] = data assert_array_equal(data, z[:]) @@ -967,6 +974,13 @@ def test_object_arrays_vlen_bytes(self): z[:] = data assert_array_equal(data, z[:]) + # convenience API + z = self.create_array(shape=data.shape, dtype=binary_type) + assert z.dtype == object + assert isinstance(z.filters[0], VLenBytes) + z[:] = data + assert_array_equal(data, z[:]) + z = self.create_array(shape=data.shape, dtype=object, object_codec=Pickle()) z[:] = data assert_array_equal(data, z[:]) @@ -977,18 +991,29 @@ def test_object_arrays_vlen_array(self): np.array([5]), np.array([2, 8, 12])] * 1000, dtype=object) + def compare_arrays(expected, actual, item_dtype): + assert isinstance(actual, np.ndarray) + assert actual.dtype == object + assert actual.shape == expected.shape + for e, a in zip(expected.flat, actual.flat): + assert isinstance(a, np.ndarray) + assert_array_equal(e, a) + assert a.dtype == item_dtype + codecs = VLenArray(int), VLenArray(' 1: + args = tokens[1].split(',') + else: + args = () + try: + object_codec = codec_registry[codec_id](*args) + except KeyError: + raise ValueError('codec %r for object type %r is not ' + 'available; please provide an ' + 'object_codec manually' % (codec_id, key)) + return dtype, object_codec + + dtype = np.dtype(dtype) + + if dtype.kind in 'mM': + raise ValueError('datetime64 and timedelta64 dtypes are not currently ' + 'supported; please store the data using int64 instead') + + return dtype, object_codec + + # noinspection PyTypeChecker def is_total_slice(item, shape): """Determine whether `item` specifies a complete slice of array with the From 3865229667b7e8add10f914aa0f655621d4af82a Mon Sep 17 00:00:00 2001 From: Alistair Miles Date: Wed, 6 Dec 2017 23:09:13 +0000 Subject: [PATCH 04/15] fix categorize warnings --- zarr/tests/test_filters.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zarr/tests/test_filters.py b/zarr/tests/test_filters.py index cf6d9974d4..3152de5997 100644 --- a/zarr/tests/test_filters.py +++ b/zarr/tests/test_filters.py @@ -169,8 +169,8 @@ def test_array_with_packbits_filter(): def test_array_with_categorize_filter(): # setup - data = np.random.choice([b'foo', b'bar', b'baz'], size=100) - flt = Categorize(dtype=data.dtype, labels=['foo', 'bar', 'baz']) + data = np.random.choice([u'foo', u'bar', u'baz'], size=100) + flt = Categorize(dtype=data.dtype, labels=[u'foo', u'bar', u'baz']) filters = [flt] for compressor in compressors: From 7a680e28dfc61c734b5b0bbdeb8140d3f1103498 Mon Sep 17 00:00:00 2001 From: Alistair Miles Date: Wed, 6 Dec 2017 23:09:48 +0000 Subject: [PATCH 05/15] fix categorize warnings --- zarr/tests/test_core.py | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/zarr/tests/test_core.py b/zarr/tests/test_core.py index cba492ccb8..953966f0d4 100644 --- a/zarr/tests/test_core.py +++ b/zarr/tests/test_core.py @@ -1027,24 +1027,11 @@ def test_object_arrays_danger(self): z[:] = 42 # do something else dangerous - labels = [ - '¡Hola mundo!', - 'Hej Världen!', - 'Servus Woid!', - 'Hei maailma!', - 'Xin chào thế giới', - 'Njatjeta Botë!', - 'Γεια σου κόσμε!', - 'こんにちは世界', - '世界，你好！', - 'Helló, világ!', - 'Zdravo svete!', - 'เฮลโลเวิลด์' - ] - data = labels * 10 + data = greetings * 10 for compressor in Zlib(1), Blosc(): z = self.create_array(shape=len(data), chunks=30, dtype=object, - object_codec=Categorize(labels, dtype=object), + object_codec=Categorize(greetings, + dtype=object), compressor=compressor) z[:] = data v = z.view(filters=[]) From 31b67e8153b9aa9715184e2935bdf61eda5ef3bf Mon Sep 17 00:00:00 2001 From: Alistair Miles Date: Wed, 6 Dec 2017 23:10:06 +0000 Subject: [PATCH 06/15] comment --- zarr/util.py | 1 + 1 file changed, 1 insertion(+) diff --git a/zarr/util.py b/zarr/util.py index 8940129aa4..412fd85dc2 100644 --- a/zarr/util.py +++ b/zarr/util.py @@ -132,6 +132,7 @@ def normalize_dtype(dtype, object_codec): if inspect.isclass(dtype): dtype = dtype.__name__ if isinstance(dtype, str): + # allow ':' to delimit class from codec arguments tokens = dtype.split(':') key = tokens[0] if key in object_codecs: From e59822a3cf0afb2ad0c8d1676ebfab778aa7218f Mon Sep 17 00:00:00 2001 From: Alistair Miles Date: Thu, 7 Dec 2017 00:02:06 +0000 Subject: [PATCH 07/15] support datetime64 and timedelta64 --- zarr/core.py | 25 ++++++++++++++----------- zarr/tests/test_core.py | 38 ++++++++++++++++++++++++-------------- zarr/util.py | 7 ++++--- 3 files changed, 42 insertions(+), 28 deletions(-) diff --git a/zarr/core.py b/zarr/core.py index ca8683ee36..cb9db4a78a 100644 --- a/zarr/core.py +++ b/zarr/core.py @@ -1590,7 +1590,10 @@ def _chunk_getitem(self, chunk_coords, chunk_selection, out, out_selection, if self._compressor: self._compressor.decode(cdata, dest) else: - chunk = np.frombuffer(cdata, dtype=self._dtype) + if isinstance(cdata, np.ndarray): + chunk = cdata.view(self._dtype) + else: + chunk = np.frombuffer(cdata, dtype=self._dtype) chunk = chunk.reshape(self._chunks, order=self._order) np.copyto(dest, chunk) return @@ -1736,7 +1739,7 @@ def _decode_chunk(self, cdata): elif isinstance(chunk, np.ndarray): chunk = chunk.view(self._dtype) else: - chunk = np.frombuffer(chunk, self._dtype) + chunk = np.frombuffer(chunk, dtype=self._dtype) # reshape chunk = chunk.reshape(self._chunks, order=self._order) @@ -2087,15 +2090,15 @@ def view(self, shape=None, chunks=None, dtype=None, >>> import zarr >>> import numpy as np >>> np.random.seed(42) - >>> labels = [b'female', b'male'] + >>> labels = ['female', 'male'] >>> data = np.random.choice(labels, size=10000) >>> filters = [zarr.Categorize(labels=labels, - ... dtype=data.dtype, - ... astype='u1')] + ... dtype=data.dtype, + ... astype='u1')] >>> a = zarr.array(data, chunks=1000, filters=filters) >>> a[:] - array([b'female', b'male', b'female', ..., b'male', b'male', b'female'], - dtype='|S6') + array(['female', 'male', 'female', ..., 'male', 'male', 'female'], + dtype='>> v = a.view(dtype='u1', filters=[]) >>> v.is_view True @@ -2110,10 +2113,10 @@ def view(self, shape=None, chunks=None, dtype=None, >>> v[:] array([1, 1, 1, ..., 2, 2, 2], dtype=uint8) >>> a[:] - array([b'female', b'female', b'female', ..., b'male', b'male', b'male'], - dtype='|S6') + array(['female', 'female', 'female', ..., 'male', 'male', 'male'], + dtype='>> data = np.random.randint(0, 2, size=10000, dtype='u1') >>> a = zarr.array(data, chunks=1000) @@ -2125,7 +2128,7 @@ def view(self, shape=None, chunks=None, dtype=None, >>> np.all(a[:].view(dtype=bool) == v[:]) True - An array can be viewed with a dtype with a different itemsize, however + An array can be viewed with a dtype with a different item size, however some care is needed to adjust the shape and chunk shape so that chunk data is interpreted correctly: diff --git a/zarr/tests/test_core.py b/zarr/tests/test_core.py index 953966f0d4..10e1d6522c 100644 --- a/zarr/tests/test_core.py +++ b/zarr/tests/test_core.py @@ -855,27 +855,37 @@ def test_structured_array(self): def test_dtypes(self): # integers - for t in 'u1', 'u2', 'u4', 'u8', 'i1', 'i2', 'i4', 'i8': - z = self.create_array(shape=10, chunks=3, dtype=t) - assert z.dtype == np.dtype(t) - a = np.arange(z.shape[0], dtype=t) + for dtype in 'u1', 'u2', 'u4', 'u8', 'i1', 'i2', 'i4', 'i8': + z = self.create_array(shape=10, chunks=3, dtype=dtype) + assert z.dtype == np.dtype(dtype) + a = np.arange(z.shape[0], dtype=dtype) z[:] = a assert_array_equal(a, z[:]) # floats - for t in 'f2', 'f4', 'f8': - z = self.create_array(shape=10, chunks=3, dtype=t) - assert z.dtype == np.dtype(t) - a = np.linspace(0, 1, z.shape[0], dtype=t) + for dtype in 'f2', 'f4', 'f8': + z = self.create_array(shape=10, chunks=3, dtype=dtype) + assert z.dtype == np.dtype(dtype) + a = np.linspace(0, 1, z.shape[0], dtype=dtype) z[:] = a assert_array_almost_equal(a, z[:]) - # datetime, timedelta are not supported for the time being - for resolution in 'D', 'us', 'ns': - with assert_raises(ValueError): - self.create_array(shape=10, dtype='datetime64[{}]'.format(resolution)) - with assert_raises(ValueError): - self.create_array(shape=10, dtype='timedelta64[{}]'.format(resolution)) + # datetime, timedelta + for base_type in 'Mm': + for resolution in 'D', 'us', 'ns': + dtype = '{}8[{}]'.format(base_type, resolution) + z = self.create_array(shape=100, dtype=dtype) + assert z.dtype == np.dtype(dtype) + a = np.random.randint(0, np.iinfo('u8').max, size=z.shape[0], + dtype='u8').view(dtype) + z[:] = a + assert_array_equal(a, z[:]) + + # check that datetime generic units are not allowed + with assert_raises(ValueError): + self.create_array(shape=100, dtype='M8') + with assert_raises(ValueError): + self.create_array(shape=100, dtype='m8') def test_object_arrays(self): diff --git a/zarr/util.py b/zarr/util.py index 412fd85dc2..1f20fbac1b 100644 --- a/zarr/util.py +++ b/zarr/util.py @@ -153,9 +153,10 @@ def normalize_dtype(dtype, object_codec): dtype = np.dtype(dtype) - if dtype.kind in 'mM': - raise ValueError('datetime64 and timedelta64 dtypes are not currently ' - 'supported; please store the data using int64 instead') + # don't allow generic datetime64 or timedelta64, require units to be specified + if dtype == np.dtype('M8') or dtype == np.dtype('m8'): + raise ValueError('datetime64 and timedelta64 dtypes with generic units ' + 'are not supported, please specify units (e.g., "M8[ns]")') return dtype, object_codec From bf376e858babc669369ee2229edf4f29112f6792 Mon Sep 17 00:00:00 2001 From: Alistair Miles Date: Thu, 7 Dec 2017 00:15:42 +0000 Subject: [PATCH 08/15] require date/time units --- docs/spec/v2.rst | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/docs/spec/v2.rst b/docs/spec/v2.rst index e1dcc31601..fa68b7eb0e 100644 --- a/docs/spec/v2.rst +++ b/docs/spec/v2.rst @@ -117,17 +117,20 @@ consists of 3 parts: ``">"``: big-endian; ``"|"``: not-relevant) * One character code giving the basic type of the array (``"b"``: Boolean (integer type where all values are only True or False); ``"i"``: integer; ``"u"``: unsigned - integer; ``"f"``: floating point; ``"c"``: complex floating point; ``"S"``: string - (fixed-length sequence of char); ``"U"``: unicode (fixed-length sequence of - Py_UNICODE); ``"V"``: other (void * – each item is a fixed-size chunk of memory)) + integer; ``"f"``: floating point; ``"c"``: complex floating point; ``"m"``: timedelta; + ``"M"``: datetime; ``"S"``: string (fixed-length sequence of char); ``"U"``: unicode + (fixed-length sequence of Py_UNICODE); ``"V"``: other (void * – each item is a + fixed-size chunk of memory)) * An integer specifying the number of bytes the type uses. The byte order MUST be specified. E.g., ``"i4"``, ``"|b1"`` and ``"|S12"`` are valid data type encodings. -Please note that NumPy's datetime64 ("M") and timedelta64 ("m") data types are **not** -currently supported. Please store data using an appropriate physical data type instead, -e.g., 64-bit integer. +For datetime64 ("M") and timedelta64 ("m") data types, these MUST also include the +units within square brackets. A list of valid units and their definitions are given in +the `NumPy documentation on Datetimes and Timedeltas +`_. +For example, ``" Date: Thu, 7 Dec 2017 00:16:21 +0000 Subject: [PATCH 09/15] numcodecs version bump --- requirements_dev.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements_dev.txt b/requirements_dev.txt index dc7b7d10b1..b01ba13e3f 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -15,7 +15,7 @@ mccabe==0.6.1 monotonic==1.3 msgpack-python==0.4.8 nose==1.3.7 -numcodecs==0.5.0 +numcodecs==0.5.1 numpy==1.13.3 packaging==16.8 pkginfo==1.4.1 diff --git a/setup.py b/setup.py index da0cff8042..698fc91a21 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ 'asciitree', 'numpy>=1.7', 'fasteners', - 'numcodecs>=0.5.0', + 'numcodecs>=0.5.1', ], package_dir={'': '.'}, packages=['zarr', 'zarr.tests'], From 31d3c6d40e176603e61f71e4e7d60a10fa96cf31 Mon Sep 17 00:00:00 2001 From: Alistair Miles Date: Thu, 7 Dec 2017 00:51:14 +0000 Subject: [PATCH 10/15] modify tutorial for datetime support --- docs/tutorial.rst | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 1e46a8322f..227a9ebdad 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -1079,24 +1079,13 @@ E.g., pickle/unpickle an array stored on disk:: Datetimes and timedeltas ------------------------ -Please note that NumPy's ``datetime64`` and ``timedelta64`` dtypes are **not** currently -supported for Zarr arrays. If you would like to store datetime or timedelta data, you -can store the data in an array with an integer dtype, e.g.:: +NumPy's ``datetime64`` ('M') and ``timedelta64`` ('m') dtypes are supported for Zarr +arrays, as long as the units are specified. E.g.:: - >>> a = np.array(['2007-07-13', '2006-01-13', '2010-08-13'], dtype='datetime64[D]') - >>> z = zarr.array(a.view('i8')) + >>> z = zarr.array(['2007-07-13', '2006-01-13', '2010-08-13'], dtype='M[D]') >>> z - + >>> z[:] - array([13707, 13161, 14834]) - >>> z[:].view(a.dtype) - array(['2007-07-13', '2006-01-13', '2010-08-13'], dtype='datetime64[D]') - -If you would like a convenient way to retrieve the data from this array viewed as the -original datetime64 dtype, try the :func:`zarr.core.Array.astype` method, e.g.:: - - >>> zv = z.astype(a.dtype) - >>> zv[:] array(['2007-07-13', '2006-01-13', '2010-08-13'], dtype='datetime64[D]') .. _tutorial_tips: From e385d270d1b6cc0a469ff68f8397004a957b38c8 Mon Sep 17 00:00:00 2001 From: Alistair Miles Date: Thu, 7 Dec 2017 01:33:07 +0000 Subject: [PATCH 11/15] finish up datetime support --- docs/tutorial.rst | 4 ++-- zarr/meta.py | 2 ++ zarr/tests/test_core.py | 2 +- zarr/util.py | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 227a9ebdad..ad9b065c71 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -1079,10 +1079,10 @@ E.g., pickle/unpickle an array stored on disk:: Datetimes and timedeltas ------------------------ -NumPy's ``datetime64`` ('M') and ``timedelta64`` ('m') dtypes are supported for Zarr +NumPy's ``datetime64`` ('M8') and ``timedelta64`` ('m8') dtypes are supported for Zarr arrays, as long as the units are specified. E.g.:: - >>> z = zarr.array(['2007-07-13', '2006-01-13', '2010-08-13'], dtype='M[D]') + >>> z = zarr.array(['2007-07-13', '2006-01-13', '2010-08-13'], dtype='M8[D]') >>> z >>> z[:] diff --git a/zarr/meta.py b/zarr/meta.py index 0ad91cfca3..674fb04649 100644 --- a/zarr/meta.py +++ b/zarr/meta.py @@ -178,5 +178,7 @@ def encode_fill_value(v, dtype): return v elif dtype.kind == 'U': return v + elif dtype.kind in 'mM': + return int(v.view('u8')) else: return v diff --git a/zarr/tests/test_core.py b/zarr/tests/test_core.py index 10e1d6522c..838078e953 100644 --- a/zarr/tests/test_core.py +++ b/zarr/tests/test_core.py @@ -874,7 +874,7 @@ def test_dtypes(self): for base_type in 'Mm': for resolution in 'D', 'us', 'ns': dtype = '{}8[{}]'.format(base_type, resolution) - z = self.create_array(shape=100, dtype=dtype) + z = self.create_array(shape=100, dtype=dtype, fill_value=0) assert z.dtype == np.dtype(dtype) a = np.random.randint(0, np.iinfo('u8').max, size=z.shape[0], dtype='u8').view(dtype) diff --git a/zarr/util.py b/zarr/util.py index 1f20fbac1b..3b43e3a0d6 100644 --- a/zarr/util.py +++ b/zarr/util.py @@ -145,7 +145,7 @@ def normalize_dtype(dtype, object_codec): args = () try: object_codec = codec_registry[codec_id](*args) - except KeyError: + except KeyError: # pragma: no cover raise ValueError('codec %r for object type %r is not ' 'available; please provide an ' 'object_codec manually' % (codec_id, key)) From 6748354323d7da2a8e9d3cf9e4be23f55b557294 Mon Sep 17 00:00:00 2001 From: Alistair Miles Date: Thu, 7 Dec 2017 09:27:57 +0000 Subject: [PATCH 12/15] edit release notes --- docs/release.rst | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/docs/release.rst b/docs/release.rst index db49568f17..8317026660 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -1,11 +1,16 @@ Release notes ============= -.. release_2.2.0rc1 +.. _release_2.2.0rc1: 2.2.0rc1 -------- +To install the release candidate version:: + + $ pip install --pre zarr==2.2.0rc1 + + Enhancements ~~~~~~~~~~~~ @@ -119,6 +124,9 @@ Enhancements continue to work, however a warning will be raised to encourage use of the ``object_codec`` parameter. :issue:`208`, :issue:`212`. +* **Added support for ``datetime64`` and ``timedelta64`` data types**; + :issue:`85`, :issue:`215`. + Bug fixes ~~~~~~~~~ @@ -146,14 +154,8 @@ Documentation * Some changes have been made to the :ref:`spec_v2` document to clarify ambiguities and add some missing information. These changes do not break compatibility with any of the material as previously implemented, and so the changes have been made - in-place in the document without incrementing the document version number. The - specification now describes how bytes fill values should be encoded and - decoded for arrays with a fixed-length byte string data type (:issue:`165`, - :issue:`176`). The specification now clarifies that datetime64 and - timedelta64 data types are not supported in this version (:issue:`85`). The - specification now clarifies that the '.zattrs' key does not have to be present for - either arrays or groups, and if absent then custom attributes should be treated as - empty. + in-place in the document without incrementing the document version number. See the + section on :ref:`spec_v2_changes` in the specification document for more information. * A new :ref:`tutorial_indexing` section has been added to the tutorial. * A new :ref:`tutorial_strings` section has been added to the tutorial (:issue:`135`, :issue:`175`). From 2bb9f7964dd7a4b7302d8c7fcd08082cfe99aaa3 Mon Sep 17 00:00:00 2001 From: Alistair Miles Date: Thu, 7 Dec 2017 09:28:10 +0000 Subject: [PATCH 13/15] add refs --- docs/spec/v2.rst | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/docs/spec/v2.rst b/docs/spec/v2.rst index fa68b7eb0e..2a3bbd9a54 100644 --- a/docs/spec/v2.rst +++ b/docs/spec/v2.rst @@ -15,6 +15,8 @@ Status This specification is the latest version. See :ref:`spec` for previous versions. +.. _spec_v2_storage: + Storage ------- @@ -32,9 +34,13 @@ resources can be read, written or deleted via HTTP. Below an "array store" refers to any system implementing this interface. +.. _spec_v2_array: + Arrays ------ +.. _spec_v2_array_metadata: + Metadata ~~~~~~~~ @@ -105,6 +111,8 @@ using the Blosc compression library prior to storage:: "zarr_format": 2 } +.. _spec_v2_array_dtype: + Data type encoding ~~~~~~~~~~~~~~~~~~ @@ -139,6 +147,8 @@ example, the JSON list ``[["r", "|u1"], ["g", "|u1"], ["b", "|u1"]]`` defines a data type composed of three single-byte unsigned integers labelled "r", "g" and "b". +.. _spec_v2_array_fill_value: + Fill value encoding ~~~~~~~~~~~~~~~~~~~ @@ -157,6 +167,8 @@ If an array has a fixed length byte string data type (e.g., ``"|S12"``), or a structured data type, and if the fill value is not null, then the fill value MUST be encoded as an ASCII string using the standard Base64 alphabet. +.. _spec_v2_array_chunks: + Chunks ~~~~~~ @@ -190,6 +202,8 @@ array dimension is not exactly divisible by the length of the corresponding chunk dimension then some chunks will overhang the edge of the array. The contents of any chunk region falling outside the array are undefined. +.. _spec_v2_array_filters: + Filters ~~~~~~~ @@ -200,9 +214,13 @@ the primary compressor. When retrieving data, stored chunk data are decompressed by the primary compressor then decoded using filters in the reverse order. +.. _spec_v2_hierarchy: + Hierarchies ----------- +.. _spec_v2_hierarchy_paths: + Logical storage paths ~~~~~~~~~~~~~~~~~~~~~ @@ -238,6 +256,8 @@ treat all keys as opaque ASCII strings; equally, an array store could map logical paths onto some kind of hierarchical storage (e.g., directories on a file system). +.. _spec_v2_hierarchy_groups: + Groups ~~~~~~ @@ -272,6 +292,8 @@ under the logical paths "foo" and "foo/bar" and an array exists at logical path "foo/baz" then the members of the group at path "foo" are the group at path "foo/bar" and the array at path "foo/baz". +.. _spec_v2_attrs: + Attributes ---------- @@ -290,6 +312,8 @@ For example, the JSON object below encodes three attributes named "baz": [1, 2, 3, 4] } +.. _spec_v2_examples: + Examples -------- @@ -466,6 +490,8 @@ What has been stored:: foo/bar/1.0 foo/bar/1.1 +.. _spec_v2_changes: + Changes ------- @@ -487,8 +513,8 @@ initially published to clarify ambiguities and add some missing information. empty. -Changes in version 2 -~~~~~~~~~~~~~~~~~~~~ +Changes from version 1 to version 2 +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The following changes were made between version 1 and version 2 of this specification: From f380374c02346e739f5dd2cdb33b43736d0d56cf Mon Sep 17 00:00:00 2001 From: Alistair Miles Date: Thu, 7 Dec 2017 10:41:16 +0000 Subject: [PATCH 14/15] edit tutorial on strings and objects; bump numcodecs --- docs/tutorial.rst | 105 ++++++++++++++++++++++++++++++++++++------- requirements_dev.txt | 2 +- setup.py | 2 +- 3 files changed, 91 insertions(+), 18 deletions(-) diff --git a/docs/tutorial.rst b/docs/tutorial.rst index ad9b065c71..e28b816edb 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -779,6 +779,8 @@ If your strings are all ASCII strings, and you know the maximum length of the st your dataset, then you can use an array with a fixed-length bytes dtype. E.g.:: >>> z = zarr.zeros(10, dtype='S6') + >>> z + >>> z[0] = b'Hello' >>> z[1] = b'world!' >>> z[:] @@ -793,37 +795,68 @@ A fixed-length unicode dtype is also available, e.g.:: ... 'เฮลโลเวิลด์'] >>> text_data = greetings * 10000 >>> z = zarr.array(text_data, dtype='U20') + >>> z + >>> z[:] array(['¡Hola mundo!', 'Hej Världen!', 'Servus Woid!', ..., 'Helló, világ!', 'Zdravo svete!', 'เฮลโลเวิลด์'], dtype='>> import numcodecs - >>> z = zarr.array(text_data, dtype=object, object_codec=numcodecs.JSON()) + >>> z = zarr.array(text_data, dtype=object, object_codec=numcodecs.VLenUTF8()) + >>> z + + >>> z.filters + [VLenUTF8()] >>> z[:] array(['¡Hola mundo!', 'Hej Världen!', 'Servus Woid!', ..., 'Helló, világ!', 'Zdravo svete!', 'เฮลโลเวิลด์'], dtype=object) -...or alternatively using msgpack (requires `msgpack-python -`_ to be installed):: +As a convenience, ``dtype=str`` (or ``dtype=unicode`` on Python 2.7) can be used, which +is a short-hand for ``dtype=object, object_codec=numcodecs.VLenUTF8()``, e.g.:: - >>> z = zarr.array(text_data, dtype=object, object_codec=numcodecs.MsgPack()) + >>> z = zarr.array(text_data, dtype=str) + >>> z + + >>> z.filters + [VLenUTF8()] >>> z[:] array(['¡Hola mundo!', 'Hej Världen!', 'Servus Woid!', ..., 'Helló, világ!', 'Zdravo svete!', 'เฮลโลเวิลด์'], dtype=object) -If you know ahead of time all the possible string values that can occur, then you could -also use the :class:`numcodecs.Categorize` codec to encode each unique value as an +Variable-length byte strings are also supported via ``dtype=object``. Again an +``object_codec`` is required, which can be one of :class:`numcodecs.VLenBytes` or +:class:`numcodecs.Pickle`. For convenience, ``dtype=bytes`` (or ``dtype=str`` on Python +2.7) can be used as a short-hand for ``dtype=object, object_codec=numcodecs.VLenBytes()``, +e.g.:: + + >>> bytes_data = [g.encode('utf-8') for g in greetings] * 10000 + >>> z = zarr.array(bytes_data, dtype=bytes) + >>> z + + >>> z.filters + [VLenBytes()] + >>> z[:] + array([b'\xc2\xa1Hola mundo!', b'Hej V\xc3\xa4rlden!', b'Servus Woid!', + ..., b'Hell\xc3\xb3, vil\xc3\xa1g!', b'Zdravo svete!', + b'\xe0\xb9\x80\xe0\xb8\xae\xe0\xb8\xa5\xe0\xb9\x82\xe0\xb8\xa5\xe0\xb9\x80\xe0\xb8\xa7\xe0\xb8\xb4\xe0\xb8\xa5\xe0\xb8\x94\xe0\xb9\x8c'], dtype=object) + +If you know ahead of time all the possible string values that can occur, you could +also use the :class:`numcodecs.Categorize` codec to encode each unique string value as an integer. E.g.:: >>> categorize = numcodecs.Categorize(greetings, dtype=object) >>> z = zarr.array(text_data, dtype=object, object_codec=categorize) + >>> z + + >>> z.filters + [Categorize(dtype='|O', astype='|u1', labels=['¡Hola mundo!', 'Hej Världen!', 'Servus Woid!', ...])] >>> z[:] array(['¡Hola mundo!', 'Hej Världen!', 'Servus Woid!', ..., 'Helló, világ!', 'Zdravo svete!', 'เฮลโลเวิลด์'], dtype=object) @@ -835,13 +868,14 @@ Object arrays ------------- Zarr supports arrays with an "object" dtype. This allows arrays to contain any type of -object, such as variable length unicode strings, or variable length lists, or other -possibilities. When creating an object array, a codec must be provided via the +object, such as variable length unicode strings, or variable length arrays of numbers, or +other possibilities. When creating an object array, a codec must be provided via the ``object_codec`` argument. This codec handles encoding (serialization) of Python objects. -At the time of writing there are three codecs available that can serve as a -general purpose object codec and support encoding of a variety of -object types: :class:`numcodecs.JSON`, :class:`numcodecs.MsgPack`. and -:class:`numcodecs.Pickle`. +The best codec to use will depend on what type of objects are present in the array. + +At the time of writing there are three codecs available that can serve as a general +purpose object codec and support encoding of a mixture of object types: +:class:`numcodecs.JSON`, :class:`numcodecs.MsgPack`. and :class:`numcodecs.Pickle`. For example, using the JSON codec:: @@ -861,6 +895,40 @@ code can be embedded within pickled data. The JSON and MsgPack codecs do not hav security issues and support encoding of unicode strings, lists and dictionaries. MsgPack is usually faster for both encoding and decoding. +Ragged arrays +~~~~~~~~~~~~~ + +If you need to store an array of arrays, where each member array can be of any length +and stores the same primitive type (a.k.a. a ragged array), the +:class:`numcodecs.VLenArray` codec can be used, e.g.:: + + >>> z = zarr.empty(4, dtype=object, object_codec=numcodecs.VLenArray(int)) + >>> z + + >>> z.filters + [VLenArray(dtype='>> z[0] = np.array([1, 3, 5]) + >>> z[1] = np.array([4]) + >>> z[2] = np.array([7, 9, 14]) + >>> z[:] + array([array([1, 3, 5]), array([4]), array([ 7, 9, 14]), + array([], dtype=int64)], dtype=object) + +As a convenience, ``dtype='array:T'`` can be used as a short-hand for +``dtype=object, object_codec=numcodecs.VLenArray('T')``, where 'T' can be any NumPy +primitive dtype such as 'i4' or 'f8'. E.g.:: + + >>> z = zarr.empty(4, dtype='array:i8') + >>> z + + >>> z.filters + [VLenArray(dtype='>> z[0] = np.array([1, 3, 5]) + >>> z[1] = np.array([4]) + >>> z[2] = np.array([7, 9, 14]) + >>> z[:] + array([array([1, 3, 5]), array([4]), array([ 7, 9, 14]), + array([], dtype=int64)], dtype=object) .. _tutorial_chunks: @@ -1087,6 +1155,11 @@ arrays, as long as the units are specified. E.g.:: >>> z[:] array(['2007-07-13', '2006-01-13', '2010-08-13'], dtype='datetime64[D]') + >>> z[0] + numpy.datetime64('2007-07-13') + >>> z[0] = '1999-12-31' + >>> z[:] + array(['1999-12-31', '2006-01-13', '2010-08-13'], dtype='datetime64[D]') .. _tutorial_tips: diff --git a/requirements_dev.txt b/requirements_dev.txt index b01ba13e3f..8517c84c7c 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -15,7 +15,7 @@ mccabe==0.6.1 monotonic==1.3 msgpack-python==0.4.8 nose==1.3.7 -numcodecs==0.5.1 +numcodecs==0.5.2 numpy==1.13.3 packaging==16.8 pkginfo==1.4.1 diff --git a/setup.py b/setup.py index 698fc91a21..5ae09c7ef2 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ 'asciitree', 'numpy>=1.7', 'fasteners', - 'numcodecs>=0.5.1', + 'numcodecs>=0.5.2', ], package_dir={'': '.'}, packages=['zarr', 'zarr.tests'], From f272f8801da64ea7513c1a59755d0251a59926c4 Mon Sep 17 00:00:00 2001 From: Alistair Miles Date: Thu, 7 Dec 2017 12:34:53 +0000 Subject: [PATCH 15/15] extra object tests --- zarr/tests/test_core.py | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/zarr/tests/test_core.py b/zarr/tests/test_core.py index 838078e953..c50328422e 100644 --- a/zarr/tests/test_core.py +++ b/zarr/tests/test_core.py @@ -948,8 +948,16 @@ def test_object_arrays_vlen_text(self): data = np.array(greetings * 1000, dtype=object) z = self.create_array(shape=data.shape, dtype=object, object_codec=VLenUTF8()) + z[0] = u'foo' + assert z[0] == u'foo' + z[1] = u'bar' + assert z[1] == u'bar' + z[2] = u'baz' + assert z[2] == u'baz' z[:] = data - assert_array_equal(data, z[:]) + a = z[:] + assert a.dtype == object + assert_array_equal(data, a) # convenience API z = self.create_array(shape=data.shape, dtype=text_type) @@ -981,8 +989,16 @@ def test_object_arrays_vlen_bytes(self): data = np.array(greetings_bytes * 1000, dtype=object) z = self.create_array(shape=data.shape, dtype=object, object_codec=VLenBytes()) + z[0] = b'foo' + assert z[0] == b'foo' + z[1] = b'bar' + assert z[1] == b'bar' + z[2] = b'baz' + assert z[2] == b'baz' z[:] = data - assert_array_equal(data, z[:]) + a = z[:] + assert a.dtype == object + assert_array_equal(data, a) # convenience API z = self.create_array(shape=data.shape, dtype=binary_type) @@ -1013,8 +1029,12 @@ def compare_arrays(expected, actual, item_dtype): codecs = VLenArray(int), VLenArray('