diff --git a/docs/release.rst b/docs/release.rst index db49568f17..8317026660 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -1,11 +1,16 @@ Release notes ============= -.. release_2.2.0rc1 +.. _release_2.2.0rc1: 2.2.0rc1 -------- +To install the release candidate version:: + + $ pip install --pre zarr==2.2.0rc1 + + Enhancements ~~~~~~~~~~~~ @@ -119,6 +124,9 @@ Enhancements continue to work, however a warning will be raised to encourage use of the ``object_codec`` parameter. :issue:`208`, :issue:`212`. +* **Added support for ``datetime64`` and ``timedelta64`` data types**; + :issue:`85`, :issue:`215`. + Bug fixes ~~~~~~~~~ @@ -146,14 +154,8 @@ Documentation * Some changes have been made to the :ref:`spec_v2` document to clarify ambiguities and add some missing information. These changes do not break compatibility with any of the material as previously implemented, and so the changes have been made - in-place in the document without incrementing the document version number. The - specification now describes how bytes fill values should be encoded and - decoded for arrays with a fixed-length byte string data type (:issue:`165`, - :issue:`176`). The specification now clarifies that datetime64 and - timedelta64 data types are not supported in this version (:issue:`85`). The - specification now clarifies that the '.zattrs' key does not have to be present for - either arrays or groups, and if absent then custom attributes should be treated as - empty. + in-place in the document without incrementing the document version number. See the + section on :ref:`spec_v2_changes` in the specification document for more information. * A new :ref:`tutorial_indexing` section has been added to the tutorial. * A new :ref:`tutorial_strings` section has been added to the tutorial (:issue:`135`, :issue:`175`). diff --git a/docs/spec/v2.rst b/docs/spec/v2.rst index e1dcc31601..2a3bbd9a54 100644 --- a/docs/spec/v2.rst +++ b/docs/spec/v2.rst @@ -15,6 +15,8 @@ Status This specification is the latest version. See :ref:`spec` for previous versions. +.. _spec_v2_storage: + Storage ------- @@ -32,9 +34,13 @@ resources can be read, written or deleted via HTTP. Below an "array store" refers to any system implementing this interface. +.. _spec_v2_array: + Arrays ------ +.. _spec_v2_array_metadata: + Metadata ~~~~~~~~ @@ -105,6 +111,8 @@ using the Blosc compression library prior to storage:: "zarr_format": 2 } +.. _spec_v2_array_dtype: + Data type encoding ~~~~~~~~~~~~~~~~~~ @@ -117,17 +125,20 @@ consists of 3 parts: ``">"``: big-endian; ``"|"``: not-relevant) * One character code giving the basic type of the array (``"b"``: Boolean (integer type where all values are only True or False); ``"i"``: integer; ``"u"``: unsigned - integer; ``"f"``: floating point; ``"c"``: complex floating point; ``"S"``: string - (fixed-length sequence of char); ``"U"``: unicode (fixed-length sequence of - Py_UNICODE); ``"V"``: other (void * – each item is a fixed-size chunk of memory)) + integer; ``"f"``: floating point; ``"c"``: complex floating point; ``"m"``: timedelta; + ``"M"``: datetime; ``"S"``: string (fixed-length sequence of char); ``"U"``: unicode + (fixed-length sequence of Py_UNICODE); ``"V"``: other (void * – each item is a + fixed-size chunk of memory)) * An integer specifying the number of bytes the type uses. The byte order MUST be specified. E.g., ``"i4"``, ``"|b1"`` and ``"|S12"`` are valid data type encodings. -Please note that NumPy's datetime64 ("M") and timedelta64 ("m") data types are **not** -currently supported. Please store data using an appropriate physical data type instead, -e.g., 64-bit integer. +For datetime64 ("M") and timedelta64 ("m") data types, these MUST also include the +units within square brackets. A list of valid units and their definitions are given in +the `NumPy documentation on Datetimes and Timedeltas +`_. +For example, ``">> z = zarr.zeros(10, dtype='S6') + >>> z + >>> z[0] = b'Hello' >>> z[1] = b'world!' >>> z[:] @@ -793,37 +795,68 @@ A fixed-length unicode dtype is also available, e.g.:: ... 'เฮลโลเวิลด์'] >>> text_data = greetings * 10000 >>> z = zarr.array(text_data, dtype='U20') + >>> z + >>> z[:] array(['¡Hola mundo!', 'Hej Världen!', 'Servus Woid!', ..., 'Helló, világ!', 'Zdravo svete!', 'เฮลโลเวิลด์'], dtype='>> import numcodecs - >>> z = zarr.array(text_data, dtype=object, object_codec=numcodecs.JSON()) + >>> z = zarr.array(text_data, dtype=object, object_codec=numcodecs.VLenUTF8()) + >>> z + + >>> z.filters + [VLenUTF8()] >>> z[:] array(['¡Hola mundo!', 'Hej Världen!', 'Servus Woid!', ..., 'Helló, világ!', 'Zdravo svete!', 'เฮลโลเวิลด์'], dtype=object) -...or alternatively using msgpack (requires `msgpack-python -`_ to be installed):: +As a convenience, ``dtype=str`` (or ``dtype=unicode`` on Python 2.7) can be used, which +is a short-hand for ``dtype=object, object_codec=numcodecs.VLenUTF8()``, e.g.:: - >>> z = zarr.array(text_data, dtype=object, object_codec=numcodecs.MsgPack()) + >>> z = zarr.array(text_data, dtype=str) + >>> z + + >>> z.filters + [VLenUTF8()] >>> z[:] array(['¡Hola mundo!', 'Hej Världen!', 'Servus Woid!', ..., 'Helló, világ!', 'Zdravo svete!', 'เฮลโลเวิลด์'], dtype=object) -If you know ahead of time all the possible string values that can occur, then you could -also use the :class:`numcodecs.Categorize` codec to encode each unique value as an +Variable-length byte strings are also supported via ``dtype=object``. Again an +``object_codec`` is required, which can be one of :class:`numcodecs.VLenBytes` or +:class:`numcodecs.Pickle`. For convenience, ``dtype=bytes`` (or ``dtype=str`` on Python +2.7) can be used as a short-hand for ``dtype=object, object_codec=numcodecs.VLenBytes()``, +e.g.:: + + >>> bytes_data = [g.encode('utf-8') for g in greetings] * 10000 + >>> z = zarr.array(bytes_data, dtype=bytes) + >>> z + + >>> z.filters + [VLenBytes()] + >>> z[:] + array([b'\xc2\xa1Hola mundo!', b'Hej V\xc3\xa4rlden!', b'Servus Woid!', + ..., b'Hell\xc3\xb3, vil\xc3\xa1g!', b'Zdravo svete!', + b'\xe0\xb9\x80\xe0\xb8\xae\xe0\xb8\xa5\xe0\xb9\x82\xe0\xb8\xa5\xe0\xb9\x80\xe0\xb8\xa7\xe0\xb8\xb4\xe0\xb8\xa5\xe0\xb8\x94\xe0\xb9\x8c'], dtype=object) + +If you know ahead of time all the possible string values that can occur, you could +also use the :class:`numcodecs.Categorize` codec to encode each unique string value as an integer. E.g.:: >>> categorize = numcodecs.Categorize(greetings, dtype=object) >>> z = zarr.array(text_data, dtype=object, object_codec=categorize) + >>> z + + >>> z.filters + [Categorize(dtype='|O', astype='|u1', labels=['¡Hola mundo!', 'Hej Världen!', 'Servus Woid!', ...])] >>> z[:] array(['¡Hola mundo!', 'Hej Världen!', 'Servus Woid!', ..., 'Helló, világ!', 'Zdravo svete!', 'เฮลโลเวิลด์'], dtype=object) @@ -835,13 +868,14 @@ Object arrays ------------- Zarr supports arrays with an "object" dtype. This allows arrays to contain any type of -object, such as variable length unicode strings, or variable length lists, or other -possibilities. When creating an object array, a codec must be provided via the +object, such as variable length unicode strings, or variable length arrays of numbers, or +other possibilities. When creating an object array, a codec must be provided via the ``object_codec`` argument. This codec handles encoding (serialization) of Python objects. -At the time of writing there are three codecs available that can serve as a -general purpose object codec and support encoding of a variety of -object types: :class:`numcodecs.JSON`, :class:`numcodecs.MsgPack`. and -:class:`numcodecs.Pickle`. +The best codec to use will depend on what type of objects are present in the array. + +At the time of writing there are three codecs available that can serve as a general +purpose object codec and support encoding of a mixture of object types: +:class:`numcodecs.JSON`, :class:`numcodecs.MsgPack`. and :class:`numcodecs.Pickle`. For example, using the JSON codec:: @@ -861,6 +895,40 @@ code can be embedded within pickled data. The JSON and MsgPack codecs do not hav security issues and support encoding of unicode strings, lists and dictionaries. MsgPack is usually faster for both encoding and decoding. +Ragged arrays +~~~~~~~~~~~~~ + +If you need to store an array of arrays, where each member array can be of any length +and stores the same primitive type (a.k.a. a ragged array), the +:class:`numcodecs.VLenArray` codec can be used, e.g.:: + + >>> z = zarr.empty(4, dtype=object, object_codec=numcodecs.VLenArray(int)) + >>> z + + >>> z.filters + [VLenArray(dtype='>> z[0] = np.array([1, 3, 5]) + >>> z[1] = np.array([4]) + >>> z[2] = np.array([7, 9, 14]) + >>> z[:] + array([array([1, 3, 5]), array([4]), array([ 7, 9, 14]), + array([], dtype=int64)], dtype=object) + +As a convenience, ``dtype='array:T'`` can be used as a short-hand for +``dtype=object, object_codec=numcodecs.VLenArray('T')``, where 'T' can be any NumPy +primitive dtype such as 'i4' or 'f8'. E.g.:: + + >>> z = zarr.empty(4, dtype='array:i8') + >>> z + + >>> z.filters + [VLenArray(dtype='>> z[0] = np.array([1, 3, 5]) + >>> z[1] = np.array([4]) + >>> z[2] = np.array([7, 9, 14]) + >>> z[:] + array([array([1, 3, 5]), array([4]), array([ 7, 9, 14]), + array([], dtype=int64)], dtype=object) .. _tutorial_chunks: @@ -1079,25 +1147,19 @@ E.g., pickle/unpickle an array stored on disk:: Datetimes and timedeltas ------------------------ -Please note that NumPy's ``datetime64`` and ``timedelta64`` dtypes are **not** currently -supported for Zarr arrays. If you would like to store datetime or timedelta data, you -can store the data in an array with an integer dtype, e.g.:: +NumPy's ``datetime64`` ('M8') and ``timedelta64`` ('m8') dtypes are supported for Zarr +arrays, as long as the units are specified. E.g.:: - >>> a = np.array(['2007-07-13', '2006-01-13', '2010-08-13'], dtype='datetime64[D]') - >>> z = zarr.array(a.view('i8')) + >>> z = zarr.array(['2007-07-13', '2006-01-13', '2010-08-13'], dtype='M8[D]') >>> z - + >>> z[:] - array([13707, 13161, 14834]) - >>> z[:].view(a.dtype) - array(['2007-07-13', '2006-01-13', '2010-08-13'], dtype='datetime64[D]') - -If you would like a convenient way to retrieve the data from this array viewed as the -original datetime64 dtype, try the :func:`zarr.core.Array.astype` method, e.g.:: - - >>> zv = z.astype(a.dtype) - >>> zv[:] array(['2007-07-13', '2006-01-13', '2010-08-13'], dtype='datetime64[D]') + >>> z[0] + numpy.datetime64('2007-07-13') + >>> z[0] = '1999-12-31' + >>> z[:] + array(['1999-12-31', '2006-01-13', '2010-08-13'], dtype='datetime64[D]') .. _tutorial_tips: diff --git a/requirements_dev.txt b/requirements_dev.txt index 5435d1a777..8517c84c7c 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -15,7 +15,7 @@ mccabe==0.6.1 monotonic==1.3 msgpack-python==0.4.8 nose==1.3.7 -numcodecs==0.4.1 +numcodecs==0.5.2 numpy==1.13.3 packaging==16.8 pkginfo==1.4.1 diff --git a/setup.py b/setup.py index 4f615d5144..5ae09c7ef2 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ 'asciitree', 'numpy>=1.7', 'fasteners', - 'numcodecs>=0.4.1', + 'numcodecs>=0.5.2', ], package_dir={'': '.'}, packages=['zarr', 'zarr.tests'], diff --git a/zarr/core.py b/zarr/core.py index ca8683ee36..cb9db4a78a 100644 --- a/zarr/core.py +++ b/zarr/core.py @@ -1590,7 +1590,10 @@ def _chunk_getitem(self, chunk_coords, chunk_selection, out, out_selection, if self._compressor: self._compressor.decode(cdata, dest) else: - chunk = np.frombuffer(cdata, dtype=self._dtype) + if isinstance(cdata, np.ndarray): + chunk = cdata.view(self._dtype) + else: + chunk = np.frombuffer(cdata, dtype=self._dtype) chunk = chunk.reshape(self._chunks, order=self._order) np.copyto(dest, chunk) return @@ -1736,7 +1739,7 @@ def _decode_chunk(self, cdata): elif isinstance(chunk, np.ndarray): chunk = chunk.view(self._dtype) else: - chunk = np.frombuffer(chunk, self._dtype) + chunk = np.frombuffer(chunk, dtype=self._dtype) # reshape chunk = chunk.reshape(self._chunks, order=self._order) @@ -2087,15 +2090,15 @@ def view(self, shape=None, chunks=None, dtype=None, >>> import zarr >>> import numpy as np >>> np.random.seed(42) - >>> labels = [b'female', b'male'] + >>> labels = ['female', 'male'] >>> data = np.random.choice(labels, size=10000) >>> filters = [zarr.Categorize(labels=labels, - ... dtype=data.dtype, - ... astype='u1')] + ... dtype=data.dtype, + ... astype='u1')] >>> a = zarr.array(data, chunks=1000, filters=filters) >>> a[:] - array([b'female', b'male', b'female', ..., b'male', b'male', b'female'], - dtype='|S6') + array(['female', 'male', 'female', ..., 'male', 'male', 'female'], + dtype='>> v = a.view(dtype='u1', filters=[]) >>> v.is_view True @@ -2110,10 +2113,10 @@ def view(self, shape=None, chunks=None, dtype=None, >>> v[:] array([1, 1, 1, ..., 2, 2, 2], dtype=uint8) >>> a[:] - array([b'female', b'female', b'female', ..., b'male', b'male', b'male'], - dtype='|S6') + array(['female', 'female', 'female', ..., 'male', 'male', 'male'], + dtype='>> data = np.random.randint(0, 2, size=10000, dtype='u1') >>> a = zarr.array(data, chunks=1000) @@ -2125,7 +2128,7 @@ def view(self, shape=None, chunks=None, dtype=None, >>> np.all(a[:].view(dtype=bool) == v[:]) True - An array can be viewed with a dtype with a different itemsize, however + An array can be viewed with a dtype with a different item size, however some care is needed to adjust the shape and chunk shape so that chunk data is interpreted correctly: diff --git a/zarr/meta.py b/zarr/meta.py index 0ad91cfca3..674fb04649 100644 --- a/zarr/meta.py +++ b/zarr/meta.py @@ -178,5 +178,7 @@ def encode_fill_value(v, dtype): return v elif dtype.kind == 'U': return v + elif dtype.kind in 'mM': + return int(v.view('u8')) else: return v diff --git a/zarr/storage.py b/zarr/storage.py index bf821f82ab..4d25e6a634 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -25,7 +25,8 @@ from zarr.util import (normalize_shape, normalize_chunks, normalize_order, - normalize_storage_path, buffer_size, normalize_fill_value, nolock) + normalize_storage_path, buffer_size, + normalize_fill_value, nolock, normalize_dtype) from zarr.meta import encode_array_metadata, encode_group_metadata from zarr.compat import PY2, binary_type from numcodecs.registry import codec_registry @@ -308,10 +309,7 @@ def _init_array_metadata(store, shape, chunks=None, dtype=None, compressor='defa # normalize metadata shape = normalize_shape(shape) - dtype = np.dtype(dtype) - if dtype.kind in 'mM': - raise ValueError('datetime64 and timedelta64 dtypes are not currently supported; ' - 'please store the data using int64 instead') + dtype, object_codec = normalize_dtype(dtype, object_codec) chunks = normalize_chunks(chunks, shape, dtype.itemsize) order = normalize_order(order) fill_value = normalize_fill_value(fill_value, dtype) diff --git a/zarr/tests/test_core.py b/zarr/tests/test_core.py index 47c3f3e3c8..c50328422e 100644 --- a/zarr/tests/test_core.py +++ b/zarr/tests/test_core.py @@ -21,10 +21,11 @@ DBMStore, LMDBStore, atexit_rmtree, atexit_rmglob) from zarr.core import Array from zarr.errors import PermissionError -from zarr.compat import PY2 +from zarr.compat import PY2, text_type, binary_type from zarr.util import buffer_size from numcodecs import (Delta, FixedScaleOffset, Zlib, Blosc, BZ2, MsgPack, Pickle, - Categorize, JSON) + Categorize, JSON, VLenUTF8, VLenBytes, VLenArray) +from numcodecs.tests.common import greetings # needed for PY2/PY3 consistent behaviour @@ -854,27 +855,37 @@ def test_structured_array(self): def test_dtypes(self): # integers - for t in 'u1', 'u2', 'u4', 'u8', 'i1', 'i2', 'i4', 'i8': - z = self.create_array(shape=10, chunks=3, dtype=t) - assert z.dtype == np.dtype(t) - a = np.arange(z.shape[0], dtype=t) + for dtype in 'u1', 'u2', 'u4', 'u8', 'i1', 'i2', 'i4', 'i8': + z = self.create_array(shape=10, chunks=3, dtype=dtype) + assert z.dtype == np.dtype(dtype) + a = np.arange(z.shape[0], dtype=dtype) z[:] = a assert_array_equal(a, z[:]) # floats - for t in 'f2', 'f4', 'f8': - z = self.create_array(shape=10, chunks=3, dtype=t) - assert z.dtype == np.dtype(t) - a = np.linspace(0, 1, z.shape[0], dtype=t) + for dtype in 'f2', 'f4', 'f8': + z = self.create_array(shape=10, chunks=3, dtype=dtype) + assert z.dtype == np.dtype(dtype) + a = np.linspace(0, 1, z.shape[0], dtype=dtype) z[:] = a assert_array_almost_equal(a, z[:]) - # datetime, timedelta are not supported for the time being - for resolution in 'D', 'us', 'ns': - with assert_raises(ValueError): - self.create_array(shape=10, dtype='datetime64[{}]'.format(resolution)) - with assert_raises(ValueError): - self.create_array(shape=10, dtype='timedelta64[{}]'.format(resolution)) + # datetime, timedelta + for base_type in 'Mm': + for resolution in 'D', 'us', 'ns': + dtype = '{}8[{}]'.format(base_type, resolution) + z = self.create_array(shape=100, dtype=dtype, fill_value=0) + assert z.dtype == np.dtype(dtype) + a = np.random.randint(0, np.iinfo('u8').max, size=z.shape[0], + dtype='u8').view(dtype) + z[:] = a + assert_array_equal(a, z[:]) + + # check that datetime generic units are not allowed + with assert_raises(ValueError): + self.create_array(shape=100, dtype='M8') + with assert_raises(ValueError): + self.create_array(shape=100, dtype='m8') def test_object_arrays(self): @@ -932,11 +943,29 @@ def test_object_arrays(self): a = z[:] assert a.dtype == object - def test_object_arrays_text(self): + def test_object_arrays_vlen_text(self): - from numcodecs.tests.common import greetings data = np.array(greetings * 1000, dtype=object) + z = self.create_array(shape=data.shape, dtype=object, object_codec=VLenUTF8()) + z[0] = u'foo' + assert z[0] == u'foo' + z[1] = u'bar' + assert z[1] == u'bar' + z[2] = u'baz' + assert z[2] == u'baz' + z[:] = data + a = z[:] + assert a.dtype == object + assert_array_equal(data, a) + + # convenience API + z = self.create_array(shape=data.shape, dtype=text_type) + assert z.dtype == object + assert isinstance(z.filters[0], VLenUTF8) + z[:] = data + assert_array_equal(data, z[:]) + z = self.create_array(shape=data.shape, dtype=object, object_codec=MsgPack()) z[:] = data assert_array_equal(data, z[:]) @@ -954,6 +983,68 @@ def test_object_arrays_text(self): z[:] = data assert_array_equal(data, z[:]) + def test_object_arrays_vlen_bytes(self): + + greetings_bytes = [g.encode('utf8') for g in greetings] + data = np.array(greetings_bytes * 1000, dtype=object) + + z = self.create_array(shape=data.shape, dtype=object, object_codec=VLenBytes()) + z[0] = b'foo' + assert z[0] == b'foo' + z[1] = b'bar' + assert z[1] == b'bar' + z[2] = b'baz' + assert z[2] == b'baz' + z[:] = data + a = z[:] + assert a.dtype == object + assert_array_equal(data, a) + + # convenience API + z = self.create_array(shape=data.shape, dtype=binary_type) + assert z.dtype == object + assert isinstance(z.filters[0], VLenBytes) + z[:] = data + assert_array_equal(data, z[:]) + + z = self.create_array(shape=data.shape, dtype=object, object_codec=Pickle()) + z[:] = data + assert_array_equal(data, z[:]) + + def test_object_arrays_vlen_array(self): + + data = np.array([np.array([1, 3, 7]), + np.array([5]), + np.array([2, 8, 12])] * 1000, dtype=object) + + def compare_arrays(expected, actual, item_dtype): + assert isinstance(actual, np.ndarray) + assert actual.dtype == object + assert actual.shape == expected.shape + for e, a in zip(expected.flat, actual.flat): + assert isinstance(a, np.ndarray) + assert_array_equal(e, a) + assert a.dtype == item_dtype + + codecs = VLenArray(int), VLenArray(' 1: + args = tokens[1].split(',') + else: + args = () + try: + object_codec = codec_registry[codec_id](*args) + except KeyError: # pragma: no cover + raise ValueError('codec %r for object type %r is not ' + 'available; please provide an ' + 'object_codec manually' % (codec_id, key)) + return dtype, object_codec + + dtype = np.dtype(dtype) + + # don't allow generic datetime64 or timedelta64, require units to be specified + if dtype == np.dtype('M8') or dtype == np.dtype('m8'): + raise ValueError('datetime64 and timedelta64 dtypes with generic units ' + 'are not supported, please specify units (e.g., "M8[ns]")') + + return dtype, object_codec + + # noinspection PyTypeChecker def is_total_slice(item, shape): """Determine whether `item` specifies a complete slice of array with the