From ca04bf9e43d1acf90109ccadb91b5e31a7b04f6c Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 22 Jun 2022 15:16:56 -0400 Subject: [PATCH 01/14] forward write_empty_chunks kwarg in group.require_dataset --- zarr/_storage/store.py | 4 +- zarr/core.py | 4 +- zarr/creation.py | 2 +- zarr/hierarchy.py | 212 ++++++++++++++++++++++------------- zarr/storage.py | 16 +-- zarr/tests/test_hierarchy.py | 41 +++++-- zarr/util.py | 2 +- 7 files changed, 180 insertions(+), 101 deletions(-) diff --git a/zarr/_storage/store.py b/zarr/_storage/store.py index 36a5c0bff5..fa19dbd9c9 100644 --- a/zarr/_storage/store.py +++ b/zarr/_storage/store.py @@ -1,6 +1,6 @@ import abc import os -from collections.abc import MutableMapping +from typing import MutableMapping from string import ascii_letters, digits from typing import Any, List, Mapping, Optional, Union @@ -29,7 +29,7 @@ def assert_zarr_v3_api_available(): ) # pragma: no cover -class BaseStore(MutableMapping): +class BaseStore(MutableMapping[str, Any]): """Abstract base class for store implementations. This is a thin wrapper over MutableMapping that provides methods to check diff --git a/zarr/core.py b/zarr/core.py index 6ce2fa8800..558fd0037e 100644 --- a/zarr/core.py +++ b/zarr/core.py @@ -6,7 +6,7 @@ import re from collections.abc import MutableMapping from functools import reduce -from typing import Any +from typing import Any, Tuple import numpy as np from numcodecs.compat import ensure_bytes, ensure_ndarray @@ -365,7 +365,7 @@ def chunk_store(self): return self._chunk_store @property - def shape(self): + def shape(self) -> Tuple[int, ...]: """A tuple of integers describing the length of each dimension of the array.""" # N.B., shape may change if array is resized, hence need to refresh diff --git a/zarr/creation.py b/zarr/creation.py index e77f26b3e2..a255eda2bc 100644 --- a/zarr/creation.py +++ b/zarr/creation.py @@ -21,7 +21,7 @@ def create(shape, chunks=True, dtype=None, compressor='default', overwrite=False, path=None, chunk_store=None, filters=None, cache_metadata=True, cache_attrs=True, read_only=False, object_codec=None, dimension_separator=None, write_empty_chunks=True, - *, zarr_version=None, **kwargs): + *, zarr_version=None, **kwargs) -> Array: """Create an array. Parameters diff --git a/zarr/hierarchy.py b/zarr/hierarchy.py index b9052408b4..bb25e3438c 100644 --- a/zarr/hierarchy.py +++ b/zarr/hierarchy.py @@ -1,9 +1,21 @@ -from collections.abc import MutableMapping from itertools import islice +from typing import ( + Any, + Callable, + Generator, + List, + Literal, + Optional, + Tuple, + Union, + cast, + MutableMapping +) import numpy as np +import numpy.typing as npt -from zarr._storage.store import (_get_metadata_suffix, data_root, meta_root, +from zarr._storage.store import (StoreV3, _get_metadata_suffix, data_root, meta_root, DEFAULT_ZARR_VERSION, assert_zarr_v3_api_available) from zarr.attrs import Attributes from zarr.core import Array @@ -16,6 +28,7 @@ ReadOnlyError, ) from zarr.storage import ( + StoreLike, _get_hierarchy_metadata, _prefix_to_group_key, BaseStore, @@ -40,8 +53,10 @@ normalize_storage_path, ) +AccessModes = Literal['r', 'r+', 'w', 'a', 'w-', 'x'] + -class Group(MutableMapping): +class Group(MutableMapping[str, Union['Group', Array]]): """Instantiate a group from an initialized store. Parameters @@ -113,9 +128,15 @@ class Group(MutableMapping): """ - def __init__(self, store, path=None, read_only=False, chunk_store=None, - cache_attrs=True, synchronizer=None, zarr_version=None): - store: BaseStore = _normalize_store_arg(store, zarr_version=zarr_version) + def __init__(self, + store: BaseStore, + path: Optional[str] = None, + read_only: bool = False, + chunk_store: Optional[StoreLike] = None, + cache_attrs: bool = True, + synchronizer: Any = None, + zarr_version: Optional[int] = None): + store = _normalize_store_arg(store, zarr_version=zarr_version) if zarr_version is None: zarr_version = getattr(store, '_store_version', DEFAULT_ZARR_VERSION) @@ -123,7 +144,7 @@ def __init__(self, store, path=None, read_only=False, chunk_store=None, assert_zarr_v3_api_available() if chunk_store is not None: - chunk_store: BaseStore = _normalize_store_arg(chunk_store, zarr_version=zarr_version) + chunk_store = _normalize_store_arg(chunk_store, zarr_version=zarr_version) self._store = store self._chunk_store = chunk_store self._path = normalize_storage_path(path) @@ -136,6 +157,7 @@ def __init__(self, store, path=None, read_only=False, chunk_store=None, self._version = zarr_version if self._version == 3: + cast(StoreV3, self._store) self._data_key_prefix = data_root + self._key_prefix self._data_path = data_root + self._path self._hierarchy_metadata = _get_hierarchy_metadata(store=self._store) @@ -182,12 +204,12 @@ def store(self): return self._store @property - def path(self): + def path(self) -> str: """Storage path.""" return self._path @property - def name(self): + def name(self) -> str: """Group name following h5py convention.""" if self._path: # follow h5py convention: add leading slash @@ -198,17 +220,17 @@ def name(self): return '/' @property - def basename(self): + def basename(self) -> str: """Final component of name.""" return self.name.split('/')[-1] @property - def read_only(self): + def read_only(self) -> bool: """A boolean, True if modification operations are not permitted.""" return self._read_only @property - def chunk_store(self): + def chunk_store(self) -> StoreLike: """A MutableMapping providing the underlying storage for array chunks.""" if self._chunk_store is None: return self._store @@ -216,22 +238,22 @@ def chunk_store(self): return self._chunk_store @property - def synchronizer(self): + def synchronizer(self) -> Any: """Object used to synchronize write access to groups and arrays.""" return self._synchronizer @property - def attrs(self): + def attrs(self) -> MutableMapping[str, Any]: """A MutableMapping containing user-defined attributes. Note that attribute values must be JSON serializable.""" return self._attrs @property - def info(self): + def info(self) -> InfoReporter: """Return diagnostic information about the group.""" return self._info - def __eq__(self, other): + def __eq__(self, other: Any) -> bool: return ( isinstance(other, Group) and self._store == other.store and @@ -241,7 +263,7 @@ def __eq__(self, other): # store comparison ) - def __iter__(self): + def __iter__(self) -> Generator[str, None, None]: """Return an iterator over group member names. Examples @@ -291,11 +313,11 @@ def __iter__(self): # Note: omit data/root/ to avoid duplicate listings # any group in data/root/ must has an entry in meta/root/ - def __len__(self): + def __len__(self) -> int: """Number of members.""" return sum(1 for _ in self) - def __repr__(self): + def __repr__(self) -> str: t = type(self) r = '<{}.{}'.format(t.__module__, t.__name__) if self.name: @@ -313,12 +335,12 @@ def __exit__(self, exc_type, exc_val, exc_tb): """Call the close method of the underlying Store.""" self.store.close() - def info_items(self): + def info_items(self) -> List[Tuple[str, Union[str, int]]]: - def typestr(o): + def typestr(o: Any): return '{}.{}'.format(type(o).__module__, type(o).__name__) - items = [] + items: List[Tuple[str, Union[int, str]]] = [] # basic info if self.name is not None: @@ -357,14 +379,14 @@ def __getstate__(self): def __setstate__(self, state): self.__init__(*state) - def _item_path(self, item): + def _item_path(self, item: str) -> str: absolute = isinstance(item, str) and item and item[0] == '/' path = normalize_storage_path(item) if not absolute and self._path: path = self._key_prefix + path return path - def __contains__(self, item): + def __contains__(self, item: str): """Test for group membership. Examples @@ -385,7 +407,7 @@ def __contains__(self, item): return contains_array(self._store, path) or \ contains_group(self._store, path, explicit_only=False) - def __getitem__(self, item): + def __getitem__(self, item: str) -> Union['Group', Array]: """Obtain a group member. Parameters @@ -542,7 +564,7 @@ def groups(self): synchronizer=self._synchronizer, zarr_version=self._version) - def array_keys(self, recurse=False): + def array_keys(self, recurse: bool = False) -> Generator[str, None, None]: """Return an iterator over member names for arrays only. Parameters @@ -596,7 +618,7 @@ def arrays(self, recurse=False): method='arrays', recurse=recurse) - def _array_iter(self, keys_only, method, recurse): + def _array_iter(self, keys_only: bool, method, recurse: bool) -> Generator[str, None, None]: if self._version == 2: for key in sorted(listdir(self._store, self._path)): path = self._key_prefix + key @@ -627,7 +649,7 @@ def _array_iter(self, keys_only, method, recurse): for i in getattr(group, method)(recurse=recurse): yield i - def visitvalues(self, func): + def visitvalues(self, func: Callable[[Union['Group', Array]], Any]) -> Any: """Run ``func`` on each object. Note: If ``func`` returns ``None`` (or doesn't return), @@ -655,7 +677,7 @@ def visitvalues(self, func): """ - def _visit(obj): + def _visit(obj: Union['Group', Array]) -> Generator[Union['Group', Array], None, None]: yield obj keys = sorted(getattr(obj, "keys", lambda: [])()) for k in keys: @@ -667,7 +689,7 @@ def _visit(obj): if value is not None: return value - def visit(self, func): + def visit(self, func: Callable[[str], Any]): """Run ``func`` on each object's path. Note: If ``func`` returns ``None`` (or doesn't return), @@ -704,7 +726,7 @@ def visitkeys(self, func): return self.visit(func) - def visititems(self, func): + def visititems(self, func: Callable[[str, Union['Group', Array]], Any]) -> Any: """Run ``func`` on each object's path and the object itself. Note: If ``func`` returns ``None`` (or doesn't return), @@ -735,7 +757,7 @@ def visititems(self, func): base_len = len(self.name) return self.visitvalues(lambda o: func(o.name[base_len:].lstrip("/"), o)) - def tree(self, expand=False, level=None): + def tree(self, expand: bool = False, level: Optional[int] = None): """Provide a ``print``-able display of the hierarchy. Parameters @@ -783,7 +805,7 @@ def tree(self, expand=False, level=None): return TreeViewer(self, expand=expand, level=level) - def _write_op(self, f, *args, **kwargs): + def _write_op(self, f: Callable[..., Any], *args: Any, **kwargs: Any) -> Any: # guard condition if self._read_only: @@ -799,7 +821,7 @@ def _write_op(self, f, *args, **kwargs): with lock: return f(*args, **kwargs) - def create_group(self, name, overwrite=False): + def create_group(self, name: str, overwrite: bool = False): """Create a sub-group. Parameters @@ -825,7 +847,7 @@ def create_group(self, name, overwrite=False): return self._write_op(self._create_group_nosync, name, overwrite=overwrite) - def _create_group_nosync(self, name, overwrite=False): + def _create_group_nosync(self, name: str, overwrite: bool = False): path = self._item_path(name) # create terminal group @@ -836,11 +858,11 @@ def _create_group_nosync(self, name, overwrite=False): chunk_store=self._chunk_store, cache_attrs=self.attrs.cache, synchronizer=self._synchronizer, zarr_version=self._version) - def create_groups(self, *names, **kwargs): + def create_groups(self, *names: str, **kwargs: Any): """Convenience method to create multiple groups in a single call.""" return tuple(self.create_group(name, **kwargs) for name in names) - def require_group(self, name, overwrite=False): + def require_group(self, name: str, overwrite: bool = False): """Obtain a sub-group, creating one if it doesn't exist. Parameters @@ -868,7 +890,7 @@ def require_group(self, name, overwrite=False): return self._write_op(self._require_group_nosync, name, overwrite=overwrite) - def _require_group_nosync(self, name, overwrite=False): + def _require_group_nosync(self, name: str, overwrite: bool = False): path = self._item_path(name) # create terminal group if necessary @@ -880,12 +902,12 @@ def _require_group_nosync(self, name, overwrite=False): chunk_store=self._chunk_store, cache_attrs=self.attrs.cache, synchronizer=self._synchronizer, zarr_version=self._version) - def require_groups(self, *names): + def require_groups(self, *names: str): """Convenience method to require multiple groups in a single call.""" return tuple(self.require_group(name) for name in names) # noinspection PyIncorrectDocstring - def create_dataset(self, name, **kwargs): + def create_dataset(self, name: str, **kwargs: Any): """Create an array. Arrays are known as "datasets" in HDF5 terminology. For compatibility @@ -924,6 +946,14 @@ def create_dataset(self, name, **kwargs): overhead depending on storage and data access pattern). dimension_separator : {'.', '/'}, optional Separator placed between the dimensions of a chunk. + write_empty_chunks : bool, optional + If True, all chunks will be stored regardless of their contents. If + False (default), each chunk is compared to the array's fill value prior + to storing. If a chunk is uniformly equal to the fill value, then that + chunk is not be stored, and the store entry for that chunk's key is + deleted. This setting enables sparser storage, as only chunks with + non-fill-value data are stored, at the expense of overhead associated + with checking the data of each chunk. Returns ------- @@ -947,7 +977,7 @@ def create_dataset(self, name, **kwargs): return self._write_op(self._create_dataset_nosync, name, **kwargs) - def _create_dataset_nosync(self, name, data=None, **kwargs): + def _create_dataset_nosync(self, name: str, data: Any = None, **kwargs: Any) -> Array: assert "mode" not in kwargs path = self._item_path(name) @@ -967,7 +997,12 @@ def _create_dataset_nosync(self, name, data=None, **kwargs): return a - def require_dataset(self, name, shape, dtype=None, exact=False, **kwargs): + def require_dataset(self, + name: str, + shape: Union[int, Tuple[int, ...]], + dtype: Optional[npt.DTypeLike] = None, + exact: bool = False, + **kwargs: Any): """Obtain an array, creating if it doesn't exist. Arrays are known as "datasets" in HDF5 terminology. For compatibility @@ -992,8 +1027,12 @@ def require_dataset(self, name, shape, dtype=None, exact=False, **kwargs): return self._write_op(self._require_dataset_nosync, name, shape=shape, dtype=dtype, exact=exact, **kwargs) - def _require_dataset_nosync(self, name, shape, dtype=None, exact=False, - **kwargs): + def _require_dataset_nosync(self, + name: str, + shape: Tuple[int, ...], + dtype: Optional[npt.DTypeLike] = None, + exact: bool = False, + **kwargs: Any) -> Array: path = self._item_path(name) @@ -1004,9 +1043,15 @@ def _require_dataset_nosync(self, name, shape, dtype=None, exact=False, synchronizer = kwargs.get('synchronizer', self._synchronizer) cache_metadata = kwargs.get('cache_metadata', True) cache_attrs = kwargs.get('cache_attrs', self.attrs.cache) - a = Array(self._store, path=path, read_only=self._read_only, - chunk_store=self._chunk_store, synchronizer=synchronizer, - cache_metadata=cache_metadata, cache_attrs=cache_attrs) + write_empty_chunks = kwargs.get('write_empty_chunks', True) + a = Array(self._store, + path=path, + read_only=self._read_only, + chunk_store=self._chunk_store, + synchronizer=synchronizer, + cache_metadata=cache_metadata, + cache_attrs=cache_attrs, + write_empty_chunks=write_empty_chunks) shape = normalize_shape(shape) if shape != a.shape: raise TypeError('shape do not match existing array; expected {}, got {}' @@ -1026,131 +1071,131 @@ def _require_dataset_nosync(self, name, shape, dtype=None, exact=False, return self._create_dataset_nosync(name, shape=shape, dtype=dtype, **kwargs) - def create(self, name, **kwargs): + def create(self, name: str, **kwargs: Any) -> Array: """Create an array. Keyword arguments as per :func:`zarr.creation.create`.""" return self._write_op(self._create_nosync, name, **kwargs) - def _create_nosync(self, name, **kwargs): + def _create_nosync(self, name: str, **kwargs: Any) -> Array: path = self._item_path(name) kwargs.setdefault('synchronizer', self._synchronizer) kwargs.setdefault('cache_attrs', self.attrs.cache) return create(store=self._store, path=path, chunk_store=self._chunk_store, **kwargs) - def empty(self, name, **kwargs): + def empty(self, name: str, **kwargs: Any) -> Array: """Create an array. Keyword arguments as per :func:`zarr.creation.empty`.""" return self._write_op(self._empty_nosync, name, **kwargs) - def _empty_nosync(self, name, **kwargs): + def _empty_nosync(self, name: str, **kwargs: Any) -> Array: path = self._item_path(name) kwargs.setdefault('synchronizer', self._synchronizer) kwargs.setdefault('cache_attrs', self.attrs.cache) return empty(store=self._store, path=path, chunk_store=self._chunk_store, **kwargs) - def zeros(self, name, **kwargs): + def zeros(self, name: str, **kwargs: Any) -> Array: """Create an array. Keyword arguments as per :func:`zarr.creation.zeros`.""" return self._write_op(self._zeros_nosync, name, **kwargs) - def _zeros_nosync(self, name, **kwargs): + def _zeros_nosync(self, name: str, **kwargs: Any) -> Array: path = self._item_path(name) kwargs.setdefault('synchronizer', self._synchronizer) kwargs.setdefault('cache_attrs', self.attrs.cache) return zeros(store=self._store, path=path, chunk_store=self._chunk_store, **kwargs) - def ones(self, name, **kwargs): + def ones(self, name: str, **kwargs: Any) -> Array: """Create an array. Keyword arguments as per :func:`zarr.creation.ones`.""" return self._write_op(self._ones_nosync, name, **kwargs) - def _ones_nosync(self, name, **kwargs): + def _ones_nosync(self, name: str, **kwargs: Any) -> Array: path = self._item_path(name) kwargs.setdefault('synchronizer', self._synchronizer) kwargs.setdefault('cache_attrs', self.attrs.cache) return ones(store=self._store, path=path, chunk_store=self._chunk_store, **kwargs) - def full(self, name, fill_value, **kwargs): + def full(self, name: str, fill_value: Any, **kwargs: Any) -> Array: """Create an array. Keyword arguments as per :func:`zarr.creation.full`.""" return self._write_op(self._full_nosync, name, fill_value, **kwargs) - def _full_nosync(self, name, fill_value, **kwargs): + def _full_nosync(self, name: str, fill_value: Any, **kwargs: Any) -> Array: path = self._item_path(name) kwargs.setdefault('synchronizer', self._synchronizer) kwargs.setdefault('cache_attrs', self.attrs.cache) return full(store=self._store, path=path, chunk_store=self._chunk_store, fill_value=fill_value, **kwargs) - def array(self, name, data, **kwargs): + def array(self, name: str, data: Any, **kwargs: Any) -> Array: """Create an array. Keyword arguments as per :func:`zarr.creation.array`.""" return self._write_op(self._array_nosync, name, data, **kwargs) - def _array_nosync(self, name, data, **kwargs): + def _array_nosync(self, name: str, data: Any, **kwargs: Any) -> Array: path = self._item_path(name) kwargs.setdefault('synchronizer', self._synchronizer) kwargs.setdefault('cache_attrs', self.attrs.cache) return array(data, store=self._store, path=path, chunk_store=self._chunk_store, **kwargs) - def empty_like(self, name, data, **kwargs): + def empty_like(self, name: str, data: Any, **kwargs: Any) -> Array: """Create an array. Keyword arguments as per :func:`zarr.creation.empty_like`.""" return self._write_op(self._empty_like_nosync, name, data, **kwargs) - def _empty_like_nosync(self, name, data, **kwargs): + def _empty_like_nosync(self, name: str, data: Any, **kwargs: Any) -> Array: path = self._item_path(name) kwargs.setdefault('synchronizer', self._synchronizer) kwargs.setdefault('cache_attrs', self.attrs.cache) return empty_like(data, store=self._store, path=path, chunk_store=self._chunk_store, **kwargs) - def zeros_like(self, name, data, **kwargs): + def zeros_like(self, name: str, data: Any, **kwargs: Any) -> Array: """Create an array. Keyword arguments as per :func:`zarr.creation.zeros_like`.""" return self._write_op(self._zeros_like_nosync, name, data, **kwargs) - def _zeros_like_nosync(self, name, data, **kwargs): + def _zeros_like_nosync(self, name: str, data: Any, **kwargs: Any) -> Array: path = self._item_path(name) kwargs.setdefault('synchronizer', self._synchronizer) kwargs.setdefault('cache_attrs', self.attrs.cache) return zeros_like(data, store=self._store, path=path, chunk_store=self._chunk_store, **kwargs) - def ones_like(self, name, data, **kwargs): + def ones_like(self, name: str, data: Any, **kwargs: Any) -> Array: """Create an array. Keyword arguments as per :func:`zarr.creation.ones_like`.""" return self._write_op(self._ones_like_nosync, name, data, **kwargs) - def _ones_like_nosync(self, name, data, **kwargs): + def _ones_like_nosync(self, name: str, data: Any, **kwargs: Any) -> Array: path = self._item_path(name) kwargs.setdefault('synchronizer', self._synchronizer) kwargs.setdefault('cache_attrs', self.attrs.cache) return ones_like(data, store=self._store, path=path, chunk_store=self._chunk_store, **kwargs) - def full_like(self, name, data, **kwargs): + def full_like(self, name: str, data: Any, **kwargs: Any) -> Array: """Create an array. Keyword arguments as per :func:`zarr.creation.full_like`.""" return self._write_op(self._full_like_nosync, name, data, **kwargs) - def _full_like_nosync(self, name, data, **kwargs): + def _full_like_nosync(self, name: str, data: Any, **kwargs: Any) -> Array: path = self._item_path(name) kwargs.setdefault('synchronizer', self._synchronizer) kwargs.setdefault('cache_attrs', self.attrs.cache) return full_like(data, store=self._store, path=path, chunk_store=self._chunk_store, **kwargs) - def _move_nosync(self, path, new_path): + def _move_nosync(self, path: str, new_path: str): rename(self._store, path, new_path) if self._chunk_store is not None: rename(self._chunk_store, path, new_path) - def move(self, source, dest): + def move(self, source: str, dest: str): """Move contents from one path to another relative to the Group. Parameters @@ -1179,8 +1224,11 @@ def move(self, source, dest): self._write_op(self._move_nosync, source, dest) -def _normalize_store_arg(store, *, storage_options=None, mode="r", - zarr_version=None): +def _normalize_store_arg(store: Optional[BaseStore], + *, + storage_options: Any = None, + mode: str = "r", + zarr_version: Optional[int] = None): if zarr_version is None: zarr_version = getattr(store, '_store_version', DEFAULT_ZARR_VERSION) @@ -1194,8 +1242,14 @@ def _normalize_store_arg(store, *, storage_options=None, mode="r", zarr_version=zarr_version) -def group(store=None, overwrite=False, chunk_store=None, - cache_attrs=True, synchronizer=None, path=None, *, zarr_version=None): +def group(store: Optional[BaseStore] = None, + overwrite: bool = False, + chunk_store: Optional[StoreLike] = None, + cache_attrs: bool = True, + synchronizer: Any = None, + path: Optional[str] = None, + *, + zarr_version: Optional[int] = None): """Create a group. Parameters @@ -1251,6 +1305,7 @@ def group(store=None, overwrite=False, chunk_store=None, raise ValueError(f"path must be provided for a v{zarr_version} group") path = normalize_storage_path(path) + requires_init = True if zarr_version == 2: requires_init = overwrite or not contains_group(store) elif zarr_version == 3: @@ -1265,8 +1320,15 @@ def group(store=None, overwrite=False, chunk_store=None, zarr_version=zarr_version) -def open_group(store=None, mode='a', cache_attrs=True, synchronizer=None, path=None, - chunk_store=None, storage_options=None, *, zarr_version=None): +def open_group(store: Optional[StoreLike] = None, + mode: AccessModes = 'a', + cache_attrs: bool = True, + synchronizer: Any = None, + path: Optional[str] = None, + chunk_store: Optional[StoreLike] = None, + storage_options: Any = None, + *, + zarr_version: Optional[int] = None) -> 'Group': """Open a group using file-mode-like semantics. Parameters diff --git a/zarr/storage.py b/zarr/storage.py index 48b6f049dd..6d1799d25e 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -27,7 +27,7 @@ import warnings import zipfile from collections import OrderedDict -from collections.abc import MutableMapping +from typing import MutableMapping from os import scandir from pickle import PicklingError from threading import Lock, RLock @@ -95,7 +95,7 @@ Path = Union[str, bytes, None] # allow MutableMapping for backwards compatibility -StoreLike = Union[BaseStore, MutableMapping] +StoreLike = Union[BaseStore, MutableMapping[str, Any]] def contains_array(store: StoreLike, path: Path = None) -> bool: @@ -282,7 +282,7 @@ def _require_parent_group( def init_array( store: StoreLike, - shape: Tuple[int, ...], + shape: Union[int, Tuple[int, ...]], chunks: Union[bool, int, Tuple[int, ...]] = True, dtype=None, compressor="default", @@ -710,19 +710,19 @@ class KVStore(Store): store which is likely to expose a MuttableMapping interface, """ - def __init__(self, mutablemapping): + def __init__(self, mutablemapping: MutableMapping[str, Any]): self._mutable_mapping = mutablemapping - def __getitem__(self, key): + def __getitem__(self, key: str): return self._mutable_mapping[key] - def __setitem__(self, key, value): + def __setitem__(self, key: str, value: Any): self._mutable_mapping[key] = value - def __delitem__(self, key): + def __delitem__(self, key: str): del self._mutable_mapping[key] - def get(self, key, default=None): + def get(self, key: str, default: Any = None): return self._mutable_mapping.get(key, default) def values(self): diff --git a/zarr/tests/test_hierarchy.py b/zarr/tests/test_hierarchy.py index 7c2eaa3f75..229f42c775 100644 --- a/zarr/tests/test_hierarchy.py +++ b/zarr/tests/test_hierarchy.py @@ -5,6 +5,7 @@ import shutil import tempfile import textwrap +from typing import Any, Optional, Tuple import unittest import numpy as np @@ -25,7 +26,7 @@ from zarr.hierarchy import Group, group, open_group from zarr.storage import (ABSStore, DBMStore, KVStore, DirectoryStore, FSStore, LMDBStore, LRUStoreCache, MemoryStore, - NestedDirectoryStore, SQLiteStore, ZipStore, + NestedDirectoryStore, SQLiteStore, StoreLike, ZipStore, array_meta_key, atexit_rmglob, atexit_rmtree, data_root, group_meta_key, init_array, init_group, meta_root) from zarr._storage.v3 import (ABSStoreV3, KVStoreV3, DirectoryStoreV3, MemoryStoreV3, @@ -47,8 +48,12 @@ def create_store(): # can be overridden in sub-classes return KVStore(dict()), None - def create_group(self, store=None, path=None, read_only=False, - chunk_store=None, synchronizer=None): + def create_group(self, + store: Optional[StoreLike] = None, + path: Optional[str] = None, + read_only: bool = False, + chunk_store: Optional[StoreLike] = None, + synchronizer: Any = None) -> Group: # can be overridden in sub-classes if store is None: store, chunk_store = self.create_store() @@ -109,7 +114,7 @@ def test_group_init_errors_2(self): Group(store, chunk_store=chunk_store) store.close() - def _subgroup_path(self, group, path): + def _subgroup_path(self, group: Group, path: str) -> Tuple[str, str]: path = path.rstrip('/') group_path = '/'.join([group.path, path]) group_path = group_path.lstrip('/') @@ -283,7 +288,7 @@ def test_rmdir_group_and_array_metadata_files(self): g1.store.rmdir(g2.path) assert group_meta_file not in g1.store - def _dataset_path(self, group, path): + def _dataset_path(self, group: Group, path: str) -> Tuple[str, str]: path = path.rstrip('/') absolute = path.startswith('/') if absolute: @@ -374,6 +379,10 @@ def test_create_dataset(self): assert d.compressor.codec_id == 'zlib' assert 1 == d.compressor.level + # write_empty_chunks + d = g.create_dataset('fff', shape=1000, dtype='u1', write_empty_chunks=False) + assert d.write_empty_chunks is False + g.store.close() def test_require_dataset(self): @@ -422,6 +431,14 @@ def test_require_dataset(self): g.require_dataset('foo', shape=1000, chunks=100, dtype='i2', exact=True) + # write_empty_chunks + d4 = g.require_dataset('baz', shape=1000, dtype='u1') + d5 = g.require_dataset('baz', shape=1000, dtype='u1', write_empty_chunks=False) + assert d5.write_empty_chunks is False + + d6 = g.require_dataset('baz', shape=1000, dtype='u1', write_empty_chunks=True) + assert d6.write_empty_chunks is True + g.store.close() def test_create_errors(self): @@ -1083,7 +1100,7 @@ def test_context_manager(self): @pytest.mark.parametrize('chunk_dict', [False, True]) -def test_group_init_from_dict(chunk_dict): +def test_group_init_from_dict(chunk_dict: bool): if chunk_dict: store, chunk_store = dict(), dict() else: @@ -1590,7 +1607,7 @@ def test_group(zarr_version): @pytest.mark.parametrize('zarr_version', _VERSIONS) -def test_open_group(zarr_version): +def test_open_group(zarr_version: int): # test the open_group() convenience function store = 'data/group.zarr' @@ -1657,7 +1674,7 @@ def test_open_group(zarr_version): @pytest.mark.parametrize('zarr_version', _VERSIONS) -def test_group_completions(zarr_version): +def test_group_completions(zarr_version: int): path = None if zarr_version == 2 else 'group1' g = group(path=path, zarr_version=zarr_version) d = dir(g) @@ -1688,7 +1705,7 @@ def test_group_completions(zarr_version): @pytest.mark.parametrize('zarr_version', _VERSIONS) -def test_group_key_completions(zarr_version): +def test_group_key_completions(zarr_version: int): path = None if zarr_version == 2 else 'group1' g = group(path=path, zarr_version=zarr_version) d = dir(g) @@ -1760,7 +1777,7 @@ def test_group_key_completions(zarr_version): assert 'asdf;' in k -def _check_tree(g, expect_bytes, expect_text): +def _check_tree(g, expect_bytes: bytes, expect_text: str): assert expect_bytes == bytes(g.tree()) assert expect_text == str(g.tree()) expect_repr = expect_text @@ -1772,7 +1789,7 @@ def _check_tree(g, expect_bytes, expect_text): @pytest.mark.parametrize('zarr_version', _VERSIONS) -def test_tree(zarr_version): +def test_tree(zarr_version: int): # setup path = None if zarr_version == 2 else 'group1' g1 = group(path=path, zarr_version=zarr_version) @@ -1873,7 +1890,7 @@ def test_group_mismatched_store_versions(): @pytest.mark.parametrize('zarr_version', _VERSIONS) -def test_open_group_from_paths(zarr_version): +def test_open_group_from_paths(zarr_version: int): """Verify zarr_version is applied to both the store and chunk_store.""" store = tempfile.mkdtemp() chunk_store = tempfile.mkdtemp() diff --git a/zarr/util.py b/zarr/util.py index cc3bd50356..8f943a9c85 100644 --- a/zarr/util.py +++ b/zarr/util.py @@ -56,7 +56,7 @@ def json_loads(s: str) -> Dict[str, Any]: return json.loads(ensure_text(s, 'ascii')) -def normalize_shape(shape) -> Tuple[int]: +def normalize_shape(shape: Any) -> Tuple[int, ...]: """Convenience function to normalize the `shape` argument.""" if shape is None: From 84ca00aeb8bf6a0a68083638f119fe2f843e6cb9 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 22 Jun 2022 15:31:37 -0400 Subject: [PATCH 02/14] expand docstring --- zarr/hierarchy.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/zarr/hierarchy.py b/zarr/hierarchy.py index bb25e3438c..91b6b3dc62 100644 --- a/zarr/hierarchy.py +++ b/zarr/hierarchy.py @@ -1008,7 +1008,10 @@ def require_dataset(self, Arrays are known as "datasets" in HDF5 terminology. For compatibility with h5py, Zarr groups also implement the create_dataset() method. - Other `kwargs` are as per :func:`zarr.hierarchy.Group.create_dataset`. + Other `kwargs` are as per :func:`zarr.hierarchy.Group.create_dataset` + if a new array is being created; if the array already exists, keyword + arguments for :func:`zarr.core.Array` are extracted from `kwargs` + and supplied to that constructor. Parameters ---------- From 2d96cf1529bd27b75783268d89f33628d24f36ae Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Thu, 23 Jun 2022 07:48:51 -0400 Subject: [PATCH 03/14] using typing_extensions to import Literal, and remove an unused variable assignment from tests --- zarr/hierarchy.py | 3 ++- zarr/tests/test_hierarchy.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/zarr/hierarchy.py b/zarr/hierarchy.py index 91b6b3dc62..2ca02c23c9 100644 --- a/zarr/hierarchy.py +++ b/zarr/hierarchy.py @@ -4,7 +4,6 @@ Callable, Generator, List, - Literal, Optional, Tuple, Union, @@ -12,6 +11,8 @@ MutableMapping ) +from typing_extensions import Literal + import numpy as np import numpy.typing as npt diff --git a/zarr/tests/test_hierarchy.py b/zarr/tests/test_hierarchy.py index 229f42c775..a1b4e85ddb 100644 --- a/zarr/tests/test_hierarchy.py +++ b/zarr/tests/test_hierarchy.py @@ -432,7 +432,7 @@ def test_require_dataset(self): exact=True) # write_empty_chunks - d4 = g.require_dataset('baz', shape=1000, dtype='u1') + _ = g.require_dataset('baz', shape=1000, dtype='u1') d5 = g.require_dataset('baz', shape=1000, dtype='u1', write_empty_chunks=False) assert d5.write_empty_chunks is False From 43c5faa97b37d8cde8f06582c73e82d46ca6dccf Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Thu, 23 Jun 2022 08:14:43 -0400 Subject: [PATCH 04/14] weaken typing of dtypes --- zarr/hierarchy.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/zarr/hierarchy.py b/zarr/hierarchy.py index 2ca02c23c9..2c585957b5 100644 --- a/zarr/hierarchy.py +++ b/zarr/hierarchy.py @@ -14,7 +14,6 @@ from typing_extensions import Literal import numpy as np -import numpy.typing as npt from zarr._storage.store import (StoreV3, _get_metadata_suffix, data_root, meta_root, DEFAULT_ZARR_VERSION, assert_zarr_v3_api_available) @@ -1001,7 +1000,7 @@ def _create_dataset_nosync(self, name: str, data: Any = None, **kwargs: Any) -> def require_dataset(self, name: str, shape: Union[int, Tuple[int, ...]], - dtype: Optional[npt.DTypeLike] = None, + dtype: Any = None, exact: bool = False, **kwargs: Any): """Obtain an array, creating if it doesn't exist. @@ -1034,7 +1033,7 @@ def require_dataset(self, def _require_dataset_nosync(self, name: str, shape: Tuple[int, ...], - dtype: Optional[npt.DTypeLike] = None, + dtype: Any = None, exact: bool = False, **kwargs: Any) -> Array: From e20d0c9a0d0a9af2640315a97e294c5620f6555c Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 26 Jun 2022 21:23:14 -0400 Subject: [PATCH 05/14] more typing --- zarr/_storage/store.py | 37 +++++++++++------------ zarr/_storage/v3.py | 16 +++++----- zarr/hierarchy.py | 34 +++++++++++---------- zarr/meta.py | 30 +++++++++---------- zarr/storage.py | 67 +++++++++++++++++++++++++----------------- 5 files changed, 101 insertions(+), 83 deletions(-) diff --git a/zarr/_storage/store.py b/zarr/_storage/store.py index fa19dbd9c9..a3a8ff977a 100644 --- a/zarr/_storage/store.py +++ b/zarr/_storage/store.py @@ -1,8 +1,9 @@ import abc import os -from typing import MutableMapping +from typing import MutableMapping, Set, Tuple from string import ascii_letters, digits from typing import Any, List, Mapping, Optional, Union +from zarr._storage.v3 import KVStoreV3 from zarr.meta import Metadata2, Metadata3 from zarr.util import normalize_storage_path @@ -135,7 +136,7 @@ def _ensure_store(store: Any): class Store(BaseStore): """Abstract store class used by implementations following the Zarr v2 spec. - Adds public `listdir`, `rename`, and `rmdir` methods on top of BaseStore. + Adds public `listdir` and `rmdir` methods on top of BaseStore. .. added: 2.11.0 @@ -155,11 +156,11 @@ def rmdir(self, path: str = "") -> None: class StoreV3(BaseStore): - _store_version = 3 + _store_version: int = 3 _metadata_class = Metadata3 _valid_key_characters = set(ascii_letters + digits + "/.-_") - def _valid_key(self, key: str) -> bool: + def _valid_key(self, key: Any) -> bool: """ Verify that a key conforms to the specification. @@ -204,16 +205,16 @@ def _validate_key(self, key: str): if key.endswith('/'): raise ValueError("keys may not end in /") - def list_prefix(self, prefix): + def list_prefix(self, prefix: str) -> List[str]: if prefix.startswith('/'): raise ValueError("prefix must not begin with /") # TODO: force prefix to end with /? return [k for k in self.list() if k.startswith(prefix)] - def erase(self, key): + def erase(self, key: str): self.__delitem__(key) - def erase_prefix(self, prefix): + def erase_prefix(self, prefix: str): assert prefix.endswith("/") if prefix == "/": @@ -223,7 +224,7 @@ def erase_prefix(self, prefix): for key in all_keys: self.erase(key) - def list_dir(self, prefix): + def list_dir(self, prefix: str) -> Tuple[List[str], List[str]]: """ TODO: carefully test this with trailing/leading slashes """ @@ -232,8 +233,8 @@ def list_dir(self, prefix): all_keys = self.list_prefix(prefix) len_prefix = len(prefix) - keys = [] - prefixes = [] + keys: List[str] = [] + prefixes: List[str] = [] for k in all_keys: trail = k[len_prefix:] if "/" not in trail: @@ -242,29 +243,29 @@ def list_dir(self, prefix): prefixes.append(prefix + trail.split("/", maxsplit=1)[0] + "/") return keys, list(set(prefixes)) - def list(self): + def list(self) -> List[str]: return list(self.keys()) - def __contains__(self, key): + def __contains__(self, key: Any) -> bool: return key in self.list() @abc.abstractmethod - def __setitem__(self, key, value): + def __setitem__(self, key: str, value: Any): """Set a value.""" @abc.abstractmethod - def __getitem__(self, key): + def __getitem__(self, key: str) -> Any: """Get a value.""" def clear(self): """Remove all items from store.""" self.erase_prefix("/") - def __eq__(self, other): + def __eq__(self, other: Any): return NotImplemented @staticmethod - def _ensure_store(store): + def _ensure_store(store: Any) -> Union['StoreV3', 'KVStoreV3', None]: """ We want to make sure internally that zarr stores are always a class with a specific interface derived from ``Store``, which is slightly @@ -306,7 +307,7 @@ def _ensure_store(store): # allow MutableMapping for backwards compatibility -StoreLike = Union[BaseStore, MutableMapping] +StoreLike = Union[BaseStore, MutableMapping[str, Any]] def _path_to_prefix(path: Optional[str]) -> str: @@ -411,7 +412,7 @@ def _rmdir_from_keys_v3(store: StoreV3, path: str = "") -> None: def _listdir_from_keys(store: BaseStore, path: Optional[str] = None) -> List[str]: # assume path already normalized prefix = _path_to_prefix(path) - children = set() + children: Set[str] = set() for key in list(store.keys()): if key.startswith(prefix) and len(key) > len(prefix): suffix = key[len(prefix):] diff --git a/zarr/_storage/v3.py b/zarr/_storage/v3.py index 540b62ef7e..345581cc32 100644 --- a/zarr/_storage/v3.py +++ b/zarr/_storage/v3.py @@ -1,14 +1,14 @@ import os import shutil from collections import OrderedDict -from collections.abc import MutableMapping from threading import Lock -from typing import Union, Dict, Any +from typing import Optional, Union, Dict, Any from zarr.errors import ( MetadataError, ReadOnlyError, ) +from zarr.hierarchy import AccessModes from zarr.util import (buffer_size, json_loads, normalize_storage_path) from zarr._storage.absstore import ABSStoreV3 # noqa: F401 @@ -32,7 +32,7 @@ StoreV3) from zarr.storage import (DBMStore, ConsolidatedMetadataStore, DirectoryStore, FSStore, KVStore, LMDBStore, LRUStoreCache, MemoryStore, MongoDBStore, RedisStore, - SQLiteStore, ZipStore, _getsize) + SQLiteStore, StoreLike, ZipStore, _getsize) __doctest_requires__ = { ('RedisStore', 'RedisStore.*'): ['redis'], @@ -52,8 +52,6 @@ Path = Union[str, bytes, None] # allow MutableMapping for backwards compatibility -StoreLike = Union[BaseStore, MutableMapping] - class RmdirV3(): """Mixin class that can be used to ensure override of any existing v2 rmdir class.""" @@ -480,7 +478,7 @@ def rmdir(self, path=None): class LRUStoreCacheV3(RmdirV3, LRUStoreCache, StoreV3): - def __init__(self, store, max_size: int): + def __init__(self, store: Any, max_size: int): self._store = StoreV3._ensure_store(store) self._max_size = max_size self._current_size = 0 @@ -551,11 +549,13 @@ def __init__(self, store: StoreLike, metadata_key=meta_root + "consolidated/.zme # decode metadata self.meta_store: Store = KVStoreV3(meta["metadata"]) - def rmdir(self, key): + def rmdir(self, path: str = ""): raise ReadOnlyError() -def _normalize_store_arg_v3(store: Any, storage_options=None, mode="r") -> BaseStore: +def _normalize_store_arg_v3(store: StoreLike, + storage_options: Optional[Dict[str, Any]] = None, + mode: AccessModes = "r") -> BaseStore: # default to v2 store for backward compatibility zarr_version = getattr(store, '_store_version', 3) if zarr_version != 3: diff --git a/zarr/hierarchy.py b/zarr/hierarchy.py index 2c585957b5..869beba345 100644 --- a/zarr/hierarchy.py +++ b/zarr/hierarchy.py @@ -2,6 +2,7 @@ from typing import ( Any, Callable, + Dict, Generator, List, Optional, @@ -15,7 +16,7 @@ import numpy as np -from zarr._storage.store import (StoreV3, _get_metadata_suffix, data_root, meta_root, +from zarr._storage.store import (Store, StoreV3, _get_metadata_suffix, data_root, meta_root, DEFAULT_ZARR_VERSION, assert_zarr_v3_api_available) from zarr.attrs import Attributes from zarr.core import Array @@ -129,7 +130,7 @@ class Group(MutableMapping[str, Union['Group', Array]]): """ def __init__(self, - store: BaseStore, + store: StoreLike, path: Optional[str] = None, read_only: bool = False, chunk_store: Optional[StoreLike] = None, @@ -176,6 +177,7 @@ def __init__(self, if self._version == 2: raise GroupNotFoundError(path) else: + cast(StoreV3, self._store) implicit_prefix = meta_root + self._key_prefix if self._store.list_prefix(implicit_prefix): # implicit group does not have any metadata @@ -243,7 +245,7 @@ def synchronizer(self) -> Any: return self._synchronizer @property - def attrs(self) -> MutableMapping[str, Any]: + def attrs(self) -> Attributes: """A MutableMapping containing user-defined attributes. Note that attribute values must be JSON serializable.""" return self._attrs @@ -283,6 +285,7 @@ def __iter__(self) -> Generator[str, None, None]: """ if getattr(self._store, '_store_version', 2) == 2: + cast(Store, self._store) for key in sorted(listdir(self._store, self._path)): path = self._key_prefix + key if (contains_array(self._store, path) or @@ -376,17 +379,17 @@ def __getstate__(self): return (self._store, self._path, self._read_only, self._chunk_store, self.attrs.cache, self._synchronizer) - def __setstate__(self, state): + def __setstate__(self, state: Tuple[Any, ...]): self.__init__(*state) - def _item_path(self, item: str) -> str: + def _item_path(self, item: Any) -> str: absolute = isinstance(item, str) and item and item[0] == '/' path = normalize_storage_path(item) if not absolute and self._path: path = self._key_prefix + path return path - def __contains__(self, item: str): + def __contains__(self, item: Any) -> bool: """Test for group membership. Examples @@ -439,6 +442,7 @@ def __getitem__(self, item: str) -> Union['Group', Array]: chunk_store=self._chunk_store, cache_attrs=self.attrs.cache, synchronizer=self._synchronizer, zarr_version=self._version) elif self._version == 3: + cast(StoreV3, self._store) implicit_group = meta_root + path + '/' # non-empty folder in the metadata path implies an implicit group if self._store.list_prefix(implicit_group): @@ -450,13 +454,13 @@ def __getitem__(self, item: str) -> Union['Group', Array]: else: raise KeyError(item) - def __setitem__(self, item, value): + def __setitem__(self, item: str, value: Any): self.array(item, value, overwrite=True) - def __delitem__(self, item): + def __delitem__(self, item: str): return self._write_op(self._delitem_nosync, item) - def _delitem_nosync(self, item): + def _delitem_nosync(self, item: str): path = self._item_path(item) if contains_array(self._store, path) or \ contains_group(self._store, path, explicit_only=False): @@ -464,14 +468,14 @@ def _delitem_nosync(self, item): else: raise KeyError(item) - def __getattr__(self, item): + def __getattr__(self, item: str): # allow access to group members via dot notation try: return self.__getitem__(item) except KeyError: raise AttributeError - def __dir__(self): + def __dir__(self) -> List[str]: # noinspection PyUnresolvedReferences base = super().__dir__() keys = sorted(set(base + list(self))) @@ -1227,10 +1231,10 @@ def move(self, source: str, dest: str): self._write_op(self._move_nosync, source, dest) -def _normalize_store_arg(store: Optional[BaseStore], +def _normalize_store_arg(store: Optional[StoreLike], *, - storage_options: Any = None, - mode: str = "r", + storage_options: Optional[Dict[str, Any]] = None, + mode: AccessModes = "r", zarr_version: Optional[int] = None): if zarr_version is None: zarr_version = getattr(store, '_store_version', DEFAULT_ZARR_VERSION) @@ -1245,7 +1249,7 @@ def _normalize_store_arg(store: Optional[BaseStore], zarr_version=zarr_version) -def group(store: Optional[BaseStore] = None, +def group(store: Optional[StoreLike] = None, overwrite: bool = False, chunk_store: Optional[StoreLike] = None, cache_attrs: bool = True, diff --git a/zarr/meta.py b/zarr/meta.py index bb4bae4199..76d7b632bc 100644 --- a/zarr/meta.py +++ b/zarr/meta.py @@ -9,7 +9,7 @@ from zarr.errors import MetadataError from zarr.util import json_dumps, json_loads -from typing import cast, Union, Any, List, Mapping as MappingType, Optional +from typing import Dict, cast, Union, Any, List, Mapping as MappingType, Optional ZARR_FORMAT = 2 ZARR_FORMAT_v3 = 3 @@ -88,7 +88,7 @@ class Metadata2: ZARR_FORMAT = ZARR_FORMAT @classmethod - def parse_metadata(cls, s: Union[MappingType, str]) -> MappingType[str, Any]: + def parse_metadata(cls, s: Union[MappingType[str, Any], str]) -> MappingType[str, Any]: # Here we allow that a store may return an already-parsed metadata object, # or a string of JSON that we will parse here. We allow for an already-parsed @@ -106,7 +106,7 @@ def parse_metadata(cls, s: Union[MappingType, str]) -> MappingType[str, Any]: return meta @classmethod - def decode_array_metadata(cls, s: Union[MappingType, str]) -> MappingType[str, Any]: + def decode_array_metadata(cls, s: Union[MappingType[str, Any], str]) -> MappingType[str, Any]: meta = cls.parse_metadata(s) # check metadata format @@ -174,14 +174,14 @@ def encode_array_metadata(cls, meta: MappingType[str, Any]) -> bytes: return json_dumps(meta) @classmethod - def encode_dtype(cls, d: np.dtype): + def encode_dtype(cls, d: np.dtype[Any]): if d.fields is None: return d.str else: return d.descr @classmethod - def _decode_dtype_descr(cls, d) -> List[Any]: + def _decode_dtype_descr(cls, d: Any) -> List[Any]: # need to convert list of lists to list of tuples if isinstance(d, list): # recurse to handle nested structures @@ -189,12 +189,12 @@ def _decode_dtype_descr(cls, d) -> List[Any]: return d @classmethod - def decode_dtype(cls, d) -> np.dtype: + def decode_dtype(cls, d: Any) -> np.dtype[Any]: d = cls._decode_dtype_descr(d) return np.dtype(d) @classmethod - def decode_group_metadata(cls, s: Union[MappingType, str]) -> MappingType[str, Any]: + def decode_group_metadata(cls, s: Union[MappingType[str, Any], str]) -> MappingType[str, Any]: meta = cls.parse_metadata(s) # check metadata format version @@ -208,13 +208,13 @@ def decode_group_metadata(cls, s: Union[MappingType, str]) -> MappingType[str, A # N.B., keep `meta` parameter as a placeholder for future # noinspection PyUnusedLocal @classmethod - def encode_group_metadata(cls, meta=None) -> bytes: + def encode_group_metadata(cls, meta: Any = None) -> bytes: meta = dict(zarr_format=cls.ZARR_FORMAT) return json_dumps(meta) @classmethod def decode_fill_value( - cls, v: Any, dtype: np.dtype, object_codec: Any = None + cls, v: Any, dtype: np.dtype[Any], object_codec: Any = None ) -> Any: # early out if v is None: @@ -264,7 +264,7 @@ def decode_fill_value( @classmethod def encode_fill_value( - cls, v: Any, dtype: np.dtype, object_codec: Any = None + cls, v: Any, dtype: np.dtype[Any], object_codec: Any = None ) -> Any: # early out if v is None: @@ -347,7 +347,7 @@ def encode_dtype(cls, d): return get_extended_dtype_info(np.dtype(d)) @classmethod - def decode_group_metadata(cls, s: Union[MappingType, str]) -> MappingType[str, Any]: + def decode_group_metadata(cls, s: Union[MappingType[str, Any], str]) -> MappingType[str, Any]: meta = cls.parse_metadata(s) # 1 / 0 # # check metadata format version @@ -386,7 +386,7 @@ def encode_hierarchy_metadata(cls, meta=None) -> bytes: @classmethod def decode_hierarchy_metadata( - cls, s: Union[MappingType, str] + cls, s: Union[MappingType[str, Any], str] ) -> MappingType[str, Any]: meta = cls.parse_metadata(s) # check metadata format @@ -403,7 +403,7 @@ def decode_hierarchy_metadata( return meta @classmethod - def _encode_codec_metadata(cls, codec: Codec) -> Optional[Mapping]: + def _encode_codec_metadata(cls, codec: Codec) -> Optional[Dict[str, Any]]: if codec is None: return None @@ -430,7 +430,7 @@ def _encode_codec_metadata(cls, codec: Codec) -> Optional[Mapping]: return meta @classmethod - def _decode_codec_metadata(cls, meta: Optional[Mapping]) -> Optional[Codec]: + def _decode_codec_metadata(cls, meta: Optional[MappingType[str, Any]]) -> Optional[Codec]: if meta is None: return None @@ -460,7 +460,7 @@ def _decode_codec_metadata(cls, meta: Optional[Mapping]) -> Optional[Codec]: return codec @classmethod - def decode_array_metadata(cls, s: Union[MappingType, str]) -> MappingType[str, Any]: + def decode_array_metadata(cls, s: Union[MappingType[str, Any], str]) -> MappingType[str, Any]: meta = cls.parse_metadata(s) # extract array metadata fields diff --git a/zarr/storage.py b/zarr/storage.py index 6d1799d25e..25a311715c 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -18,6 +18,7 @@ import errno import glob import multiprocessing +from multiprocessing.sharedctypes import Value import operator import os import re @@ -27,8 +28,8 @@ import warnings import zipfile from collections import OrderedDict -from typing import MutableMapping -from os import scandir +from typing import Callable, Generator, MutableMapping +from os import PathLike, scandir from pickle import PicklingError from threading import Lock, RLock from typing import Optional, Union, List, Tuple, Dict, Any @@ -51,6 +52,7 @@ FSPathExistNotDir, ReadOnlyError, ) +from zarr.hierarchy import AccessModes from zarr.meta import encode_array_metadata, encode_group_metadata from zarr.util import (buffer_size, json_loads, nolock, normalize_chunks, normalize_dimension_separator, @@ -128,7 +130,7 @@ def contains_group(store: StoreLike, path: Path = None, explicit_only=True) -> b return False -def _normalize_store_arg_v2(store: Any, storage_options=None, mode="r") -> BaseStore: +def _normalize_store_arg_v2(store: Any, storage_options: Optional[Dict[str, Any]] = None, mode="r") -> BaseStore: # default to v2 store for backward compatibility zarr_version = getattr(store, '_store_version', 2) if zarr_version != 2: @@ -155,19 +157,24 @@ def _normalize_store_arg_v2(store: Any, storage_options=None, mode="r") -> BaseS return store -def normalize_store_arg(store: Any, storage_options=None, mode="r", *, - zarr_version=None) -> BaseStore: +def normalize_store_arg(store: StoreLike, + storage_options: Optional[Dict[str, Any]] = None, + mode: AccessModes = "r", *, + zarr_version: Optional[int] = None) -> BaseStore: + + result: BaseStore if zarr_version is None: # default to v2 store for backward compatibility zarr_version = getattr(store, "_store_version", DEFAULT_ZARR_VERSION) - elif zarr_version not in [2, 3]: - raise ValueError("zarr_version must be either 2 or 3") + if zarr_version == 2: - normalize_store = _normalize_store_arg_v2 + result = _normalize_store_arg_v2(store, storage_options, mode) elif zarr_version == 3: from zarr._storage.v3 import _normalize_store_arg_v3 - normalize_store = _normalize_store_arg_v3 - return normalize_store(store, storage_options, mode) + result = _normalize_store_arg_v3(store, storage_options, mode) + else: + raise ValueError("zarr_version must be either 2 or 3") + return result def rmdir(store: StoreLike, path: Path = None): @@ -590,7 +597,7 @@ def init_group( store: StoreLike, overwrite: bool = False, path: Path = None, - chunk_store: StoreLike = None, + chunk_store: Optional[StoreLike] = None, ): """Initialize a group store. Note that this is a low-level function and there should be no need to call this directly from user code. @@ -637,7 +644,7 @@ def _init_group_metadata( store: StoreLike, overwrite: Optional[bool] = False, path: Optional[str] = None, - chunk_store: StoreLike = None, + chunk_store: Optional[StoreLike] = None, ): store_version = getattr(store, '_store_version', 2) @@ -691,7 +698,7 @@ def _init_group_metadata( store[key] = encode_group_metadata(meta) -def _dict_store_keys(d: Dict, prefix="", cls=dict): +def _dict_store_keys(d: Dict[str, Any], prefix="", cls: Any = dict) -> Generator[Any, None, None]: for k in d.keys(): v = d[k] if isinstance(v, cls): @@ -770,7 +777,10 @@ class MemoryStore(Store): """ - def __init__(self, root=None, cls=dict, dimension_separator=None): + def __init__(self, + root: Any = None, + cls: Any = dict, + dimension_separator: Optional[str] = None): if root is None: self.root = cls() else: @@ -779,10 +789,10 @@ def __init__(self, root=None, cls=dict, dimension_separator=None): self.write_mutex = Lock() self._dimension_separator = dimension_separator - def __getstate__(self): + def __getstate__(self) -> Tuple[Any, Any]: return self.root, self.cls - def __setstate__(self, state): + def __setstate__(self, state: Tuple[Any, ...]): root, cls = state self.__init__(root=root, cls=cls) @@ -797,7 +807,7 @@ def _get_parent(self, item: str): raise KeyError(item) return parent, segments[-1] - def _require_parent(self, item): + def _require_parent(self, item: str): parent = self.root # split the item segments = item.split('/') @@ -839,7 +849,7 @@ def __delitem__(self, item: str): except KeyError: raise KeyError(item) - def __contains__(self, item: str): # type: ignore[override] + def __contains__(self, item: str) -> bool: # type: ignore[override] try: parent, key = self._get_parent(item) value = parent[key] @@ -848,18 +858,18 @@ def __contains__(self, item: str): # type: ignore[override] else: return not isinstance(value, self.cls) - def __eq__(self, other): + def __eq__(self, other) -> bool: return ( isinstance(other, MemoryStore) and self.root == other.root and self.cls == other.cls ) - def keys(self): + def keys(self) -> Generator[str, None, None]: for k in _dict_store_keys(self.root, cls=self.cls): yield k - def __iter__(self): + def __iter__(self) -> Generator[str, None, None]: return self.keys() def __len__(self) -> int: @@ -1008,7 +1018,10 @@ class DirectoryStore(Store): """ - def __init__(self, path, normalize_keys=False, dimension_separator=None): + def __init__(self, + path: str, + normalize_keys: bool = False, + dimension_separator: Optional[str] = None): # guard conditions path = os.path.abspath(path) @@ -1019,11 +1032,11 @@ def __init__(self, path, normalize_keys=False, dimension_separator=None): self.normalize_keys = normalize_keys self._dimension_separator = dimension_separator - def _normalize_key(self, key): + def _normalize_key(self, key: str): return key.lower() if self.normalize_keys else key @staticmethod - def _fromfile(fn): + def _fromfile(fn: str): """ Read data from a file Parameters @@ -1040,7 +1053,7 @@ def _fromfile(fn): return f.read() @staticmethod - def _tofile(a, fn): + def _tofile(a: Any, fn: str): """ Write data to a file Parameters @@ -1058,7 +1071,7 @@ def _tofile(a, fn): with open(fn, mode='wb') as f: f.write(a) - def __getitem__(self, key): + def __getitem__(self, key: str): key = self._normalize_key(key) filepath = os.path.join(self.path, key) if os.path.isfile(filepath): @@ -1066,7 +1079,7 @@ def __getitem__(self, key): else: raise KeyError(key) - def __setitem__(self, key, value): + def __setitem__(self, key: str, value: Any): key = self._normalize_key(key) # coerce to flat, contiguous array (ideally without copying) From 2443b0aba690031d27481ab7bbd5ba2c44cc80d0 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 26 Jun 2022 22:17:58 -0400 Subject: [PATCH 06/14] type annotations and some explicit casts --- zarr/_storage/v3.py | 31 +++++++++++++++++-------------- zarr/hierarchy.py | 38 ++++++++++++++++++++++++-------------- zarr/storage.py | 15 ++++++++------- 3 files changed, 49 insertions(+), 35 deletions(-) diff --git a/zarr/_storage/v3.py b/zarr/_storage/v3.py index 345581cc32..61d740aad2 100644 --- a/zarr/_storage/v3.py +++ b/zarr/_storage/v3.py @@ -1,4 +1,5 @@ import os + import shutil from collections import OrderedDict from threading import Lock @@ -53,6 +54,7 @@ Path = Union[str, bytes, None] # allow MutableMapping for backwards compatibility + class RmdirV3(): """Mixin class that can be used to ensure override of any existing v2 rmdir class.""" @@ -555,36 +557,37 @@ def rmdir(self, path: str = ""): def _normalize_store_arg_v3(store: StoreLike, storage_options: Optional[Dict[str, Any]] = None, - mode: AccessModes = "r") -> BaseStore: + mode: AccessModes = "r") -> StoreV3: + result: StoreV3 # default to v2 store for backward compatibility zarr_version = getattr(store, '_store_version', 3) if zarr_version != 3: raise ValueError("store must be a version 3 store") if store is None: - store = KVStoreV3(dict()) + result = KVStoreV3(dict()) # add default zarr.json metadata - store['zarr.json'] = store._metadata_class.encode_hierarchy_metadata(None) - return store + result['zarr.json'] = store._metadata_class.encode_hierarchy_metadata(None) + return result if isinstance(store, os.PathLike): - store = os.fspath(store) + result = os.fspath(store) if isinstance(store, str): if "://" in store or "::" in store: - store = FSStoreV3(store, mode=mode, **(storage_options or {})) + result = FSStoreV3(store, mode=mode, **(storage_options or {})) elif storage_options: raise ValueError("storage_options passed with non-fsspec path") elif store.endswith('.zip'): - store = ZipStoreV3(store, mode=mode) + result = ZipStoreV3(store, mode=mode) elif store.endswith('.n5'): raise NotImplementedError("N5Store not yet implemented for V3") # return N5StoreV3(store) else: - store = DirectoryStoreV3(store) + result = DirectoryStoreV3(store) # add default zarr.json metadata - store['zarr.json'] = store._metadata_class.encode_hierarchy_metadata(None) - return store + result['zarr.json'] = result._metadata_class.encode_hierarchy_metadata(None) + return result else: - store = StoreV3._ensure_store(store) - if 'zarr.json' not in store: + result = StoreV3._ensure_store(store) + if 'zarr.json' not in result: # add default zarr.json metadata - store['zarr.json'] = store._metadata_class.encode_hierarchy_metadata(None) - return store + result['zarr.json'] = store._metadata_class.encode_hierarchy_metadata(None) + return result diff --git a/zarr/hierarchy.py b/zarr/hierarchy.py index 869beba345..433906f3b5 100644 --- a/zarr/hierarchy.py +++ b/zarr/hierarchy.py @@ -158,11 +158,11 @@ def __init__(self, self._version = zarr_version if self._version == 3: - cast(StoreV3, self._store) + _store = cast(StoreV3, self._store) self._data_key_prefix = data_root + self._key_prefix self._data_path = data_root + self._path - self._hierarchy_metadata = _get_hierarchy_metadata(store=self._store) - self._metadata_key_suffix = _get_metadata_suffix(store=self._store) + self._hierarchy_metadata = _get_hierarchy_metadata(store=_store) + self._metadata_key_suffix = _get_metadata_suffix(store=_store) # guard conditions if contains_array(store, path=self._path): @@ -177,9 +177,9 @@ def __init__(self, if self._version == 2: raise GroupNotFoundError(path) else: - cast(StoreV3, self._store) + _store = cast(StoreV3, self._store) implicit_prefix = meta_root + self._key_prefix - if self._store.list_prefix(implicit_prefix): + if _store.list_prefix(implicit_prefix): # implicit group does not have any metadata self._meta = None else: @@ -295,6 +295,7 @@ def __iter__(self) -> Generator[str, None, None]: # TODO: Should this iterate over data folders and/or metadata # folders and/or metadata files + self._store = cast(StoreV3, self._store) dir_path = meta_root + self._key_prefix name_start = len(dir_path) keys, prefixes = self._store.list_dir(dir_path) @@ -442,7 +443,7 @@ def __getitem__(self, item: str) -> Union['Group', Array]: chunk_store=self._chunk_store, cache_attrs=self.attrs.cache, synchronizer=self._synchronizer, zarr_version=self._version) elif self._version == 3: - cast(StoreV3, self._store) + self._store = cast(StoreV3, self._store) implicit_group = meta_root + path + '/' # non-empty folder in the metadata path implies an implicit group if self._store.list_prefix(implicit_group): @@ -477,7 +478,7 @@ def __getattr__(self, item: str): def __dir__(self) -> List[str]: # noinspection PyUnresolvedReferences - base = super().__dir__() + base = list(super().__dir__()) keys = sorted(set(base + list(self))) keys = [k for k in keys if is_valid_python_name(k)] return keys @@ -1235,18 +1236,27 @@ def _normalize_store_arg(store: Optional[StoreLike], *, storage_options: Optional[Dict[str, Any]] = None, mode: AccessModes = "r", - zarr_version: Optional[int] = None): + zarr_version: Optional[int] = None) -> BaseStore: + result: BaseStore if zarr_version is None: zarr_version = getattr(store, '_store_version', DEFAULT_ZARR_VERSION) - - if zarr_version != 2: + if zarr_version == 2: + pass + elif zarr_version == 3: assert_zarr_v3_api_available() + else: + raise NotImplementedError(f"Zarr version must be 2 or 3, not {zarr_version}") if store is None: - return MemoryStore() if zarr_version == 2 else MemoryStoreV3() - return normalize_store_arg(store, - storage_options=storage_options, mode=mode, - zarr_version=zarr_version) + if zarr_version == 2: + result = MemoryStore() + else: + result = MemoryStoreV3() + else: + result = normalize_store_arg(store, + storage_options=storage_options, mode=mode, + zarr_version=zarr_version) + return result def group(store: Optional[StoreLike] = None, diff --git a/zarr/storage.py b/zarr/storage.py index 25a311715c..fcb6774cbd 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -18,7 +18,6 @@ import errno import glob import multiprocessing -from multiprocessing.sharedctypes import Value import operator import os import re @@ -28,8 +27,8 @@ import warnings import zipfile from collections import OrderedDict -from typing import Callable, Generator, MutableMapping -from os import PathLike, scandir +from typing import Generator, MutableMapping +from os import scandir from pickle import PicklingError from threading import Lock, RLock from typing import Optional, Union, List, Tuple, Dict, Any @@ -60,7 +59,7 @@ normalize_shape, normalize_storage_path, retry_call) from zarr._storage.absstore import ABSStore # noqa: F401 -from zarr._storage.store import (_get_hierarchy_metadata, # noqa: F401 +from zarr._storage.store import (StoreV3, _get_hierarchy_metadata, # noqa: F401 _get_metadata_suffix, _listdir_from_keys, _rename_from_keys, @@ -130,7 +129,9 @@ def contains_group(store: StoreLike, path: Path = None, explicit_only=True) -> b return False -def _normalize_store_arg_v2(store: Any, storage_options: Optional[Dict[str, Any]] = None, mode="r") -> BaseStore: +def _normalize_store_arg_v2(store: Any, + storage_options: Optional[Dict[str, Any]] = None, + mode="r") -> BaseStore: # default to v2 store for backward compatibility zarr_version = getattr(store, '_store_version', 2) if zarr_version != 2: @@ -162,7 +163,7 @@ def normalize_store_arg(store: StoreLike, mode: AccessModes = "r", *, zarr_version: Optional[int] = None) -> BaseStore: - result: BaseStore + result: Store if zarr_version is None: # default to v2 store for backward compatibility zarr_version = getattr(store, "_store_version", DEFAULT_ZARR_VERSION) @@ -194,7 +195,7 @@ def rmdir(store: StoreLike, path: Path = None): _rmdir_from_keys_v3(store, path) # type: ignore -def rename(store: Store, src_path: Path, dst_path: Path): +def rename(store: BaseStore, src_path: Path, dst_path: Path): """Rename all items under the given path. If `store` provides a `rename` method, this will be called, otherwise will fall back to implementation via the `Store` interface.""" From 6d6dbc8cb961e10a70bf4ee369dbaef6c8620ece Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 26 Jun 2022 22:48:35 -0400 Subject: [PATCH 07/14] revert some bad numpy dtype annotations and move AccessModes literal to utils --- zarr/_storage/store.py | 3 +-- zarr/_storage/v3.py | 3 +-- zarr/hierarchy.py | 5 +---- zarr/meta.py | 8 ++++---- zarr/storage.py | 12 ++++++------ zarr/util.py | 3 +++ 6 files changed, 16 insertions(+), 18 deletions(-) diff --git a/zarr/_storage/store.py b/zarr/_storage/store.py index a3a8ff977a..96140edcc4 100644 --- a/zarr/_storage/store.py +++ b/zarr/_storage/store.py @@ -3,7 +3,6 @@ from typing import MutableMapping, Set, Tuple from string import ascii_letters, digits from typing import Any, List, Mapping, Optional, Union -from zarr._storage.v3 import KVStoreV3 from zarr.meta import Metadata2, Metadata3 from zarr.util import normalize_storage_path @@ -265,7 +264,7 @@ def __eq__(self, other: Any): return NotImplemented @staticmethod - def _ensure_store(store: Any) -> Union['StoreV3', 'KVStoreV3', None]: + def _ensure_store(store: Any) -> Union['StoreV3', None]: """ We want to make sure internally that zarr stores are always a class with a specific interface derived from ``Store``, which is slightly diff --git a/zarr/_storage/v3.py b/zarr/_storage/v3.py index 61d740aad2..82146e65ba 100644 --- a/zarr/_storage/v3.py +++ b/zarr/_storage/v3.py @@ -9,8 +9,7 @@ MetadataError, ReadOnlyError, ) -from zarr.hierarchy import AccessModes -from zarr.util import (buffer_size, json_loads, normalize_storage_path) +from zarr.util import (buffer_size, json_loads, normalize_storage_path, AccessModes) from zarr._storage.absstore import ABSStoreV3 # noqa: F401 from zarr._storage.store import (_get_hierarchy_metadata, # noqa: F401 diff --git a/zarr/hierarchy.py b/zarr/hierarchy.py index 433906f3b5..3fa57e8098 100644 --- a/zarr/hierarchy.py +++ b/zarr/hierarchy.py @@ -12,8 +12,6 @@ MutableMapping ) -from typing_extensions import Literal - import numpy as np from zarr._storage.store import (Store, StoreV3, _get_metadata_suffix, data_root, meta_root, @@ -46,6 +44,7 @@ ) from zarr._storage.v3 import MemoryStoreV3 from zarr.util import ( + AccessModes, InfoReporter, TreeViewer, is_valid_python_name, @@ -54,8 +53,6 @@ normalize_storage_path, ) -AccessModes = Literal['r', 'r+', 'w', 'a', 'w-', 'x'] - class Group(MutableMapping[str, Union['Group', Array]]): """Instantiate a group from an initialized store. diff --git a/zarr/meta.py b/zarr/meta.py index 76d7b632bc..cb845f9db0 100644 --- a/zarr/meta.py +++ b/zarr/meta.py @@ -174,7 +174,7 @@ def encode_array_metadata(cls, meta: MappingType[str, Any]) -> bytes: return json_dumps(meta) @classmethod - def encode_dtype(cls, d: np.dtype[Any]): + def encode_dtype(cls, d: np.dtype): if d.fields is None: return d.str else: @@ -189,7 +189,7 @@ def _decode_dtype_descr(cls, d: Any) -> List[Any]: return d @classmethod - def decode_dtype(cls, d: Any) -> np.dtype[Any]: + def decode_dtype(cls, d: Any) -> np.dtype: d = cls._decode_dtype_descr(d) return np.dtype(d) @@ -214,7 +214,7 @@ def encode_group_metadata(cls, meta: Any = None) -> bytes: @classmethod def decode_fill_value( - cls, v: Any, dtype: np.dtype[Any], object_codec: Any = None + cls, v: Any, dtype: np.dtype, object_codec: Any = None ) -> Any: # early out if v is None: @@ -264,7 +264,7 @@ def decode_fill_value( @classmethod def encode_fill_value( - cls, v: Any, dtype: np.dtype[Any], object_codec: Any = None + cls, v: Any, dtype: np.dtype, object_codec: Any = None ) -> Any: # early out if v is None: diff --git a/zarr/storage.py b/zarr/storage.py index fcb6774cbd..08ca3239d6 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -27,11 +27,12 @@ import warnings import zipfile from collections import OrderedDict -from typing import Generator, MutableMapping +from typing import Generator, MutableMapping, cast from os import scandir from pickle import PicklingError from threading import Lock, RLock from typing import Optional, Union, List, Tuple, Dict, Any + import uuid import time @@ -51,12 +52,11 @@ FSPathExistNotDir, ReadOnlyError, ) -from zarr.hierarchy import AccessModes from zarr.meta import encode_array_metadata, encode_group_metadata from zarr.util import (buffer_size, json_loads, nolock, normalize_chunks, normalize_dimension_separator, normalize_dtype, normalize_fill_value, normalize_order, - normalize_shape, normalize_storage_path, retry_call) + normalize_shape, normalize_storage_path, retry_call, AccessModes) from zarr._storage.absstore import ABSStore # noqa: F401 from zarr._storage.store import (StoreV3, _get_hierarchy_metadata, # noqa: F401 @@ -93,7 +93,6 @@ from zarr.codecs import Zlib default_compressor = Zlib() - Path = Union[str, bytes, None] # allow MutableMapping for backwards compatibility StoreLike = Union[BaseStore, MutableMapping[str, Any]] @@ -131,7 +130,7 @@ def contains_group(store: StoreLike, path: Path = None, explicit_only=True) -> b def _normalize_store_arg_v2(store: Any, storage_options: Optional[Dict[str, Any]] = None, - mode="r") -> BaseStore: + mode="r") -> Store: # default to v2 store for backward compatibility zarr_version = getattr(store, '_store_version', 2) if zarr_version != 2: @@ -163,7 +162,7 @@ def normalize_store_arg(store: StoreLike, mode: AccessModes = "r", *, zarr_version: Optional[int] = None) -> BaseStore: - result: Store + result: BaseStore if zarr_version is None: # default to v2 store for backward compatibility zarr_version = getattr(store, "_store_version", DEFAULT_ZARR_VERSION) @@ -230,6 +229,7 @@ def listdir(store: BaseStore, path: Path = None): def _getsize(store: BaseStore, path: Path = None) -> int: # compute from size of values if path and path in store: + path = cast(str, path) v = store[path] size = buffer_size(v) else: diff --git a/zarr/util.py b/zarr/util.py index 8f943a9c85..6723bd2519 100644 --- a/zarr/util.py +++ b/zarr/util.py @@ -15,6 +15,9 @@ from numcodecs.blosc import cbuffer_sizes, cbuffer_metainfo from typing import Any, Callable, Dict, Optional, Tuple, Union +from typing_extensions import Literal + +AccessModes = Literal['r', 'r+', 'w', 'a', 'w-', 'x'] def flatten(arg: Iterable) -> Iterable: From d0cb795b179c64367d3c962b489d33c8e771d303 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 26 Jun 2022 23:14:25 -0400 Subject: [PATCH 08/14] fix breakage in v3 store validation --- zarr/_storage/store.py | 2 +- zarr/_storage/v3.py | 6 +++--- zarr/convenience.py | 13 +++++++++---- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/zarr/_storage/store.py b/zarr/_storage/store.py index 96140edcc4..2d45fb1fc4 100644 --- a/zarr/_storage/store.py +++ b/zarr/_storage/store.py @@ -264,7 +264,7 @@ def __eq__(self, other: Any): return NotImplemented @staticmethod - def _ensure_store(store: Any) -> Union['StoreV3', None]: + def _ensure_store(store: Any) -> Optional['StoreV3']: """ We want to make sure internally that zarr stores are always a class with a specific interface derived from ``Store``, which is slightly diff --git a/zarr/_storage/v3.py b/zarr/_storage/v3.py index 82146e65ba..bce7106bb6 100644 --- a/zarr/_storage/v3.py +++ b/zarr/_storage/v3.py @@ -565,10 +565,10 @@ def _normalize_store_arg_v3(store: StoreLike, if store is None: result = KVStoreV3(dict()) # add default zarr.json metadata - result['zarr.json'] = store._metadata_class.encode_hierarchy_metadata(None) + result['zarr.json'] = result._metadata_class.encode_hierarchy_metadata(None) return result if isinstance(store, os.PathLike): - result = os.fspath(store) + store = os.fspath(store) if isinstance(store, str): if "://" in store or "::" in store: result = FSStoreV3(store, mode=mode, **(storage_options or {})) @@ -588,5 +588,5 @@ def _normalize_store_arg_v3(store: StoreLike, result = StoreV3._ensure_store(store) if 'zarr.json' not in result: # add default zarr.json metadata - result['zarr.json'] = store._metadata_class.encode_hierarchy_metadata(None) + result['zarr.json'] = result._metadata_class.encode_hierarchy_metadata(None) return result diff --git a/zarr/convenience.py b/zarr/convenience.py index 93dc860477..5e8d94d166 100644 --- a/zarr/convenience.py +++ b/zarr/convenience.py @@ -17,9 +17,9 @@ from zarr.storage import (_get_metadata_suffix, contains_array, contains_group, normalize_store_arg, BaseStore, ConsolidatedMetadataStore) from zarr._storage.v3 import ConsolidatedMetadataStoreV3 -from zarr.util import TreeViewer, buffer_size, normalize_storage_path +from zarr.util import AccessModes, TreeViewer, buffer_size, normalize_storage_path -from typing import Union +from typing import Any, Optional, Union StoreLike = Union[BaseStore, MutableMapping, str, None] @@ -31,7 +31,12 @@ def _check_and_update_path(store: BaseStore, path): # noinspection PyShadowingBuiltins -def open(store: StoreLike = None, mode: str = "a", *, zarr_version=None, path=None, **kwargs): +def open(store: StoreLike = None, + mode: Optional[AccessModes] = "a", + *, + zarr_version: Optional[int] = None, + path: Optional[str] = None, + **kwargs: Any): """Convenience function to open a group or array using file-mode-like semantics. Parameters @@ -93,7 +98,7 @@ def open(store: StoreLike = None, mode: str = "a", *, zarr_version=None, path=No # handle polymorphic store arg # we pass storage options explicitly, since normalize_store_arg might construct # a store if the input is a fsspec-compatible URL - _store: BaseStore = normalize_store_arg( + _store = normalize_store_arg( store, storage_options=kwargs.pop("storage_options", {}), mode=mode, zarr_version=zarr_version, ) From 2bb7a2f4c05787116fab89ab74f98ac4336ea342 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 27 Jun 2022 08:19:24 -0400 Subject: [PATCH 09/14] more type hints --- zarr/_storage/v3.py | 7 ++++--- zarr/convenience.py | 2 +- zarr/hierarchy.py | 6 +++--- zarr/storage.py | 2 +- zarr/tests/test_hierarchy.py | 7 ++++--- 5 files changed, 13 insertions(+), 11 deletions(-) diff --git a/zarr/_storage/v3.py b/zarr/_storage/v3.py index bce7106bb6..2a09594766 100644 --- a/zarr/_storage/v3.py +++ b/zarr/_storage/v3.py @@ -3,7 +3,7 @@ import shutil from collections import OrderedDict from threading import Lock -from typing import Optional, Union, Dict, Any +from typing import Optional, Union, Dict, Any, cast from zarr.errors import ( MetadataError, @@ -554,7 +554,7 @@ def rmdir(self, path: str = ""): raise ReadOnlyError() -def _normalize_store_arg_v3(store: StoreLike, +def _normalize_store_arg_v3(store: Union[StoreLike, str, None], storage_options: Optional[Dict[str, Any]] = None, mode: AccessModes = "r") -> StoreV3: result: StoreV3 @@ -585,7 +585,8 @@ def _normalize_store_arg_v3(store: StoreLike, result['zarr.json'] = result._metadata_class.encode_hierarchy_metadata(None) return result else: - result = StoreV3._ensure_store(store) + store_v3 = StoreV3._ensure_store(store) + result = cast(StoreV3, store_v3) if 'zarr.json' not in result: # add default zarr.json metadata result['zarr.json'] = result._metadata_class.encode_hierarchy_metadata(None) diff --git a/zarr/convenience.py b/zarr/convenience.py index 5e8d94d166..9777624ef6 100644 --- a/zarr/convenience.py +++ b/zarr/convenience.py @@ -32,7 +32,7 @@ def _check_and_update_path(store: BaseStore, path): # noinspection PyShadowingBuiltins def open(store: StoreLike = None, - mode: Optional[AccessModes] = "a", + mode: AccessModes = "a", *, zarr_version: Optional[int] = None, path: Optional[str] = None, diff --git a/zarr/hierarchy.py b/zarr/hierarchy.py index 3fa57e8098..4800044ced 100644 --- a/zarr/hierarchy.py +++ b/zarr/hierarchy.py @@ -1229,7 +1229,7 @@ def move(self, source: str, dest: str): self._write_op(self._move_nosync, source, dest) -def _normalize_store_arg(store: Optional[StoreLike], +def _normalize_store_arg(store: Union[StoreLike, str, None], *, storage_options: Optional[Dict[str, Any]] = None, mode: AccessModes = "r", @@ -1334,12 +1334,12 @@ def group(store: Optional[StoreLike] = None, zarr_version=zarr_version) -def open_group(store: Optional[StoreLike] = None, +def open_group(store: Union[StoreLike, str, None] = None, mode: AccessModes = 'a', cache_attrs: bool = True, synchronizer: Any = None, path: Optional[str] = None, - chunk_store: Optional[StoreLike] = None, + chunk_store: Union[StoreLike, str, None] = None, storage_options: Any = None, *, zarr_version: Optional[int] = None) -> 'Group': diff --git a/zarr/storage.py b/zarr/storage.py index 08ca3239d6..42e9bd7161 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -157,7 +157,7 @@ def _normalize_store_arg_v2(store: Any, return store -def normalize_store_arg(store: StoreLike, +def normalize_store_arg(store: Union[StoreLike, str, None], storage_options: Optional[Dict[str, Any]] = None, mode: AccessModes = "r", *, zarr_version: Optional[int] = None) -> BaseStore: diff --git a/zarr/tests/test_hierarchy.py b/zarr/tests/test_hierarchy.py index a1b4e85ddb..9b4e8f9107 100644 --- a/zarr/tests/test_hierarchy.py +++ b/zarr/tests/test_hierarchy.py @@ -5,7 +5,7 @@ import shutil import tempfile import textwrap -from typing import Any, Optional, Tuple +from typing import Any, Dict, Optional, Tuple import unittest import numpy as np @@ -32,7 +32,7 @@ from zarr._storage.v3 import (ABSStoreV3, KVStoreV3, DirectoryStoreV3, MemoryStoreV3, FSStoreV3, ZipStoreV3, DBMStoreV3, LMDBStoreV3, SQLiteStoreV3, LRUStoreCacheV3) -from zarr.util import InfoReporter, buffer_size +from zarr.util import AccessModes, InfoReporter, buffer_size from zarr.tests.util import skip_test_env_var, have_fsspec, abs_container @@ -1101,6 +1101,8 @@ def test_context_manager(self): @pytest.mark.parametrize('chunk_dict', [False, True]) def test_group_init_from_dict(chunk_dict: bool): + store: Dict[str, Any] + chunk_store: Optional[Dict[str, Any]] if chunk_dict: store, chunk_store = dict(), dict() else: @@ -1613,7 +1615,6 @@ def test_open_group(zarr_version: int): store = 'data/group.zarr' expected_store_type = DirectoryStore if zarr_version == 2 else DirectoryStoreV3 - # mode == 'w' path = None if zarr_version == 2 else 'group1' g = open_group(store, path=path, mode='w', zarr_version=zarr_version) From 0bae3f2f89614bc0a03fbac4fc3a2c2dd212b761 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 27 Jun 2022 08:27:12 -0400 Subject: [PATCH 10/14] flake8 --- zarr/tests/test_hierarchy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zarr/tests/test_hierarchy.py b/zarr/tests/test_hierarchy.py index 9b4e8f9107..d8843db634 100644 --- a/zarr/tests/test_hierarchy.py +++ b/zarr/tests/test_hierarchy.py @@ -32,7 +32,7 @@ from zarr._storage.v3 import (ABSStoreV3, KVStoreV3, DirectoryStoreV3, MemoryStoreV3, FSStoreV3, ZipStoreV3, DBMStoreV3, LMDBStoreV3, SQLiteStoreV3, LRUStoreCacheV3) -from zarr.util import AccessModes, InfoReporter, buffer_size +from zarr.util import InfoReporter, buffer_size from zarr.tests.util import skip_test_env_var, have_fsspec, abs_container From 6b2be43a72d4209f3f90b9beafc454d6b23c3e2c Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 27 Jun 2022 08:49:11 -0400 Subject: [PATCH 11/14] remove version handling in _normalize_store_arg --- zarr/hierarchy.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/zarr/hierarchy.py b/zarr/hierarchy.py index 4800044ced..078184df41 100644 --- a/zarr/hierarchy.py +++ b/zarr/hierarchy.py @@ -1237,12 +1237,8 @@ def _normalize_store_arg(store: Union[StoreLike, str, None], result: BaseStore if zarr_version is None: zarr_version = getattr(store, '_store_version', DEFAULT_ZARR_VERSION) - if zarr_version == 2: - pass - elif zarr_version == 3: + if zarr_version == 3: assert_zarr_v3_api_available() - else: - raise NotImplementedError(f"Zarr version must be 2 or 3, not {zarr_version}") if store is None: if zarr_version == 2: From 0143d2e2b74c3a4eb22de8c3439282baf6adc969 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 27 Jun 2022 09:11:51 -0400 Subject: [PATCH 12/14] add casts for access modes in hierarchy tests --- zarr/tests/test_hierarchy.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/zarr/tests/test_hierarchy.py b/zarr/tests/test_hierarchy.py index d8843db634..0b0296e2ce 100644 --- a/zarr/tests/test_hierarchy.py +++ b/zarr/tests/test_hierarchy.py @@ -5,7 +5,7 @@ import shutil import tempfile import textwrap -from typing import Any, Dict, Optional, Tuple +from typing import Any, Dict, Optional, Tuple, cast import unittest import numpy as np @@ -32,7 +32,7 @@ from zarr._storage.v3 import (ABSStoreV3, KVStoreV3, DirectoryStoreV3, MemoryStoreV3, FSStoreV3, ZipStoreV3, DBMStoreV3, LMDBStoreV3, SQLiteStoreV3, LRUStoreCacheV3) -from zarr.util import InfoReporter, buffer_size +from zarr.util import AccessModes, InfoReporter, buffer_size from zarr.tests.util import skip_test_env_var, have_fsspec, abs_container @@ -1627,6 +1627,8 @@ def test_open_group(zarr_version: int): # mode in 'r', 'r+' open_array('data/array.zarr', shape=100, chunks=10, mode='w') for mode in 'r', 'r+': + # todo: figure out how to get mypy to understand subsets of literals + mode = cast(AccessModes, mode) with pytest.raises(ValueError): open_group('doesnotexist', mode=mode) with pytest.raises(ValueError): @@ -1655,6 +1657,8 @@ def test_open_group(zarr_version: int): # mode in 'w-', 'x' for mode in 'w-', 'x': + # todo: figure out how to get mypy to understand subsets of literals + mode = cast(AccessModes, mode) shutil.rmtree(store) g = open_group(store, path=path, mode=mode, zarr_version=zarr_version) assert isinstance(g, Group) From cf87dde75ff9eab66112514c68a27e21ed66cc51 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 27 Jun 2022 22:18:00 -0400 Subject: [PATCH 13/14] rename AccessModes to AccessMode and fix type errors in hierarchy tests --- zarr/_storage/v3.py | 4 ++-- zarr/convenience.py | 4 ++-- zarr/hierarchy.py | 6 +++--- zarr/storage.py | 4 ++-- zarr/tests/test_hierarchy.py | 27 ++++++++++++--------------- zarr/util.py | 2 +- 6 files changed, 22 insertions(+), 25 deletions(-) diff --git a/zarr/_storage/v3.py b/zarr/_storage/v3.py index 2a09594766..7143324f0a 100644 --- a/zarr/_storage/v3.py +++ b/zarr/_storage/v3.py @@ -9,7 +9,7 @@ MetadataError, ReadOnlyError, ) -from zarr.util import (buffer_size, json_loads, normalize_storage_path, AccessModes) +from zarr.util import (buffer_size, json_loads, normalize_storage_path, AccessMode) from zarr._storage.absstore import ABSStoreV3 # noqa: F401 from zarr._storage.store import (_get_hierarchy_metadata, # noqa: F401 @@ -556,7 +556,7 @@ def rmdir(self, path: str = ""): def _normalize_store_arg_v3(store: Union[StoreLike, str, None], storage_options: Optional[Dict[str, Any]] = None, - mode: AccessModes = "r") -> StoreV3: + mode: AccessMode = "r") -> StoreV3: result: StoreV3 # default to v2 store for backward compatibility zarr_version = getattr(store, '_store_version', 3) diff --git a/zarr/convenience.py b/zarr/convenience.py index 9777624ef6..c5e34c8266 100644 --- a/zarr/convenience.py +++ b/zarr/convenience.py @@ -17,7 +17,7 @@ from zarr.storage import (_get_metadata_suffix, contains_array, contains_group, normalize_store_arg, BaseStore, ConsolidatedMetadataStore) from zarr._storage.v3 import ConsolidatedMetadataStoreV3 -from zarr.util import AccessModes, TreeViewer, buffer_size, normalize_storage_path +from zarr.util import AccessMode, TreeViewer, buffer_size, normalize_storage_path from typing import Any, Optional, Union @@ -32,7 +32,7 @@ def _check_and_update_path(store: BaseStore, path): # noinspection PyShadowingBuiltins def open(store: StoreLike = None, - mode: AccessModes = "a", + mode: AccessMode = "a", *, zarr_version: Optional[int] = None, path: Optional[str] = None, diff --git a/zarr/hierarchy.py b/zarr/hierarchy.py index 078184df41..7d945e54f5 100644 --- a/zarr/hierarchy.py +++ b/zarr/hierarchy.py @@ -44,7 +44,7 @@ ) from zarr._storage.v3 import MemoryStoreV3 from zarr.util import ( - AccessModes, + AccessMode, InfoReporter, TreeViewer, is_valid_python_name, @@ -1232,7 +1232,7 @@ def move(self, source: str, dest: str): def _normalize_store_arg(store: Union[StoreLike, str, None], *, storage_options: Optional[Dict[str, Any]] = None, - mode: AccessModes = "r", + mode: AccessMode = "r", zarr_version: Optional[int] = None) -> BaseStore: result: BaseStore if zarr_version is None: @@ -1331,7 +1331,7 @@ def group(store: Optional[StoreLike] = None, def open_group(store: Union[StoreLike, str, None] = None, - mode: AccessModes = 'a', + mode: AccessMode = 'a', cache_attrs: bool = True, synchronizer: Any = None, path: Optional[str] = None, diff --git a/zarr/storage.py b/zarr/storage.py index 42e9bd7161..7c500fdb28 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -56,7 +56,7 @@ from zarr.util import (buffer_size, json_loads, nolock, normalize_chunks, normalize_dimension_separator, normalize_dtype, normalize_fill_value, normalize_order, - normalize_shape, normalize_storage_path, retry_call, AccessModes) + normalize_shape, normalize_storage_path, retry_call, AccessMode) from zarr._storage.absstore import ABSStore # noqa: F401 from zarr._storage.store import (StoreV3, _get_hierarchy_metadata, # noqa: F401 @@ -159,7 +159,7 @@ def _normalize_store_arg_v2(store: Any, def normalize_store_arg(store: Union[StoreLike, str, None], storage_options: Optional[Dict[str, Any]] = None, - mode: AccessModes = "r", *, + mode: AccessMode = "r", *, zarr_version: Optional[int] = None) -> BaseStore: result: BaseStore diff --git a/zarr/tests/test_hierarchy.py b/zarr/tests/test_hierarchy.py index 0b0296e2ce..a06edf1fe5 100644 --- a/zarr/tests/test_hierarchy.py +++ b/zarr/tests/test_hierarchy.py @@ -5,7 +5,7 @@ import shutil import tempfile import textwrap -from typing import Any, Dict, Optional, Tuple, cast +from typing import Any, Dict, List, Optional, Tuple, cast import unittest import numpy as np @@ -32,7 +32,7 @@ from zarr._storage.v3 import (ABSStoreV3, KVStoreV3, DirectoryStoreV3, MemoryStoreV3, FSStoreV3, ZipStoreV3, DBMStoreV3, LMDBStoreV3, SQLiteStoreV3, LRUStoreCacheV3) -from zarr.util import AccessModes, InfoReporter, buffer_size +from zarr.util import AccessMode, InfoReporter, buffer_size from zarr.tests.util import skip_test_env_var, have_fsspec, abs_container @@ -1611,13 +1611,12 @@ def test_group(zarr_version): @pytest.mark.parametrize('zarr_version', _VERSIONS) def test_open_group(zarr_version: int): # test the open_group() convenience function - + mode: AccessMode = 'w' store = 'data/group.zarr' expected_store_type = DirectoryStore if zarr_version == 2 else DirectoryStoreV3 - # mode == 'w' path = None if zarr_version == 2 else 'group1' - g = open_group(store, path=path, mode='w', zarr_version=zarr_version) + g = open_group(store, path=path, mode=mode, zarr_version=zarr_version) assert isinstance(g, Group) assert isinstance(g.store, expected_store_type) assert 0 == len(g) @@ -1625,10 +1624,10 @@ def test_open_group(zarr_version: int): assert 2 == len(g) # mode in 'r', 'r+' - open_array('data/array.zarr', shape=100, chunks=10, mode='w') - for mode in 'r', 'r+': + open_array('data/array.zarr', shape=100, chunks=10, mode=mode) + modes: List[AccessMode] = ['r', 'r+'] + for mode in modes: # todo: figure out how to get mypy to understand subsets of literals - mode = cast(AccessModes, mode) with pytest.raises(ValueError): open_group('doesnotexist', mode=mode) with pytest.raises(ValueError): @@ -1644,21 +1643,19 @@ def test_open_group(zarr_version: int): g.create_groups('baz', 'quux') assert 4 == len(g) - # mode == 'a' + mode = 'a' shutil.rmtree(store) - g = open_group(store, path=path, mode='a', zarr_version=zarr_version) + g = open_group(store, path=path, mode=mode, zarr_version=zarr_version) assert isinstance(g, Group) assert isinstance(g.store, expected_store_type) assert 0 == len(g) g.create_groups('foo', 'bar') assert 2 == len(g) with pytest.raises(ValueError): - open_group('data/array.zarr', mode='a', zarr_version=zarr_version) + open_group('data/array.zarr', mode=mode, zarr_version=zarr_version) - # mode in 'w-', 'x' - for mode in 'w-', 'x': - # todo: figure out how to get mypy to understand subsets of literals - mode = cast(AccessModes, mode) + modes = ['w-', 'x'] + for mode in modes: shutil.rmtree(store) g = open_group(store, path=path, mode=mode, zarr_version=zarr_version) assert isinstance(g, Group) diff --git a/zarr/util.py b/zarr/util.py index 6723bd2519..18c3a8a960 100644 --- a/zarr/util.py +++ b/zarr/util.py @@ -17,7 +17,7 @@ from typing import Any, Callable, Dict, Optional, Tuple, Union from typing_extensions import Literal -AccessModes = Literal['r', 'r+', 'w', 'a', 'w-', 'x'] +AccessMode = Literal['r', 'r+', 'w', 'a', 'w-', 'x'] def flatten(arg: Iterable) -> Iterable: From 7647c3c30ced727b7089eb0b65d82740c58b2f96 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 27 Jun 2022 22:23:12 -0400 Subject: [PATCH 14/14] flake8 --- zarr/tests/test_hierarchy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zarr/tests/test_hierarchy.py b/zarr/tests/test_hierarchy.py index a06edf1fe5..d7c2dc950c 100644 --- a/zarr/tests/test_hierarchy.py +++ b/zarr/tests/test_hierarchy.py @@ -5,7 +5,7 @@ import shutil import tempfile import textwrap -from typing import Any, Dict, List, Optional, Tuple, cast +from typing import Any, Dict, List, Optional, Tuple import unittest import numpy as np