From c36ec3d1bd8d9bcdf7831b6cecaf9c3986d56515 Mon Sep 17 00:00:00 2001 From: jmoore Date: Mon, 18 Oct 2021 22:56:47 +0200 Subject: [PATCH 1/7] Fix #840 --- zarr/core.py | 13 ++++++++++++- zarr/meta.py | 4 +++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/zarr/core.py b/zarr/core.py index ba3f2c1e2d..d5f919f8a8 100644 --- a/zarr/core.py +++ b/zarr/core.py @@ -195,7 +195,18 @@ def _load_metadata_nosync(self): self._dtype = meta['dtype'] self._fill_value = meta['fill_value'] self._order = meta['order'] - self._dimension_separator = meta.get('dimension_separator', '.') + + dimension_separator = meta.get('dimension_separator', None) + if dimension_separator is None: + try: + dimension_separator = self._store._dimension_separator + except KeyError: + pass + + # Fallback for any stores which do not choose a default + if dimension_separator is None: + dimension_separator = "." + self._dimension_separator = dimension_separator # setup compressor config = meta['compressor'] diff --git a/zarr/meta.py b/zarr/meta.py index 53b2b779db..d4f0fc1900 100644 --- a/zarr/meta.py +++ b/zarr/meta.py @@ -47,6 +47,7 @@ def decode_array_metadata(s: Union[MappingType, str]) -> MappingType[str, Any]: else: object_codec = None + dimension_separator=meta.get('dimension_separator', None) fill_value = decode_fill_value(meta['fill_value'], dtype, object_codec) meta = dict( zarr_format=meta['zarr_format'], @@ -57,8 +58,9 @@ def decode_array_metadata(s: Union[MappingType, str]) -> MappingType[str, Any]: fill_value=fill_value, order=meta['order'], filters=meta['filters'], - dimension_separator=meta.get('dimension_separator', '.'), ) + if dimension_separator: + meta['dimension_separator'] = dimension_separator except Exception as e: raise MetadataError('error decoding metadata: %s' % e) From a35a1a5e13b0f58f3246652143a5da06790d4696 Mon Sep 17 00:00:00 2001 From: jmoore Date: Mon, 18 Oct 2021 23:05:20 +0200 Subject: [PATCH 2/7] Add legacy tests (See #840) Each fixture (flat & nested) should have a version which does not include any new metadata. --- fixture/flat/.zarray | 3 +- fixture/flat_legacy/.zarray | 22 ++++++++++ fixture/flat_legacy/0.0 | Bin 0 -> 48 bytes fixture/nested_legacy/.zarray | 22 ++++++++++ fixture/nested_legacy/0/0 | Bin 0 -> 48 bytes zarr/tests/test_dim_separator.py | 67 ++++++++++++++++++++++++------- 6 files changed, 99 insertions(+), 15 deletions(-) create mode 100644 fixture/flat_legacy/.zarray create mode 100644 fixture/flat_legacy/0.0 create mode 100644 fixture/nested_legacy/.zarray create mode 100644 fixture/nested_legacy/0/0 diff --git a/fixture/flat/.zarray b/fixture/flat/.zarray index 8ec79419da..f265bb0674 100644 --- a/fixture/flat/.zarray +++ b/fixture/flat/.zarray @@ -10,6 +10,7 @@ "id": "blosc", "shuffle": 1 }, + "dimension_separator": ".", "dtype": " Date: Tue, 19 Oct 2021 08:33:08 +0200 Subject: [PATCH 3/7] Fix PEP8 issues --- zarr/meta.py | 2 +- zarr/tests/test_dim_separator.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/zarr/meta.py b/zarr/meta.py index d4f0fc1900..8a7d760c0f 100644 --- a/zarr/meta.py +++ b/zarr/meta.py @@ -47,7 +47,7 @@ def decode_array_metadata(s: Union[MappingType, str]) -> MappingType[str, Any]: else: object_codec = None - dimension_separator=meta.get('dimension_separator', None) + dimension_separator = meta.get('dimension_separator', None) fill_value = decode_fill_value(meta['fill_value'], dtype, object_codec) meta = dict( zarr_format=meta['zarr_format'], diff --git a/zarr/tests/test_dim_separator.py b/zarr/tests/test_dim_separator.py index e984cc40bf..c9988594cf 100644 --- a/zarr/tests/test_dim_separator.py +++ b/zarr/tests/test_dim_separator.py @@ -76,7 +76,7 @@ def dataset(tmpdir, request): def verify(array, expect_failure=False): try: assert_array_equal(array[:], [[1, 2], [3, 4]]) - except: + except AssertionError: if expect_failure: pytest.xfail() else: @@ -122,8 +122,8 @@ def test_nested(dataset): datasets without any metadata, NestedDirectoryStore will fail. """ failure = ( - "flat_legacy" in dataset or \ - "directory_default" in dataset or \ + "flat_legacy" in dataset or + "directory_default" in dataset or "fs_default" in dataset ) verify(Array(store=NestedDirectoryStore(dataset)), failure) From 7ee2deddb7bf5d45adf40d8dc13343a5589177ad Mon Sep 17 00:00:00 2001 From: jmoore Date: Tue, 19 Oct 2021 08:46:41 +0200 Subject: [PATCH 4/7] Try dropping editable installs see: https://github.com/pypa/pip/issues/10573 --- .github/workflows/minimal.yml | 2 +- .github/workflows/python-package.yml | 2 +- .github/workflows/windows-testing.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/minimal.yml b/.github/workflows/minimal.yml index b1705bdf98..0ce211cbde 100644 --- a/.github/workflows/minimal.yml +++ b/.github/workflows/minimal.yml @@ -24,5 +24,5 @@ jobs: shell: "bash -l {0}" run: | conda activate minimal - python -m pip install -e . + python -m pip install . pytest -svx diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index d9bc362d12..9261187caf 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -57,7 +57,7 @@ jobs: python -m pip install --upgrade pip python -m pip install -U pip setuptools wheel codecov line_profiler python -m pip install -rrequirements_dev_minimal.txt numpy${{ matrix.numpy_version}} -rrequirements_dev_optional.txt pymongo redis - python -m pip install -e . + python -m pip install . python -m pip freeze - name: Tests shell: "bash -l {0}" diff --git a/.github/workflows/windows-testing.yml b/.github/workflows/windows-testing.yml index 0d746ced99..a4e20b7131 100644 --- a/.github/workflows/windows-testing.yml +++ b/.github/workflows/windows-testing.yml @@ -39,7 +39,7 @@ jobs: python -m pip install --upgrade pip python -m pip install -U pip setuptools wheel python -m pip install -r requirements_dev_numpy.txt -r requirements_dev_minimal.txt -r requirements_dev_optional.txt - python -m pip install -e . + python -m pip install . python -m pip freeze npm install -g azurite - name: Run Tests From 6bc47d5654f1057294a6e68c6453d78fd0912bcd Mon Sep 17 00:00:00 2001 From: jmoore Date: Tue, 19 Oct 2021 09:36:03 +0200 Subject: [PATCH 5/7] Handle case of store being a dict --- zarr/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zarr/core.py b/zarr/core.py index d5f919f8a8..c1a1c341ff 100644 --- a/zarr/core.py +++ b/zarr/core.py @@ -200,7 +200,7 @@ def _load_metadata_nosync(self): if dimension_separator is None: try: dimension_separator = self._store._dimension_separator - except KeyError: + except (AttributeError, KeyError): pass # Fallback for any stores which do not choose a default From 6754503c3e1ce15415dd597f54b25c62dbe19d53 Mon Sep 17 00:00:00 2001 From: jmoore Date: Tue, 19 Oct 2021 09:49:13 +0200 Subject: [PATCH 6/7] Exclude unexpected failure from code coverage --- zarr/tests/test_dim_separator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zarr/tests/test_dim_separator.py b/zarr/tests/test_dim_separator.py index c9988594cf..5e17bbe279 100644 --- a/zarr/tests/test_dim_separator.py +++ b/zarr/tests/test_dim_separator.py @@ -80,7 +80,7 @@ def verify(array, expect_failure=False): if expect_failure: pytest.xfail() else: - raise + raise # pragma: no cover def test_open(dataset): From ff1b9a9542749ed48444a5d0a183f7a4f5b55507 Mon Sep 17 00:00:00 2001 From: jmoore Date: Tue, 19 Oct 2021 21:48:21 +0200 Subject: [PATCH 7/7] Add 2.10.2 release notes --- docs/release.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/release.rst b/docs/release.rst index d37516f8a6..6c814cbedb 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -6,6 +6,17 @@ Release notes Unreleased ---------- +.. _release_2.10.2: + +2.10.2 +------ + +Bug fixes +~~~~~~~~~ + +* Fix NestedDirectoryStore datasets without dimension_separator metadata. + By :user:`Josh Moore `; :issue:`850`. + .. _release_2.10.1: 2.10.1