From 5d49b9208d6d283f408311209094f56fe1cfedb6 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Mon, 11 May 2026 21:22:26 -0500 Subject: [PATCH 1/3] BUG: Fix empty Group nodes lacking a "consolidated_metadata" field All nodes opened via `open_consolidated` are expected to have a non-null `.metadata.consolidated_metadata`. https://github.com/zarr-developers/zarr-python/issues/3954 showed an example, with a data file written by zarr-python 2.x, where that condition failed to hold. This happened because of a slight difference between how zarr-python 3.x writes consolidated metadata and how zarr-python 2.x wrote it. For leaf `Group` nodes (a `Group` with no `Group` or `Array` children), zarr-python 3.x includes an "empty" `consolidated_metadata` field. In zarr-python 3.x, we include a `consolidated_metadata: metadata: {}` field to indicate that it's empty. zarr-python 2.x *didn't* include that. The fix is to update how we load this type of data. We effectively normalize a node missing consolidated metadata to one that is known not to have consolidated metadata (which is consistent with the idea behind consolidated metadata: just do one metadata read up front). --- changes/3954.bugfix.rst | 1 + src/zarr/core/group.py | 20 +++---- tests/test_metadata/test_consolidated.py | 68 ++++++++++++++++++++++++ tests/test_metadata/test_v2.py | 29 ++++++++++ 4 files changed, 109 insertions(+), 9 deletions(-) create mode 100644 changes/3954.bugfix.rst diff --git a/changes/3954.bugfix.rst b/changes/3954.bugfix.rst new file mode 100644 index 0000000000..1a6c93a8ac --- /dev/null +++ b/changes/3954.bugfix.rst @@ -0,0 +1 @@ +Handle missing consolidated metadata in leaf Group nodes. diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index cbce1c1ed0..ae18cbaf85 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -235,8 +235,6 @@ def _flat_to_nested( # array metadata of its immediate children. # In the example, the group at `/a/b` will have consolidated metadata # for its children `array-0` and `array-1`. - # - # metadata = dict(metadata) keys = sorted(metadata, key=lambda k: k.count("/")) grouped = { @@ -269,13 +267,17 @@ def _flat_to_nested( # These are already present, either thanks to being an array in the # root, or by being collected as a child in the else clause continue - children_keys = list(children_keys) - # We pop from metadata, since we're *moving* this under group - children = { - child_key.split("/")[-1]: metadata.pop(child_key) - for child_key in children_keys - if child_key != key - } + children: dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata] = {} + # We pop from metadata, since we're *moving* this under group. + # While doing this, normalize leaf groups to carry empty consolidated metadata. + for child_key in children_keys: + if child_key == key: + continue + child = metadata.pop(child_key) + if isinstance(child, GroupMetadata) and child.consolidated_metadata is None: + child = replace(child, consolidated_metadata=ConsolidatedMetadata(metadata={})) + children[child_key.split("/")[-1]] = child + parent[name] = replace( node, consolidated_metadata=ConsolidatedMetadata(metadata=children) ) diff --git a/tests/test_metadata/test_consolidated.py b/tests/test_metadata/test_consolidated.py index 9e8b763ef7..3596d2bcaa 100644 --- a/tests/test_metadata/test_consolidated.py +++ b/tests/test_metadata/test_consolidated.py @@ -51,6 +51,74 @@ async def memory_store_with_hierarchy(memory_store: Store) -> Store: class TestConsolidated: + @pytest.mark.filterwarnings("ignore:Consolidated metadata") + async def test_getitem_consolidated_empty_leaf_group( + self, memory_store: zarr.storage.MemoryStore, zarr_format: ZarrFormat + ) -> None: + # This test writes the bytes directly, rather than using the zarr API, to mimic + # how older versions of zarr-python wrote the consolidated metadata. + # Notably, zarr-python 2.x does not include a + # + # "consolidated_metadata": {"metadata": {}} + # + # field on the leaf group nodes. + if zarr_format == 2: + zmetadata: dict[str, JSON] = { + "metadata": { + ".zattrs": {}, + ".zgroup": {"zarr_format": 2}, + "raw/.zattrs": {}, + "raw/.zgroup": {"zarr_format": 2}, + "raw/varm/.zattrs": {}, + "raw/varm/.zgroup": {"zarr_format": 2}, + }, + "zarr_consolidated_format": 1, + } + await memory_store.set( + ".zgroup", cpu.Buffer.from_bytes(json.dumps({"zarr_format": 2}).encode()) + ) + await memory_store.set(".zattrs", cpu.Buffer.from_bytes(json.dumps({}).encode())) + await memory_store.set( + ".zmetadata", cpu.Buffer.from_bytes(json.dumps(zmetadata).encode()) + ) + + else: + zmetadata = { + "attributes": {}, + "zarr_format": 3, + "consolidated_metadata": { + "kind": "inline", + "must_understand": False, + "metadata": { + "raw": { + "attributes": {}, + "zarr_format": 3, + "node_type": "group", + }, + "raw/varm": { + "attributes": {}, + "zarr_format": 3, + "node_type": "group", + }, + }, + }, + "node_type": "group", + } + await memory_store.set( + "zarr.json", cpu.Buffer.from_bytes(json.dumps(zmetadata).encode()) + ) + + group = await zarr.api.asynchronous.open_consolidated( + store=memory_store, zarr_format=zarr_format + ) + raw = await group.getitem("raw") + assert isinstance(raw, zarr.AsyncGroup) + assert raw.metadata.consolidated_metadata is not None + + varm = await raw.getitem("varm") + assert isinstance(varm, zarr.AsyncGroup) + assert varm.metadata.consolidated_metadata == ConsolidatedMetadata(metadata={}) + async def test_open_consolidated_false_raises(self) -> None: store = zarr.storage.MemoryStore() with pytest.raises(TypeError, match="use_consolidated"): diff --git a/tests/test_metadata/test_v2.py b/tests/test_metadata/test_v2.py index 8c3082e924..4e1edd0a83 100644 --- a/tests/test_metadata/test_v2.py +++ b/tests/test_metadata/test_v2.py @@ -281,6 +281,35 @@ async def test_getitem_consolidated( assert isinstance(air, zarr.AsyncArray) assert air.metadata.shape == (730,) + async def test_getitem_consolidated_empty_leaf_group( + self, memory_store: zarr.storage.MemoryStore + ) -> None: + zmetadata: dict[str, JSON] = { + "metadata": { + ".zattrs": {}, + ".zgroup": {"zarr_format": 2}, + "raw/.zattrs": {}, + "raw/.zgroup": {"zarr_format": 2}, + "raw/varm/.zattrs": {}, + "raw/varm/.zgroup": {"zarr_format": 2}, + }, + "zarr_consolidated_format": 1, + } + await memory_store.set( + ".zgroup", cpu.Buffer.from_bytes(json.dumps({"zarr_format": 2}).encode()) + ) + await memory_store.set(".zattrs", cpu.Buffer.from_bytes(json.dumps({}).encode())) + await memory_store.set(".zmetadata", cpu.Buffer.from_bytes(json.dumps(zmetadata).encode())) + + group = await zarr.api.asynchronous.open_consolidated(store=memory_store, zarr_format=2) + raw = await group.getitem("raw") + assert isinstance(raw, zarr.AsyncGroup) + assert raw.metadata.consolidated_metadata is not None + + varm = await raw.getitem("varm") + assert isinstance(varm, zarr.AsyncGroup) + assert varm.metadata.consolidated_metadata == ConsolidatedMetadata(metadata={}) + def test_from_dict_extra_fields() -> None: data = { From eee40b2915744efb3350a70e9d9b7d193739e609 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Mon, 11 May 2026 21:26:35 -0500 Subject: [PATCH 2/3] remove dup test --- tests/test_metadata/test_v2.py | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/tests/test_metadata/test_v2.py b/tests/test_metadata/test_v2.py index 4e1edd0a83..8c3082e924 100644 --- a/tests/test_metadata/test_v2.py +++ b/tests/test_metadata/test_v2.py @@ -281,35 +281,6 @@ async def test_getitem_consolidated( assert isinstance(air, zarr.AsyncArray) assert air.metadata.shape == (730,) - async def test_getitem_consolidated_empty_leaf_group( - self, memory_store: zarr.storage.MemoryStore - ) -> None: - zmetadata: dict[str, JSON] = { - "metadata": { - ".zattrs": {}, - ".zgroup": {"zarr_format": 2}, - "raw/.zattrs": {}, - "raw/.zgroup": {"zarr_format": 2}, - "raw/varm/.zattrs": {}, - "raw/varm/.zgroup": {"zarr_format": 2}, - }, - "zarr_consolidated_format": 1, - } - await memory_store.set( - ".zgroup", cpu.Buffer.from_bytes(json.dumps({"zarr_format": 2}).encode()) - ) - await memory_store.set(".zattrs", cpu.Buffer.from_bytes(json.dumps({}).encode())) - await memory_store.set(".zmetadata", cpu.Buffer.from_bytes(json.dumps(zmetadata).encode())) - - group = await zarr.api.asynchronous.open_consolidated(store=memory_store, zarr_format=2) - raw = await group.getitem("raw") - assert isinstance(raw, zarr.AsyncGroup) - assert raw.metadata.consolidated_metadata is not None - - varm = await raw.getitem("varm") - assert isinstance(varm, zarr.AsyncGroup) - assert varm.metadata.consolidated_metadata == ConsolidatedMetadata(metadata={}) - def test_from_dict_extra_fields() -> None: data = { From 6932421873fb9f4dc6f6659b7a10868d7877935b Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Tue, 12 May 2026 06:42:10 -0500 Subject: [PATCH 3/3] suffix --- changes/{3954.bugfix.rst => 3954.bugfix.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename changes/{3954.bugfix.rst => 3954.bugfix.md} (100%) diff --git a/changes/3954.bugfix.rst b/changes/3954.bugfix.md similarity index 100% rename from changes/3954.bugfix.rst rename to changes/3954.bugfix.md