diff --git a/changes/3954.bugfix.md b/changes/3954.bugfix.md new file mode 100644 index 0000000000..1a6c93a8ac --- /dev/null +++ b/changes/3954.bugfix.md @@ -0,0 +1 @@ +Handle missing consolidated metadata in leaf Group nodes. diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index cbce1c1ed0..ae18cbaf85 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -235,8 +235,6 @@ def _flat_to_nested( # array metadata of its immediate children. # In the example, the group at `/a/b` will have consolidated metadata # for its children `array-0` and `array-1`. - # - # metadata = dict(metadata) keys = sorted(metadata, key=lambda k: k.count("/")) grouped = { @@ -269,13 +267,17 @@ def _flat_to_nested( # These are already present, either thanks to being an array in the # root, or by being collected as a child in the else clause continue - children_keys = list(children_keys) - # We pop from metadata, since we're *moving* this under group - children = { - child_key.split("/")[-1]: metadata.pop(child_key) - for child_key in children_keys - if child_key != key - } + children: dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata] = {} + # We pop from metadata, since we're *moving* this under group. + # While doing this, normalize leaf groups to carry empty consolidated metadata. + for child_key in children_keys: + if child_key == key: + continue + child = metadata.pop(child_key) + if isinstance(child, GroupMetadata) and child.consolidated_metadata is None: + child = replace(child, consolidated_metadata=ConsolidatedMetadata(metadata={})) + children[child_key.split("/")[-1]] = child + parent[name] = replace( node, consolidated_metadata=ConsolidatedMetadata(metadata=children) ) diff --git a/tests/test_metadata/test_consolidated.py b/tests/test_metadata/test_consolidated.py index 9e8b763ef7..3596d2bcaa 100644 --- a/tests/test_metadata/test_consolidated.py +++ b/tests/test_metadata/test_consolidated.py @@ -51,6 +51,74 @@ async def memory_store_with_hierarchy(memory_store: Store) -> Store: class TestConsolidated: + @pytest.mark.filterwarnings("ignore:Consolidated metadata") + async def test_getitem_consolidated_empty_leaf_group( + self, memory_store: zarr.storage.MemoryStore, zarr_format: ZarrFormat + ) -> None: + # This test writes the bytes directly, rather than using the zarr API, to mimic + # how older versions of zarr-python wrote the consolidated metadata. + # Notably, zarr-python 2.x does not include a + # + # "consolidated_metadata": {"metadata": {}} + # + # field on the leaf group nodes. + if zarr_format == 2: + zmetadata: dict[str, JSON] = { + "metadata": { + ".zattrs": {}, + ".zgroup": {"zarr_format": 2}, + "raw/.zattrs": {}, + "raw/.zgroup": {"zarr_format": 2}, + "raw/varm/.zattrs": {}, + "raw/varm/.zgroup": {"zarr_format": 2}, + }, + "zarr_consolidated_format": 1, + } + await memory_store.set( + ".zgroup", cpu.Buffer.from_bytes(json.dumps({"zarr_format": 2}).encode()) + ) + await memory_store.set(".zattrs", cpu.Buffer.from_bytes(json.dumps({}).encode())) + await memory_store.set( + ".zmetadata", cpu.Buffer.from_bytes(json.dumps(zmetadata).encode()) + ) + + else: + zmetadata = { + "attributes": {}, + "zarr_format": 3, + "consolidated_metadata": { + "kind": "inline", + "must_understand": False, + "metadata": { + "raw": { + "attributes": {}, + "zarr_format": 3, + "node_type": "group", + }, + "raw/varm": { + "attributes": {}, + "zarr_format": 3, + "node_type": "group", + }, + }, + }, + "node_type": "group", + } + await memory_store.set( + "zarr.json", cpu.Buffer.from_bytes(json.dumps(zmetadata).encode()) + ) + + group = await zarr.api.asynchronous.open_consolidated( + store=memory_store, zarr_format=zarr_format + ) + raw = await group.getitem("raw") + assert isinstance(raw, zarr.AsyncGroup) + assert raw.metadata.consolidated_metadata is not None + + varm = await raw.getitem("varm") + assert isinstance(varm, zarr.AsyncGroup) + assert varm.metadata.consolidated_metadata == ConsolidatedMetadata(metadata={}) + async def test_open_consolidated_false_raises(self) -> None: store = zarr.storage.MemoryStore() with pytest.raises(TypeError, match="use_consolidated"):