Validate out-of-order tables incrementally while parsing

CHANGELOG.md

@@ -12,6 +12,7 @@
 - Speed up parsing by comparing `StringType` members by identity (`is`) instead of building a set on every `is_basic`/`is_literal`/`is_singleline`/`is_multiline` call, avoiding millions of enum hashes while parsing. ([#502](https://github.com/python-poetry/tomlkit/pull/502))
 - Speed up merging super tables by merging in place instead of deep-copying the growing target on every merge, turning the parse of documents with many subtables under a shared super table (e.g. consecutive `[a.b.c]` / `[a.b.d]` headers) from O(n²) into O(n). ([#503](https://github.com/python-poetry/tomlkit/pull/503))
 - Speed up membership tests (`key in ...`) on out-of-order tables with a native `OutOfOrderTableProxy.__contains__`, completing [#483](https://github.com/python-poetry/tomlkit/issues/483) for the last mapping type that still inherited the slow `MutableMapping` mixin (which resolves the value and builds an exception on every absent key). ([#515](https://github.com/python-poetry/tomlkit/pull/515))
+- Speed up parsing documents with many dotted keys or table headers sharing a prefix by validating out-of-order tables incrementally: each new fragment is merged into a cached validation container once, instead of re-merging (and deep-copying) every earlier fragment on each append, turning a super-cubic worst case into linear time (80 shared-prefix dotted keys: ~8 s → ~10 ms). ([#479](https://github.com/python-poetry/tomlkit/issues/479))
 - Speed up parsing of arrays that close right after a value (e.g. the `files = [...]` blocks that dominate lock files): the parser no longer attempts to read a value while sitting on the closing `]`, which previously built an `UnexpectedCharError` just to discard it — and constructing that exception eagerly computes a line/column by scanning the whole document, making it O(document size) per such array. ([#517](https://github.com/python-poetry/tomlkit/pull/517))
 
 ### Fixed

tests/test_api.py

@@ -438,6 +438,32 @@ def test_value_rejects_values_with_appendage(raw: str) -> None:
         tomlkit.value(raw)
 
 
+def test_parse_many_dotted_keys_with_shared_prefix() -> None:
+    """Each dotted key adds a fragment to the same out-of-order key; parsing
+    validates incrementally instead of re-merging every fragment (#479)."""
+    content = "\n".join(f"a.b.c.key{i} = {i}" for i in range(200)) + "\n"
+    doc = parse(content)
+    assert doc["a"]["b"]["c"]["key0"] == 0
+    assert doc["a"]["b"]["c"]["key199"] == 199
+    assert doc.as_string() == content
+
+
+@pytest.mark.parametrize(
+    "content",
+    [
+        "[a]\nb.c = 1\n[q]\nx = 1\n[a]\nb.c = 2",
+        "[t.a]\nk = 1\n[u]\nz = 1\n[t.a]\nk = 2",
+    ],
+)
+def test_parse_rejects_collisions_across_out_of_order_fragments(
+    content: str,
+) -> None:
+    """Key collisions between out-of-order table parts still fail at parse
+    time with the incremental validation (#479)."""
+    with pytest.raises(tomlkit.exceptions.ParseError):
+        parse(content)
+
+
 def test_create_super_table_with_table() -> None:
     data = {"foo": {"bar": {"a": 1}}}
     assert dumps(data) == "[foo.bar]\na = 1\n"

tomlkit/container.py

@@ -44,6 +44,15 @@ def __init__(self, parsed: bool = False) -> None:
         self._body: list[tuple[Key | None, Item]] = []
         self._parsed = parsed
         self._table_keys: list[Key] = []
+        # number of already-validated fragments and the temp container they
+        # were merged into, per out-of-order key; lets parse-time validation
+        # resume where the previous pass stopped instead of re-merging every
+        # fragment (quadratic) on each append
+        self._validation_cache: dict[Key, tuple[int, Container]] = {}
+        # superset of the keys mapped to an index tuple, so validating all
+        # out-of-order tables doesn't have to scan every key in the map;
+        # stale entries are filtered by the per-key isinstance check
+        self._out_of_order_keys: set[Key] = set()
 
     @property
     def body(self) -> list[tuple[Key | None, Item]]:
@@ -89,6 +98,7 @@ def value(self) -> dict[str, Any]:
 
     def parsing(self, parsing: bool) -> None:
         self._parsed = parsing
+        self._validation_cache.clear()
 
         for _, v in self._body:
             if isinstance(v, Table):
@@ -153,7 +163,7 @@ def _get_last_index_before_table(self) -> int:
 
     def _validate_out_of_order_table(self, key: Key | None = None) -> None:
         if key is None:
-            for k in self._map:
+            for k in list(self._out_of_order_keys):
                 assert k is not None
                 self._validate_out_of_order_table(k)
             return
@@ -162,6 +172,27 @@ def _validate_out_of_order_table(self, key: Key | None = None) -> None:
         current_idx = self._map[key]
         if not isinstance(current_idx, tuple):
             return
+        if self._parsed:
+            # while parsing, every fragment appended to an out-of-order key
+            # triggers a validation pass; resume from the cached temp
+            # container so each fragment is merged (and deep-copied) once
+            # instead of on every later pass. Fragments are only ever
+            # appended during parsing, so a count prefix stays valid; any
+            # other mutation clears the cache.
+            validated, temp = self._validation_cache.get(key, (0, None))
+            if validated > len(current_idx):
+                validated, temp = 0, None
+            try:
+                temp = OutOfOrderTableProxy.validate(
+                    self, current_idx[validated:], temp
+                )
+            except Exception:
+                # the temp container may be partially mutated; don't let a
+                # caught-and-retried failure resume from a poisoned cache
+                self._validation_cache.pop(key, None)
+                raise
+            self._validation_cache[key] = (len(current_idx), temp)
+            return
         OutOfOrderTableProxy.validate(self, current_idx)
 
     def append(
@@ -367,6 +398,7 @@ def _raw_append(self, key: Key | None, item: Item) -> None:
                 raise KeyAlreadyPresent(key)
 
             self._map[key] = (*current_idx, len(self._body))
+            self._out_of_order_keys.add(key)
         elif key is not None:
             self._map[key] = len(self._body)
 
@@ -383,6 +415,7 @@ def _remove_at(self, idx: int) -> None:
         index = self._map.get(key)
         if index is None:
             raise NonExistentKey(key)
+        self._validation_cache.clear()
         self._body[idx] = (None, Null())
 
         if isinstance(index, tuple):
@@ -405,6 +438,7 @@ def remove(self, key: Key | str) -> Container:
         if idx is None:
             raise NonExistentKey(key)
 
+        self._validation_cache.clear()
         if isinstance(idx, tuple):
             for i in idx:
                 self._body[i] = (None, Null())
@@ -500,6 +534,7 @@ def _insert_at(self, idx: int, key: Key | str, item: Any) -> Container:
             if not isinstance(current_idx, tuple):
                 current_idx = (current_idx,)
             self._map[key] = (*current_idx, idx)
+            self._out_of_order_keys.add(key)
         else:
             self._map[key] = idx
         self._body.insert(idx, (key, item))
@@ -735,7 +770,13 @@ def __contains__(self, key: object) -> bool:
         if idx is None:
             return False
         if isinstance(idx, tuple):
-            OutOfOrderTableProxy(self, idx)
+            # during parsing every fragment append re-probes membership;
+            # if the validation cache covers exactly these fragments they
+            # are already known to merge cleanly, so skip rebuilding the
+            # proxy (which is linear in the number of fragments)
+            validated, _ = self._validation_cache.get(key, (0, None))
+            if not (self._parsed and validated == len(idx)):
+                OutOfOrderTableProxy(self, idx)
         return True
 
     def __setitem__(self, key: Key | str, value: Any) -> None:
@@ -767,6 +808,7 @@ def _replace_at(
         self, idx: int | tuple[int, ...], new_key: Key | str, value: Item
     ) -> None:
         value = _item(value)
+        self._validation_cache.clear()
 
         if isinstance(idx, tuple):
             for i in idx[1:]:
@@ -872,6 +914,9 @@ def __setstate__(self, state: tuple[Any, ...]) -> None:
         self._body = state[1]
         self._parsed = state[2]
         self._table_keys = state[3]
+        self._out_of_order_keys = {
+            k for k, v in self._map.items() if isinstance(v, tuple)
+        }
 
         for key, item in self._body:
             if key is not None:
@@ -887,6 +932,7 @@ def __copy__(self) -> Self:
 
         c._body += self.body
         c._map.update(self._map)
+        c._out_of_order_keys |= self._out_of_order_keys
 
         return c
 
@@ -914,7 +960,11 @@ def _previous_item(
 
 class OutOfOrderTableProxy(_CustomDict):  # type: ignore[type-arg]
     @staticmethod
-    def validate(container: Container, indices: tuple[int, ...]) -> None:
+    def validate(
+        container: Container,
+        indices: tuple[int, ...],
+        temp_container: Container | None = None,
+    ) -> Container:
         """Validate out of order tables in the given container"""
         # Append all items to a temp container to see if there is any error.
         # We deep-copy each value before appending: appending a super table
@@ -923,7 +973,11 @@ def validate(container: Container, indices: tuple[int, ...]) -> None:
         # the merge would mutate the caller's tables (and corrupt a later
         # validation pass). The container merge itself is now copy-free for
         # speed, so this is where the isolation lives.
-        temp_container = Container(True)
+        # Passing a previous pass's temp container back in (with the already
+        # validated indices stripped from `indices`) resumes the validation
+        # incrementally.
+        if temp_container is None:
+            temp_container = Container(True)
         for i in indices:
             _, item = container._body[i]
 
@@ -932,6 +986,7 @@ def validate(container: Container, indices: tuple[int, ...]) -> None:
                     temp_container.append(k, copy.deepcopy(v), validate=True)
 
         temp_container._validate_out_of_order_table()
+        return temp_container
 
     def __init__(self, container: Container, indices: tuple[int, ...]) -> None:
         self._container = container