diff --git a/CHANGELOG.md b/CHANGELOG.md index 53741346..df6e33cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ - Speed up parsing by comparing `StringType` members by identity (`is`) instead of building a set on every `is_basic`/`is_literal`/`is_singleline`/`is_multiline` call, avoiding millions of enum hashes while parsing. ([#502](https://github.com/python-poetry/tomlkit/pull/502)) - Speed up merging super tables by merging in place instead of deep-copying the growing target on every merge, turning the parse of documents with many subtables under a shared super table (e.g. consecutive `[a.b.c]` / `[a.b.d]` headers) from O(n²) into O(n). ([#503](https://github.com/python-poetry/tomlkit/pull/503)) - Speed up membership tests (`key in ...`) on out-of-order tables with a native `OutOfOrderTableProxy.__contains__`, completing [#483](https://github.com/python-poetry/tomlkit/issues/483) for the last mapping type that still inherited the slow `MutableMapping` mixin (which resolves the value and builds an exception on every absent key). ([#515](https://github.com/python-poetry/tomlkit/pull/515)) +- Speed up parsing documents with many dotted keys or table headers sharing a prefix by validating out-of-order tables incrementally: each new fragment is merged into a cached validation container once, instead of re-merging (and deep-copying) every earlier fragment on each append, turning a super-cubic worst case into linear time (80 shared-prefix dotted keys: ~8 s → ~10 ms). ([#479](https://github.com/python-poetry/tomlkit/issues/479)) - Speed up parsing of arrays that close right after a value (e.g. the `files = [...]` blocks that dominate lock files): the parser no longer attempts to read a value while sitting on the closing `]`, which previously built an `UnexpectedCharError` just to discard it — and constructing that exception eagerly computes a line/column by scanning the whole document, making it O(document size) per such array. ([#517](https://github.com/python-poetry/tomlkit/pull/517)) ### Fixed diff --git a/tests/test_api.py b/tests/test_api.py index fba6a826..ed04e8a2 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -438,6 +438,32 @@ def test_value_rejects_values_with_appendage(raw: str) -> None: tomlkit.value(raw) +def test_parse_many_dotted_keys_with_shared_prefix() -> None: + """Each dotted key adds a fragment to the same out-of-order key; parsing + validates incrementally instead of re-merging every fragment (#479).""" + content = "\n".join(f"a.b.c.key{i} = {i}" for i in range(200)) + "\n" + doc = parse(content) + assert doc["a"]["b"]["c"]["key0"] == 0 + assert doc["a"]["b"]["c"]["key199"] == 199 + assert doc.as_string() == content + + +@pytest.mark.parametrize( + "content", + [ + "[a]\nb.c = 1\n[q]\nx = 1\n[a]\nb.c = 2", + "[t.a]\nk = 1\n[u]\nz = 1\n[t.a]\nk = 2", + ], +) +def test_parse_rejects_collisions_across_out_of_order_fragments( + content: str, +) -> None: + """Key collisions between out-of-order table parts still fail at parse + time with the incremental validation (#479).""" + with pytest.raises(tomlkit.exceptions.ParseError): + parse(content) + + def test_create_super_table_with_table() -> None: data = {"foo": {"bar": {"a": 1}}} assert dumps(data) == "[foo.bar]\na = 1\n" diff --git a/tomlkit/container.py b/tomlkit/container.py index f8896602..43b160f2 100644 --- a/tomlkit/container.py +++ b/tomlkit/container.py @@ -44,6 +44,15 @@ def __init__(self, parsed: bool = False) -> None: self._body: list[tuple[Key | None, Item]] = [] self._parsed = parsed self._table_keys: list[Key] = [] + # number of already-validated fragments and the temp container they + # were merged into, per out-of-order key; lets parse-time validation + # resume where the previous pass stopped instead of re-merging every + # fragment (quadratic) on each append + self._validation_cache: dict[Key, tuple[int, Container]] = {} + # superset of the keys mapped to an index tuple, so validating all + # out-of-order tables doesn't have to scan every key in the map; + # stale entries are filtered by the per-key isinstance check + self._out_of_order_keys: set[Key] = set() @property def body(self) -> list[tuple[Key | None, Item]]: @@ -89,6 +98,7 @@ def value(self) -> dict[str, Any]: def parsing(self, parsing: bool) -> None: self._parsed = parsing + self._validation_cache.clear() for _, v in self._body: if isinstance(v, Table): @@ -153,7 +163,7 @@ def _get_last_index_before_table(self) -> int: def _validate_out_of_order_table(self, key: Key | None = None) -> None: if key is None: - for k in self._map: + for k in list(self._out_of_order_keys): assert k is not None self._validate_out_of_order_table(k) return @@ -162,6 +172,27 @@ def _validate_out_of_order_table(self, key: Key | None = None) -> None: current_idx = self._map[key] if not isinstance(current_idx, tuple): return + if self._parsed: + # while parsing, every fragment appended to an out-of-order key + # triggers a validation pass; resume from the cached temp + # container so each fragment is merged (and deep-copied) once + # instead of on every later pass. Fragments are only ever + # appended during parsing, so a count prefix stays valid; any + # other mutation clears the cache. + validated, temp = self._validation_cache.get(key, (0, None)) + if validated > len(current_idx): + validated, temp = 0, None + try: + temp = OutOfOrderTableProxy.validate( + self, current_idx[validated:], temp + ) + except Exception: + # the temp container may be partially mutated; don't let a + # caught-and-retried failure resume from a poisoned cache + self._validation_cache.pop(key, None) + raise + self._validation_cache[key] = (len(current_idx), temp) + return OutOfOrderTableProxy.validate(self, current_idx) def append( @@ -367,6 +398,7 @@ def _raw_append(self, key: Key | None, item: Item) -> None: raise KeyAlreadyPresent(key) self._map[key] = (*current_idx, len(self._body)) + self._out_of_order_keys.add(key) elif key is not None: self._map[key] = len(self._body) @@ -383,6 +415,7 @@ def _remove_at(self, idx: int) -> None: index = self._map.get(key) if index is None: raise NonExistentKey(key) + self._validation_cache.clear() self._body[idx] = (None, Null()) if isinstance(index, tuple): @@ -405,6 +438,7 @@ def remove(self, key: Key | str) -> Container: if idx is None: raise NonExistentKey(key) + self._validation_cache.clear() if isinstance(idx, tuple): for i in idx: self._body[i] = (None, Null()) @@ -500,6 +534,7 @@ def _insert_at(self, idx: int, key: Key | str, item: Any) -> Container: if not isinstance(current_idx, tuple): current_idx = (current_idx,) self._map[key] = (*current_idx, idx) + self._out_of_order_keys.add(key) else: self._map[key] = idx self._body.insert(idx, (key, item)) @@ -735,7 +770,13 @@ def __contains__(self, key: object) -> bool: if idx is None: return False if isinstance(idx, tuple): - OutOfOrderTableProxy(self, idx) + # during parsing every fragment append re-probes membership; + # if the validation cache covers exactly these fragments they + # are already known to merge cleanly, so skip rebuilding the + # proxy (which is linear in the number of fragments) + validated, _ = self._validation_cache.get(key, (0, None)) + if not (self._parsed and validated == len(idx)): + OutOfOrderTableProxy(self, idx) return True def __setitem__(self, key: Key | str, value: Any) -> None: @@ -767,6 +808,7 @@ def _replace_at( self, idx: int | tuple[int, ...], new_key: Key | str, value: Item ) -> None: value = _item(value) + self._validation_cache.clear() if isinstance(idx, tuple): for i in idx[1:]: @@ -872,6 +914,9 @@ def __setstate__(self, state: tuple[Any, ...]) -> None: self._body = state[1] self._parsed = state[2] self._table_keys = state[3] + self._out_of_order_keys = { + k for k, v in self._map.items() if isinstance(v, tuple) + } for key, item in self._body: if key is not None: @@ -887,6 +932,7 @@ def __copy__(self) -> Self: c._body += self.body c._map.update(self._map) + c._out_of_order_keys |= self._out_of_order_keys return c @@ -914,7 +960,11 @@ def _previous_item( class OutOfOrderTableProxy(_CustomDict): # type: ignore[type-arg] @staticmethod - def validate(container: Container, indices: tuple[int, ...]) -> None: + def validate( + container: Container, + indices: tuple[int, ...], + temp_container: Container | None = None, + ) -> Container: """Validate out of order tables in the given container""" # Append all items to a temp container to see if there is any error. # We deep-copy each value before appending: appending a super table @@ -923,7 +973,11 @@ def validate(container: Container, indices: tuple[int, ...]) -> None: # the merge would mutate the caller's tables (and corrupt a later # validation pass). The container merge itself is now copy-free for # speed, so this is where the isolation lives. - temp_container = Container(True) + # Passing a previous pass's temp container back in (with the already + # validated indices stripped from `indices`) resumes the validation + # incrementally. + if temp_container is None: + temp_container = Container(True) for i in indices: _, item = container._body[i] @@ -932,6 +986,7 @@ def validate(container: Container, indices: tuple[int, ...]) -> None: temp_container.append(k, copy.deepcopy(v), validate=True) temp_container._validate_out_of_order_table() + return temp_container def __init__(self, container: Container, indices: tuple[int, ...]) -> None: self._container = container