diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 678b550..6273f8b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,7 +56,7 @@ jobs: - name: Install the reviewed CI lock run: python -m pip install --require-hashes -r requirements/ci.lock - run: ruff check . - - run: python -m compileall -q src tests scripts + - run: python -m compileall -q src tests scripts benchmarks - name: Run production doctests run: | python -m doctest \ @@ -143,6 +143,9 @@ jobs: cd "$temp_dir" python - <<'PY' from threadweave import ( + IncrementalThreadIndex, + IndexedMessage, + MailboxChangeSet, Message, __version__, serialize_thread_response, @@ -170,6 +173,39 @@ jobs: "later@example.com", ] assert serialize_thread_response(roots) == "* THREAD (1)(2)\r\n" + + index = IncrementalThreadIndex() + delta = index.apply( + MailboxChangeSet( + expected_version=0, + additions=( + IndexedMessage( + "root-key", + Message(message_id="root", sequence_number=1, uid=101), + email_id="Email_101", + thread_id="Thread_7", + ), + IndexedMessage( + "reply-key", + Message( + message_id="reply", + references=["root"], + sequence_number=2, + uid=102, + ), + email_id="Email_102", + thread_id="Thread_7", + ), + ), + ) + ) + assert delta.version == 1 + assert index.projections[0].message_keys == ("root-key", "reply-key") + assert serialize_thread_response(index.roots, identifier="uid") == ( + "* THREAD (101 102)\r\n" + ) + restored = IncrementalThreadIndex.restore(index.snapshot()) + assert restored.projections == index.projections PY - run: python -m pip check - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 diff --git a/.github/workflows/diagnose-hourly-product-workflow.yml b/.github/workflows/diagnose-hourly-product-workflow.yml new file mode 100644 index 0000000..3da59a9 --- /dev/null +++ b/.github/workflows/diagnose-hourly-product-workflow.yml @@ -0,0 +1,69 @@ +name: Diagnose repository workflows + +on: + push: + branches: + - feature/incremental-thread-index + paths: + - .github/workflows/diagnose-hourly-product-workflow.yml + +permissions: + contents: read + +jobs: + actionlint: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - name: Check out the exact branch + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: feature/incremental-thread-index + persist-credentials: false + + - name: Install pinned actionlint + shell: bash + run: | + set -euo pipefail + archive="${RUNNER_TEMP}/actionlint.tar.gz" + curl -fsSL \ + -o "$archive" \ + https://github.com/rhysd/actionlint/releases/download/v1.7.12/actionlint_1.7.12_linux_amd64.tar.gz + printf '%s %s\n' \ + 8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8 \ + "$archive" | sha256sum -c - + tar -xzf "$archive" -C "$RUNNER_TEMP" actionlint + + - name: Apply known corrections only in the diagnostic workspace + shell: bash + run: | + set -euo pipefail + python - <<'PY' + from pathlib import Path + + path = Path('.github/workflows/hourly-product-development.yml') + text = path.read_text(encoding='utf-8') + indentation_old = " printf 'threadweave @ file://%s \\\n --hash=sha256:%s\\n' \\\n" + indentation_new = " printf 'threadweave @ file://%s \\\n --hash=sha256:%s\\n' \\\n" + if text.count(indentation_old) != 1: + raise SystemExit('unexpected hourly workflow indentation state') + text = text.replace(indentation_old, indentation_new) + + shellcheck_old = " if timeout --kill-after=30s \"${OPENCODE_RUN_TIMEOUT_SECONDS}s\" \\\n" + shellcheck_new = ( + " # shellcheck disable=SC2016\n" + " if timeout --kill-after=30s \"${OPENCODE_RUN_TIMEOUT_SECONDS}s\" \\\n" + ) + if text.count(shellcheck_old) != 1: + raise SystemExit('unexpected nested-shell state') + path.write_text(text.replace(shellcheck_old, shellcheck_new), encoding='utf-8') + PY + + - name: Diagnose every repository workflow after known corrections + shell: bash + run: | + set -o pipefail + "${RUNNER_TEMP}/actionlint" \ + -color=false \ + .github/workflows/*.yml \ + 2>&1 | tee "${RUNNER_TEMP}/actionlint-output.txt" diff --git a/.github/workflows/incremental-benchmark.yml b/.github/workflows/incremental-benchmark.yml new file mode 100644 index 0000000..7296aa1 --- /dev/null +++ b/.github/workflows/incremental-benchmark.yml @@ -0,0 +1,96 @@ +name: Incremental Mailbox Benchmark + +on: + workflow_dispatch: + inputs: + messages: + description: Number of existing mailbox messages + required: false + default: "100000" + type: string + schedule: + - cron: "17 3 * * 1" + +permissions: + contents: read + +concurrency: + group: incremental-mailbox-benchmark-${{ github.repository }} + cancel-in-progress: true + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + PYTHONPATH: src + +jobs: + benchmark: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - name: Harden runner and block undeclared egress + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: block + disable-telemetry: true + allowed-endpoints: | + codeload.github.com:443 + files.pythonhosted.org:443 + github.com:443 + objects.githubusercontent.com:443 + pypi.org:443 + release-assets.githubusercontent.com:443 + results-receiver.actions.githubusercontent.com:443 + *.actions.githubusercontent.com:443 + *.blob.core.windows.net:443 + + - name: Check out the reviewed source without persisted credentials + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.13" + cache: pip + cache-dependency-path: requirements/ci.lock + + - name: Install the reviewed benchmark toolchain + run: python -m pip install --require-hashes -r requirements/ci.lock + + - name: Run the isolated mailbox benchmark + env: + BENCHMARK_MESSAGES: ${{ inputs.messages || '100000' }} + run: | + set -euo pipefail + python benchmarks/incremental_mailbox.py \ + --messages "$BENCHMARK_MESSAGES" \ + --thread-size 10 \ + --output incremental-benchmark.json + + - name: Enforce parity and the mailbox-scale delta target + run: | + python - <<'PY' + import json + from pathlib import Path + + result = json.loads( + Path("incremental-benchmark.json").read_text(encoding="utf-8") + ) + incremental = result["incremental"] + full_rebuild = result["full_rebuild"] + assert incremental["projection_sha256"] == full_rebuild["projection_sha256"] + assert incremental["affected_message_count"] == 21 + if result["message_count"] >= 100_001: + assert incremental["delta_apply_seconds"] < full_rebuild[ + "full_rebuild_seconds" + ] + PY + + - name: Upload benchmark evidence + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: incremental-mailbox-benchmark-${{ github.run_id }}-${{ github.run_attempt }} + path: incremental-benchmark.json + if-no-files-found: error + retention-days: 90 diff --git a/AGENTS.md b/AGENTS.md index b625b3f..d5b25b8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -5,11 +5,12 @@ Operating guide for automated agents working on this repository. `threadweave` implements the JWZ container model with RFC 5256 `REFERENCES` threading semantics, RFC 5322 identification-field parsing, RFC 2047 encoded-word decoding, RFC 5256 base-subject extraction, RFC 5051 Unicode casemap comparison, -optional RFC 5256 sent-date ordering, and RFC 5256 IMAP `THREAD` response -serialization. Its value is correctness: mail clients and ingestion systems rely -on threading being deterministic, standards-grounded, and impossible to hang on -malformed input. Treat changes to `threading.py`, `container.py`, `subject.py`, -`collation.py`, `dates.py`, `headers.py`, and `imap.py` as behavior-sensitive. +optional RFC 5256 sent-date ordering, RFC 5256 IMAP `THREAD` response +serialization, and atomic incremental mailbox indexing. Its value is correctness: +mail clients and ingestion systems rely on threading being deterministic, +standards-grounded, and impossible to hang on malformed input. Treat changes to +`threading.py`, `incremental.py`, `container.py`, `subject.py`, `collation.py`, +`dates.py`, `headers.py`, and `imap.py` as behavior-sensitive. ## Invariants that must not regress @@ -55,6 +56,17 @@ malformed input. Treat changes to `threading.py`, `container.py`, `subject.py`, dummy-root grouping after search projection, reject cyclic or shared graphs, and leave the source `Container` tree unchanged. Rendering stays iterative, and response framing accepts only CRLF or a caller-owned empty suffix. +13. **Incremental updates remain batch-equivalent and atomic.** Immutable caller + message keys—not sequence numbers—identify indexed records. Recompute every + affected old/new reference or subject component through `thread_messages`, + but do not pass unrelated components to the batch delegate. Validation or + recomputation failure must leave records, version, roots, and projections + unchanged. Snapshot state excludes arbitrary payloads and graph pointers. +14. **External identities follow RFC 8474.** `EMAILID` and `THREADID` use exact + 1–255 character ObjectID grammar, are case-sensitive, and use disjoint + namespaces. Equal EMAILIDs require equal THREADIDs. Once a non-null value is + reported, replacement cannot remove or change it; merges and splits remain + explicit transitions rather than silent identity rewrites. ## Architecture and dependency rules @@ -63,10 +75,15 @@ malformed input. Treat changes to `threading.py`, `container.py`, `subject.py`, justified. - Preserve the standalone package API and its use as a naruon module. The header primitives originated in naruon; port behavioral fixes in both directions. -- Keep IMAP response serialization separate from the transport-neutral tree and - date layers so non-IMAP callers do not inherit protocol-specific state. -- Public behavior, compatibility aliases, and typing markers are release - contracts. Record changes in `CHANGELOG.md` and update user/research docs. +- Keep batch threading authoritative. The incremental layer owns caller keys, + component bookkeeping, deltas, and payload-free snapshots; it must delegate + every recomputed component to the existing batch engine rather than fork the + threading algorithm. +- Keep IMAP response serialization separate from the transport-neutral batch, + incremental, and date layers so non-IMAP callers do not inherit protocol state. +- Public behavior, compatibility aliases, typing markers, snapshot schemas, and + external-ID handoff are release contracts. Record changes in `CHANGELOG.md` + and update user/research docs. - Unicode collation results depend on the Unicode Character Database bundled with the supported Python runtime. Tests must cover stable RFC examples and security-sensitive non-equivalences rather than version-specific new codepoints. diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 0000000..1fb9ac6 --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,88 @@ +# ThreadWeave Architecture + +## Decision status + +This document is the repository-level architecture decision record. `AGENTS.md` is +the canonical operating policy; this file explains component boundaries and data +flow for human reviewers and embedding services such as naruon. + +## Architectural goal + +ThreadWeave provides one standards-grounded threading kernel that works both as a +zero-runtime-dependency Python package and as a module inside a larger mail or +knowledge platform. Protocol, incremental state, automation, and release concerns +remain separate from the canonical batch algorithm. + +## Modules + +| Boundary | Responsibility | Must not own | +|---|---|---| +| `headers`, `encoded_words`, `subject`, `collation`, `dates` | RFC parsing, normalization, and comparison primitives | graph state, sockets, databases | +| `threading`, `container` | authoritative JWZ/RFC 5256 batch forest | mailbox sessions, persistence, IMAP framing | +| `incremental` | caller keys, atomic change sets, component indexes, deltas, payload-free snapshots | a second threading algorithm, database/network state | +| `imap` | non-mutating RFC 5256 `THREAD`/`UID THREAD` presentation | authentication, command parsing, mailbox storage | +| stdlib adapters | conversion from Python `email` messages | transport sessions or durable state | +| GitHub workflows and `scripts/ci` | review-first automation, NIM isolation, release evidence | runtime package behavior | + +## Authoritative data flow + +```text +caller message metadata + -> RFC normalization + -> canonical batch thread_messages(component) + -> Container forest + -> optional incremental component cache and ThreadDelta + -> optional IMAP response projection +``` + +The incremental layer over-approximates connectivity with normalized message IDs, +effective reference IDs, and optional RFC 5051 subject keys. Every affected +component is still evaluated by `thread_messages`; no copy of the threading rules +is maintained in incremental code. + +## State and mutation policy + +- `IndexedMessage.message_key` is caller-owned and immutable across revisions. +- Public reads, snapshots, and `apply` calls on one index are serialized by an + in-process reentrant lock. Two transactions targeting one version cannot both + commit; the later transaction observes the new version and fails explicitly. +- The process-local lock is not a distributed lock. Naruon or another host must + serialize durable writes across processes and persist the optimistic version. +- `apply` validates and computes on isolated transaction overlays, then commits once. +- Default-mode updates retain the existing state-map objects and publish only touched + record, position, token, component, EMAILID, and THREADID entries. +- Reverse connectivity buckets use copy-on-write mutation; RFC 8474 indexes retain + compact association/count state rather than one set object per message. +- RFC sent-date ordering may still scan the mailbox to derive global ranks and reject + effective sequence-number collisions; the default first-appearance mode does not. +- Complete root/projection views are lazy caches invalidated by a successful change. +- `roots` returns a defensive structural copy; payload references stay caller-owned. +- Snapshot schema version 1 contains structural metadata only, never payload objects + or graph pointers. +- RFC 8474 `EMAILID` and `THREADID` remain external identities. Structural merges + and splits are explicit transitions rather than silent identifier rewrites. + +## Ordering and protocol policy + +Implicit input positions may be used internally for RFC 5256 sent-date ordering, +but they are not mailbox sequence numbers and are cleared before public roots are +returned. IMAP serialization therefore fails closed unless callers supply valid +sequence numbers or UIDs. + +## Scale evidence + +The deterministic benchmark runs incremental and full-rebuild workers in separate +processes. It compares projection SHA-256 values and records initial-build time, +delta-application time, full-view materialization time, full-rebuild time, affected +message count, root count, delta retained/transient traced bytes, and peak RSS. +Scheduled evidence defaults to 100,000 existing messages. Focused performance +contracts additionally reject default-mode +small-delta implementations that iterate or replace unrelated state maps. + +## Integration policy + +Naruon and other services should own persistence, tenancy, authentication, +mailbox synchronization, distributed write serialization, and external stable-ID +policy. They pass immutable caller keys and `Message` metadata into ThreadWeave and +consume `ThreadDelta`, snapshots, or IMAP presentation output through public +interfaces. diff --git a/CHANGELOG.md b/CHANGELOG.md index a121fa6..34aa4ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,51 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## Unreleased +- Record retained and transient traced allocation for each isolated incremental + benchmark delta so scheduled 100,000-message evidence detects mailbox-wide + copying regressions alongside wall time, affected-message count, and peak RSS. +- Apply default-mode incremental deltas through bounded overlay mappings and + touched reverse indexes instead of cloning or scanning every mailbox state map; + unchanged records, positions, components, and RFC 8474 identity namespaces stay + in place until one validated commit. +- Preflight incremental snapshot root fields and record counts before nested + traversal, and cap plain-container validation by the configured byte limit so + oversized hostile trees fail before JSON encoder construction. +- Bound incremental snapshot size checks to streaming UTF-8 code-point + accounting without encoding each JSON chunk into a second bytes object, and + reject reused container identities so compact Python object graphs cannot + trigger exponential JSON expansion. +- Reject cyclic built-in dictionaries and lists at the incremental snapshot + restore boundary without recursion or unbounded traversal. +- Reject container and scalar subclasses plus non-plain-string object keys at + the incremental snapshot restore boundary before sorted JSON encoding can + invoke attacker-controlled iteration or comparison methods. +- Serialize every read and write on one `IncrementalThreadIndex` with a + process-local reentrant lock so same-version concurrent writers yield one + commit and one explicit conflict, while readers observe only committed state. +- Keep implicit RFC 5256 sent-date tie-break positions internal to the incremental + engine instead of exposing invented IMAP sequence numbers on public roots. +- Return defensive structural copies from `IncrementalThreadIndex.roots` so caller + graph edits cannot corrupt reusable index state while payload objects remain + caller-owned references. +- Replace quadratic disconnected-component partitioning and pairwise thread-delta + comparisons with bounded indexed passes, copy reverse-token buckets only when + touched, and defer complete forest materialization until a caller requests it. +- Add deterministic 100,000-message incremental-versus-full-rebuild benchmark + evidence with projection parity, affected-message counts, wall time, and peak RSS. +- Harden incremental snapshot publication and restore so schema versions require + exact non-boolean integers and hostile nesting or unencodable Unicode fails + closed with `IncrementalThreadError` instead of leaking runtime exceptions. + +- Add an atomic `IncrementalThreadIndex` for mailbox additions, replacements, + and removals with stable caller message keys, affected-component + recomputation, batch-result parity, and explicit thread merge/split deltas. +- Add strict RFC 8474 `EMAILID` and `THREADID` handoff, including immutable + values, exact ObjectID grammar, disjoint namespaces, and consistent THREADID + values for equal EMAILIDs. +- Add versioned, bounded, JSON-safe incremental snapshots that omit arbitrary + caller payloads and rebuild derived graph state through validated metadata. + ## [0.2.0] - 2026-08-04 - Add iterative RFC 5256 `THREAD` and `UID THREAD` response serialization with diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..bf19006 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,19 @@ +# Claude / Coding-Agent Instructions + +Read and follow [`AGENTS.md`](AGENTS.md) as the canonical repository policy. Do not +create a second, conflicting rule set here. + +For every change: + +1. Preserve the batch threader as the correctness oracle. +2. Work test-first and retain 100% production statement and branch coverage. +3. Add beginner-readable docstrings to every authored production callable. +4. Keep runtime dependencies at zero unless an approved architecture decision says + otherwise. +5. Update `CHANGELOG.md`, user documentation, and standards references when public + behavior changes. +6. Never bypass current-head CI, security scans, independent review, release + identity checks, or the release-blocker contract. +7. GitHub product-development automation uses the isolated NVIDIA NIM/OpenCode + boundary. Do not introduce `COPILOT_GITHUB_TOKEN` or alter existing review-agent + credentials. diff --git a/README.md b/README.md index 2e78d16..5106ba4 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ runtime dependencies.** trees. It combines the JWZ container model with RFC 5322 identification fields, RFC 2047 encoded-word decoding, RFC 5256 base-subject extraction and optional sent-date ordering, RFC 5051 `i;unicode-casemap` comparison, and RFC 5256 IMAP -`THREAD` response serialization. +`THREAD` response serialization, and atomic incremental mailbox updates. It accepts normalized identifiers, raw header strings, or Python standard-library `email.message.Message` objects. Malformed historical mail, missing roots, @@ -167,6 +167,70 @@ source containers are never mutated. Cycles, shared nodes, duplicate numbers, missing UIDs, values outside the non-zero unsigned 32-bit range, and unsafe line endings fail closed. Both deep chains and nested splits are rendered iteratively. +## Incremental mailbox updates + +`IncrementalThreadIndex` applies atomic additions, replacements, and removals +without re-threading unrelated reference components. Caller-owned message keys +remain stable across expunge or mailbox sequence-number changes; optional RFC +8474 `EMAILID` and `THREADID` values are validated and never silently rewritten. + +```python +from threadweave import ( + IncrementalThreadIndex, + IndexedMessage, + MailboxChangeSet, + Message, +) + +index = IncrementalThreadIndex() +delta = index.apply( + MailboxChangeSet( + expected_version=0, + additions=( + IndexedMessage( + "mailbox:101", + Message(message_id="root", sequence_number=1, uid=101), + email_id="Email_101", + thread_id="Thread_7", + ), + IndexedMessage( + "mailbox:102", + Message( + message_id="reply", + references=["root"], + sequence_number=2, + uid=102, + ), + email_id="Email_102", + thread_id="Thread_7", + ), + ), + ) +) + +assert delta.version == 1 +assert index.projections[0].message_keys == ("mailbox:101", "mailbox:102") +assert IncrementalThreadIndex.restore(index.snapshot()).projections == ( + index.projections +) +``` + +Every affected component is recomputed through the canonical batch threader, and +full-rebuild parity is the correctness oracle. Default-mode transactions stage +records, positions, connectivity, component ownership, and RFC 8474 identity +changes through bounded overlays; they do not copy or iterate unrelated mailbox +state before one validated commit. Structural merges and splits are reported +explicitly. `roots` returns a defensive structural copy, so callers may +traverse or edit that graph without corrupting reusable index state; payload objects +remain caller-owned references. Internal sent-date tie-break positions never become +public IMAP sequence numbers. Versioned snapshots omit arbitrary payloads, reject +unknown, malformed, cyclic, aliased, or oversized input, and count UTF-8 +code-point width to stop at the configured byte limit without encoding a second +full bytes copy. Restore checks the fixed root schema and record-count bound before +nested traversal, then caps the structural scan with the same byte limit. See +[`docs/incremental-threading.md`](docs/incremental-threading.md) for the atomicity, +identity, snapshot, complexity, and RFC boundaries. + ## Public API | Symbol | Purpose | @@ -176,6 +240,11 @@ endings fail closed. Both deep chains and nested splits are rendered iteratively | `thread_messages(...)` | Build JWZ/RFC 5256 thread roots from any iterable. | | `message_from_email(...)` | Convert one stdlib email object. | | `thread_email_messages(...)` | Convert and thread stdlib email objects. | +| `IncrementalThreadIndex` | Apply atomic mailbox deltas and expose batch-equivalent roots. | +| `IndexedMessage` | Bind one stable caller key and optional RFC 8474 identities to a message. | +| `MailboxChangeSet` | Describe one optimistic additions/replacements/removals transaction. | +| `ThreadDelta` | Report affected keys, projection changes, merges, and splits. | +| `ThreadProjection` | Describe one root with traversal-ordered caller keys and THREADIDs. | | `serialize_thread_data(...)` | Render RFC 5256 `thread-data` without response framing. | | `serialize_thread_response(...)` | Render one untagged `* THREAD` response. | | `ThreadSerializationError` | Report invalid graph or mailbox identifier state. | @@ -202,8 +271,13 @@ endings fail closed. Both deep chains and nested splits are rendered iteratively on Python 3.10, 3.11, 3.12, and 3.13. - CI builds wheel and source distributions, verifies `py.typed`, installs the wheel outside the source tree, and executes a smoke test. -- Graph operations and IMAP rendering are iterative and identity-guarded; deep - or cyclic malformed input cannot recurse indefinitely. +- Graph operations, IMAP rendering, and incremental component traversal are + iterative and identity-guarded; deep or cyclic malformed input cannot recurse + indefinitely. +- `benchmarks/incremental_mailbox.py` compares isolated incremental and full + rebuild processes at 100,000 messages, verifies an identical projection digest, + and records affected-message count, wall time, delta retained/transient traced + bytes, and peak RSS as JSON evidence. ## Reproducible CI supply chain @@ -273,12 +347,14 @@ unavailable safe proposal stops the cycle without mutation. ## Architecture and standards boundary The package remains useful both as a standalone dependency and as a module in -`naruon` or another service. The threading, subject, collation, and date layers -are transport-neutral. IMAP `THREAD` response serialization is a separate -presentation layer rather than protocol state embedded in the core model. +`naruon` or another service. The batch-threading, incremental-index, subject, +collation, and date layers are transport-neutral. IMAP `THREAD` response +serialization is a separate presentation layer rather than protocol state +embedded in the core model. See [`docs/research`](docs/research/README.md) for JWZ, RFC 5322, RFC 2047, -RFC 5051, RFC 5256, RFC 6532, RFC 9051, Unicode-version boundaries, and PEP 561. +RFC 5051, RFC 5256, RFC 6532, RFC 7162, RFC 8474, RFC 8621, RFC 9051, +Unicode-version boundaries, and PEP 561. ## License diff --git a/benchmarks/incremental_mailbox.py b/benchmarks/incremental_mailbox.py new file mode 100644 index 0000000..fdaedc5 --- /dev/null +++ b/benchmarks/incremental_mailbox.py @@ -0,0 +1,241 @@ +"""Deterministic mailbox-scale benchmark for the incremental threading layer. + +The parent process runs incremental and full-rebuild workers separately so each +scenario reports its own peak resident-set size. Both workers hash the same +caller-key projection to prove structural parity without serializing a 100,000- +message forest between processes. +""" + +from __future__ import annotations + +import argparse +import gc +import hashlib +import json +import resource +import subprocess +import sys +import tracemalloc +from collections.abc import Iterable +from time import perf_counter + +from threadweave import ( + Container, + IncrementalThreadIndex, + IndexedMessage, + MailboxChangeSet, + Message, + thread_messages, +) + + +def _records(message_count: int, thread_size: int) -> tuple[IndexedMessage, ...]: + """Create deterministic thread chains with caller keys retained as payloads.""" + records: list[IndexedMessage] = [] + for index in range(message_count): + position = index % thread_size + references = () if position == 0 else (f"message_{index - 1}",) + records.append( + IndexedMessage( + f"key_{index}", + Message( + message_id=f"message_{index}", + references=references, + payload=f"key_{index}", + ), + ) + ) + return tuple(records) + + +def _bridge_record(thread_size: int) -> IndexedMessage: + """Return one message that joins the first two deterministic components.""" + return IndexedMessage( + "bridge_key", + Message( + message_id="bridge_message", + references=( + f"message_{thread_size - 1}", + f"message_{(thread_size * 2) - 1}", + ), + payload="bridge_key", + ), + ) + + +def _projection_digest(projections: Iterable[tuple[str, ...]]) -> str: + """Hash ordered root projections with unambiguous length framing.""" + digest = hashlib.sha256() + for projection in projections: + digest.update(len(projection).to_bytes(8, "big")) + for message_key in projection: + encoded = message_key.encode("utf-8") + digest.update(len(encoded).to_bytes(8, "big")) + digest.update(encoded) + return digest.hexdigest() + + +def _forest_projection(roots: Iterable[Container]) -> tuple[tuple[str, ...], ...]: + """Project a batch forest into iterative traversal-ordered payload keys.""" + result: list[tuple[str, ...]] = [] + for root in roots: + keys: list[str] = [] + seen: set[int] = set() + stack = [root] + while stack: + node = stack.pop() + if id(node) in seen: + continue + seen.add(id(node)) + if node.message is not None: + if not isinstance(node.message.payload, str): + raise RuntimeError("benchmark payload must be a caller key string") + keys.append(node.message.payload) + stack.extend(reversed(node.children)) + result.append(tuple(keys)) + return tuple(result) + + +def _peak_rss_bytes() -> int: + """Return the process peak RSS in bytes on the Linux benchmark runner.""" + return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * 1024 + + +def _incremental_worker(message_count: int, thread_size: int) -> dict[str, object]: + """Build an index, apply one bridge delta, and materialize its final view.""" + records = _records(message_count, thread_size) + index = IncrementalThreadIndex() + started = perf_counter() + index.apply(MailboxChangeSet(expected_version=0, additions=records)) + initial_seconds = perf_counter() - started + + bridge = _bridge_record(thread_size) + gc.collect() + tracemalloc.start() + baseline_current, _ = tracemalloc.get_traced_memory() + started = perf_counter() + delta = index.apply( + MailboxChangeSet(expected_version=1, additions=(bridge,)) + ) + delta_seconds = perf_counter() - started + current_bytes, peak_bytes = tracemalloc.get_traced_memory() + tracemalloc.stop() + + started = perf_counter() + projections = tuple( + projection.message_keys for projection in index.projections + ) + materialize_seconds = perf_counter() - started + return { + "scenario": "incremental", + "message_count": message_count + 1, + "root_count": len(projections), + "affected_message_count": len(delta.affected_message_keys), + "initial_build_seconds": initial_seconds, + "delta_apply_seconds": delta_seconds, + "delta_retained_bytes": current_bytes - baseline_current, + "delta_transient_peak_bytes": peak_bytes - baseline_current, + "materialize_seconds": materialize_seconds, + "peak_rss_bytes": _peak_rss_bytes(), + "projection_sha256": _projection_digest(projections), + } + + +def _full_worker(message_count: int, thread_size: int) -> dict[str, object]: + """Build the same final mailbox through the canonical batch oracle.""" + records = _records(message_count, thread_size) + messages = [record.message for record in records] + messages.append(_bridge_record(thread_size).message) + started = perf_counter() + roots = thread_messages(messages) + rebuild_seconds = perf_counter() - started + projections = _forest_projection(roots) + return { + "scenario": "full_rebuild", + "message_count": message_count + 1, + "root_count": len(projections), + "full_rebuild_seconds": rebuild_seconds, + "peak_rss_bytes": _peak_rss_bytes(), + "projection_sha256": _projection_digest(projections), + } + + +def _run_worker(scenario: str, message_count: int, thread_size: int) -> dict[str, object]: + """Execute one isolated worker and decode its single JSON result.""" + completed = subprocess.run( + [ + sys.executable, + __file__, + "--worker", + scenario, + "--messages", + str(message_count), + "--thread-size", + str(thread_size), + ], + check=True, + capture_output=True, + text=True, + ) + return json.loads(completed.stdout) + + +def _validated_positive(value: int, name: str) -> int: + """Require one positive non-boolean benchmark integer.""" + if isinstance(value, bool) or value <= 0: + raise ValueError(f"{name} must be a positive integer") + return value + + +def run_benchmark(message_count: int, thread_size: int) -> dict[str, object]: + """Run isolated scenarios, require parity, and return one evidence record.""" + message_count = _validated_positive(message_count, "message_count") + thread_size = _validated_positive(thread_size, "thread_size") + if message_count < thread_size * 2: + raise ValueError("message_count must contain at least two complete threads") + incremental = _run_worker("incremental", message_count, thread_size) + full_rebuild = _run_worker("full_rebuild", message_count, thread_size) + if incremental["projection_sha256"] != full_rebuild["projection_sha256"]: + raise RuntimeError("incremental and full-rebuild projections disagree") + return { + "schema_version": 1, + "message_count": message_count + 1, + "thread_size": thread_size, + "incremental": incremental, + "full_rebuild": full_rebuild, + } + + +def _parser() -> argparse.ArgumentParser: + """Build the command-line parser for parent and worker modes.""" + parser = argparse.ArgumentParser() + parser.add_argument("--messages", type=int, default=100_000) + parser.add_argument("--thread-size", type=int, default=10) + parser.add_argument( + "--worker", + choices=("incremental", "full_rebuild"), + ) + parser.add_argument("--output") + return parser + + +def main(argv: list[str] | None = None) -> int: + """Run a worker or publish the combined deterministic benchmark JSON.""" + arguments = _parser().parse_args(argv) + if arguments.worker == "incremental": + result = _incremental_worker(arguments.messages, arguments.thread_size) + elif arguments.worker == "full_rebuild": + result = _full_worker(arguments.messages, arguments.thread_size) + else: + result = run_benchmark(arguments.messages, arguments.thread_size) + encoded = json.dumps(result, indent=2, sort_keys=True) + "\n" + if arguments.output: + with open(arguments.output, "w", encoding="utf-8") as output_file: + output_file.write(encoded) + else: + sys.stdout.write(encoded) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docs/incremental-threading.md b/docs/incremental-threading.md new file mode 100644 index 0000000..aa967fa --- /dev/null +++ b/docs/incremental-threading.md @@ -0,0 +1,259 @@ +# Incremental mailbox threading + +`IncrementalThreadIndex` applies mailbox additions, replacements, and removals +without rebuilding unrelated reference components. It delegates every affected +component to the same `thread_messages` batch implementation used by the public +batch API, so full reconstruction remains the correctness oracle. + +The incremental layer is transport-neutral. It contains no database, network, +authentication, IMAP session, or JMAP state and can be used as a standalone +library or embedded in naruon and other services. + +## Basic use + +```python +from threadweave import ( + IncrementalThreadIndex, + IndexedMessage, + MailboxChangeSet, + Message, + serialize_thread_response, +) + +index = IncrementalThreadIndex(sort_by_sent_date=True) + +initial_delta = index.apply( + MailboxChangeSet( + expected_version=0, + additions=( + IndexedMessage( + message_key="mailbox:101", + message=Message( + message_id="", + sent_date="1 Jan 2026 00:00:00 +0000", + sequence_number=1, + uid=101, + ), + email_id="Email_101", + thread_id="Thread_7", + ), + IndexedMessage( + message_key="mailbox:102", + message=Message( + message_id="", + references="", + sent_date="2 Jan 2026 00:00:00 +0000", + sequence_number=2, + uid=102, + ), + email_id="Email_102", + thread_id="Thread_7", + ), + ), + ) +) + +assert initial_delta.version == 1 +assert index.projections[0].message_keys == ( + "mailbox:101", + "mailbox:102", +) +assert serialize_thread_response(index.roots, identifier="uid") == ( + "* THREAD (101 102)\r\n" +) +``` + +The caller-owned `message_key` is the index identity. It must not be derived from +an IMAP sequence number, because sequence numbers change after expunge. A stable +mailbox row key, object key, or application identifier is appropriate. + +## Atomic changes and optimistic versions + +One `MailboxChangeSet` may contain additions, replacements, and removals. Its key +sets must be disjoint. Additions require absent keys; replacements and removals +require existing keys. The request is validated and recomputed on copied state; +any failure leaves the index version, records, roots, and projections unchanged. + +```python +updated = index.apply( + MailboxChangeSet( + expected_version=index.version, + replacements=( + IndexedMessage( + message_key="mailbox:102", + message=Message( + message_id="", + references="", + sequence_number=2, + uid=102, + ), + email_id="Email_102", + thread_id="Thread_7", + ), + ), + ) +) + +assert updated.previous_version == 1 +assert updated.version == 2 +``` + +`VersionConflictError` reports a stale `expected_version`. Reapplying a change +with the old version therefore fails explicitly instead of duplicating records or +edges. An empty change set is idempotent and does not advance the version. + +### Concurrent access + +All public reads and writes on one index acquire the same process-local reentrant +lock. A transaction owns that lock from optimistic-version validation through the +single state publication point. A second writer using the same version therefore +waits, observes the committed version, and raises `VersionConflictError`; a reader +waits and sees either the complete old state or the complete new state. + +```mermaid +flowchart LR + W1[Writer A: expected version n] --> L[Per-index reentrant lock] + W2[Writer B: expected version n] --> L + R[Reader: roots, projections, snapshot] --> L + L --> V{Validate current version} + V -->|Writer A| C[Compute isolated transaction] + C --> P[Publish one committed state] + P --> O[Release lock] + V -->|Writer B after A| X[Explicit version conflict] + V -->|Reader| S[Return one committed snapshot] +``` + +The lock coordinates threads inside one Python process only. A host such as naruon +must still serialize durable writes across workers or replicas and persist the +optimistic version beside its mailbox state. + +## Affected-component recomputation + +Each record contributes connectivity tokens for: + +- normalized `Message-ID`; +- the RFC 5256 effective `References` chain, or the first valid `In-Reply-To` + identifier when `References` is unavailable; +- the RFC 5051 base-subject key when `group_by_subject=True`. + +The token graph deliberately over-approximates the final thread graph. A change +may therefore recompute a component that proves unchanged, but it cannot omit a +reference or subject dependency. Replacing or removing a record seeds its whole +old component; a new bridge message includes every old component touched by its +new tokens. The candidate region is repartitioned iteratively and each resulting +component is processed by `thread_messages`. + +Unchanged components retain their existing internal `Container` roots. In the +default first-appearance ordering mode, a transaction uses overlay mappings and +copy-on-write reverse buckets for only the changed records and affected components. +It does not clone or iterate the unrelated record, position, token, component, +EMAILID, or THREADID maps before the single commit point. Sent-date ordering still +requires a global rank and effective-sequence validation because IMAP ordering is a +mailbox-wide contract. Applying a change does not eagerly rebuild the complete +public forest: only the affected old and new component views are composed for +`ThreadDelta`. The complete ordered forest +is materialized once, on demand, when `roots` or `projections` is requested. Public +roots are defensive structural copies; editing their parent, child, or message +metadata cannot corrupt internal state, while caller payload objects remain shared by +reference. Internal implicit sent-date ranks are cleared before roots are exposed, so +ordinary IMAP `THREAD` output still requires real caller-supplied sequence numbers. + +`ThreadDelta.affected_message_keys` reports the candidate region. The delta also +contains added, removed, and structurally updated projections. A metadata-only +replacement can leave a projection unchanged while its key still appears in the +affected set. + +## External EMAILID and THREADID handoff + +`email_id` and `thread_id` are optional caller-owned RFC 8474 values. ThreadWeave +validates the RFC `objectid` grammar: 1 through 255 ASCII letters, digits, +underscore, or hyphen. The values are case-sensitive. + +The incremental layer enforces these rules: + +- a reported `email_id` or `thread_id` cannot be removed or changed by a + replacement; +- equal non-null EMAILID values must expose the same THREADID value; +- the EMAILID and THREADID namespaces cannot reuse the same ObjectID value; +- ThreadWeave never selects a canonical external THREADID; +- a structural merge or split is returned as an explicit `ThreadTransition`. + +This prevents a server adapter from silently changing a THREADID that has already +been exposed. A transport-specific service can consume the transition and apply +its own documented policy. + +## Snapshot and restore + +```python +snapshot = index.snapshot() +restored = IncrementalThreadIndex.restore(snapshot) + +assert restored.version == index.version +assert restored.projections == index.projections +``` + +Schema version 1 stores: + +- batch options; +- optimistic version; +- stable record order; +- structural message metadata; +- optional EMAILID and THREADID values. + +Payload objects and derived graph pointers are never serialized. Restored +messages therefore have `payload=None`. Date values use an explicit tagged text +or ISO-8601 datetime representation. Schema versions must be exact non-boolean +integers. Restore rejects unknown versions, extra or missing fields, duplicate +keys, malformed types, invalid external IDs, hostile nesting, unencodable Unicode, +and configured record or byte limits through `IncrementalThreadError` before +publishing state. Only built-in JSON dictionaries, lists, scalar values, and plain +string object keys are accepted. Container, key, and scalar subclasses are rejected +before serialization so hostile iteration, comparison, or conversion methods cannot +execute inside the restore boundary. RFC 8259 represents JSON as nested arrays, +objects, and scalar values rather than an identity-bearing object graph. ThreadWeave +therefore rejects cyclic or reused built-in container identities with an iterative +active-path and seen-object guard. This prevents a compact Python DAG from expanding +exponentially during encoding. The configured UTF-8 byte limit is counted from +incremental encoder chunks with allocation-free code-point width accounting. The +check aborts without calling ``str.encode`` on a complete chunk or materializing a +second complete JSON string or byte array. Restore also preflights the exact +root/options schema and the record-count limit before inspecting nested values. The +plain-container walk stops when its visited-value count exceeds the configured byte +limit; every JSON value requires at least one encoded byte, so that condition proves +the snapshot is oversized before the JSON encoder is constructed. + +## Correctness and operational boundaries + +The test suite compares additions, delayed ancestors, bridge merges, component +splits, replacement, and root/internal/leaf removal with a complete batch rebuild. +It also covers RFC 5051 subject buckets, RFC 5256 sent-date ordering, ordinary and +UID THREAD output, duplicate and missing Message-ID values, deep chains, +optimistic conflicts, hostile snapshots, and payload omission. + +`benchmarks/incremental_mailbox.py` runs incremental and full-rebuild scenarios in +separate processes, requires identical projection hashes, and reports wall time, peak +RSS, affected-message count, delta retained/transient traced bytes, and full-view +materialization time. The scheduled/manual workflow defaults to 100,000 existing +messages and stores the JSON evidence for 90 +days. The update contract promises that unrelated records are not passed to the batch +threader; full-view materialization remains proportional to the number of roots and is +therefore reported separately from delta application. + +## References + +Bray, T. (Ed.). (2017). *The JavaScript Object Notation (JSON) data interchange +format* (RFC 8259). RFC Editor. https://doi.org/10.17487/RFC8259 + +Gondwana, B. (2018). *IMAP extension for object identifiers* (RFC 8474). RFC +Editor. https://doi.org/10.17487/RFC8474 + +Jenkins, N., & Newman, C. (2019). *The JSON Meta Application Protocol (JMAP) for +mail* (RFC 8621). RFC Editor. https://doi.org/10.17487/RFC8621 + +Melnikov, A., & Cridland, D. (2014). *IMAP extensions: Quick flag changes +resynchronization (CONDSTORE) and quick mailbox resynchronization (QRESYNC)* +(RFC 7162). RFC Editor. https://doi.org/10.17487/RFC7162 + +Melnikov, A., & Leiba, B. (Eds.). (2021). *Internet Message Access Protocol +(IMAP)—Version 4rev2* (RFC 9051). RFC Editor. +https://doi.org/10.17487/RFC9051 diff --git a/docs/superpowers/plans/2026-08-05-incremental-thread-index.md b/docs/superpowers/plans/2026-08-05-incremental-thread-index.md new file mode 100644 index 0000000..846adec --- /dev/null +++ b/docs/superpowers/plans/2026-08-05-incremental-thread-index.md @@ -0,0 +1,149 @@ +# Incremental Thread Index Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add an atomic, snapshot-capable incremental mailbox index that recomputes only affected reference/subject components while remaining exactly equivalent to the existing batch threader. + +**Architecture:** A new `threadweave.incremental` module owns caller keys, copied message metadata, reverse connectivity buckets, component-local batch results, projections, deltas, and snapshot validation. Existing batch/thread/container/IMAP modules remain transport-neutral and authoritative. The release-frozen `main` branch is not modified until release blocker #17 closes. + +**Tech Stack:** Python 3.10-3.13 standard library, existing `Message`, `Container`, `thread_messages`, RFC parsing/collation/date helpers, pytest, coverage, Ruff, Hatchling. + +## Global Constraints + +- Runtime dependencies remain empty. +- Production statement and branch coverage remain 100%. +- Every authored production module, class, function, method, and property has a beginner-readable docstring. +- Graph processing is iterative and identity-safe. +- Caller payloads never enter snapshots, delta equality, logs, or error messages. +- RFC 7162, RFC 8474, RFC 8621, RFC 9051, and existing RFC 5256 behavior remain traceable in APA 7th documentation. +- The PR remains draft and unmerged while issue #17 is open. + +--- + +### Task 1: Public change, projection, and error contracts + +**Files:** +- Create: `src/threadweave/incremental.py` +- Create: `tests/test_incremental_contract.py` +- Modify: `src/threadweave/__init__.py` +- Modify: `tests/test_documentation.py` + +**Interfaces:** +- Produces: `IndexedMessage`, `MailboxChangeSet`, `ThreadProjection`, `ThreadDelta`, `IncrementalThreadError`, `VersionConflictError`, `ExternalIdentityError`, `IncrementalThreadIndex`. + +- [ ] Write failing import, constructor-default, immutable-record, invalid-key, disjoint-change-set, and docstring tests. +- [ ] Run the focused tests and confirm missing symbols fail. +- [ ] Implement frozen public records, bounded key validation, exception hierarchy, and empty index properties. +- [ ] Export the new API and include the module in documentation inspection. +- [ ] Run focused tests and commit. + +### Task 2: Atomic record validation and copied metadata + +**Files:** +- Modify: `src/threadweave/incremental.py` +- Create: `tests/test_incremental_atomicity.py` + +**Interfaces:** +- Consumes: Task 1 public records. +- Produces: `IncrementalThreadIndex.apply(change_set) -> ThreadDelta` for record ownership and versioning before graph behavior. + +- [ ] Write failing tests for additions, replacement-in-place, removal, disjoint keys, missing/existing ownership errors, boolean/negative version values, and optimistic conflicts. +- [ ] Add tests proving payload identity is retained in memory while reference sequences are copied and later caller mutation cannot change indexed metadata. +- [ ] Run focused tests and confirm failures. +- [ ] Implement validation on copied state and commit only after every input passes. +- [ ] Verify failures leave version, records, roots, and projections unchanged. +- [ ] Run focused tests and commit. + +### Task 3: Connectivity indexes and bounded component recomputation + +**Files:** +- Modify: `src/threadweave/incremental.py` +- Create: `tests/test_incremental_components.py` + +**Interfaces:** +- Produces: token extraction, reverse buckets, component membership, component-local roots, affected-message reporting. + +- [ ] Write failing batch-parity tests for independent roots, linear references, shared missing roots, delayed ancestors, duplicate Message-IDs, replacement, root/internal/leaf removal, and bridge merges. +- [ ] Add a spy around the module-level batch delegate proving an unrelated component is not passed to the delegate during a one-component update. +- [ ] Run focused tests and confirm failures. +- [ ] Implement normalized ID/reference/subject tokens, copied reverse buckets, old-component seeding, touched-token expansion, iterative repartition, and component-local `thread_messages` calls. +- [ ] Compose roots in global insertion order and produce deterministic projections. +- [ ] Run focused tests and commit. + +### Task 4: Sent-date, subject, IMAP, and deep-tree parity + +**Files:** +- Modify: `src/threadweave/incremental.py` +- Create: `tests/test_incremental_parity.py` + +**Interfaces:** +- Consumes: Task 3 component state. +- Produces: exact output parity for both batch options and protocol serialization. + +- [ ] Write failing tests for RFC 5051 subject grouping, RFC 5256 sent-date ordering, explicit/implicit sequence collisions, Unicode subjects, raw headers, UID THREAD output, and one-shot source construction. +- [ ] Add deep-chain, split-tree, and malformed cycle-oriented metadata cases without recursive index traversal. +- [ ] Run focused tests and confirm failures. +- [ ] Implement global effective-sequence validation, root ordering from the first concrete node, and deterministic root composition. +- [ ] Compare projections and RFC 5256 serialization with complete batch rebuilds. +- [ ] Run focused tests and commit. + +### Task 5: RFC 8474 external identity transitions + +**Files:** +- Modify: `src/threadweave/incremental.py` +- Create: `tests/test_incremental_identity.py` + +**Interfaces:** +- Produces: immutable EMAILID/THREADID validation and explicit merge/split groups in `ThreadDelta`. + +- [ ] Write failing tests for replacement changes/removal of reported IDs, same EMAILID with inconsistent/missing THREADID, structurally merged distinct IDs, and structural splits. +- [ ] Run focused tests and confirm failures. +- [ ] Implement cross-record identity validation and deterministic overlap-based transition classification. +- [ ] Ensure no canonical thread ID is invented and sequence numbers never participate in identity. +- [ ] Run focused tests and commit. + +### Task 6: JSON-safe snapshot and restore + +**Files:** +- Modify: `src/threadweave/incremental.py` +- Create: `tests/test_incremental_snapshot.py` + +**Interfaces:** +- Produces: `snapshot()` and `IncrementalThreadIndex.restore(...)` schema version 1. + +- [ ] Write failing deterministic round-trip tests covering strings, aware/naive datetimes, references, options, external IDs, and payload omission. +- [ ] Write failing tests for unknown/missing/extra fields, duplicate keys, invalid types, noncanonical records, oversized record counts/bytes, and unsupported schema versions. +- [ ] Run focused tests and confirm failures. +- [ ] Implement tagged date encoding, strict field readers, JSON-size checks, restore through validated records, and derived-state rebuild. +- [ ] Run focused tests and commit. + +### Task 7: Buyer and research documentation, changelog, package smoke + +**Files:** +- Modify: `README.md` +- Modify: `CHANGELOG.md` +- Modify: `docs/research/README.md` +- Modify: `.github/workflows/ci.yml` +- Modify: `AGENTS.md` +- Modify: PR body + +**Interfaces:** +- Documents public behavior, complexity boundary, release freeze, and standards. + +- [ ] Add a complete incremental example with atomic changes, delta inspection, snapshot/restore, and RFC 8474 identity notes. +- [ ] Add APA 7th references and explicitly separate caller IDs from managed policy. +- [ ] Add `[Unreleased]` notes without changing version `0.2.0`. +- [ ] Extend the installed-wheel smoke test to construct and update an index. +- [ ] Add invariants to `AGENTS.md`. +- [ ] Run all repository checks and commit. + +### Task 8: Exact-head review and merge gating + +**Files:** +- No product files unless a review or check identifies a defect. + +- [ ] Run lock regeneration, Ruff, compileall, doctests, full pytest/coverage, autonomous/release-boundary coverage, build, hash-installed wheel smoke, SAST, and Security Scan. +- [ ] Confirm all production and trusted-boundary statements/branches are 100% covered. +- [ ] Review every unresolved thread and apply focused fixes. +- [ ] Keep the PR draft while issue #17 is open; do not enable auto-merge. +- [ ] After the verified `0.2.0` release closes #17, rebase/update the exact head, rerun all gates, request independent review, and merge only when policy is satisfied. diff --git a/docs/superpowers/specs/2026-08-05-incremental-thread-index-design.md b/docs/superpowers/specs/2026-08-05-incremental-thread-index-design.md new file mode 100644 index 0000000..380f2cd --- /dev/null +++ b/docs/superpowers/specs/2026-08-05-incremental-thread-index-design.md @@ -0,0 +1,200 @@ +# Incremental Thread Index Design + +## Status + +Approved product direction for issue #19. This design is implemented on a feature +branch while release blocker #17 keeps the `0.2.0` source on `main` frozen. The +feature must remain draft and must not merge until the `0.2.0` release succeeds. + +## Product problem + +`thread_messages` is a deterministic and standards-grounded batch operation, but +mail servers, migration products, archive viewers, and naruon-style control planes +receive mailbox deltas. Rebuilding an unrelated million-message mailbox after every +arrival, expunge, or metadata correction wastes CPU and memory and gives integrators +no explicit stable-identity handoff. + +## Goals + +1. Accept additions, replacements, and removals as one atomic change set. +2. Use an immutable caller-owned message key that is independent of IMAP sequence + numbers and UIDs. +3. Recompute only the old components and newly connected buckets touched by a + change while preserving exact parity with a full `thread_messages` rebuild. +4. Expose deterministic thread projections and explicit merge/split transitions. +5. Carry optional caller-owned RFC 8474 `EMAILID` and `THREADID` metadata without + silently changing a reported identifier. +6. Snapshot and restore versioned JSON-safe state without serializing payloads. +7. Preserve zero runtime dependencies, Python 3.10-3.13 support, iterative graph + processing, complete docstrings, and 100% statement and branch coverage. + +## Non-goals for this slice + +- Implementing an IMAP, JMAP, database, socket, authentication, or tenant layer. +- Inventing a server-owned stable `THREADID` policy. +- Persisting arbitrary caller payloads. +- Replacing `thread_messages`; the batch function remains the correctness oracle. +- Claiming sublinear behavior for an initial build or snapshot restore. The bounded + update path, not construction or restore, carries the incremental requirement. + +## Public API + +### `IndexedMessage` + +A frozen record containing: + +- `message_key: str`: immutable caller-owned key. +- `message: Message`: message metadata; the index copies structural fields on entry. +- `email_id: str | None`: optional caller-owned RFC 8474 immutable content ID. +- `thread_id: str | None`: optional caller-owned RFC 8474 thread correlator. + +Payload objects remain caller-owned and are retained only in memory. They never enter +snapshots or equality/delta calculations. + +### `MailboxChangeSet` + +A frozen atomic request containing: + +- `expected_version: int` +- `additions: tuple[IndexedMessage, ...]` +- `replacements: tuple[IndexedMessage, ...]` +- `removals: tuple[str, ...]` + +The three key sets must be disjoint. Additions require absent keys; replacements and +removals require existing keys. Any failure leaves the index unchanged. + +### `ThreadProjection` + +A frozen, JSON-safe description of one returned thread root: + +- `message_keys`: traversal-ordered caller keys. +- `thread_ids`: sorted distinct external thread IDs represented by the root. + +### `ThreadDelta` + +A deterministic result containing previous/new versions, affected message keys, +added/removed/updated projections, and explicit merge/split external-ID groups. +Structural overlap is computed from caller keys; mutable IMAP sequence numbers never +serve as identity. + +### `IncrementalThreadIndex` + +Constructor options mirror the batch API: + +- `group_by_subject: bool = False` +- `sort_by_sent_date: bool = False` +- `max_snapshot_records: int = 1_000_000` +- `max_snapshot_bytes: int = 256 * 1024 * 1024` + +Methods and properties: + +- `apply(change_set) -> ThreadDelta` +- `roots -> tuple[Container, ...]` +- `projections -> tuple[ThreadProjection, ...]` +- `version -> int` +- `snapshot() -> dict[str, object]` +- `restore(snapshot, *, max_snapshot_records=..., max_snapshot_bytes=...)` + +## Incremental algorithm + +The index keeps four kinds of state: + +1. Ordered records and stable insertion positions. +2. Per-message connectivity tokens. +3. Reverse token buckets. +4. Component membership and component-local `thread_messages` output. + +A record contributes tokens for its normalized `Message-ID`, every effective RFC +5256 reference identifier, and—when subject grouping is enabled—its RFC 5051 base +subject key. Token connectivity deliberately over-approximates the final root graph; +that can recompute a larger component but cannot omit a dependency. + +For one change set: + +1. Validate version, key ownership, immutable external IDs, numeric metadata, and + JSON-safe identifier bounds without mutating state. +2. Seed the affected set with every old component containing a replaced/removed key. +3. Remove old token memberships and insert new memberships on copied indexes. +4. Add every component touching an old or new token, including bridge additions. +5. Expand only that candidate region through current token buckets. +6. Repartition the candidate region iteratively and invoke `thread_messages` for + each new component in global insertion order. +7. Reuse untouched component roots and compose the global root list. Default output + uses first insertion position; RFC sent-date mode uses the same UTC/date, + sequence-number, and input-position ordering contract as the batch API. +8. Compare before/after projections and commit copied state only after all steps + succeed. + +Removing an internal or root message seeds its entire old component, so a split is +fully rediscovered. Adding a bridge message includes every touched old component, so +merges are explicit. Unrelated components are neither passed to `thread_messages` +nor traversed during the update. + +## External identity rules + +- Message keys never change; replacement uses the same key. +- A non-null `email_id` or `thread_id` cannot be removed or changed by replacement. +- All records sharing one non-null `email_id` must expose the same non-null + `thread_id`, matching RFC 8474. +- Structurally merged roots retain every caller thread ID and emit a merge group. +- Structural splits emit a split group rather than silently rewriting IDs. +- The core does not choose a canonical external ID. A server-specific policy may + consume the explicit transition later. + +## Snapshot contract + +Snapshot schema version `1` contains options, optimistic version, ordered records, +message metadata, and optional external IDs. It excludes payloads and derived graph +state. References are stored as normalized identifier lists. Datetimes are encoded +as tagged ISO 8601 strings; textual date inputs remain textual. + +Restore rejects unknown schema versions, duplicate or malformed keys, unexpected +fields, non-finite/boolean numeric metadata, oversized records/documents, and invalid +external-ID invariants. It rebuilds derived indexes through the same public +validation path and compares no untrusted serialized graph pointers. + +## Error model + +- `IncrementalThreadError`: invalid change or snapshot. +- `VersionConflictError`: optimistic version mismatch. +- `ExternalIdentityError`: immutable/cross-record EMAILID or THREADID violation. + +Messages are bounded and do not include payload representations, local paths, or +secrets. + +## Verification + +Required tests include: + +- additions, replacement, removal, delayed ancestor, bridge merge, and component + split against a complete batch rebuild; +- duplicate/missing Message-ID, raw headers, Unicode subjects, date sorting, UID + output, and RFC 5256 serialization parity; +- explicit merge/split external-ID transitions and RFC 8474 invariants; +- atomic rollback and optimistic version conflict; +- deterministic snapshot round trip, payload omission, malformed/oversized input, + and idempotency through version conflicts; +- a spy proving an unrelated component is not passed to `thread_messages` during a + bounded update; +- deep and hostile metadata cases without recursion; +- installed-wheel public API smoke coverage. + +A scheduled/manual benchmark follows in a separate bounded PR with at least 100,000 +messages, bounded deltas, wall time, peak RSS, affected-message count, and full +rebuild comparison. + +## References + +Gondwana, B. (2018). *IMAP extension for object identifiers* (RFC 8474). RFC +Editor. https://doi.org/10.17487/RFC8474 + +Jenkins, N., & Newman, C. (2019). *The JSON Meta Application Protocol (JMAP) for +mail* (RFC 8621). RFC Editor. https://doi.org/10.17487/RFC8621 + +Melnikov, A., & Cridland, D. (2014). *IMAP extensions: Quick flag changes +resynchronization (CONDSTORE) and quick mailbox resynchronization (QRESYNC)* +(RFC 7162). RFC Editor. https://doi.org/10.17487/RFC7162 + +Melnikov, A., & Leiba, B. (Eds.). (2021). *Internet Message Access Protocol +(IMAP)—Version 4rev2* (RFC 9051). RFC Editor. +https://doi.org/10.17487/RFC9051 diff --git a/src/threadweave/__init__.py b/src/threadweave/__init__.py index 24500b0..e6082a4 100644 --- a/src/threadweave/__init__.py +++ b/src/threadweave/__init__.py @@ -3,8 +3,8 @@ Assemble flat iterables of email messages into conversation trees using Jamie Zawinski's container algorithm and RFC 5256 REFERENCES semantics, on top of RFC 5322 identification-field parsing, RFC 5051 Unicode casemap comparison, exact -RFC 5256 base-subject extraction, optional sent-date ordering, and IMAP THREAD -response serialization. +RFC 5256 base-subject extraction, optional sent-date ordering, IMAP THREAD +response serialization, and atomic incremental mailbox updates. from threadweave import Message, thread_messages @@ -16,9 +16,9 @@ # -> one root Container; a is an ancestor of b, b of c. The RFC 5322 header primitives (:mod:`threadweave.headers`) are extracted -behaviour-preserving from the naruon control plane; the threading, subject, -collation, date, and protocol-projection layers are standalone implementations -grounded in published standards. +behaviour-preserving from the naruon control plane; the threading, incremental, +subject, collation, date, and protocol-projection layers are standalone +implementations grounded in published standards. """ from threadweave.adapters import message_from_email, thread_email_messages @@ -31,6 +31,17 @@ generate_email_fingerprint, normalize_message_id, ) +from threadweave.incremental import ( + ExternalIdentityError, + IncrementalThreadError, + IncrementalThreadIndex, + IndexedMessage, + MailboxChangeSet, + ThreadDelta, + ThreadProjection, + ThreadTransition, + VersionConflictError, +) from threadweave.imap import ( IdentifierResolver, MessageFilter, @@ -52,10 +63,19 @@ __all__ = [ "Container", "DateValue", + "ExternalIdentityError", "IdentifierResolver", + "IncrementalThreadError", + "IncrementalThreadIndex", + "IndexedMessage", + "MailboxChangeSet", "Message", "MessageFilter", + "ThreadDelta", + "ThreadProjection", "ThreadSerializationError", + "ThreadTransition", + "VersionConflictError", "decode_header_text", "extract_reference_ids", "generate_email_fingerprint", diff --git a/src/threadweave/incremental.py b/src/threadweave/incremental.py new file mode 100644 index 0000000..b52fb88 --- /dev/null +++ b/src/threadweave/incremental.py @@ -0,0 +1,1646 @@ +"""Incremental, identity-aware mailbox threading over the batch RFC engine. + +The batch :func:`threadweave.thread_messages` function remains the correctness +oracle. This module adds an atomic state boundary that indexes caller-owned +message keys, recomputes only affected connectivity components, reports explicit +thread merge/split transitions, and snapshots JSON-safe metadata without caller +payloads. +""" + +from __future__ import annotations + +import json +from collections.abc import Iterable, Iterator, Mapping, Sequence +from dataclasses import dataclass +from datetime import datetime +from _thread import RLock +from typing import Literal, TypeVar + +from threadweave.collation import unicode_casemap_key +from threadweave.container import Container +from threadweave.headers import extract_reference_ids, normalize_message_id +from threadweave.subject import normalize_subject +from threadweave.threading import Message, thread_messages + +__all__ = [ + "ExternalIdentityError", + "IncrementalThreadError", + "IncrementalThreadIndex", + "IndexedMessage", + "MailboxChangeSet", + "ThreadDelta", + "ThreadProjection", + "ThreadTransition", + "VersionConflictError", +] + +_batch_thread_messages = thread_messages +_MAX_MESSAGE_KEY_LENGTH = 512 +_MAX_EXTERNAL_ID_LENGTH = 255 +_OBJECT_ID_CHARACTERS = frozenset( + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_-" +) +_MAX_IMAP_NUMBER = 4_294_967_295 +_DEFAULT_MAX_SNAPSHOT_RECORDS = 1_000_000 +_DEFAULT_MAX_SNAPSHOT_BYTES = 256 * 1024 * 1024 +_SNAPSHOT_SCHEMA_VERSION = 1 + +_KeyT = TypeVar("_KeyT") +_ValueT = TypeVar("_ValueT") + + +class _OverlayMapping(Mapping[_KeyT, _ValueT]): + """Expose staged updates over a base mapping without copying unrelated entries.""" + + def __init__( + self, + base: Mapping[_KeyT, _ValueT], + updates: Mapping[_KeyT, _ValueT], + removals: frozenset[_KeyT] = frozenset(), + ) -> None: + """Retain immutable transaction views of one base mapping and its delta.""" + self._base = base + self._updates = updates + self._removals = removals + + def __getitem__(self, key: _KeyT) -> _ValueT: + """Return a staged value, excluding keys removed by the transaction.""" + if key in self._updates: + return self._updates[key] + if key in self._removals: + raise KeyError(key) + return self._base[key] + + def __iter__(self) -> Iterator[_KeyT]: + """Iterate the logical mapping only when a global operation requires it.""" + for key in self._base: + if key not in self._removals and key not in self._updates: + yield key + yield from self._updates + + def __len__(self) -> int: + """Return logical size from the bounded transaction delta.""" + removed = sum( + 1 + for key in self._removals + if key in self._base and key not in self._updates + ) + added = sum(1 for key in self._updates if key not in self._base) + return len(self._base) - removed + added + + def __contains__(self, key: object) -> bool: + """Test logical membership without iterating the base mapping.""" + if key in self._updates: + return True + if key in self._removals: + return False + return key in self._base + + +class IncrementalThreadError(ValueError): + """Raised when an incremental change or snapshot violates its contract.""" + + +class VersionConflictError(IncrementalThreadError): + """Raised when an optimistic change targets a stale index version.""" + + +class ExternalIdentityError(IncrementalThreadError): + """Raised when caller-owned EMAILID or THREADID metadata is inconsistent.""" + + +def _validated_positive_limit(value: object, name: str) -> int: + """Return a positive non-boolean integer denial-of-service limit.""" + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + raise IncrementalThreadError(f"{name} must be a positive integer") + return value + + +def _validated_nonnegative_integer(value: object, name: str) -> int: + """Return a non-negative non-boolean integer revision value.""" + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise IncrementalThreadError(f"{name} must be a non-negative integer") + return value + + +def _validated_identifier( + value: object, + name: str, + *, + allow_none: bool, + maximum_length: int, +) -> str | None: + """Validate one bounded printable caller-owned identifier.""" + if value is None and allow_none: + return None + if not isinstance(value, str) or not value or len(value) > maximum_length: + raise IncrementalThreadError( + f"{name} must be a non-empty string of at most {maximum_length} characters" + ) + if any(not character.isprintable() for character in value): + raise IncrementalThreadError(f"{name} must contain only printable characters") + return value + + +def _validated_message_key(value: object) -> str: + """Return one safe immutable caller message key.""" + validated = _validated_identifier( + value, + "message_key", + allow_none=False, + maximum_length=_MAX_MESSAGE_KEY_LENGTH, + ) + assert validated is not None + return validated + + +def _validated_external_id(value: object, name: str) -> str | None: + """Return one optional RFC 8474 ``objectid`` value.""" + validated = _validated_identifier( + value, + name, + allow_none=True, + maximum_length=_MAX_EXTERNAL_ID_LENGTH, + ) + if validated is not None and any( + character not in _OBJECT_ID_CHARACTERS for character in validated + ): + raise IncrementalThreadError( + f"{name} must use only ASCII letters, digits, underscore, or hyphen" + ) + return validated + + +def _tuple_without_duplicate_keys( + values: Iterable[object], + name: str, + *, + indexed: bool, +) -> tuple[object, ...]: + """Materialize one request sequence and reject duplicate caller keys.""" + materialized = tuple(values) + keys: list[str] = [] + for value in materialized: + if indexed: + if not isinstance(value, IndexedMessage): + raise IncrementalThreadError(f"{name} must contain IndexedMessage values") + key = value.message_key + else: + key = _validated_message_key(value) + keys.append(key) + if len(set(keys)) != len(keys): + raise IncrementalThreadError(f"duplicate message keys in {name}") + return materialized + + +@dataclass(frozen=True, slots=True) +class IndexedMessage: + """One message plus immutable caller and optional external identities. + + Args: + message_key: Caller-owned key that is stable across mailbox revisions. + message: Structural email metadata and an arbitrary in-memory payload. + email_id: Optional RFC 8474 immutable message-content identifier. + thread_id: Optional RFC 8474 caller/server thread correlator. + """ + + message_key: str + message: Message + email_id: str | None = None + thread_id: str | None = None + + def __post_init__(self) -> None: + """Validate public identity values without copying message metadata yet.""" + object.__setattr__(self, "message_key", _validated_message_key(self.message_key)) + if not isinstance(self.message, Message): + raise IncrementalThreadError("message must be a threadweave.Message") + object.__setattr__( + self, + "email_id", + _validated_external_id(self.email_id, "email_id"), + ) + object.__setattr__( + self, + "thread_id", + _validated_external_id(self.thread_id, "thread_id"), + ) + + +@dataclass(frozen=True, slots=True) +class MailboxChangeSet: + """One optimistic and atomic mailbox mutation request.""" + + expected_version: int + additions: tuple[IndexedMessage, ...] = () + replacements: tuple[IndexedMessage, ...] = () + removals: tuple[str, ...] = () + + def __post_init__(self) -> None: + """Normalize request sequences and require disjoint unique key sets.""" + object.__setattr__( + self, + "expected_version", + _validated_nonnegative_integer(self.expected_version, "expected_version"), + ) + additions = _tuple_without_duplicate_keys( + self.additions, + "additions", + indexed=True, + ) + replacements = _tuple_without_duplicate_keys( + self.replacements, + "replacements", + indexed=True, + ) + removals = _tuple_without_duplicate_keys( + self.removals, + "removals", + indexed=False, + ) + addition_keys = {item.message_key for item in additions} + replacement_keys = {item.message_key for item in replacements} + removal_keys = set(removals) + if ( + addition_keys & replacement_keys + or addition_keys & removal_keys + or replacement_keys & removal_keys + ): + raise IncrementalThreadError( + "addition, replacement, and removal message keys must be disjoint" + ) + object.__setattr__(self, "additions", additions) + object.__setattr__(self, "replacements", replacements) + object.__setattr__(self, "removals", removals) + + +@dataclass(frozen=True, slots=True) +class ThreadProjection: + """A deterministic caller-key view of one returned thread root.""" + + message_keys: tuple[str, ...] + thread_ids: tuple[str, ...] = () + + +@dataclass(frozen=True, slots=True) +class ThreadTransition: + """One explicit structural merge or split across caller thread IDs.""" + + kind: Literal["merge", "split"] + before: tuple[ThreadProjection, ...] + after: tuple[ThreadProjection, ...] + thread_ids: tuple[str, ...] + + +@dataclass(frozen=True, slots=True) +class ThreadDelta: + """Deterministic differences produced by one successful atomic change.""" + + previous_version: int + version: int + affected_message_keys: tuple[str, ...] + added_threads: tuple[ThreadProjection, ...] + removed_threads: tuple[ThreadProjection, ...] + updated_threads: tuple[ThreadProjection, ...] + merges: tuple[ThreadTransition, ...] + splits: tuple[ThreadTransition, ...] + + +def _validated_optional_number( + value: object, + name: str, + *, + maximum: int | None = None, +) -> int | None: + """Validate optional positive integer mailbox metadata.""" + if value is None: + return None + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + raise IncrementalThreadError(f"{name} must be a positive integer or None") + if maximum is not None and value > maximum: + raise IncrementalThreadError(f"{name} exceeds the unsigned 32-bit range") + return value + + +def _validated_optional_text(value: object, name: str) -> str | None: + """Validate an optional textual email field.""" + if value is None or isinstance(value, str): + return value + raise IncrementalThreadError(f"{name} must be a string or None") + + +def _copied_reference_value( + value: str | Sequence[str] | None, + name: str, +) -> str | tuple[str, ...] | None: + """Copy one raw or already-split identification header value.""" + if value is None or isinstance(value, str): + return value + try: + copied = tuple(value) + except TypeError as error: + raise IncrementalThreadError( + f"{name} must be a string, sequence of strings, or None" + ) from error + if not all(isinstance(item, str) for item in copied): + raise IncrementalThreadError(f"{name} must contain only strings") + return copied + + +def _copied_date_value(value: object, name: str) -> str | datetime | None: + """Copy one supported date value after validating its runtime type.""" + if value is None or isinstance(value, (str, datetime)): + return value + raise IncrementalThreadError(f"{name} must be a datetime, string, or None") + + +def _copied_indexed_message(record: IndexedMessage) -> IndexedMessage: + """Copy structural metadata while retaining the caller payload by reference.""" + message = record.message + copied = Message( + message_id=_validated_optional_text(message.message_id, "message_id"), + in_reply_to=_copied_reference_value(message.in_reply_to, "in_reply_to"), + references=_copied_reference_value(message.references, "references") or (), + subject=_validated_optional_text(message.subject, "subject"), + payload=message.payload, + sent_date=_copied_date_value(message.sent_date, "sent_date"), + internal_date=_copied_date_value(message.internal_date, "internal_date"), + sequence_number=_validated_optional_number( + message.sequence_number, + "sequence_number", + ), + uid=_validated_optional_number(message.uid, "uid", maximum=_MAX_IMAP_NUMBER), + ) + return IndexedMessage( + message_key=record.message_key, + message=copied, + email_id=record.email_id, + thread_id=record.thread_id, + ) + + +def _reference_ids(value: str | Sequence[str] | None) -> tuple[str, ...]: + """Parse and de-duplicate one raw or split identification field.""" + if value is None: + return () + values = (value,) if isinstance(value, str) else value + result: list[str] = [] + seen: set[str] = set() + for raw_value in values: + for identifier in extract_reference_ids(raw_value): + if identifier not in seen: + seen.add(identifier) + result.append(identifier) + return tuple(result) + + +def _effective_reference_ids(message: Message) -> tuple[str, ...]: + """Return the RFC 5256 ancestry identifiers used by the batch algorithm.""" + references = _reference_ids(message.references) + if references: + return references + return _reference_ids(message.in_reply_to)[:1] + + +def _connectivity_tokens( + record: IndexedMessage, + *, + group_by_subject: bool, +) -> frozenset[str]: + """Return an over-approximating connectivity token set for one record.""" + tokens: set[str] = set() + message_id = normalize_message_id(record.message.message_id) + if message_id is not None: + tokens.add(f"id\x00{message_id}") + for reference_id in _effective_reference_ids(record.message): + tokens.add(f"id\x00{reference_id}") + if group_by_subject: + subject_key = unicode_casemap_key(normalize_subject(record.message.subject)) + if subject_key: + tokens.add(f"subject\x00{subject_key}") + return frozenset(tokens) + + +def _writable_bucket( + bucket_key: str, + base_buckets: Mapping[str, set[str]], + bucket_updates: dict[str, set[str]], +) -> set[str]: + """Return one transaction-owned copy of a reverse-index bucket.""" + if bucket_key not in bucket_updates: + bucket_updates[bucket_key] = set(base_buckets.get(bucket_key, set())) + return bucket_updates[bucket_key] + + +def _remove_key_from_buckets( + key: str, + bucket_keys: Iterable[str], + base_buckets: Mapping[str, set[str]], + bucket_updates: dict[str, set[str]], +) -> None: + """Stage removal of one record key from selected reverse-index buckets.""" + for bucket_key in bucket_keys: + _writable_bucket(bucket_key, base_buckets, bucket_updates).discard(key) + + +def _add_key_to_buckets( + key: str, + bucket_keys: Iterable[str], + base_buckets: Mapping[str, set[str]], + bucket_updates: dict[str, set[str]], +) -> None: + """Stage insertion of one record key into selected reverse-index buckets.""" + for bucket_key in bucket_keys: + _writable_bucket(bucket_key, base_buckets, bucket_updates).add(key) + + +def _commit_bucket_updates( + target: dict[str, set[str]], + bucket_updates: Mapping[str, set[str]], +) -> None: + """Publish touched reverse buckets and discard buckets that became empty.""" + for bucket_key, values in bucket_updates.items(): + if values: + target[bucket_key] = values + else: + target.pop(bucket_key, None) + + +def _email_state_after( + email_id: str, + base_states: Mapping[str, tuple[str | None, int]], + state_updates: Mapping[str, tuple[str | None, int] | None], +) -> tuple[str | None, int] | None: + """Return one staged EMAILID association and reference count.""" + if email_id in state_updates: + return state_updates[email_id] + return base_states.get(email_id) + + +def _thread_count_after( + thread_id: str, + base_counts: Mapping[str, int], + count_updates: Mapping[str, int], +) -> int: + """Return one staged THREADID reference count.""" + return count_updates.get(thread_id, base_counts.get(thread_id, 0)) + + +def _stage_external_identity( + record: IndexedMessage, + adjustment: Literal[-1, 1], + base_email_states: Mapping[str, tuple[str | None, int]], + email_state_updates: dict[str, tuple[str | None, int] | None], + base_thread_counts: Mapping[str, int], + thread_count_updates: dict[str, int], + touched_values: set[str], +) -> None: + """Stage one record's RFC 8474 identity contribution without a global scan.""" + email_id = record.email_id + if email_id is not None: + touched_values.add(email_id) + current_state = _email_state_after( + email_id, + base_email_states, + email_state_updates, + ) + if adjustment < 0: + if current_state is None or current_state[0] != record.thread_id: + raise IncrementalThreadError("internal EMAILID index is inconsistent") + next_count = current_state[1] - 1 + email_state_updates[email_id] = ( + None + if next_count == 0 + else (current_state[0], next_count) + ) + else: + if current_state is not None and current_state[0] != record.thread_id: + raise ExternalIdentityError( + f"messages with EMAILID {email_id!r} must expose the same THREADID" + ) + email_state_updates[email_id] = ( + record.thread_id, + 1 if current_state is None else current_state[1] + 1, + ) + + thread_id = record.thread_id + if thread_id is not None: + touched_values.add(thread_id) + next_count = ( + _thread_count_after( + thread_id, + base_thread_counts, + thread_count_updates, + ) + + adjustment + ) + if next_count < 0: + raise IncrementalThreadError("internal THREADID index is inconsistent") + thread_count_updates[thread_id] = next_count + + +def _validate_touched_identity_namespaces( + touched_values: Iterable[str], + base_email_states: Mapping[str, tuple[str | None, int]], + email_state_updates: Mapping[str, tuple[str | None, int] | None], + base_thread_counts: Mapping[str, int], + thread_count_updates: Mapping[str, int], +) -> None: + """Reject touched ObjectID values present in both RFC 8474 namespaces.""" + reused_values = sorted( + identity_value + for identity_value in touched_values + if _email_state_after( + identity_value, + base_email_states, + email_state_updates, + ) + is not None + and _thread_count_after( + identity_value, + base_thread_counts, + thread_count_updates, + ) + > 0 + ) + if reused_values: + raise ExternalIdentityError( + "EMAILID and THREADID must use disjoint ObjectID values: " + f"{reused_values!r}" + ) + + +def _commit_external_identity_updates( + email_states: dict[str, tuple[str | None, int]], + email_state_updates: Mapping[str, tuple[str | None, int] | None], + thread_counts: dict[str, int], + thread_count_updates: Mapping[str, int], +) -> None: + """Publish touched compact RFC 8474 indexes after transaction validation.""" + for email_id, state in email_state_updates.items(): + if state is None: + email_states.pop(email_id, None) + else: + email_states[email_id] = state + for thread_id, count in thread_count_updates.items(): + if count == 0: + thread_counts.pop(thread_id, None) + else: + thread_counts[thread_id] = count + + +def _ordered_keys(keys: Iterable[str], positions: Mapping[str, int]) -> tuple[str, ...]: + """Return keys in stable insertion order with a lexical safety tie-breaker.""" + return tuple(sorted(keys, key=lambda key: (positions[key], key))) + + +def _current_ranks(positions: Mapping[str, int]) -> dict[str, int]: + """Return compact one-based input positions for the current record order.""" + return { + key: rank + for rank, key in enumerate(_ordered_keys(positions, positions), start=1) + } + + +def _validate_effective_sequence_numbers( + records: Mapping[str, IndexedMessage], + ranks: Mapping[str, int], +) -> None: + """Reject global explicit/implicit sequence collisions before recomputation.""" + used: dict[int, str] = {} + for key in _ordered_keys(records, ranks): + explicit = records[key].message.sequence_number + sequence_number = ranks[key] if explicit is None else explicit + previous = used.get(sequence_number) + if previous is not None: + raise IncrementalThreadError( + f"duplicate sequence number: {sequence_number} ({previous}, {key})" + ) + used[sequence_number] = key + + +def _validate_replacement_identity( + old_record: IndexedMessage, + new_record: IndexedMessage, +) -> None: + """Prevent removal or change of an already reported external identifier.""" + if old_record.email_id is not None and new_record.email_id != old_record.email_id: + raise ExternalIdentityError("reported email_id is immutable on replacement") + if old_record.thread_id is not None and new_record.thread_id != old_record.thread_id: + raise ExternalIdentityError("reported thread_id is immutable on replacement") + + +def _expand_candidate_keys( + seeds: set[str], + tokens_by_key: Mapping[str, frozenset[str]], + keys_by_token: Mapping[str, set[str]], +) -> set[str]: + """Expand candidate keys through current token connectivity iteratively.""" + expanded = {key for key in seeds if key in tokens_by_key} + queue = list(expanded) + while queue: + key = queue.pop() + for token in tokens_by_key.get(key, frozenset()): + for neighbor in keys_by_token.get(token, set()): + if neighbor not in expanded: + expanded.add(neighbor) + queue.append(neighbor) + return expanded + + +def _partition_components( + keys: set[str], + positions: Mapping[str, int], + tokens_by_key: Mapping[str, frozenset[str]], + keys_by_token: Mapping[str, set[str]], +) -> tuple[tuple[str, ...], ...]: + """Partition candidate keys into deterministic current connectivity components.""" + remaining = set(keys) + components: list[tuple[str, ...]] = [] + for seed in _ordered_keys(keys, positions): + if seed not in remaining: + continue + component = {seed} + queue = [seed] + remaining.remove(seed) + while queue: + key = queue.pop() + for token in tokens_by_key.get(key, frozenset()): + for neighbor in keys_by_token.get(token, set()): + if neighbor in remaining: + remaining.remove(neighbor) + component.add(neighbor) + queue.append(neighbor) + components.append(_ordered_keys(component, positions)) + return tuple(components) + + +def _message_for_batch(message: Message, sequence_number: int | None) -> Message: + """Return a computation copy with an explicit global sequence when required.""" + if sequence_number is None or message.sequence_number is not None: + return message + return Message( + message_id=message.message_id, + in_reply_to=message.in_reply_to, + references=message.references, + subject=message.subject, + payload=message.payload, + sent_date=message.sent_date, + internal_date=message.internal_date, + sequence_number=sequence_number, + uid=message.uid, + ) + + +def _public_message_copy(message: Message) -> Message: + """Copy structural message metadata while retaining the caller payload.""" + return Message( + message_id=message.message_id, + in_reply_to=message.in_reply_to, + references=message.references, + subject=message.subject, + payload=message.payload, + sent_date=message.sent_date, + internal_date=message.internal_date, + sequence_number=message.sequence_number, + uid=message.uid, + ) + + +def _public_forest_copy(roots: Iterable[Container]) -> tuple[Container, ...]: + """Return a loop-safe defensive copy of the index's internal forest.""" + copied_roots: list[Container] = [] + seen: set[int] = set() + for root in roots: + root_identity = id(root) + if root_identity in seen: + raise IncrementalThreadError( + "internal thread forest contains a shared or cyclic container" + ) + root_copy = Container( + message=None + if root.message is None + else _public_message_copy(root.message) + ) + copied_roots.append(root_copy) + seen.add(root_identity) + stack: list[tuple[Container, Container]] = [(root, root_copy)] + while stack: + source, target = stack.pop() + for child in source.children: + child_identity = id(child) + if child_identity in seen: + raise IncrementalThreadError( + "internal thread forest contains a shared or cyclic container" + ) + child_copy = Container( + message=None + if child.message is None + else _public_message_copy(child.message), + parent=target, + ) + target.children.append(child_copy) + seen.add(child_identity) + stack.append((child, child_copy)) + return tuple(copied_roots) + + +def _projection_for_root( + root: Container, + key_by_message_identity: Mapping[int, str], + records: Mapping[str, IndexedMessage], +) -> ThreadProjection: + """Project one root into traversal-ordered caller keys and external IDs.""" + message_keys: list[str] = [] + seen: set[int] = set() + stack = [root] + while stack: + node = stack.pop() + if id(node) in seen: + continue + seen.add(id(node)) + if node.message is not None: + key = key_by_message_identity.get(id(node.message)) + if key is None: + raise IncrementalThreadError( + "batch output contained a message outside its component" + ) + message_keys.append(key) + stack.extend(reversed(node.children)) + thread_ids = tuple( + sorted( + { + records[key].thread_id + for key in message_keys + if records[key].thread_id is not None + } + ) + ) + return ThreadProjection(tuple(message_keys), thread_ids) + + +def _build_forest( + keys: tuple[str, ...], + records: Mapping[str, IndexedMessage], + ranks: Mapping[str, int], + *, + group_by_subject: bool, + sort_by_sent_date: bool, +) -> tuple[tuple[Container, ...], tuple[ThreadProjection, ...]]: + """Run the canonical batch engine for one ordered record subset.""" + messages: list[Message] = [] + key_by_message_identity: dict[int, str] = {} + for key in keys: + message = _message_for_batch( + records[key].message, + ranks[key] if sort_by_sent_date else None, + ) + messages.append(message) + key_by_message_identity[id(message)] = key + roots = tuple( + _batch_thread_messages( + messages, + group_by_subject=group_by_subject, + sort_by_sent_date=sort_by_sent_date, + ) + ) + projections = tuple( + _projection_for_root(root, key_by_message_identity, records) for root in roots + ) + for key, message in zip(keys, messages): + if records[key].message.sequence_number is None: + message.sequence_number = None + return roots, projections + + +def _transition_thread_ids( + before: Iterable[ThreadProjection], + after: Iterable[ThreadProjection], +) -> tuple[str, ...]: + """Return every distinct caller THREADID represented by a transition.""" + return tuple( + sorted( + { + thread_id + for projection in (*tuple(before), *tuple(after)) + for thread_id in projection.thread_ids + } + ) + ) + + +def _projection_membership( + projections: Sequence[ThreadProjection], + name: str, +) -> dict[str, int]: + """Index each caller key once and reject overlapping root projections.""" + membership: dict[str, int] = {} + for projection_index, projection in enumerate(projections): + for message_key in projection.message_keys: + if message_key in membership: + raise IncrementalThreadError( + f"{name} projections contain duplicate message_key: {message_key}" + ) + membership[message_key] = projection_index + return membership + + +def _thread_delta( + previous_version: int, + version: int, + affected_message_keys: tuple[str, ...], + before: Sequence[ThreadProjection], + after: Sequence[ThreadProjection], +) -> ThreadDelta: + """Classify projection changes and transitions in linear message-key work.""" + before_tuple = tuple(before) + after_tuple = tuple(after) + before_membership = _projection_membership(before_tuple, "before") + after_membership = _projection_membership(after_tuple, "after") + + before_by_after: list[set[int]] = [set() for _ in after_tuple] + after_by_before: list[set[int]] = [set() for _ in before_tuple] + for message_key, after_index in after_membership.items(): + before_index = before_membership.get(message_key) + if before_index is not None: + before_by_after[after_index].add(before_index) + after_by_before[before_index].add(after_index) + + before_values = set(before_tuple) + added = tuple( + projection + for projection, overlaps in zip(after_tuple, before_by_after) + if not overlaps + ) + removed = tuple( + projection + for projection, overlaps in zip(before_tuple, after_by_before) + if not overlaps + ) + updated = tuple( + projection + for projection, overlaps in zip(after_tuple, before_by_after) + if overlaps and projection not in before_values + ) + merges = tuple( + ThreadTransition( + "merge", + tuple(before_tuple[index] for index in sorted(overlaps)), + (projection,), + _transition_thread_ids( + tuple(before_tuple[index] for index in sorted(overlaps)), + (projection,), + ), + ) + for projection, overlaps in zip(after_tuple, before_by_after) + if len(overlaps) > 1 + ) + splits = tuple( + ThreadTransition( + "split", + (projection,), + tuple(after_tuple[index] for index in sorted(overlaps)), + _transition_thread_ids( + (projection,), + tuple(after_tuple[index] for index in sorted(overlaps)), + ), + ) + for projection, overlaps in zip(before_tuple, after_by_before) + if len(overlaps) > 1 + ) + return ThreadDelta( + previous_version, + version, + affected_message_keys, + added, + removed, + updated, + merges, + splits, + ) + + +def _encoded_date(value: str | datetime | None) -> object: + """Encode one date value into a deterministic JSON-safe tagged form.""" + if value is None: + return None + if isinstance(value, datetime): + return {"kind": "datetime", "value": value.isoformat()} + return {"kind": "text", "value": value} + + +def _decoded_date(value: object, name: str) -> str | datetime | None: + """Decode one strict tagged date value from an untrusted snapshot.""" + if value is None: + return None + if not isinstance(value, Mapping) or set(value) != {"kind", "value"}: + raise IncrementalThreadError(f"{name} date value is malformed") + kind = value["kind"] + encoded = value["value"] + if not isinstance(encoded, str): + raise IncrementalThreadError(f"{name} date value must be textual") + if kind == "text": + return encoded + if kind == "datetime": + try: + return datetime.fromisoformat(encoded) + except ValueError as error: + raise IncrementalThreadError(f"{name} datetime is invalid") from error + raise IncrementalThreadError(f"{name} date kind is unsupported") + + +def _require_plain_json_containers( + value: object, + *, + maximum_nodes: int | None = None, +) -> None: + """Reject executable, cyclic, aliased, or structurally oversized JSON trees. + + JSON decoding produces a tree of built-in dictionaries, lists, string + keys, and scalar values. Requiring exact runtime types prevents attacker- + controlled iteration, comparison, or scalar subclasses from executing, + while rejecting repeated container identities prevents a compact Python + object graph from expanding exponentially during JSON encoding. A node + ceiling derived from the byte limit bounds validation before serialization. + """ + pending = [(value, False)] + active_containers: set[int] = set() + seen_containers: set[int] = set() + visited_nodes = 0 + while pending: + current, exiting = pending.pop() + if not exiting: + visited_nodes += 1 + if maximum_nodes is not None and visited_nodes > maximum_nodes: + raise IncrementalThreadError("snapshot exceeds max_snapshot_bytes") + if type(current) in {dict, list}: + identity = id(current) + if exiting: + active_containers.remove(identity) + continue + if identity in active_containers: + raise IncrementalThreadError( + "snapshot must not contain cyclic JSON containers" + ) + if identity in seen_containers: + raise IncrementalThreadError( + "snapshot must not contain reused JSON container objects" + ) + if type(current) is dict and any( + type(key) is not str for key in dict.keys(current) + ): + raise IncrementalThreadError( + "snapshot object keys must be plain strings" + ) + seen_containers.add(identity) + active_containers.add(identity) + pending.append((current, True)) + children = dict.values(current) if type(current) is dict else current + pending.extend((child, False) for child in children) + elif current is None or type(current) in {str, int, float, bool}: + continue + else: + raise IncrementalThreadError( + "snapshot must contain only plain JSON containers and scalar values" + ) + + +def _bounded_utf8_size(value: str, maximum_bytes: int) -> int: + """Count UTF-8 bytes up to a limit without allocating an encoded copy.""" + if str.isascii(value): + return str.__len__(value) + + encoded_bytes = 0 + for index in range(str.__len__(value)): + code_point = ord(str.__getitem__(value, index)) + if code_point <= 0x7F: + width = 1 + elif code_point <= 0x7FF: + width = 2 + elif 0xD800 <= code_point <= 0xDFFF: + raise UnicodeEncodeError( + "utf-8", + value, + index, + index + 1, + "surrogates not allowed", + ) + elif code_point <= 0xFFFF: + width = 3 + else: + width = 4 + encoded_bytes += width + if encoded_bytes > maximum_bytes: + break + return encoded_bytes + + +def _bounded_snapshot_json_size(value: object, maximum_bytes: int) -> int: + """Return canonical UTF-8 size while stopping at the configured byte limit.""" + encoder = json.JSONEncoder( + ensure_ascii=False, + allow_nan=False, + sort_keys=True, + separators=(",", ":"), + ) + encoded_bytes = 0 + try: + for chunk in encoder.iterencode(value): + encoded_bytes += _bounded_utf8_size( + chunk, + maximum_bytes - encoded_bytes, + ) + if encoded_bytes > maximum_bytes: + raise IncrementalThreadError("snapshot exceeds max_snapshot_bytes") + except IncrementalThreadError: + raise + except (RecursionError, TypeError, UnicodeError, ValueError) as error: + raise IncrementalThreadError( + "snapshot must contain only JSON-safe values" + ) from error + return encoded_bytes + + +def _required_plain_object( + value: object, + expected: set[str], + name: str, +) -> dict[str, object]: + """Return one exact built-in JSON object with the required plain-string keys.""" + if not isinstance(value, Mapping): + raise IncrementalThreadError(f"{name} must be a mapping") + if type(value) is not dict: + raise IncrementalThreadError( + "snapshot must contain only plain JSON containers and scalar values" + ) + if dict.__len__(value) != len(expected): + raise IncrementalThreadError(f"{name} fields do not match the schema") + keys = dict.keys(value) + if any(type(key) is not str for key in keys): + raise IncrementalThreadError("snapshot object keys must be plain strings") + if set(keys) != expected: + raise IncrementalThreadError(f"{name} fields do not match the schema") + return value + + +class IncrementalThreadIndex: + """Maintain batch-equivalent thread roots across atomic mailbox changes.""" + + def __init__( + self, + *, + group_by_subject: bool = False, + sort_by_sent_date: bool = False, + max_snapshot_records: int = _DEFAULT_MAX_SNAPSHOT_RECORDS, + max_snapshot_bytes: int = _DEFAULT_MAX_SNAPSHOT_BYTES, + ) -> None: + """Create an empty index with batch-compatible options and snapshot bounds.""" + if not isinstance(group_by_subject, bool): + raise IncrementalThreadError("group_by_subject must be a boolean") + if not isinstance(sort_by_sent_date, bool): + raise IncrementalThreadError("sort_by_sent_date must be a boolean") + self._state_lock = RLock() + self._group_by_subject = group_by_subject + self._sort_by_sent_date = sort_by_sent_date + self._max_snapshot_records = _validated_positive_limit( + max_snapshot_records, + "max_snapshot_records", + ) + self._max_snapshot_bytes = _validated_positive_limit( + max_snapshot_bytes, + "max_snapshot_bytes", + ) + self._version = 0 + self._records: dict[str, IndexedMessage] = {} + self._positions: dict[str, int] = {} + self._next_position = 1 + self._tokens_by_key: dict[str, frozenset[str]] = {} + self._keys_by_token: dict[str, set[str]] = {} + self._email_id_states: dict[str, tuple[str | None, int]] = {} + self._thread_id_counts: dict[str, int] = {} + self._component_by_key: dict[str, str] = {} + self._keys_by_component: dict[str, tuple[str, ...]] = {} + self._roots: tuple[Container, ...] | None = () + self._projections: tuple[ThreadProjection, ...] | None = () + + def __len__(self) -> int: + """Return the number of indexed caller message keys.""" + with self._state_lock: + return len(self._records) + + def _materialize_forest(self) -> None: + """Build and cache the complete canonical forest only when requested.""" + if self._roots is not None and self._projections is not None: + return + ranks = ( + _current_ranks(self._positions) + if self._sort_by_sent_date + else self._positions + ) + roots, projections = _build_forest( + self.message_keys, + self._records, + ranks, + group_by_subject=self._group_by_subject, + sort_by_sent_date=self._sort_by_sent_date, + ) + self._roots = roots + self._projections = projections + + @property + def version(self) -> int: + """Return the optimistic mailbox-state version.""" + with self._state_lock: + return self._version + + @property + def message_keys(self) -> tuple[str, ...]: + """Return current caller keys in stable batch input order.""" + with self._state_lock: + return _ordered_keys(self._records, self._positions) + + @property + def roots(self) -> tuple[Container, ...]: + """Return defensive transport-neutral copies of current thread roots.""" + with self._state_lock: + self._materialize_forest() + assert self._roots is not None + return _public_forest_copy(self._roots) + + @property + def projections(self) -> tuple[ThreadProjection, ...]: + """Return deterministic caller-key projections for current roots.""" + with self._state_lock: + self._materialize_forest() + assert self._projections is not None + return self._projections + + def apply(self, change_set: MailboxChangeSet) -> ThreadDelta: + """Atomically apply one optimistic mailbox change set. + + Concurrent callers are serialized. A second writer using the same + ``expected_version`` observes the first commit and raises + :class:`VersionConflictError` instead of interleaving copied state. + + Raises: + VersionConflictError: ``expected_version`` is stale. + IncrementalThreadError: Key ownership, metadata, or graph processing + violates the public contract. The existing state remains unchanged. + ExternalIdentityError: Reported EMAILID/THREADID metadata changes or + conflicts across equal EMAILID values. + """ + with self._state_lock: + return self._apply_locked(change_set) + + def _apply_locked(self, change_set: MailboxChangeSet) -> ThreadDelta: + """Apply one change while ``_state_lock`` protects every state field.""" + if not isinstance(change_set, MailboxChangeSet): + raise IncrementalThreadError("change_set must be a MailboxChangeSet") + if change_set.expected_version != self._version: + raise VersionConflictError( + f"expected version {change_set.expected_version}; " + f"current version {self._version}" + ) + if not ( + change_set.additions or change_set.replacements or change_set.removals + ): + return ThreadDelta( + self._version, + self._version, + (), + (), + (), + (), + (), + (), + ) + + addition_keys = {record.message_key for record in change_set.additions} + replacement_keys = {record.message_key for record in change_set.replacements} + removal_keys = set(change_set.removals) + already_present = {key for key in addition_keys if key in self._records} + missing_replacements = { + key for key in replacement_keys if key not in self._records + } + missing_removals = {key for key in removal_keys if key not in self._records} + if already_present: + raise IncrementalThreadError( + f"addition keys already exist: {sorted(already_present)!r}" + ) + if missing_replacements: + raise IncrementalThreadError( + f"replacement keys do not exist: {sorted(missing_replacements)!r}" + ) + if missing_removals: + raise IncrementalThreadError( + f"removal keys do not exist: {sorted(missing_removals)!r}" + ) + + copied_additions = tuple( + _copied_indexed_message(record) for record in change_set.additions + ) + copied_replacements = tuple( + _copied_indexed_message(record) for record in change_set.replacements + ) + for replacement in copied_replacements: + _validate_replacement_identity( + self._records[replacement.message_key], + replacement, + ) + + record_updates = { + record.message_key: record + for record in (*copied_replacements, *copied_additions) + } + records = _OverlayMapping( + self._records, + record_updates, + frozenset(removal_keys), + ) + position_updates: dict[str, int] = {} + next_position = self._next_position + for addition in copied_additions: + position_updates[addition.message_key] = next_position + next_position += 1 + positions = _OverlayMapping( + self._positions, + position_updates, + frozenset(removal_keys), + ) + + token_updates: dict[str, frozenset[str]] = {} + token_bucket_updates: dict[str, set[str]] = {} + email_state_updates: dict[str, tuple[str | None, int] | None] = {} + thread_count_updates: dict[str, int] = {} + changed_existing_keys = replacement_keys | removal_keys + candidate_seeds: set[str] = set() + touched_tokens: set[str] = set() + touched_identity_values: set[str] = set() + + for key in changed_existing_keys: + component_id = self._component_by_key.get(key) + if component_id is not None: + candidate_seeds.update(self._keys_by_component[component_id]) + old_tokens = self._tokens_by_key.get(key, frozenset()) + touched_tokens.update(old_tokens) + _remove_key_from_buckets( + key, + old_tokens, + self._keys_by_token, + token_bucket_updates, + ) + _stage_external_identity( + self._records[key], + -1, + self._email_id_states, + email_state_updates, + self._thread_id_counts, + thread_count_updates, + touched_identity_values, + ) + + for record in (*copied_replacements, *copied_additions): + tokens = _connectivity_tokens( + record, + group_by_subject=self._group_by_subject, + ) + token_updates[record.message_key] = tokens + touched_tokens.update(tokens) + _add_key_to_buckets( + record.message_key, + tokens, + self._keys_by_token, + token_bucket_updates, + ) + _stage_external_identity( + record, + 1, + self._email_id_states, + email_state_updates, + self._thread_id_counts, + thread_count_updates, + touched_identity_values, + ) + candidate_seeds.add(record.message_key) + + tokens_by_key = _OverlayMapping( + self._tokens_by_key, + token_updates, + frozenset(removal_keys), + ) + keys_by_token = _OverlayMapping(self._keys_by_token, token_bucket_updates) + + for token in touched_tokens: + candidate_seeds.update(keys_by_token.get(token, set())) + for key in tuple(candidate_seeds): + component_id = self._component_by_key.get(key) + if component_id is not None: + candidate_seeds.update(self._keys_by_component[component_id]) + + old_component_ids = { + self._component_by_key[key] + for key in candidate_seeds + if key in self._component_by_key + } + old_ranks = ( + _current_ranks(self._positions) + if self._sort_by_sent_date + else self._positions + ) + before_affected_keys = _ordered_keys( + { + key + for component_id in old_component_ids + for key in self._keys_by_component[component_id] + }, + self._positions, + ) + _, before_affected_projections = _build_forest( + before_affected_keys, + self._records, + old_ranks, + group_by_subject=self._group_by_subject, + sort_by_sent_date=self._sort_by_sent_date, + ) + + _validate_touched_identity_namespaces( + touched_identity_values, + self._email_id_states, + email_state_updates, + self._thread_id_counts, + thread_count_updates, + ) + ranks = ( + _current_ranks(positions) + if self._sort_by_sent_date + else positions + ) + if self._sort_by_sent_date: + _validate_effective_sequence_numbers(records, ranks) + + current_candidate_keys = _expand_candidate_keys( + candidate_seeds, + tokens_by_key, + keys_by_token, + ) + candidate_keys = current_candidate_keys | set(candidate_seeds) + new_components = _partition_components( + current_candidate_keys, + positions, + tokens_by_key, + keys_by_token, + ) + + after_affected_keys = _ordered_keys(current_candidate_keys, positions) + _, after_affected_projections = _build_forest( + after_affected_keys, + records, + ranks, + group_by_subject=self._group_by_subject, + sort_by_sent_date=self._sort_by_sent_date, + ) + affected_positions = { + key: positions.get(key, self._positions.get(key, _MAX_IMAP_NUMBER)) + for key in candidate_keys + } + affected = tuple( + sorted(candidate_keys, key=lambda key: (affected_positions[key], key)) + ) + previous_version = self._version + version = previous_version + 1 + delta = _thread_delta( + previous_version, + version, + affected, + before_affected_projections, + after_affected_projections, + ) + + for key in removal_keys: + self._records.pop(key) + self._positions.pop(key) + self._tokens_by_key.pop(key, None) + self._records.update(record_updates) + self._positions.update(position_updates) + self._tokens_by_key.update(token_updates) + _commit_bucket_updates(self._keys_by_token, token_bucket_updates) + _commit_external_identity_updates( + self._email_id_states, + email_state_updates, + self._thread_id_counts, + thread_count_updates, + ) + + for component_id in old_component_ids: + self._keys_by_component.pop(component_id, None) + for key in candidate_keys: + self._component_by_key.pop(key, None) + for keys in new_components: + component_id = keys[0] + self._keys_by_component[component_id] = keys + for key in keys: + self._component_by_key[key] = component_id + + self._next_position = next_position + self._roots = None + self._projections = None + self._version = version + return delta + + def snapshot(self) -> dict[str, object]: + """Return deterministic versioned JSON-safe state without payload objects.""" + with self._state_lock: + return self._snapshot_locked() + + def _snapshot_locked(self) -> dict[str, object]: + """Build one snapshot while ``_state_lock`` protects current state.""" + if len(self._records) > self._max_snapshot_records: + raise IncrementalThreadError( + "snapshot exceeds max_snapshot_records" + ) + records: list[dict[str, object]] = [] + for key in self.message_keys: + record = self._records[key] + message = record.message + records.append( + { + "message_key": key, + "email_id": record.email_id, + "thread_id": record.thread_id, + "message": { + "message_id": message.message_id, + "in_reply_to": list(_reference_ids(message.in_reply_to)), + "references": list(_reference_ids(message.references)), + "subject": message.subject, + "sent_date": _encoded_date(message.sent_date), + "internal_date": _encoded_date(message.internal_date), + "sequence_number": message.sequence_number, + "uid": message.uid, + }, + } + ) + snapshot: dict[str, object] = { + "schema_version": _SNAPSHOT_SCHEMA_VERSION, + "version": self._version, + "options": { + "group_by_subject": self._group_by_subject, + "sort_by_sent_date": self._sort_by_sent_date, + }, + "records": records, + } + _bounded_snapshot_json_size(snapshot, self._max_snapshot_bytes) + return snapshot + + @classmethod + def restore( + cls, + snapshot: Mapping[str, object], + *, + max_snapshot_records: int = _DEFAULT_MAX_SNAPSHOT_RECORDS, + max_snapshot_bytes: int = _DEFAULT_MAX_SNAPSHOT_BYTES, + ) -> IncrementalThreadIndex: + """Restore strict schema-version-1 state and rebuild all derived indexes.""" + max_records = _validated_positive_limit( + max_snapshot_records, + "max_snapshot_records", + ) + max_bytes = _validated_positive_limit( + max_snapshot_bytes, + "max_snapshot_bytes", + ) + snapshot_object = _required_plain_object( + snapshot, + {"schema_version", "version", "options", "records"}, + "snapshot", + ) + raw_options = snapshot_object["options"] + if not isinstance(raw_options, Mapping): + raise IncrementalThreadError("snapshot options must be a mapping") + options = _required_plain_object( + raw_options, + {"group_by_subject", "sort_by_sent_date"}, + "option", + ) + encoded_records = snapshot_object["records"] + if not isinstance(encoded_records, list): + raise IncrementalThreadError("snapshot records must be a list") + if type(encoded_records) is not list: + raise IncrementalThreadError( + "snapshot must contain only plain JSON containers and scalar values" + ) + if len(encoded_records) > max_records: + raise IncrementalThreadError("snapshot exceeds max_snapshot_records") + _require_plain_json_containers( + snapshot_object, + maximum_nodes=max_bytes, + ) + _bounded_snapshot_json_size(snapshot_object, max_bytes) + schema_version = snapshot_object["schema_version"] + if ( + isinstance(schema_version, bool) + or not isinstance(schema_version, int) + or schema_version != _SNAPSHOT_SCHEMA_VERSION + ): + raise IncrementalThreadError("unsupported snapshot schema_version") + version = _validated_nonnegative_integer( + snapshot_object["version"], + "version", + ) + group_by_subject = options["group_by_subject"] + sort_by_sent_date = options["sort_by_sent_date"] + if not isinstance(group_by_subject, bool): + raise IncrementalThreadError("group_by_subject must be a boolean") + if not isinstance(sort_by_sent_date, bool): + raise IncrementalThreadError("sort_by_sent_date must be a boolean") + records: list[IndexedMessage] = [] + seen_keys: set[str] = set() + record_fields = {"message_key", "email_id", "thread_id", "message"} + message_fields = { + "message_id", + "in_reply_to", + "references", + "subject", + "sent_date", + "internal_date", + "sequence_number", + "uid", + } + for encoded_record in encoded_records: + encoded_record = _required_plain_object( + encoded_record, + record_fields, + "record", + ) + key = _validated_message_key(encoded_record["message_key"]) + if key in seen_keys: + raise IncrementalThreadError(f"duplicate message_key in snapshot: {key}") + seen_keys.add(key) + encoded_message = encoded_record["message"] + encoded_message = _required_plain_object( + encoded_message, + message_fields, + "message", + ) + in_reply_to = encoded_message["in_reply_to"] + references = encoded_message["references"] + if not isinstance(in_reply_to, list) or not all( + isinstance(value, str) for value in in_reply_to + ): + raise IncrementalThreadError("in_reply_to must be a list of strings") + if not isinstance(references, list) or not all( + isinstance(value, str) for value in references + ): + raise IncrementalThreadError("references must be a list of strings") + message = Message( + message_id=_validated_optional_text( + encoded_message["message_id"], + "message_id", + ), + in_reply_to=tuple(in_reply_to), + references=tuple(references), + subject=_validated_optional_text( + encoded_message["subject"], + "subject", + ), + payload=None, + sent_date=_decoded_date(encoded_message["sent_date"], "sent_date"), + internal_date=_decoded_date( + encoded_message["internal_date"], + "internal_date", + ), + sequence_number=_validated_optional_number( + encoded_message["sequence_number"], + "sequence_number", + ), + uid=_validated_optional_number( + encoded_message["uid"], + "uid", + maximum=_MAX_IMAP_NUMBER, + ), + ) + records.append( + IndexedMessage( + message_key=key, + message=message, + email_id=_validated_external_id( + encoded_record["email_id"], + "email_id", + ), + thread_id=_validated_external_id( + encoded_record["thread_id"], + "thread_id", + ), + ) + ) + + index = cls( + group_by_subject=group_by_subject, + sort_by_sent_date=sort_by_sent_date, + max_snapshot_records=max_records, + max_snapshot_bytes=max_bytes, + ) + if records: + index.apply( + MailboxChangeSet(expected_version=0, additions=tuple(records)) + ) + index._version = version + return index diff --git a/tests/test_documentation.py b/tests/test_documentation.py index a2dde8e..29906aa 100644 --- a/tests/test_documentation.py +++ b/tests/test_documentation.py @@ -11,6 +11,7 @@ import threadweave.encoded_words import threadweave.headers import threadweave.imap +import threadweave.incremental import threadweave.subject import threadweave.threading @@ -22,6 +23,7 @@ threadweave.encoded_words, threadweave.headers, threadweave.imap, + threadweave.incremental, threadweave.subject, threadweave.threading, ) diff --git a/tests/test_incremental_benchmark.py b/tests/test_incremental_benchmark.py new file mode 100644 index 0000000..0163ee5 --- /dev/null +++ b/tests/test_incremental_benchmark.py @@ -0,0 +1,67 @@ +"""Contract tests for the mailbox-scale incremental benchmark.""" + +from __future__ import annotations + +import importlib.util +import json +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).parents[1] +BENCHMARK_PATH = ROOT / "benchmarks" / "incremental_mailbox.py" +SPEC = importlib.util.spec_from_file_location( + "threadweave_incremental_benchmark", + BENCHMARK_PATH, +) +assert SPEC is not None and SPEC.loader is not None +benchmark = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = benchmark +SPEC.loader.exec_module(benchmark) + + +def test_small_benchmark_proves_projection_parity_and_reports_resources(): + """The CI-sized run emits matching digests and complete evidence fields.""" + result = benchmark.run_benchmark(1_000, 10) + + assert result["schema_version"] == 1 + assert result["message_count"] == 1_001 + assert result["incremental"]["projection_sha256"] == ( + result["full_rebuild"]["projection_sha256"] + ) + assert result["incremental"]["affected_message_count"] == 21 + assert result["incremental"]["delta_apply_seconds"] >= 0 + assert result["incremental"]["delta_retained_bytes"] >= 0 + assert result["incremental"]["delta_transient_peak_bytes"] >= 0 + assert result["incremental"]["peak_rss_bytes"] > 0 + assert result["full_rebuild"]["full_rebuild_seconds"] >= 0 + assert result["full_rebuild"]["peak_rss_bytes"] > 0 + + +def test_benchmark_validates_size_contracts(): + """Invalid or undersized workloads fail before spawning workers.""" + with pytest.raises(ValueError, match="message_count"): + benchmark.run_benchmark(0, 10) + with pytest.raises(ValueError, match="thread_size"): + benchmark.run_benchmark(100, 0) + with pytest.raises(ValueError, match="two complete threads"): + benchmark.run_benchmark(10, 10) + + +def test_main_writes_deterministic_json_shape(tmp_path: Path): + """The command-line entry point writes one parseable evidence document.""" + output = tmp_path / "benchmark.json" + assert benchmark.main( + [ + "--messages", + "1000", + "--thread-size", + "10", + "--output", + str(output), + ] + ) == 0 + parsed = json.loads(output.read_text(encoding="utf-8")) + assert parsed["schema_version"] == 1 + assert parsed["message_count"] == 1_001 diff --git a/tests/test_incremental_benchmark_workflow.py b/tests/test_incremental_benchmark_workflow.py new file mode 100644 index 0000000..3c9120d --- /dev/null +++ b/tests/test_incremental_benchmark_workflow.py @@ -0,0 +1,56 @@ +"""Governance contract for the mailbox-scale benchmark workflow.""" + +from pathlib import Path + +WORKFLOW = ( + Path(__file__).parents[1] + / ".github" + / "workflows" + / "incremental-benchmark.yml" +) + + +def _workflow() -> str: + """Return the workflow source without adding a YAML parser dependency.""" + return WORKFLOW.read_text(encoding="utf-8") + + +def test_benchmark_is_manual_and_scheduled_with_a_100k_default(): + """Operators can reproduce evidence while a weekly run catches regressions.""" + workflow = _workflow() + assert "workflow_dispatch:" in workflow + assert 'default: "100000"' in workflow + assert 'cron: "17 3 * * 1"' in workflow + assert "cancel-in-progress: true" in workflow + + +def test_benchmark_uses_immutable_actions_and_the_reviewed_dependency_lock(): + """Performance evidence runs through the same pinned supply-chain boundary.""" + workflow = _workflow() + required_actions = { + "step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920", + "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97", + "actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a", + } + observed_actions = { + line.strip().removeprefix("uses: ").split(" #", 1)[0] + for line in workflow.splitlines() + if line.strip().startswith("uses: ") + } + assert required_actions <= observed_actions + assert "pip install --require-hashes -r requirements/ci.lock" in workflow + assert "persist-credentials: false" in workflow + + +def test_benchmark_requires_parity_bounded_impact_and_evidence_upload(): + """The workflow fails closed on incorrect output or a regressed 100k delta.""" + workflow = _workflow() + assert "incremental_mailbox.py" in workflow + assert 'incremental["projection_sha256"] == full_rebuild["projection_sha256"]' in workflow + assert 'incremental["affected_message_count"] == 21' in workflow + assert 'incremental["delta_apply_seconds"] < full_rebuild[' in workflow + assert "incremental-mailbox-benchmark-${{ github.run_id }}" in workflow + assert "retention-days: 90" in workflow + assert "NVIDIA_NIM_API_KEY" not in workflow + assert "COPILOT_GITHUB_TOKEN" not in workflow diff --git a/tests/test_incremental_components.py b/tests/test_incremental_components.py new file mode 100644 index 0000000..bd5ec4b --- /dev/null +++ b/tests/test_incremental_components.py @@ -0,0 +1,330 @@ +"""Component and delta tests for incremental mailbox threading.""" + +from __future__ import annotations + +import pytest + +from collections.abc import Iterable + +import threadweave.incremental as incremental_module +from threadweave import ( + Container, + IncrementalThreadIndex, + IndexedMessage, + MailboxChangeSet, + Message, + thread_messages, +) + + +def _record( + key: str, + *, + message_id: str | None = None, + references: tuple[str, ...] = (), + in_reply_to: tuple[str, ...] = (), + subject: str | None = None, + sent_date: str | None = None, + sequence_number: int | None = None, + uid: int | None = None, + email_id: str | None = None, + thread_id: str | None = None, +) -> IndexedMessage: + """Build one record whose payload exposes the caller key to tests.""" + return IndexedMessage( + message_key=key, + message=Message( + message_id=message_id if message_id is not None else key, + references=references, + in_reply_to=in_reply_to, + subject=subject, + payload=key, + sent_date=sent_date, + sequence_number=sequence_number, + uid=uid, + ), + email_id=email_id, + thread_id=thread_id, + ) + + +def _shape(roots: Iterable[Container]) -> tuple[object, ...]: + """Return one deterministic, iterative forest representation.""" + rendered: list[object] = [] + for root in roots: + output: list[object] = [] + stack: list[tuple[Container, bool]] = [(root, False)] + built: dict[int, object] = {} + while stack: + node, exiting = stack.pop() + if not exiting: + stack.append((node, True)) + for child in reversed(node.children): + stack.append((child, False)) + continue + children = tuple(built[id(child)] for child in node.children) + key = None if node.message is None else node.message.payload + built[id(node)] = (key, children) + output.append(built[id(root)]) + rendered.extend(output) + return tuple(rendered) + + +def _batch( + records: Iterable[IndexedMessage], + *, + group_by_subject: bool = False, + sort_by_sent_date: bool = False, +) -> tuple[object, ...]: + """Return the canonical batch shape for indexed records.""" + return _shape( + thread_messages( + (record.message for record in records), + group_by_subject=group_by_subject, + sort_by_sent_date=sort_by_sent_date, + ) + ) + + +def test_additions_match_batch_and_preserve_independent_root_order(): + """Initial component construction is exactly equivalent to one batch call.""" + records = ( + _record("a"), + _record("b", references=("a",)), + _record("x"), + _record("y", references=("x",)), + ) + index = IncrementalThreadIndex() + delta = index.apply(MailboxChangeSet(expected_version=0, additions=records)) + + assert _shape(index.roots) == _batch(records) + assert index.message_keys == ("a", "b", "x", "y") + assert delta.affected_message_keys == ("a", "b", "x", "y") + assert tuple(projection.message_keys for projection in index.projections) == ( + ("a", "b"), + ("x", "y"), + ) + assert len(delta.added_threads) == 2 + assert delta.removed_threads == () + + +def test_delayed_missing_ancestor_recomputes_only_the_affected_component(): + """A late parent replaces a dummy without touching an unrelated thread.""" + child = _record("child", references=("root",)) + other = _record("other") + index = IncrementalThreadIndex() + index.apply( + MailboxChangeSet(expected_version=0, additions=(child, other)) + ) + + root = _record("root") + delta = index.apply( + MailboxChangeSet(expected_version=1, additions=(root,)) + ) + + assert _shape(index.roots) == _batch((child, other, root)) + assert delta.affected_message_keys == ("child", "root") + assert tuple(projection.message_keys for projection in index.projections) == ( + ("root", "child"), + ("other",), + ) + + +def test_bridge_message_emits_an_explicit_external_identity_merge(): + """Joining two exposed groups reports both caller THREADIDs.""" + first = _record("a", thread_id="T1") + second = _record("b", thread_id="T2") + index = IncrementalThreadIndex() + index.apply( + MailboxChangeSet(expected_version=0, additions=(first, second)) + ) + + bridge = _record("bridge", references=("a", "b"), thread_id="T3") + delta = index.apply( + MailboxChangeSet(expected_version=1, additions=(bridge,)) + ) + + assert _shape(index.roots) == _batch((first, second, bridge)) + assert len(delta.merges) == 1 + merge = delta.merges[0] + assert merge.kind == "merge" + assert merge.thread_ids == ("T1", "T2", "T3") + assert tuple(item.message_keys for item in merge.before) == (("a",), ("b",)) + assert tuple(item.message_keys for item in merge.after) == (("a", "b", "bridge"),) + + +def test_replacing_bridge_references_emits_a_split_and_matches_batch(): + """Removing a structural bridge rediscovers both resulting components.""" + first = _record("a", thread_id="T1") + second = _record("b", thread_id="T2") + bridge = _record("bridge", references=("a", "b"), thread_id="T3") + index = IncrementalThreadIndex() + index.apply( + MailboxChangeSet(expected_version=0, additions=(first, second, bridge)) + ) + + replacement = _record("bridge", references=("a",), thread_id="T3") + delta = index.apply( + MailboxChangeSet(expected_version=1, replacements=(replacement,)) + ) + + assert _shape(index.roots) == _batch((first, second, replacement)) + assert len(delta.splits) == 1 + split = delta.splits[0] + assert split.kind == "split" + assert split.thread_ids == ("T1", "T2", "T3") + assert tuple(item.message_keys for item in split.before) == ( + ("a", "b", "bridge"), + ) + assert {item.message_keys for item in split.after} == { + ("a", "bridge"), + ("b",), + } + + +def test_removing_root_internal_leaf_and_duplicate_id_matches_batch(): + """Every removal location follows canonical dummy pruning and promotion.""" + records = ( + _record("root"), + _record("middle", references=("root",)), + _record("leaf", references=("root", "middle")), + _record("duplicate", message_id="middle"), + _record("missing", message_id=None), + ) + + for removed_key in ("root", "middle", "leaf", "duplicate", "missing"): + index = IncrementalThreadIndex() + index.apply(MailboxChangeSet(expected_version=0, additions=records)) + index.apply( + MailboxChangeSet(expected_version=1, removals=(removed_key,)) + ) + remaining = tuple(record for record in records if record.message_key != removed_key) + assert _shape(index.roots) == _batch(remaining) + + +def test_unrelated_components_are_not_passed_to_the_batch_delegate(monkeypatch): + """A bounded update avoids rescanning a structurally unrelated component.""" + records = ( + _record("a"), + _record("b", references=("a",)), + _record("x"), + _record("y", references=("x",)), + ) + index = IncrementalThreadIndex() + index.apply(MailboxChangeSet(expected_version=0, additions=records)) + + calls: list[tuple[str, ...]] = [] + real_delegate = incremental_module._batch_thread_messages + + def recording_delegate(messages, **options): + """Record component payload keys before invoking the canonical batcher.""" + materialized = tuple(messages) + calls.append(tuple(message.payload for message in materialized)) + return real_delegate(materialized, **options) + + monkeypatch.setattr(incremental_module, "_batch_thread_messages", recording_delegate) + replacement = _record("b", references=("a",), subject="changed") + index.apply( + MailboxChangeSet(expected_version=1, replacements=(replacement,)) + ) + + assert calls == [("a", "b"), ("a", "b")] + assert all("x" not in call and "y" not in call for call in calls) + + +def test_noop_change_set_advances_no_version_and_returns_empty_delta(): + """An empty request is idempotent and does not fabricate a new revision.""" + index = IncrementalThreadIndex() + delta = index.apply(MailboxChangeSet(expected_version=0)) + + assert delta.previous_version == 0 + assert delta.version == 0 + assert delta.affected_message_keys == () + assert delta.added_threads == () + assert delta.removed_threads == () + assert delta.updated_threads == () + assert delta.merges == () + assert delta.splits == () + + +def test_delta_classification_receives_only_affected_component_projections( + monkeypatch: pytest.MonkeyPatch, +): + """Unrelated roots never enter one small change's delta classifier.""" + records = tuple( + _record(f"key_{index}", message_id=f"message_{index}") + for index in range(128) + ) + index = IncrementalThreadIndex() + index.apply(MailboxChangeSet(expected_version=0, additions=records)) + + original = incremental_module._thread_delta + observed_sizes: list[tuple[int, int]] = [] + + def recording_delta(*args): + """Record old/new projection counts before delegating.""" + observed_sizes.append((len(args[3]), len(args[4]))) + return original(*args) + + monkeypatch.setattr(incremental_module, "_thread_delta", recording_delta) + index.apply( + MailboxChangeSet( + expected_version=1, + additions=( + _record( + "new_key", + message_id="new_message", + references=("message_0",), + ), + ), + ) + ) + + assert observed_sizes == [(1, 1)] + + + +def test_small_delta_defers_the_complete_canonical_batch_view( + monkeypatch: pytest.MonkeyPatch, +): + """Apply batches only affected records until a full view is requested.""" + records = tuple( + _record(f"key_{index}", message_id=f"message_{index}") + for index in range(128) + ) + index = IncrementalThreadIndex() + index.apply(MailboxChangeSet(expected_version=0, additions=records)) + + original = incremental_module._batch_thread_messages + observed_message_counts: list[int] = [] + + def recording_batch(messages, **options): + """Record each canonical batch size before delegating.""" + materialized = tuple(messages) + observed_message_counts.append(len(materialized)) + return original(materialized, **options) + + monkeypatch.setattr( + incremental_module, + "_batch_thread_messages", + recording_batch, + ) + index.apply( + MailboxChangeSet( + expected_version=1, + additions=( + _record( + "new_key", + message_id="new_message", + references=("message_0",), + ), + ), + ) + ) + + assert observed_message_counts == [1, 2] + assert len(index.projections) == 128 + assert observed_message_counts == [1, 2, 129] + assert len(index.projections) == 128 + assert len(index.roots) == 128 + assert observed_message_counts == [1, 2, 129] diff --git a/tests/test_incremental_concurrency.py b/tests/test_incremental_concurrency.py new file mode 100644 index 0000000..922a84a --- /dev/null +++ b/tests/test_incremental_concurrency.py @@ -0,0 +1,174 @@ +"""Concurrency contracts for the incremental mailbox index.""" + +from __future__ import annotations + +import os +import subprocess +import sys +import threading +from pathlib import Path + +import threadweave.incremental as incremental_module +from threadweave import ( + IncrementalThreadIndex, + IndexedMessage, + MailboxChangeSet, + Message, + VersionConflictError, +) + + +def _record(message_key: str) -> IndexedMessage: + """Return one independent indexed message for a concurrent update.""" + return IndexedMessage(message_key, Message(message_id=message_key)) + + +def test_concurrent_writers_are_serialized_by_optimistic_version(monkeypatch): + """Two writers targeting one version yield one commit and one conflict.""" + index = IncrementalThreadIndex() + real_copy = incremental_module._copied_indexed_message + first_entered = threading.Event() + second_entered = threading.Event() + release_first = threading.Event() + activity_lock = threading.Lock() + active_calls = 0 + maximum_active_calls = 0 + + def controlled_copy(record: IndexedMessage) -> IndexedMessage: + """Expose whether two transactions copy records at the same time.""" + nonlocal active_calls, maximum_active_calls + with activity_lock: + active_calls += 1 + maximum_active_calls = max(maximum_active_calls, active_calls) + is_first_call = active_calls == 1 + if is_first_call: + first_entered.set() + else: + second_entered.set() + release_first.set() + if is_first_call: + release_first.wait(timeout=1) + try: + return real_copy(record) + finally: + with activity_lock: + active_calls -= 1 + + monkeypatch.setattr( + incremental_module, + "_copied_indexed_message", + controlled_copy, + ) + + outcomes: list[tuple[str, object]] = [] + + def apply_record(message_key: str) -> None: + """Apply one expected-version-zero transaction and capture its outcome.""" + try: + delta = index.apply( + MailboxChangeSet( + expected_version=0, + additions=(_record(message_key),), + ) + ) + except VersionConflictError as error: + outcomes.append(("conflict", error)) + else: + outcomes.append(("committed", delta)) + + first = threading.Thread(target=apply_record, args=("first",)) + second = threading.Thread(target=apply_record, args=("second",)) + first.start() + assert first_entered.wait(timeout=1) + second.start() + if not second_entered.wait(timeout=0.1): + release_first.set() + first.join(timeout=2) + second.join(timeout=2) + + assert not first.is_alive() + assert not second.is_alive() + assert maximum_active_calls == 1 + assert sorted(kind for kind, _ in outcomes) == ["committed", "conflict"] + assert index.version == 1 + assert len(index.message_keys) == 1 + + +def test_readers_observe_only_committed_versions(monkeypatch): + """A reader blocks behind a writer and never observes partial transaction state.""" + index = IncrementalThreadIndex() + real_copy = incremental_module._copied_indexed_message + writer_entered = threading.Event() + release_writer = threading.Event() + reader_started = threading.Event() + reader_finished = threading.Event() + observed: list[dict[str, object]] = [] + + def controlled_copy(record: IndexedMessage) -> IndexedMessage: + """Pause one writer after it owns the state lock but before commit.""" + writer_entered.set() + assert release_writer.wait(timeout=2) + return real_copy(record) + + monkeypatch.setattr( + incremental_module, + "_copied_indexed_message", + controlled_copy, + ) + + def write() -> None: + """Apply one transaction while the test controls its commit point.""" + index.apply( + MailboxChangeSet( + expected_version=0, + additions=(_record("message"),), + ) + ) + + def read() -> None: + """Capture one snapshot after acquiring the same state lock.""" + reader_started.set() + observed.append(index.snapshot()) + reader_finished.set() + + writer = threading.Thread(target=write) + reader = threading.Thread(target=read) + writer.start() + assert writer_entered.wait(timeout=1) + reader.start() + assert reader_started.wait(timeout=1) + assert not reader_finished.wait(timeout=0.05) + + release_writer.set() + writer.join(timeout=2) + reader.join(timeout=2) + + assert not writer.is_alive() + assert not reader.is_alive() + assert reader_finished.is_set() + assert observed[0]["version"] == 1 + records = observed[0]["records"] + assert isinstance(records, list) + assert [record["message_key"] for record in records] == ["message"] + + +def test_package_import_survives_a_top_level_threading_name_collision(): + """The built-in lock remains importable beside ``threadweave/threading.py``.""" + repository_root = Path(__file__).resolve().parents[1] + import_script = ( + "import sys; " + "sys.path.insert(0, 'src/threadweave'); " + "sys.path.insert(1, 'src'); " + "import subject" + ) + + result = subprocess.run( + [sys.executable, "-S", "-c", import_script], + cwd=repository_root, + env=os.environ.copy(), + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 0, result.stderr diff --git a/tests/test_incremental_contract.py b/tests/test_incremental_contract.py new file mode 100644 index 0000000..d374c83 --- /dev/null +++ b/tests/test_incremental_contract.py @@ -0,0 +1,271 @@ +"""Public and atomicity contracts for incremental mailbox threading.""" + +from __future__ import annotations + +from dataclasses import FrozenInstanceError + +import pytest + +from threadweave import ( + ExternalIdentityError, + IncrementalThreadError, + IncrementalThreadIndex, + IndexedMessage, + MailboxChangeSet, + Message, + VersionConflictError, +) + + +def _record( + key: str, + *, + message_id: str | None = None, + references: tuple[str, ...] = (), + subject: str | None = None, + payload: object | None = None, + sequence_number: int | None = None, + uid: int | None = None, + email_id: str | None = None, + thread_id: str | None = None, +) -> IndexedMessage: + """Build one indexed message with explicit caller identity.""" + return IndexedMessage( + message_key=key, + message=Message( + message_id=message_id or key, + references=references, + subject=subject, + payload=key if payload is None else payload, + sequence_number=sequence_number, + uid=uid, + ), + email_id=email_id, + thread_id=thread_id, + ) + + +def test_empty_index_exposes_immutable_empty_state(): + """A new index starts at version zero with no roots or projections.""" + index = IncrementalThreadIndex() + + assert index.version == 0 + assert index.message_keys == () + assert index.roots == () + assert index.projections == () + assert len(index) == 0 + + +def test_public_records_are_frozen_and_sequences_are_normalized(): + """Change records cannot be mutated after validation or retain caller lists.""" + additions = [_record("a")] + changes = MailboxChangeSet(expected_version=0, additions=additions) + + assert changes.additions == tuple(additions) + assert changes.replacements == () + assert changes.removals == () + with pytest.raises(FrozenInstanceError): + changes.expected_version = 1 # type: ignore[misc] + + +def test_indexed_message_rejects_unsafe_keys_and_external_identifiers(): + """Caller identity values remain bounded, printable, and non-empty.""" + for key in ("", "bad\nkey", "x" * 513): + with pytest.raises(IncrementalThreadError, match="message_key"): + _record(key) + + with pytest.raises(IncrementalThreadError, match="email_id"): + _record("a", email_id="bad\x00id") + with pytest.raises(IncrementalThreadError, match="thread_id"): + _record("a", thread_id="") + + +def test_change_set_rejects_invalid_versions_duplicate_and_overlapping_keys(): + """An atomic request has one non-negative version and disjoint unique keys.""" + with pytest.raises(IncrementalThreadError, match="expected_version"): + MailboxChangeSet(expected_version=True) + with pytest.raises(IncrementalThreadError, match="expected_version"): + MailboxChangeSet(expected_version=-1) + with pytest.raises(IncrementalThreadError, match="duplicate.*additions"): + MailboxChangeSet(expected_version=0, additions=(_record("a"), _record("a"))) + with pytest.raises(IncrementalThreadError, match="disjoint"): + MailboxChangeSet( + expected_version=0, + additions=(_record("a"),), + removals=("a",), + ) + with pytest.raises(IncrementalThreadError, match="removals"): + MailboxChangeSet(expected_version=0, removals=("a", "a")) + + +def test_add_replace_remove_are_atomic_and_optimistically_versioned(): + """Successful changes advance once while rejected requests preserve state.""" + original_payload = object() + original = _record("a", subject="Original", payload=original_payload) + index = IncrementalThreadIndex() + + added = index.apply(MailboxChangeSet(expected_version=0, additions=(original,))) + assert (added.previous_version, added.version) == (0, 1) + assert added.affected_message_keys == ("a",) + assert index.message_keys == ("a",) + assert index.roots[0].message.payload is original_payload + + replacement = _record("a", subject="Replacement", payload=original_payload) + replaced = index.apply( + MailboxChangeSet(expected_version=1, replacements=(replacement,)) + ) + assert (replaced.previous_version, replaced.version) == (1, 2) + assert index.roots[0].message.subject == "Replacement" + + before = index.snapshot() + with pytest.raises(VersionConflictError, match="expected version 1.*current version 2"): + index.apply(MailboxChangeSet(expected_version=1, removals=("a",))) + assert index.snapshot() == before + + removed = index.apply(MailboxChangeSet(expected_version=2, removals=("a",))) + assert (removed.previous_version, removed.version) == (2, 3) + assert index.message_keys == () + assert index.roots == () + + +def test_invalid_ownership_requests_leave_the_index_unchanged(): + """Add/replace/remove ownership errors fail before committing copied state.""" + index = IncrementalThreadIndex() + index.apply(MailboxChangeSet(expected_version=0, additions=(_record("a"),))) + before = index.snapshot() + + invalid_changes = ( + MailboxChangeSet(expected_version=1, additions=(_record("a"),)), + MailboxChangeSet(expected_version=1, replacements=(_record("missing"),)), + MailboxChangeSet(expected_version=1, removals=("missing",)), + ) + for changes in invalid_changes: + with pytest.raises(IncrementalThreadError): + index.apply(changes) + assert index.snapshot() == before + + +def test_structural_metadata_is_copied_while_payload_remains_caller_owned(): + """Later caller mutation cannot rewrite indexed references or subject data.""" + references = ["root"] + message = Message( + message_id="child", + references=references, + subject="Original", + payload={"caller": "owned"}, + ) + index = IncrementalThreadIndex() + index.apply( + MailboxChangeSet( + expected_version=0, + additions=(IndexedMessage("child_key", message),), + ) + ) + + references.append("other") + message.subject = "Mutated" + + indexed = index.roots[0].message + assert indexed.references == ("root",) + assert indexed.subject == "Original" + assert indexed.payload is message.payload + + +def test_public_roots_are_defensive_structural_copies(): + """Caller graph edits cannot corrupt the index's reusable internal forest.""" + payload = object() + index = IncrementalThreadIndex() + index.apply( + MailboxChangeSet( + expected_version=0, + additions=( + _record( + "root", + message_id="root", + subject="Original", + payload=payload, + ), + _record( + "child", + message_id="child", + references=("root",), + ), + ), + ) + ) + + exposed = index.roots + exposed[0].message.subject = "Mutated" + exposed[0].children.clear() + + current = index.roots + assert current is not exposed + assert current[0] is not exposed[0] + assert current[0].message is not exposed[0].message + assert current[0].message.subject == "Original" + assert current[0].message.payload is payload + assert [child.message.payload for child in current[0].children] == ["child"] + assert index.projections[0].message_keys == ("root", "child") + + +def test_reported_external_identity_is_immutable_on_replacement(): + """RFC 8474 identity metadata cannot disappear or change after exposure.""" + index = IncrementalThreadIndex() + index.apply( + MailboxChangeSet( + expected_version=0, + additions=( + _record("a", email_id="M1", thread_id="T1"), + ), + ) + ) + before = index.snapshot() + + for replacement in ( + _record("a", email_id=None, thread_id="T1"), + _record("a", email_id="M2", thread_id="T1"), + _record("a", email_id="M1", thread_id=None), + _record("a", email_id="M1", thread_id="T2"), + ): + with pytest.raises(ExternalIdentityError, match="immutable"): + index.apply( + MailboxChangeSet(expected_version=1, replacements=(replacement,)) + ) + assert index.snapshot() == before + + +def test_same_email_id_requires_one_consistent_thread_id(): + """Messages sharing immutable content identity cannot disagree on THREADID.""" + index = IncrementalThreadIndex() + + with pytest.raises(ExternalIdentityError, match="EMAILID.*THREADID"): + index.apply( + MailboxChangeSet( + expected_version=0, + additions=( + _record("a", email_id="M1", thread_id="T1"), + _record("b", email_id="M1", thread_id="T2"), + ), + ) + ) + assert index.version == 0 + + with pytest.raises(ExternalIdentityError, match="EMAILID.*THREADID"): + index.apply( + MailboxChangeSet( + expected_version=0, + additions=( + _record("a", email_id="M1", thread_id="T1"), + _record("b", email_id="M1", thread_id=None), + ), + ) + ) + + +def test_constructor_rejects_boolean_and_nonpositive_snapshot_limits(): + """Snapshot denial-of-service limits are explicit positive integers.""" + for keyword in ("max_snapshot_records", "max_snapshot_bytes"): + with pytest.raises(IncrementalThreadError, match=keyword): + IncrementalThreadIndex(**{keyword: True}) + with pytest.raises(IncrementalThreadError, match=keyword): + IncrementalThreadIndex(**{keyword: 0}) diff --git a/tests/test_incremental_parity.py b/tests/test_incremental_parity.py new file mode 100644 index 0000000..2162dd1 --- /dev/null +++ b/tests/test_incremental_parity.py @@ -0,0 +1,371 @@ +"""RFC, ordering, protocol, and depth parity for the incremental index.""" + +from __future__ import annotations + +import random + +import pytest + +from threadweave import ( + IncrementalThreadIndex, + IncrementalThreadError, + IndexedMessage, + MailboxChangeSet, + Message, + ThreadSerializationError, + serialize_thread_response, + thread_messages, +) + + +def _record(key: str, **message_fields) -> IndexedMessage: + """Build one record whose payload remains its caller key.""" + return IndexedMessage( + message_key=key, + message=Message(payload=key, **message_fields), + ) + + +def _preorder(roots) -> tuple[tuple[str | None, ...], ...]: + """Return root-by-root preorder payload keys without recursion.""" + forest: list[tuple[str | None, ...]] = [] + for root in roots: + keys: list[str | None] = [] + stack = [root] + seen: set[int] = set() + while stack: + node = stack.pop() + if id(node) in seen: + continue + seen.add(id(node)) + keys.append(None if node.message is None else node.message.payload) + stack.extend(reversed(node.children)) + forest.append(tuple(keys)) + return tuple(forest) + + +def test_subject_grouping_uses_rfc_5051_and_replacement_updates_bucket(): + """Compatibility-width and reply variants merge and later split by replacement.""" + first = _record("a", message_id="a", subject="Ｔｏｐｉｃ") + second = _record("b", message_id="b", subject="Re: Topic") + index = IncrementalThreadIndex(group_by_subject=True) + index.apply(MailboxChangeSet(expected_version=0, additions=(first, second))) + + batch = thread_messages( + (first.message, second.message), + group_by_subject=True, + ) + assert _preorder(index.roots) == _preorder(batch) + assert len(index.roots) == 1 + + replacement = _record("b", message_id="b", subject="Different") + delta = index.apply( + MailboxChangeSet(expected_version=1, replacements=(replacement,)) + ) + batch_after = thread_messages( + (first.message, replacement.message), + group_by_subject=True, + ) + assert _preorder(index.roots) == _preorder(batch_after) + assert len(index.roots) == 2 + assert len(delta.splits) == 1 + + +def test_sent_date_order_matches_batch_across_unrelated_components(): + """Global root ordering uses RFC date recovery and sequence tie-breaking.""" + records = ( + _record( + "later", + message_id="later", + sent_date="2 Jan 2026 00:00:00 +0000", + sequence_number=2, + ), + _record( + "earlier", + message_id="earlier", + sent_date="1 Jan 2026 09:00:00 +0900", + sequence_number=1, + ), + _record( + "same_time", + message_id="same-time", + sent_date="1 Jan 2026 00:00:00 +0000", + sequence_number=3, + ), + ) + index = IncrementalThreadIndex(sort_by_sent_date=True) + index.apply(MailboxChangeSet(expected_version=0, additions=records)) + + batch = thread_messages( + (record.message for record in records), + sort_by_sent_date=True, + ) + assert _preorder(index.roots) == _preorder(batch) + assert tuple(projection.message_keys for projection in index.projections) == ( + ("earlier",), + ("same_time",), + ("later",), + ) + + +def test_implicit_sort_positions_do_not_leak_into_public_metadata(): + """Internal sort ranks must not become caller-visible IMAP identifiers.""" + records = ( + _record( + "earlier", + message_id="earlier", + sent_date="1 Jan 2026 00:00:00 +0000", + ), + _record( + "later", + message_id="later", + sent_date="2 Jan 2026 00:00:00 +0000", + ), + ) + index = IncrementalThreadIndex(sort_by_sent_date=True) + index.apply(MailboxChangeSet(expected_version=0, additions=records)) + + batch = thread_messages( + (record.message for record in records), + sort_by_sent_date=True, + ) + assert [root.message.sequence_number for root in index.roots] == [ + root.message.sequence_number for root in batch + ] == [None, None] + with pytest.raises(ThreadSerializationError, match="positive integer"): + serialize_thread_response(index.roots) + + +def test_global_sequence_number_collision_fails_atomically(): + """Separate components cannot hide duplicate RFC ordering tie-breakers.""" + index = IncrementalThreadIndex(sort_by_sent_date=True) + index.apply( + MailboxChangeSet( + expected_version=0, + additions=( + _record("a", message_id="a", sequence_number=1), + ), + ) + ) + before = index.snapshot() + + with pytest.raises(IncrementalThreadError, match="duplicate sequence number"): + index.apply( + MailboxChangeSet( + expected_version=1, + additions=( + _record("b", message_id="b", sequence_number=1), + ), + ) + ) + assert index.snapshot() == before + + +def test_raw_reference_headers_and_uid_thread_output_match_batch(): + """The index preserves raw RFC input and feeds the installed IMAP projector.""" + records = ( + _record( + "root", + message_id="", + sequence_number=1, + uid=101, + ), + _record( + "child", + message_id="", + references="", + in_reply_to=" ", + sequence_number=2, + uid=102, + ), + ) + index = IncrementalThreadIndex() + index.apply(MailboxChangeSet(expected_version=0, additions=records)) + + batch = thread_messages(record.message for record in records) + assert _preorder(index.roots) == _preorder(batch) + assert serialize_thread_response(index.roots) == "* THREAD (1 2)\r\n" + assert serialize_thread_response(index.roots, identifier="uid") == ( + "* THREAD (101 102)\r\n" + ) + + +def test_deep_delayed_ancestry_update_remains_iterative(): + """A deep component can be indexed and extended without recursion limits.""" + depth = 1500 + records = tuple( + _record( + f"message_{index}", + message_id=f"message_{index}", + references=() if index == 0 else (f"message_{index - 1}",), + ) + for index in range(depth) + ) + index = IncrementalThreadIndex() + index.apply(MailboxChangeSet(expected_version=0, additions=records)) + + tail = _record( + "tail", + message_id="tail", + references=(f"message_{depth - 1}",), + ) + delta = index.apply( + MailboxChangeSet(expected_version=1, additions=(tail,)) + ) + + assert len(index.projections) == 1 + assert len(index.projections[0].message_keys) == depth + 1 + assert index.projections[0].message_keys[-1] == "tail" + assert len(delta.affected_message_keys) == depth + 1 + + +def test_implicit_sequence_positions_remain_stable_after_replacement(): + """Replacing metadata does not move a caller key to the end of input order.""" + first = _record("a", message_id="a", sent_date="1 Jan 2026 00:00:00 +0000") + second = _record("b", message_id="b", sent_date="1 Jan 2026 00:00:00 +0000") + index = IncrementalThreadIndex(sort_by_sent_date=True) + index.apply(MailboxChangeSet(expected_version=0, additions=(first, second))) + + replacement = _record( + "a", + message_id="a", + sent_date="1 Jan 2026 00:00:00 +0000", + subject="updated", + ) + index.apply( + MailboxChangeSet(expected_version=1, replacements=(replacement,)) + ) + + assert tuple(projection.message_keys for projection in index.projections) == ( + ("a",), + ("b",), + ) + + +def test_missing_ancestor_creation_order_matches_canonical_batch_root_order(): + """A late missing-root placeholder cannot reorder an earlier independent root.""" + records = ( + _record( + "early_child", + message_id="child@example.test", + references=("late-parent@example.test",), + ), + _record("independent", message_id="independent@example.test"), + _record( + "late_parent", + message_id="late-parent@example.test", + references=("missing-ancestor@example.test",), + ), + ) + index = IncrementalThreadIndex() + index.apply(MailboxChangeSet(expected_version=0, additions=records)) + + batch = thread_messages(record.message for record in records) + + assert _preorder(index.roots) == _preorder(batch) == ( + ("independent",), + ("late_parent", "early_child"), + ) + + +@pytest.mark.parametrize("group_by_subject", [False, True]) +@pytest.mark.parametrize("sort_by_sent_date", [False, True]) +def test_bounded_randomized_change_stream_matches_full_batch_oracle( + group_by_subject: bool, + sort_by_sent_date: bool, +): + """Deterministic mixed mailbox changes preserve canonical forest parity.""" + option_code = int(group_by_subject) * 10 + int(sort_by_sent_date) + subjects = (None, "Topic", "Re: Topic", "Ｔｏｐｉｃ", "Other", "Fwd: Other") + dates = ( + None, + "1 Jan 2026 00:00:00 +0000", + "2 Jan 2026 09:00:00 +0900", + "3 Jan 2026 00:00:00 -0500", + ) + + for seed in range(4): + random_source = random.Random(10_000 + option_code * 100 + seed) + index = IncrementalThreadIndex( + group_by_subject=group_by_subject, + sort_by_sent_date=sort_by_sent_date, + ) + records: dict[str, IndexedMessage] = {} + ordered_keys: list[str] = [] + next_key = 0 + + def random_record(key: str) -> IndexedMessage: + """Build one deterministic adversarial record for this seed.""" + existing_ids = [ + record.message.message_id + for record in records.values() + if record.message.message_id is not None + ] + message_id_mode = random_source.randrange(5) + if message_id_mode == 0: + message_id = None + elif message_id_mode == 1 and existing_ids: + message_id = random_source.choice(existing_ids) + else: + message_id = f"id-{key}@example.test" + + reference_candidates = list(existing_ids) + reference_candidates.extend( + f"missing-{index}@example.test" for index in range(3) + ) + reference_count = random_source.randrange(3) + references = tuple( + random_source.choice(reference_candidates) + for _ in range(reference_count) + ) if reference_candidates else () + return _record( + key, + message_id=message_id, + references=references, + subject=random_source.choice(subjects), + sent_date=random_source.choice(dates), + ) + + for _step in range(24): + operation_roll = random_source.random() + if not records or operation_roll < 0.50: + key = f"seed_{seed}_message_{next_key}" + next_key += 1 + record = random_record(key) + changes = MailboxChangeSet( + expected_version=index.version, + additions=(record,), + ) + records[key] = record + ordered_keys.append(key) + elif operation_roll < 0.78: + key = random_source.choice(ordered_keys) + record = random_record(key) + changes = MailboxChangeSet( + expected_version=index.version, + replacements=(record,), + ) + records[key] = record + else: + key = random_source.choice(ordered_keys) + changes = MailboxChangeSet( + expected_version=index.version, + removals=(key,), + ) + del records[key] + ordered_keys.remove(key) + + index.apply(changes) + batch = thread_messages( + (records[key].message for key in ordered_keys), + group_by_subject=group_by_subject, + sort_by_sent_date=sort_by_sent_date, + ) + expected_shape = _preorder(batch) + assert _preorder(index.roots) == expected_shape + assert tuple( + projection.message_keys for projection in index.projections + ) == tuple( + tuple(key for key in root if key is not None) + for root in expected_shape + ) diff --git a/tests/test_incremental_private_graph.py b/tests/test_incremental_private_graph.py new file mode 100644 index 0000000..e2f45a8 --- /dev/null +++ b/tests/test_incremental_private_graph.py @@ -0,0 +1,376 @@ +"""Defensive graph-branch coverage for the incremental index.""" + +from __future__ import annotations + +import pytest + +import threadweave.incremental as incremental +from threadweave import ( + Container, + IncrementalThreadError, + IncrementalThreadIndex, + IndexedMessage, + MailboxChangeSet, + Message, + ThreadProjection, +) + + +def _record(key: str, message: Message | None = None, **identity: str) -> IndexedMessage: + """Build one record for focused graph tests.""" + return IndexedMessage( + key, + Message(message_id=key) if message is None else message, + **identity, + ) + + +def test_duplicate_references_missing_id_and_consistent_email_id_are_supported(): + """Deduplication and optional ID branches preserve valid historical mail.""" + index = IncrementalThreadIndex() + index.apply( + MailboxChangeSet( + expected_version=0, + additions=( + _record( + "a", + Message(message_id=None, references=("", "")), + email_id="M1", + thread_id="T1", + ), + _record( + "b", + Message(message_id="b"), + email_id="M1", + thread_id="T1", + ), + ), + ) + ) + assert index.message_keys == ("a", "b") + + +def test_candidate_expansion_adds_neighbors_reached_through_tokens(): + """The iterative candidate queue crosses every current reverse bucket.""" + assert incremental._expand_candidate_keys( + {"a"}, + {"a": frozenset({"token"}), "b": frozenset({"token"})}, + {"token": {"a", "b"}}, + ) == {"a", "b"} + + +def test_projection_defenses_cover_dummy_cycle_and_foreign_messages(): + """Private batch-output validation remains loop-safe and fail-closed.""" + message = Message(message_id="a") + record = _record("a", message) + concrete = Container(message=message) + concrete.children = [concrete] + projection = incremental._projection_for_root( + concrete, + {id(message): "a"}, + {"a": record}, + ) + assert projection.message_keys == ("a",) + + dummy = Container() + child = Container(message=message) + dummy.children = [child] + assert incremental._projection_for_root( + dummy, + {id(message): "a"}, + {"a": record}, + ).message_keys == ("a",) + + with pytest.raises(IncrementalThreadError, match="outside its component"): + incremental._projection_for_root(child, {}, {"a": record}) + + + + +def test_thread_delta_scans_projection_sequences_a_bounded_number_of_times(): + """Large independent forests avoid pairwise projection comparisons.""" + + class CountingSequence: + """Wrap projections and record full-sequence operations.""" + + def __init__(self, values: tuple[ThreadProjection, ...]) -> None: + """Store values with zeroed iteration and membership counters.""" + self.values = values + self.iterations = 0 + self.membership_checks = 0 + + def __iter__(self): + """Iterate while recording one full-sequence pass.""" + self.iterations += 1 + return iter(self.values) + + def __len__(self) -> int: + """Return the wrapped projection count.""" + return len(self.values) + + def __getitem__(self, index: int) -> ThreadProjection: + """Return one wrapped projection by position.""" + return self.values[index] + + def __contains__(self, value: object) -> bool: + """Record whole-sequence membership checks.""" + self.membership_checks += 1 + return value in self.values + + projection_count = 128 + values = tuple( + ThreadProjection((f"message_{index}",)) + for index in range(projection_count) + ) + before = CountingSequence(values) + after = CountingSequence(values + (ThreadProjection(("new_message",)),)) + + delta = incremental._thread_delta(0, 1, ("new_message",), before, after) + + assert delta.added_threads == (ThreadProjection(("new_message",)),) + assert before.iterations <= 2 + assert after.iterations <= 2 + assert before.membership_checks == 0 + assert after.membership_checks == 0 + + +def test_projection_membership_rejects_duplicate_keys_between_roots(): + """Malformed projections cannot make merge/split classification ambiguous.""" + with pytest.raises(IncrementalThreadError, match="duplicate message_key"): + incremental._thread_delta( + 0, + 1, + (), + ( + ThreadProjection(("shared",)), + ThreadProjection(("shared",)), + ), + (), + ) + + +def test_reverse_token_buckets_are_copied_only_when_mutated(): + """Atomic changes preserve every shared pre-transaction bucket.""" + original = {"token": {"a", "b"}} + updates: dict[str, set[str]] = {} + + incremental._remove_key_from_buckets( + "a", + ("token",), + original, + updates, + ) + incremental._add_key_to_buckets( + "c", + ("token",), + original, + updates, + ) + + assert original == {"token": {"a", "b"}} + assert updates == {"token": {"b", "c"}} + incremental._commit_bucket_updates(original, updates) + assert original == {"token": {"b", "c"}} + incremental._commit_bucket_updates(original, {"token": set()}) + assert original == {} + + +def test_component_partition_reads_positions_linearly_for_disconnected_keys(): + """Disconnected mailboxes avoid a repeated-minimum quadratic scan.""" + + class CountingPositions(dict[str, int]): + """Count position lookups performed by the partitioner.""" + + def __init__(self, values: dict[str, int]) -> None: + """Create the mapping with a zeroed lookup counter.""" + super().__init__(values) + self.lookups = 0 + + def __getitem__(self, key: str) -> int: + """Return one position while recording algorithmic work.""" + self.lookups += 1 + return super().__getitem__(key) + + key_count = 128 + keys = {f"message_{index}" for index in range(key_count)} + positions = CountingPositions( + {key: index for index, key in enumerate(sorted(keys), start=1)} + ) + components = incremental._partition_components( + keys, + positions, + {key: frozenset() for key in keys}, + {}, + ) + + assert len(components) == key_count + assert positions.lookups <= key_count * 2 + + +def test_public_forest_copy_rejects_shared_and_cyclic_internal_nodes(): + """Defensive copies fail closed if derived graph invariants are corrupted.""" + index = IncrementalThreadIndex() + shared = Container(message=Message(message_id="shared")) + index._roots = (shared, shared) + with pytest.raises(IncrementalThreadError, match="shared or cyclic"): + _ = index.roots + + cyclic = Container(message=Message(message_id="cyclic")) + cyclic.children = [cyclic] + index._roots = (cyclic,) + with pytest.raises(IncrementalThreadError, match="shared or cyclic"): + _ = index.roots + + root = Container(message=Message(message_id="root")) + root.children = [Container(parent=root)] + index._roots = (root,) + copied = index.roots + assert copied[0].children[0].message is None + assert copied[0].children[0].parent is copied[0] + + +def test_replacement_recovers_when_derived_component_mapping_is_missing(): + """A defensive update rebuilds a record after derived-state loss.""" + index = IncrementalThreadIndex() + index.apply(MailboxChangeSet(expected_version=0, additions=(_record("a"),))) + index._component_by_key.pop("a") + + index.apply( + MailboxChangeSet( + expected_version=1, + replacements=( + _record("a", Message(message_id="a", subject="updated")), + ), + ) + ) + assert index.projections[0].message_keys == ("a",) + + +def test_default_delta_does_not_copy_or_iterate_unrelated_state_maps(): + """A small default-mode update touches bounded state instead of the whole mailbox.""" + + class NoFullIterationDict(dict[str, object]): + """Permit keyed access and mutation while rejecting whole-map scans.""" + + def __iter__(self): + """Reject direct iteration over unrelated state.""" + raise AssertionError("unexpected full-state iteration") + + def keys(self): + """Reject key-view scans over unrelated state.""" + raise AssertionError("unexpected full-state key scan") + + def items(self): + """Reject item-view scans over unrelated state.""" + raise AssertionError("unexpected full-state item scan") + + def values(self): + """Reject value-view scans over unrelated state.""" + raise AssertionError("unexpected full-state value scan") + + records = tuple( + _record( + f"message_{index}", + Message(message_id=f"message_{index}"), + email_id=f"Email_{index}", + thread_id=f"Thread_{index}", + ) + for index in range(128) + ) + index = IncrementalThreadIndex() + index.apply(MailboxChangeSet(expected_version=0, additions=records)) + + state_names = ( + "_records", + "_positions", + "_tokens_by_key", + "_keys_by_token", + "_email_id_states", + "_thread_id_counts", + "_component_by_key", + "_keys_by_component", + ) + for name in state_names: + setattr(index, name, NoFullIterationDict(getattr(index, name))) + state_identities = {name: id(getattr(index, name)) for name in state_names} + + delta = index.apply( + MailboxChangeSet( + expected_version=1, + replacements=( + _record( + "message_0", + Message(message_id="message_0", subject="updated"), + email_id="Email_0", + thread_id="Thread_0", + ), + ), + ) + ) + + assert delta.affected_message_keys == ("message_0",) + assert index.version == 2 + assert {name: id(getattr(index, name)) for name in state_names} == state_identities + + +def test_external_identity_indexes_update_only_touched_namespaces(): + """Identity removals and additions remain atomic without a mailbox-wide scan.""" + index = IncrementalThreadIndex() + index.apply( + MailboxChangeSet( + expected_version=0, + additions=( + _record("a", email_id="Mail_A", thread_id="Thread_A"), + _record("b", email_id="Mail_B", thread_id="Thread_B"), + ), + ) + ) + + index.apply( + MailboxChangeSet( + expected_version=1, + removals=("a",), + additions=( + _record("c", email_id="Thread_A", thread_id="Mail_A"), + ), + ) + ) + + assert index._email_id_states == { + "Mail_B": ("Thread_B", 1), + "Thread_A": ("Mail_A", 1), + } + assert index._thread_id_counts == {"Thread_B": 1, "Mail_A": 1} + + before = index.snapshot() + with pytest.raises(IncrementalThreadError, match="disjoint ObjectID"): + index.apply( + MailboxChangeSet( + expected_version=2, + additions=( + _record("d", email_id="Thread_B", thread_id="Thread_D"), + ), + ) + ) + assert index.snapshot() == before + + +def test_external_identity_index_corruption_fails_before_commit(): + """Broken private identity indexes cannot be published by a transaction.""" + index = IncrementalThreadIndex() + index.apply( + MailboxChangeSet( + expected_version=0, + additions=( + _record("a", email_id="Mail_A", thread_id="Thread_A"), + ), + ) + ) + index._email_id_states.clear() + with pytest.raises(IncrementalThreadError, match="EMAILID index"): + index.apply(MailboxChangeSet(expected_version=1, removals=("a",))) + + index._email_id_states["Mail_A"] = ("Thread_A", 1) + index._thread_id_counts.clear() + with pytest.raises(IncrementalThreadError, match="THREADID index"): + index.apply(MailboxChangeSet(expected_version=1, removals=("a",))) diff --git a/tests/test_incremental_rfc8474.py b/tests/test_incremental_rfc8474.py new file mode 100644 index 0000000..98da780 --- /dev/null +++ b/tests/test_incremental_rfc8474.py @@ -0,0 +1,63 @@ +"""RFC 8474 identity grammar and namespace tests for incremental threading.""" + +import pytest + +from threadweave import ( + ExternalIdentityError, + IncrementalThreadError, + IncrementalThreadIndex, + IndexedMessage, + MailboxChangeSet, + Message, +) + + +def _record( + key: str, + *, + email_id: str | None = None, + thread_id: str | None = None, +) -> IndexedMessage: + """Build one message carrying optional RFC 8474 identity metadata.""" + return IndexedMessage( + message_key=key, + message=Message(message_id=key), + email_id=email_id, + thread_id=thread_id, + ) + + +def test_object_ids_use_exact_ascii_grammar_and_length(): + """EMAILID and THREADID accept only 1-255 RFC ``objectid`` characters.""" + valid = "Abc_012-Z" + record = _record("a", email_id=valid, thread_id="Thread_1") + assert record.email_id == valid + assert record.thread_id == "Thread_1" + + for field in ("email_id", "thread_id"): + for invalid in ("contains space", "é", "x" * 256, "slash/value"): + with pytest.raises(IncrementalThreadError, match=field): + _record("a", **{field: invalid}) + + +def test_email_id_and_thread_id_namespaces_are_disjoint(): + """One ObjectID value cannot be reused across EMAILID and THREADID data items.""" + index = IncrementalThreadIndex() + with pytest.raises(ExternalIdentityError, match="disjoint ObjectID"): + index.apply( + MailboxChangeSet( + expected_version=0, + additions=( + _record("a", email_id="Shared1", thread_id="Thread1"), + _record("b", email_id="Mail2", thread_id="Shared1"), + ), + ) + ) + assert index.version == 0 + + +def test_message_keys_reject_nonprintable_unicode_without_encoding_failure(): + """Caller keys fail during validation instead of later UTF-8 serialization.""" + for invalid in ("bad\ud800key", "line\u2028separator"): + with pytest.raises(IncrementalThreadError, match="message_key.*printable"): + _record(invalid) diff --git a/tests/test_incremental_snapshot.py b/tests/test_incremental_snapshot.py new file mode 100644 index 0000000..8bb237a --- /dev/null +++ b/tests/test_incremental_snapshot.py @@ -0,0 +1,557 @@ +"""Versioned JSON-safe snapshot tests for the incremental index.""" + +from __future__ import annotations + +import json +import sys +from copy import deepcopy +from datetime import datetime, timezone + +import pytest +import threadweave.incremental as incremental_module + +from threadweave import ( + IncrementalThreadError, + IncrementalThreadIndex, + IndexedMessage, + MailboxChangeSet, + Message, +) + + +def _index() -> IncrementalThreadIndex: + """Return a representative index containing every serializable field.""" + index = IncrementalThreadIndex( + group_by_subject=True, + sort_by_sent_date=True, + max_snapshot_records=100, + max_snapshot_bytes=100_000, + ) + index.apply( + MailboxChangeSet( + expected_version=0, + additions=( + IndexedMessage( + message_key="root_key", + message=Message( + message_id="", + subject="Topic", + payload={"must": "not persist"}, + sent_date=datetime(2026, 1, 1, tzinfo=timezone.utc), + internal_date="1 Jan 2026 00:00:00 +0000", + sequence_number=1, + uid=101, + ), + email_id="M1", + thread_id="T1", + ), + IndexedMessage( + message_key="child_key", + message=Message( + message_id="", + references="", + in_reply_to="", + subject="Re: Topic", + payload=object(), + sent_date=datetime(2026, 1, 2), + sequence_number=2, + uid=102, + ), + email_id="M2", + thread_id="T1", + ), + ), + ) + ) + return index + + +def test_snapshot_is_deterministic_json_safe_and_omits_payloads(): + """State serializes reproducibly without arbitrary caller objects.""" + index = _index() + + first = index.snapshot() + second = index.snapshot() + + assert first == second + encoded = json.dumps(first, sort_keys=True, separators=(",", ":")) + assert "must not persist" not in encoded + assert "payload" not in encoded + assert first["schema_version"] == 1 + assert first["version"] == 1 + assert first["options"] == { + "group_by_subject": True, + "sort_by_sent_date": True, + } + assert [record["message_key"] for record in first["records"]] == [ + "root_key", + "child_key", + ] + + +def test_snapshot_restore_round_trip_preserves_structure_and_version(): + """Restored derived state matches the source while payloads become None.""" + source = _index() + snapshot = source.snapshot() + + restored = IncrementalThreadIndex.restore( + snapshot, + max_snapshot_records=100, + max_snapshot_bytes=100_000, + ) + + assert restored.version == source.version + assert restored.message_keys == source.message_keys + assert restored.projections == source.projections + assert restored.snapshot() == snapshot + assert all( + node.message.payload is None + for root in restored.roots + for node in (root, *tuple(root.iter_descendants())) + if node.message is not None + ) + + +def test_restored_index_continues_with_optimistic_versioning(): + """A restored revision accepts the next atomic change exactly once.""" + restored = IncrementalThreadIndex.restore(_index().snapshot()) + delta = restored.apply( + MailboxChangeSet( + expected_version=1, + additions=( + IndexedMessage( + "new_key", + Message(message_id="new", payload="not persisted"), + ), + ), + ) + ) + + assert (delta.previous_version, delta.version) == (1, 2) + assert restored.version == 2 + + +def test_restore_rejects_unknown_root_fields_and_schema_versions(): + """Untrusted snapshots use an exact schema rather than permissive decoding.""" + snapshot = _index().snapshot() + + unknown = deepcopy(snapshot) + unknown["unexpected"] = True + with pytest.raises(IncrementalThreadError, match="snapshot fields"): + IncrementalThreadIndex.restore(unknown) + + unsupported = deepcopy(snapshot) + unsupported["schema_version"] = 2 + with pytest.raises(IncrementalThreadError, match="schema_version"): + IncrementalThreadIndex.restore(unsupported) + + missing = deepcopy(snapshot) + del missing["records"] + with pytest.raises(IncrementalThreadError, match="snapshot fields"): + IncrementalThreadIndex.restore(missing) + + +@pytest.mark.parametrize("invalid_schema_version", [True, 1.0]) +def test_restore_requires_an_exact_integer_schema_version( + invalid_schema_version: object, +): + """Boolean and floating-point lookalikes cannot select a snapshot schema.""" + snapshot = _index().snapshot() + snapshot["schema_version"] = invalid_schema_version + + with pytest.raises(IncrementalThreadError, match="schema_version"): + IncrementalThreadIndex.restore(snapshot) + + +def test_snapshot_reports_unencodable_unicode_as_a_domain_error(): + """Lone surrogates fail closed instead of leaking a codec exception.""" + index = IncrementalThreadIndex() + index.apply( + MailboxChangeSet( + expected_version=0, + additions=( + IndexedMessage( + "surrogate_key", + Message(message_id="surrogate", subject="bad\ud800subject"), + ), + ), + ) + ) + + with pytest.raises(IncrementalThreadError, match="JSON-safe"): + index.snapshot() + + +def test_restore_reports_excessive_json_nesting_as_a_domain_error(): + """Hostile nesting cannot escape the snapshot boundary as RecursionError.""" + nested: object = [] + for _ in range(sys.getrecursionlimit() * 20): + nested = [nested] + snapshot = { + "schema_version": 1, + "version": 0, + "options": { + "group_by_subject": False, + "sort_by_sent_date": False, + }, + "records": nested, + } + + with pytest.raises(IncrementalThreadError, match="JSON-safe"): + IncrementalThreadIndex.restore(snapshot) + + +def test_restore_rejects_malformed_record_fields_and_duplicate_keys(): + """Record decoding rejects ambiguity before constructing any graph state.""" + snapshot = _index().snapshot() + + duplicate = deepcopy(snapshot) + duplicate["records"].append(deepcopy(duplicate["records"][0])) + with pytest.raises(IncrementalThreadError, match="duplicate.*message_key"): + IncrementalThreadIndex.restore(duplicate) + + extra = deepcopy(snapshot) + extra["records"][0]["unexpected"] = "value" + with pytest.raises(IncrementalThreadError, match="record fields"): + IncrementalThreadIndex.restore(extra) + + invalid_records = deepcopy(snapshot) + invalid_records["records"] = "not-a-list" + with pytest.raises(IncrementalThreadError, match="records"): + IncrementalThreadIndex.restore(invalid_records) + + invalid_message = deepcopy(snapshot) + invalid_message["records"][0]["message"] = [] + with pytest.raises(IncrementalThreadError, match="message"): + IncrementalThreadIndex.restore(invalid_message) + + +def test_restore_rejects_invalid_date_tags_and_numeric_metadata(): + """Date and IMAP metadata preserve strict tagged and integer contracts.""" + snapshot = _index().snapshot() + + invalid_date = deepcopy(snapshot) + invalid_date["records"][0]["message"]["sent_date"] = { + "kind": "unknown", + "value": "2026-01-01", + } + with pytest.raises(IncrementalThreadError, match="date"): + IncrementalThreadIndex.restore(invalid_date) + + invalid_datetime = deepcopy(snapshot) + invalid_datetime["records"][0]["message"]["sent_date"] = { + "kind": "datetime", + "value": "not-a-date", + } + with pytest.raises(IncrementalThreadError, match="datetime"): + IncrementalThreadIndex.restore(invalid_datetime) + + invalid_sequence = deepcopy(snapshot) + invalid_sequence["records"][0]["message"]["sequence_number"] = True + with pytest.raises(IncrementalThreadError, match="sequence_number"): + IncrementalThreadIndex.restore(invalid_sequence) + + +def test_restore_and_snapshot_enforce_record_and_byte_limits(): + """Configured bounds stop oversized state before allocation or publication.""" + snapshot = _index().snapshot() + + with pytest.raises(IncrementalThreadError, match="max_snapshot_records"): + IncrementalThreadIndex.restore(snapshot, max_snapshot_records=1) + with pytest.raises(IncrementalThreadError, match="max_snapshot_bytes"): + IncrementalThreadIndex.restore(snapshot, max_snapshot_bytes=10) + + index = IncrementalThreadIndex(max_snapshot_bytes=10) + index.apply( + MailboxChangeSet( + expected_version=0, + additions=(IndexedMessage("a", Message(message_id="a")),), + ) + ) + with pytest.raises(IncrementalThreadError, match="max_snapshot_bytes"): + index.snapshot() + + +def test_restore_rejects_non_mapping_and_non_json_safe_values(): + """The public boundary converts encoder failures into domain errors.""" + with pytest.raises(IncrementalThreadError, match="mapping"): + IncrementalThreadIndex.restore([]) # type: ignore[arg-type] + + snapshot = _index().snapshot() + invalid = deepcopy(snapshot) + invalid["records"][0]["email_id"] = object() + with pytest.raises(IncrementalThreadError, match="JSON"): + IncrementalThreadIndex.restore(invalid) + + +class _HostileDictionary(dict): + """Dictionary subclass whose iteration must never cross the snapshot boundary.""" + + def items(self): + """Raise if generic JSON encoding invokes subclass behavior.""" + raise RuntimeError("hostile dictionary iteration") + + +class _HostileList(list): + """List subclass whose iteration must never cross the snapshot boundary.""" + + def __iter__(self): + """Raise if generic JSON encoding invokes subclass behavior.""" + raise RuntimeError("hostile list iteration") + + +class _HostileString(str): + """String subclass whose comparisons must not run during sorted encoding.""" + + def __lt__(self, _other: object) -> bool: + """Reject less-than comparison if sorted encoding reaches this key.""" + raise TypeError("hostile string comparison") + + def __le__(self, _other: object) -> bool: + """Reject less-than-or-equal comparison if encoding reaches this key.""" + raise TypeError("hostile string comparison") + + def __gt__(self, _other: object) -> bool: + """Reject greater-than comparison if sorted encoding reaches this key.""" + raise TypeError("hostile string comparison") + + def __ge__(self, _other: object) -> bool: + """Reject greater-than-or-equal comparison if encoding reaches this key.""" + raise TypeError("hostile string comparison") + + +def test_restore_rejects_container_subclasses_before_they_execute(): + """Only plain JSON containers may enter the untrusted snapshot decoder.""" + root = _HostileDictionary( + { + "schema_version": 1, + "version": 0, + "options": { + "group_by_subject": False, + "sort_by_sent_date": False, + }, + "records": [], + } + ) + with pytest.raises(IncrementalThreadError, match="plain JSON containers"): + IncrementalThreadIndex.restore(root) + + nested = { + "schema_version": 1, + "version": 0, + "options": { + "group_by_subject": False, + "sort_by_sent_date": False, + }, + "records": _HostileList(), + } + with pytest.raises(IncrementalThreadError, match="plain JSON containers"): + IncrementalThreadIndex.restore(nested) + + +def test_plain_json_guard_rejects_executable_key_and_scalar_subclasses(): + """Sorted JSON encoding cannot invoke attacker-controlled scalar methods.""" + hostile_key = _HostileString("unexpected") + with pytest.raises(IncrementalThreadError, match="plain strings"): + incremental_module._require_plain_json_containers( + {hostile_key: "value", "schema_version": 1} + ) + + hostile_snapshot = { + "schema_version": 1, + "version": 0, + "options": { + "group_by_subject": False, + "sort_by_sent_date": False, + }, + _HostileString("records"): [], + } + with pytest.raises(IncrementalThreadError, match="plain strings"): + IncrementalThreadIndex.restore(hostile_snapshot) + + with pytest.raises(IncrementalThreadError, match="scalar values"): + incremental_module._require_plain_json_containers( + {"value": _HostileString("hostile")} + ) + + +def test_plain_container_guard_rejects_cycles_and_reused_containers(): + """JSON snapshots are trees: cycles and shared container identities fail closed.""" + mapping_cycle: dict[str, object] = {} + mapping_cycle["self"] = mapping_cycle + mapping_snapshot = { + "schema_version": 1, + "version": 0, + "options": { + "group_by_subject": False, + "sort_by_sent_date": False, + }, + "records": [mapping_cycle], + } + with pytest.raises(IncrementalThreadError, match="cyclic"): + IncrementalThreadIndex.restore(mapping_snapshot) + + list_cycle: list[object] = [] + list_cycle.append(list_cycle) + nested_cycle = { + "schema_version": 1, + "version": 0, + "options": { + "group_by_subject": False, + "sort_by_sent_date": False, + }, + "records": list_cycle, + } + with pytest.raises(IncrementalThreadError, match="cyclic"): + IncrementalThreadIndex.restore(nested_cycle) + + shared: list[object] = [{"value": 1}] + with pytest.raises(IncrementalThreadError, match="reused"): + incremental_module._require_plain_json_containers( + {"first": shared, "second": shared} + ) + + compact_graph: object = [] + for _ in range(28): + compact_graph = [compact_graph, compact_graph] + hostile_snapshot = { + "schema_version": 1, + "version": 0, + "options": { + "group_by_subject": False, + "sort_by_sent_date": False, + }, + "records": compact_graph, + } + with pytest.raises(IncrementalThreadError, match="reused"): + IncrementalThreadIndex.restore(hostile_snapshot) + + +def test_utf8_size_counting_avoids_chunk_encoding(monkeypatch): + """Large JSON chunks are counted without allocating an encoded bytes copy.""" + + class _NoEncodeString(str): + """Raise if the size checker calls ``str.encode`` on an encoder chunk.""" + + def encode( + self, + encoding: str = "utf-8", + errors: str = "strict", + ) -> bytes: + """Expose accidental full-chunk byte allocation as a test failure.""" + raise AssertionError((encoding, errors)) + + class _SingleChunkEncoder: + """Yield one non-ASCII JSON chunk whose ``encode`` method is forbidden.""" + + def __init__(self, **_options: object) -> None: + """Accept the production encoder configuration.""" + + def iterencode(self, _value: object): + """Yield one representative built JSON string chunk.""" + yield _NoEncodeString('"aé€😀"') + + monkeypatch.setattr(incremental_module.json, "JSONEncoder", _SingleChunkEncoder) + + assert incremental_module._bounded_snapshot_json_size({}, 100) == 12 + + +def test_snapshot_size_validation_stops_at_the_utf8_limit(monkeypatch): + """Byte-limit enforcement stops the incremental encoder before later chunks.""" + later_chunks_requested: list[bool] = [] + + class _ChunkedEncoder: + """Yield one oversized UTF-8 chunk, then fail if iteration continues.""" + + def __init__(self, **_options: object) -> None: + """Accept the production encoder options used by the helper.""" + + def iterencode(self, _value: object): + """Yield one four-byte chunk and record any forbidden second request.""" + yield '"é"' + later_chunks_requested.append(True) + raise AssertionError("encoder continued after the configured byte limit") + + monkeypatch.setattr(incremental_module.json, "JSONEncoder", _ChunkedEncoder) + + with pytest.raises(IncrementalThreadError, match="max_snapshot_bytes"): + incremental_module._bounded_snapshot_json_size({}, 3) + assert later_chunks_requested == [] + + +def test_restore_checks_record_limit_before_nested_record_values(): + """The record-count limit rejects an oversized list before nested traversal.""" + snapshot = { + "schema_version": 1, + "version": 0, + "options": { + "group_by_subject": False, + "sort_by_sent_date": False, + }, + "records": [object(), object()], + } + + with pytest.raises(IncrementalThreadError, match="max_snapshot_records"): + IncrementalThreadIndex.restore( + snapshot, + max_snapshot_records=1, + max_snapshot_bytes=10_000, + ) + + +def test_restore_checks_root_fields_before_untrusted_extra_values(): + """Unknown root fields fail before their nested values are traversed.""" + snapshot = { + "schema_version": 1, + "version": 0, + "options": { + "group_by_subject": False, + "sort_by_sent_date": False, + }, + "records": [], + "unexpected": [object()], + } + + with pytest.raises(IncrementalThreadError, match="snapshot fields"): + IncrementalThreadIndex.restore(snapshot) + + same_size_wrong_fields = { + "schema_version": 1, + "version": 0, + "options": { + "group_by_subject": False, + "sort_by_sent_date": False, + }, + "unexpected": None, + } + with pytest.raises(IncrementalThreadError, match="snapshot fields"): + IncrementalThreadIndex.restore(same_size_wrong_fields) + + +def test_restore_bounds_plain_container_scan_by_snapshot_byte_limit(monkeypatch): + """A small byte limit stops structural traversal before JSON encoding.""" + + class _ForbiddenEncoder: + """Expose an attempt to encode after the structural limit is exceeded.""" + + def __init__(self, **_options: object) -> None: + """Fail because bounded validation must reject before encoding.""" + raise AssertionError("JSON encoder should not be constructed") + + monkeypatch.setattr(incremental_module.json, "JSONEncoder", _ForbiddenEncoder) + snapshot = { + "schema_version": 1, + "version": 0, + "options": { + "group_by_subject": False, + "sort_by_sent_date": False, + }, + "records": [None] * 100, + } + + with pytest.raises(IncrementalThreadError, match="max_snapshot_bytes"): + IncrementalThreadIndex.restore( + snapshot, + max_snapshot_records=200, + max_snapshot_bytes=10, + ) diff --git a/tests/test_incremental_validation.py b/tests/test_incremental_validation.py new file mode 100644 index 0000000..102f8dc --- /dev/null +++ b/tests/test_incremental_validation.py @@ -0,0 +1,144 @@ +"""Defensive validation coverage for incremental mailbox boundaries.""" + +from __future__ import annotations + +from copy import deepcopy + +import pytest + +from threadweave import ( + IncrementalThreadError, + IncrementalThreadIndex, + IndexedMessage, + MailboxChangeSet, + Message, +) + + +def _record(key: str, message: Message | None = None, **identity: str) -> IndexedMessage: + """Build one record for focused validation tests.""" + return IndexedMessage( + key, + Message(message_id=key) if message is None else message, + **identity, + ) + + +def test_public_containers_reject_wrong_element_and_message_types(): + """Runtime boundaries reject values hidden behind static annotations.""" + with pytest.raises(IncrementalThreadError, match="IndexedMessage"): + MailboxChangeSet( + expected_version=0, + additions=(object(),), # type: ignore[arg-type] + ) + with pytest.raises(IncrementalThreadError, match="threadweave.Message"): + IndexedMessage("a", object()) # type: ignore[arg-type] + + +def test_apply_rejects_invalid_message_metadata_before_mutation(): + """Malformed text, references, dates, and IMAP numbers fail atomically.""" + bad_messages = ( + Message(message_id=object()), # type: ignore[arg-type] + Message(subject=object()), # type: ignore[arg-type] + Message(references=1), # type: ignore[arg-type] + Message(references=("ok", 1)), # type: ignore[arg-type] + Message(sent_date=object()), # type: ignore[arg-type] + Message(uid=4_294_967_296), + ) + for position, message in enumerate(bad_messages): + index = IncrementalThreadIndex() + with pytest.raises(IncrementalThreadError): + index.apply( + MailboxChangeSet( + expected_version=0, + additions=(_record(f"bad_{position}", message),), + ) + ) + assert index.version == 0 + + +def test_constructor_and_apply_reject_wrong_runtime_boundary_types(): + """Boolean options and arbitrary change objects never enter mutable state.""" + with pytest.raises(IncrementalThreadError, match="group_by_subject"): + IncrementalThreadIndex(group_by_subject=1) # type: ignore[arg-type] + with pytest.raises(IncrementalThreadError, match="sort_by_sent_date"): + IncrementalThreadIndex(sort_by_sent_date=1) # type: ignore[arg-type] + with pytest.raises(IncrementalThreadError, match="MailboxChangeSet"): + IncrementalThreadIndex().apply(object()) # type: ignore[arg-type] + + +def test_snapshot_record_limit_is_enforced_at_publication_time(): + """A lowered publication bound cannot emit an oversized state document.""" + index = IncrementalThreadIndex(max_snapshot_records=1) + index.apply( + MailboxChangeSet( + expected_version=0, + additions=(_record("a"), _record("b")), + ) + ) + with pytest.raises(IncrementalThreadError, match="max_snapshot_records"): + index.snapshot() + + +def test_restore_rejects_remaining_shape_and_scalar_ambiguities(): + """Nested snapshot fields use exact mapping, list, and scalar types.""" + source = IncrementalThreadIndex() + source.apply(MailboxChangeSet(expected_version=0, additions=(_record("a"),))) + snapshot = source.snapshot() + + invalid_values: list[tuple[dict[str, object], str]] = [] + options_not_mapping = deepcopy(snapshot) + options_not_mapping["options"] = [] + invalid_values.append((options_not_mapping, "options")) + + invalid_group_option = deepcopy(snapshot) + invalid_group_option["options"]["group_by_subject"] = 1 # type: ignore[index] + invalid_values.append((invalid_group_option, "group_by_subject")) + + invalid_sort_option = deepcopy(snapshot) + invalid_sort_option["options"]["sort_by_sent_date"] = 1 # type: ignore[index] + invalid_values.append((invalid_sort_option, "sort_by_sent_date")) + + record_not_mapping = deepcopy(snapshot) + record_not_mapping["records"] = [1] + invalid_values.append((record_not_mapping, "record")) + + invalid_reply = deepcopy(snapshot) + invalid_reply["records"][0]["message"]["in_reply_to"] = [1] # type: ignore[index] + invalid_values.append((invalid_reply, "in_reply_to")) + + invalid_references = deepcopy(snapshot) + invalid_references["records"][0]["message"]["references"] = [1] # type: ignore[index] + invalid_values.append((invalid_references, "references")) + + malformed_date = deepcopy(snapshot) + malformed_date["records"][0]["message"]["sent_date"] = [] # type: ignore[index] + invalid_values.append((malformed_date, "date")) + + nontext_date = deepcopy(snapshot) + nontext_date["records"][0]["message"]["sent_date"] = { # type: ignore[index] + "kind": "text", + "value": 1, + } + invalid_values.append((nontext_date, "textual")) + + for invalid, pattern in invalid_values: + with pytest.raises(IncrementalThreadError, match=pattern): + IncrementalThreadIndex.restore(invalid) + + +def test_empty_snapshot_restore_skips_initial_rebuild(): + """A versioned empty mailbox restores without a fabricated change.""" + restored = IncrementalThreadIndex.restore( + { + "schema_version": 1, + "version": 7, + "options": { + "group_by_subject": False, + "sort_by_sent_date": False, + }, + "records": [], + } + ) + assert restored.version == 7 + assert restored.message_keys == ()