feat: add incremental mailbox threading with stable identity handoff

.github/workflows/ci.yml

@@ -56,7 +56,7 @@ jobs:
       - name: Install the reviewed CI lock
         run: python -m pip install --require-hashes -r requirements/ci.lock
       - run: ruff check .
-      - run: python -m compileall -q src tests scripts
+      - run: python -m compileall -q src tests scripts benchmarks
       - name: Run production doctests
         run: |
           python -m doctest \
@@ -143,6 +143,9 @@ jobs:
           cd "$temp_dir"
           python - <<'PY'
           from threadweave import (
+              IncrementalThreadIndex,
+              IndexedMessage,
+              MailboxChangeSet,
               Message,
               __version__,
               serialize_thread_response,
@@ -170,6 +173,39 @@ jobs:
               "later@example.com",
           ]
           assert serialize_thread_response(roots) == "* THREAD (1)(2)\r\n"
+
+          index = IncrementalThreadIndex()
+          delta = index.apply(
+              MailboxChangeSet(
+                  expected_version=0,
+                  additions=(
+                      IndexedMessage(
+                          "root-key",
+                          Message(message_id="root", sequence_number=1, uid=101),
+                          email_id="Email_101",
+                          thread_id="Thread_7",
+                      ),
+                      IndexedMessage(
+                          "reply-key",
+                          Message(
+                              message_id="reply",
+                              references=["root"],
+                              sequence_number=2,
+                              uid=102,
+                          ),
+                          email_id="Email_102",
+                          thread_id="Thread_7",
+                      ),
+                  ),
+              )
+          )
+          assert delta.version == 1
+          assert index.projections[0].message_keys == ("root-key", "reply-key")
+          assert serialize_thread_response(index.roots, identifier="uid") == (
+              "* THREAD (101 102)\r\n"
+          )
+          restored = IncrementalThreadIndex.restore(index.snapshot())
+          assert restored.projections == index.projections
           PY
       - run: python -m pip check
       - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1

.github/workflows/diagnose-hourly-product-workflow.yml

@@ -0,0 +1,69 @@
+name: Diagnose repository workflows
+
+on:
+  push:
+    branches:
+      - feature/incremental-thread-index
+    paths:
+      - .github/workflows/diagnose-hourly-product-workflow.yml
+
+permissions:
+  contents: read
+
+jobs:
+  actionlint:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 10
+    steps:
+      - name: Check out the exact branch
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: feature/incremental-thread-index
+          persist-credentials: false
+
+      - name: Install pinned actionlint
+        shell: bash
+        run: |
+          set -euo pipefail
+          archive="${RUNNER_TEMP}/actionlint.tar.gz"
+          curl -fsSL \
+            -o "$archive" \
+            https://github.com/rhysd/actionlint/releases/download/v1.7.12/actionlint_1.7.12_linux_amd64.tar.gz
+          printf '%s  %s\n' \
+            8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8 \
+            "$archive" | sha256sum -c -
+          tar -xzf "$archive" -C "$RUNNER_TEMP" actionlint
+
+      - name: Apply known corrections only in the diagnostic workspace
+        shell: bash
+        run: |
+          set -euo pipefail
+          python - <<'PY'
+          from pathlib import Path
+
+          path = Path('.github/workflows/hourly-product-development.yml')
+          text = path.read_text(encoding='utf-8')
+          indentation_old = "          printf 'threadweave @ file://%s \\\n    --hash=sha256:%s\\n' \\\n"
+          indentation_new = "          printf 'threadweave @ file://%s \\\n            --hash=sha256:%s\\n' \\\n"
+          if text.count(indentation_old) != 1:
+              raise SystemExit('unexpected hourly workflow indentation state')
+          text = text.replace(indentation_old, indentation_new)
+
+          shellcheck_old = "            if timeout --kill-after=30s \"${OPENCODE_RUN_TIMEOUT_SECONDS}s\" \\\n"
+          shellcheck_new = (
+              "            # shellcheck disable=SC2016\n"
+              "            if timeout --kill-after=30s \"${OPENCODE_RUN_TIMEOUT_SECONDS}s\" \\\n"
+          )
+          if text.count(shellcheck_old) != 1:
+              raise SystemExit('unexpected nested-shell state')
+          path.write_text(text.replace(shellcheck_old, shellcheck_new), encoding='utf-8')
+          PY
+
+      - name: Diagnose every repository workflow after known corrections
+        shell: bash
+        run: |
+          set -o pipefail
+          "${RUNNER_TEMP}/actionlint" \
+            -color=false \
+            .github/workflows/*.yml \
+            2>&1 | tee "${RUNNER_TEMP}/actionlint-output.txt"

.github/workflows/incremental-benchmark.yml

@@ -0,0 +1,96 @@
+name: Incremental Mailbox Benchmark
+
+on:
+  workflow_dispatch:
+    inputs:
+      messages:
+        description: Number of existing mailbox messages
+        required: false
+        default: "100000"
+        type: string
+  schedule:
+    - cron: "17 3 * * 1"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: incremental-mailbox-benchmark-${{ github.repository }}
+  cancel-in-progress: true
+
+env:
+  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
+  PYTHONPATH: src
+
+jobs:
+  benchmark:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 30
+    steps:
+      - name: Harden runner and block undeclared egress
+        uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
+        with:
+          egress-policy: block
+          disable-telemetry: true
+          allowed-endpoints: |
+            codeload.github.com:443
+            files.pythonhosted.org:443
+            github.com:443
+            objects.githubusercontent.com:443
+            pypi.org:443
+            release-assets.githubusercontent.com:443
+            results-receiver.actions.githubusercontent.com:443
+            *.actions.githubusercontent.com:443
+            *.blob.core.windows.net:443
+
+      - name: Check out the reviewed source without persisted credentials
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: "3.13"
+          cache: pip
+          cache-dependency-path: requirements/ci.lock
+
+      - name: Install the reviewed benchmark toolchain
+        run: python -m pip install --require-hashes -r requirements/ci.lock
+
+      - name: Run the isolated mailbox benchmark
+        env:
+          BENCHMARK_MESSAGES: ${{ inputs.messages || '100000' }}
+        run: |
+          set -euo pipefail
+          python benchmarks/incremental_mailbox.py \
+            --messages "$BENCHMARK_MESSAGES" \
+            --thread-size 10 \
+            --output incremental-benchmark.json
+
+      - name: Enforce parity and the mailbox-scale delta target
+        run: |
+          python - <<'PY'
+          import json
+          from pathlib import Path
+
+          result = json.loads(
+              Path("incremental-benchmark.json").read_text(encoding="utf-8")
+          )
+          incremental = result["incremental"]
+          full_rebuild = result["full_rebuild"]
+          assert incremental["projection_sha256"] == full_rebuild["projection_sha256"]
+          assert incremental["affected_message_count"] == 21
+          if result["message_count"] >= 100_001:
+              assert incremental["delta_apply_seconds"] < full_rebuild[
+                  "full_rebuild_seconds"
+              ]
+          PY
+
+      - name: Upload benchmark evidence
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: incremental-mailbox-benchmark-${{ github.run_id }}-${{ github.run_attempt }}
+          path: incremental-benchmark.json
+          if-no-files-found: error
+          retention-days: 90

AGENTS.md

@@ -5,11 +5,12 @@ Operating guide for automated agents working on this repository.
 `threadweave` implements the JWZ container model with RFC 5256 `REFERENCES`
 threading semantics, RFC 5322 identification-field parsing, RFC 2047 encoded-word
 decoding, RFC 5256 base-subject extraction, RFC 5051 Unicode casemap comparison,
-optional RFC 5256 sent-date ordering, and RFC 5256 IMAP `THREAD` response
-serialization. Its value is correctness: mail clients and ingestion systems rely
-on threading being deterministic, standards-grounded, and impossible to hang on
-malformed input. Treat changes to `threading.py`, `container.py`, `subject.py`,
-`collation.py`, `dates.py`, `headers.py`, and `imap.py` as behavior-sensitive.
+optional RFC 5256 sent-date ordering, RFC 5256 IMAP `THREAD` response
+serialization, and atomic incremental mailbox indexing. Its value is correctness:
+mail clients and ingestion systems rely on threading being deterministic,
+standards-grounded, and impossible to hang on malformed input. Treat changes to
+`threading.py`, `incremental.py`, `container.py`, `subject.py`, `collation.py`,
+`dates.py`, `headers.py`, and `imap.py` as behavior-sensitive.
 
 ## Invariants that must not regress
 
@@ -55,6 +56,17 @@ malformed input. Treat changes to `threading.py`, `container.py`, `subject.py`,
     dummy-root grouping after search projection, reject cyclic or shared graphs,
     and leave the source `Container` tree unchanged. Rendering stays iterative,
     and response framing accepts only CRLF or a caller-owned empty suffix.
+13. **Incremental updates remain batch-equivalent and atomic.** Immutable caller
+    message keys—not sequence numbers—identify indexed records. Recompute every
+    affected old/new reference or subject component through `thread_messages`,
+    but do not pass unrelated components to the batch delegate. Validation or
+    recomputation failure must leave records, version, roots, and projections
+    unchanged. Snapshot state excludes arbitrary payloads and graph pointers.
+14. **External identities follow RFC 8474.** `EMAILID` and `THREADID` use exact
+    1–255 character ObjectID grammar, are case-sensitive, and use disjoint
+    namespaces. Equal EMAILIDs require equal THREADIDs. Once a non-null value is
+    reported, replacement cannot remove or change it; merges and splits remain
+    explicit transitions rather than silent identity rewrites.
 
 ## Architecture and dependency rules
 
@@ -63,10 +75,15 @@ malformed input. Treat changes to `threading.py`, `container.py`, `subject.py`,
   justified.
 - Preserve the standalone package API and its use as a naruon module. The header
   primitives originated in naruon; port behavioral fixes in both directions.
-- Keep IMAP response serialization separate from the transport-neutral tree and
-  date layers so non-IMAP callers do not inherit protocol-specific state.
-- Public behavior, compatibility aliases, and typing markers are release
-  contracts. Record changes in `CHANGELOG.md` and update user/research docs.
+- Keep batch threading authoritative. The incremental layer owns caller keys,
+  component bookkeeping, deltas, and payload-free snapshots; it must delegate
+  every recomputed component to the existing batch engine rather than fork the
+  threading algorithm.
+- Keep IMAP response serialization separate from the transport-neutral batch,
+  incremental, and date layers so non-IMAP callers do not inherit protocol state.
+- Public behavior, compatibility aliases, typing markers, snapshot schemas, and
+  external-ID handoff are release contracts. Record changes in `CHANGELOG.md`
+  and update user/research docs.
 - Unicode collation results depend on the Unicode Character Database bundled
   with the supported Python runtime. Tests must cover stable RFC examples and
   security-sensitive non-equivalences rather than version-specific new codepoints.

ARCHITECTURE.md

@@ -0,0 +1,88 @@
+# ThreadWeave Architecture
+
+## Decision status
+
+This document is the repository-level architecture decision record. `AGENTS.md` is
+the canonical operating policy; this file explains component boundaries and data
+flow for human reviewers and embedding services such as naruon.
+
+## Architectural goal
+
+ThreadWeave provides one standards-grounded threading kernel that works both as a
+zero-runtime-dependency Python package and as a module inside a larger mail or
+knowledge platform. Protocol, incremental state, automation, and release concerns
+remain separate from the canonical batch algorithm.
+
+## Modules
+
+| Boundary | Responsibility | Must not own |
+|---|---|---|
+| `headers`, `encoded_words`, `subject`, `collation`, `dates` | RFC parsing, normalization, and comparison primitives | graph state, sockets, databases |
+| `threading`, `container` | authoritative JWZ/RFC 5256 batch forest | mailbox sessions, persistence, IMAP framing |
+| `incremental` | caller keys, atomic change sets, component indexes, deltas, payload-free snapshots | a second threading algorithm, database/network state |
+| `imap` | non-mutating RFC 5256 `THREAD`/`UID THREAD` presentation | authentication, command parsing, mailbox storage |
+| stdlib adapters | conversion from Python `email` messages | transport sessions or durable state |
+| GitHub workflows and `scripts/ci` | review-first automation, NIM isolation, release evidence | runtime package behavior |
+
+## Authoritative data flow
+
+```text
+caller message metadata
+  -> RFC normalization
+  -> canonical batch thread_messages(component)
+  -> Container forest
+  -> optional incremental component cache and ThreadDelta
+  -> optional IMAP response projection
+```
+
+The incremental layer over-approximates connectivity with normalized message IDs,
+effective reference IDs, and optional RFC 5051 subject keys. Every affected
+component is still evaluated by `thread_messages`; no copy of the threading rules
+is maintained in incremental code.
+
+## State and mutation policy
+
+- `IndexedMessage.message_key` is caller-owned and immutable across revisions.
+- Public reads, snapshots, and `apply` calls on one index are serialized by an
+  in-process reentrant lock. Two transactions targeting one version cannot both
+  commit; the later transaction observes the new version and fails explicitly.
+- The process-local lock is not a distributed lock. Naruon or another host must
+  serialize durable writes across processes and persist the optimistic version.
+- `apply` validates and computes on isolated transaction overlays, then commits once.
+- Default-mode updates retain the existing state-map objects and publish only touched
+  record, position, token, component, EMAILID, and THREADID entries.
+- Reverse connectivity buckets use copy-on-write mutation; RFC 8474 indexes retain
+  compact association/count state rather than one set object per message.
+- RFC sent-date ordering may still scan the mailbox to derive global ranks and reject
+  effective sequence-number collisions; the default first-appearance mode does not.
+- Complete root/projection views are lazy caches invalidated by a successful change.
+- `roots` returns a defensive structural copy; payload references stay caller-owned.
+- Snapshot schema version 1 contains structural metadata only, never payload objects
+  or graph pointers.
+- RFC 8474 `EMAILID` and `THREADID` remain external identities. Structural merges
+  and splits are explicit transitions rather than silent identifier rewrites.
+
+## Ordering and protocol policy
+
+Implicit input positions may be used internally for RFC 5256 sent-date ordering,
+but they are not mailbox sequence numbers and are cleared before public roots are
+returned. IMAP serialization therefore fails closed unless callers supply valid
+sequence numbers or UIDs.
+
+## Scale evidence
+
+The deterministic benchmark runs incremental and full-rebuild workers in separate
+processes. It compares projection SHA-256 values and records initial-build time,
+delta-application time, full-view materialization time, full-rebuild time, affected
+message count, root count, delta retained/transient traced bytes, and peak RSS.
+Scheduled evidence defaults to 100,000 existing messages. Focused performance
+contracts additionally reject default-mode
+small-delta implementations that iterate or replace unrelated state maps.
+
+## Integration policy
+
+Naruon and other services should own persistence, tenancy, authentication,
+mailbox synchronization, distributed write serialization, and external stable-ID
+policy. They pass immutable caller keys and `Message` metadata into ThreadWeave and
+consume `ThreadDelta`, snapshots, or IMAP presentation output through public
+interfaces.