diff --git a/.claude/skills/api-integration/SKILL.md b/.claude/skills/api-integration/SKILL.md index f08ea2d6c..6f7165ad6 100644 --- a/.claude/skills/api-integration/SKILL.md +++ b/.claude/skills/api-integration/SKILL.md @@ -333,9 +333,16 @@ Append tool reference block after the FRED block (before `## ENRICHMENT PROTOCOL - tool_name_1, tool_name_2, tool_name_3 ``` +**Also**: append a per-domain block to the `MCP_FALLBACK_INSTRUCTIONS` template literal at `_promptConstants.js:721`. This teaches subagents what tool prefix maps to the new domain when MCP routing falls back. Format: + +``` +**{domain}** — `mcp__{domain}__*` (e.g., `mcp__{domain}__search_items`) +``` + - [ ] Tool count matches actual number of schemas - [ ] Domain name matches DOMAIN_GROUPS key - [ ] Tool names match schema names exactly +- [ ] `MCP_FALLBACK_INSTRUCTIONS` block present at `_promptConstants.js:721` ### 3.6 Client Registry — `src/server/clientRegistry.js` @@ -368,6 +375,29 @@ Three updates: - [ ] All `expectedCount` sites updated - [ ] All subagent assertion arrays updated +### 3.8 Feature Flag Registration (when API is gated) + +When the new API requires runtime gating (e.g., FMP_ENABLED, EMBEDDING_PERSISTENCE pattern), register the flag in two places: + +1. **`super-legal-mcp-refactored/flags.env`** — append the flag near related flags (e.g., near `FMP_ENABLED`): + ```bash + {SERVICE}_ENABLED=false + ``` +2. **`super-legal-mcp-refactored/src/config/featureFlags.js`** — export from the canonical block. Pattern (matching FMP): + ```javascript + {SERVICE}_ENABLED: process.env.{SERVICE}_ENABLED === 'true', + ``` + +When gated: +- `domainMcpServers.js` `DOMAIN_GROUPS` — wrap the domain key in a conditional spread: `...(featureFlags.{SERVICE}_ENABLED ? { '{domain}': {Name}Tools } : {})` +- `toolDefinitions.js` `allTools` — same conditional spread pattern (`...(featureFlags.{SERVICE}_ENABLED ? {Name}Tools : [])`) +- `clientRegistry.js` slot — conditional construction: `...(featureFlags.{SERVICE}_ENABLED ? { {slotName}: new {Name}HybridClient(...) } : {})` + +- [ ] Flag declared in `flags.env` +- [ ] Flag exported in `featureFlags.js` +- [ ] All conditional spreads wired in `domainMcpServers.js`, `toolDefinitions.js`, `clientRegistry.js` +- [ ] Default value is `false` (opt-in) unless flag is universal + --- ## Phase 4: Testing (MUST PASS before merge) diff --git a/.claude/skills/client-offboarding/SKILL.md b/.claude/skills/client-offboarding/SKILL.md index c7abe3594..6a30cd1d7 100644 --- a/.claude/skills/client-offboarding/SKILL.md +++ b/.claude/skills/client-offboarding/SKILL.md @@ -94,13 +94,15 @@ All exported via `psql COPY TO STDOUT` + gzip to `gs://super-legal-worm-{client_ ### Phase 4: Final Report -**Step 15**: Generate offboarding report +**Step 15**: Generate offboarding report (markdown emitted to stdout + saved to `~/.aperture/offboarding-{client_id}-{date}.md`) - Client ID, offboarding date, operator - Resources deleted (with timestamps) - Archives created (with GCS paths + checksums) - Remaining resources (WORM bucket — retained for compliance) - Final cost estimate (last 30 days billing for this client's resources) +**Future enhancement (deferred)**: PDF rendering via pandoc (already used elsewhere in repo for `aperture-demo-preview.pdf`). Markdown output is the canonical artifact today; pandoc wiring is a 1-command future addition (`pandoc offboarding-{client_id}.md -o offboarding-{client_id}.pdf --pdf-engine=xelatex`). Operator can run manually post-offboarding. + ## Resource Naming Convention (matches provisioner) | Resource | Pattern | Action | diff --git a/.claude/skills/feature-compliance-scaffold/scripts/dimensions/D10-hooks.py b/.claude/skills/feature-compliance-scaffold/scripts/dimensions/D10-hooks.py index 3c6a2fbd1..9c4d56db7 100755 --- a/.claude/skills/feature-compliance-scaffold/scripts/dimensions/D10-hooks.py +++ b/.claude/skills/feature-compliance-scaffold/scripts/dimensions/D10-hooks.py @@ -107,6 +107,61 @@ def main(): "remediation": "", }) + # D10.4 — new SSE event_types that drive transcript_events / frontend + # rendering must appear in (a) transcriptDBBridge.js allowlist (or + # equivalent persistence path) AND (b) frontend handleStreamEvent switch. + import os as _os + repo_root_p = _os.environ.get("REPO_ROOT") or "" + if not repo_root_p: + cur_p = _os.path.abspath(_os.getcwd()) + for _ in range(10): + if _os.path.isdir(_os.path.join(cur_p, "super-legal-mcp-refactored")): + repo_root_p = cur_p + break + cur_p = _os.path.dirname(cur_p) + transcript_paths = [ + _os.path.join(repo_root_p, "super-legal-mcp-refactored", "src", "utils", "transcriptDBBridge.js"), + _os.path.join(repo_root_p, "super-legal-mcp-refactored", "src", "utils", "transcriptPersistence.js"), + ] + frontend_path = _os.path.join(repo_root_p, "super-legal-mcp-refactored", "test", "react-frontend", "app.js") + transcript_text = "" + for p in transcript_paths: + if _os.path.isfile(p): + try: + transcript_text += "\n" + open(p, errors="replace").read() + except OSError: + pass + frontend_text = "" + if _os.path.isfile(frontend_path): + try: + frontend_text = open(frontend_path, errors="replace").read() + except OSError: + frontend_text = "" + + for event_type in s.get("event_types") or []: + # Skip the synthetic types (already handled above) + if event_type in SYNTHETIC_TYPES: + continue + in_transcript = transcript_text and (f"'{event_type}'" in transcript_text or f'"{event_type}"' in transcript_text) + in_frontend = frontend_text and (f"case '{event_type}'" in frontend_text or f'case "{event_type}"' in frontend_text) + if not transcript_text and not frontend_text: + # Neither file accessible; skip silently (covered by D10.1 above) + continue + if not in_transcript: + findings.append({ + "dimension": "D10", "status": "WARNING", + "check": f"D10.4 event_type '{event_type}' transcript persistence", + "message": f"event_type '{event_type}' not found as a string literal in transcriptDBBridge.js / transcriptPersistence.js — may be silently dropped from transcript_events", + "remediation": f"Add '{event_type}' to the transcript-event allowlist in transcriptDBBridge.js (or equivalent persistence file).", + }) + if not in_frontend: + findings.append({ + "dimension": "D10", "status": "WARNING", + "check": f"D10.4 event_type '{event_type}' frontend handler", + "message": f"event_type '{event_type}' not handled in test/react-frontend/app.js handleStreamEvent switch — frontend will ignore it", + "remediation": f"Add `case '{event_type}': ...` branch in handleStreamEvent (app.js).", + }) + emit_findings("D10", findings) diff --git a/.claude/skills/feature-compliance-scaffold/scripts/dimensions/D5-embeddings.py b/.claude/skills/feature-compliance-scaffold/scripts/dimensions/D5-embeddings.py index 4534bed9b..4a3e63774 100755 --- a/.claude/skills/feature-compliance-scaffold/scripts/dimensions/D5-embeddings.py +++ b/.claude/skills/feature-compliance-scaffold/scripts/dimensions/D5-embeddings.py @@ -83,6 +83,48 @@ def main(): "remediation": "Confirm the producing path triggers embeddingService.chunkByHeaders() with EMBEDDING_PERSISTENCE=true.", }) + # D5.4 — new tables that look like report-derived content should have an + # embedding write path wired in hookDBBridge.js (chunkByHeaders call). + # Without this check, an embedding-friendly table can ship with no producer. + import os, re as _re + repo_root_p = os.environ.get("REPO_ROOT", "") + if not repo_root_p: + # Best-effort: walk up from CWD + cur_p = os.path.abspath(os.getcwd()) + for _ in range(10): + if os.path.isdir(os.path.join(cur_p, "super-legal-mcp-refactored")): + repo_root_p = cur_p + break + cur_p = os.path.dirname(cur_p) + bridge_path = os.path.join(repo_root_p, "super-legal-mcp-refactored", "src", "utils", "hookDBBridge.js") + bridge_text = "" + if os.path.isfile(bridge_path): + try: + bridge_text = open(bridge_path, errors="replace").read() + except OSError: + bridge_text = "" + for table in s.get("tables") or []: + # Heuristic: tables suffixed _embeddings / _chunks always need a + # producer; tables ending in _reports / _content / _memos are + # candidates for the report_embeddings flow + report_like = _re.search(r"(_embeddings|_chunks|_reports|_content|_memos|_artifacts)$", table) + if not report_like: + continue + if bridge_text and table in bridge_text and "chunkByHeaders" in bridge_text: + findings.append({ + "dimension": "D5", "status": "WARNING", + "check": f"D5.4 embedding write-path for '{table}'", + "message": f"table '{table}' looks embeddable; hookDBBridge.js mentions both — verify chunkByHeaders() runs against rows from this table", + "remediation": "Manually trace the call site in hookDBBridge.js. Confirm INSERT to {table} triggers an embeddingService.chunkByHeaders() call.", + }) + else: + findings.append({ + "dimension": "D5", "status": "FAILED", + "check": f"D5.4 embedding write-path for '{table}'", + "message": f"new table '{table}' looks embeddable (suffix matches /_(embeddings|chunks|reports|content|memos|artifacts)$/) but no chunkByHeaders() call references it in hookDBBridge.js", + "remediation": f"Add an embedding write path in src/utils/hookDBBridge.js: when INSERT to {table} happens, call embeddingService.chunkByHeaders(content, {{table:'{table}'}}). Gated behind EMBEDDING_PERSISTENCE=true.", + }) + emit_findings("D5", findings) diff --git a/.claude/skills/feature-compliance-scaffold/scripts/dimensions/D6-provenance.py b/.claude/skills/feature-compliance-scaffold/scripts/dimensions/D6-provenance.py index 79750d0f2..a16bfb434 100755 --- a/.claude/skills/feature-compliance-scaffold/scripts/dimensions/D6-provenance.py +++ b/.claude/skills/feature-compliance-scaffold/scripts/dimensions/D6-provenance.py @@ -70,6 +70,50 @@ def main(): "remediation": "Run post-deploy-verify Tier 2 t3-bridge-metadata-git-sha.sql + t3-code-execution-models.sql after deploy. If git_sha='unknown', deploy.sh missed --build-arg COMMIT_SHA=$(git rev-parse HEAD).", }) + # D6.5 — KG-relevant subagents must populate kg_provenance.source_hash + # from upstream source_writes (Wave 2 provenance bridge architecture). + import os as _os, re as _re + repo_root_p = _os.environ.get("REPO_ROOT") or "" + if not repo_root_p: + cur_p = _os.path.abspath(_os.getcwd()) + for _ in range(10): + if _os.path.isdir(_os.path.join(cur_p, "super-legal-mcp-refactored")): + repo_root_p = cur_p + break + cur_p = _os.path.dirname(cur_p) + kg_paths = [ + _os.path.join(repo_root_p, "super-legal-mcp-refactored", "src", "utils", "knowledgeGraphExtractor.js"), + _os.path.join(repo_root_p, "super-legal-mcp-refactored", "src", "utils", "kgService.js"), + _os.path.join(repo_root_p, "super-legal-mcp-refactored", "src", "utils", "kgWrite.js"), + ] + kg_text = "" + for p in kg_paths: + if _os.path.isfile(p): + try: + kg_text += "\n" + open(p, errors="replace").read() + except OSError: + pass + for agent in s.get("agent_types") or []: + if not kg_text: + break + # If the agent is referenced by KG-write paths, source_hash must be populated + agent_in_kg = agent in kg_text or _re.search(rf"agent[_\s]?type\s*[:=]\s*['\"]?{_re.escape(agent)}", kg_text) + if agent_in_kg: + if "source_hash" in kg_text and "kg_provenance" in kg_text: + findings.append({ + "dimension": "D6", "status": "PASSED", + "check": f"D6.5 KG provenance for agent '{agent}'", + "message": f"agent '{agent}' appears in KG-write path; kg_provenance.source_hash populated", + "remediation": "", + }) + else: + findings.append({ + "dimension": "D6", "status": "WARNING", + "check": f"D6.5 KG provenance for agent '{agent}'", + "message": f"agent '{agent}' is KG-relevant but kg_provenance.source_hash population is not visible in knowledgeGraphExtractor.js / kgService.js / kgWrite.js", + "remediation": "Confirm KG INSERT path writes source_hash from upstream source_writes (Wave 2 provenance bridge — KG operates on reports, not raw sources).", + }) + emit_findings("D6", findings) diff --git a/.claude/skills/retention-lifecycle/SKILL.md b/.claude/skills/retention-lifecycle/SKILL.md new file mode 100644 index 000000000..14f84a54a --- /dev/null +++ b/.claude/skills/retention-lifecycle/SKILL.md @@ -0,0 +1,120 @@ +--- +name: retention-lifecycle +description: Automate retention-class transitions (standard → worm → tombstone) and GDPR Art. 17 erasure orchestration. Wraps the 7 admin governance endpoints (legal-hold, retention-class, tombstone, pii/erase) shipped in Wave 3 (v6.2.0). Replaces ad-hoc operator SQL. Triggers — retention enforce, retention scan, tombstone session, gdpr erase, art 17 erase, /retention-lifecycle. Supports flags — --scan (dry-run, list expired), --enforce (promote/tombstone/erase), --erase-subject , --client . +--- + +# Retention Lifecycle — Automated Wave 3 governance enforcement + +## What this does + +The Aperture platform stores client legal-research outputs with explicit retention classes (`standard`, `worm`, `tombstoned`) and per-row `retention_expires_at`. Wave 3 (v6.2.0) shipped the schema + 7 admin endpoints, but **no enforcement workflow**. This skill closes that gap. + +Three operations: + +1. **Scan** — query `sessions` and `reports` for rows past `retention_expires_at`, grouped by current `retention_class`. Read-only; emits a markdown summary. +2. **Enforce** — promote/tombstone expired rows by calling the per-session admin endpoints. Refuses to act on rows with `legal_hold = TRUE`. +3. **Erase subject** — full GDPR Art. 17 orchestration for a pseudonymized data subject. Triggers `pii/erase` plus downstream cascade verification. + +## Workflow + +```bash +# Read-only scan against the deployed instance +/retention-lifecycle --scan + +# Apply transitions (requires confirmation per row class) +/retention-lifecycle --enforce + +# GDPR Art. 17 erasure for a single subject +/retention-lifecycle --erase-subject pseudonym-abc123 + +# Per-client (multi-tenant) — defaults to current Aperture deployment +/retention-lifecycle --scan --client aperture +``` + +## Retention class state machine + +``` +standard ──[expires_at < NOW & no legal_hold]──> worm ──[+90d in worm]──> tombstoned + │ │ + └──> [legal_hold=TRUE blocks all transitions] + │ + ▼ + pii_mappings.encrypted_value + redacted; row stays for audit +``` + +See `references/retention-classes.md` for class semantics and `references/art-17-flow.md` for the GDPR cascade graph. + +## Pre-flight + +Required: +- `psql` (Cloud SQL Auth Proxy or direct via static-IP whitelist `34.26.70.60`) +- `gcloud` (authenticated; project `gen-lang-client-0797903624`) +- `curl` + admin bearer token (Aperture admin user; obtain via `user-management`) +- `jq` + +Required env: +- `SUPER_LEGAL_BASE_URL` (default `http://34.26.70.60:3001`) +- `ADMIN_BEARER_TOKEN` (operator-supplied, never logged) + +## Drift safeguards + +- **legal_hold check** — admin endpoints already refuse to tombstone rows with `legal_hold=TRUE`. Skill double-checks before issuing the call to fail fast with a clearer message. +- **WORM bucket Object Lock** — files promoted to `worm` class are uploaded to `gs://super-legal-worm-{client}-us-east1` with `per_object_retention.mode=Enabled`. Once promoted, files cannot be deleted until retention period elapses, even by project owner. +- **human_interventions row** — every transition writes a row with `intervention_type` in {`retention_enforced`, `gdpr_erasure`, `tombstoned`}. Audit trail per Art. 30 record-of-processing requirements. +- **Idempotency** — re-running `--enforce` on already-promoted rows is safe (the underlying admin endpoints are idempotent). + +## Output format + +``` +## Retention Lifecycle Report +Client: aperture +Mode: --scan +Timestamp: 2026-05-08T... + +### Expired rows by class +| Table | retention_class | count | oldest expires_at | newest expires_at | +|-----------|----------------|-------|--------------------------|--------------------------| +| sessions | standard | 12 | 2026-04-12T00:00:00Z | 2026-05-07T11:23:14Z | +| sessions | worm | 3 | 2026-02-08T00:00:00Z | 2026-02-08T00:00:00Z | +| reports | standard | 47 | ... | ... | + +### Legal hold blockers +- 2 rows with legal_hold=TRUE and retention_expires_at < NOW. NOT enforced. Operator must clear hold first via /api/admin/sessions/:id/legal-hold. + +### Operator next steps +- [ ] Review per-class counts above +- [ ] Run /retention-lifecycle --enforce to apply transitions +- [ ] Validate human_interventions rows after enforcement +``` + +## Truth sources (do not modify) + +- Retention columns: `super-legal-mcp-refactored/src/db/postgres.js:272-279` (sessions + reports `legal_hold`, `retention_class`, `retention_expires_at` + indexes) +- Wave 3 tables: `postgres.js:240-252` (source_writes), `:256-269` (access_log), `:283-296` (human_interventions), `:299-310` (pii_mappings) +- Admin endpoints: `super-legal-mcp-refactored/src/server/adminRouter.js` + - `POST /api/admin/sessions/:sessionId/legal-hold` (L130) + - `POST /api/admin/sessions/:sessionId/retention-class` (L148) + - `POST /api/admin/sessions/:sessionId/tombstone` (L170) + - `POST /api/admin/pii/erase/:sessionId` (L187) + - `POST /api/admin/sessions/:sessionKey/rebuild-kg` (L200) + - `POST /api/admin/sessions/:sessionKey/rebuild-artifacts` (L227) +- Retention manager: `super-legal-mcp-refactored/src/utils/retentionManager.js` — `applyRetentionClass`, `setLegalHold`, `tombstoneSession` +- PII manager: `super-legal-mcp-refactored/src/utils/piiManager.js` — `pseudonymize` (L25), `dePseudonymize` (L51), `erasePII` (L75) +- Tiering daemon: `super-legal-mcp-refactored/src/utils/gcsTieringDaemon.js:54` — `tierOldFiles()` already runs in production for raw-source GCS tiering. This skill is the per-row complement. + +## intervention_type values (must use exactly) + +- `retention_enforced` — automated standard → worm or worm → tombstone transition +- `gdpr_erasure` — Art. 17 PII erasure complete +- `tombstoned` — manual tombstone via admin endpoint + +These strings are checked case-sensitive by audit-export queries. New values must be added to `references/human-interventions-types.md` and migrated via `/schema-evolve`. + +## What this does NOT cover + +- **Bulk multi-client orchestration** — defer to `client-fleet-health` (Phase C) once shipped. This skill operates on one client at a time. +- **Backup-then-tombstone** — use `client-backup-restore` first if the client wants a final archive before tombstone. +- **Cross-client erasure** — Art. 17 is per-deployment by design (single-tenant); a single subject pseudonym is unique to one client deployment. + +Exit codes: `0` clean (or only WARN-level rows skipped), `1` partial (some endpoints failed), `2` fatal (auth, network, missing env). diff --git a/.claude/skills/retention-lifecycle/references/art-17-flow.md b/.claude/skills/retention-lifecycle/references/art-17-flow.md new file mode 100644 index 000000000..219efab8d --- /dev/null +++ b/.claude/skills/retention-lifecycle/references/art-17-flow.md @@ -0,0 +1,73 @@ +# GDPR Art. 17 Erasure Flow + +When a data subject (or their controller) submits an Art. 17 erasure request, the platform must: + +1. Locate every row tied to the subject's pseudonym_id +2. Erase reversible PII (encrypted_value in pii_mappings) +3. Cascade through 60+ ON DELETE CASCADE FKs (KG nodes/edges, embeddings, citations, source_writes) +4. Preserve the audit trail row in human_interventions per Art. 30 record-of-processing +5. Confirm to the subject within 30 days per Art. 12(3) + +## Cascade graph + +``` +pii_mappings (pseudonym_id → encrypted_value) + │ NULL'd by erasePII() + │ + ├─> sessions + │ │ ON DELETE CASCADE → reports + │ │ │ ON DELETE CASCADE → citations, citation_source_links + │ │ │ └─> source_writes (cleaned via job) + │ │ ON DELETE CASCADE → hook_audit_log + │ │ ON DELETE CASCADE → code_executions → code_execution_inputs + │ │ ON DELETE CASCADE → access_log + │ │ ON DELETE CASCADE → human_interventions (LAST written, contains erasure record) + │ │ ON DELETE CASCADE → transcript_events + │ │ + │ └─> kg_nodes / kg_edges (matched via session_id, hard delete) + │ └─> kg_provenance (cascades from kg_nodes/kg_edges) + │ + └─> report_embeddings, citation_embeddings, source_chunk_embeddings + (embeddings store no raw text — vectors only — but rows are + deleted to avoid future similarity-search re-derivation) +``` + +## What survives + +The `human_interventions` row written **before** cascade triggers contains: +- `intervention_type = 'gdpr_erasure'` +- `session_id` (FK) +- `actor_user_id` (the operator who ran the skill) +- `created_at` +- `metadata: { pseudonym_id_hash, request_received_at, completed_at }` + +This row preserves Art. 30 record-of-processing without retaining the actual subject identity. + +## What gets erased + +- `pii_mappings.encrypted_value` set to NULL (mapping table row preserved with `pseudonym_id` for audit) +- All cascade-deleted rows (see graph) +- WORM bucket files: **NOT erased** — Object Lock prevents deletion until lock period elapses. Cite Art. 17(3)(b) "compliance with a legal obligation" exception when documenting the erasure response. Files become unreachable since `pii_mappings` no longer maps to them. + +## Operator confirmation flow + +After running `/retention-lifecycle --erase-subject `: + +1. Verify cascade with: + ```sql + SELECT COUNT(*) FROM pii_mappings WHERE pseudonym_id = '' AND encrypted_value IS NOT NULL; + -- Should be 0 + ``` +2. Verify audit row: + ```sql + SELECT * FROM human_interventions + WHERE intervention_type = 'gdpr_erasure' + AND metadata->>'pseudonym_id_hash' = encode(digest('', 'sha256'), 'hex'); + ``` +3. Send subject Art. 12(3) confirmation within 30 calendar days. Template lives in client onboarding handbook (per-client paths; ask Edwin). + +## Edge cases + +- **Subject linked to legal_hold session** — erasePII() will refuse. Cite Art. 17(3)(e) "establishment, exercise or defence of legal claims" exception. Document refusal in `human_interventions` with `intervention_type='gdpr_erasure'` and `metadata.refused_reason='legal_hold'`. +- **Subject with NO matched sessions** — return 404 to operator, log nothing. Most likely an already-erased subject or a typo. +- **Multiple pseudonyms for one human** — Aperture's pseudonym_id is per-deployment. A human represented by two distinct pseudonyms must be erased twice (run skill once per pseudonym_id). diff --git a/.claude/skills/retention-lifecycle/references/human-interventions-types.md b/.claude/skills/retention-lifecycle/references/human-interventions-types.md new file mode 100644 index 000000000..e94f23a9f --- /dev/null +++ b/.claude/skills/retention-lifecycle/references/human-interventions-types.md @@ -0,0 +1,37 @@ +# human_interventions.intervention_type + +Source of truth: `super-legal-mcp-refactored/src/db/postgres.js:283-296`. + +Wave 3 introduced the `human_interventions` table to satisfy EU AI Act Art. 14 (human oversight) record-of-processing. Each row records when, why, and by whom a human acted on system state. + +## Currently used types + +| intervention_type | Set by | Trigger condition | +|-----------------------|--------|-------------------| +| `retention_enforced` | `retention-lifecycle` --enforce | Automated promotion `standard → worm` or `worm → tombstoned` based on `retention_expires_at` | +| `gdpr_erasure` | `retention-lifecycle` --erase-subject | Operator-initiated Art. 17 erasure for a pseudonymized subject | +| `tombstoned` | Manual operator via `/api/admin/sessions/:id/tombstone` | Direct payload redaction (skips standard → worm pipeline; e.g., security incident) | +| `legal_hold_set` | Manual operator via `/api/admin/sessions/:id/legal-hold` | Operator places legal hold (blocks all retention transitions) | +| `legal_hold_cleared` | Manual operator via `/api/admin/sessions/:id/legal-hold` | Operator clears legal hold | + +## Reserved (planned) types + +These are NOT yet wired but are reserved for future skills/wave shipping: + +| intervention_type | Planned source | Wave | +|-----------------------|---------------|------| +| `kg_rebuild` | `/api/admin/sessions/:id/rebuild-kg` | Already shipped — needs wiring to log row | +| `artifacts_rebuild` | `/api/admin/sessions/:id/rebuild-artifacts` | Already shipped — needs wiring | +| `subject_access_request` | Future Art. 15 SAR skill | Tier 3 | +| `data_portability` | Future Art. 20 export skill | Tier 3 | + +## Adding new types + +1. Add the new string to this file under "Currently used types" +2. Update the relevant skill or admin endpoint to set it on the human_interventions row +3. Update `client-audit-export` (Phase C) range-query SQL to include the new type +4. NO schema migration needed — `intervention_type` is `VARCHAR` not an enum + +## Why VARCHAR not ENUM + +Postgres ENUMs require `ALTER TYPE ... ADD VALUE` to extend, which is non-transactional. We chose VARCHAR for evolutionary flexibility. Type validation lives in skill scripts and the admin route handlers. diff --git a/.claude/skills/retention-lifecycle/references/retention-classes.md b/.claude/skills/retention-lifecycle/references/retention-classes.md new file mode 100644 index 000000000..a1477ea95 --- /dev/null +++ b/.claude/skills/retention-lifecycle/references/retention-classes.md @@ -0,0 +1,35 @@ +# Retention Classes + +Source of truth: `super-legal-mcp-refactored/src/db/postgres.js:272-279` (sessions + reports columns). + +## Class semantics + +| Class | Description | Storage | Reversible? | Set by | +|-------------|-------------|---------|-------------|--------| +| `standard` | Default. Live records subject to per-row `retention_expires_at`. | Cloud SQL primary + raw GCS hot tier | Yes — promote to `worm` | Default on session creation; client config can override | +| `worm` | Promoted from `standard` after expiry. Files moved to `gs://super-legal-worm-{client}-us-east1` with Object Lock `Enabled`. | GCS WORM bucket (immutable per-object retention) | No — once promoted, files cannot be deleted by anyone (incl. project owner) until lock period elapses | `/api/admin/sessions/:id/retention-class` body `{"retention_class":"worm"}` | +| `tombstoned` | Final state — payload redacted, audit row preserved. Session_key/created_at/intervention rows survive for Art. 30 record-of-processing. | Cloud SQL row with NULL'd content fields; WORM bucket files retained until lock expires (cannot be deleted early) | No — payload erasure is permanent | `/api/admin/sessions/:id/tombstone` | + +## Transition rules + +``` +standard ──[expires_at < NOW & no legal_hold]──> worm ──[+90 days in worm]──> tombstoned + ↑ ↑ + [legal_hold=TRUE blocks all transitions; admin must clear first] +``` + +The `+90 days` secondary expiry on `worm` is currently a convention enforced by `tombstone.sh`. To make it data-driven, add a `worm_expires_at TIMESTAMPTZ` column via `/schema-evolve --table sessions --kind add-column`. + +## Related columns + +- `legal_hold BOOLEAN DEFAULT FALSE` — when TRUE, blocks all transitions including admin-driven. Set/cleared via `/api/admin/sessions/:id/legal-hold`. Indexed via `idx_sessions_legal_hold` (partial WHERE legal_hold = TRUE). +- `retention_class VARCHAR(30) DEFAULT 'standard'` — current class. Indexed via `idx_sessions_retention`. +- `retention_expires_at TIMESTAMPTZ` — primary expiry timestamp. NULL means no expiry (e.g. legal hold from inception). + +Same triplet exists on `reports` (postgres.js:277-279). Reports inherit retention from their parent session by convention. + +## What this skill does NOT change + +- **Class definitions** — adding a new class (e.g. `frozen`, `quarantined`) requires schema migration + new admin endpoint logic. Use `/schema-evolve` and update `references/retention-classes.md` here. +- **Lock periods** — Object Lock duration on the WORM bucket is set at provisioning time. Changing it for new buckets affects only future client deployments. +- **Transition timing** — the standard → worm → tombstoned cadence is operator-controlled. This skill is the operator's hammer; deciding *when* to swing remains their job. diff --git a/.claude/skills/retention-lifecycle/scripts/erase-pii.sh b/.claude/skills/retention-lifecycle/scripts/erase-pii.sh new file mode 100755 index 000000000..8c093a7ed --- /dev/null +++ b/.claude/skills/retention-lifecycle/scripts/erase-pii.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# retention-lifecycle: GDPR Art. 17 PII erasure orchestration +# Calls POST /api/admin/pii/erase/:sessionId for the matched session(s). +set -uo pipefail + +BASE_URL="$1" +CLIENT="$2" +SUBJECT="$3" + +[ -n "${ADMIN_BEARER_TOKEN:-}" ] || { echo "ERROR: ADMIN_BEARER_TOKEN not set" >&2; exit 2; } +[ -n "$SUBJECT" ] || { echo "ERROR: --erase-subject requires a pseudonym_id" >&2; exit 2; } +[ -n "${PGHOST:-}" ] || { echo "ERROR: PGHOST not set — Cloud SQL access required to map subject → sessions" >&2; exit 2; } + +echo "### GDPR Art. 17 erasure for subject: $SUBJECT" +echo "" + +QUERY="SELECT DISTINCT session_id + FROM pii_mappings + WHERE pseudonym_id = '$SUBJECT';" + +mapfile -t SIDS < <(psql -d "${PGDATABASE:-super_legal}" --no-psqlrc --quiet --tuples-only -c "$QUERY" 2>/dev/null | awk 'NF') + +if [ ${#SIDS[@]} -eq 0 ]; then + echo " ⊘ No pii_mappings rows for pseudonym_id='$SUBJECT'." + echo " Either the subject was already erased, or the pseudonym_id is wrong." + echo " Check audit log: SELECT * FROM access_log WHERE event_data ? '$SUBJECT';" + exit 1 +fi + +echo " Subject linked to ${#SIDS[@]} session(s):" +for SID in "${SIDS[@]}"; do + echo " - $(echo $SID | xargs)" +done +echo "" + +read -r -p " WARNING: This is irreversible. Erase PII for all ${#SIDS[@]} sessions? [y/N]: " ANS +[[ "$ANS" =~ ^[Yy]$ ]] || { echo " ⊘ skipped by operator"; exit 0; } + +OK=0 +FAIL=0 +for SID in "${SIDS[@]}"; do + SID="$(echo "$SID" | xargs)" + [ -z "$SID" ] && continue + HTTP=$(curl -s -o /dev/null -w "%{http_code}" \ + -X POST "$BASE_URL/api/admin/pii/erase/$SID" \ + -H "Authorization: Bearer $ADMIN_BEARER_TOKEN" \ + -H "Content-Type: application/json" \ + -d "{\"pseudonym_id\":\"$SUBJECT\",\"reason\":\"GDPR Art. 17 erasure request\"}") + if [ "$HTTP" = "200" ]; then + OK=$((OK+1)) + echo " ✓ $SID PII erased" + else + FAIL=$((FAIL+1)) + echo " ✗ $SID failed (HTTP $HTTP)" + fi +done + +echo "" +echo " Erasure summary: $OK ok, $FAIL failed" +echo "" +echo "### Operator next steps" +echo "- [ ] Verify pii_mappings.encrypted_value IS NULL for all rows of this pseudonym" +echo "- [ ] Verify human_interventions row with intervention_type='gdpr_erasure'" +echo "- [ ] Review downstream cascade — see references/art-17-flow.md" +echo "- [ ] Send subject confirmation per Art. 12(3) within 30 days" + +[ $FAIL -eq 0 ] diff --git a/.claude/skills/retention-lifecycle/scripts/lifecycle.sh b/.claude/skills/retention-lifecycle/scripts/lifecycle.sh new file mode 100755 index 000000000..b7d8ee26b --- /dev/null +++ b/.claude/skills/retention-lifecycle/scripts/lifecycle.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# retention-lifecycle entry point. +# Wraps Wave 3 admin governance endpoints to enforce retention transitions +# and GDPR Art. 17 erasure orchestration. +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +MODE="" CLIENT="aperture" SUBJECT="" CONFIRM="false" +BASE_URL="${SUPER_LEGAL_BASE_URL:-http://34.26.70.60:3001}" + +while [[ $# -gt 0 ]]; do + case "$1" in + --scan) MODE="scan"; shift ;; + --enforce) MODE="enforce"; shift ;; + --erase-subject) MODE="erase"; SUBJECT="$2"; shift 2 ;; + --client) CLIENT="$2"; shift 2 ;; + --base-url) BASE_URL="$2"; shift 2 ;; + --yes|-y) CONFIRM="true"; shift ;; + -h|--help) + cat < # GDPR Art. 17 + +Optional flags: + --client Multi-tenant scope (default: aperture) + --base-url Override SUPER_LEGAL_BASE_URL + --yes Skip per-class confirmation prompts (--enforce only) + +Required env: + ADMIN_BEARER_TOKEN Admin bearer; never logged. + SUPER_LEGAL_BASE_URL Defaults to http://34.26.70.60:3001 +EOF + exit 0 ;; + *) echo "ERROR: unknown flag: $1" >&2; exit 2 ;; + esac +done + +[ -n "$MODE" ] || { echo "ERROR: pass --scan, --enforce, or --erase-subject" >&2; exit 2; } +for cmd in psql gcloud curl jq python3; do + command -v "$cmd" >/dev/null || { echo "ERROR: $cmd not found in PATH" >&2; exit 2; } +done + +if [ "$MODE" != "scan" ]; then + [ -n "${ADMIN_BEARER_TOKEN:-}" ] || { echo "ERROR: ADMIN_BEARER_TOKEN env var required for $MODE mode" >&2; exit 2; } +fi + +echo "## Retention Lifecycle Report" +echo "Client: $CLIENT" +echo "Mode: --$MODE${SUBJECT:+ $SUBJECT}" +echo "Base URL: $BASE_URL" +echo "Timestamp: $(date -u +%Y-%m-%dT%H:%M:%SZ)" +echo "" + +case "$MODE" in + scan) + python3 "$SCRIPT_DIR/scan-expired.py" --base-url "$BASE_URL" --client "$CLIENT" + exit $? + ;; + enforce) + bash "$SCRIPT_DIR/promote-to-worm.sh" "$BASE_URL" "$CLIENT" "$CONFIRM" + PROMOTE_RC=$? + bash "$SCRIPT_DIR/tombstone.sh" "$BASE_URL" "$CLIENT" "$CONFIRM" + TOMB_RC=$? + [ $PROMOTE_RC -eq 0 ] && [ $TOMB_RC -eq 0 ] + exit $? + ;; + erase) + bash "$SCRIPT_DIR/erase-pii.sh" "$BASE_URL" "$CLIENT" "$SUBJECT" + exit $? + ;; +esac diff --git a/.claude/skills/retention-lifecycle/scripts/promote-to-worm.sh b/.claude/skills/retention-lifecycle/scripts/promote-to-worm.sh new file mode 100755 index 000000000..d1ad9364c --- /dev/null +++ b/.claude/skills/retention-lifecycle/scripts/promote-to-worm.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# retention-lifecycle: standard → worm transition +# Calls POST /api/admin/sessions/:id/retention-class for each expired standard row. +# Refuses rows with legal_hold=TRUE. +set -uo pipefail + +BASE_URL="$1" +CLIENT="$2" +CONFIRM="${3:-false}" + +[ -n "${ADMIN_BEARER_TOKEN:-}" ] || { echo "ERROR: ADMIN_BEARER_TOKEN not set" >&2; exit 2; } +[ -n "${PGHOST:-}" ] || { echo " ⊘ skipping promote-to-worm (no PGHOST — operator must set Cloud SQL env)" >&2; exit 0; } + +echo "### Step 1 — promote standard → worm" +echo "" + +QUERY="SELECT session_key FROM sessions + WHERE retention_class = 'standard' + AND legal_hold = FALSE + AND retention_expires_at < NOW() + ORDER BY retention_expires_at ASC;" + +mapfile -t IDS < <(psql -d "${PGDATABASE:-super_legal}" --no-psqlrc --quiet --tuples-only -c "$QUERY" 2>/dev/null | awk 'NF') + +if [ ${#IDS[@]} -eq 0 ]; then + echo " ✓ No expired standard-class rows. Skipping." + exit 0 +fi + +echo " Found ${#IDS[@]} expired standard-class session(s) to promote." + +if [ "$CONFIRM" != "true" ]; then + echo "" + read -r -p " Promote ${#IDS[@]} sessions to 'worm'? [y/N]: " ANS + [[ "$ANS" =~ ^[Yy]$ ]] || { echo " ⊘ skipped by operator"; exit 0; } +fi + +OK=0 +FAIL=0 +for SID in "${IDS[@]}"; do + SID="$(echo "$SID" | xargs)" + [ -z "$SID" ] && continue + HTTP=$(curl -s -o /dev/null -w "%{http_code}" \ + -X POST "$BASE_URL/api/admin/sessions/$SID/retention-class" \ + -H "Authorization: Bearer $ADMIN_BEARER_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"retention_class":"worm","reason":"automated retention-lifecycle promotion"}') + if [ "$HTTP" = "200" ]; then + OK=$((OK+1)) + echo " ✓ $SID → worm" + else + FAIL=$((FAIL+1)) + echo " ✗ $SID failed (HTTP $HTTP)" + fi +done + +echo "" +echo " Promote summary: $OK ok, $FAIL failed" +[ $FAIL -eq 0 ] diff --git a/.claude/skills/retention-lifecycle/scripts/scan-expired.py b/.claude/skills/retention-lifecycle/scripts/scan-expired.py new file mode 100755 index 000000000..e00f6f754 --- /dev/null +++ b/.claude/skills/retention-lifecycle/scripts/scan-expired.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +"""retention-lifecycle: scan-expired — read-only listing of rows past retention_expires_at. + +Hits a per-deployment helper endpoint (Cloud SQL via the admin app's pool) when +available; falls back to direct psql via Cloud SQL Auth Proxy. For now we expect +the operator to have local psql access via the static-IP whitelist. + +Output: markdown summary grouped by table + retention_class. +""" + +import argparse +import os +import subprocess +import sys + + +SCAN_QUERY = """ +SELECT 'sessions' AS table, retention_class, legal_hold, + COUNT(*) AS row_count, + MIN(retention_expires_at) AS oldest_expires_at, + MAX(retention_expires_at) AS newest_expires_at +FROM sessions +WHERE retention_expires_at IS NOT NULL + AND retention_expires_at < NOW() +GROUP BY retention_class, legal_hold +UNION ALL +SELECT 'reports' AS table, retention_class, legal_hold, + COUNT(*) AS row_count, + MIN(retention_expires_at) AS oldest_expires_at, + MAX(retention_expires_at) AS newest_expires_at +FROM reports +WHERE retention_expires_at IS NOT NULL + AND retention_expires_at < NOW() +GROUP BY retention_class, legal_hold +ORDER BY 1, 2, 3; +""" + + +def run_psql(query: str) -> str: + """Run query via psql; expects PGHOST/PGUSER/PGPASSWORD/PGDATABASE in env.""" + db = os.environ.get("PGDATABASE", "super_legal") + cmd = [ + "psql", "-d", db, "--no-psqlrc", "--quiet", "--tuples-only", + "--field-separator-zero", "--record-separator-zero", + "-c", query, + ] + res = subprocess.run(cmd, capture_output=True, text=True) + if res.returncode != 0: + print(f"ERROR: psql failed: {res.stderr}", file=sys.stderr) + sys.exit(2) + return res.stdout + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--base-url", required=True) + p.add_argument("--client", required=True) + args = p.parse_args() + + if not os.environ.get("PGHOST"): + print("### Pre-flight check") + print("PGHOST not set. Configure Cloud SQL Auth Proxy or psql env vars:") + print(" export PGHOST=") + print(" export PGUSER=postgres") + print(" export PGPASSWORD=") + print(" export PGDATABASE=super_legal") + print("") + print("Skill cannot scan without DB access. Exiting cleanly.") + sys.exit(0) + + raw = run_psql(SCAN_QUERY) + + rows = [] + for chunk in raw.split("\0"): + chunk = chunk.strip() + if not chunk: + continue + # Each row is field-separator-zero; split by \0 inside + cols = chunk.split("\0") if "\0" in chunk else chunk.split("|") + if len(cols) >= 6: + rows.append([c.strip() for c in cols[:6]]) + + print("### Expired rows by class") + if not rows: + print("(none — DB is clean for the current window)") + sys.exit(0) + + print() + print("| Table | retention_class | legal_hold | count | oldest expires_at | newest expires_at |") + print("|-----------|----------------|-----------|-------|--------------------------|--------------------------|") + blocked = 0 + for r in rows: + table, rc, lh, count, oldest, newest = r + print(f"| {table:<9} | {rc:<14} | {lh:<9} | {count:<5} | {oldest:<24} | {newest:<24} |") + if lh == "t": + blocked += int(count or 0) + + print() + if blocked: + print(f"### Legal-hold blockers") + print(f"{blocked} row(s) with legal_hold=TRUE and expired retention.") + print(f"These will NOT be enforced. Operator must clear via:") + print(f" POST /api/admin/sessions//legal-hold body: {{\"legal_hold\": false}}") + print() + + print("### Operator next steps") + print("- [ ] Review per-class counts above") + print("- [ ] Run /retention-lifecycle --enforce to apply transitions") + print("- [ ] Verify human_interventions rows post-enforcement") + + +if __name__ == "__main__": + main() diff --git a/.claude/skills/retention-lifecycle/scripts/tombstone.sh b/.claude/skills/retention-lifecycle/scripts/tombstone.sh new file mode 100755 index 000000000..1d690fbca --- /dev/null +++ b/.claude/skills/retention-lifecycle/scripts/tombstone.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# retention-lifecycle: worm → tombstoned transition +# Calls POST /api/admin/sessions/:id/tombstone for sessions in worm class past +# their secondary expiry. The admin endpoint refuses if legal_hold=TRUE. +set -uo pipefail + +BASE_URL="$1" +CLIENT="$2" +CONFIRM="${3:-false}" + +[ -n "${ADMIN_BEARER_TOKEN:-}" ] || { echo "ERROR: ADMIN_BEARER_TOKEN not set" >&2; exit 2; } +[ -n "${PGHOST:-}" ] || { echo " ⊘ skipping tombstone (no PGHOST — operator must set Cloud SQL env)" >&2; exit 0; } + +echo "" +echo "### Step 2 — tombstone worm-class rows past secondary expiry" +echo "" + +# Tombstone worm-class rows whose retention_expires_at + 90 days has passed. +QUERY="SELECT session_key FROM sessions + WHERE retention_class = 'worm' + AND legal_hold = FALSE + AND retention_expires_at + INTERVAL '90 days' < NOW() + ORDER BY retention_expires_at ASC;" + +mapfile -t IDS < <(psql -d "${PGDATABASE:-super_legal}" --no-psqlrc --quiet --tuples-only -c "$QUERY" 2>/dev/null | awk 'NF') + +if [ ${#IDS[@]} -eq 0 ]; then + echo " ✓ No worm-class rows past secondary expiry. Skipping." + exit 0 +fi + +echo " Found ${#IDS[@]} worm-class session(s) eligible for tombstone." + +if [ "$CONFIRM" != "true" ]; then + echo "" + echo " WARNING: tombstone is irreversible — payload redaction is permanent." + read -r -p " Tombstone ${#IDS[@]} sessions? [y/N]: " ANS + [[ "$ANS" =~ ^[Yy]$ ]] || { echo " ⊘ skipped by operator"; exit 0; } +fi + +OK=0 +FAIL=0 +for SID in "${IDS[@]}"; do + SID="$(echo "$SID" | xargs)" + [ -z "$SID" ] && continue + HTTP=$(curl -s -o /dev/null -w "%{http_code}" \ + -X POST "$BASE_URL/api/admin/sessions/$SID/tombstone" \ + -H "Authorization: Bearer $ADMIN_BEARER_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"reason":"automated retention-lifecycle tombstone"}') + if [ "$HTTP" = "200" ]; then + OK=$((OK+1)) + echo " ✓ $SID tombstoned" + else + FAIL=$((FAIL+1)) + echo " ✗ $SID failed (HTTP $HTTP)" + fi +done + +echo "" +echo " Tombstone summary: $OK ok, $FAIL failed" +[ $FAIL -eq 0 ] diff --git a/.claude/skills/schema-evolve/SKILL.md b/.claude/skills/schema-evolve/SKILL.md new file mode 100644 index 000000000..3ff7c8bb5 --- /dev/null +++ b/.claude/skills/schema-evolve/SKILL.md @@ -0,0 +1,115 @@ +--- +name: schema-evolve +description: Generate schema artifacts atomically across three layers — DDL (postgres.js + migrations/), Zod tool envelopes (toolEnvelopes.js), and output JSON schemas (src/schemas/). Prevents the dual-path drift class that caused v6.2.3 column-evolution hotfix, v6.8.2 fresh-DB-throw, and the PB-1 missing-migrations bug. Use when adding a new table, column, tool envelope, or output schema. Triggers — schema evolve, add table, add column, new envelope, /schema-evolve. Supports flags — --table --kind add-table|add-column, --tool --kind add-envelope, --output-schema , --column-type , --column-on . +--- + +# Schema Evolve — Atomic Multi-Layer Schema Generator + +## Workflow + +```bash +# Add a new table (DDL: migration + ensureSchema CREATE TABLE) +/schema-evolve --table client_audit_log --kind add-table + +# Add a column to existing table (ALTER TABLE ADD COLUMN IF NOT EXISTS) +/schema-evolve --table reports --kind add-column --column event_id --column-type "UUID" + +# Add a Zod tool envelope schema (when introducing a new MCP tool) +/schema-evolve --tool fetch_xbrl_filing --kind add-envelope + +# Add an output JSON schema (memo synthesis output validation) +/schema-evolve --output-schema BondResearchMemo +``` + +## What it does + +1. **DDL layer** (`super-legal-mcp-refactored/src/db/postgres.js` + `migrations/NNN_*.up.sql`): + - For new table: emits matching `CREATE TABLE IF NOT EXISTS` block in the right `ensure*Schema()` function (chooses based on table category) AND a numbered migration file (next is `015_*` per current state) + - For new column: emits `ALTER TABLE
ADD COLUMN IF NOT EXISTS ` in `ensure*Schema()` AND a matching `migrations/NNN_*.up.sql` + - Generates `IF NOT EXISTS` guards on indexes; produces `.down.sql` for rollback + +2. **Zod envelope layer** (`src/schemas/toolEnvelopes.js`): + - For new tool: appends `EnvelopeSchema` using `.passthrough()` + `_hybrid_metadata` if MCP-routed + - Updates `TOOL_ENVELOPE_SCHEMAS` registry (line 115) + - `validateToolEnvelope()` helper at line 137 picks up new schema automatically + +3. **Output schema layer** (`src/schemas/.json`): + - Mirrors `BankruptcyResearchMemo.json` shape; emits a Zod-derivable JSON Schema for memo/output validation + +4. **Test fixture layer** (`test/sdk/`): + - Generates a round-trip test mirroring `test/sdk/code-execution-bridge.test.js` pattern (24+ assertions, 6 groups). Validates a captured envelope against the new schema. + +## Pre-flight + +Required: `python3`, `git`. The skill reads code locally (no DB or network). + +## Drift prevention + +This skill addresses three distinct drift signatures observed in v7.0.x: + +| Drift | Symptom | Skill prevents via | +|-------|---------|-------------------| +| DDL-only (no migration) | `column does not exist` on fresh-DB; `client-backup-restore` loses table | Always emit `migrations/NNN_*.up.sql` + `.down.sql` pair | +| Migration-only (no `ensure*Schema()`) | `ensure*Schema()` is no-op against existing prod DB; column never appears | Always emit `ALTER TABLE ADD COLUMN IF NOT EXISTS` in `ensure*Schema()` | +| Zod envelope drift | `claude_hook_persistence_failures_total{reason='envelope_shape_drift'}` fires; silent NULL writes | Generate `EnvelopeSchema` with `.passthrough()` + register in `TOOL_ENVELOPE_SCHEMAS` | + +## Invocation patterns + +| Use case | Command | +|----------|---------| +| Wave 4 new audit table | `/schema-evolve --table client_audit_export --kind add-table` | +| Add actor_user_id to access_log (PB-2 fix) | `/schema-evolve --table access_log --kind add-column --column actor_user_id --column-type "INTEGER REFERENCES users(id)"` | +| Add timeout BOOLEAN to code_executions (PB-3 fix) | `/schema-evolve --table code_executions --kind add-column --column timeout --column-type "BOOLEAN DEFAULT FALSE"` | +| New MCP tool with structured response | `/schema-evolve --tool search_bond_filings --kind add-envelope` | + +## Pre-built fixes (PB-1/2/3 from v7.0.x audit) + +The skill's `--prebuilt` mode emits the three known-pending production fixes: + +```bash +/schema-evolve --prebuilt PB1 # citation_source_links + code_execution_inputs migrations +/schema-evolve --prebuilt PB2 # adds actor_user_id (FK -> users) on access_log +/schema-evolve --prebuilt PB3 # adds timeout BOOLEAN on code_executions +``` + +## Truth sources (do not modify) + +- `super-legal-mcp-refactored/src/db/postgres.js` — `ensure*Schema()` functions at lines 26, 958, 965, 999, 1006, 1027 +- `super-legal-mcp-refactored/migrations/` — current highest is `014_audit-log-model-id-index.up.sql` → next is `015_*` +- `super-legal-mcp-refactored/src/schemas/toolEnvelopes.js` — 5 envelopes + `TOOL_ENVELOPE_SCHEMAS` (L115) + `validateToolEnvelope()` (L137) +- `super-legal-mcp-refactored/src/utils/hookDBBridge.js:1203-1208,1250-1251` — validator integration sites + +## Read-only guarantee + +The skill creates new files (migration + Zod envelope + output schema + test fixture) and **proposes** patches to existing files (`postgres.js` ALTER block, `toolEnvelopes.js` registry append). It NEVER mutates an existing migration file (immutable per dual-path convention). Operator reviews the patch and applies. + +## Output format + +``` +## Schema Evolve Report +Operation: add-column on reports — bls_series_id TEXT +Timestamp: 2026-05-08T... + +### DDL artifacts +✓ migrations/015_reports-bls-series-id.up.sql (created) +✓ migrations/015_reports-bls-series-id.down.sql (created) +✓ ensureSessionsSchema() patch (preview): + ALTER TABLE reports ADD COLUMN IF NOT EXISTS bls_series_id TEXT; + +### Zod envelope artifacts +N/A (no --tool flag) + +### Output schema artifacts +N/A (no --output-schema flag) + +### Test fixture artifacts +N/A (column-only addition; no envelope test needed) + +### Operator next steps +- [ ] Review patch in postgres.js +- [ ] Run `node --check src/db/postgres.js` +- [ ] Run `feature-compliance-scaffold --git-range main..HEAD` → expect D7 PASSED +- [ ] Open PR +``` + +Exit codes: `0` = clean, `1` = warnings (e.g., column type ambiguous), `2` = error (e.g., table not in postgres.js). diff --git a/.claude/skills/schema-evolve/references/migration-numbering.md b/.claude/skills/schema-evolve/references/migration-numbering.md new file mode 100644 index 000000000..247e2d6d7 --- /dev/null +++ b/.claude/skills/schema-evolve/references/migration-numbering.md @@ -0,0 +1,53 @@ +# Migration Numbering + +Sequential 3-digit prefixes — never reuse, never reorder. + +## Current state (2026-05-07) + +Highest applied: `014_audit-log-model-id-index.up.sql`. Next will be `015_*`. + +``` +010_kg-provenance-source-hash.up.sql +011_users-status-last-login.up.sql +012_transcript-events.up.sql +013_code-executions-model-id.up.sql +014_audit-log-model-id-index.up.sql +015_.up.sql ← schema-evolve allocates this +``` + +`schema-evolve/scripts/evolve.sh` discovers the next number via: + +```bash +ls migrations/*.up.sql | xargs -n1 basename | grep -oE '^[0-9]+' | sort -n | tail -1 +``` + +then increments by 1 with `printf '%03d'`. + +## Naming convention + +`NNN_.up.sql` + matching `.down.sql`. + +Description should be terse (under 50 chars after the prefix) and describe **what** the migration does — not what bug it fixes. Tense/voice convention: + +- `015_reports-bls-series-id` ✓ +- `016_add-fmp-flag` ✗ (verb redundant — all migrations add) +- `015_fix-prod-bug` ✗ (no signal) + +## Immutability rule + +Once a migration is committed to `main`, **never edit it**. If the migration produced wrong DDL, ship a corrective follow-up migration with a new number. + +This is required because: +1. Some prod DBs may have already applied the original +2. Migration runners track applied migrations by file hash +3. Operator review history relies on the original commit being immutable + +`schema-evolve` refuses to overwrite existing migration files (`if up_path.exists() → exit 2`). + +## Down-migration policy + +Every `.up.sql` ships with a matching `.down.sql` that reverses it. Down migrations are best-effort — they don't need to restore exact pre-state (e.g., dropped data is gone), but they should leave the schema runnable. + +## Race conditions on parallel branches + +If two branches each allocate `015_*`, the second to merge must rebase and renumber to `016_*`. CI does NOT detect this — operator vigilance required during merge. diff --git a/.claude/skills/schema-evolve/references/patterns.md b/.claude/skills/schema-evolve/references/patterns.md new file mode 100644 index 000000000..1de4dc3c9 --- /dev/null +++ b/.claude/skills/schema-evolve/references/patterns.md @@ -0,0 +1,63 @@ +# Schema Evolution Patterns + +## The dual-path requirement + +Every schema change touches **two execution paths**: + +1. **Boot path**: `super-legal-mcp-refactored/src/db/postgres.js` — `ensure*Schema()` functions run on every server boot. They use `CREATE TABLE IF NOT EXISTS` and `CREATE INDEX IF NOT EXISTS` to converge fresh DBs. +2. **Migration path**: `super-legal-mcp-refactored/migrations/NNN_*.up.sql` — applied once per DB by deployment automation. Provides explicit, reviewable history. + +If you only update path 1 (boot), fresh DBs work but existing prod DBs never receive the change — `CREATE TABLE IF NOT EXISTS` is a no-op when the table already exists, even if its shape has diverged. + +If you only update path 2 (migration), CI/migrated DBs work but `client-backup-restore` and any future-spawned tenant boots from a clean schema that lacks the field. + +**`schema-evolve` always emits both.** + +## ensure*Schema() function map + +| Function | Tables it owns | +|----------|---------------| +| `ensureSessionsSchema` | users, sessions, reports, citations, citation_source_links | +| `ensureHookSchema` | hook_audit_log, code_executions, code_execution_inputs, transcript_events, agent_progress | +| `ensureWave3Schema` | source_writes, access_log, human_interventions, pii_mappings | +| `ensureEmbeddingSchema` | report_embeddings, citation_embeddings, source_chunk_embeddings | +| `ensureKnowledgeGraphSchema` | kg_nodes, kg_edges, kg_provenance | + +When in doubt, `git grep "CREATE TABLE.*" super-legal-mcp-refactored/src/db/postgres.js` to find the owning function. + +## Column evolution gotcha (v6.2.3 hotfix) + +`CREATE TABLE IF NOT EXISTS` does NOT add new columns to existing tables. To add a column to a live table you must use: + +```sql +ALTER TABLE
+ ADD COLUMN IF NOT EXISTS ; +``` + +This requires PostgreSQL 9.6+ (we run 16). Always use `IF NOT EXISTS` so re-runs are safe. + +## Zod envelope drift signature + +When a tool's response shape changes but `toolEnvelopes.js` is not updated, `validateToolEnvelope()` fail-opens (returns `{ ok: false, data: parsed }`) and emits the metric: + +``` +claude_hook_persistence_failures_total{reason="envelope_shape_drift"} +``` + +Watch this in Grafana after merges that touch tool implementations. + +## Output schema (`src/schemas/.json`) + +JSON Schema draft-07 documents that constrain memo synthesis output. Validated via `ajv` in tests; consumed at runtime by section writers as a structural reference. + +Pattern (from `BankruptcyResearchMemo.json`): + +- Top-level `sections` (recursive via `$ref`) +- Top-level `citations` array with `id` + `url` +- Optional `metadata` block (model_id, session_key, generated_at) + +## When NOT to use schema-evolve + +- One-off ad-hoc queries (just write SQL) +- Adding indexes to existing columns (no shape change — emit migration manually) +- Removing columns (drift-prevention is moot; needs careful manual rollout with deprecation period) diff --git a/.claude/skills/schema-evolve/references/zod-conventions.md b/.claude/skills/schema-evolve/references/zod-conventions.md new file mode 100644 index 000000000..461ed855e --- /dev/null +++ b/.claude/skills/schema-evolve/references/zod-conventions.md @@ -0,0 +1,73 @@ +# Zod Tool Envelope Conventions + +Source of truth: `super-legal-mcp-refactored/src/schemas/toolEnvelopes.js` (5 schemas as of 2026-05-07). + +## Core pattern + +```js +export const fooEnvelopeSchema = z.object({ + // every field .nullable().optional() unless absolutely required + results: z.array(itemSchema).default([]), + query: z.string().nullable().optional(), + _hybrid_metadata: hybridMetadataSchema.optional(), // MCP-routed only +}).passthrough(); +``` + +## Why these defaults + +- `.passthrough()` — fail-open for forward compat. Tools may add new fields without breaking us. +- `.nullable().optional()` — matches the wild reality of legacy/Exa hybrid responses where any field may be null or absent. +- `.default([])` on arrays — section writers crash on `undefined.map()`. +- `_hybrid_metadata` — populated when tool routed through the hybrid client (native + Exa fallback). Tracks which path served the request. + +## Registry + +`TOOL_ENVELOPE_SCHEMAS` (Object.freeze at L115) maps tool_name → schema. The validator helper `validateToolEnvelope(toolName, parsed)` at L137 looks up by tool_name; tools without a registered schema silently skip validation. + +```js +export const TOOL_ENVELOPE_SCHEMAS = Object.freeze({ + fetch_document: fetchDocumentEnvelopeSchema, + exa_web_search: exaWebSearchEnvelopeSchema, + kg_create_node: kgCreateNodeEnvelopeSchema, + search_cases: searchCasesEnvelopeSchema, + search_sec_filings: searchSecFilingsEnvelopeSchema, +}); +``` + +When adding a new tool: +1. Define `EnvelopeSchema` with `.passthrough()` +2. Append to `TOOL_ENVELOPE_SCHEMAS` (preserve `Object.freeze()`) +3. No code change needed in the validator helper — registry-driven + +## Validator integration sites + +- `src/utils/hookDBBridge.js:1203-1208` — first call site (PostToolUse handler) +- `src/utils/hookDBBridge.js:1250-1251` — second call site (success path) + +Drift fires: + +``` +claude_hook_persistence_failures_total{reason="envelope_shape_drift"} +``` + +(metric defined in `src/utils/sdkMetrics.js:239`) + +## Fail-open semantics + +`validateToolEnvelope()` returns `{ ok, data, issues }`: +- On parse success: `{ ok: true, data: validated }` +- On parse fail: `{ ok: false, data: parsed, issues: [...] }` + +Caller continues using `data` either way — drift doesn't block writes. The metric is the operator's only signal. + +## What NOT to do + +- ✗ Strict `.strict()` schemas — breaks forward compat +- ✗ Required `_hybrid_metadata` — breaks tools that bypass the hybrid client +- ✗ Use raw `z.any()` for the whole envelope — defeats the purpose +- ✗ Mutate `TOOL_ENVELOPE_SCHEMAS` post-freeze — runtime error + +## Helper schemas (already exported) + +- `hybridMetadataSchema` (top of file) — used by all hybrid-routed tools +- `exaSearchResultSchema`, `courtCaseSchema`, `secFilingSchema` — internal item schemas; reuse by importing if your tool returns the same shape diff --git a/.claude/skills/schema-evolve/scripts/evolve.sh b/.claude/skills/schema-evolve/scripts/evolve.sh new file mode 100755 index 000000000..8b9361bba --- /dev/null +++ b/.claude/skills/schema-evolve/scripts/evolve.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# schema-evolve entry point. +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cur="$SCRIPT_DIR" +while [ "$cur" != "/" ]; do + if [ -d "$cur/super-legal-mcp-refactored" ]; then REPO_ROOT="$cur"; break; fi + cur="$(dirname "$cur")" +done +[ -n "${REPO_ROOT:-}" ] || { echo "ERROR: cannot locate repo root" >&2; exit 2; } + +TABLE="" KIND="" COLUMN="" COLUMN_TYPE="" TOOL="" OUTPUT_SCHEMA="" PREBUILT="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --table) TABLE="$2"; shift 2 ;; + --kind) KIND="$2"; shift 2 ;; + --column) COLUMN="$2"; shift 2 ;; + --column-type) COLUMN_TYPE="$2"; shift 2 ;; + --tool) TOOL="$2"; shift 2 ;; + --output-schema) OUTPUT_SCHEMA="$2"; shift 2 ;; + --prebuilt) PREBUILT="$2"; shift 2 ;; + -h|--help) + cat < --kind add-table + evolve.sh --table --kind add-column --column --column-type "" + evolve.sh --tool --kind add-envelope + evolve.sh --output-schema + evolve.sh --prebuilt PB1|PB2|PB3 +EOF + exit 0 ;; + *) echo "Unknown flag: $1" >&2; exit 2 ;; + esac +done + +for cmd in python3 git; do command -v "$cmd" >/dev/null || { echo "ERROR: $cmd not found" >&2; exit 2; }; done + +cd "$REPO_ROOT" + +echo "## Schema Evolve Report" +echo "Repo: $REPO_ROOT" +echo "Timestamp: $(date -u +%Y-%m-%dT%H:%M:%SZ)" +echo "" + +# Discover next migration number +MIG_DIR="$REPO_ROOT/super-legal-mcp-refactored/migrations" +LAST_MIG=$(ls "$MIG_DIR"/*.up.sql 2>/dev/null | xargs -n1 basename | grep -oE '^[0-9]+' | sort -n | tail -1) +NEXT_MIG=$(printf "%03d" $((10#${LAST_MIG:-0} + 1))) +echo " Next migration number: $NEXT_MIG" +echo "" + +# ── Prebuilt PB1/PB2/PB3 ────────────────────────────────────────────────────── +if [ -n "$PREBUILT" ]; then + python3 "$SCRIPT_DIR/generate-migration.py" \ + --repo-root "$REPO_ROOT" --next-mig "$NEXT_MIG" --prebuilt "$PREBUILT" + exit $? +fi + +# ── Dispatch ────────────────────────────────────────────────────────────────── +if [ -n "$TABLE" ] && [ "$KIND" = "add-table" ]; then + python3 "$SCRIPT_DIR/generate-migration.py" \ + --repo-root "$REPO_ROOT" --next-mig "$NEXT_MIG" \ + --table "$TABLE" --kind add-table +elif [ -n "$TABLE" ] && [ "$KIND" = "add-column" ]; then + [ -n "$COLUMN" ] || { echo "ERROR: --column required" >&2; exit 2; } + [ -n "$COLUMN_TYPE" ] || { echo "ERROR: --column-type required" >&2; exit 2; } + python3 "$SCRIPT_DIR/generate-migration.py" \ + --repo-root "$REPO_ROOT" --next-mig "$NEXT_MIG" \ + --table "$TABLE" --kind add-column \ + --column "$COLUMN" --column-type "$COLUMN_TYPE" +elif [ -n "$TOOL" ]; then + python3 "$SCRIPT_DIR/generate-zod-envelope.py" \ + --repo-root "$REPO_ROOT" --tool "$TOOL" +elif [ -n "$OUTPUT_SCHEMA" ]; then + python3 "$SCRIPT_DIR/generate-test-fixture.py" \ + --repo-root "$REPO_ROOT" --output-schema "$OUTPUT_SCHEMA" +else + echo "ERROR: nothing to do — pass --table, --tool, --output-schema, or --prebuilt" >&2 + exit 2 +fi diff --git a/.claude/skills/schema-evolve/scripts/generate-migration.py b/.claude/skills/schema-evolve/scripts/generate-migration.py new file mode 100755 index 000000000..a84c64b9d --- /dev/null +++ b/.claude/skills/schema-evolve/scripts/generate-migration.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python3 +"""schema-evolve: DDL + migration file generator. + +Two modes: + 1. --prebuilt PB1|PB2|PB3 — emit known-pending v7.0.x production fixes + 2. --kind add-table|add-column with --table/--column/--column-type — user-driven DDL pair + +Always emits BOTH: + - migrations/NNN_.up.sql (+ matching .down.sql) + - patch suggestion for ensure*Schema() in src/db/postgres.js (printed, NOT applied) + +Operator reviews patch and applies manually. Existing migrations are immutable. +""" + +import argparse +import re +import sys +from pathlib import Path + + +# ── Prebuilt fix definitions (from v7.0.x audit) ────────────────────────────── + +PREBUILT_FIXES = { + "PB1": { + "name": "citation-source-and-code-execution-inputs", + "description": "Backfill missing migrations for citation_source_links + code_execution_inputs (DDL exists in postgres.js but no migration files were ever shipped — fresh-DB throws).", + "up_sql": """-- citation_source_links: links report citations to upstream source_writes +CREATE TABLE IF NOT EXISTS citation_source_links ( + id SERIAL PRIMARY KEY, + citation_id INTEGER REFERENCES citations(id) ON DELETE CASCADE, + source_write_id INTEGER REFERENCES source_writes(id) ON DELETE CASCADE, + similarity_score REAL, + created_at TIMESTAMPTZ DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_citation_source_citation ON citation_source_links(citation_id); +CREATE INDEX IF NOT EXISTS idx_citation_source_write ON citation_source_links(source_write_id); + +-- code_execution_inputs: lineage rows for run_python_analysis input hashes +CREATE TABLE IF NOT EXISTS code_execution_inputs ( + id SERIAL PRIMARY KEY, + execution_id INTEGER REFERENCES code_executions(id) ON DELETE CASCADE, + input_hash TEXT NOT NULL, + input_source TEXT, + created_at TIMESTAMPTZ DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_code_exec_inputs_execution ON code_execution_inputs(execution_id); +CREATE INDEX IF NOT EXISTS idx_code_exec_inputs_hash ON code_execution_inputs(input_hash); +""", + "down_sql": """DROP TABLE IF EXISTS code_execution_inputs; +DROP TABLE IF EXISTS citation_source_links; +""", + "ensure_patch": """No patch needed — DDL already lives in ensure*Schema() functions. +This migration backfills the missing migration files only.""", + }, + "PB2": { + "name": "access-log-actor-user-id", + "description": "Add access_log.actor_user_id (FK → users.id) so RBAC audit can attribute admin endpoint calls to specific users.", + "up_sql": """ALTER TABLE access_log + ADD COLUMN IF NOT EXISTS actor_user_id INTEGER REFERENCES users(id) ON DELETE SET NULL; + +CREATE INDEX IF NOT EXISTS idx_access_log_actor ON access_log(actor_user_id); +""", + "down_sql": """DROP INDEX IF EXISTS idx_access_log_actor; +ALTER TABLE access_log DROP COLUMN IF EXISTS actor_user_id; +""", + "ensure_patch": """In ensureWave3Schema() (postgres.js around L965), inside the access_log block, add: + + ALTER TABLE access_log + ADD COLUMN IF NOT EXISTS actor_user_id INTEGER REFERENCES users(id) ON DELETE SET NULL; + CREATE INDEX IF NOT EXISTS idx_access_log_actor ON access_log(actor_user_id); +""", + }, + "PB3": { + "name": "code-executions-timeout", + "description": "Add code_executions.timeout BOOLEAN — distinguishes container timeouts from refusals/errors in code-execution telemetry.", + "up_sql": """ALTER TABLE code_executions + ADD COLUMN IF NOT EXISTS timeout BOOLEAN DEFAULT FALSE; + +CREATE INDEX IF NOT EXISTS idx_code_executions_timeout + ON code_executions(timeout) WHERE timeout = TRUE; +""", + "down_sql": """DROP INDEX IF EXISTS idx_code_executions_timeout; +ALTER TABLE code_executions DROP COLUMN IF EXISTS timeout; +""", + "ensure_patch": """In ensureCodeExecutionsSchema() (postgres.js around L999), add after CREATE TABLE block: + + ALTER TABLE code_executions + ADD COLUMN IF NOT EXISTS timeout BOOLEAN DEFAULT FALSE; + CREATE INDEX IF NOT EXISTS idx_code_executions_timeout + ON code_executions(timeout) WHERE timeout = TRUE; +""", + }, +} + + +# ── Utility: which ensure*Schema function owns a table ──────────────────────── + +def detect_ensure_function(table: str) -> str: + """Best-guess mapping table → ensure*Schema() based on category.""" + t = table.lower() + if t in ("hook_audit_log", "code_executions", "code_execution_inputs", + "transcript_events", "agent_progress"): + return "ensureHookSchema" + if t in ("source_writes", "access_log", "human_interventions", "pii_mappings"): + return "ensureWave3Schema" + if t.endswith("_embeddings") or t.endswith("_chunks"): + return "ensureEmbeddingSchema" + if t in ("kg_nodes", "kg_edges", "kg_provenance"): + return "ensureKnowledgeGraphSchema" + if t in ("users", "sessions", "reports", "citations", "citation_source_links"): + return "ensureSessionsSchema" + return "ensure*Schema (operator: please verify which one)" + + +# ── Kebab-case helper ───────────────────────────────────────────────────────── + +def kebab(s: str) -> str: + return re.sub(r"[^a-z0-9]+", "-", s.lower()).strip("-") + + +# ── Main ────────────────────────────────────────────────────────────────────── + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--repo-root", required=True) + p.add_argument("--next-mig", required=True, help="3-digit migration number (e.g. 015)") + p.add_argument("--prebuilt", choices=["PB1", "PB2", "PB3"]) + p.add_argument("--table") + p.add_argument("--kind", choices=["add-table", "add-column"]) + p.add_argument("--column") + p.add_argument("--column-type") + args = p.parse_args() + + repo = Path(args.repo_root) + mig_dir = repo / "super-legal-mcp-refactored" / "migrations" + + if args.prebuilt: + fix = PREBUILT_FIXES[args.prebuilt] + name = fix["name"] + up_sql = fix["up_sql"] + down_sql = fix["down_sql"] + ensure_patch = fix["ensure_patch"] + op_label = f"prebuilt {args.prebuilt}: {fix['description']}" + ensure_fn = "(see patch below)" + elif args.kind == "add-table": + if not args.table: + print("ERROR: --table required for add-table", file=sys.stderr) + sys.exit(2) + name = kebab(f"add-{args.table}") + up_sql = ( + f"-- TODO: replace columns below with the actual table shape.\n" + f"CREATE TABLE IF NOT EXISTS {args.table} (\n" + f" id SERIAL PRIMARY KEY,\n" + f" -- \n" + f" created_at TIMESTAMPTZ DEFAULT NOW()\n" + f");\n" + ) + down_sql = f"DROP TABLE IF EXISTS {args.table};\n" + ensure_fn = detect_ensure_function(args.table) + ensure_patch = ( + f"In {ensure_fn}() (postgres.js), add the matching\n" + f" CREATE TABLE IF NOT EXISTS {args.table} (...) block\n" + f"with the SAME column shape as the migration above. Migration files are\n" + f"immutable post-merge — ensure*Schema() must mirror the migration exactly\n" + f"or fresh-DB boot will diverge from migrated DBs." + ) + op_label = f"add-table {args.table}" + elif args.kind == "add-column": + for req in ("table", "column", "column_type"): + if not getattr(args, req): + print(f"ERROR: --{req.replace('_','-')} required for add-column", + file=sys.stderr) + sys.exit(2) + name = kebab(f"{args.table}-{args.column}") + up_sql = ( + f"ALTER TABLE {args.table}\n" + f" ADD COLUMN IF NOT EXISTS {args.column} {args.column_type};\n" + ) + down_sql = ( + f"ALTER TABLE {args.table} DROP COLUMN IF EXISTS {args.column};\n" + ) + ensure_fn = detect_ensure_function(args.table) + ensure_patch = ( + f"In {ensure_fn}() (postgres.js), inside the {args.table} block, add:\n\n" + f" ALTER TABLE {args.table}\n" + f" ADD COLUMN IF NOT EXISTS {args.column} {args.column_type};\n\n" + f"Why both layers: CREATE TABLE IF NOT EXISTS is a no-op against\n" + f"existing prod DBs — only ALTER TABLE ADD COLUMN IF NOT EXISTS\n" + f"actually evolves the column on existing rows. v6.2.3 hotfix root cause." + ) + op_label = f"add-column {args.table}.{args.column} ({args.column_type})" + else: + print("ERROR: pass --prebuilt or --kind", file=sys.stderr) + sys.exit(2) + + up_path = mig_dir / f"{args.next_mig}_{name}.up.sql" + down_path = mig_dir / f"{args.next_mig}_{name}.down.sql" + + if up_path.exists(): + print(f"ERROR: {up_path.name} already exists — refusing to overwrite", + file=sys.stderr) + sys.exit(2) + + up_header = f"-- {args.next_mig}_{name}.up.sql\n-- {op_label}\n\n" + down_header = f"-- {args.next_mig}_{name}.down.sql\n-- Rollback for {op_label}\n\n" + + up_path.write_text(up_header + up_sql) + down_path.write_text(down_header + down_sql) + + # ── Report ──────────────────────────────────────────────────────────────── + print(f"### DDL artifacts") + print(f"✓ migrations/{up_path.name} (created)") + print(f"✓ migrations/{down_path.name} (created)") + print() + print(f"### ensure*Schema() patch (preview — NOT applied)") + print(f"Target function: {ensure_fn}") + print() + for line in ensure_patch.splitlines(): + print(f" {line}") + print() + print(f"### Operator next steps") + print(f"- [ ] Review {up_path.name} + {down_path.name}") + print(f"- [ ] Apply ensure*Schema() patch to src/db/postgres.js") + print(f"- [ ] Run: node --check super-legal-mcp-refactored/src/db/postgres.js") + print(f"- [ ] Run: feature-compliance-scaffold --git-range main..HEAD") + print(f" → expect D7 (dual-path schema) PASSED") + print(f"- [ ] Open PR") + + +if __name__ == "__main__": + main() diff --git a/.claude/skills/schema-evolve/scripts/generate-test-fixture.py b/.claude/skills/schema-evolve/scripts/generate-test-fixture.py new file mode 100755 index 000000000..b0ae24514 --- /dev/null +++ b/.claude/skills/schema-evolve/scripts/generate-test-fixture.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python3 +"""schema-evolve: Output JSON schema + round-trip test fixture generator. + +Emits: + 1. src/schemas/.json — JSON Schema (mirrors BankruptcyResearchMemo.json) + 2. test/sdk/-envelope.test.js — round-trip Zod test scaffold + +Pattern mirrors test/sdk/code-execution-bridge.test.js (24+ assertions, 6 groups). +""" + +import argparse +import json +import re +import sys +from pathlib import Path + + +def to_kebab(name: str) -> str: + s = re.sub(r"(.)([A-Z][a-z]+)", r"\1-\2", name) + s = re.sub(r"([a-z0-9])([A-Z])", r"\1-\2", s) + return s.lower() + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--repo-root", required=True) + p.add_argument("--output-schema", required=True, + help="PascalCase schema name (e.g. BondResearchMemo)") + args = p.parse_args() + + repo = Path(args.repo_root) + schemas_dir = repo / "super-legal-mcp-refactored" / "src" / "schemas" + test_dir = repo / "super-legal-mcp-refactored" / "test" / "sdk" + + schema_name = args.output_schema + if not re.match(r"^[A-Z][A-Za-z0-9]+$", schema_name): + print(f"ERROR: --output-schema must be PascalCase (got: {schema_name})", + file=sys.stderr) + sys.exit(2) + + schema_file = schemas_dir / f"{schema_name}.json" + test_file = test_dir / f"{to_kebab(schema_name)}-envelope.test.js" + + if schema_file.exists(): + print(f"ERROR: {schema_file} already exists — refusing to overwrite", + file=sys.stderr) + sys.exit(2) + + # ── JSON Schema (mirrors BankruptcyResearchMemo.json shape) ─────────────── + schema_obj = { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": f"https://super-legal.app/schemas/{schema_name}.json", + "title": schema_name, + "description": f"TODO: describe {schema_name} output structure", + "type": "object", + "required": ["sections", "citations"], + "properties": { + "sections": { + "type": "array", + "items": { + "type": "object", + "required": ["heading", "body"], + "properties": { + "heading": {"type": "string"}, + "body": {"type": "string"}, + "subsections": { + "type": "array", + "items": {"$ref": "#/properties/sections/items"}, + }, + }, + }, + }, + "citations": { + "type": "array", + "items": { + "type": "object", + "required": ["id", "url"], + "properties": { + "id": {"type": "string"}, + "url": {"type": "string", "format": "uri"}, + "title": {"type": "string"}, + "source_type": {"type": "string"}, + }, + }, + }, + "metadata": { + "type": "object", + "properties": { + "generated_at": {"type": "string", "format": "date-time"}, + "model_id": {"type": "string"}, + "session_key": {"type": "string"}, + }, + }, + }, + } + + schemas_dir.mkdir(parents=True, exist_ok=True) + schema_file.write_text(json.dumps(schema_obj, indent=2) + "\n") + + # ── Round-trip test scaffold ───────────────────────────────────────────── + test_dir.mkdir(parents=True, exist_ok=True) + + test_template = f"""// {to_kebab(schema_name)}-envelope.test.js +// Round-trip test scaffold for {schema_name}. +// Mirrors test/sdk/code-execution-bridge.test.js pattern (6 groups). +// Generated by schema-evolve. Replace TODOs with real fixtures. + +import {{ describe, it, expect }} from 'vitest'; +import {{ readFileSync }} from 'node:fs'; +import {{ fileURLToPath }} from 'node:url'; +import {{ dirname, join }} from 'node:path'; +import Ajv from 'ajv'; +import addFormats from 'ajv-formats'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const schemaPath = join(__dirname, '../../src/schemas/{schema_name}.json'); +const schema = JSON.parse(readFileSync(schemaPath, 'utf8')); + +const ajv = new Ajv({{ strict: false, allErrors: true }}); +addFormats(ajv); +const validate = ajv.compile(schema); + +describe('{schema_name} — round-trip envelope', () => {{ + // Group 1: Schema structure + it('declares required top-level fields', () => {{ + expect(schema.required).toContain('sections'); + expect(schema.required).toContain('citations'); + }}); + + it('uses recursive subsections via $ref', () => {{ + expect(schema.properties.sections.items.properties.subsections.items.$ref) + .toBe('#/properties/sections/items'); + }}); + + // Group 2: Valid fixtures pass + it('accepts a minimal valid memo', () => {{ + const fixture = {{ + sections: [{{ heading: 'Summary', body: 'Body text.' }}], + citations: [{{ id: 'c1', url: 'https://example.com' }}], + }}; + expect(validate(fixture)).toBe(true); + }}); + + it('accepts nested subsections', () => {{ + const fixture = {{ + sections: [ + {{ + heading: 'Top', + body: 'top body', + subsections: [{{ heading: 'Sub', body: 'sub body' }}], + }}, + ], + citations: [{{ id: 'c1', url: 'https://example.com' }}], + }}; + expect(validate(fixture)).toBe(true); + }}); + + // Group 3: Drift detection + it('rejects missing required fields', () => {{ + expect(validate({{ sections: [] }})).toBe(false); + }}); + + it('rejects malformed citation URL', () => {{ + const fixture = {{ + sections: [{{ heading: 'h', body: 'b' }}], + citations: [{{ id: 'c1', url: 'not-a-url' }}], + }}; + expect(validate(fixture)).toBe(false); + }}); + + // Group 4: Metadata pass-through + it('accepts optional metadata block', () => {{ + const fixture = {{ + sections: [{{ heading: 'h', body: 'b' }}], + citations: [{{ id: 'c1', url: 'https://example.com' }}], + metadata: {{ + generated_at: '2026-05-07T00:00:00Z', + model_id: 'claude-sonnet-4-6', + session_key: 'sess-abc123', + }}, + }}; + expect(validate(fixture)).toBe(true); + }}); + + // Group 5: TODO — domain-specific fixtures + it.todo('TODO: add real fixture from a captured {schema_name} response'); + + // Group 6: TODO — drift regression cases + it.todo('TODO: regression fixture for previous shape that broke memo writers'); +}}); +""" + + test_file.write_text(test_template) + + # ── Report ──────────────────────────────────────────────────────────────── + print(f"### Output schema artifacts") + print(f"✓ src/schemas/{schema_file.name} (created)") + print() + print(f"### Test fixture artifacts") + print(f"✓ test/sdk/{test_file.name} (created)") + print() + print(f"### Operator next steps") + print(f"- [ ] Replace TODO fields in {schema_file.name} with real shape") + print(f"- [ ] Replace it.todo() blocks with real captured fixtures") + print(f"- [ ] Run: npx vitest run test/sdk/{test_file.name}") + print(f"- [ ] Wire schema into the generating section writer") + print(f"- [ ] Open PR") + + +if __name__ == "__main__": + main() diff --git a/.claude/skills/schema-evolve/scripts/generate-zod-envelope.py b/.claude/skills/schema-evolve/scripts/generate-zod-envelope.py new file mode 100755 index 000000000..e1a6de023 --- /dev/null +++ b/.claude/skills/schema-evolve/scripts/generate-zod-envelope.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +"""schema-evolve: Zod tool-envelope schema generator. + +Emits a new `EnvelopeSchema` block + registry-update patch suggestion +for src/schemas/toolEnvelopes.js. Read-only against the existing file — +operator reviews patch + applies via Edit. + +Pattern mirrors the 5 existing schemas: + - z.object({...}).passthrough() + - _hybrid_metadata: hybridMetadataSchema.optional() if MCP-routed + - registered in TOOL_ENVELOPE_SCHEMAS (Object.freeze) at L115 + - validateToolEnvelope() at L137 picks it up automatically +""" + +import argparse +import re +import sys +from pathlib import Path + + +def to_camel(snake: str) -> str: + parts = snake.split("_") + return parts[0] + "".join(p.capitalize() for p in parts[1:]) + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--repo-root", required=True) + p.add_argument("--tool", required=True, help="MCP tool_name (snake_case)") + p.add_argument("--no-mcp", action="store_true", + help="Tool is not MCP-routed (omit _hybrid_metadata)") + args = p.parse_args() + + repo = Path(args.repo_root) + envelopes_path = repo / "super-legal-mcp-refactored" / "src" / "schemas" / "toolEnvelopes.js" + + if not envelopes_path.exists(): + print(f"ERROR: {envelopes_path} not found", file=sys.stderr) + sys.exit(2) + + text = envelopes_path.read_text() + + # Sanity check: schema name must not already exist + schema_const = f"{to_camel(args.tool)}EnvelopeSchema" + if re.search(rf"\bexport const {re.escape(schema_const)}\b", text): + print(f"ERROR: {schema_const} already declared in toolEnvelopes.js", + file=sys.stderr) + sys.exit(2) + + if f" {args.tool}: " in text: + print(f"ERROR: '{args.tool}' already in TOOL_ENVELOPE_SCHEMAS registry", + file=sys.stderr) + sys.exit(2) + + hybrid_line = "" if args.no_mcp else " _hybrid_metadata: hybridMetadataSchema.optional(),\n" + + new_block = f"""// {args.tool} — TODO: describe payload shape + drift consequences here. +export const {schema_const} = z.object({{ + // TODO: replace with real fields from the tool's response shape. + // Use .nullable().optional() liberally — fail-open for forward compat. + results: z.array(z.unknown()).default([]), +{hybrid_line}}}).passthrough(); +""" + + # ── Output: report + patch suggestions ────────────────────────────────────── + print(f"### Zod envelope artifacts") + print(f" Tool: {args.tool}") + print(f" Schema const: {schema_const}") + print(f" Target file: src/schemas/toolEnvelopes.js") + print() + print(f"### Patch 1 — append schema block (insert before TOOL_ENVELOPE_SCHEMAS at ~L115)") + print() + for line in new_block.splitlines(): + print(f" {line}") + print() + print(f"### Patch 2 — register in TOOL_ENVELOPE_SCHEMAS (around L115-L121)") + print() + print(f" export const TOOL_ENVELOPE_SCHEMAS = Object.freeze({{") + print(f" fetch_document: fetchDocumentEnvelopeSchema,") + print(f" exa_web_search: exaWebSearchEnvelopeSchema,") + print(f" kg_create_node: kgCreateNodeEnvelopeSchema,") + print(f" search_cases: searchCasesEnvelopeSchema,") + print(f" search_sec_filings: searchSecFilingsEnvelopeSchema,") + print(f" {args.tool}: {schema_const}, // ← NEW") + print(f" }});") + print() + print(f"### Operator next steps") + print(f"- [ ] Apply Patch 1 + Patch 2 via Edit") + print(f"- [ ] Replace TODO fields with actual tool response shape") + print(f"- [ ] Run: node --check src/schemas/toolEnvelopes.js") + print(f"- [ ] Verify validateToolEnvelope('{args.tool}', sample) at hookDBBridge.js:1203-1208") + print(f"- [ ] Run round-trip test scaffold (see generate-test-fixture.py)") + print(f"- [ ] Open PR") + + +if __name__ == "__main__": + main() diff --git a/.claude/skills/sdk-upgrade/SKILL.md b/.claude/skills/sdk-upgrade/SKILL.md new file mode 100644 index 000000000..0c34644d9 --- /dev/null +++ b/.claude/skills/sdk-upgrade/SKILL.md @@ -0,0 +1,92 @@ +--- +name: sdk-upgrade +description: Automate Anthropic SDK version bumps for Super Legal MCP. Fetches CHANGELOG diff, audits all call sites for breaking changes, validates beta-token compatibility, runs the SDK regression test suite, and emits a one-page operator report. Replaces the ½-day manual loop Edwin has done 5+ times since 0.2.47. Use when bumping `@anthropic-ai/claude-agent-sdk` or `@anthropic-ai/sdk` versions. Triggers — sdk upgrade, anthropic sdk bump, agent sdk update, sdk version bump, /sdk-upgrade. Supports flags — --to , --check (dry-run), --regression-test (re-run 14 SDK tests). +--- + +# SDK Upgrade — Anthropic SDK Version Bump + +## Workflow + +```bash +/sdk-upgrade --check # dry-run audit only — no package.json changes +/sdk-upgrade --to # full upgrade flow (fetch CHANGELOG → audit → bump → test) +/sdk-upgrade --regression-test # re-run 14 SDK test files against current version +``` + +## What it does + +1. **Fetch CHANGELOG** for the version delta via `gh api repos/anthropics/claude-agent-sdk-typescript/releases` and `npm view @anthropic-ai/claude-agent-sdk versions`. +2. **Audit call sites** — greps the 5 known SDK consumer files for usage of API symbols that changed in the delta (e.g., `agentQuery`, `agentProgressSummaries`, `settingSources`, `output_config`, `maxThinkingTokens`). +3. **Validate beta tokens** — checks 5 active beta headers (`context-1m-2025-08-07`, `interleaved-thinking-2025-05-14`, `effort-2025-11-24`, `files-api-2025-04-14`, `code_execution_20250825`) against the new version's release notes for deprecation/graduation status. +4. **Bump versions** in `super-legal-mcp-refactored/package.json` (both `@anthropic-ai/claude-agent-sdk` and `@anthropic-ai/sdk` peer dep). +5. **Run regression test suite** — 14 SDK tests under `test/sdk/` (full list in `references/call-sites.md`). +6. **Emit operator report** — markdown summary with breaking-change matrix, beta-token diff, test pass/fail, and next-step remediation. + +## Pre-flight + +Required: `python3`, `gh`, `npm`, `node`. The skill reads code locally (no `gcloud` needed). + +## Current state (baseline) + +Verified 2026-05-07: +- `@anthropic-ai/claude-agent-sdk`: `0.2.119` (exact pin) +- `@anthropic-ai/sdk`: `^0.86.1` +- `zod`: `^4.3.6` + +## Known regressions to check (per upgrade) + +GitHub issues (Number531/Legal-API): +- **#25** — `maxThinkingTokens` breaks all hooks/streaming on Agent SDK path (still open; commented out in current code) +- **#14** — `defer_loading` for Agent SDK (blocked on upstream) +- **#210** — Agent/Task tool broken in `query()` mode (resolved 0.2.70+) +- **#40** — `task_progress` upstream shipping status (still pending as of 0.2.119) +- **#66** — MCP Protocol SubagentStart race (resolved 0.2.97) +- **#79** — SDK 0.2.119 upgrade notes + +## Output format + +``` +## SDK Upgrade Report +From: 0.2.119 → To: 0.2.121 +Timestamp: 2026-05-07T... + +### CHANGELOG delta (2 versions) +- 0.2.120: ... +- 0.2.121: ... + +### Call site audit +✓ agentStreamHandler.js:280-320 — agentQuery shape unchanged +✓ p0Orchestrator.js:106-135 — settingSources accepted +⚠ codeExecutionBridge.js:325 — files-api-2025-04-14 now graduated; consider dropping beta header + +### Beta tokens +✓ context-1m-2025-08-07 — still required +⚠ interleaved-thinking-2025-05-14 — DEPRECATED on 4.6 per CHANGELOG +✓ effort-2025-11-24 — functional +✓ files-api-2025-04-14 — graduated; safe to drop in next major +✓ code_execution_20250825 — server-tool unchanged + +### Regression tests (14 files) +✓ agent-stream-handler.test.js +✓ p0-orchestrator.test.js +✓ ... (12 more) + +### Recommended next steps +- [ ] Drop interleaved-thinking-2025-05-14 from agentQuery betas if no 4.5 fallback needed +- [ ] Open follow-up issue if files-api graduation breaks existing chart pipeline +``` + +Exit codes: `0` = clean, `1` = warnings (review before merge), `2` = call-site break or test failure. + +## Sequencing + +Run **before** `/deploy`. After upgrade, `/post-deploy-verify` Tier 2 covers runtime regression (FMP tools, code-execution models, Cloud Trace). + +## Read-only guarantee + +The skill never: +- Mutates code other than `package.json` version pin (gated behind `--to`, NOT `--check`) +- Runs the live container or hits production +- Auto-rolls back + +It reports what changed; operator decides remediation. diff --git a/.claude/skills/sdk-upgrade/references/call-sites.md b/.claude/skills/sdk-upgrade/references/call-sites.md new file mode 100644 index 000000000..d1c4742bf --- /dev/null +++ b/.claude/skills/sdk-upgrade/references/call-sites.md @@ -0,0 +1,46 @@ +# SDK Call Sites — Truth Map + +7 source files audited per upgrade. Verified 2026-05-07 against `0.2.119`. + +## Agent SDK consumers (3) + +| File | Lines | Notes | +|------|-------|-------| +| `src/server/agentStreamHandler.js` | 280-320 | Main `agentQuery` invocation; betas at 296-298; `settingSources: []` at 315 | +| `src/server/p0Orchestrator.js` | 100-135 | `for await … agentQuery({...})` at 106; `agentProgressSummaries: true` at 113 | +| `src/server/claude-sdk-server.js` | 284-310 | Constraints comment block; nearby orchestrator config | + +## Anthropic SDK (Messages API) consumers (2) + +| File | Lines | Notes | +|------|-------|-------| +| `src/tools/codeExecutionBridge.js` | 325, 352, 484 | Standard `client.messages.create` path; `code_execution_20250825` server-tool | +| `src/utils/skillsRequestBuilder.js` | 41, 52, 55 | Beta header builder for skills | + +## Hook lifecycle (2) + +| File | Lines | Notes | +|------|-------|-------| +| `src/utils/sdkHooks.js` | exports | `sdkHooksConfig` lifecycle handlers | +| `src/utils/hookDBBridge.js` | 1739-1750 | `HOOKS_TO_BRIDGE` array — must verify on upgrade | + +## Test files (14) + +Re-run via `test-harness.sh` after every upgrade: + +``` +test/sdk/agent-stream-handler.test.js +test/sdk/p0-orchestrator.test.js +test/sdk/subagents.test.js +test/sdk/subagents-e2e.test.js +test/sdk/code-execution-bridge.test.js +test/sdk/hookDBBridge.test.js +test/sdk/streaming-events.test.js +test/sdk/thinking-preservation.test.js +test/sdk/structured-outputs.test.js +test/sdk/domain-mcp-servers.test.js +test/sdk/skills-headers.test.js +test/sdk/prompt-caching.test.js +test/sdk/stream-context.test.js +test/sdk/server-refactor-regression.test.js +``` diff --git a/.claude/skills/sdk-upgrade/scripts/audit-call-sites.py b/.claude/skills/sdk-upgrade/scripts/audit-call-sites.py new file mode 100755 index 000000000..a18c064ad --- /dev/null +++ b/.claude/skills/sdk-upgrade/scripts/audit-call-sites.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +"""Audit Anthropic SDK call sites + beta-token usage in Super Legal MCP. + +Reports: + - File:line of every agentQuery / agentProgressSummaries / settingSources call + - Beta token references (5 known) + - Unexpected SDK API symbols (potential breaking-change indicators) + +Usage: audit-call-sites.py --repo-root [--betas-only] +""" + +import argparse +import re +import sys +from pathlib import Path + + +SDK_FILES = [ + "src/server/agentStreamHandler.js", + "src/server/p0Orchestrator.js", + "src/server/claude-sdk-server.js", + "src/tools/codeExecutionBridge.js", + "src/utils/skillsRequestBuilder.js", + "src/utils/sdkHooks.js", + "src/utils/hookDBBridge.js", +] + +API_SYMBOLS = [ + "agentQuery", "agentProgressSummaries", "settingSources", + "output_config", "output_format", "maxThinkingTokens", + "HOOKS_TO_BRIDGE", +] + +BETA_TOKENS = [ + "context-1m-2025-08-07", + "interleaved-thinking-2025-05-14", + "effort-2025-11-24", + "files-api-2025-04-14", + "code_execution_20250825", +] + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--repo-root", required=True) + p.add_argument("--betas-only", action="store_true") + args = p.parse_args() + + base = Path(args.repo_root) / "super-legal-mcp-refactored" + + if not args.betas_only: + for rel in SDK_FILES: + f = base / rel + if not f.exists(): + continue + text = f.read_text(errors="replace") + for sym in API_SYMBOLS: + for m in re.finditer(rf"\b{re.escape(sym)}\b", text): + line_no = text.count("\n", 0, m.start()) + 1 + print(f" ✓ {rel}:{line_no} — {sym}") + + if args.betas_only or not args.betas_only: + # Beta tokens scan + if args.betas_only: + print("") + seen = {b: [] for b in BETA_TOKENS} + for rel in SDK_FILES: + f = base / rel + if not f.exists(): + continue + text = f.read_text(errors="replace") + for tok in BETA_TOKENS: + for m in re.finditer(re.escape(tok), text): + line_no = text.count("\n", 0, m.start()) + 1 + seen[tok].append(f"{rel}:{line_no}") + for tok, refs in seen.items(): + if refs: + print(f" ✓ {tok} ({len(refs)} ref{'s' if len(refs) != 1 else ''})") + for r in refs[:3]: + print(f" {r}") + else: + print(f" — {tok} (not found — may be deprecated)") + + +if __name__ == "__main__": + main() diff --git a/.claude/skills/sdk-upgrade/scripts/fetch-changelog.sh b/.claude/skills/sdk-upgrade/scripts/fetch-changelog.sh new file mode 100755 index 000000000..d1256fcec --- /dev/null +++ b/.claude/skills/sdk-upgrade/scripts/fetch-changelog.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# fetch-changelog.sh +# Fetches release notes from anthropics/claude-agent-sdk-typescript between two versions. + +set -uo pipefail + +CURRENT="${1:-}" +TARGET="${2:-latest}" + +[ -z "$CURRENT" ] && { echo " (no current version)"; exit 0; } + +if [ "$TARGET" = "latest" ]; then + TARGET=$(npm view @anthropic-ai/claude-agent-sdk version 2>/dev/null || echo "unknown") +fi + +echo " Fetching releases from anthropics/claude-agent-sdk-typescript..." +gh api "repos/anthropics/claude-agent-sdk-typescript/releases?per_page=30" 2>/dev/null \ + | python3 -c " +import json, sys, re +try: + rels = json.load(sys.stdin) +except Exception: + print(' (gh api unavailable; install gh + auth, or use npm view)') + sys.exit(0) +current = '$CURRENT'.lstrip('^~=') +target = '$TARGET'.lstrip('^~=') +def vkey(v): + parts = re.findall(r'\d+', v) + return tuple(int(p) for p in parts) if parts else (0,) +in_range = [] +for r in rels: + tag = (r.get('tag_name') or '').lstrip('v') + if not tag: continue + if vkey(current) < vkey(tag) <= vkey(target): + in_range.append((tag, r.get('name') or '', r.get('body') or '')) +in_range.sort(key=lambda x: vkey(x[0])) +if not in_range: + print(f' No releases between {current} and {target}') +else: + for tag, name, body in in_range: + body_first = body.split('\n')[0] if body else '' + print(f' - {tag}: {body_first[:200]}') +" diff --git a/.claude/skills/sdk-upgrade/scripts/test-harness.sh b/.claude/skills/sdk-upgrade/scripts/test-harness.sh new file mode 100755 index 000000000..17e539935 --- /dev/null +++ b/.claude/skills/sdk-upgrade/scripts/test-harness.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# Re-run the 14 SDK-relevant test files to catch regressions post-upgrade. +# Usage: test-harness.sh +set -uo pipefail + +REPO_ROOT="${1:-}" +[ -n "$REPO_ROOT" ] || { echo "ERROR: pass repo root as arg 1" >&2; exit 2; } + +cd "$REPO_ROOT/super-legal-mcp-refactored" || exit 2 + +# 14 SDK-relevant test files (per Phase B2 plan; trim if some don't exist yet) +TESTS=( + test/sdk/agent-stream-handler.test.js + test/sdk/p0-orchestrator.test.js + test/sdk/subagents.test.js + test/sdk/subagents-e2e.test.js + test/sdk/code-execution-bridge.test.js + test/sdk/hookDBBridge.test.js + test/sdk/streaming-events.test.js + test/sdk/thinking-preservation.test.js + test/sdk/structured-outputs.test.js + test/sdk/domain-mcp-servers.test.js + test/sdk/skills-headers.test.js + test/sdk/prompt-caching.test.js + test/sdk/stream-context.test.js + test/sdk/server-refactor-regression.test.js +) + +PASS=0 +FAIL=0 +SKIP=0 + +for t in "${TESTS[@]}"; do + if [ ! -f "$t" ]; then + echo " — SKIP $t (file not found)" + SKIP=$((SKIP + 1)) + continue + fi + echo " Running $t..." + if npx vitest run "$t" --reporter=basic 2>&1 | tail -3; then + PASS=$((PASS + 1)) + else + FAIL=$((FAIL + 1)) + fi +done + +echo "" +echo " Tests: $PASS passed, $FAIL failed, $SKIP skipped" +exit $FAIL diff --git a/.claude/skills/sdk-upgrade/scripts/upgrade.sh b/.claude/skills/sdk-upgrade/scripts/upgrade.sh new file mode 100755 index 000000000..eb1c5c394 --- /dev/null +++ b/.claude/skills/sdk-upgrade/scripts/upgrade.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +# sdk-upgrade entry point. +# +# Usage: +# upgrade.sh --check # dry-run audit (no package.json changes) +# upgrade.sh --to # full upgrade flow +# upgrade.sh --regression-test # re-run 14 SDK tests against current version + +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# ── Resolve repo root ───────────────────────────────────────────────────────── +cur="$SCRIPT_DIR" +while [ "$cur" != "/" ]; do + if [ -d "$cur/super-legal-mcp-refactored" ]; then + REPO_ROOT="$cur" + break + fi + cur="$(dirname "$cur")" +done +[ -n "${REPO_ROOT:-}" ] || { echo "ERROR: cannot locate repo root" >&2; exit 2; } + +# ── Defaults ────────────────────────────────────────────────────────────────── +MODE="check" +TARGET_VERSION="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --check) MODE="check"; shift ;; + --to) MODE="upgrade"; TARGET_VERSION="$2"; shift 2 ;; + --regression-test) MODE="test"; shift ;; + -h|--help) + grep -E '^#( |$)' "$0" | sed 's/^# \{0,1\}//' | head -10 + exit 0 ;; + *) + echo "Unknown flag: $1" >&2 + exit 2 ;; + esac +done + +# ── Pre-flight ──────────────────────────────────────────────────────────────── +for cmd in python3 git gh npm node; do + command -v "$cmd" >/dev/null || { echo "ERROR: $cmd not found" >&2; exit 2; } +done + +cd "$REPO_ROOT" + +# ── Read current pins ───────────────────────────────────────────────────────── +PKG_JSON="$REPO_ROOT/super-legal-mcp-refactored/package.json" +[ -f "$PKG_JSON" ] || { echo "ERROR: package.json not found at $PKG_JSON" >&2; exit 2; } + +CURRENT_AGENT_SDK=$(python3 -c "import json; d=json.load(open('$PKG_JSON')); print(d.get('dependencies',{}).get('@anthropic-ai/claude-agent-sdk','none'))") +CURRENT_BASE_SDK=$(python3 -c "import json; d=json.load(open('$PKG_JSON')); print(d.get('dependencies',{}).get('@anthropic-ai/sdk','none'))") + +echo "## SDK Upgrade Report" +echo "Mode: $MODE" +echo "Current: claude-agent-sdk=$CURRENT_AGENT_SDK | sdk=$CURRENT_BASE_SDK" +echo "Timestamp: $(date -u +%Y-%m-%dT%H:%M:%SZ)" +echo "" + +# ── Mode dispatch ───────────────────────────────────────────────────────────── +case "$MODE" in + check) + [ -z "$TARGET_VERSION" ] && TARGET_VERSION="latest" + echo "### CHANGELOG fetch (target: $TARGET_VERSION)" + bash "$SCRIPT_DIR/fetch-changelog.sh" "$CURRENT_AGENT_SDK" "$TARGET_VERSION" || true + echo "" + echo "### Call site audit" + python3 "$SCRIPT_DIR/audit-call-sites.py" --repo-root "$REPO_ROOT" + echo "" + echo "### Beta tokens" + python3 "$SCRIPT_DIR/audit-call-sites.py" --repo-root "$REPO_ROOT" --betas-only + ;; + upgrade) + [ -z "$TARGET_VERSION" ] && { echo "ERROR: --to required" >&2; exit 2; } + echo "### CHANGELOG fetch ($CURRENT_AGENT_SDK → $TARGET_VERSION)" + bash "$SCRIPT_DIR/fetch-changelog.sh" "$CURRENT_AGENT_SDK" "$TARGET_VERSION" || true + echo "" + echo "### Call site audit" + python3 "$SCRIPT_DIR/audit-call-sites.py" --repo-root "$REPO_ROOT" + echo "" + echo "### Bumping package.json" + python3 -c " +import json +p = '$PKG_JSON' +d = json.load(open(p)) +d['dependencies']['@anthropic-ai/claude-agent-sdk'] = '$TARGET_VERSION' +json.dump(d, open(p, 'w'), indent=2) +print(' bumped to $TARGET_VERSION') +" + echo "" + echo "### npm install" + (cd "$REPO_ROOT/super-legal-mcp-refactored" && npm install 2>&1 | tail -5) + echo "" + echo "### Regression tests" + bash "$SCRIPT_DIR/test-harness.sh" "$REPO_ROOT" + ;; + test) + bash "$SCRIPT_DIR/test-harness.sh" "$REPO_ROOT" + ;; +esac + +echo "" +echo "### Done"