diff --git a/.env.example b/.env.example index 31327c5a7..e26a5c667 100644 --- a/.env.example +++ b/.env.example @@ -260,3 +260,17 @@ OTEL_EXPORTER_OTLP_PROTOCOL=grpc # Metrics export interval in milliseconds (default: 60 seconds) OTEL_METRIC_EXPORT_INTERVAL=60000 + +# =========================================== +# CANARY INVARIANT HARNESS (Optional, staging/dev) +# =========================================== + +# Continuous orchestration-invariant watcher (CANARY-001 / Issue #411). +# Set to 1 on staging/dev to run the 5-min check loop. Production stays 0. +CANARY_ENABLED=0 + +# Slack incoming webhook URL for canary green→red transitions. +# Get from: https://api.slack.com/apps → your app → Incoming Webhooks → Add +# The URL is the credential — anyone with it can post to that one channel. +# Unset = canary cycles run silently (violations still persisted to DB). +CANARY_SLACK_WEBHOOK_URL= diff --git a/config/canary-fleet.yaml b/config/canary-fleet.yaml new file mode 100644 index 000000000..06825807d --- /dev/null +++ b/config/canary-fleet.yaml @@ -0,0 +1,63 @@ +# Canary Invariant Harness — load-generator fleet (CANARY-001 / Issue #411). +# +# This manifest defines the synthetic agents that the canary watcher +# observes. Without traffic, every invariant holds trivially and the +# harness produces no signal. The fleet is what makes the harness useful. +# +# Deploy on staging/dev with: +# +# curl -sS -X POST -H "Authorization: Bearer " \ +# -H "Content-Type: application/json" \ +# -d "{\"manifest\": $(jq -Rs . < config/canary-fleet.yaml)}" \ +# http://localhost:8000/api/systems/deploy +# +# The watcher service (src/backend/services/canary_service.py) runs every +# 5 min independently of this fleet — it'll just observe trivially-green +# state until the fleet generates load. +# +# Permissions are explicitly empty (preset: none) — fleet members observe +# their own orchestration, not each other. + +name: canary-fleet +description: Canary harness load generators (Issue #411 Phase 1) + +agents: + # Constant slot churn — fires every minute (cron's minimum). Exercises + # S-01 (slot–row bijection) and E-02 (no phantom reversal) by producing + # a steady stream of acquire / release cycles. + # + # `test-echo` is Trinity's minimal stock template (no MCP servers, no + # credentials, deterministic short reply) — exactly what slot-churn + # invariants need. + burst: + template: local:test-echo + resources: + cpu: "1" + memory: "512m" + schedules: + - name: heartbeat + cron: "* * * * *" # every minute + message: "ok" + timezone: UTC + description: Constant load to exercise slot–row bijection (S-01) and reversal (E-02) + + # Slower cadence so multiple slots overlap across cycles. Phase 2 + # invariants (S-03 / E-01 / E-06) want a longer-lived slot to scrutinize; + # Phase 1 only needs the slower cron to broaden snapshot coverage, so + # reusing `test-echo` is fine here too. + long: + template: local:test-echo + resources: + cpu: "1" + memory: "1g" + schedules: + - name: long-task + cron: "*/5 * * * *" # every 5 min + message: "long-task ping" + timezone: UTC + description: Slower cadence to overlap slots across canary cycles (Phase 2 will swap in a real long task) + +# No cross-agent permissions — the canary fleet observes the orchestration +# layer, not each other. +permissions: + preset: none diff --git a/docker-compose.yml b/docker-compose.yml index 5ed64f780..fbdb9b189 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -51,6 +51,14 @@ services: - SLACK_SIGNING_SECRET=${SLACK_SIGNING_SECRET:-} # SSH access host override - SSH_HOST=${SSH_HOST:-} + # Canary invariant harness (CANARY-001 / Issue #411). + # When 1, services/canary_service.py runs the 5-min watcher loop on + # staging/dev. Default 0 — production users see no canary activity. + - CANARY_ENABLED=${CANARY_ENABLED:-0} + # Slack alert sink for canary green→red transitions. URL is the + # credential — leaking it lets anyone post to that one channel. + # Unset = canary cycles run silently (still persists violations). + - CANARY_SLACK_WEBHOOK_URL=${CANARY_SLACK_WEBHOOK_URL:-} volumes: - /var/run/docker.sock:/var/run/docker.sock:ro - ./src/backend:/app diff --git a/docs/memory/architecture.md b/docs/memory/architecture.md index 24b518669..d2d89860a 100644 --- a/docs/memory/architecture.md +++ b/docs/memory/architecture.md @@ -405,6 +405,7 @@ Services that run continuously in the backend process: | **Capacity Maintenance** | `capacity_manager.py` | Calls `CapacityManager.run_maintenance()` every 60s — expires stale queued tasks (>24h) and drains orphans after restart. (BACKLOG-001 / CAPACITY-CONSOLIDATE #428) | | **Audit Retention** | `audit_retention_service.py` | Daily APScheduler job at 04:15 UTC that DELETEs `audit_log` rows past the retention window. Configured via `AUDIT_LOG_RETENTION_DAYS` (default 365, floored at 365 — the `audit_log_no_delete` trigger refuses younger rows). Pruning ages out hash-chain history past the cutoff by design. (#552) | | **Session Cleanup** | `session_cleanup_service.py` | Periodic JSONL reaper for the Session tab. Default 6h cycle (`poll_interval_seconds`); each cycle diffs every running agent's `~/.claude/projects/-home-developer/.jsonl` set against `agent_sessions.cached_claude_session_id` and deletes JSONLs not in the keep set whose mtime is older than `min_age_seconds` (default 1h race guard). Synchronous best-effort `reap_jsonl()` is also called by the session router on user-initiated reset/delete so the disk reclaim is immediate. Uses `execute_command_in_container` (no agent-server endpoint required). (SESSION_TAB Phase 4.2) | +| **Canary Watcher** | `canary_service.py` | Continuous orchestration-invariant harness (CANARY-001 / Issue #411). Every 5 min: `collect_snapshot()` over Redis × SQLite × agent registries, runs deterministic invariant library (S-01, E-02, L-03 in Phase 1), persists violations to `canary_violations`, classifies green→red transitions and fires one Slack webhook POST per transition (`CANARY_SLACK_WEBHOOK_URL` env var; unset = silent sink). Disabled by default; enable on staging/dev with `CANARY_ENABLED=1`. | The **agent server** also runs a 15-min `auto_sync` heartbeat loop (gated by `GIT_SYNC_AUTO` env var; default-on for non-source-mode GitHub-template @@ -684,6 +685,55 @@ middleware. Phase 3: MCP tool call audit via transparent wrapper (all 66+ tools, zero per-tool code). Phase 4: hash chain verification, CSV/JSON export, enable/disable toggle. Issue #20 can be closed. +### Canary Invariant Harness (CANARY-001 — Phase 1, NEW: 2026-05-04) + +| Method | Path | Auth | Description | +|--------|------|------|-------------| +| GET | `/api/canary/violations` | Admin | List violations (filters: invariant_id, severity, tier, start_time, end_time, limit, offset) | +| GET | `/api/canary/violations/stats` | Admin | Aggregate counts by invariant_id and severity | +| GET | `/api/canary/violations/{id}` | Admin | Single violation by row id | +| POST | `/api/canary/run-cycle` | Admin | Run one cycle on demand (delegates to the same `CanaryService.run_cycle()` invoked by the 5-min background loop). Optional body filters which invariants to run. Returns `{snapshot_time, cycle_duration_ms, checks_run, sources_unavailable, violations[], transitions[]}`. Returns 409 with `detail="cycle in progress"` when a background or sibling on-demand cycle is mid-run — empty payload is never silently returned. | + +**Storage**: `canary_violations` table in main SQLite DB. JSON-encoded +`observed_state` column carries invariant-specific payload. + +**Phase 1 invariants** (S-01, E-02, L-03): +- **S-01 — Slot–row bijection**: per agent, set of execution_ids in + `agent:slots:{name}` (Redis ZSET, drain sentinels filtered) equals set + of execution_ids in `schedule_executions WHERE status='running'`. + Severity: critical. Catches PR #378/#403 bug class. +- **E-02 — No phantom reversal**: an execution row that was in a + terminal status in the previous cycle must not appear non-terminal in + this snapshot. Phase 1 uses Redis-backed state comparison (key + `canary:e02:terminal_seen`) instead of Vector log diff for simplicity. + Severity: critical. +- **L-03 — Delete cascades**: no live row in any cross-cutting table + (agent_sharing, agent_schedules, schedule_executions [non-terminal], + agent_skills, agent_tags, agent_shared_files, agent_public_links, + pending operator_queue, pending access_requests, agent-scoped + mcp_api_keys, active chat_sessions) may reference an `agent_name` not + in `agent_ownership`; no Redis `agent:slots:{name}` for missing agent. + Severity: critical for orphaned `schedule_executions` or Redis slots, + major otherwise. Catches Issue #129 bug class. + +**Fleet**: `config/canary-fleet.yaml` — synthetic load generators +(`canary-fleet-burst`, `canary-fleet-long`) deployed via the existing +systems-deploy API. Without traffic the harness produces trivially-green +checks; the fleet is what gives the watcher something to watch on +staging/dev. + +**Architecture**: deterministic library (`src/backend/canary/`) shared +between the 5-min watcher service and the on-demand admin endpoint. +Library reads state but writes nothing; service writes violations and +classifies green→red transitions. **Alert sink**: Slack via incoming +webhook URL configured by `CANARY_SLACK_WEBHOOK_URL` env var (admin-side, +no Settings UI — the canary is staging/dev-only and the operator already +has shell access). Unset = silent sink (cycles still run, violations +still persist). Each transition fires exactly one webhook POST with a +Block Kit payload (header + body + context with "last red Xm ago" +badge). Continuing-red invariants don't re-post. No LLM reasoning +anywhere — the canary's value depends on determinism. + ### Nevermined Payments (NVM-001) | Method | Path | Auth | Description | @@ -1380,6 +1430,36 @@ BEGIN SELECT RAISE(ABORT, 'Audit log entries cannot be deleted within retention - Cross-cutting platform audit for lifecycle, auth, MCP, credentials events - Phase 1 ships infrastructure only; write integration into routers happens in Phase 2 +**canary_violations:** (CANARY-001 / Issue #411 — Phase 1, NEW: 2026-05-04) +```sql +CREATE TABLE canary_violations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + invariant_id TEXT NOT NULL, -- 'S-01', 'E-02', 'L-03', ... + tier TEXT NOT NULL, -- 'A' | 'B' + severity TEXT NOT NULL, -- 'critical' | 'major' | 'minor' + snapshot_time TEXT NOT NULL, -- ISO 8601 UTC + observed_state TEXT NOT NULL, -- JSON, invariant-specific + signal_query TEXT, -- the check that fired (debugging aid) + created_at TEXT NOT NULL DEFAULT (datetime('now')) +); +CREATE INDEX idx_canary_violations_invariant + ON canary_violations(invariant_id, snapshot_time DESC); +CREATE INDEX idx_canary_violations_severity + ON canary_violations(severity, snapshot_time DESC); +CREATE INDEX idx_canary_violations_snapshot + ON canary_violations(snapshot_time DESC); +``` + +**canary_violations Features:** +- Append-only in practice (no UPDATE / DELETE in the read API surface). +- One row per fired check per cycle. `observed_state` carries + invariant-specific JSON (slot diffs, ghost agent names, terminal-status + reversals). +- Read via `GET /api/canary/violations`; `GET /api/canary/violations/stats` + drives the dashboard tiles. +- Populated by `services/canary_service.py` on a 5-min loop or on-demand + via `POST /api/canary/run-cycle`. + ### Redis **Credential Storage (DEPRECATED - CRED-002):** diff --git a/docs/memory/requirements.md b/docs/memory/requirements.md index 9bd9ee0dd..5dd4c7a58 100644 --- a/docs/memory/requirements.md +++ b/docs/memory/requirements.md @@ -2005,6 +2005,58 @@ Standalone mobile-friendly admin page for managing agents on the go. Designed as --- +## 31. Canary Invariant Harness (CANARY-001) + +### 31.1 Continuous Orchestration-Invariant Watcher (CANARY-001 — Phase 1) +- **Implements**: Issue #411 — first three invariants (S-01, E-02, L-03) +- **Description**: Background watcher service that runs deterministic + orchestration-invariant checks against live platform state every 5 + minutes. Persists violations to a queryable table and classifies + green→red transitions for an external alert sink. Catches the bug + class behind PRs #378, #403, #129, #226 — race conditions and + cross-component state drift that unit tests miss. +- **Architecture**: deterministic Python library (`src/backend/canary/`) + shared between the watcher service (`services/canary_service.py`) and + the on-demand admin endpoint (`POST /api/canary/run-cycle`). Library + reads state but writes nothing; service writes violations and + classifies transitions. +- **Phase 1 invariants**: + - **S-01** Slot–row bijection (Redis ZRANGE vs SQL running rows, drain + sentinels filtered) + - **E-02** No phantom reversal (terminal executions stay terminal, + detected via Redis-backed state comparison) + - **L-03** Delete cascades (no orphan rows referencing removed agents + in any cross-cutting table; no orphan Redis slot keys) +- **Storage**: `canary_violations` table; observed_state JSON column. +- **Activation**: gated by `CANARY_ENABLED=1` env var; disabled by + default. Production deployment is staging/dev — the harness watches + there, not in user-facing prod. +- **Fleet**: `config/canary-fleet.yaml` deploys two synthetic agents + (`canary-fleet-burst` minute-cron, `canary-fleet-long` 5-min cron) via + the existing `/api/systems/deploy` endpoint. Without the fleet, the + watcher reports trivially-green cycles with no signal. +- **Alert sink**: Slack via incoming webhook URL configured by the + `CANARY_SLACK_WEBHOOK_URL` env var (admin-side, no Settings UI — the + audience is operators with shell access on staging/dev). Each + green→red transition fires exactly one webhook POST with a Block Kit + payload (severity emoji header, rendered violation summary, context + line with snapshot_time + violation count + "last red Xm ago" + badge). Unset = silent sink: cycles still run, violations still + persist to `canary_violations`, only the outbound POST is skipped. + Continuing-red invariants don't re-post. The dashboard-notifications + path (writing `agent_notifications` rows via `db.create_notification`) + was rejected on the product call. +- **Determinism**: invariant checks are pure functions + `check(snapshot) → list[ViolationReport]`. Same snapshot input always + yields the same output. No LLM reasoning anywhere in the canary path. +- **Phase 2 (deferred)**: S-02, S-03, E-01, E-05, E-06, B-01, B-02, + G-01, R-01 (per the catalog at + `docs/testing/orchestration-invariant-catalog.md`). Each adds as a new + file under `src/backend/canary/invariants/` and a registry entry; the + service and API surface stay unchanged. + +--- + ## Out of Scope - Multi-tenant deployment (single org only) diff --git a/docs/security-reports/cso-2026-05-09-411-diff.json b/docs/security-reports/cso-2026-05-09-411-diff.json new file mode 100644 index 000000000..9b1448094 --- /dev/null +++ b/docs/security-reports/cso-2026-05-09-411-diff.json @@ -0,0 +1,42 @@ +{ + "version": "1.0", + "date": "2026-05-09", + "mode": "daily", + "scope": "diff", + "branch": "feature/411-canary-migration", + "base": "dev", + "phases_run": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], + "attack_surface": { + "public_endpoints": 0, + "authenticated_endpoints": 0, + "admin_endpoints": 4, + "file_upload_points": 0, + "websocket_channels": 0, + "external_integrations": 1, + "background_jobs": 1, + "new_env_vars": 2 + }, + "findings": [], + "supply_chain_summary": { + "new_python_deps": 0, + "new_node_deps": 0, + "new_lockfile_changes": 0 + }, + "filter_stats": { + "candidates_considered": 5, + "dropped_below_gate": 5, + "hard_exclusions_applied": ["#1 DoS/resource", "#3 memory leak"] + }, + "totals": { + "critical": 0, + "high": 0, + "medium": 0, + "low": 0 + }, + "trend": { + "prior_report": "cso-2026-05-04-589-diff.md", + "carryover": 0, + "note": "Prior diff report covered a different change set (#589 network split). No findings carry over." + }, + "summary": "Phase 1 of #411 (canary invariant harness). Admin-only API surface, outbound-only Slack integration with explicit URL-leak mitigation, SQL composition via constants and enum-derived values, env-gated activation. Zero findings at the 8/10 daily-mode gate." +} diff --git a/docs/security-reports/cso-2026-05-09-411-diff.md b/docs/security-reports/cso-2026-05-09-411-diff.md new file mode 100644 index 000000000..642a03aa7 --- /dev/null +++ b/docs/security-reports/cso-2026-05-09-411-diff.md @@ -0,0 +1,66 @@ +# CSO Diff Audit — feature/411-canary-migration → dev + +**Date**: 2026-05-09 +**Mode**: daily (8/10 confidence gate) +**Scope**: branch diff (19 source files, ~4144 added lines, excluding docs/ and tests/) + +## Summary + +Zero findings at the daily-mode confidence gate. + +| Severity | Count | +|----------|-------| +| Critical | 0 | +| High | 0 | +| Medium | 0 | +| Low | 0 | + +## Change Set + +Phase 1 of #411 — continuous orchestration-invariant harness: + +- New module `src/backend/canary/` (snapshot collector + 3 invariants: S-01, E-02, L-03) +- New service `services/canary_service.py` (5-min watcher, env-gated `CANARY_ENABLED=1`) +- New service `services/canary_alerts.py` (Slack Block Kit composer, outbound webhook POST) +- New router `routers/canary.py` (4 admin endpoints) +- New table `canary_violations` (append-mostly, JSON `observed_state`) +- New env vars: `CANARY_ENABLED`, `CANARY_SLACK_WEBHOOK_URL` + +## Attack Surface (diff scope) + +| Surface | Count | +|---------|-------| +| Public endpoints | 0 | +| Authenticated endpoints | 0 | +| Admin-only endpoints | 4 | +| File upload points | 0 | +| WebSocket channels | 0 | +| External integrations | 1 (Slack outbound) | +| Background jobs | 1 (env-gated) | + +## Why no findings + +1. **Admin-only API surface**: every route uses `Depends(require_admin)`. No public, agent, or user-facing exposure. +2. **Outbound-only Slack integration**: signature verification N/A. URL-leak surface explicitly addressed at `slack_service.py:310-315` — exception class name only, never `str(e)`. +3. **No SQL injection**: f-string SQL composition draws from `ORPHAN_SCAN_TABLES` constants, the `TaskExecutionStatus` enum, and `pk_col` from `PRAGMA table_info` results. No user input crosses into f-string interpolation. +4. **Env-gated activation**: `CANARY_ENABLED=0` default; silent-sink fallback when `CANARY_SLACK_WEBHOOK_URL` unset. +5. **Read-only data flow**: snapshot collector reads Redis and SQLite, writes only to the new dedicated `canary_violations` table. +6. **No new agent-network exposure**: env vars passed only to backend container; no new ports/volumes/capabilities. + +## Candidates Filtered Below the Gate + +| Candidate | Drop reason | +|-----------|-------------| +| Multi-worker duplicate Slack alerts | Theoretical — Trinity ships single backend worker. Documented in `/review I4` as a forward-looking concern, not currently exploitable. | +| `canary_violations` unbounded growth | Hard exclusion #3 (resource without proven security impact). | +| Slack mrkdwn injection via agent_name | Agent names sanitized to `[a-zA-Z0-9_.-]`; admin-only channel; webhooks don't `link_names` by default. | +| httpx may log webhook URL at DEBUG level | Only at `LOG_LEVEL=DEBUG`; prod runs INFO. Confidence 4/10. | +| `print()` instead of structured logger in `main.py` startup paths | Logging hygiene, not security. | + +## Repudiation Note (non-blocking) + +`POST /api/canary/run-cycle` is not currently written to `audit_log`. Admin-gated, so blast radius is bounded, but on-demand cycles aren't traceable to the operator who triggered them. The existing platform audit log doesn't cover every admin endpoint either; flag this for Phase 2 if traceability becomes a stakeholder requirement. + +## Trend + +Prior report `cso-2026-05-04-589-diff.md` covered #589 (Redis/network split) — different change set, no overlap. No carryover findings. diff --git a/src/backend/canary/__init__.py b/src/backend/canary/__init__.py new file mode 100644 index 000000000..1fa91d636 --- /dev/null +++ b/src/backend/canary/__init__.py @@ -0,0 +1,38 @@ +""" +Canary invariant harness (CANARY-001 / Issue #411). + +Continuous orchestration-invariant testing harness running against +staging/dev. Ships in phases per docs/planning/CANARY_HARNESS_PHASE_1.md; +Phase 1 covers S-01, E-02, L-03 with the snapshot collector and +deterministic invariant library colocated here. + +Public API: +- `collect_snapshot()` — gather a roughly-simultaneous read of Redis × SQL + × agent registry state, returning a typed `Snapshot`. +- `run_invariants(snapshot, ids)` — apply the named invariants to the + snapshot and return a list of `ViolationReport`. +- `INVARIANTS` — registry of invariant id → check function. + +`services/canary_service.py` drives these on a 5-minute background loop +in the backend process; `POST /api/canary/run-cycle` exposes the same +entrypoint for on-demand smoke tests. +""" + +from .snapshot import ( + Snapshot, + AgentSnapshot, + OrphanRef, + ViolationReport, + collect_snapshot, +) +from .invariants import INVARIANTS, run_invariants + +__all__ = [ + "Snapshot", + "AgentSnapshot", + "OrphanRef", + "ViolationReport", + "collect_snapshot", + "run_invariants", + "INVARIANTS", +] diff --git a/src/backend/canary/invariants/__init__.py b/src/backend/canary/invariants/__init__.py new file mode 100644 index 000000000..47843ad6d --- /dev/null +++ b/src/backend/canary/invariants/__init__.py @@ -0,0 +1,62 @@ +""" +Canary invariant library — Phase 1 (CANARY-001 / Issue #411). + +Each invariant is a pure function `check(snapshot) → list[ViolationReport]`. +The library is registry-driven so the run-cycle endpoint can enable/disable +invariants per request. + +Phase 1 ships three: + +- S-01: slot–row bijection (Redis ZSET vs SQL running rows) +- E-02: no phantom state reversal (terminal executions stay terminal) +- L-03: delete cascades (no orphan rows referencing removed agents) + +Subsequent phases register additional invariants here without changes to +the snapshot collector or the run-cycle endpoint. +""" + +from typing import Callable, Dict, Iterable, List + +from ..snapshot import Snapshot, ViolationReport +from .s01_slot_row_bijection import check as s01_check +from .e02_no_phantom_reversal import check as e02_check +from .l03_delete_cascades import check as l03_check + + +# Public registry. Keys are the invariant ids the run-cycle endpoint +# accepts in its `invariants` filter. +INVARIANTS: Dict[str, Callable[[Snapshot], List[ViolationReport]]] = { + "S-01": s01_check, + "E-02": e02_check, + "L-03": l03_check, +} + + +def run_invariants( + snapshot: Snapshot, + ids: Iterable[str] | None = None, +) -> Dict[str, List[ViolationReport]]: + """Apply the named invariants to the snapshot. + + Returns dict {invariant_id: [violations]}. Empty list = invariant held. + A check raising is logged and surfaces as `{}` for that id (caller can + distinguish skipped via the absence of the key, but Phase 1 treats both + as "no violation written"). + """ + selected = list(ids) if ids is not None else list(INVARIANTS.keys()) + out: Dict[str, List[ViolationReport]] = {} + for inv_id in selected: + check_fn = INVARIANTS.get(inv_id) + if check_fn is None: + continue + try: + out[inv_id] = check_fn(snapshot) + except Exception: + import logging + logging.getLogger(__name__).exception( + "canary invariant %s raised; skipping cycle for this id", inv_id + ) + # Do not write a violation for a check error — that would be + # noise. Surface via logs and let operators investigate. + out[inv_id] = [] + return out diff --git a/src/backend/canary/invariants/e02_no_phantom_reversal.py b/src/backend/canary/invariants/e02_no_phantom_reversal.py new file mode 100644 index 000000000..eec683cc7 --- /dev/null +++ b/src/backend/canary/invariants/e02_no_phantom_reversal.py @@ -0,0 +1,187 @@ +""" +E-02 — No phantom state reversal (CANARY-001 / Issue #411). + +A `schedule_executions` row that has been observed in a terminal status +must never appear in a non-terminal status in a later snapshot. Catches +the bug class behind PR #378 / #403 (phantom stale-slot failures), where +a success/failed/cancelled/skipped execution silently flips back to running. + +## Phase 1 implementation note + +The design doc proposed "Vector log diff (`update_execution_status` lines)" +as the snapshot input. That requires log-file plumbing into the canary's +container — non-trivial and orthogonal to the check itself. + +Phase 1 instead uses a **state-comparison** detector: each cycle, the set +of recently-terminal execution_ids (last 30 min, per the snapshot +collector) is compared against the previous cycle's set, persisted in a +Redis sorted set `canary:e02:terminal_seen` with `score = unix ts last +seen terminal`. Any execution_id that was in the previous set, is *not* +terminal in this snapshot's running/queued tables, **and** still exists +in the DB → reversal violation. + +This is strictly more sensitive than log diffing for this bug class: +even a reversal that happens silently (no log line, e.g. via direct DB +write) is caught. The trade-off is a small Redis side-table; fine in +exchange for a cleaner Phase 1. + +The side-table is bounded by **age**, not by a hard count. Each cycle +calls `ZREMRANGEBYSCORE` to drop entries older than +`PREV_TERMINAL_RETENTION_SECONDS`. Earlier versions of this code +DEL'd the entire key once it crossed a 5000-row hard cap and re-added +only the current cycle's terminals, which left a one-cycle E-02 blind +spot every time the cap was crossed on busy installs. Score-based +trimming bounds memory by `activity_rate × retention_window` and never +loses an in-window terminal id to a hard reset. + +When Phase 2 wires up Vector log access, we may add a complementary +log-based detector — but the state-comparison check stands on its own. + +Tier A, severity critical. Backsliding past terminal is exactly the +class of bug this harness exists to catch. +""" + +import logging +import time +from typing import Dict, List, Set + +from ..snapshot import Snapshot, ViolationReport, TERMINAL_EXECUTION_STATUSES + + +logger = logging.getLogger(__name__) + + +INVARIANT_ID = "E-02" +TIER = "A" +SEVERITY = "critical" + +# Redis sorted set storing the previous-cycle terminal ids. +# Members are execution_ids; scores are the unix ts the id was last +# observed in a snapshot's terminal set. +REDIS_KEY_PREV_TERMINAL = "canary:e02:terminal_seen" + +# Parallel hash carrying the actual terminal status (success / failed / +# cancelled / skipped) for each id in the ZSET. Read on reversal so the +# violation report (and the Slack forensic line) can render the real +# prior status — earlier code stored the placeholder string "terminal" +# which made on-call alerts read "terminal → running" instead of e.g. +# "success → running". +REDIS_KEY_PREV_TERMINAL_STATUS = "canary:e02:terminal_status" + +# Trim ids older than this many seconds. Comfortably larger than the +# snapshot collector's 30-min terminal window so an id does not age out +# between the cycle that records it and the cycle that detects a +# reversal of it. Age-based trimming replaced an earlier hard count cap +# (5000) that DEL'd the entire key on overflow, creating one-cycle blind +# spots on busy installs. +PREV_TERMINAL_RETENTION_SECONDS = 60 * 60 # 1 hour + + +def _redis(): + """Lazy import of the slot service's Redis client.""" + from services.slot_service import get_slot_service + + return get_slot_service().redis + + +def check(snapshot: Snapshot) -> List[ViolationReport]: + """Detect terminal→non-terminal reversals across snapshots.""" + violations: List[ViolationReport] = [] + + # If SQL terminal-set read failed, skip — the comparison is meaningless. + if any( + s.startswith("sqlite.terminal_executions") for s in snapshot.sources_unavailable + ): + return violations + + now_ts = time.time() + cutoff_ts = now_ts - PREV_TERMINAL_RETENTION_SECONDS + + try: + redis_client = _redis() + # Snapshot the about-to-expire ids before trimming so we can + # garbage-collect their entries from the parallel status hash in + # the same cycle. Done via ZRANGEBYSCORE before ZREMRANGEBYSCORE + # because the latter does not return the dropped members. + expired_ids: List[str] = list( + redis_client.zrangebyscore(REDIS_KEY_PREV_TERMINAL, "-inf", cutoff_ts) or [] + ) + redis_client.zremrangebyscore(REDIS_KEY_PREV_TERMINAL, "-inf", cutoff_ts) + if expired_ids: + redis_client.hdel(REDIS_KEY_PREV_TERMINAL_STATUS, *expired_ids) + previous: Set[str] = set( + redis_client.zrange(REDIS_KEY_PREV_TERMINAL, 0, -1) or [] + ) + except Exception: + # Redis unreachable — record once via the snapshot mechanism on + # subsequent cycles, but for this cycle there's nothing to compare. + logger.exception("E-02: previous terminal set unreadable; skipping") + return violations + + current_terminal: Dict[str, str] = snapshot.terminal_exec_statuses + + # Reversal candidates: ids that were terminal previously but are now + # in the running/queued sets. Cross-reference against per-agent + # snapshots (the only place running/queued sets live). + running_now: Set[str] = set() + queued_now: Set[str] = set() + for agent in snapshot.agents: + running_now |= agent.running_exec_ids + queued_now |= agent.queued_exec_ids + + reversed_ids = previous & (running_now | queued_now) + if reversed_ids: + try: + prev_status_values = redis_client.hmget( + REDIS_KEY_PREV_TERMINAL_STATUS, *sorted(reversed_ids) + ) + except Exception: + logger.exception("E-02: status lookup failed; reporting status as unknown") + prev_status_values = [None] * len(reversed_ids) + prev_status_by_eid = dict(zip(sorted(reversed_ids), prev_status_values)) + else: + prev_status_by_eid = {} + + for eid in sorted(reversed_ids): + current_status = "running" if eid in running_now else "queued" + previous_status = prev_status_by_eid.get(eid) or "unknown" + violations.append( + ViolationReport( + invariant_id=INVARIANT_ID, + tier=TIER, + severity=SEVERITY, + observed_state={ + "execution_id": eid, + "previous_status": previous_status, + "current_status": current_status, + "snapshot_time": snapshot.snapshot_time, + "terminal_statuses_tracked": list(TERMINAL_EXECUTION_STATUSES), + }, + signal_query=( + f"execution_id {eid} was {previous_status} in previous " + f"cycle; now {current_status}" + ), + ) + ) + + # Update the side-table with this cycle's terminal set so the next + # cycle has something to compare against. ZADD updates the score on + # existing members, so a still-terminal id has its retention clock + # refreshed and will not age out while it's still being observed. + # The parallel hash carries the row's actual terminal status; HSET + # overwrites on transitions between terminal statuses (rare in + # practice but cheap to honour). + try: + if current_terminal: + redis_client.zadd( + REDIS_KEY_PREV_TERMINAL, + {eid: now_ts for eid in current_terminal}, + ) + redis_client.hset( + REDIS_KEY_PREV_TERMINAL_STATUS, + mapping=current_terminal, + ) + except Exception: + logger.exception("E-02: failed to persist terminal set; next cycle will skip") + + return violations diff --git a/src/backend/canary/invariants/l03_delete_cascades.py b/src/backend/canary/invariants/l03_delete_cascades.py new file mode 100644 index 000000000..dab4ee903 --- /dev/null +++ b/src/backend/canary/invariants/l03_delete_cascades.py @@ -0,0 +1,100 @@ +""" +L-03 — Delete cascades (CANARY-001 / Issue #411). + +No live row in any cross-cutting table (agent_sharing, agent_schedules, +non-terminal schedule_executions, agent_skills, agent_tags, agent_shared_files, +agent_public_links, pending operator_queue, pending access_requests, +agent-scoped mcp_api_keys, active chat_sessions) may reference an +agent_name that is not present in `agent_ownership`. + +Additionally, no Redis `agent:slots:{name}` key may exist for a name not +in `agent_ownership`. + +This invariant catches the bug class where deletion of an agent leaves +dangling references — the symptom of the original Issue #129 family. The +list of scanned tables is in `canary.snapshot.ORPHAN_SCAN_TABLES`. + +Tier A. Severity scales with the table: +- `schedule_executions` (running/queued), Redis slot keys → critical + (active orchestration state pointing at a ghost agent) +- All other orphan refs → major + (dangling permissions/sharing/tags; user-visible but not active orchestration) +""" + +from collections import defaultdict +from typing import Dict, List + +from ..snapshot import Snapshot, ViolationReport + + +INVARIANT_ID = "L-03" +TIER = "A" + +# Tables whose orphan rows constitute *active* orchestration state and +# warrant the `critical` severity tier. Rest are `major`. +CRITICAL_TABLES = frozenset({"schedule_executions"}) + + +def _severity_for(table: str) -> str: + return "critical" if table in CRITICAL_TABLES else "major" + + +def check(snapshot: Snapshot) -> List[ViolationReport]: + """Emit one violation per orphaned agent_name (grouped across tables).""" + violations: List[ViolationReport] = [] + + # If the SQL orphan scan failed, skip — partial result is misleading. + if any(s.startswith("sqlite.orphan_refs") for s in snapshot.sources_unavailable): + return violations + + # Group orphan refs by referenced_agent_name so one ghost agent shows + # up as one violation report (not one per dangling row). + by_agent: Dict[str, List] = defaultdict(list) + for ref in snapshot.orphan_refs: + by_agent[ref.referenced_agent_name].append(ref) + + # Redis orphan slots also belong to the same agent grouping. + redis_orphans = dict(snapshot.orphan_redis_slots) + for name in redis_orphans: + by_agent.setdefault(name, []) + + for agent_name, refs in sorted(by_agent.items()): + tables_hit = {ref.table for ref in refs} + # If this ghost agent has Redis slot membership, that's an + # active-orchestration signal regardless of which SQL tables fired. + has_redis_slots = redis_orphans.get(agent_name, 0) > 0 + if has_redis_slots: + tables_hit.add("redis:agent:slots") + + severity = ( + "critical" + if any(t in CRITICAL_TABLES for t in tables_hit) or has_redis_slots + else "major" + ) + + violations.append( + ViolationReport( + invariant_id=INVARIANT_ID, + tier=TIER, + severity=severity, + observed_state={ + "ghost_agent_name": agent_name, + "snapshot_time": snapshot.snapshot_time, + "orphan_count": len(refs), + "redis_slot_count": redis_orphans.get(agent_name, 0), + "tables_hit": sorted(tables_hit), + "sample_refs": [ + {"table": r.table, "column": r.column, "row_id": r.row_id} + for r in refs[:10] + ], + }, + signal_query=( + f"agent_name '{agent_name}' referenced by " + f"{len(refs)} SQL row(s) and " + f"{redis_orphans.get(agent_name, 0)} Redis slot(s) " + "but absent from agent_ownership" + ), + ) + ) + + return violations diff --git a/src/backend/canary/invariants/s01_slot_row_bijection.py b/src/backend/canary/invariants/s01_slot_row_bijection.py new file mode 100644 index 000000000..670911865 --- /dev/null +++ b/src/backend/canary/invariants/s01_slot_row_bijection.py @@ -0,0 +1,86 @@ +""" +S-01 — Slot–row bijection (CANARY-001 / Issue #411). + +Per agent A: the set of execution_ids in `agent:slots:A` (Redis ZSET) must +equal the set of execution_ids in `schedule_executions` with status='running' +and agent_name=A. + +Drain sentinels (members starting with `drain-`) are filtered out — see +services/backlog_service.py for why they exist. + +Tier A, severity critical. A bijection violation always indicates either +leaked Redis slots (capacity is wrong) or phantom SQL running rows (cleanup +service is failing) — both of which directly cause user-visible breakage. +""" + +import time +from datetime import datetime +from typing import List + +from ..snapshot import Snapshot, ViolationReport + + +INVARIANT_ID = "S-01" +TIER = "A" +SEVERITY = "critical" + +DRAIN_PREFIX = "drain-" +# Suppress race-window false positives: SQL row commits before the Redis ZADD +# on start (~30ms typ), and SQL terminal flip precedes ZREM on stop (~5ms). +# Real leaks (PR #378/#403 class) survive multiple cycles, so 3s is generous. +GRACE_SECONDS = 3.0 + + +def check(snapshot: Snapshot) -> List[ViolationReport]: + """Compare Redis slot ZSET membership to SQL running rows per agent.""" + violations: List[ViolationReport] = [] + + # If Redis was unreachable this cycle, skip — better silence than a + # false positive that trains operators to mute the alert. + if any(s.startswith("redis") for s in snapshot.sources_unavailable): + return violations + + for agent in snapshot.agents: + # Filter drain sentinels: they hold a slot for a few seconds during + # backlog drain and are intentionally not present in SQL. + slot_ids = {sid for sid in agent.slot_ids if not sid.startswith(DRAIN_PREFIX)} + running_ids = agent.running_exec_ids + + if slot_ids == running_ids: + continue + + cutoff = time.time() - GRACE_SECONDS + in_redis_only = sorted( + sid for sid in slot_ids - running_ids + if agent.slot_scores.get(sid, 0) < cutoff + ) + in_sql_only = sorted( + eid for eid in running_ids - slot_ids + if (ts := agent.running_started_at.get(eid)) is None + or datetime.fromisoformat(ts).timestamp() < cutoff + ) + if not in_redis_only and not in_sql_only: + continue + + violations.append( + ViolationReport( + invariant_id=INVARIANT_ID, + tier=TIER, + severity=SEVERITY, + observed_state={ + "agent_name": agent.name, + "redis_slot_count": len(slot_ids), + "sql_running_count": len(running_ids), + "in_redis_only": in_redis_only, + "in_sql_only": in_sql_only, + "snapshot_time": snapshot.snapshot_time, + }, + signal_query=( + "set(ZRANGE agent:slots:{name}) - drain sentinels " + "vs set(SELECT id FROM schedule_executions " + "WHERE agent_name = '{name}' AND status = 'running')" + ).format(name=agent.name), + ) + ) + + return violations diff --git a/src/backend/canary/snapshot.py b/src/backend/canary/snapshot.py new file mode 100644 index 000000000..df0be8800 --- /dev/null +++ b/src/backend/canary/snapshot.py @@ -0,0 +1,438 @@ +""" +Canary snapshot collector (CANARY-001 / Issue #411 — Phase 1). + +Gathers a roughly-simultaneous read of orchestration state across: + +- SQLite — agent ownership, execution rows (running + queued), plus per-table + agent_name references for the L-03 orphan scan. +- Redis — agent slot ZSETs (`agent:slots:{name}`). +- Vector logs — deferred to Phase 2; E-02 uses a state-comparison detector + in this phase (see invariants/e02_no_phantom_reversal.py for rationale). +- Agent registries / container exec — deferred to Phase 2 invariants. + +The collector is pure read. It writes nothing. Invariant library functions +take the resulting `Snapshot` and return zero-or-more `ViolationReport`s. + +Phase 1 scope is S-01, E-02, L-03 — the rest of the design doc's snapshot +fields are placeholders until their invariants land. + +## Why a separate module from the invariants + +The three Phase 1 invariants (S-01, E-02, L-03) all read overlapping +state. Splitting state collection out gives three things: + +1. **One consistent view per cycle.** All invariants see the same + `Snapshot` instance, so per-check timing drift cannot introduce + spurious mismatches — e.g. L-03 reading the SQL `agent_ownership` + set after S-01 has already started ZRANGEing on agents that were + live a moment earlier. +2. **No duplicated query code.** New invariants are pure functions + `(snapshot) → list[ViolationReport]`; they never re-implement + SELECTs or ZRANGEs against live state. This keeps the registry in + `invariants/__init__.py` the only file the catalog grows in. +3. **Test-friendly.** Tests pass synthetic `Snapshot` dataclasses + straight in (see `tests/test_canary_invariants.py`) and never + need a live Redis or SQLite to exercise the checking logic. + +Note: the snapshot is *not* atomic across Redis and SQLite — those +don't share transactions, and our reads are sequential. The harness +deliberately accepts sub-second inconsistencies (a real bug persists +across a 5-minute cycle by definition; transient races self-resolve +and are not what we're trying to catch). +""" + +import logging +import os +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional, Set + +from models import TaskExecutionStatus +from utils.helpers import iso_cutoff, utc_now_iso + + +logger = logging.getLogger(__name__) + + +# Statuses considered "terminal" for execution rows. Derived directly +# from `TaskExecutionStatus` (models.py) — the same set PR #524's CAS +# state machine treats as write-once. Used by E-02 (phantom reversal +# detection) and the L-03 orphan scan filter. Sourcing from the enum +# means a new terminal status added there flows here automatically; +# the previous hand-maintained tuple silently drifted (see /review I3). +TERMINAL_EXECUTION_STATUSES = ( + TaskExecutionStatus.SUCCESS.value, + TaskExecutionStatus.FAILED.value, + TaskExecutionStatus.CANCELLED.value, + TaskExecutionStatus.SKIPPED.value, +) +_TERMINAL_SQL_LIST = ", ".join(f"'{s}'" for s in TERMINAL_EXECUTION_STATUSES) + + +# Tables whose `agent_name` column references `agent_ownership.agent_name`. +# Used by L-03 (delete cascades) to scan for orphan rows. +# +# Exclusions: +# - `chat_messages` — denormalized via `chat_sessions`; covered transitively. +# - `agent_health_checks`, `agent_dashboard_values` — observational tables +# that legitimately retain history of deleted agents (rolled up by retention). +# - `nevermined_payment_log` — append-only audit; deletes do not cascade by design. +# - `monitoring_alert_cooldowns` — cooldown TTL handles cleanup. +# +# The list intentionally errs on the side of catching more orphans rather +# than fewer; false positives surface as L-03 violations operators triage. +ORPHAN_SCAN_TABLES = [ + ("agent_sharing", "agent_name", None), + ("agent_schedules", "agent_name", None), + # Only non-terminal executions; terminal rows are immutable history per + # PR #524's CAS-guarded state machine and may legitimately reference a + # later-deleted agent. + ( + "schedule_executions", + "agent_name", + f"status NOT IN ({_TERMINAL_SQL_LIST})", + ), + ("chat_sessions", "agent_name", "status = 'active'"), + ("agent_skills", "agent_name", None), + ("agent_tags", "agent_name", None), + ("agent_shared_files", "agent_name", None), + ("agent_public_links", "agent_name", None), + ("operator_queue", "agent_name", "status = 'pending'"), + ("access_requests", "agent_name", "status = 'pending'"), +] + + +@dataclass +class OrphanRef: + """One orphan row found during the L-03 scan.""" + + table: str + column: str + referenced_agent_name: str + row_id: str # Stringified primary key (TEXT or INTEGER) + + +@dataclass +class ViolationReport: + """Output of an invariant check that fired. + + Mirrors the canary_violations table schema so the run-cycle endpoint + can persist these directly. + """ + + invariant_id: str + tier: str # 'A' or 'B' + severity: str # 'critical' | 'major' | 'minor' + observed_state: Dict[str, Any] + signal_query: Optional[str] = None + + +@dataclass +class AgentSnapshot: + """Per-agent slice of the snapshot.""" + + name: str + is_system: bool + max_parallel: int + execution_timeout_seconds: int + # Redis ZSET membership for `agent:slots:{name}`. Drain sentinels + # (members starting with 'drain-') are filtered out by S-01 before the + # bijection check; we keep the raw set here so other invariants can see + # them if needed. + slot_ids: Set[str] = field(default_factory=set) + # ZSET score per slot (Unix epoch seconds at acquire); used by S-01 grace. + slot_scores: Dict[str, float] = field(default_factory=dict) + # SQLite execution_id sets, partitioned by status. + running_exec_ids: Set[str] = field(default_factory=set) + # `started_at` per running id (ISO); used by S-01 grace. + running_started_at: Dict[str, str] = field(default_factory=dict) + queued_exec_ids: Set[str] = field(default_factory=set) + + +@dataclass +class Snapshot: + """Full snapshot at one moment in time.""" + + snapshot_time: str # ISO 8601 UTC + agents: List[AgentSnapshot] = field(default_factory=list) + # All known agent names (from agent_ownership). Source of truth for L-03. + known_agents: Set[str] = field(default_factory=set) + # L-03 inputs: orphan rows found via cross-table scan. + orphan_refs: List[OrphanRef] = field(default_factory=list) + # Redis slot keys observed for agents NOT in known_agents (also L-03). + orphan_redis_slots: Dict[str, int] = field(default_factory=dict) + # E-02 inputs: terminal-state map per execution_id in the most recent + # snapshot. The check compares this against a stored "previously + # terminal" set fetched from Redis to detect reversals. The status + # value (success/failed/cancelled/skipped) is preserved so reversal + # alerts can render the real prior status, not a placeholder. + terminal_exec_statuses: Dict[str, str] = field(default_factory=dict) + # Diagnostics — empty on a clean cycle. + sources_unavailable: List[str] = field(default_factory=list) + + +# --------------------------------------------------------------------------- +# Collection helpers +# --------------------------------------------------------------------------- + + +def _collect_known_agents() -> List[Dict[str, Any]]: + """Read agent_ownership rows. One source of truth for valid agent names.""" + from db.connection import get_db_connection + + with get_db_connection() as conn: + cursor = conn.cursor() + cursor.execute( + """ + SELECT agent_name, + COALESCE(is_system, 0) AS is_system, + COALESCE(max_parallel_tasks, 3) AS max_parallel_tasks, + COALESCE(execution_timeout_seconds, 900) AS execution_timeout_seconds + FROM agent_ownership + """ + ) + return [dict(row) for row in cursor.fetchall()] + + +def _collect_executions(agent_name: str) -> Dict[str, Set[str]]: + """Per-agent running + queued execution_ids.""" + from db.connection import get_db_connection + + with get_db_connection() as conn: + cursor = conn.cursor() + cursor.execute( + "SELECT id, status, started_at FROM schedule_executions " + "WHERE agent_name = ? AND status IN ('running', 'queued')", + (agent_name,), + ) + out: Dict[str, Any] = {"running": set(), "queued": set(), "started_at": {}} + for row in cursor.fetchall(): + if row["status"] == "running": + out["running"].add(row["id"]) + if row["started_at"]: + out["started_at"][row["id"]] = row["started_at"] + elif row["status"] == "queued": + out["queued"].add(row["id"]) + return out + + +def _collect_terminal_executions(window_minutes: int = 30) -> Dict[str, str]: + """Recent terminal execution_ids → status (for E-02 reversal detection). + + Bounding the window keeps the comparison set small. Reversals are + expected within minutes of the original transition; older terminal + rows reverting would also indicate corruption but at vanishingly low + base rate, and would be caught by E-01 (terminal-state closure) too. + + Returns a dict so E-02 can persist the *real* prior status (success + / failed / cancelled / skipped) into its Redis side-table — the + reversal alert prints that back to the operator, and a placeholder + string ("terminal") would erase the forensic value of the alert. + """ + from db.connection import get_db_connection + + placeholders = ",".join("?" * len(TERMINAL_EXECUTION_STATUSES)) + cutoff = iso_cutoff(minutes=int(window_minutes)) + with get_db_connection() as conn: + cursor = conn.cursor() + cursor.execute( + f""" + SELECT id, status FROM schedule_executions + WHERE status IN ({placeholders}) + AND completed_at > ? + """, + (*TERMINAL_EXECUTION_STATUSES, cutoff), + ) + return {row["id"]: row["status"] for row in cursor.fetchall()} + + +def _collect_orphan_refs(known_agents: Set[str]) -> List[OrphanRef]: + """Scan cross-table agent_name refs for any not in known_agents. + + Driven by ORPHAN_SCAN_TABLES. Each tuple is (table, column, optional + SQL filter clause that further narrows what counts as 'live'). + """ + from db.connection import get_db_connection + + refs: List[OrphanRef] = [] + if not known_agents: + return refs # nothing to compare against; scan would mark every row + + placeholder_list = ",".join("?" * len(known_agents)) + known_params = list(known_agents) + + with get_db_connection() as conn: + cursor = conn.cursor() + for table, column, extra_filter in ORPHAN_SCAN_TABLES: + # Discover the primary-key column name so we can return a + # stable row_id without hardcoding per-table schemas. + cursor.execute(f"PRAGMA table_info({table})") + cols = cursor.fetchall() + if not cols: + # Table not present (test DB or partial install). Skip. + continue + pk_col = next((c["name"] for c in cols if c["pk"]), None) + if pk_col is None: + # Composite-PK or no-PK tables get a synthetic row_id. + pk_expr = f"'{table}-row'" + else: + pk_expr = pk_col + + where = f"{column} NOT IN ({placeholder_list})" + if extra_filter: + where += f" AND ({extra_filter})" + + cursor.execute( + f"SELECT {pk_expr} AS row_id, {column} AS agent_name " + f"FROM {table} WHERE {where}", + known_params, + ) + for row in cursor.fetchall(): + refs.append( + OrphanRef( + table=table, + column=column, + referenced_agent_name=row["agent_name"], + row_id=str(row["row_id"]), + ) + ) + + # Agent-scoped MCP keys: same logic, separate filter on `scope`. + cursor.execute("PRAGMA table_info(mcp_api_keys)") + cols = cursor.fetchall() + if cols: + cursor.execute( + f""" + SELECT id, agent_name FROM mcp_api_keys + WHERE scope = 'agent' + AND agent_name IS NOT NULL + AND agent_name NOT IN ({placeholder_list}) + """, + known_params, + ) + for row in cursor.fetchall(): + refs.append( + OrphanRef( + table="mcp_api_keys", + column="agent_name", + referenced_agent_name=row["agent_name"], + row_id=str(row["id"]), + ) + ) + + return refs + + +def _collect_redis_slot_state(known_agents: Set[str]) -> Dict[str, Dict[str, Any]]: + """Per-agent Redis slot ZSET membership + scan for orphan slot keys. + + Returns dict with two keys: + "by_agent": {agent_name: set(execution_ids)} for known agents + "orphan_slots": {agent_name_in_key: count} for keys matching agents + NOT in agent_ownership + """ + from services.slot_service import get_slot_service + + slot_service = get_slot_service() + redis_client = slot_service.redis + prefix = slot_service.slots_prefix + + by_agent: Dict[str, Set[str]] = {} + scores: Dict[str, Dict[str, float]] = {} + orphan_slots: Dict[str, int] = {} + + # Per-agent ZRANGE for known agents (with scores for S-01 grace). + for name in known_agents: + with_scores = redis_client.zrange(f"{prefix}{name}", 0, -1, withscores=True) + by_agent[name] = {m for m, _ in with_scores} + scores[name] = {m: float(s) for m, s in with_scores} + + # SCAN for orphan keys (agent name in the key but not in known set). + cursor = 0 + while True: + cursor, keys = redis_client.scan( + cursor=cursor, match=f"{prefix}*", count=200 + ) + for key in keys: + # `decode_responses=True` on the slot_service client; key is str. + name = key[len(prefix):] + if name not in known_agents: + orphan_slots[name] = redis_client.zcard(key) + if cursor == 0: + break + + return {"by_agent": by_agent, "scores": scores, "orphan_slots": orphan_slots} + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + + +def collect_snapshot() -> Snapshot: + """Collect one complete snapshot. + + Sources that fail (e.g. Redis unreachable) are recorded in + `sources_unavailable` and the snapshot is still returned with whatever + succeeded. Invariant checks are responsible for skipping cycles when + their required sources are absent — see each invariant for the policy. + """ + snap = Snapshot(snapshot_time=utc_now_iso()) + + # SQLite: agent_ownership is the source of truth for "known agents". + try: + agent_rows = _collect_known_agents() + except Exception as exc: + logger.exception("canary snapshot: agent_ownership read failed") + snap.sources_unavailable.append(f"sqlite.agent_ownership: {exc}") + return snap + + snap.known_agents = {row["agent_name"] for row in agent_rows} + + # Redis slot state (scan once for both per-agent and orphan keys). + redis_state: Dict[str, Any] = {"by_agent": {}, "scores": {}, "orphan_slots": {}} + try: + redis_state = _collect_redis_slot_state(snap.known_agents) + snap.orphan_redis_slots = redis_state["orphan_slots"] + except Exception as exc: + logger.exception("canary snapshot: redis read failed") + snap.sources_unavailable.append(f"redis: {exc}") + + # SQLite: per-agent running/queued executions. + for row in agent_rows: + name = row["agent_name"] + try: + execs = _collect_executions(name) + except Exception as exc: + logger.exception("canary snapshot: executions read failed for %s", name) + snap.sources_unavailable.append(f"sqlite.executions[{name}]: {exc}") + execs = {"running": set(), "queued": set()} + + snap.agents.append( + AgentSnapshot( + name=name, + is_system=bool(row["is_system"]), + max_parallel=int(row["max_parallel_tasks"]), + execution_timeout_seconds=int(row["execution_timeout_seconds"]), + slot_ids=redis_state["by_agent"].get(name, set()), + slot_scores=redis_state["scores"].get(name, {}), + running_exec_ids=execs["running"], + running_started_at=execs.get("started_at", {}), + queued_exec_ids=execs["queued"], + ) + ) + + # SQLite: orphan refs across cross-cutting tables (L-03). + try: + snap.orphan_refs = _collect_orphan_refs(snap.known_agents) + except Exception as exc: + logger.exception("canary snapshot: orphan ref scan failed") + snap.sources_unavailable.append(f"sqlite.orphan_refs: {exc}") + + # SQLite: terminal execution ids → status for E-02 detector. + try: + snap.terminal_exec_statuses = _collect_terminal_executions() + except Exception as exc: + logger.exception("canary snapshot: terminal executions read failed") + snap.sources_unavailable.append(f"sqlite.terminal_executions: {exc}") + + return snap diff --git a/src/backend/database.py b/src/backend/database.py index a2f997d5b..4e1abc3e7 100644 --- a/src/backend/database.py +++ b/src/backend/database.py @@ -133,6 +133,7 @@ from db.whatsapp_channels import WhatsAppChannelOperations from db.access_requests import AccessRequestOperations from db.audit import PlatformAuditOperations +from db.canary import CanaryOperations from db.sync_state import SyncStateOperations @@ -289,6 +290,7 @@ def __init__(self): self._whatsapp_channel_ops = WhatsAppChannelOperations() self._access_request_ops = AccessRequestOperations() self._audit_ops = PlatformAuditOperations() + self._canary_ops = CanaryOperations() self._sync_state_ops = SyncStateOperations() # #389 sync health # ========================================================================= @@ -1914,6 +1916,49 @@ def prune_audit_log(self, retention_days: int) -> int: """Delete audit_log entries older than ``retention_days``. Returns count removed.""" return self._audit_ops.prune_audit_log(retention_days) + # ========================================================================= + # Canary Invariant Violations (CANARY-001 / Issue #411 — Phase 1) + # ========================================================================= + + def insert_canary_violation( + self, + invariant_id: str, + tier: str, + severity: str, + snapshot_time: str, + observed_state: dict, + signal_query: str = None, + ) -> int: + """Insert a violation row from the canary harness; returns row id.""" + return self._canary_ops.insert_violation( + invariant_id=invariant_id, + tier=tier, + severity=severity, + snapshot_time=snapshot_time, + observed_state=observed_state, + signal_query=signal_query, + ) + + def list_canary_violations(self, **filters): + """Query violations with optional filters (newest first). See CanaryOperations.""" + return self._canary_ops.list_violations(**filters) + + def count_canary_violations(self, **filters): + """Count violations matching filters (independent of limit/offset).""" + return self._canary_ops.count_violations(**filters) + + def get_canary_violation(self, violation_id: int): + """Fetch a single violation by id.""" + return self._canary_ops.get_violation(violation_id) + + def get_latest_canary_violation_per_invariant(self): + """Latest violation per invariant_id; used for green→red transition detection.""" + return self._canary_ops.get_latest_per_invariant() + + def get_canary_stats(self, start_time: str = None, end_time: str = None): + """Aggregate canary violation counts by invariant_id and severity.""" + return self._canary_ops.stats_by_invariant(start_time=start_time, end_time=end_time) + # Global database manager instance db = DatabaseManager() diff --git a/src/backend/db/canary.py b/src/backend/db/canary.py new file mode 100644 index 000000000..8c9fde0ac --- /dev/null +++ b/src/backend/db/canary.py @@ -0,0 +1,262 @@ +""" +Canary invariant violations database operations (CANARY-001 / Issue #411). + +Append-mostly access to the `canary_violations` table populated by the +continuous orchestration-invariant harness. `services/canary_service.py` +writes one row per fired check each cycle; the read API surfaces them to +admins for triage. + +`observed_state` is stored as a JSON string per invariant; the helpers +parse it on the way out so callers see a dict. +""" + +import json +from typing import Any, Dict, List, Optional + +from .connection import get_db_connection + + +# Tier and severity values are validated at write time so the read API can +# expose them as plain strings without a DB-level CHECK constraint. +_VALID_TIERS = {"A", "B"} +_VALID_SEVERITIES = {"critical", "major", "minor"} + + +class CanaryOperations: + """Database operations for the canary invariant violations table.""" + + # --------------------------------------------------------------------- + # Write + # --------------------------------------------------------------------- + + def insert_violation( + self, + invariant_id: str, + tier: str, + severity: str, + snapshot_time: str, + observed_state: Dict[str, Any], + signal_query: Optional[str] = None, + ) -> int: + """Insert a violation row, returning the new id. + + `observed_state` is JSON-serialized here so the caller passes a dict. + """ + if tier not in _VALID_TIERS: + raise ValueError(f"invalid tier {tier!r}; expected one of {_VALID_TIERS}") + if severity not in _VALID_SEVERITIES: + raise ValueError( + f"invalid severity {severity!r}; expected one of {_VALID_SEVERITIES}" + ) + + with get_db_connection() as conn: + cursor = conn.cursor() + cursor.execute( + """ + INSERT INTO canary_violations ( + invariant_id, tier, severity, snapshot_time, + observed_state, signal_query + ) VALUES (?, ?, ?, ?, ?, ?) + """, + ( + invariant_id, + tier, + severity, + snapshot_time, + json.dumps(observed_state), + signal_query, + ), + ) + return int(cursor.lastrowid) + + # --------------------------------------------------------------------- + # Read + # --------------------------------------------------------------------- + + def list_violations( + self, + invariant_id: Optional[str] = None, + severity: Optional[str] = None, + tier: Optional[str] = None, + start_time: Optional[str] = None, + end_time: Optional[str] = None, + limit: int = 100, + offset: int = 0, + ) -> List[Dict[str, Any]]: + """Query violations with optional filters, newest first.""" + conditions: List[str] = [] + params: List[Any] = [] + + if invariant_id: + conditions.append("invariant_id = ?") + params.append(invariant_id) + if severity: + conditions.append("severity = ?") + params.append(severity) + if tier: + conditions.append("tier = ?") + params.append(tier) + if start_time: + conditions.append("snapshot_time >= ?") + params.append(start_time) + if end_time: + conditions.append("snapshot_time <= ?") + params.append(end_time) + + where_clause = " AND ".join(conditions) if conditions else "1=1" + params.extend([int(limit), int(offset)]) + + with get_db_connection() as conn: + cursor = conn.cursor() + cursor.execute( + f""" + SELECT * FROM canary_violations + WHERE {where_clause} + ORDER BY snapshot_time DESC, id DESC + LIMIT ? OFFSET ? + """, + params, + ) + return [self._row_to_dict(row) for row in cursor.fetchall()] + + def count_violations( + self, + invariant_id: Optional[str] = None, + severity: Optional[str] = None, + tier: Optional[str] = None, + start_time: Optional[str] = None, + end_time: Optional[str] = None, + ) -> int: + """Return total count for a filter (independent of limit/offset).""" + conditions: List[str] = [] + params: List[Any] = [] + + if invariant_id: + conditions.append("invariant_id = ?") + params.append(invariant_id) + if severity: + conditions.append("severity = ?") + params.append(severity) + if tier: + conditions.append("tier = ?") + params.append(tier) + if start_time: + conditions.append("snapshot_time >= ?") + params.append(start_time) + if end_time: + conditions.append("snapshot_time <= ?") + params.append(end_time) + + where_clause = " AND ".join(conditions) if conditions else "1=1" + + with get_db_connection() as conn: + cursor = conn.cursor() + cursor.execute( + f"SELECT COUNT(*) FROM canary_violations WHERE {where_clause}", + params, + ) + return int(cursor.fetchone()[0]) + + def get_violation(self, violation_id: int) -> Optional[Dict[str, Any]]: + """Fetch a single violation by primary key.""" + with get_db_connection() as conn: + cursor = conn.cursor() + cursor.execute( + "SELECT * FROM canary_violations WHERE id = ?", + (int(violation_id),), + ) + row = cursor.fetchone() + return self._row_to_dict(row) if row else None + + def get_latest_per_invariant(self) -> Dict[str, Dict[str, Any]]: + """Return the most recent violation per invariant_id. + + Used by `CanaryService` for green→red transition detection: if the + latest stored violation for an invariant predates the current + snapshot, this cycle is a fresh transition that warrants a Slack + webhook post. + """ + with get_db_connection() as conn: + cursor = conn.cursor() + cursor.execute( + """ + SELECT v.* FROM canary_violations v + JOIN ( + SELECT invariant_id, MAX(id) AS max_id + FROM canary_violations + GROUP BY invariant_id + ) latest ON v.id = latest.max_id + """ + ) + return {row["invariant_id"]: self._row_to_dict(row) for row in cursor.fetchall()} + + def stats_by_invariant( + self, + start_time: Optional[str] = None, + end_time: Optional[str] = None, + ) -> Dict[str, Any]: + """Aggregate counts by invariant_id and severity for dashboard tiles.""" + time_filter = "" + params: List[Any] = [] + if start_time: + time_filter += " AND snapshot_time >= ?" + params.append(start_time) + if end_time: + time_filter += " AND snapshot_time <= ?" + params.append(end_time) + + with get_db_connection() as conn: + cursor = conn.cursor() + + cursor.execute( + f"SELECT COUNT(*) FROM canary_violations WHERE 1=1 {time_filter}", + params, + ) + total = int(cursor.fetchone()[0]) + + cursor.execute( + f""" + SELECT invariant_id, COUNT(*) AS cnt + FROM canary_violations + WHERE 1=1 {time_filter} + GROUP BY invariant_id + ORDER BY cnt DESC + """, + params, + ) + by_invariant = {row["invariant_id"]: int(row["cnt"]) for row in cursor.fetchall()} + + cursor.execute( + f""" + SELECT severity, COUNT(*) AS cnt + FROM canary_violations + WHERE 1=1 {time_filter} + GROUP BY severity + ORDER BY cnt DESC + """, + params, + ) + by_severity = {row["severity"]: int(row["cnt"]) for row in cursor.fetchall()} + + return { + "total": total, + "by_invariant": by_invariant, + "by_severity": by_severity, + } + + # --------------------------------------------------------------------- + # Helpers + # --------------------------------------------------------------------- + + @staticmethod + def _row_to_dict(row) -> Dict[str, Any]: + """Convert sqlite3.Row to dict; parse `observed_state` JSON.""" + result = {key: row[key] for key in row.keys()} + observed = result.get("observed_state") + if observed: + try: + result["observed_state"] = json.loads(observed) + except (TypeError, ValueError): + # Leave as raw string if not valid JSON. + pass + return result diff --git a/src/backend/db/migrations.py b/src/backend/db/migrations.py index 5e4174cb9..ce0412a79 100644 --- a/src/backend/db/migrations.py +++ b/src/backend/db/migrations.py @@ -4,7 +4,7 @@ Each migration function handles a specific schema change. Migrations are idempotent - safe to run multiple times. -Migration Order (as of 2026-02-28): +Migration Order (as of 2026-05-07): 1. agent_sharing - Email-based sharing (from user_id) 2. schedule_executions_observability - Context/cost/tools columns 3. mcp_api_keys_agent_scope - Agent collaboration support @@ -34,10 +34,34 @@ 27. agent_ownership_execution_timeout - TIMEOUT-001 per-agent execution timeout 28. public_user_memory_table - MEM-001 per-user persistent memory for public link agents 29. subscription_rate_limit_tracking - SUB-003 rate-limit event tracking for auto-switch -30. execution_fan_out_id - FANOUT-001 fan-out operation linkage -31. scheduler_retry_support - RETRY-001 scheduler retry mechanism -32. validation_support - VALIDATE-001 post-execution business validation -33. agent_git_config_pat - #347 per-agent GitHub PAT support +30. chat_messages_source_column - Chat message source tracking +31. agent_ownership_voice_prompt - Per-agent voice system prompt +32. slack_channel_agents - SLACK-002 channel-agent bindings +33. execution_fan_out_id - FANOUT-001 fan-out operation linkage +34. telegram_bindings - TELEGRAM-001 Telegram bot integration tables +35. subscription_usage_tracking - SUB-002 usage tracking columns +36. telegram_group_configs - TGRAM-GROUP Telegram group chat configs +37. access_control - #311 unified channel access control (access_requests, verified_email) +38. public_link_require_email_unified - #311 unified require_email flag on public links +39. email_whitelist_default_role - #314 default role on email whitelist rows +40. backlog_support - BACKLOG-001 persistent FIFO overflow store +41. scheduler_retry_support - RETRY-001 scheduler retry mechanism +42. validation_support - VALIDATE-001 post-execution business validation +43. audit_log_table - SEC-001 / #20 platform audit log +44. group_auth_mode - Telegram/Slack group auth mode +45. agent_ownership_guardrails - GUARD-001 per-agent guardrails overrides +46. agent_git_config_pat - #347 per-agent GitHub PAT support +47. proactive_messaging - #321 proactive agent-to-user messaging +48. agent_git_config_branch_ownership - Working-branch ownership tracking +49. sync_health - #389 agent_sync_state for sync health observability +50. whatsapp_bindings - WHATSAPP-001 Twilio WhatsApp integration +51. agent_schedules_webhook - WEBHOOK-001 webhook tokens for schedules +52. agent_shared_files - FILES-001 outbound file sharing +53. agent_sessions_tables - SESSION_TAB --resume-default Session tab +54. session_compact_events - Session compact event tracking +55. public_links_type - SITE-001 type column on public links (chat | site) +56. slack_bot_token_encryption - #453 encrypt Slack bot tokens at rest +57. canary_violations_table - CANARY-001 / Issue #411 invariant harness violations """ import logging import sqlite3 @@ -1944,6 +1968,44 @@ def _encrypt_table(table_name: str, token_column: str) -> tuple[int, int]: conn.commit() +def _migrate_canary_violations_table(cursor, conn): + """Create canary_violations table + indexes (CANARY-001 / Issue #411 — Phase 1). + + Stores orchestration-invariant violations recorded by the continuous + canary harness. Schema is also defined in db/schema.py for fresh + installs; this migration handles existing installs. + """ + cursor.execute("PRAGMA table_info(canary_violations)") + if cursor.fetchall(): + return # already created (fresh-install path via init_schema) + + cursor.execute(""" + CREATE TABLE IF NOT EXISTS canary_violations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + invariant_id TEXT NOT NULL, + tier TEXT NOT NULL, + severity TEXT NOT NULL, + snapshot_time TEXT NOT NULL, + observed_state TEXT NOT NULL, + signal_query TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')) + ) + """) + + for ddl in [ + "CREATE INDEX IF NOT EXISTS idx_canary_violations_invariant " + "ON canary_violations(invariant_id, snapshot_time DESC)", + "CREATE INDEX IF NOT EXISTS idx_canary_violations_severity " + "ON canary_violations(severity, snapshot_time DESC)", + "CREATE INDEX IF NOT EXISTS idx_canary_violations_snapshot " + "ON canary_violations(snapshot_time DESC)", + ]: + cursor.execute(ddl) + + conn.commit() + print("Created canary_violations table with indexes (CANARY-001)") + + MIGRATIONS = [ ("agent_sharing", _migrate_agent_sharing_table), ("schedule_executions_observability", _migrate_schedule_executions_observability), @@ -2001,4 +2063,5 @@ def _encrypt_table(table_name: str, token_column: str) -> tuple[int, int]: ("session_compact_events", _migrate_session_compact_events), ("public_links_type", _migrate_public_links_type), ("slack_bot_token_encryption", _migrate_slack_bot_token_encryption), + ("canary_violations_table", _migrate_canary_violations_table), ] diff --git a/src/backend/db/schema.py b/src/backend/db/schema.py index 93bd85bcf..cc68c0d0a 100644 --- a/src/backend/db/schema.py +++ b/src/backend/db/schema.py @@ -1014,6 +1014,28 @@ created_at TEXT NOT NULL DEFAULT (datetime('now')) ) """, + + # ------------------------------------------------------------------------- + # Canary Invariant Harness (CANARY-001 / Issue #411 — Phase 1) + # ------------------------------------------------------------------------- + # Continuous orchestration-invariant violations recorded by the canary + # watcher service (`services/canary_service.py`). Each row is one fired + # check; the row stores the invariant id, tier, severity, snapshot + # timestamp, and a JSON `observed_state` payload specific to the + # invariant. The service writes here every cycle and posts to a Slack + # webhook (`CANARY_SLACK_WEBHOOK_URL`) on green→red transitions. + "canary_violations": """ + CREATE TABLE IF NOT EXISTS canary_violations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + invariant_id TEXT NOT NULL, + tier TEXT NOT NULL, + severity TEXT NOT NULL, + snapshot_time TEXT NOT NULL, + observed_state TEXT NOT NULL, + signal_query TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')) + ) + """, } # ============================================================================= @@ -1190,6 +1212,11 @@ "CREATE INDEX IF NOT EXISTS idx_audit_log_mcp_key ON audit_log(mcp_key_id, timestamp DESC)", "CREATE INDEX IF NOT EXISTS idx_audit_log_request ON audit_log(request_id)", + # Canary violations indexes (CANARY-001 / Issue #411 — Phase 1) + "CREATE INDEX IF NOT EXISTS idx_canary_violations_invariant ON canary_violations(invariant_id, snapshot_time DESC)", + "CREATE INDEX IF NOT EXISTS idx_canary_violations_severity ON canary_violations(severity, snapshot_time DESC)", + "CREATE INDEX IF NOT EXISTS idx_canary_violations_snapshot ON canary_violations(snapshot_time DESC)", + # Subscription credentials indexes (SUB-001) "CREATE INDEX IF NOT EXISTS idx_subscriptions_name ON subscription_credentials(name)", "CREATE INDEX IF NOT EXISTS idx_subscriptions_owner ON subscription_credentials(owner_id)", diff --git a/src/backend/main.py b/src/backend/main.py index 19b260256..8287abe9e 100644 --- a/src/backend/main.py +++ b/src/backend/main.py @@ -72,6 +72,7 @@ from routers.logs import router as logs_router from routers.agent_dashboard import router as agent_dashboard_router from routers.audit_log import router as audit_log_router # SEC-001 / Issue #20 +from routers.canary import router as canary_router # CANARY-001 / Issue #411 from routers.skills import router as skills_router from routers.internal import router as internal_router from routers.tags import router as tags_router @@ -113,6 +114,7 @@ # Import cleanup service from services.cleanup_service import cleanup_service, set_cleanup_ws_manager +from services.canary_service import canary_service # CANARY-001 / Issue #411 from services.platform_audit_service import platform_audit_service, AuditEventType @@ -420,6 +422,14 @@ async def _start_sync_health_delayed(): print(f"Error starting sync health service: {e}") asyncio.create_task(_start_sync_health_delayed()) + # CANARY-001 / Issue #411: Canary watcher — 5-min cycle. Disabled by + # default (CANARY_ENABLED=1 to enable on staging/dev). Service self- + # gates internally; the start() call is a no-op when not enabled. + try: + canary_service.start() + except Exception as e: + print(f"Error starting canary service: {e}") + # BACKLOG-001 / CAPACITY-CONSOLIDATE (#428): instantiate the unified # CapacityManager (this also wires the slot-release → backlog-drain # callback internally) and spawn the 60s maintenance loop. The @@ -638,6 +648,13 @@ async def _capacity_maintenance_loop(): except Exception as e: print(f"Error stopping sync health service: {e}") + # Shutdown canary service (CANARY-001 / Issue #411) + try: + canary_service.stop() + print("Canary service stopped") + except Exception as e: + print(f"Error stopping canary service: {e}") + # Shutdown operator queue sync service try: operator_queue_service.stop() @@ -781,6 +798,7 @@ async def add_security_headers(request: Request, call_next): app.include_router(logs_router) app.include_router(agent_dashboard_router) app.include_router(audit_log_router) # SEC-001 / #20: Platform audit log (Phase 1) +app.include_router(canary_router) # CANARY-001 / #411: Invariant violations app.include_router(skills_router) # Skills Management System app.include_router(internal_router) # Internal agent-to-backend endpoints (no auth) app.include_router(tags_router) # Agent Tags (ORG-001) diff --git a/src/backend/routers/canary.py b/src/backend/routers/canary.py new file mode 100644 index 000000000..5cbea0a05 --- /dev/null +++ b/src/backend/routers/canary.py @@ -0,0 +1,281 @@ +""" +Canary Invariant Harness API (CANARY-001 / Issue #411). + +Admin-only query interface over the `canary_violations` table populated by +the continuous canary harness. Phase 1 ships this read endpoint plus the +`CanaryService` 5-minute background loop that posts to a Slack webhook +(`CANARY_SLACK_WEBHOOK_URL`) on green→red transitions; the table itself is +the source of truth for forensic replay and 24-hour trend tiles. + +Mounted at `/api/canary` to keep the canary surface area distinct from the +platform audit log. +""" + +import logging +import time +from typing import List, Optional + +from fastapi import APIRouter, Depends, HTTPException, Query +from pydantic import BaseModel, Field + +from canary import INVARIANTS +from database import db +from dependencies import require_admin +from models import User +from services.canary_alerts import severity_rank +from services.canary_service import canary_service + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/api/canary", tags=["canary"]) + + +# --------------------------------------------------------------------------- +# Response models +# --------------------------------------------------------------------------- + + +class CanaryViolation(BaseModel): + """Single canary_violations row as returned to API clients.""" + + id: int + invariant_id: str + tier: str + severity: str + snapshot_time: str + observed_state: dict = Field(default_factory=dict) + signal_query: Optional[str] = None + created_at: Optional[str] = None + + +class CanaryViolationListResponse(BaseModel): + """Paginated list response.""" + + violations: List[CanaryViolation] + total: int + limit: int + offset: int + + +class CanaryStatsResponse(BaseModel): + """Aggregate violation counts for dashboard tiles.""" + + total: int + by_invariant: dict = Field(default_factory=dict) + by_severity: dict = Field(default_factory=dict) + + +# --------------------------------------------------------------------------- +# Run-cycle request / response models +# --------------------------------------------------------------------------- + + +class RunCycleRequest(BaseModel): + """Optional filter on which invariants to run this cycle.""" + + invariants: Optional[List[str]] = Field( + None, + description=( + "Subset of invariant ids to run. Default: all enabled " + f"({sorted(INVARIANTS.keys())})." + ), + ) + + +class CycleViolation(BaseModel): + """One violation persisted during a run-cycle call.""" + + id: int + invariant_id: str + tier: str + severity: str + snapshot_time: str + observed_state: dict + signal_query: Optional[str] = None + + +class CycleTransition(BaseModel): + """A green→red transition detected this cycle. + + `CanaryService` posts exactly one Slack webhook message per entry, + mapping severity to the message styling. Surfaced here so the run-cycle + response mirrors what the service actually emitted. + """ + + invariant_id: str + severity: str + violations_in_cycle: int + previous_violation_at: Optional[str] = Field( + None, + description=( + "snapshot_time of the most recent prior violation for this " + "invariant; null if the invariant has never violated before." + ), + ) + + +class RunCycleResponse(BaseModel): + """Result of one canary cycle.""" + + snapshot_time: str + cycle_duration_ms: int + # Invariants this cycle attempted (= the request's `invariants` filter, + # or all registered ids if unfiltered). Whether each one *fired* is + # surfaced via `violations` and `transitions`. Sources that were down + # this cycle are listed in `sources_unavailable` — invariants that + # depend on them returned no violations regardless of state. + checks_run: List[str] + sources_unavailable: List[str] + violations: List[CycleViolation] + transitions: List[CycleTransition] + + +# --------------------------------------------------------------------------- +# Endpoints +# --------------------------------------------------------------------------- + + +@router.get("/violations", response_model=CanaryViolationListResponse) +async def list_canary_violations( + invariant_id: Optional[str] = Query(None, description="Filter by invariant id (e.g. 'S-01')"), + severity: Optional[str] = Query(None, description="Filter by severity (critical|major|minor)"), + tier: Optional[str] = Query(None, description="Filter by tier (A|B)"), + start_time: Optional[str] = Query(None, description="Filter snapshot_time >= ISO 8601"), + end_time: Optional[str] = Query(None, description="Filter snapshot_time <= ISO 8601"), + limit: int = Query(100, ge=1, le=1000, description="Max rows returned"), + offset: int = Query(0, ge=0, description="Rows to skip"), + _: User = Depends(require_admin), +) -> CanaryViolationListResponse: + """List canary invariant violations, newest first. Admin only.""" + filters = { + "invariant_id": invariant_id, + "severity": severity, + "tier": tier, + "start_time": start_time, + "end_time": end_time, + } + rows = db.list_canary_violations(limit=limit, offset=offset, **filters) + total = db.count_canary_violations(**filters) + return CanaryViolationListResponse( + violations=[CanaryViolation(**row) for row in rows], + total=total, + limit=limit, + offset=offset, + ) + + +@router.get("/violations/stats", response_model=CanaryStatsResponse) +async def get_canary_stats( + start_time: Optional[str] = Query(None, description="Filter snapshot_time >= ISO 8601"), + end_time: Optional[str] = Query(None, description="Filter snapshot_time <= ISO 8601"), + _: User = Depends(require_admin), +) -> CanaryStatsResponse: + """Aggregate violation counts by invariant_id and severity. Admin only.""" + stats = db.get_canary_stats(start_time=start_time, end_time=end_time) + return CanaryStatsResponse(**stats) + + +@router.get("/violations/{violation_id}", response_model=CanaryViolation) +async def get_canary_violation( + violation_id: int, + _: User = Depends(require_admin), +) -> CanaryViolation: + """Fetch a single violation by id. Admin only.""" + row = db.get_canary_violation(violation_id) + if not row: + raise HTTPException(status_code=404, detail="Violation not found") + return CanaryViolation(**row) + + +# --------------------------------------------------------------------------- +# Run-cycle endpoint +# --------------------------------------------------------------------------- + + +@router.post("/run-cycle", response_model=RunCycleResponse) +async def run_canary_cycle( + body: RunCycleRequest | None = None, + _: User = Depends(require_admin), +) -> RunCycleResponse: + """Run one canary cycle on demand. + + Admin only. Delegates to the same `CanaryService.run_cycle()` invoked + by the 5-minute background loop, so the operator-on-demand path and + the scheduled path share their entire implementation. Useful for: + - smoke-testing the harness right after deploy (don't wait 5 min) + - confirming a violation cleared after a fix + - integration tests that need deterministic cycle timing + + The response surfaces exactly the transitions the service emitted — + no recomputation here — so the endpoint and the Slack webhook cannot + disagree. + """ + body = body or RunCycleRequest() + requested_ids = body.invariants or list(INVARIANTS.keys()) + invalid_ids = [i for i in requested_ids if i not in INVARIANTS] + if invalid_ids: + # 422 keeps unknown ids from silently no-op'ing — easier to debug. + raise HTTPException( + status_code=422, + detail=f"Unknown invariant id(s): {invalid_ids}. " + f"Available: {sorted(INVARIANTS.keys())}", + ) + started = time.monotonic() + cycle = await canary_service.run_cycle(invariant_ids=requested_ids) + duration_ms = int((time.monotonic() - started) * 1000) + + # 409 is the operator-friendly signal for "another cycle was mid-run, this + # call did nothing." Without it the response is structurally identical to + # a real green cycle (empty violations, empty transitions) and the caller + # has no way to tell their request was skipped — see /review I2. + if cycle.skipped: + raise HTTPException(status_code=409, detail="cycle in progress") + + # Row ids come straight from the service via `persisted_violation_ids` + # (index-aligned with `cycle.violations`); no re-query needed. A `None` + # slot means the insert failed — we drop those from the response rather + # than surface a stale row. + snapshot_time = cycle.snapshot_time + persisted: List[CycleViolation] = [] + transitions_out: List[CycleTransition] = [] + transition_set = set(cycle.transition_invariant_ids) + + for invariant_id, vlist in cycle.violations.items(): + ids = cycle.persisted_violation_ids.get(invariant_id, []) + for v, row_id in zip(vlist, ids): + if row_id is None: + continue + persisted.append(CycleViolation( + id=row_id, + invariant_id=v.invariant_id, + tier=v.tier, + severity=v.severity, + snapshot_time=snapshot_time, + observed_state=v.observed_state, + signal_query=v.signal_query, + )) + + # Build a transition entry only for invariants the SERVICE actually + # decided fired a notification this cycle. Continuing-red invariants + # have rows in `persisted` but are absent from `transition_set`. + if invariant_id in transition_set and vlist: + worst = max(vlist, key=lambda v: severity_rank(v.severity)) + transitions_out.append(CycleTransition( + invariant_id=invariant_id, + severity=worst.severity, + violations_in_cycle=len(vlist), + # The service captured this from `previous_latest` BEFORE + # the cycle's inserts, so it's the prior cycle's tail — + # not the row we just wrote. `None` means first-ever + # violation for this invariant. + previous_violation_at=cycle.previous_violation_at.get(invariant_id), + )) + + return RunCycleResponse( + snapshot_time=snapshot_time, + cycle_duration_ms=duration_ms, + checks_run=list(requested_ids), + sources_unavailable=cycle.sources_unavailable, + violations=persisted, + transitions=transitions_out, + ) diff --git a/src/backend/services/canary_alerts.py b/src/backend/services/canary_alerts.py new file mode 100644 index 000000000..e986c9d7d --- /dev/null +++ b/src/backend/services/canary_alerts.py @@ -0,0 +1,370 @@ +""" +Canary alert sink — Slack Block Kit composition + webhook post (CANARY-001 / #411). + +Extracted from `services/canary_service.py` to keep the cycle orchestrator +focused on lifecycle + invariant runs. The watcher imports `CanaryAlerts` +and calls `emit_transition` once per green→red transition; everything +Slack-shaped lives here. + +The split is purely organisational — there's no behaviour change vs. when +these methods lived on `CanaryService` as classmethods. Tests pivoted from +`CanaryService._foo` to `CanaryAlerts._foo` accordingly. +""" + +import logging +import os +from datetime import datetime +from typing import List, Optional, Tuple + +from canary.snapshot import ViolationReport + + +logger = logging.getLogger(__name__) + + +class CanaryAlerts: + """Stateless Slack alert composer + sink for canary transitions.""" + + # Severity → Slack emoji. Common monitoring convention; rendered in + # the header block so the alert is scannable at a glance even with + # the channel collapsed in the sidebar. + _SEVERITY_EMOJI = { + "critical": "🚨", + "major": "⚠️", + "minor": "🟡", + } + + # Friendly invariant names — paired with the catalog at + # docs/testing/orchestration-invariant-catalog.md. The bare ID + # (S-01, E-02, …) is opaque to anyone not steeped in the catalog; + # the name is what makes the Slack alert immediately interpretable. + _INVARIANT_NAMES = { + "S-01": "Slot–row bijection", + "E-02": "Phantom execution reversal", + "L-03": "Delete cascades", + } + + # One-line runbook hint per invariant. Kept short on purpose — + # the alert is the entry point, the catalog has the full prose. + # Tells the on-call where to start looking, not what to do. + _INVARIANT_RUNBOOKS = { + "S-01": ( + "Redis slot ZSET diverged from running schedule_executions rows. " + "Inspect for crashed `slot.release()` calls; `cleanup_service` " + "should reconcile within one cycle." + ), + "E-02": ( + "An execution went terminal then non-terminal. Look for retry " + "logic that resurrects completed rows or a status-write race." + ), + "L-03": ( + "An agent was deleted but a referencing row wasn't cascaded. " + "Check the delete handler for the table(s) listed above." + ), + } + + @classmethod + async def emit_transition( + cls, + invariant_id: str, + violations: List[ViolationReport], + snapshot_time: str, + previous_violation_at: Optional[str], + persisted_ids: List[Optional[int]], + ) -> None: + """Fire a Slack alert for a green→red transition. + + Reads the webhook URL from the `CANARY_SLACK_WEBHOOK_URL` env var. + If unset, logs at debug and returns — green→red detection still + runs and rows are still persisted to `canary_violations`, the + sink is just silent. Mirrors the `CANARY_ENABLED` env-gating + pattern for the watcher itself. + + The webhook URL is the credential. We don't echo it in any log + line. Failures are logged and swallowed so a hung webhook can't + break the cycle — `slack_service.post_webhook` already enforces + a 5s timeout. + """ + webhook_url = os.getenv("CANARY_SLACK_WEBHOOK_URL", "").strip() + if not webhook_url: + # Emit a structured debug line so operators can confirm the + # transition was *detected* even when alerts are silent. + worst = max(violations, key=lambda v: severity_rank(v.severity)) + logger.debug( + "canary transition (slack disabled — set CANARY_SLACK_WEBHOOK_URL): " + "%s severity=%s violations_in_cycle=%d snapshot_time=%s", + invariant_id, + worst.severity, + len(violations), + snapshot_time, + ) + return + + worst = max(violations, key=lambda v: severity_rank(v.severity)) + text, blocks = cls._build_slack_payload( + invariant_id, + violations, + snapshot_time, + previous_violation_at, + worst.severity, + persisted_ids, + ) + + # Lazy import — avoids dragging the SlackService init (and its + # httpx client) into test paths that exercise the canary library + # without the wider services tree. + from services.slack_service import slack_service + + success, error = await slack_service.post_webhook(webhook_url, text, blocks=blocks) + if not success: + logger.warning( + "canary slack webhook failed for %s: %s (cycle continues, row persisted)", + invariant_id, + error, + ) + else: + logger.info( + "canary slack alert sent: %s severity=%s violations_in_cycle=%d", + invariant_id, + worst.severity, + len(violations), + ) + + @classmethod + def _build_slack_payload( + cls, + invariant_id: str, + violations: List[ViolationReport], + snapshot_time: str, + previous_violation_at: Optional[str], + severity: str, + persisted_ids: List[Optional[int]], + ) -> Tuple[str, list]: + """Compose the Slack message text + Block Kit blocks. + + Layout: header → summary → forensic detail → runbook hint → + context (snapshot_time, count, last red, row ids). Tests + identify blocks by `type` rather than index so adding/removing + sections doesn't break them. + + Returns `(text, blocks)` — `text` is the fallback used by + clients that don't render blocks (notifications, screen + readers). + """ + emoji = cls._SEVERITY_EMOJI.get(severity, "•") + name = cls._INVARIANT_NAMES.get(invariant_id, invariant_id) + body = cls._render_message(invariant_id, violations, snapshot_time) + forensic = cls._render_forensic(invariant_id, violations) + runbook = cls._INVARIANT_RUNBOOKS.get(invariant_id) + last_red = cls._format_last_red(previous_violation_at, snapshot_time) + row_ref = cls._format_row_refs(persisted_ids) + + text = f"{emoji} canary {invariant_id} {name} ({severity}): {body}" + blocks = [ + { + "type": "header", + "text": { + "type": "plain_text", + "text": f"{emoji} {invariant_id} {name} — {severity}", + "emoji": True, + }, + }, + { + "type": "section", + "text": {"type": "mrkdwn", "text": body}, + }, + ] + if forensic: + blocks.append({ + "type": "section", + "text": {"type": "mrkdwn", "text": forensic}, + }) + if runbook: + blocks.append({ + "type": "section", + "text": {"type": "mrkdwn", "text": f"_{runbook}_"}, + }) + # Context line: row refs first if present (most actionable), + # then snapshot_time + count + last-red badge. + ctx_parts: List[str] = [] + if row_ref: + ctx_parts.append(row_ref) + ctx_parts.extend([ + f"`{snapshot_time}`", + f"{len(violations)} violation(s) this cycle", + last_red, + ]) + blocks.append({ + "type": "context", + "elements": [ + {"type": "mrkdwn", "text": " · ".join(ctx_parts)} + ], + }) + return text, blocks + + @classmethod + def _render_forensic( + cls, + invariant_id: str, + violations: List[ViolationReport], + ) -> Optional[str]: + """Per-invariant rendering of the forensic detail. + + The shape of `observed_state` differs per invariant — there's + no useful generic rendering. Each branch picks the fields that + actually help triage, in a Slack-mrkdwn format. Truncated to + keep the message scannable; full state is in the violation + row referenced by id in the context line. + + Returns `None` when the rendering would be empty — caller + omits the block entirely rather than emit an empty one. + """ + if invariant_id == "L-03": + tables: set = set() + refs: list = [] + for v in violations: + obs = v.observed_state or {} + tables.update(obs.get("tables_hit") or []) + for r in obs.get("sample_refs") or []: + refs.append(r) + lines: List[str] = [] + if tables: + lines.append(f"*Tables hit:* {', '.join(sorted(tables))}") + if refs: + lines.append("*Sample refs:*") + for r in refs[:5]: + lines.append( + f" • `{r.get('table')}.{r.get('column')}` " + f"(row `{r.get('row_id')}`)" + ) + if len(refs) > 5: + lines.append(f" • _… +{len(refs) - 5} more_") + return "\n".join(lines) if lines else None + + if invariant_id == "S-01": + lines: List[str] = [] + for v in violations[:5]: + obs = v.observed_state or {} + agent = obs.get("agent_name", "?") + redis_n = obs.get("redis_slot_count", "?") + sql_n = obs.get("sql_running_count", "?") + in_redis_only = obs.get("in_redis_only") or [] + in_sql_only = obs.get("in_sql_only") or [] + line = f"*{agent}*: redis={redis_n} vs sql={sql_n}" + diff_bits: List[str] = [] + if in_redis_only: + diff_bits.append( + f"redis-only: `{', '.join(in_redis_only[:3])}`" + + (f" +{len(in_redis_only) - 3}" if len(in_redis_only) > 3 else "") + ) + if in_sql_only: + diff_bits.append( + f"sql-only: `{', '.join(in_sql_only[:3])}`" + + (f" +{len(in_sql_only) - 3}" if len(in_sql_only) > 3 else "") + ) + if diff_bits: + line += "\n " + " · ".join(diff_bits) + lines.append(line) + if len(violations) > 5: + lines.append(f"_… +{len(violations) - 5} more agent(s)_") + return "\n".join(lines) if lines else None + + if invariant_id == "E-02": + lines: List[str] = [] + for v in violations[:5]: + obs = v.observed_state or {} + eid = obs.get("execution_id", "?") + prev = obs.get("previous_status") or "unknown" + curr = obs.get("current_status", "?") + lines.append(f" • `{eid}`: *{prev}* → *{curr}*") + if len(violations) > 5: + lines.append(f" • _… +{len(violations) - 5} more_") + return "\n".join(lines) if lines else None + + return None + + @staticmethod + def _format_row_refs(persisted_ids: List[Optional[int]]) -> Optional[str]: + """Render "violation #21" / "violations #21,#22,#23" / range form. + + Drops `None` slots (insert failures). Returns `None` when no + rows persisted — caller skips the row-ref segment entirely + rather than emit "violation None". + """ + ids = [i for i in (persisted_ids or []) if i is not None] + if not ids: + return None + if len(ids) == 1: + return f"violation #{ids[0]}" + if len(ids) <= 3: + return f"violations {', '.join(f'#{i}' for i in ids)}" + # 4+: collapse to range with count to keep the line tidy. + return f"violations #{min(ids)}–#{max(ids)} ({len(ids)} total)" + + @staticmethod + def _format_last_red( + previous_violation_at: Optional[str], + snapshot_time: str, + ) -> str: + """Render "last red Xm ago" / "first red" for the context block. + + Best-effort: if either timestamp fails to parse we fall back to + "first red" rather than crash the alert. Slack will render the + block fine without the badge. + """ + if not previous_violation_at: + return "first red for this invariant" + try: + prev = datetime.fromisoformat(previous_violation_at.replace("Z", "+00:00")) + now = datetime.fromisoformat(snapshot_time.replace("Z", "+00:00")) + delta = now - prev + secs = int(delta.total_seconds()) + if secs < 60: + return f"last red {secs}s ago" + if secs < 3600: + return f"last red {secs // 60}m ago" + if secs < 86400: + return f"last red {secs // 3600}h ago" + return f"last red {secs // 86400}d ago" + except Exception: + return "first red for this invariant" + + @staticmethod + def _render_message( + invariant_id: str, + violations: List[ViolationReport], + snapshot_time: str, + ) -> str: + """Human-readable one-liner for the Slack message body. + + Time is intentionally omitted — the Slack Block Kit payload + carries a relative "just now / 4m ago" context badge, and the + precise ISO `snapshot_time` is preserved in the `canary_violations` + row for forensic correlation. Embedding it in the message text + would be redundant. + """ + if invariant_id == "S-01": + agents = sorted({v.observed_state.get("agent_name") for v in violations}) + return ( + f"Slot–row bijection broke on {len(agents)} agent(s): " + f"{', '.join(agents)[:160]}." + ) + if invariant_id == "E-02": + return ( + f"{len(violations)} execution(s) reverted from terminal " + f"to non-terminal status." + ) + if invariant_id == "L-03": + ghosts = sorted( + {v.observed_state.get("ghost_agent_name") for v in violations} + ) + return ( + f"{len(ghosts)} ghost agent(s) referenced by orphan rows: " + f"{', '.join(ghosts)[:160]}." + ) + return f"{invariant_id} fired {len(violations)} violation(s)." + + +def severity_rank(severity: str) -> int: + """Higher = worse. Used to pick the loudest violation for a transition.""" + return {"minor": 1, "major": 2, "critical": 3}.get(severity, 0) diff --git a/src/backend/services/canary_service.py b/src/backend/services/canary_service.py new file mode 100644 index 000000000..fe75887eb --- /dev/null +++ b/src/backend/services/canary_service.py @@ -0,0 +1,353 @@ +""" +Canary watcher service (CANARY-001 / Issue #411). + +Runs in the backend process. Every 5 minutes: + +1. `collect_snapshot()` — read Redis, SQLite, agent registries. +2. `run_invariants(snapshot)` — apply S-01 / E-02 / L-03 (Phase 1 set). +3. Persist any violations to `canary_violations`. +4. Detect green→red transitions per invariant against the previously-stored + latest violation; fire one Slack alert per transition via incoming + webhook (`CANARY_SLACK_WEBHOOK_URL` env var; unset = silent sink). + +Modeled on `services/cleanup_service.py` — single asyncio task, idempotent +start/stop, lock-guarded re-entrancy. Disabled by default; enable per +deployment with `CANARY_ENABLED=1`. Production deployment is staging/dev. + +Why a service and not a Trinity agent (Issue #411 design discussion): +the watcher does no LLM reasoning — it's a deterministic library invocation +on a 5-minute timer. Running it as a Trinity agent would add an LLM call +per cycle and a separate container for no benefit. Deterministic checks +belong in the backend; the agents are the *fleet* (load generators), which +are deployed via the canary-fleet manifest. +""" + +import asyncio +import logging +import os +from dataclasses import dataclass, field +from typing import Dict, Iterable, List, Optional + +from canary import collect_snapshot, run_invariants +from canary.snapshot import ViolationReport +from database import db +from services.canary_alerts import CanaryAlerts + + +@dataclass +class CycleResult: + """Outcome of one canary cycle. + + `violations` is the per-invariant list of `ViolationReport`s the + deterministic library produced (now persisted to `canary_violations`). + + `transition_invariant_ids` is the subset the service classified as + a fresh green→red flip this cycle, not a continuation of an + already-known violation. The router exposes this directly to + operators so the on-demand `/api/canary/run-cycle` response matches + what the alert sink (when wired) will see. + + `persisted_violation_ids` is index-aligned with `violations`: for + each `ViolationReport` in `violations[inv_id][i]`, the row id + returned by `insert_canary_violation` is at + `persisted_violation_ids[inv_id][i]` — or `None` if the insert + failed. Lets the router surface row ids without re-querying. + + `skipped` is True when the lock was already held (concurrent cycle + in progress) and this call returned without running. The router + maps this to HTTP 409 so an empty response can never be confused + with a real green cycle. + """ + + violations: Dict[str, List[ViolationReport]] = field(default_factory=dict) + persisted_violation_ids: Dict[str, List[Optional[int]]] = field(default_factory=dict) + transition_invariant_ids: List[str] = field(default_factory=list) + # snapshot_time of the most recent prior violation per transitioning + # invariant — used by the alert sink to render "last red 2h ago" and + # by the run-cycle endpoint to surface `previous_violation_at`. Only + # populated for invariants in `transition_invariant_ids`. Value is + # `None` if this is the first-ever violation for that invariant. + previous_violation_at: Dict[str, Optional[str]] = field(default_factory=dict) + snapshot_time: str = "" + sources_unavailable: List[str] = field(default_factory=list) + skipped: bool = False + +logger = logging.getLogger(__name__) + + +# Five-minute cadence per the design doc. Deliberately the same as +# cleanup_service to share the operator's mental model (both are "every +# 5 min the backend reconciles state"). +CANARY_INTERVAL_SECONDS = 300 + +# Redis key holding the snapshot_time of the most recent cycle that ran. +# Used by transition detection so a continuously-red invariant fires a +# notification once (on the first cycle that catches it) rather than every +# cycle thereafter — see `_run_cycle_inner` for the rule. +REDIS_KEY_LAST_CYCLE = "canary:last_cycle_at" + + +class CanaryService: + """Background watcher loop for the canary invariant harness.""" + + def __init__(self, interval_seconds: int = CANARY_INTERVAL_SECONDS): + self.interval = interval_seconds + self._task: Optional[asyncio.Task] = None + self._running = False + self._lock = asyncio.Lock() + # Counters surface in /api/health-style monitoring; useful for + # confirming the service is actually firing on deployed instances. + self.cumulative_cycles: int = 0 + self.cumulative_violations: int = 0 + self.cumulative_transitions: int = 0 + self.last_run_at: Optional[str] = None + + # ------------------------------------------------------------------ + # Lifecycle + # ------------------------------------------------------------------ + + def start(self): + """Start the background loop. No-op if already running or disabled.""" + if not self._is_enabled(): + logger.info("canary: disabled (set CANARY_ENABLED=1 to enable)") + return + if self._running: + return + self._running = True + self._task = asyncio.create_task(self._loop()) + logger.info(f"canary watcher started (interval={self.interval}s)") + + def stop(self): + """Stop the background loop cleanly.""" + self._running = False + if self._task: + self._task.cancel() + self._task = None + logger.info("canary watcher stopped") + + @staticmethod + def _is_enabled() -> bool: + return os.getenv("CANARY_ENABLED", "0") == "1" + + # ------------------------------------------------------------------ + # Loop + # ------------------------------------------------------------------ + + async def _loop(self): + """Loop forever: cycle → sleep → cycle.""" + # Short delay so the backend is fully ready before the first cycle — + # avoids a noisy "sources_unavailable" log line during cold start. + await asyncio.sleep(30) + while self._running: + try: + await self.run_cycle() + except asyncio.CancelledError: + raise + except Exception: + # Never let a cycle exception kill the loop. Log and retry + # next interval. Mirrors cleanup_service. + logger.exception("canary cycle raised; will retry next interval") + try: + await asyncio.sleep(self.interval) + except asyncio.CancelledError: + raise + + # ------------------------------------------------------------------ + # One cycle + # ------------------------------------------------------------------ + + async def run_cycle( + self, + invariant_ids: Optional[Iterable[str]] = None, + ) -> CycleResult: + """Run one canary cycle. Public so it can be invoked from tests + or by the operator via `POST /api/canary/run-cycle`. + + Returns a `CycleResult` carrying the per-invariant violation + lists this cycle produced *and* the subset classified as + green→red transitions. Both pieces of truth come from the same + code path the background loop uses, so the on-demand endpoint + cannot disagree with the alert sink (when wired). + """ + if self._lock.locked(): + logger.debug("canary: cycle already in progress, skipping") + return CycleResult(skipped=True) + async with self._lock: + return await self._run_cycle_inner(invariant_ids) + + async def _run_cycle_inner( + self, + invariant_ids: Optional[Iterable[str]], + ) -> CycleResult: + # Capture pre-cycle state for green→red transition detection. + # Two reads BEFORE the cycle runs: + # 1. previous_latest — the most recent persisted violation per + # invariant id. Tells us "when was this invariant last red". + # 2. prev_cycle_at — snapshot_time of the prior cycle (any cycle, + # regardless of outcome). Tells us "when did we last look". + # + # An invariant transition (green→red) fires only when (a) the + # invariant has violations this cycle AND (b) the previous cycle + # was green for it — i.e. the latest stored violation predates + # the previous cycle's snapshot. This is the only rule that + # silences continuously-red invariants without losing real + # green→red flips, including red→green→red. + previous_latest = db.get_latest_canary_violation_per_invariant() + prev_cycle_at = self._read_prev_cycle_at() + + # Heavy work — synchronous SQLite + Redis reads. Offload to a thread + # so we don't block the asyncio loop while sqlite3 is blocking. + snapshot = await asyncio.to_thread(collect_snapshot) + results = await asyncio.to_thread(run_invariants, snapshot, invariant_ids) + + persisted_count = 0 + # Index-aligned with `results[inv_id]` — `None` slot means insert + # failed. The router uses these ids directly instead of re-querying + # by (invariant_id, snapshot_time). + persisted_ids: Dict[str, List[Optional[int]]] = {} + for inv_id, vlist in results.items(): + inv_ids: List[Optional[int]] = [] + for v in vlist: + try: + row_id = db.insert_canary_violation( + invariant_id=v.invariant_id, + tier=v.tier, + severity=v.severity, + snapshot_time=snapshot.snapshot_time, + observed_state=v.observed_state, + signal_query=v.signal_query, + ) + inv_ids.append(row_id) + persisted_count += 1 + except Exception: + inv_ids.append(None) + logger.exception( + "canary: failed to persist violation %s; continuing", + v.invariant_id, + ) + persisted_ids[inv_id] = inv_ids + + # Detect green→red transitions and emit one notification per. + transition_ids: List[str] = [] + previous_violation_at: Dict[str, Optional[str]] = {} + for inv_id, vlist in results.items(): + if not vlist: + continue + if not self._is_green_to_red(inv_id, previous_latest, prev_cycle_at): + continue + # Capture the prior snapshot_time BEFORE emit so the alert + # sink can render "last red Xm ago". `previous_latest` was + # loaded pre-cycle (line ~214) and is None when this is the + # first-ever violation for the invariant — pass that through + # honestly rather than papering over with the current cycle. + prev = previous_latest.get(inv_id) or {} + previous_violation_at[inv_id] = prev.get("snapshot_time") + try: + await CanaryAlerts.emit_transition( + inv_id, + vlist, + snapshot.snapshot_time, + previous_violation_at[inv_id], + persisted_ids.get(inv_id, []), + ) + transition_ids.append(inv_id) + except Exception: + logger.exception( + "canary: failed to emit transition notification for %s", + inv_id, + ) + + # Update counters + last-run. + self.cumulative_cycles += 1 + self.cumulative_violations += persisted_count + self.cumulative_transitions += len(transition_ids) + self.last_run_at = snapshot.snapshot_time + # Persist this cycle's snapshot_time for the NEXT cycle's transition + # check. Done AFTER notifications so a crash mid-emit doesn't + # advance the cursor and silence a real transition on retry. + self._write_prev_cycle_at(snapshot.snapshot_time) + + if persisted_count or snapshot.sources_unavailable: + logger.info( + "canary cycle: violations=%d transitions=%d unavailable=%s", + persisted_count, + len(transition_ids), + snapshot.sources_unavailable, + ) + + return CycleResult( + violations=results, + persisted_violation_ids=persisted_ids, + transition_invariant_ids=transition_ids, + previous_violation_at=previous_violation_at, + snapshot_time=snapshot.snapshot_time, + sources_unavailable=list(snapshot.sources_unavailable), + ) + + + # ------------------------------------------------------------------ + # Cycle-state side-table (Redis) + # ------------------------------------------------------------------ + + @staticmethod + def _redis(): + """Redis client shared with the slot service. Lazy import so this + module stays loadable in tests without a live Redis.""" + from services.slot_service import get_slot_service + + return get_slot_service().redis + + def _read_prev_cycle_at(self) -> Optional[str]: + """Snapshot_time of the prior cycle, or None on first ever run. + + Falls back to None on any Redis error — that turns the next + cycle's transition detection into "all violations are transitions" + for that single cycle, which is verbose but never misses a real + flip. We chose verbose-on-failure over silent-on-failure because + the canary's whole reason to exist is catching transitions. + """ + try: + return self._redis().get(REDIS_KEY_LAST_CYCLE) + except Exception: + logger.exception("canary: failed to read previous-cycle marker") + return None + + def _write_prev_cycle_at(self, snapshot_time: str) -> None: + """Advance the previous-cycle cursor to this cycle's snapshot_time.""" + try: + self._redis().set(REDIS_KEY_LAST_CYCLE, snapshot_time) + except Exception: + logger.exception("canary: failed to persist previous-cycle marker") + + @staticmethod + def _is_green_to_red( + invariant_id: str, + previous_latest: dict, + prev_cycle_at: Optional[str], + ) -> bool: + """Decide whether this cycle's violation is a fresh transition. + + Green→red iff the latest persisted violation for this invariant + predates the previous cycle's snapshot_time. Cases: + + - First-ever cycle (prev_cycle_at is None): every violation is a + transition. Operators expect to be told once when the harness + first starts seeing problems. + - First-ever violation (previous_latest entry absent): transition. + - Continuing-red (latest violation timestamp == prev_cycle_at): + continuation, no notification — the previous cycle saw it too. + - Red→green→red (latest violation predates prev_cycle_at): + transition — there was at least one clean cycle in between. + """ + prev = previous_latest.get(invariant_id) + if prev is None: + return True + if prev_cycle_at is None: + return True + # `<` so a same-snapshot replay from an immediate manual rerun + # is treated as a continuation rather than re-firing. + return prev["snapshot_time"] < prev_cycle_at + + +# Module-level singleton, mirrors cleanup_service. +canary_service = CanaryService() diff --git a/src/backend/services/slack_service.py b/src/backend/services/slack_service.py index 8235e0a39..bb57f3691 100644 --- a/src/backend/services/slack_service.py +++ b/src/backend/services/slack_service.py @@ -266,6 +266,54 @@ async def send_message( logger.error(f"Failed to send Slack message: {e}") return False, str(e) + async def post_webhook( + self, + webhook_url: str, + text: str, + blocks: Optional[list] = None, + timeout_seconds: float = 5.0, + ) -> Tuple[bool, Optional[str]]: + """Post to a Slack incoming webhook URL. + + Used by the canary alert sink (CANARY-001 Phase 1). Distinct from + `send_message` which uses an OAuth bot token: webhooks are + URL-as-credential, single-channel, fire-and-forget. Slack's + response is HTTP 200 with body "ok", or non-200 with an error + string — we surface the body verbatim so the admin Test button + can show "invalid_token" / "channel_not_found" verbatim. + + Short timeout (5s default) so a hung webhook can't stall the + canary cycle. Caller still wraps in its own try/except — this + already returns `(False, error)` on httpx errors. + """ + if not webhook_url: + return False, "webhook_url not configured" + payload: dict = {"text": text} + if blocks: + payload["blocks"] = blocks + try: + response = await self.client.post( + webhook_url, + json=payload, + timeout=timeout_seconds, + ) + if response.status_code != 200: + # Slack returns body like "invalid_token" / "no_service" for + # webhook errors; preserve verbatim for the admin Test UI. + body = (response.text or "").strip() or f"HTTP {response.status_code}" + logger.warning(f"Slack webhook post failed: {body}") + return False, body + return True, None + except httpx.TimeoutException: + return False, f"timeout after {timeout_seconds}s" + except Exception as e: + # Don't echo `str(e)` — httpx exceptions typically embed the + # request URL in their message, and for canary alerts that URL + # IS the credential. Log + return only the exception class. + err = type(e).__name__ + logger.error("Slack webhook post error: %s", err) + return False, f"{err}: webhook unreachable" + async def get_user_email( self, bot_token: str, diff --git a/src/backend/utils/helpers.py b/src/backend/utils/helpers.py index c8abc10e0..f0c7a8ff2 100644 --- a/src/backend/utils/helpers.py +++ b/src/backend/utils/helpers.py @@ -43,7 +43,7 @@ def utc_now_iso() -> str: return datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%S.%fZ') -def iso_cutoff(hours: int) -> str: +def iso_cutoff(hours: int = 0, *, minutes: int = 0) -> str: """ Compute a past-cutoff ISO timestamp in the same format as utc_now_iso(). @@ -58,16 +58,23 @@ def iso_cutoff(hours: int) -> str: "SELECT COUNT(*) FROM events WHERE occurred_at > ?", (iso_cutoff(2),) ) + cursor.execute( + "SELECT COUNT(*) FROM events WHERE occurred_at > ?", + (iso_cutoff(minutes=30),) + ) Args: hours: Hours in the past (positive). `iso_cutoff(0)` ≈ `utc_now_iso()`. + minutes: Additional minutes in the past (positive, keyword-only). + Combine with `hours` for arbitrary windows, or use alone for + sub-hour windows. Returns: ISO timestamp matching the format of utc_now_iso(). """ - return (datetime.now(timezone.utc) - timedelta(hours=hours)).strftime( - '%Y-%m-%dT%H:%M:%S.%fZ' - ) + return ( + datetime.now(timezone.utc) - timedelta(hours=hours, minutes=minutes) + ).strftime('%Y-%m-%dT%H:%M:%S.%fZ') def to_utc_iso(dt: datetime) -> str: diff --git a/tests/test_canary_invariants.py b/tests/test_canary_invariants.py new file mode 100644 index 000000000..9565295db --- /dev/null +++ b/tests/test_canary_invariants.py @@ -0,0 +1,1478 @@ +""" +Canary invariant harness unit tests (CANARY-001 / Issue #411 — Phase 1). + +Covers: +- CanaryOperations: insert (with validation), list/count with filters, + latest-per-invariant, stats aggregation +- Snapshot collector: agent_ownership read, per-agent execution + partitioning, orphan-ref scan, terminal-execution window +- Invariant library: S-01 (slot–row bijection), E-02 (phantom reversal + detection via state comparison), L-03 (delete-cascade orphan scan) +- End-to-end Option-1 smoke fixture: orphan agent_sharing row triggers + exactly one L-03 violation with correct severity + +Tests run with isolated temp SQLite + an in-memory fake Redis. No live +backend required. +""" + +import json +import os +import sqlite3 +import sys +import tempfile +import types +from collections import defaultdict +from datetime import datetime +from typing import Any, Dict, List + +import pytest + +# Add backend to path for direct imports. +_backend_path = os.path.abspath( + os.path.join(os.path.dirname(__file__), "..", "src", "backend") +) +if _backend_path not in sys.path: + sys.path.insert(0, _backend_path) + +# Stub utils.helpers (the test harness shadows src/backend/utils otherwise). +from datetime import timedelta as _td + +if "utils.helpers" not in sys.modules: + _helpers = types.ModuleType("utils.helpers") + _helpers.utc_now = lambda: datetime.utcnow() + _helpers.utc_now_iso = lambda: datetime.utcnow().isoformat() + "Z" + _helpers.to_utc_iso = lambda v: str(v) + _helpers.parse_iso_timestamp = lambda s: datetime.fromisoformat(s.rstrip("Z")) + _helpers.iso_cutoff = lambda hours=0, minutes=0, seconds=0: ( + (datetime.utcnow() - _td(hours=hours, minutes=minutes, seconds=seconds)) + .isoformat() + "Z" + ) + sys.modules["utils.helpers"] = _helpers + + +# Stub `croniter` so importing `db.__init__` doesn't fail outside the +# backend container. The canary code path never calls into croniter. +if "croniter" not in sys.modules: + _croniter_mod = types.ModuleType("croniter") + _croniter_mod.croniter = type("croniter", (), {}) + sys.modules["croniter"] = _croniter_mod + + +# Stub `models.TaskExecutionStatus` so canary/snapshot.py can derive its +# terminal-status tuple from the canonical enum without dragging the +# real `models` module (and its pydantic / db_models dependencies) into +# unit-test imports. Mirrors the four terminal values from +# src/backend/models.py:TaskExecutionStatus. +if "models" not in sys.modules: + from enum import Enum as _Enum + + class _StubTaskExecutionStatus(str, _Enum): + SUCCESS = "success" + FAILED = "failed" + CANCELLED = "cancelled" + SKIPPED = "skipped" + # Non-terminal values still listed so tests that touch them + # match the real enum's surface area. + QUEUED = "queued" + RUNNING = "running" + PENDING_RETRY = "pending_retry" + + _models_mod = types.ModuleType("models") + _models_mod.TaskExecutionStatus = _StubTaskExecutionStatus + sys.modules["models"] = _models_mod + + +# --------------------------------------------------------------------------- +# Tiny in-memory Redis substitute — covers only the surface canary uses. +# --------------------------------------------------------------------------- + + +class FakeRedis: + """Minimal Redis stand-in for canary tests (ZSET + HASH + SCAN).""" + + def __init__(self): + self._zsets: Dict[str, Dict[str, float]] = defaultdict(dict) + self._hashes: Dict[str, Dict[str, str]] = defaultdict(dict) + self._strings: Dict[str, str] = {} + + # ZSET ------------------------------------------------------------------ + + def zadd(self, key: str, mapping: Dict[str, float]) -> int: + added = 0 + for member, score in mapping.items(): + if member not in self._zsets[key]: + added += 1 + self._zsets[key][member] = score + return added + + def zrange(self, key: str, start: int, end: int, withscores: bool = False): + items = sorted(self._zsets.get(key, {}).items(), key=lambda kv: kv[1]) + sliced = items[start:] if end == -1 else items[start : end + 1] + return list(sliced) if withscores else [m for m, _ in sliced] + + def zcard(self, key: str) -> int: + return len(self._zsets.get(key, {})) + + def zrem(self, key: str, member: str) -> int: + if member in self._zsets.get(key, {}): + del self._zsets[key][member] + return 1 + return 0 + + def zremrangebyscore(self, key: str, min_score, max_score) -> int: + # Accepts numerics or the strings "-inf" / "+inf" — same as redis-py. + def _coerce(v): + if isinstance(v, str): + if v in ("-inf", "inf", "+inf"): + return float(v) + return float(v) + + lo = _coerce(min_score) + hi = _coerce(max_score) + if key not in self._zsets: + return 0 + to_remove = [m for m, s in self._zsets[key].items() if lo <= s <= hi] + for m in to_remove: + del self._zsets[key][m] + return len(to_remove) + + def zrangebyscore(self, key: str, min_score, max_score) -> List[str]: + def _coerce(v): + if isinstance(v, str) and v in ("-inf", "inf", "+inf"): + return float(v) + return float(v) + + lo = _coerce(min_score) + hi = _coerce(max_score) + items = sorted( + (kv for kv in self._zsets.get(key, {}).items() if lo <= kv[1] <= hi), + key=lambda kv: kv[1], + ) + return [m for m, _ in items] + + # HASH ------------------------------------------------------------------ + + def hset(self, key: str, field: str = None, value: str = None, mapping: Dict[str, str] = None) -> int: + added = 0 + if mapping: + for k, v in mapping.items(): + if k not in self._hashes[key]: + added += 1 + self._hashes[key][k] = v + elif field is not None: + if field not in self._hashes[key]: + added = 1 + self._hashes[key][field] = value + return added + + def hget(self, key: str, field: str): + return self._hashes.get(key, {}).get(field) + + def hmget(self, key: str, *fields: str) -> List: + bucket = self._hashes.get(key, {}) + return [bucket.get(f) for f in fields] + + def hdel(self, key: str, *fields: str) -> int: + bucket = self._hashes.get(key) + if not bucket: + return 0 + removed = 0 + for f in fields: + if f in bucket: + del bucket[f] + removed += 1 + return removed + + def hkeys(self, key: str) -> List[str]: + return list(self._hashes.get(key, {}).keys()) + + def hlen(self, key: str) -> int: + return len(self._hashes.get(key, {})) + + def delete(self, key: str) -> int: + deleted = 0 + if key in self._zsets: + del self._zsets[key] + deleted += 1 + if key in self._hashes: + del self._hashes[key] + deleted += 1 + return deleted + + # SCAN ------------------------------------------------------------------ + + def scan(self, cursor: int = 0, match: str = "*", count: int = 100): + # No real cursoring — return everything once, then stop. + if cursor != 0: + return 0, [] + import fnmatch + + keys = list(self._zsets.keys()) + list(self._hashes.keys()) + matched = [k for k in keys if fnmatch.fnmatch(k, match)] + return 0, matched + + # STRING ---------------------------------------------------------------- + # Used by CanaryService for the previous-cycle snapshot_time cursor + # (REDIS_KEY_LAST_CYCLE) — see services/canary_service.py. + + def get(self, key: str): + return self._strings.get(key) + + def set(self, key: str, value: str) -> bool: + self._strings[key] = str(value) + return True + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture +def canary_db(monkeypatch): + """Temp SQLite with the tables canary touches; patch db.connection.""" + db_file = tempfile.NamedTemporaryFile(suffix="_canary_test.db", delete=False) + db_file.close() + db_path = db_file.name + + conn = sqlite3.connect(db_path) + conn.row_factory = sqlite3.Row + cur = conn.cursor() + cur.executescript( + """ + CREATE TABLE canary_violations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + invariant_id TEXT NOT NULL, + tier TEXT NOT NULL, + severity TEXT NOT NULL, + snapshot_time TEXT NOT NULL, + observed_state TEXT NOT NULL, + signal_query TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + CREATE TABLE agent_ownership ( + agent_name TEXT PRIMARY KEY, + owner_id TEXT NOT NULL, + is_system INTEGER DEFAULT 0, + max_parallel_tasks INTEGER DEFAULT 3, + execution_timeout_seconds INTEGER DEFAULT 900 + ); + CREATE TABLE schedule_executions ( + id TEXT PRIMARY KEY, + schedule_id TEXT, + agent_name TEXT NOT NULL, + status TEXT NOT NULL, + started_at TEXT NOT NULL, + completed_at TEXT, + message TEXT NOT NULL DEFAULT '', + triggered_by TEXT NOT NULL DEFAULT 'test' + ); + CREATE TABLE agent_sharing ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + agent_name TEXT NOT NULL, + shared_with_email TEXT NOT NULL, + shared_by_id TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + CREATE TABLE agent_schedules ( + id TEXT PRIMARY KEY, + agent_name TEXT NOT NULL, + name TEXT NOT NULL DEFAULT '', + cron_expression TEXT NOT NULL DEFAULT '', + message TEXT NOT NULL DEFAULT '', + enabled INTEGER DEFAULT 1, + owner_id INTEGER NOT NULL DEFAULT 0 + ); + CREATE TABLE chat_sessions ( + id TEXT PRIMARY KEY, + agent_name TEXT NOT NULL, + user_id INTEGER NOT NULL DEFAULT 0, + user_email TEXT NOT NULL DEFAULT '', + started_at TEXT NOT NULL DEFAULT '', + last_message_at TEXT NOT NULL DEFAULT '', + status TEXT DEFAULT 'active' + ); + CREATE TABLE agent_skills ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + agent_name TEXT NOT NULL, + skill_name TEXT NOT NULL + ); + CREATE TABLE agent_tags ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + agent_name TEXT NOT NULL, + tag TEXT NOT NULL + ); + CREATE TABLE agent_shared_files ( + id TEXT PRIMARY KEY, + agent_name TEXT NOT NULL, + filename TEXT NOT NULL, + stored_filename TEXT NOT NULL, + size_bytes INTEGER NOT NULL, + download_token TEXT UNIQUE NOT NULL, + created_by TEXT NOT NULL, + created_at TEXT NOT NULL + ); + CREATE TABLE agent_public_links ( + id TEXT PRIMARY KEY, + agent_name TEXT NOT NULL, + token TEXT NOT NULL + ); + CREATE TABLE operator_queue ( + id TEXT PRIMARY KEY, + agent_name TEXT NOT NULL, + type TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + priority TEXT NOT NULL DEFAULT 'medium', + title TEXT NOT NULL DEFAULT '', + question TEXT NOT NULL DEFAULT '', + created_at TEXT NOT NULL DEFAULT '' + ); + CREATE TABLE access_requests ( + id TEXT PRIMARY KEY, + agent_name TEXT NOT NULL, + email TEXT NOT NULL, + requested_at TEXT NOT NULL DEFAULT '', + status TEXT NOT NULL DEFAULT 'pending' + ); + CREATE TABLE mcp_api_keys ( + id TEXT PRIMARY KEY, + user_id TEXT NOT NULL, + agent_name TEXT, + scope TEXT NOT NULL, + key_hash TEXT UNIQUE NOT NULL, + key_prefix TEXT NOT NULL DEFAULT '', + name TEXT NOT NULL DEFAULT '', + created_at TEXT NOT NULL DEFAULT '' + ); + """ + ) + conn.commit() + conn.close() + + class _ConnCtx: + def __enter__(self): + self._conn = sqlite3.connect(db_path) + self._conn.row_factory = sqlite3.Row + return self._conn + + def __exit__(self, exc_type, exc_val, exc_tb): + try: + if exc_type is None: + self._conn.commit() + else: + self._conn.rollback() + finally: + self._conn.close() + + fake_db_connection = types.ModuleType("db.connection") + fake_db_connection.get_db_connection = lambda: _ConnCtx() + monkeypatch.setitem(sys.modules, "db.connection", fake_db_connection) + + yield db_path + os.unlink(db_path) + + +@pytest.fixture +def fake_redis(monkeypatch): + """Patch services.slot_service.get_slot_service to a fake.""" + redis_inst = FakeRedis() + + class _FakeSlotService: + slots_prefix = "agent:slots:" + + def __init__(self): + self.redis = redis_inst + + fake_module = types.ModuleType("services.slot_service") + fake_module.get_slot_service = lambda: _FakeSlotService() + monkeypatch.setitem(sys.modules, "services.slot_service", fake_module) + + return redis_inst + + +@pytest.fixture +def reload_canary(canary_db, fake_redis): + """Force reimport of canary modules so they bind to the patched modules.""" + for mod in list(sys.modules): + if mod.startswith("canary") or mod == "db.canary": + del sys.modules[mod] + import canary as canary_pkg # noqa: F401 + import db.canary as db_canary + + return {"canary": canary_pkg, "db_canary": db_canary, "redis": fake_redis} + + +# Override the package-wide autouse fixtures. +@pytest.fixture(scope="session") +def api_client(): + yield None + + +@pytest.fixture(autouse=True) +def cleanup_after_test(): + yield + + +# --------------------------------------------------------------------------- +# Helpers — populate fixtures +# --------------------------------------------------------------------------- + + +def _conn(path): + c = sqlite3.connect(path) + c.row_factory = sqlite3.Row + return c + + +def _add_agent(path, name, max_parallel=3, timeout=900, is_system=0): + c = _conn(path) + c.execute( + "INSERT INTO agent_ownership (agent_name, owner_id, is_system, max_parallel_tasks, execution_timeout_seconds) VALUES (?, ?, ?, ?, ?)", + (name, "test-owner", is_system, max_parallel, timeout), + ) + c.commit() + c.close() + + +def _add_execution(path, eid, agent_name, status, started_at=None, completed_at=None): + c = _conn(path) + c.execute( + "INSERT INTO schedule_executions (id, agent_name, status, started_at, completed_at) VALUES (?, ?, ?, ?, ?)", + (eid, agent_name, status, started_at or "2026-04-30T00:00:00Z", completed_at), + ) + c.commit() + c.close() + + +def _add_orphan_sharing(path, agent_name): + c = _conn(path) + c.execute( + "INSERT INTO agent_sharing (agent_name, shared_with_email, shared_by_id) VALUES (?, ?, ?)", + (agent_name, "ghost@example.com", "test-owner"), + ) + c.commit() + c.close() + + +# --------------------------------------------------------------------------- +# CanaryOperations tests +# --------------------------------------------------------------------------- + + +class TestCanaryOperations: + def test_insert_and_fetch(self, reload_canary): + ops = reload_canary["db_canary"].CanaryOperations() + rid = ops.insert_violation( + invariant_id="S-01", + tier="A", + severity="critical", + snapshot_time="2026-04-30T12:00:00Z", + observed_state={"agent": "a", "redis": 1}, + ) + assert rid > 0 + v = ops.get_violation(rid) + assert v["invariant_id"] == "S-01" + # observed_state is parsed back to dict + assert v["observed_state"]["agent"] == "a" + + def test_insert_validates_tier(self, reload_canary): + ops = reload_canary["db_canary"].CanaryOperations() + with pytest.raises(ValueError, match="invalid tier"): + ops.insert_violation("S-01", "X", "critical", "t", {}) + + def test_insert_validates_severity(self, reload_canary): + ops = reload_canary["db_canary"].CanaryOperations() + with pytest.raises(ValueError, match="invalid severity"): + ops.insert_violation("S-01", "A", "fatal", "t", {}) + + def test_filters_and_count(self, reload_canary): + ops = reload_canary["db_canary"].CanaryOperations() + ops.insert_violation("S-01", "A", "critical", "2026-04-30T12:00:00Z", {}) + ops.insert_violation("S-01", "A", "major", "2026-04-30T12:05:00Z", {}) + ops.insert_violation("E-02", "A", "critical", "2026-04-30T12:05:00Z", {}) + + assert ops.count_violations() == 3 + assert ops.count_violations(invariant_id="S-01") == 2 + assert ops.count_violations(severity="critical") == 2 + assert ( + ops.count_violations(start_time="2026-04-30T12:03:00Z") == 2 + ), "time-window filter must use lexicographic ISO-Z compare" + + def test_latest_per_invariant(self, reload_canary): + ops = reload_canary["db_canary"].CanaryOperations() + ops.insert_violation("S-01", "A", "critical", "2026-04-30T12:00:00Z", {}) + latest_s01 = ops.insert_violation( + "S-01", "A", "critical", "2026-04-30T12:05:00Z", {} + ) + latest_e02 = ops.insert_violation( + "E-02", "A", "critical", "2026-04-30T12:05:00Z", {} + ) + + latest = ops.get_latest_per_invariant() + assert latest["S-01"]["id"] == latest_s01 + assert latest["E-02"]["id"] == latest_e02 + + def test_stats(self, reload_canary): + ops = reload_canary["db_canary"].CanaryOperations() + ops.insert_violation("S-01", "A", "critical", "2026-04-30T12:00:00Z", {}) + ops.insert_violation("S-01", "A", "major", "2026-04-30T12:05:00Z", {}) + ops.insert_violation("L-03", "A", "critical", "2026-04-30T12:10:00Z", {}) + + stats = ops.stats_by_invariant() + assert stats["total"] == 3 + assert stats["by_invariant"] == {"S-01": 2, "L-03": 1} + assert stats["by_severity"] == {"critical": 2, "major": 1} + + +# --------------------------------------------------------------------------- +# Snapshot collector tests +# --------------------------------------------------------------------------- + + +class TestSnapshotCollector: + def test_empty_platform(self, reload_canary): + snap = reload_canary["canary"].collect_snapshot() + assert snap.known_agents == set() + assert snap.agents == [] + assert snap.orphan_refs == [] + assert snap.sources_unavailable == [] + + def test_agents_partitioned_by_status(self, canary_db, reload_canary): + _add_agent(canary_db, "a1") + _add_execution(canary_db, "e-run-1", "a1", "running") + _add_execution(canary_db, "e-q-1", "a1", "queued") + _add_execution(canary_db, "e-done", "a1", "success") + + snap = reload_canary["canary"].collect_snapshot() + assert snap.known_agents == {"a1"} + assert len(snap.agents) == 1 + agent = snap.agents[0] + assert agent.running_exec_ids == {"e-run-1"} + assert agent.queued_exec_ids == {"e-q-1"} + + def test_redis_slots_collected(self, canary_db, reload_canary): + _add_agent(canary_db, "a1") + redis = reload_canary["redis"] + redis.zadd("agent:slots:a1", {"e-run-1": 1.0, "drain-a1-9": 2.0}) + + snap = reload_canary["canary"].collect_snapshot() + agent = snap.agents[0] + assert agent.slot_ids == {"e-run-1", "drain-a1-9"} + + def test_orphan_redis_slots_for_unknown_agent(self, canary_db, reload_canary): + _add_agent(canary_db, "real") + redis = reload_canary["redis"] + redis.zadd("agent:slots:ghost", {"e-1": 1.0}) + + snap = reload_canary["canary"].collect_snapshot() + assert snap.orphan_redis_slots == {"ghost": 1} + + def test_orphan_ref_scan(self, canary_db, reload_canary): + _add_agent(canary_db, "real") + _add_orphan_sharing(canary_db, "ghost-1") + _add_orphan_sharing(canary_db, "ghost-2") + + snap = reload_canary["canary"].collect_snapshot() + ghost_names = {r.referenced_agent_name for r in snap.orphan_refs} + assert ghost_names == {"ghost-1", "ghost-2"} + + def test_terminal_executions_window(self, canary_db, reload_canary): + _add_agent(canary_db, "a1") + # Recent terminal — included + _add_execution( + canary_db, "e-recent", "a1", "success", + completed_at=datetime.utcnow().isoformat(), + ) + # Old terminal — excluded by 30-min window + _add_execution( + canary_db, "e-old", "a1", "success", + completed_at="2025-01-01T00:00:00", + ) + snap = reload_canary["canary"].collect_snapshot() + assert "e-recent" in snap.terminal_exec_statuses + assert snap.terminal_exec_statuses["e-recent"] == "success" + assert "e-old" not in snap.terminal_exec_statuses + + +# --------------------------------------------------------------------------- +# Invariant: S-01 slot–row bijection +# --------------------------------------------------------------------------- + + +class TestInvariantS01: + def test_holds_when_sets_match(self, canary_db, reload_canary): + _add_agent(canary_db, "a1") + _add_execution(canary_db, "e1", "a1", "running") + reload_canary["redis"].zadd("agent:slots:a1", {"e1": 1.0}) + + snap = reload_canary["canary"].collect_snapshot() + from canary.invariants import s01_slot_row_bijection as s01 + + assert s01.check(snap) == [] + + def test_fires_when_redis_has_phantom(self, canary_db, reload_canary): + _add_agent(canary_db, "a1") + _add_execution(canary_db, "e1", "a1", "running") + # Phantom in Redis only. + reload_canary["redis"].zadd("agent:slots:a1", {"e1": 1.0, "phantom": 2.0}) + + snap = reload_canary["canary"].collect_snapshot() + from canary.invariants import s01_slot_row_bijection as s01 + + violations = s01.check(snap) + assert len(violations) == 1 + v = violations[0] + assert v.invariant_id == "S-01" + assert v.severity == "critical" + assert v.observed_state["in_redis_only"] == ["phantom"] + assert v.observed_state["in_sql_only"] == [] + assert v.observed_state["agent_name"] == "a1" + + def test_drain_sentinels_ignored(self, canary_db, reload_canary): + _add_agent(canary_db, "a1") + _add_execution(canary_db, "e1", "a1", "running") + reload_canary["redis"].zadd( + "agent:slots:a1", {"e1": 1.0, "drain-a1-12345": 2.0} + ) + snap = reload_canary["canary"].collect_snapshot() + from canary.invariants import s01_slot_row_bijection as s01 + + assert s01.check(snap) == [], "drain sentinels must not trip S-01" + + def test_fires_when_sql_orphan(self, canary_db, reload_canary): + _add_agent(canary_db, "a1") + _add_execution(canary_db, "e-running-no-slot", "a1", "running") + # No Redis slot. + + snap = reload_canary["canary"].collect_snapshot() + from canary.invariants import s01_slot_row_bijection as s01 + + violations = s01.check(snap) + assert len(violations) == 1 + assert violations[0].observed_state["in_sql_only"] == ["e-running-no-slot"] + + def test_skipped_when_redis_unavailable(self, reload_canary): + from canary.snapshot import Snapshot, AgentSnapshot + + snap = Snapshot( + snapshot_time="2026-04-30T12:00:00Z", + sources_unavailable=["redis: connection refused"], + agents=[ + AgentSnapshot( + name="a1", + is_system=False, + max_parallel=3, + execution_timeout_seconds=900, + slot_ids=set(), + running_exec_ids={"e1"}, + ) + ], + ) + from canary.invariants import s01_slot_row_bijection as s01 + + # Even with mismatch, must not fire if Redis was unreachable. + assert s01.check(snap) == [] + + def test_grace_suppresses_fresh_sql_orphan(self, canary_db, reload_canary): + """Start-path race: SQL row freshly written, ZADD not landed yet.""" + import time + _add_agent(canary_db, "a1") + fresh = datetime.utcfromtimestamp(time.time()).isoformat() + "Z" + _add_execution(canary_db, "e-fresh", "a1", "running", started_at=fresh) + + snap = reload_canary["canary"].collect_snapshot() + from canary.invariants import s01_slot_row_bijection as s01 + + assert s01.check(snap) == [] + + def test_grace_suppresses_fresh_redis_phantom(self, canary_db, reload_canary): + """Stop-path race: ZSET score within grace, SQL already terminal.""" + import time + _add_agent(canary_db, "a1") + reload_canary["redis"].zadd("agent:slots:a1", {"e-fresh": time.time()}) + + snap = reload_canary["canary"].collect_snapshot() + from canary.invariants import s01_slot_row_bijection as s01 + + assert s01.check(snap) == [] + + def test_grace_does_not_suppress_durable_mismatch(self, canary_db, reload_canary): + """Old `started_at` + old ZSET score → real leak, must fire.""" + _add_agent(canary_db, "a1") + _add_execution(canary_db, "e-stale-sql", "a1", "running") # default 2026-04-30 + reload_canary["redis"].zadd("agent:slots:a1", {"e-stale-redis": 1.0}) # 1970 + + snap = reload_canary["canary"].collect_snapshot() + from canary.invariants import s01_slot_row_bijection as s01 + + violations = s01.check(snap) + assert len(violations) == 1 + obs = violations[0].observed_state + assert obs["in_sql_only"] == ["e-stale-sql"] + assert obs["in_redis_only"] == ["e-stale-redis"] + + +# --------------------------------------------------------------------------- +# Invariant: E-02 phantom reversal +# --------------------------------------------------------------------------- + + +class TestInvariantE02: + def test_holds_on_first_cycle(self, canary_db, reload_canary): + _add_agent(canary_db, "a1") + _add_execution( + canary_db, "e-done", "a1", "success", + completed_at=datetime.utcnow().isoformat(), + ) + snap = reload_canary["canary"].collect_snapshot() + from canary.invariants import e02_no_phantom_reversal as e02 + + assert e02.check(snap) == [] + + def test_fires_on_terminal_to_running_reversal(self, canary_db, reload_canary): + _add_agent(canary_db, "a1") + # Cycle 1: e-done is terminal. + _add_execution( + canary_db, "e-done", "a1", "success", + completed_at=datetime.utcnow().isoformat(), + ) + snap1 = reload_canary["canary"].collect_snapshot() + from canary.invariants import e02_no_phantom_reversal as e02 + + # First call seeds the side-table with terminal ids. + e02.check(snap1) + + # Simulate a phantom reversal: same id now appears as running. + c = _conn(canary_db) + c.execute( + "UPDATE schedule_executions SET status='running', completed_at=NULL WHERE id='e-done'" + ) + c.commit() + c.close() + + snap2 = reload_canary["canary"].collect_snapshot() + violations = e02.check(snap2) + assert len(violations) == 1 + v = violations[0] + assert v.invariant_id == "E-02" + assert v.observed_state["execution_id"] == "e-done" + assert v.observed_state["current_status"] == "running" + # Forensic value of the alert: the reversal report must carry the + # actual prior status (success / failed / cancelled / skipped), + # not the placeholder string "terminal" the early Phase 1 cut + # used to write into the side-table. The Slack renderer prints + # this verbatim — "terminal → running" is useless to on-call. + assert v.observed_state["previous_status"] == "success" + assert v.signal_query and "success" in v.signal_query + + def test_reversal_renders_real_prior_status_for_each_terminal_kind( + self, canary_db, reload_canary + ): + """The four terminal statuses round-trip through the side-table. + + Regression for the placeholder-string bug: the previous-cycle + side-table only carried the literal "terminal" string, so a + reversal of e.g. a `cancelled` row reported "terminal → running" + instead of "cancelled → running". Run all four through the + seed/reverse cycle and assert each comes back labelled correctly. + """ + from canary.invariants import e02_no_phantom_reversal as e02 + + # Seed cycle: one row per terminal status. + _add_agent(canary_db, "a1") + for eid, status in ( + ("e-success", "success"), + ("e-failed", "failed"), + ("e-cancelled", "cancelled"), + ("e-skipped", "skipped"), + ): + _add_execution( + canary_db, eid, "a1", status, + completed_at=datetime.utcnow().isoformat(), + ) + snap1 = reload_canary["canary"].collect_snapshot() + e02.check(snap1) + + # Reversal: flip them all to running. + c = _conn(canary_db) + c.execute( + "UPDATE schedule_executions SET status='running', completed_at=NULL" + ) + c.commit() + c.close() + + snap2 = reload_canary["canary"].collect_snapshot() + violations = e02.check(snap2) + prev_by_eid = { + v.observed_state["execution_id"]: v.observed_state["previous_status"] + for v in violations + } + assert prev_by_eid == { + "e-success": "success", + "e-failed": "failed", + "e-cancelled": "cancelled", + "e-skipped": "skipped", + } + + def test_side_table_trims_by_age_not_by_hard_reset( + self, canary_db, reload_canary, fake_redis + ): + """Aged-out ids are dropped; in-window ids survive. + + Regression for the pre-fix hard-reset trim: when the side-table + crossed a 5000-entry hash cap the entire key was DEL'd, leaving + a one-cycle E-02 blind spot. The fix uses a sorted set scored + by unix ts and trims via `ZREMRANGEBYSCORE`, so only entries + older than the retention window age out — never an in-window + terminal id. Verifies that property directly. + """ + from canary.invariants import e02_no_phantom_reversal as e02 + + # Seed: one stale entry (well past retention) and one fresh. + # Use scores < cutoff and > cutoff to test the boundary. + retention = e02.PREV_TERMINAL_RETENTION_SECONDS + import time as _time + now = _time.time() + fake_redis.zadd( + e02.REDIS_KEY_PREV_TERMINAL, + { + "stale-eid": now - retention - 60, # past cutoff + "fresh-eid": now - 30, # well inside window + }, + ) + + # Run one check; both pre-existing ids are non-running, so no + # violation, but the trim path should drop "stale-eid" and keep + # "fresh-eid". + _add_agent(canary_db, "a1") + snap = reload_canary["canary"].collect_snapshot() + e02.check(snap) + + survivors = set( + fake_redis.zrange(e02.REDIS_KEY_PREV_TERMINAL, 0, -1) + ) + assert "stale-eid" not in survivors, "aged-out id must be trimmed" + assert "fresh-eid" in survivors, ( + "in-window id must NOT be lost to a hard reset" + ) + + +# --------------------------------------------------------------------------- +# Invariant: L-03 delete cascades — primary smoke test (Option 1) +# --------------------------------------------------------------------------- + + +class TestInvariantL03: + def test_holds_with_no_orphans(self, canary_db, reload_canary): + _add_agent(canary_db, "a1") + snap = reload_canary["canary"].collect_snapshot() + from canary.invariants import l03_delete_cascades as l03 + + assert l03.check(snap) == [] + + def test_fires_on_orphan_agent_sharing_row(self, canary_db, reload_canary): + """Option-1 smoke fixture: insert one orphan row → exactly one L-03.""" + _add_agent(canary_db, "real-agent") + # Ghost agent has no agent_ownership row. + _add_orphan_sharing(canary_db, "ghost-canary-zzz") + + snap = reload_canary["canary"].collect_snapshot() + from canary.invariants import l03_delete_cascades as l03 + + violations = l03.check(snap) + assert len(violations) == 1, "one orphan agent → one violation report" + v = violations[0] + assert v.invariant_id == "L-03" + assert v.tier == "A" + # agent_sharing alone is non-active orchestration → major, not critical. + assert v.severity == "major" + assert v.observed_state["ghost_agent_name"] == "ghost-canary-zzz" + assert v.observed_state["orphan_count"] == 1 + assert "agent_sharing" in v.observed_state["tables_hit"] + + def test_critical_severity_for_orphan_running_execution( + self, canary_db, reload_canary + ): + _add_agent(canary_db, "real-agent") + # Direct INSERT of an execution row pointing at a ghost agent — + # this is the bug class #129 caught: agent deleted but a running + # execution row still references it. + c = _conn(canary_db) + c.execute( + "INSERT INTO schedule_executions (id, agent_name, status, started_at) " + "VALUES ('e-orphan', 'ghost', 'running', '2026-04-30T00:00:00Z')" + ) + c.commit() + c.close() + + snap = reload_canary["canary"].collect_snapshot() + from canary.invariants import l03_delete_cascades as l03 + + violations = l03.check(snap) + assert len(violations) == 1 + v = violations[0] + assert v.severity == "critical", "active-orchestration orphan is critical" + assert "schedule_executions" in v.observed_state["tables_hit"] + + def test_groups_multiple_orphan_rows_under_one_violation( + self, canary_db, reload_canary + ): + _add_agent(canary_db, "real-agent") + _add_orphan_sharing(canary_db, "ghost-1") + _add_orphan_sharing(canary_db, "ghost-1") # second sharing row, same ghost + _add_orphan_sharing(canary_db, "ghost-2") + + snap = reload_canary["canary"].collect_snapshot() + from canary.invariants import l03_delete_cascades as l03 + + violations = l03.check(snap) + # Two ghost agents → two violations regardless of multiple rows per ghost. + ghost_names = {v.observed_state["ghost_agent_name"] for v in violations} + assert ghost_names == {"ghost-1", "ghost-2"} + + # And the row count is captured in observed_state. + ghost1 = next(v for v in violations if v.observed_state["ghost_agent_name"] == "ghost-1") + assert ghost1.observed_state["orphan_count"] == 2 + + def test_redis_orphan_slot_alone_fires_critical(self, canary_db, reload_canary): + _add_agent(canary_db, "real-agent") + # Redis slot for ghost agent — no SQL orphan rows. + reload_canary["redis"].zadd("agent:slots:ghost-redis", {"e-1": 1.0}) + + snap = reload_canary["canary"].collect_snapshot() + from canary.invariants import l03_delete_cascades as l03 + + violations = l03.check(snap) + assert len(violations) == 1 + v = violations[0] + assert v.severity == "critical" + assert v.observed_state["redis_slot_count"] == 1 + assert "redis:agent:slots" in v.observed_state["tables_hit"] + + +# --------------------------------------------------------------------------- +# Registry / runner +# --------------------------------------------------------------------------- + + +class TestRunner: + def test_run_invariants_all(self, canary_db, reload_canary): + _add_agent(canary_db, "a1") + _add_orphan_sharing(canary_db, "ghost") + + snap = reload_canary["canary"].collect_snapshot() + results = reload_canary["canary"].run_invariants(snap) + + assert set(results.keys()) == {"S-01", "E-02", "L-03"} + assert results["S-01"] == [] + assert results["E-02"] == [] + assert len(results["L-03"]) == 1 + + def test_run_invariants_subset(self, canary_db, reload_canary): + _add_agent(canary_db, "a1") + _add_orphan_sharing(canary_db, "ghost") + + snap = reload_canary["canary"].collect_snapshot() + results = reload_canary["canary"].run_invariants(snap, ids=["L-03"]) + assert set(results.keys()) == {"L-03"} + + def test_unknown_id_silently_ignored_by_runner(self, canary_db, reload_canary): + snap = reload_canary["canary"].collect_snapshot() + results = reload_canary["canary"].run_invariants(snap, ids=["NOPE", "L-03"]) + assert "NOPE" not in results + assert "L-03" in results + + +# --------------------------------------------------------------------------- +# CanaryService.run_cycle orchestration +# --------------------------------------------------------------------------- +# +# These tests exercise the orchestrator that ties snapshot collection, +# invariant evaluation, persistence, and green→red transition detection +# together. The deterministic-library tests above cover individual parts; +# these cover the wiring — which is where the demo-driven bugs lived: +# +# - e7c11b2e: `_is_green_to_red` was firing on every continuing-red +# cycle. Fixed via a Redis previous-cycle cursor. +# - ef40cf98: `TERMINAL_EXECUTION_STATUSES` listed wrong strings +# ("completed"/"timeout") so E-02's Redis side-table never seeded +# against real-world `success` rows. +# +# Both bugs passed the unit suite and were caught only by hand-driven +# demo runs. This class is the regression net. + + +@pytest.fixture +def canary_service(canary_db, fake_redis, reload_canary, monkeypatch): + """Build a CanaryService bound to the test fixtures. + + Routes the two `db.*` calls canary_service makes through the real + `CanaryOperations` (already wired to the temp SQLite via + `canary_db`). The Slack alert path is observed via the + `slack_capture` fixture below — this fixture leaves it alone. + """ + db_canary = reload_canary["db_canary"] + canary_ops = db_canary.CanaryOperations() + + class _FakeDB: + def get_latest_canary_violation_per_invariant(self): + return canary_ops.get_latest_per_invariant() + + def insert_canary_violation(self, **kwargs): + return canary_ops.insert_violation(**kwargs) + + fake_database = types.ModuleType("database") + fake_database.db = _FakeDB() + monkeypatch.setitem(sys.modules, "database", fake_database) + + # Drop any cached canary_service so it picks up the stubs above. + sys.modules.pop("services.canary_service", None) + + from services.canary_service import CanaryService + + return { + "service": CanaryService(), + "canary_ops": canary_ops, + } + + +def _run(coro): + """Run a coroutine to completion in a fresh event loop.""" + import asyncio as _asyncio + return _asyncio.run(coro) + + +class TestCanaryService: + """End-to-end tests for `CanaryService.run_cycle()`.""" + + def test_first_cycle_violation_classifies_as_transition( + self, canary_db, canary_service + ): + """First cycle that sees a violation classifies it as a green→red flip.""" + _add_agent(canary_db, "real") + _add_orphan_sharing(canary_db, "ghost-1") # triggers L-03 + + svc = canary_service["service"] + result = _run(svc.run_cycle()) + + assert result.transition_invariant_ids == ["L-03"] + assert svc.cumulative_transitions == 1 + + def test_continuing_red_does_not_re_classify(self, canary_db, canary_service): + """Same orphan, three cycles → 3 violations persisted, 1 transition. + + Regression for e7c11b2e: transition detection was firing on every + continuing-red cycle. The fix uses a Redis previous-cycle cursor + so a continuously-red invariant is classified once, not every cycle. + """ + _add_agent(canary_db, "real") + _add_orphan_sharing(canary_db, "ghost-1") + + svc = canary_service["service"] + _run(svc.run_cycle()) + _run(svc.run_cycle()) + _run(svc.run_cycle()) + + # All three cycles still persist the violation — the forensic + # record is intact even when the transition counter stays flat. + ops = canary_service["canary_ops"] + assert ops.count_violations(invariant_id="L-03") == 3 + assert svc.cumulative_transitions == 1, ( + "continuing-red must not re-classify on every cycle" + ) + + def test_red_green_red_classifies_twice(self, canary_db, canary_service): + """red → green → red registers two transitions. + + A clean cycle in the middle "re-arms" the invariant; the next + violation is a fresh transition, not a continuation. + """ + _add_agent(canary_db, "real") + _add_orphan_sharing(canary_db, "ghost-1") + + svc = canary_service["service"] + + # Cycle 1: red. + _run(svc.run_cycle()) + assert svc.cumulative_transitions == 1 + + # Cycle 2: clean it up → green. + c = _conn(canary_db) + c.execute("DELETE FROM agent_sharing WHERE agent_name='ghost-1'") + c.commit() + c.close() + _run(svc.run_cycle()) + assert svc.cumulative_transitions == 1, "green cycle must not classify" + + # Cycle 3: re-introduce → red again. + _add_orphan_sharing(canary_db, "ghost-1") + _run(svc.run_cycle()) + + assert svc.cumulative_transitions == 2, ( + "red→green→red must register a fresh transition on the second red" + ) + + def test_terminal_status_set_seeds_e02_side_table( + self, canary_db, canary_service, fake_redis + ): + """Regression for ef40cf98 — the terminal-status-set typo. + + `TERMINAL_EXECUTION_STATUSES` previously listed + ("completed", "failed", "cancelled", "timeout"), but Trinity + actually writes ("success", "failed", "cancelled", "skipped"). + With the wrong list, a `success` row never made it into + `canary:e02:terminal_seen`, so a later reversal of the same id + would go undetected. This test fails against the pre-fix list. + """ + _add_agent(canary_db, "real") + _add_execution( + canary_db, + "e-real-success", + "real", + "success", + completed_at=datetime.utcnow().isoformat(), + ) + + _run(canary_service["service"].run_cycle()) + + terminal_seen = fake_redis.zrange("canary:e02:terminal_seen", 0, -1) + assert "e-real-success" in terminal_seen, ( + "'success' must be in TERMINAL_EXECUTION_STATUSES" + ) + # Parallel hash must carry the row's real terminal status so a + # later reversal renders "success → running", not the + # placeholder "terminal → running" that an earlier Phase 1 cut + # was emitting into Slack alerts. + assert ( + fake_redis.hget("canary:e02:terminal_status", "e-real-success") + == "success" + ) + + +# --------------------------------------------------------------------------- +# Slack alert sink (CANARY-001 Phase 2) +# --------------------------------------------------------------------------- +# +# These tests exercise the env-gated Slack webhook emit path. The pure +# message-building helpers (CanaryAlerts._build_slack_payload, +# CanaryAlerts._format_last_red) are tested without any fixtures — they're +# static/classmethods. The `CanaryAlerts.emit_transition` integration path +# piggybacks on the existing `canary_service` fixture and stubs the +# slack_service module so we can observe the outbound call without +# touching httpx. + + +class TestCanarySlackPayload: + """Pure rendering tests for the Slack payload builder. + + Takes the `canary_service` fixture for its side-effect — importing + `services.canary_service` reaches `from database import db` at module + top, which triggers production DB init unless `database` is stubbed. + The fixture already does that stubbing; we ignore its return value. + """ + + def test_format_last_red_first_red_when_none(self, canary_service): + from services.canary_alerts import CanaryAlerts + assert ( + CanaryAlerts._format_last_red(None, "2026-05-04T12:00:00Z") + == "first red for this invariant" + ) + + def test_format_last_red_seconds(self, canary_service): + from services.canary_alerts import CanaryAlerts + out = CanaryAlerts._format_last_red( + "2026-05-04T11:59:30Z", "2026-05-04T12:00:00Z" + ) + assert out == "last red 30s ago" + + def test_format_last_red_minutes(self, canary_service): + from services.canary_alerts import CanaryAlerts + out = CanaryAlerts._format_last_red( + "2026-05-04T11:55:00Z", "2026-05-04T12:00:00Z" + ) + assert out == "last red 5m ago" + + def test_format_last_red_hours(self, canary_service): + from services.canary_alerts import CanaryAlerts + out = CanaryAlerts._format_last_red( + "2026-05-04T10:00:00Z", "2026-05-04T12:30:00Z" + ) + assert out == "last red 2h ago" + + def test_format_last_red_falls_back_on_garbage(self, canary_service): + from services.canary_alerts import CanaryAlerts + out = CanaryAlerts._format_last_red("not-a-timestamp", "2026-05-04T12:00:00Z") + assert out == "first red for this invariant" + + def test_build_payload_severity_emoji(self, canary_service): + from services.canary_alerts import CanaryAlerts + from canary.snapshot import ViolationReport + + v = ViolationReport( + invariant_id="S-01", + tier="A", + severity="critical", + observed_state={"agent_name": "alpha"}, + ) + text, blocks = CanaryAlerts._build_slack_payload( + "S-01", [v], "2026-05-04T12:00:00Z", None, "critical", [42], + ) + assert text.startswith("🚨") + assert "S-01" in text + # Header block uses the same emoji + friendly name. + header = blocks[0] + assert header["type"] == "header" + assert "🚨" in header["text"]["text"] + assert "Slot–row bijection" in header["text"]["text"] + + def test_build_payload_includes_last_red_badge(self, canary_service): + from services.canary_alerts import CanaryAlerts + from canary.snapshot import ViolationReport + + v = ViolationReport( + invariant_id="L-03", + tier="A", + severity="major", + observed_state={"ghost_agent_name": "ghost-1"}, + ) + _, blocks = CanaryAlerts._build_slack_payload( + "L-03", + [v], + "2026-05-04T12:00:00Z", + "2026-05-04T11:55:00Z", + "major", + [21], + ) + # Context is the last block; assert by type, not index, so + # added/removed sections don't break this test. + ctx = next(b for b in blocks if b["type"] == "context") + assert "last red 5m ago" in ctx["elements"][0]["text"] + assert "violation #21" in ctx["elements"][0]["text"] + + def test_build_payload_l03_forensic_block(self, canary_service): + from services.canary_alerts import CanaryAlerts + from canary.snapshot import ViolationReport + + v = ViolationReport( + invariant_id="L-03", + tier="A", + severity="major", + observed_state={ + "ghost_agent_name": "ghost-1", + "tables_hit": ["agent_sharing", "agent_schedules"], + "sample_refs": [ + {"table": "agent_sharing", "column": "agent_name", "row_id": "5"}, + {"table": "agent_schedules", "column": "agent_name", "row_id": "9"}, + ], + }, + ) + _, blocks = CanaryAlerts._build_slack_payload( + "L-03", [v], "2026-05-04T12:00:00Z", None, "major", [21], + ) + sections = [b for b in blocks if b["type"] == "section"] + forensic_text = " ".join(s["text"]["text"] for s in sections) + assert "agent_sharing" in forensic_text + assert "agent_schedules" in forensic_text + assert "row `5`" in forensic_text + assert "row `9`" in forensic_text + + def test_build_payload_includes_runbook_hint(self, canary_service): + from services.canary_alerts import CanaryAlerts + from canary.snapshot import ViolationReport + + v = ViolationReport( + invariant_id="L-03", + tier="A", + severity="major", + observed_state={"ghost_agent_name": "ghost-1"}, + ) + _, blocks = CanaryAlerts._build_slack_payload( + "L-03", [v], "2026-05-04T12:00:00Z", None, "major", [21], + ) + all_text = " ".join( + b["text"]["text"] for b in blocks if b.get("text") + ) + assert "deleted" in all_text # runbook hint mentions delete handler + + def test_format_row_refs_variants(self, canary_service): + from services.canary_alerts import CanaryAlerts + assert CanaryAlerts._format_row_refs([]) is None + assert CanaryAlerts._format_row_refs([None]) is None + assert CanaryAlerts._format_row_refs([21]) == "violation #21" + assert ( + CanaryAlerts._format_row_refs([21, 22, 23]) + == "violations #21, #22, #23" + ) + assert ( + CanaryAlerts._format_row_refs([21, 22, 23, 24, 25]) + == "violations #21–#25 (5 total)" + ) + # Drops Nones (insert failures) before counting. + assert CanaryAlerts._format_row_refs([21, None, 23]) == "violations #21, #23" + + +@pytest.fixture +def slack_capture(monkeypatch): + """Replace services.slack_service.slack_service with a recorder. + + The lazy `from services.slack_service import slack_service` inside + `CanaryAlerts.emit_transition` resolves through `sys.modules`, so + seeding the module entry up-front captures every call without a + live httpx client. + """ + calls: List[Dict[str, Any]] = [] + return_value: Dict[str, Any] = {"value": (True, None)} + + class _Recorder: + async def post_webhook(self, webhook_url, text, blocks=None, timeout_seconds=5.0): + calls.append({ + "url": webhook_url, + "text": text, + "blocks": blocks, + "timeout": timeout_seconds, + }) + return return_value["value"] + + fake = types.ModuleType("services.slack_service") + fake.slack_service = _Recorder() + monkeypatch.setitem(sys.modules, "services.slack_service", fake) + + return {"calls": calls, "return_value": return_value} + + +class TestCanarySlackEmit: + """Integration tests for `CanaryAlerts.emit_transition` against a recorded sink.""" + + def test_no_webhook_url_skips_silently( + self, canary_db, canary_service, slack_capture, monkeypatch + ): + """No env var = no POST. Cycle still runs, violation still persists.""" + monkeypatch.delenv("CANARY_SLACK_WEBHOOK_URL", raising=False) + _add_agent(canary_db, "real") + _add_orphan_sharing(canary_db, "ghost-1") + + svc = canary_service["service"] + result = _run(svc.run_cycle()) + + assert result.transition_invariant_ids == ["L-03"] + assert slack_capture["calls"] == [], "no webhook URL must not POST" + # Violation still persisted. + ops = canary_service["canary_ops"] + assert ops.count_violations(invariant_id="L-03") == 1 + + def test_webhook_url_set_fires_one_post_per_transition( + self, canary_db, canary_service, slack_capture, monkeypatch + ): + """With env var set, exactly one webhook POST per transition.""" + monkeypatch.setenv( + "CANARY_SLACK_WEBHOOK_URL", + "https://hooks.slack.com/services/TEST/TEST/TEST", + ) + _add_agent(canary_db, "real") + _add_orphan_sharing(canary_db, "ghost-1") + + svc = canary_service["service"] + _run(svc.run_cycle()) + + assert len(slack_capture["calls"]) == 1 + call = slack_capture["calls"][0] + assert call["url"] == "https://hooks.slack.com/services/TEST/TEST/TEST" + assert "L-03" in call["text"] + # Block layout has grown beyond the original 3 — assert by type + # rather than count so future copy edits don't trip this test. + block_types = [b["type"] for b in call["blocks"]] + assert block_types[0] == "header" + assert block_types[-1] == "context" + assert "section" in block_types + + def test_continuing_red_does_not_re_post( + self, canary_db, canary_service, slack_capture, monkeypatch + ): + """Three cycles with the same red invariant = one webhook POST. + + Mirrors `test_continuing_red_does_not_re_classify` — green→red + gating runs upstream of the sink, so the sink also fires once. + """ + monkeypatch.setenv( + "CANARY_SLACK_WEBHOOK_URL", + "https://hooks.slack.com/services/TEST/TEST/TEST", + ) + _add_agent(canary_db, "real") + _add_orphan_sharing(canary_db, "ghost-1") + + svc = canary_service["service"] + _run(svc.run_cycle()) + _run(svc.run_cycle()) + _run(svc.run_cycle()) + + assert len(slack_capture["calls"]) == 1, ( + "continuing-red must POST once, not every cycle" + ) + + def test_webhook_failure_swallowed_cycle_continues( + self, canary_db, canary_service, slack_capture, monkeypatch + ): + """A failing webhook must not break cycle accounting. + + The row is already persisted before `CanaryAlerts.emit_transition` runs; + a hung Slack endpoint can't roll that back. We assert the + transition is still counted and the violation is still in the + DB even when the recorder returns a failure tuple. + """ + monkeypatch.setenv( + "CANARY_SLACK_WEBHOOK_URL", + "https://hooks.slack.com/services/TEST/TEST/TEST", + ) + slack_capture["return_value"]["value"] = (False, "invalid_token") + _add_agent(canary_db, "real") + _add_orphan_sharing(canary_db, "ghost-1") + + svc = canary_service["service"] + result = _run(svc.run_cycle()) + + assert result.transition_invariant_ids == ["L-03"] + assert svc.cumulative_transitions == 1 + ops = canary_service["canary_ops"] + assert ops.count_violations(invariant_id="L-03") == 1 + # Recorder still saw the call — failure happened on Slack's side. + assert len(slack_capture["calls"]) == 1 + + def test_previous_violation_at_threaded_into_payload( + self, canary_db, canary_service, slack_capture, monkeypatch + ): + """red→green→red: second-red POST carries the prior snapshot_time + so the alert can render "last red Xm ago". + """ + monkeypatch.setenv( + "CANARY_SLACK_WEBHOOK_URL", + "https://hooks.slack.com/services/TEST/TEST/TEST", + ) + _add_agent(canary_db, "real") + _add_orphan_sharing(canary_db, "ghost-1") + + svc = canary_service["service"] + + # Cycle 1: first-ever transition → "first red" badge. + _run(svc.run_cycle()) + first_ctx = next( + b for b in slack_capture["calls"][0]["blocks"] if b["type"] == "context" + )["elements"][0]["text"] + assert "first red" in first_ctx + + # Cycle 2: clean → green. + c = _conn(canary_db) + c.execute("DELETE FROM agent_sharing WHERE agent_name='ghost-1'") + c.commit() + c.close() + _run(svc.run_cycle()) + # Cycle 3: re-introduce → second transition. + _add_orphan_sharing(canary_db, "ghost-1") + _run(svc.run_cycle()) + + assert len(slack_capture["calls"]) == 2 + second_ctx = next( + b for b in slack_capture["calls"][1]["blocks"] if b["type"] == "context" + )["elements"][0]["text"] + assert "last red" in second_ctx, ( + "second transition must carry the prior snapshot_time" + ) diff --git a/tests/unit/test_iso_cutoff.py b/tests/unit/test_iso_cutoff.py index 445205151..ed40026a0 100644 --- a/tests/unit/test_iso_cutoff.py +++ b/tests/unit/test_iso_cutoff.py @@ -82,3 +82,58 @@ def test_very_old_cutoff_still_valid_format(self): assert cutoff[10] == "T" # And is comparable lexicographically with a fresh now assert cutoff < utc_now_iso() + + +class TestMinutesKwarg: + """`minutes=` keyword extends iso_cutoff to sub-hour windows. + + Added when wiring CANARY-001 E-02's 30-minute terminal-row window + onto iso_cutoff (snapshot._collect_terminal_executions). + """ + + def test_minutes_only_returns_sub_hour_cutoff(self): + cutoff = iso_cutoff(minutes=30) + # Same canonical format + assert len(cutoff) == 27 + assert cutoff.endswith("Z") + assert cutoff[10] == "T" + # ~30 minutes before now (±1s for test runtime) + dt = parse_iso_timestamp(cutoff) + delta = (utc_now() - dt).total_seconds() + assert 30 * 60 - 1 < delta < 30 * 60 + 1 + + def test_minutes_is_keyword_only(self): + # Positional second arg must NOT bind to minutes — it should + # raise TypeError. This pins the signature so future refactors + # don't accidentally make `iso_cutoff(2, 30)` mean "2h30m". + import pytest + with pytest.raises(TypeError): + iso_cutoff(2, 30) # type: ignore[misc] + + def test_hours_and_minutes_combine(self): + cutoff = iso_cutoff(1, minutes=15) + dt = parse_iso_timestamp(cutoff) + delta = (utc_now() - dt).total_seconds() + # 1h15m == 4500s + assert 4500 - 1 < delta < 4500 + 1 + + def test_minutes_zero_equals_hours_only(self): + # Generated within the same wall-clock millisecond — strings may + # differ in the last digit, but should be within 1s of each other. + a = iso_cutoff(2) + b = iso_cutoff(2, minutes=0) + da = parse_iso_timestamp(a) + db = parse_iso_timestamp(b) + assert abs((da - db).total_seconds()) < 1 + + def test_larger_minutes_means_earlier_timestamp(self): + # Lexicographic == chronological still holds for sub-hour windows + assert iso_cutoff(minutes=30) < iso_cutoff(minutes=1) + assert iso_cutoff(minutes=1) < iso_cutoff(minutes=0) + + def test_no_args_equals_now_within_tolerance(self): + # iso_cutoff() with no args is equivalent to iso_cutoff(0) + before = utc_now_iso() + cutoff = iso_cutoff() + after = utc_now_iso() + assert before <= cutoff <= after