From 31d90a024b0e13093b7c3b516e42e6a6f4b5b157 Mon Sep 17 00:00:00 2001 From: Lin Jia Date: Sat, 11 Jul 2026 00:25:53 -0700 Subject: [PATCH 01/23] feat(router): add escalation_router profile with judge-latched strong/weak routing --- .agents/skills/switchyard-lib-core/SKILL.md | 6 +- ...-lite-escalation-opus-deepseek-gemini.yaml | 32 ++ .../escalation_router_routing.md | 120 ++++ docs/routing_algorithms/overview.md | 7 + mkdocs.yml | 1 + switchyard/__init__.py | 4 + switchyard/cli/route_bundle.py | 169 +++++- .../escalation_judge_request_processor.py | 515 ++++++++++++++++++ switchyard/lib/profiles/__init__.py | 6 + .../lib/profiles/escalation_router_config.py | 111 ++++ .../escalation_router_profile_config.py | 125 +++++ ...test_escalation_judge_request_processor.py | 274 ++++++++++ tests/test_escalation_router_profile.py | 94 ++++ tests/test_route_bundle.py | 83 +++ 14 files changed, 1543 insertions(+), 4 deletions(-) create mode 100644 benchmark/routing-profiles/tb-lite-escalation-opus-deepseek-gemini.yaml create mode 100644 docs/routing_algorithms/escalation_router_routing.md create mode 100644 switchyard/lib/processors/escalation_judge_request_processor.py create mode 100644 switchyard/lib/profiles/escalation_router_config.py create mode 100644 switchyard/lib/profiles/escalation_router_profile_config.py create mode 100644 tests/test_escalation_judge_request_processor.py create mode 100644 tests/test_escalation_router_profile.py diff --git a/.agents/skills/switchyard-lib-core/SKILL.md b/.agents/skills/switchyard-lib-core/SKILL.md index 17f7409d..1a513e64 100644 --- a/.agents/skills/switchyard-lib-core/SKILL.md +++ b/.agents/skills/switchyard-lib-core/SKILL.md @@ -62,9 +62,9 @@ Profile-owned construction keeps the full behavior in one reviewable module: 5. Use `ProfileSwitchyard(config.build())` only when serving through the existing Python endpoint contract. -Random routing, deterministic routing, plan-execute, passthrough, no-op, -stage-router, latency-service, RouteLLM, and OSS-router profiles are the reference -set under `switchyard/lib/profiles/`. +Random routing, deterministic routing, escalation-router, plan-execute, +passthrough, no-op, stage-router, latency-service, RouteLLM, and OSS-router +profiles are the reference set under `switchyard/lib/profiles/`. ## Anti-Patterns diff --git a/benchmark/routing-profiles/tb-lite-escalation-opus-deepseek-gemini.yaml b/benchmark/routing-profiles/tb-lite-escalation-opus-deepseek-gemini.yaml new file mode 100644 index 00000000..3336ec2e --- /dev/null +++ b/benchmark/routing-profiles/tb-lite-escalation-opus-deepseek-gemini.yaml @@ -0,0 +1,32 @@ +routes: + switchyard: + type: escalation_router + enable_stats: true + fallback_target_on_evict: strong + # Distinguish k>1 repeated-task trials against one shared server: the + # session key includes the first 2 post-task messages, so trials diverge + # via early model responses instead of sharing an escalation latch. + # Requires nonzero sampling temperature; keep 0 outside benchmarks. + session_key_depth: 2 + strong: + model: anthropic/claude-opus-4.7 + api_key: ${OPENROUTER_API_KEY} + base_url: https://openrouter.ai/api/v1 + format: openai + timeout_secs: 600 + extra_body: + reasoning: + effort: high + weak: + model: deepseek/deepseek-v4-pro + api_key: ${OPENROUTER_API_KEY} + base_url: https://openrouter.ai/api/v1 + format: openai + timeout_secs: 600 + judge: + model: google/gemini-3.5-flash + api_key: ${OPENROUTER_API_KEY} + base_url: https://openrouter.ai/api/v1 + timeout_secs: 5 + min_turn: 3 + recent_turn_window: 14 diff --git a/docs/routing_algorithms/escalation_router_routing.md b/docs/routing_algorithms/escalation_router_routing.md new file mode 100644 index 00000000..2c3548fd --- /dev/null +++ b/docs/routing_algorithms/escalation_router_routing.md @@ -0,0 +1,120 @@ +# Escalation-Router Routing + +The escalation router starts every conversation on the cheap **weak** tier. An +LLM **judge** watches the trajectory each turn and, when the run shows a clear +pattern of trouble (repeated errors, loops, false progress, drift, giving up), +escalates the conversation to the **strong** tier — one-way for the rest of +the task. A new conversation starts weak again. + +Unlike [LLM Classifier Routing](llm_classifier_routing.md), which predicts +per-turn difficulty from request content, the escalation router judges whether +the work is *going well*. Hard-but-smooth runs stay on the weak tier; runs the +weak tier is fumbling get rescued by the strong tier. + +## How it works + +Per turn, the `EscalationJudgeRequestProcessor`: + +1. Checks the session-affinity latch. A pinned conversation routes strong with + no judge call — the latch is one-way per task. +2. Otherwise routes weak. From `judge.min_turn` on (default 3), it sends the + judge a condensed transcript: the system + first-user anchors (individually + capped so harness boilerplate cannot crowd out the evidence), the last + `judge.recent_turn_window` messages (head/tail-truncated), and a coverage + header stating how much history is not shown. +3. The judge returns `{"escalate": bool, "reason": ""}`. On + `escalate: true` the conversation is pinned to strong, effective the same + turn. The reason is logged and stamped into the request metadata + (`_escalation_verdict`) as the audit record. +4. Any judge failure — timeout, error, invalid JSON — fails open to the weak + tier and never pins. A judge outage costs quality risk, never money. + +The latch reuses the [session-affinity](sticky_routing.md) store: conversations +are keyed on the stable prefix (system prompt + first user message), so a new +task gets a fresh key and resets to weak automatically. The weak tier is never +pinned — "not pinned" means weak-and-still-watching. + +## Configuration + +The CLI accepts `type: escalation_router` in a `routes:` bundle loaded with +`--routing-profiles` (same path the `deterministic` router uses): + +```yaml +routes: + switchyard: + type: escalation_router + fallback_target_on_evict: strong + judge: + model: google/gemini-3.5-flash + api_key: ${OPENROUTER_API_KEY} + base_url: https://openrouter.ai/api/v1 + timeout_secs: 5.0 + min_turn: 3 # first turn the judge runs on + recent_turn_window: 14 # trailing messages shown to the judge + strong: + model: anthropic/claude-opus-4.7 + api_key: ${OPENROUTER_API_KEY} + base_url: https://openrouter.ai/api/v1 + weak: + model: deepseek/deepseek-v4-pro + api_key: ${OPENROUTER_API_KEY} + base_url: https://openrouter.ai/api/v1 +``` + +| Key | Default | Meaning | +|---|---|---| +| `judge.model` / `api_key` / `base_url` | required | Judge LLM target. Pick something small and fast — it sits on the request path of every pre-escalation turn. | +| `judge.timeout_secs` | `5.0` | Judge wall-clock ceiling; fails open to weak. | +| `judge.min_turn` | `3` | First conversation turn the judge runs on. Earlier turns have no trajectory to judge. | +| `judge.recent_turn_window` | `14` | Trailing messages shown to the judge. Loops longer than the window are invisible — widen before concluding the judge misses them. | +| `judge.prompt` | built-in | Judge system-prompt override. | +| `judge.max_request_chars` | `12000` | Cap on the judge transcript; oldest window messages are dropped first. | +| `fallback_target_on_evict` | required | `strong` or `weak`; context-window-eviction reroute target. | +| `session_key_depth` | `0` | See [Repeated-trial benchmarking](#repeated-trial-benchmarking) below. | +| `tier_timeout_s` | `600` | Default per-call timeout for strong/weak targets without their own `timeout_secs`. | +| `affinity_max_sessions` | `10000` | LRU capacity of the escalation latch. | + +In Python, build it from the exported config pair: + +```python +from switchyard import EscalationRouterConfig, EscalationRouterProfileConfig + +profile = EscalationRouterProfileConfig.from_config( + EscalationRouterConfig.model_validate({...}) +).build() +``` + +## Repeated-trial benchmarking (k>1) + +The session key is a content hash of the system prompt + first user message, +so **k repeated trials of the same task against one long-lived server share +one latch**: trial 1's escalation makes trials 2..k start on strong from turn +1, silently corrupting pass@k and cost numbers. Two ways to keep trials +independent: + +- **Run-level isolation (zero config):** run each trial set as a separate + `benchmark/run-baseline.sh` invocation. Each run gets a fresh Switchyard + container; latch state cannot survive between runs. +- **`session_key_depth: N` (within-run k>1):** extends the session key with + the first `N` post-first-user messages. With nonzero sampling temperature, + trials diverge in their early model responses and get distinct keys. The + prefix of a conversation never changes as it grows, so the key stays stable + within a trial; until the prefix is complete, affinity is untouched. + Caveats: useless at temperature 0 (identical trajectories still collide), + and mid-session history rewrites (context compaction) change the key and + silently drop the latch — keep `0` outside benchmarks. + +Also set Harbor `--max-retries 0` for escalation-router runs: a retry re-runs +the failed task against the same warm server, and failed attempts are exactly +the ones likely to have escalated, so the retry would warm-start on strong. + +## Known limits and v2 levers + +- **Blocking judge:** the judge call adds ~100–300 ms to each pre-escalation + turn. If benchmarks show latency pain, the designed successor is running the + judge concurrently with the weak call and applying the verdict next turn. +- **Window-bounded loop detection:** a repeat cycle longer than + `recent_turn_window` is invisible. The designed successor is a per-turn + trajectory digest distilled from the whole history. +- **No difficulty prediction:** this router rescues struggling runs; it does + not predict hard ones. Composing it with the LLM classifier is future work. diff --git a/docs/routing_algorithms/overview.md b/docs/routing_algorithms/overview.md index 3b80c732..2a6a3bb1 100644 --- a/docs/routing_algorithms/overview.md +++ b/docs/routing_algorithms/overview.md @@ -18,6 +18,7 @@ for configuration and tuning. | [Random Routing](random_routing.md) | You need a fixed strong/weak split for A/B tests, baselines, or cost experiments. | `random-routing` | | [LLM Classifier Routing](llm_classifier_routing.md) | Request content should decide whether a turn needs the weak or strong tier. | `llm-routing` | | [Stage-Router Routing](stage_router_routing.md) | Tool-result and agent-progress signals should route most turns without an extra classifier call. | `stage_router` | +| [Escalation-Router Routing](escalation_router_routing.md) | Start every task on the weak tier and let an LLM judge escalate to strong — one-way per task — when the run is in trouble. | `escalation_router` (`routes:` bundle) | [Session Affinity (Sticky Routing)](sticky_routing.md) is an opt-in feature of LLM classifier routing, not a standalone routing strategy. The classifier @@ -152,6 +153,12 @@ They are not shared across workers or restarts. See Random and stage-router routing do not expose session-affinity settings; they continue to make a routing decision for each request. +The escalation router uses the same affinity store with the opposite policy: +affinity is always on (the pin *is* the escalation latch), only the strong +tier is ever pinned, and the judge keeps running every turn until the latch +fires. See +[Escalation-Router Routing](escalation_router_routing.md). + !!! note "CLI schema availability" The CLI currently accepts these settings in a `deterministic` entry in a `routes:` bundle loaded with `--routing-profiles`. The Rust `llm-routing` diff --git a/mkdocs.yml b/mkdocs.yml index 8d4ffa84..537c1af6 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -32,6 +32,7 @@ nav: - LLM Classifier Routing: routing_algorithms/llm_classifier_routing.md - Sticky Routing: routing_algorithms/sticky_routing.md - Stage-Router Routing: routing_algorithms/stage_router_routing.md + - Escalation-Router Routing: routing_algorithms/escalation_router_routing.md - Operations: - Context-Window Handling: operations/context_window.md - Reference: diff --git a/switchyard/__init__.py b/switchyard/__init__.py index cb179043..6a0688ad 100644 --- a/switchyard/__init__.py +++ b/switchyard/__init__.py @@ -60,6 +60,8 @@ DeterministicRoutingConfig, DeterministicRoutingPresets, DeterministicRoutingProfileConfig, + EscalationRouterConfig, + EscalationRouterProfileConfig, LatencyServiceProfileConfig, NoopProfile, NoopProfileConfig, @@ -165,6 +167,8 @@ def __getattr__(name: str) -> Any: "DeterministicRoutingConfig", "DeterministicRoutingProfileConfig", "DeterministicRoutingPresets", + "EscalationRouterConfig", + "EscalationRouterProfileConfig", "LatencyServiceProfileConfig", "OSSRouterConfig", "OSSRouterProfileConfig", diff --git a/switchyard/cli/route_bundle.py b/switchyard/cli/route_bundle.py index c41a13a8..bb342e97 100644 --- a/switchyard/cli/route_bundle.py +++ b/switchyard/cli/route_bundle.py @@ -37,6 +37,7 @@ from switchyard.lib.processors.llm_classifier.presets import PROFILE_FACTORIES from switchyard.lib.profiles import ( DeterministicRoutingProfileConfig, + EscalationRouterProfileConfig, LatencyServiceProfileConfig, PlanExecuteProfileConfig, ProfileSwitchyard, @@ -44,6 +45,7 @@ StageRouterProfileConfig, ) from switchyard.lib.profiles.deterministic_routing_config import DeterministicRoutingConfig +from switchyard.lib.profiles.escalation_router_config import EscalationRouterConfig from switchyard.lib.profiles.plan_execute_config import PlanExecuteConfig from switchyard.lib.profiles.plan_execute_presets import PlanExecutePresets from switchyard.lib.profiles.random_routing import RandomRoutingConfig @@ -264,6 +266,31 @@ def llm_target_to_route_dict(target: LlmTarget) -> dict[str, Any]: "fallback_target_on_evict", }) ) +_ESCALATION_ROUTE_KEYS = ( + _ROUTE_METADATA_KEYS + | _TARGET_DEFAULT_ROUTE_KEYS + | frozenset({ + "judge", + "strong", + "weak", + "enable_stats", + "fallback_target_on_evict", + "tier_timeout_s", + "session_key_depth", + "affinity_max_sessions", + }) +) +_ESCALATION_JUDGE_KEYS = frozenset({ + "model", + "api_key", + "base_url", + "timeout", + "timeout_secs", + "min_turn", + "recent_turn_window", + "prompt", + "max_request_chars", +}) _DETERMINISTIC_CLASSIFIER_KEYS = frozenset({ "model", "api_key", @@ -298,6 +325,7 @@ def llm_target_to_route_dict(target: LlmTarget) -> dict[str, Any]: "noop": _NOOP_ROUTE_KEYS, "passthrough": _PASSTHROUGH_ROUTE_KEYS, "deterministic": _DETERMINISTIC_ROUTE_KEYS, + "escalation_router": _ESCALATION_ROUTE_KEYS, "stage_router": _STAGE_ROUTER_ROUTE_KEYS, "plan_execute": _PLAN_EXECUTE_ROUTE_KEYS, } @@ -309,6 +337,7 @@ def llm_target_to_route_dict(target: LlmTarget) -> dict[str, Any]: "passthrough": _PASSTHROUGH_SETTING_KEYS, "noop": frozenset(), "deterministic": _TARGET_DEFAULT_KEYS, + "escalation_router": _TARGET_DEFAULT_KEYS, "stage_router": _TARGET_DEFAULT_KEYS, "plan_execute": _TARGET_DEFAULT_KEYS, } @@ -548,7 +577,7 @@ def build_table_from_bundle( # at the route key AND hydrate each tier's catalog (`strong` + `weak`) # into the table as direct passthroughs — same client-facing # model-picker experience as random_routing's discovery path. - if route_type in ("stage_router", "deterministic"): + if route_type in ("stage_router", "deterministic", "escalation_router"): _merge_multi_target_discovery( table, model_id, @@ -836,6 +865,16 @@ def _build_switchyard_for_route( extra_response_processors=extra_response_processors, ) + if route_type == "escalation_router": + return _escalation_router_switchyard( + model_id, + route, + target_defaults=target_defaults, + stats=stats, + pre_routing_request_processors=pre_routing_request_processors, + extra_response_processors=extra_response_processors, + ) + if route_type == "stage_router": return _stage_router_switchyard( model_id, @@ -1008,6 +1047,132 @@ def _deterministic_switchyard( ) +def _escalation_router_switchyard( + model_id: str, + route: Mapping[str, object], + target_defaults: Mapping[str, object], + stats: StatsAccumulator, + pre_routing_request_processors: Sequence[Any], + extra_response_processors: Sequence[Any], +) -> ChainRuntime: + """Build the judge-latched escalation-routing chain for a route. + + Every conversation starts on ``weak``; an LLM judge watches the + trajectory each turn and, on a clear pattern of trouble, latches the + conversation to ``strong`` for the rest of the task. + + YAML schema:: + + type: escalation_router + judge: + model: google/gemini-3.5-flash + api_key: ${OPENROUTER_API_KEY} + base_url: https://openrouter.ai/api/v1 + timeout_secs: 5.0 + min_turn: 3 # first turn the judge runs on + recent_turn_window: 14 # trailing messages shown to the judge + strong: + model: anthropic/claude-opus-4.7 + api_key: ${OPENROUTER_API_KEY} + base_url: https://openrouter.ai/api/v1 + weak: + model: deepseek/deepseek-v4-pro + api_key: ${OPENROUTER_API_KEY} + base_url: https://openrouter.ai/api/v1 + fallback_target_on_evict: strong + session_key_depth: 0 # >0 only for k>1 benchmark trials + """ + judge_raw = route.get("judge") + if not isinstance(judge_raw, Mapping): + raise RouteBundleConfigError( + f"route {model_id!r}: type=escalation_router requires a `judge:` " + "mapping with model/api_key/base_url", + ) + judge = _classifier_mapping( + judge_raw, + target_defaults, + allowed_keys=_ESCALATION_JUDGE_KEYS, + where=f"{model_id}.judge", + ) + + strong = _target_value( + route.get("strong"), target_defaults, default_id="strong", where="strong", + ) + weak = _target_value( + route.get("weak"), target_defaults, default_id="weak", where="weak", + ) + + fallback_target_on_evict = _required_str( + route.get("fallback_target_on_evict"), + f"{model_id}.fallback_target_on_evict", + ) + valid_ids = {strong.id, weak.id} + if fallback_target_on_evict not in valid_ids: + raise RouteBundleConfigError( + f"route {model_id!r}: fallback_target_on_evict=" + f"{fallback_target_on_evict!r} must match one of {sorted(valid_ids)} " + f"(the configured strong/weak target ids)", + ) + + config_data: dict[str, object] = { + "strong": strong, + "weak": weak, + "judge": { + "id": "judge", + "model": _required_str( + judge.get("model"), f"{model_id}.judge.model" + ), + "api_key": _required_str( + judge.get("api_key"), f"{model_id}.judge.api_key" + ), + "base_url": _required_str( + judge.get("base_url"), f"{model_id}.judge.base_url" + ), + "timeout_secs": _optional_float( + judge.get("timeout_secs"), + default=5.0, + ), + }, + "fallback_target_on_evict": fallback_target_on_evict, + "judge_min_turn": _optional_int(judge.get("min_turn"), default=3), + "judge_recent_turn_window": _optional_int( + judge.get("recent_turn_window"), default=14, + ), + "judge_system_prompt": _optional_str(judge.get("prompt")), + "judge_timeout_s": _optional_float(judge.get("timeout_secs"), default=5.0), + "enable_stats": _optional_bool(route.get("enable_stats"), default=True), + } + if "max_request_chars" in judge: + config_data["judge_max_request_chars"] = _optional_int( + judge.get("max_request_chars"), default=12_000, + ) + if "tier_timeout_s" in route: + config_data["tier_timeout_s"] = _optional_float( + route.get("tier_timeout_s"), + default=None, + ) + if "session_key_depth" in route: + config_data["session_key_depth"] = _optional_int( + route.get("session_key_depth"), default=0 + ) + if "affinity_max_sessions" in route: + config_data["affinity_max_sessions"] = _optional_int( + route.get("affinity_max_sessions"), default=10_000 + ) + + config = EscalationRouterConfig.model_validate(config_data) + return ProfileSwitchyard( + EscalationRouterProfileConfig.from_config(config) + .build() + .with_runtime_components( + stats_accumulator=stats, + enable_stats=config.enable_stats, + pre_request_processors=pre_routing_request_processors, + response_processors=extra_response_processors, + ) + ) + + def _stage_router_switchyard( model_id: str, route: Mapping[str, object], @@ -1354,6 +1519,8 @@ def _route_type(model_id: str, route: Mapping[str, object]) -> str: "deterministic": "deterministic", "llm_classifier": "deterministic", "llm_classifier_routing": "deterministic", + "escalation": "escalation_router", + "escalation_router": "escalation_router", "stage_router": "stage_router", "stage_router_routing": "stage_router", "plan": "plan_execute", diff --git a/switchyard/lib/processors/escalation_judge_request_processor.py b/switchyard/lib/processors/escalation_judge_request_processor.py new file mode 100644 index 00000000..957dfa8f --- /dev/null +++ b/switchyard/lib/processors/escalation_judge_request_processor.py @@ -0,0 +1,515 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Trajectory judge for the escalation router: weak until in trouble, then strong.""" + +from __future__ import annotations + +import hashlib +import logging +from typing import Any, Literal + +from pydantic import BaseModel, ConfigDict, Field + +from switchyard.lib.backends.deterministic_routing_llm_backend import ( + CTX_DETERMINISTIC_ROUTING_TIER, +) +from switchyard.lib.conversation_turn import conversation_turn_number +from switchyard.lib.llm_client import OpenAILLMClient +from switchyard.lib.processors.llm_classifier.request_processor import ( + LLMClassifierClient, + OpenAIChatLLMClassifierClient, + _strip_markdown_fence, + _trim_messages, +) +from switchyard.lib.proxy_context import ProxyContext +from switchyard.lib.session_affinity import CTX_SESSION_KEY, SessionAffinity +from switchyard_rust.core import ChatRequest + +log = logging.getLogger(__name__) + +#: ``ProxyContext.metadata`` key holding the audit record of the judge's +#: decision for this turn: ``{"escalate", "reason", "turn", "source"}``. +CTX_ESCALATION_VERDICT = "_escalation_verdict" + +#: Tier labels stamped into ``CTX_DETERMINISTIC_ROUTING_TIER``; must match the +#: labels the profile registers on ``DeterministicRoutingLLMBackend``. +TIER_STRONG = "strong" +TIER_WEAK = "weak" + +ESCALATION_JUDGE_SYSTEM_PROMPT = """\ +You are an escalation judge inside an agentic coding router. The session +started on the EFFICIENT tier (a cheap but top-class 2026 model). Your +job is to detect when the run is genuinely in trouble so the router can +escalate the rest of the task to the STRONG tier (frontier, expensive). + +You see a condensed view of one session: the task framing (system prompt ++ first user message) and the most recent turns of activity (assistant +messages and tool results). Judge the *trajectory* — is the agent making +real progress toward the stated task — not the difficulty of the task +itself. Return exactly one JSON object: + +{"escalate": boolean, "reason": "one short sentence naming the pattern"} + +Escalation is one-way for the rest of the task and expensive. Escalate +only on a clear PATTERN of trouble, never on a single failed command. +When the evidence is thin or ambiguous, return {"escalate": false}. + +# Trouble patterns — escalate when you see these + +Repetition and loops (the most common way agent runs die): +- The same command or edit failing 2+ times with materially the same + error, especially with unrelated changes in between. +- Near-identical tool calls repeated, or the same files re-read, without + new information gained — including longer cycles (A -> B -> C -> A). +- Fighting the environment: repeatedly invoking a missing executable, + retrying installs that fail the same way, or trying variations of a + command the environment has already rejected, instead of adapting. + +False progress (looks like progress, is not): +- Declaring success or moving on while the latest visible evidence + (test output, exit code, error text) shows failure. +- Finishing without running the verification the task specifies, when + the task states how success is checked (e.g. "make the provided + tests pass") and running it was possible. +- A reproduction or test the agent wrote that passes trivially without + exercising the actual issue, then building on that false signal. +- The agent's stated reading of a tool result contradicts what the + result actually says (treating an error or empty output as success). + +Drift and dead ends: +- Recent activity no longer serves the task in the first user message + (e.g. polishing style while the required feature is unstarted). +- Violating an explicit task constraint (modifying files the task says + not to touch, changing the tests instead of the code under test). +- Editing or reasoning about code without ever having opened the files + the errors point to — acting on guessed file contents. +- Contradicting or re-deriving something already established earlier in + the session (forgetting its own findings). +- Many turns elapsed with nothing durable produced (no successful + writes, no passing checks) and no visible narrowing of the problem — + the run is on pace to exhaust its turn budget. + +Desperation: +- Giving up: declaring the task impossible, asking to stop, or drifting + into restating the problem instead of acting on it. +- Destructive flailing: rm -rf, wholesale reinstalls, chmod -R, or + reverting everything as a reaction to being stuck rather than a + reasoned step. + +# Expected friction — do NOT escalate on these + +Agentic coding is full of failures that are part of healthy work: +- A test written to fail first (TDD) or a bug being reproduced on + purpose. +- A compile, lint, or test error fixed or meaningfully acted on in the + immediately following turn. +- Exploration dead-ends early in a session (grep with no matches, + reading a file that turns out to be irrelevant) while the agent is + still orienting. +- A missing tool handled adaptively (tries `rg`, falls back to `grep`). +- A long-running command (build, install, test suite) that simply has + not finished, or the agent waiting on information it asked for. + +The distinguishing question: is each failure producing new information +that changes the next action? Failing forward is fine; failing in place +is trouble. + +# Worked examples (none drawn from any benchmark task set) + +* Turn 3; the agent ran the test suite, 4 tests fail, and it is now + reading the first failing test. -> {"escalate": false} — reproducing + failures is the job. +* The agent has run `pytest tests/test_api.py` 4 times with the same + ImportError, editing an unrelated config file between attempts. -> + {"escalate": true, "reason": "same ImportError 4 times while editing + unrelated files"} +* `conda` is not installed; the agent has tried `conda install` five + ways instead of using the `pip` that earlier output showed present. + -> {"escalate": true, "reason": "fighting missing executable instead + of adapting"} +* Task: "make the provided integration tests pass." Recent turns: + renaming variables and reformatting docstrings; tests not run in 8 + turns. -> {"escalate": true, "reason": "drifted to cosmetic edits, + verification abandoned"} +* The agent says "All tests pass, task complete" but the last visible + test output shows "2 failed, 11 passed". -> {"escalate": true, + "reason": "claims success contradicted by latest test output"} +* The agent wrote a reproduction script that exits 0 without invoking + the code path the issue describes, concluded "bug not reproducible", + and is wrapping up. -> {"escalate": true, "reason": "reproduction + never exercised the reported code path"} +* Two turns of edits, one failed build, then a fixed build and a + passing test. -> {"escalate": false} +* `npm install` has been running for one turn with no output yet. -> + {"escalate": false} — slow command, not a stall. + +Do not emit markdown, commentary, or chain-of-thought — only the JSON +object. +""" + + +class EscalationVerdict(BaseModel): + """Binary judge verdict: escalate to the strong tier or stay on weak.""" + + model_config = ConfigDict(frozen=True) + + escalate: bool + reason: str = "" + + +class EscalationJudgeConfig(BaseModel): + """Configuration for :class:`EscalationJudgeRequestProcessor`. + + The judge call is OpenAI-chat-compatible; tests inject any object + implementing the classifier's ``LLMClassifierClient`` protocol. + """ + + model_config = ConfigDict(frozen=True) + + model: str = Field(min_length=1) + api_key: str | None = None + base_url: str | None = None + timeout_s: float = Field(default=5.0, gt=0.0) + """Judge wall-clock ceiling. The judge sits on the request path of every + pre-escalation turn; a slow judge taxes the whole session, so keep this + tight — any failure or timeout fails open to the weak tier.""" + + max_completion_tokens: int = Field(default=128, ge=16) + system_prompt: str = Field(default=ESCALATION_JUDGE_SYSTEM_PROMPT, min_length=1) + structured_output_mode: Literal["json_schema", "json_object"] = "json_schema" + disable_reasoning: bool = True + extra_headers: dict[str, str] | None = None + + min_judge_turn: int = Field(default=3, ge=1) + """First conversation turn on which the judge runs. Earlier turns have no + trajectory to judge and always route weak.""" + + recent_turn_window: int = Field(default=14, ge=1) + """Trailing messages shown to the judge on top of the anchors. Wider than + the classifier's default because loop detection needs to see the repeats: + a cycle longer than the window is invisible.""" + + max_request_chars: int = Field(default=12_000, ge=1_000) + """Cap on the assembled judge transcript. When exceeded, the *oldest* + window messages are dropped first — for a trajectory judge the newest + evidence is strictly the most valuable.""" + + system_chars: int = Field(default=1_000, ge=100) + """Per-message cap for system/developer anchors. Coding-agent harnesses + (Claude Code) inject very large boilerplate system prompts with no + trajectory signal; without this cap they would crowd out the window.""" + + first_user_chars: int = Field(default=2_000, ge=100) + """Cap for the first user message — the task statement the judge needs + for drift detection, so it gets the most generous anchor budget.""" + + window_message_chars: int = Field(default=300, ge=50) + """Per-message cap inside the trailing window. Error signatures and + command shapes survive this easily; full file dumps do not need to.""" + + +class EscalationJudgeRequestProcessor: + """Route weak until an LLM judge says the run is in trouble, then latch strong. + + Per-turn policy (the latch): a pinned conversation routes strong with no + judge call; an unpinned one routes weak, and from ``min_judge_turn`` on the + judge reads a condensed transcript. An ``escalate`` verdict pins the + conversation to the strong tier via :class:`SessionAffinity` — one-way for + the rest of the task. Weak is never pinned, and any judge failure fails + open to weak without pinning, so an outage costs quality risk, never money. + """ + + def __init__( + self, + config: EscalationJudgeConfig, + *, + affinity: SessionAffinity, + client: LLMClassifierClient | None = None, + session_key_depth: int = 0, + ) -> None: + if session_key_depth < 0: + raise ValueError("session_key_depth must be >= 0") + self._config = config + self._affinity = affinity + self._session_key_depth = session_key_depth + self._client = client or OpenAIChatLLMClassifierClient( + OpenAILLMClient( + api_key=config.api_key, + base_url=config.base_url, + timeout=config.timeout_s, + # Fail-open to weak is the retry surface; let the SDK fail fast. + max_retries=0, + ), + signal_schema=EscalationVerdict, + structured_output_mode=config.structured_output_mode, + max_completion_tokens=config.max_completion_tokens, + disable_reasoning=config.disable_reasoning, + extra_headers=config.extra_headers, + ) + + async def process(self, ctx: ProxyContext, request: ChatRequest) -> ChatRequest: + """Stamp the tier for this turn and maybe escalate. Leaves the request unchanged.""" + turn = conversation_turn_number(request) + # With a deep key, affinity is untouchable until the key prefix is + # complete — hashing a shorter prefix would produce a key that later + # turns of the same conversation no longer match. + affinity_ready = self._seed_session_key(ctx, request) + + if affinity_ready: + pinned = await self._affinity.pinned(ctx, request) + if pinned == TIER_STRONG: + ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] = TIER_STRONG + ctx.metadata[CTX_ESCALATION_VERDICT] = { + "escalate": True, + "reason": "", + "turn": turn, + "source": "pinned", + } + return request + + ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] = TIER_WEAK + if not affinity_ready or turn < self._config.min_judge_turn: + return request + + summary = _summarize_for_judge(request, turn=turn, config=self._config) + try: + completion = await self._client.classify( + model=self._config.model, + system_prompt=self._config.system_prompt, + request_summary=summary, + ) + verdict = EscalationVerdict.model_validate_json( + _strip_markdown_fence(completion.content), + ) + except Exception as exc: + # Fail open to the current (weak) tier and never pin: a judge + # outage must not silently burn strong-model tokens. + log.warning( + "EscalationJudgeRequestProcessor: judge failed; staying on weak tier: %s", + exc, + ) + ctx.metadata[CTX_ESCALATION_VERDICT] = { + "escalate": False, + "reason": f"fail_open: {str(exc)[:200]}", + "turn": turn, + "source": "fail_open", + } + return request + + ctx.metadata[CTX_ESCALATION_VERDICT] = { + "escalate": verdict.escalate, + "reason": verdict.reason, + "turn": turn, + "source": "judge", + } + if verdict.escalate: + log.info( + "EscalationJudgeRequestProcessor: escalating to strong tier " + "(turn %d): %s", + turn, + verdict.reason, + ) + await self._affinity.pin(ctx, request, TIER_STRONG) + ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] = TIER_STRONG + return request + + def _seed_session_key(self, ctx: ProxyContext, request: ChatRequest) -> bool: + """Seed the deep session key when configured; return whether affinity is usable. + + ``session_key_depth == 0`` keeps the default Rust key (system + first + user message) — always usable. Depth ``N`` extends the hash with the + first ``N`` post-first-user messages so repeated runs of the same task + diverge via early model responses; until those messages exist, the key + is not yet stable and affinity must not be read or written. + """ + if self._session_key_depth == 0: + return True + key = _deep_session_key(request, self._session_key_depth) + if key is None: + return False + ctx.metadata[CTX_SESSION_KEY] = key + return True + + +def _deep_session_key(request: ChatRequest, depth: int) -> str | None: + """Hash system + first user + first ``depth`` later messages, or ``None`` if short. + + Mirrors the anchor semantics of the Rust ``session_key_from_body`` and + extends the prefix by ``depth`` messages. The prefix of a conversation + never changes as it grows, so the key is stable from the moment it exists. + """ + body = getattr(request, "body", None) + if not isinstance(body, dict): + return None + hasher = hashlib.sha256() + top_system = body.get("system") + if isinstance(top_system, str | list): + hasher.update(_content_text(top_system).encode("utf-8")) + hasher.update(b"\x00") + + messages = body.get("messages") + if not isinstance(messages, list): + messages = body.get("input") + if not isinstance(messages, list): + return None + + first_user_seen = False + tail_taken = 0 + for m in messages: + if not isinstance(m, dict): + continue + role = m.get("role") + if role in ("system", "developer"): + hasher.update(_message_text(m).encode("utf-8")) + hasher.update(b"\x00") + elif not first_user_seen and role == "user": + first_user_seen = True + hasher.update(_message_text(m).encode("utf-8")) + hasher.update(b"\x00") + elif first_user_seen and tail_taken < depth: + tail_taken += 1 + hasher.update(_message_text(m).encode("utf-8")) + hasher.update(b"\x00") + if first_user_seen and tail_taken >= depth: + break + if not first_user_seen or tail_taken < depth: + return None + return hasher.hexdigest()[:16] + + +def _summarize_for_judge( + request: ChatRequest, + *, + turn: int, + config: EscalationJudgeConfig, +) -> str: + """Render a compact role-labeled transcript for the judge. + + Anchors (system + first user) are individually capped; the trailing window + keeps the last ``recent_turn_window`` messages at ``window_message_chars`` + each. A coverage header states how much history is *not* shown so the + judge can reason about pace. If the global cap still binds, the oldest + window lines are dropped first. + """ + body = getattr(request, "body", {}) + messages: list[Any] = [] + if isinstance(body, dict): + raw = body.get("messages") + if not isinstance(raw, list): + raw = body.get("input") + if isinstance(raw, list): + messages = raw + + trimmed = _trim_messages(messages, recent_turn_window=config.recent_turn_window) + + anchor_lines: list[str] = [] + window_lines: list[str] = [] + if isinstance(body, dict): + top_system = body.get("system") + if isinstance(top_system, str | list): + anchor_lines.append( + "[system] " + _truncate(_content_text(top_system), config.system_chars) + ) + first_user_seen = False + for m in trimmed: + if not isinstance(m, dict): + continue + role = m.get("role") + text = _message_text(m) + if role in ("system", "developer"): + anchor_lines.append(f"[{role}] " + _truncate(text, config.system_chars)) + elif role == "user" and not first_user_seen: + first_user_seen = True + anchor_lines.append("[user (task)] " + _truncate(text, config.first_user_chars)) + else: + label = role or m.get("type") or "message" + window_lines.append(f"[{label}] " + _truncate(text, config.window_message_chars)) + + n_shown = len(window_lines) + header = ( + f"Conversation turn {turn}; showing the last {n_shown} of " + f"{len(messages)} messages after the task framing." + ) + + def _assemble() -> str: + return "\n".join([header, *anchor_lines, *window_lines]) + + text = _assemble() + # Drop oldest window lines first: for a trajectory judge the newest + # evidence is strictly the most valuable. + while len(text) > config.max_request_chars and window_lines: + window_lines.pop(0) + text = _assemble() + if len(text) > config.max_request_chars: + text = text[: config.max_request_chars - 15] + "..." + return text + + +def _message_text(message: dict[str, Any]) -> str: + """Flatten a chat message (or Responses item) to plain text, tool calls included.""" + parts: list[str] = [] + content = message.get("content") + if isinstance(content, str | list): + text = _content_text(content) + if text: + parts.append(text) + # OpenAI chat tool calls: the command shapes the judge needs for loop detection. + tool_calls = message.get("tool_calls") + if isinstance(tool_calls, list): + for call in tool_calls: + if not isinstance(call, dict): + continue + fn = call.get("function") + if isinstance(fn, dict): + parts.append(f"tool_call {fn.get('name')}({fn.get('arguments', '')})") + # OpenAI Responses items carry their payloads in type-specific fields. + if message.get("type") == "function_call": + parts.append(f"tool_call {message.get('name')}({message.get('arguments', '')})") + output = message.get("output") + if isinstance(output, str | list): + text = _content_text(output) + if text: + parts.append(text) + return " ".join(parts) + + +def _content_text(content: str | list[Any]) -> str: + """Flatten string-or-content-block message content to text.""" + if isinstance(content, str): + return content + parts: list[str] = [] + for block in content: + if isinstance(block, str): + parts.append(block) + continue + if not isinstance(block, dict): + continue + text = block.get("text") + if isinstance(text, str): + parts.append(text) + continue + inner = block.get("content") + if isinstance(inner, str | list): + parts.append(_content_text(inner)) + return " ".join(p for p in parts if p) + + +def _truncate(text: str, limit: int) -> str: + """Keep the head and tail of ``text`` within ``limit`` chars.""" + if len(text) <= limit: + return text + marker = " ...[trimmed] " + keep = max(limit - len(marker), 20) + head = (keep * 2) // 3 + tail = keep - head + return text[:head] + marker + text[-tail:] + + +__all__ = [ + "CTX_ESCALATION_VERDICT", + "ESCALATION_JUDGE_SYSTEM_PROMPT", + "EscalationJudgeConfig", + "EscalationJudgeRequestProcessor", + "EscalationVerdict", +] diff --git a/switchyard/lib/profiles/__init__.py b/switchyard/lib/profiles/__init__.py index e71f5dca..421c2746 100644 --- a/switchyard/lib/profiles/__init__.py +++ b/switchyard/lib/profiles/__init__.py @@ -12,6 +12,10 @@ from switchyard.lib.profiles.deterministic_routing_profile_config import ( DeterministicRoutingProfileConfig, ) +from switchyard.lib.profiles.escalation_router_config import EscalationRouterConfig +from switchyard.lib.profiles.escalation_router_profile_config import ( + EscalationRouterProfileConfig, +) from switchyard.lib.profiles.header_routing import ( HeaderRoutingConfig, HeaderRoutingDecision, @@ -62,6 +66,8 @@ "DeterministicRoutingConfig", "DeterministicRoutingProfileConfig", "DeterministicRoutingPresets", + "EscalationRouterConfig", + "EscalationRouterProfileConfig", "HeaderRoutingConfig", "HeaderRoutingDecision", "HeaderRoutingProfile", diff --git a/switchyard/lib/profiles/escalation_router_config.py b/switchyard/lib/profiles/escalation_router_config.py new file mode 100644 index 00000000..1dbfd494 --- /dev/null +++ b/switchyard/lib/profiles/escalation_router_config.py @@ -0,0 +1,111 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Config model for the escalation-router profile (judge-latched strong/weak).""" + +from __future__ import annotations + +from pydantic import ( + BaseModel, + ConfigDict, + Field, + ValidationInfo, + field_validator, +) + +from switchyard.lib.backends.llm_target import LlmTarget, coerce_llm_target +from switchyard.lib.profiles.deterministic_routing_config import ( + DEFAULT_DETERMINISTIC_TIER_TIMEOUT_S, +) + + +class EscalationRouterConfig(BaseModel): + """Configuration for the escalation-router profile. + + Every conversation starts on the ``weak`` tier. An LLM ``judge`` watches + the trajectory each turn and, on a clear pattern of trouble, escalates the + conversation to the ``strong`` tier — one-way for the rest of the task + (session-affinity latch). A new conversation resets to weak. + + Attributes: + strong: Escalation target (frontier / expensive model). + weak: Starting tier (cheap / efficient model). + judge: Target for the judge LLM call. ``model`` / ``base_url`` / + ``api_key`` / ``timeout_secs`` are extracted at build time; + other target fields are ignored. + fallback_target_on_evict: Target id the chain executor reroutes to on + eviction (context-window overflow). Must match ``strong.id`` or + ``weak.id``; the judge target is not a routing candidate. + judge_min_turn: First conversation turn on which the judge runs. + judge_recent_turn_window: Trailing messages shown to the judge on top + of the system + first-user anchors. + judge_max_request_chars: Cap on the assembled judge transcript. + judge_system_prompt: Optional judge prompt override. ``None`` uses the + built-in prompt. + judge_timeout_s: Per-call judge timeout (seconds); the judge fails + open to the weak tier at timeout. + session_key_depth: ``0`` (default) keys conversations on system + + first user message (the shared Rust session key). ``N > 0`` + extends the key with the first ``N`` post-first-user messages so + repeated runs of the identical task (k>1 benchmark trials against + one server) diverge via early model responses instead of sharing + an escalation latch. Requires nonzero sampling temperature to be + effective; production traffic should keep ``0``. + tier_timeout_s: Default per-call timeout for strong/weak tier calls + when a target does not set its own ``timeout_secs``. + enable_stats: Wire stats processors and per-tier stats wrappers. + affinity_max_sessions: LRU capacity of the escalation latch store. + """ + + model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) + + strong: LlmTarget + weak: LlmTarget + judge: LlmTarget + fallback_target_on_evict: str + judge_min_turn: int = Field(default=3, ge=1) + judge_recent_turn_window: int = Field(default=14, ge=1) + judge_max_request_chars: int = Field(default=12_000, ge=1_000) + judge_system_prompt: str | None = Field(default=None, min_length=1) + judge_timeout_s: float = Field(default=5.0, gt=0.0) + session_key_depth: int = Field(default=0, ge=0) + tier_timeout_s: float | None = Field( + default=DEFAULT_DETERMINISTIC_TIER_TIMEOUT_S, + gt=0.0, + ) + enable_stats: bool = True + affinity_max_sessions: int = Field(default=10_000, gt=0) + + @field_validator("strong", "weak", "judge", mode="before") + @classmethod + def _coerce_target(cls, value: object, info: ValidationInfo) -> LlmTarget: + return coerce_llm_target(value, default_id=info.field_name or "target") + + @field_validator("strong", "weak", "judge") + @classmethod + def _target_model_non_empty(cls, tier: LlmTarget) -> LlmTarget: + if not tier.model: + raise ValueError("target.model must be a non-empty string") + return tier + + @field_validator("judge_system_prompt", mode="before") + @classmethod + def _blank_judge_prompt_is_unset(cls, value: object) -> object: + if isinstance(value, str) and not value.strip(): + return None + return value + + @field_validator("fallback_target_on_evict") + @classmethod + def _fallback_matches_existing_target(cls, value: str, info: ValidationInfo) -> str: + valid_ids = {info.data[key].id for key in ("strong", "weak") if key in info.data} + if value not in valid_ids: + raise ValueError( + f"fallback_target_on_evict={value!r} must match one of " + f"{sorted(valid_ids)} (the configured strong/weak target ids; " + f"the judge target is not a routing candidate)" + ) + return value + + +__all__ = ["EscalationRouterConfig"] diff --git a/switchyard/lib/profiles/escalation_router_profile_config.py b/switchyard/lib/profiles/escalation_router_profile_config.py new file mode 100644 index 00000000..40945806 --- /dev/null +++ b/switchyard/lib/profiles/escalation_router_profile_config.py @@ -0,0 +1,125 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Profile-owned escalation-router construction (judge-latched strong/weak).""" + +from __future__ import annotations + +from typing import Any, Self + +from switchyard.lib.profiles.chain import ComponentChainProfile +from switchyard.lib.profiles.escalation_router_config import EscalationRouterConfig +from switchyard.lib.profiles.table import profile_config + +_TIER_STRONG = "strong" +_TIER_WEAK = "weak" + + +@profile_config("escalation_router") +class EscalationRouterProfileConfig: + """Profile config wrapper for judge-latched escalation routing.""" + + config: EscalationRouterConfig + + @classmethod + def from_config(cls, config: EscalationRouterConfig) -> Self: + """Create a profile config from the validated parsing model.""" + return cls(config=config) + + def build(self) -> ComponentChainProfile: + """Build the escalation-router profile runtime.""" + from switchyard.lib.backends.anthropic_cache_breakpoint_backend import ( + maybe_wrap_anthropic_cache, + ) + from switchyard.lib.backends.deterministic_routing_llm_backend import ( + DeterministicRoutingLLMBackend, + ) + from switchyard.lib.backends.multi_llm_backend import ( + build_native_backend, + resolve_llm_target, + ) + from switchyard.lib.processors.escalation_judge_request_processor import ( + ESCALATION_JUDGE_SYSTEM_PROMPT, + EscalationJudgeConfig, + EscalationJudgeRequestProcessor, + ) + from switchyard.lib.processors.reasoning_effort_normalizer import ( + ReasoningEffortNormalizer, + ) + from switchyard.lib.profiles.deterministic_routing_profile_config import ( + _apply_deepseek_overrides, + _apply_default_tier_timeout, + ) + from switchyard.lib.session_affinity import SessionAffinity + + config = self.config + + # The affinity store IS the escalation latch, so it is always on. + # Warmup gating lives in the judge processor (min_judge_turn), not + # here — affinity warmup would delay a decided escalation from + # taking effect, not just from being decided. + affinity = SessionAffinity( + enabled=True, + max_sessions=config.affinity_max_sessions, + warmup_turns=0, + ) + + judge_config = EscalationJudgeConfig( + model=config.judge.model, + api_key=config.judge.api_key, + base_url=config.judge.base_url, + timeout_s=config.judge.endpoint.timeout_secs or 5.0, + system_prompt=config.judge_system_prompt or ESCALATION_JUDGE_SYSTEM_PROMPT, + min_judge_turn=config.judge_min_turn, + recent_turn_window=config.judge_recent_turn_window, + max_request_chars=config.judge_max_request_chars, + extra_headers=config.judge.extra_headers or None, + ) + request_processors: list[Any] = [ + ReasoningEffortNormalizer(), + EscalationJudgeRequestProcessor( + judge_config, + affinity=affinity, + session_key_depth=config.session_key_depth, + ), + ] + + # Resolve format='auto' once after tier defaults are applied so + # backend selection and Anthropic cache wrapping see the same + # concrete target (mirrors the deterministic profile). + strong_target = resolve_llm_target( + _apply_deepseek_overrides( + _apply_default_tier_timeout(config.strong, config.tier_timeout_s), + ), + ) + weak_target = resolve_llm_target( + _apply_deepseek_overrides( + _apply_default_tier_timeout(config.weak, config.tier_timeout_s), + ), + ) + strong_backend = maybe_wrap_anthropic_cache( + build_native_backend(strong_target), + strong_target, + ) + weak_backend = maybe_wrap_anthropic_cache( + build_native_backend(weak_target), + weak_target, + ) + + backend = DeterministicRoutingLLMBackend( + tiers={ + _TIER_STRONG: (strong_backend, strong_target.model), + _TIER_WEAK: (weak_backend, weak_target.model), + }, + # Weak is the resting state; the judge processor stamps a tier on + # every turn, so the default only covers malformed metadata. + default_tier=_TIER_WEAK, + ) + return ComponentChainProfile( + request_processors=request_processors, + backend=backend, + fallback_target_on_evict=config.fallback_target_on_evict, + ) + + +__all__ = ["EscalationRouterProfileConfig"] diff --git a/tests/test_escalation_judge_request_processor.py b/tests/test_escalation_judge_request_processor.py new file mode 100644 index 00000000..56a56779 --- /dev/null +++ b/tests/test_escalation_judge_request_processor.py @@ -0,0 +1,274 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the escalation-router judge processor and its latch policy.""" + +from __future__ import annotations + +import json +from typing import Any, cast + +from switchyard.lib.backends.deterministic_routing_llm_backend import ( + CTX_DETERMINISTIC_ROUTING_TIER, +) +from switchyard.lib.processors.escalation_judge_request_processor import ( + CTX_ESCALATION_VERDICT, + EscalationJudgeConfig, + EscalationJudgeRequestProcessor, +) +from switchyard.lib.processors.llm_classifier import ClassifierCompletion +from switchyard.lib.proxy_context import ProxyContext +from switchyard.lib.session_affinity import CTX_SESSION_KEY, SessionAffinity +from switchyard_rust.core import ChatRequest + + +class _FakeJudgeClient: + def __init__(self, response: str | Exception) -> None: + self.response = response + self.calls: list[dict[str, str]] = [] + + async def classify( + self, + *, + model: str, + system_prompt: str, + request_summary: str, + ) -> ClassifierCompletion: + self.calls.append({ + "model": model, + "system_prompt": system_prompt, + "request_summary": request_summary, + }) + if isinstance(self.response, Exception): + raise self.response + return ClassifierCompletion(content=self.response) + + +def _verdict_json(escalate: bool, reason: str = "same error repeating") -> str: + return json.dumps({"escalate": escalate, "reason": reason}) + + +def _request(n_assistant_turns: int = 3, **body_overrides: Any) -> ChatRequest: + """Chat request with enough prior assistant turns to pass the judge gate.""" + messages: list[dict[str, str]] = [ + {"role": "system", "content": "You are a coding agent."}, + {"role": "user", "content": "fix the failing tests"}, + ] + for i in range(n_assistant_turns): + messages.append({"role": "assistant", "content": f"attempt {i + 1}"}) + messages.append({"role": "user", "content": f"tool result {i + 1}: error"}) + body: dict[str, Any] = {"model": "client-model", "messages": messages} + body.update(body_overrides) + return ChatRequest.openai_chat(cast(Any, body)) + + +def _processor( + response: str | Exception, + *, + affinity: SessionAffinity | None = None, + session_key_depth: int = 0, + **config_overrides: Any, +) -> tuple[EscalationJudgeRequestProcessor, _FakeJudgeClient, SessionAffinity]: + affinity = affinity or SessionAffinity(enabled=True) + fake = _FakeJudgeClient(response) + processor = EscalationJudgeRequestProcessor( + EscalationJudgeConfig(model="judge-model", **config_overrides), + affinity=affinity, + client=fake, + session_key_depth=session_key_depth, + ) + return processor, fake, affinity + + +async def test_no_escalate_routes_weak_without_pin() -> None: + processor, fake, affinity = _processor(_verdict_json(False, "")) + ctx = ProxyContext() + req = _request() + + await processor.process(ctx, req) + + assert ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "weak" + assert len(fake.calls) == 1 + assert ctx.metadata[CTX_ESCALATION_VERDICT]["escalate"] is False + assert await affinity.pinned(ProxyContext(), _request()) is None + + +async def test_escalate_routes_strong_and_latches() -> None: + processor, fake, _ = _processor(_verdict_json(True)) + ctx = ProxyContext() + + await processor.process(ctx, _request()) + assert ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "strong" + assert ctx.metadata[CTX_ESCALATION_VERDICT]["reason"] == "same error repeating" + + # Later turn of the same conversation: pinned, judge not called again. + later_ctx = ProxyContext() + await processor.process(later_ctx, _request(n_assistant_turns=5)) + assert later_ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "strong" + assert later_ctx.metadata[CTX_ESCALATION_VERDICT]["source"] == "pinned" + assert len(fake.calls) == 1 + + +async def test_judge_skipped_before_min_turn() -> None: + processor, fake, _ = _processor(_verdict_json(True), min_judge_turn=3) + ctx = ProxyContext() + + await processor.process(ctx, _request(n_assistant_turns=1)) + + assert ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "weak" + assert fake.calls == [] + + +async def test_judge_failure_fails_open_to_weak_without_pin() -> None: + processor, fake, affinity = _processor(RuntimeError("judge down")) + ctx = ProxyContext() + + await processor.process(ctx, _request()) + + assert ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "weak" + assert ctx.metadata[CTX_ESCALATION_VERDICT]["source"] == "fail_open" + assert await affinity.pinned(ProxyContext(), _request()) is None + # A garbage verdict is the same failure surface. + processor, fake, affinity = _processor("not json at all") + ctx = ProxyContext() + await processor.process(ctx, _request()) + assert ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "weak" + assert await affinity.pinned(ProxyContext(), _request()) is None + + +async def test_markdown_fenced_verdict_is_parsed() -> None: + processor, _, _ = _processor("```json\n" + _verdict_json(True) + "\n```") + ctx = ProxyContext() + + await processor.process(ctx, _request()) + + assert ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "strong" + + +async def test_summary_has_header_anchors_and_truncated_window() -> None: + processor, fake, _ = _processor( + _verdict_json(False, ""), + window_message_chars=80, + first_user_chars=200, + ) + req = _request(n_assistant_turns=3, messages=[ + {"role": "system", "content": "harness boilerplate " * 200}, + {"role": "user", "content": "fix the failing tests in tests/test_api.py"}, + {"role": "assistant", "content": "running pytest " + "x" * 500}, + {"role": "user", "content": "tool result: ImportError " + "y" * 500}, + {"role": "assistant", "content": "attempt 2"}, + {"role": "assistant", "content": "attempt 3"}, + ]) + + await processor.process(ProxyContext(), req) + + summary = fake.calls[0]["request_summary"] + assert summary.startswith("Conversation turn ") + # Anchors survive: capped system + full task statement. + assert "harness boilerplate" in summary + assert "fix the failing tests in tests/test_api.py" in summary + # Oversized system anchor and window messages are head/tail truncated. + assert "...[trimmed]" in summary + # Window content is present with role labels. + assert "[assistant] attempt 3" in summary + for line in summary.splitlines()[1:]: + assert len(line) <= 2_000 + + +async def test_global_cap_drops_oldest_window_messages_first() -> None: + processor, fake, _ = _processor( + _verdict_json(False, ""), + max_request_chars=1_000, + window_message_chars=200, + ) + messages: list[dict[str, str]] = [ + {"role": "user", "content": "the task"}, + ] + for i in range(20): + messages.append({"role": "assistant", "content": f"attempt {i} " + "z" * 150}) + + await processor.process(ProxyContext(), _request(messages=messages)) + + summary = fake.calls[0]["request_summary"] + assert len(summary) <= 1_000 + # The newest evidence survives; the oldest window lines are dropped. + assert "attempt 19" in summary + assert "attempt 5" not in summary + + +async def test_deep_session_key_diverges_on_early_responses() -> None: + processor, _, _ = _processor(_verdict_json(False, ""), session_key_depth=2) + + def _trial(first_response: str) -> ChatRequest: + return _request(messages=[ + {"role": "system", "content": "You are a coding agent."}, + {"role": "user", "content": "identical task text"}, + {"role": "assistant", "content": first_response}, + {"role": "user", "content": "tool result"}, + {"role": "assistant", "content": "next"}, + ]) + + ctx_a, ctx_b = ProxyContext(), ProxyContext() + await processor.process(ctx_a, _trial("I'll start by reading the tests.")) + await processor.process(ctx_b, _trial("Let me look at the repo layout.")) + + assert ctx_a.metadata[CTX_SESSION_KEY] != ctx_b.metadata[CTX_SESSION_KEY] + + # The key is a prefix hash: the same conversation grown later keeps it. + grown = _trial("I'll start by reading the tests.") + grown.body["messages"].extend([ + {"role": "assistant", "content": "much later turn"}, + {"role": "user", "content": "more results"}, + ]) + ctx_grown = ProxyContext() + await processor.process(ctx_grown, grown) + assert ctx_grown.metadata[CTX_SESSION_KEY] == ctx_a.metadata[CTX_SESSION_KEY] + + +async def test_deep_key_incomplete_prefix_skips_affinity() -> None: + processor, fake, affinity = _processor( + _verdict_json(True), + session_key_depth=4, + min_judge_turn=1, + ) + ctx = ProxyContext() + # Only one post-first-user message: key prefix incomplete. + req = _request(messages=[ + {"role": "user", "content": "the task"}, + {"role": "assistant", "content": "first response"}, + ]) + + await processor.process(ctx, req) + + assert ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "weak" + assert CTX_SESSION_KEY not in ctx.metadata + assert fake.calls == [] + + +async def test_escalation_latch_isolated_by_deep_key() -> None: + """Trial 2 of an identical task starts weak when trajectories diverge.""" + affinity = SessionAffinity(enabled=True) + processor, fake, _ = _processor( + _verdict_json(True), affinity=affinity, session_key_depth=1, + ) + + def _trial(first_response: str, extra_turns: int = 2) -> ChatRequest: + messages = [ + {"role": "user", "content": "identical task text"}, + {"role": "assistant", "content": first_response}, + ] + for i in range(extra_turns): + messages.append({"role": "user", "content": f"tool result {i}"}) + messages.append({"role": "assistant", "content": f"attempt {i}"}) + return _request(messages=messages) + + ctx1 = ProxyContext() + await processor.process(ctx1, _trial("trial one path")) + assert ctx1.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "strong" + + # Same task text, different first response: fresh latch, judge runs. + fake.response = _verdict_json(False, "") + ctx2 = ProxyContext() + await processor.process(ctx2, _trial("trial two path")) + assert ctx2.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "weak" + assert len(fake.calls) == 2 diff --git a/tests/test_escalation_router_profile.py b/tests/test_escalation_router_profile.py new file mode 100644 index 00000000..b88478bb --- /dev/null +++ b/tests/test_escalation_router_profile.py @@ -0,0 +1,94 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the escalation-router config model and profile construction.""" + +from __future__ import annotations + +from typing import Any + +import pytest +from pydantic import ValidationError + +from switchyard.lib.backends.deterministic_routing_llm_backend import ( + DeterministicRoutingLLMBackend, +) +from switchyard.lib.backends.llm_target import LlmTarget +from switchyard.lib.processors.escalation_judge_request_processor import ( + ESCALATION_JUDGE_SYSTEM_PROMPT, + EscalationJudgeRequestProcessor, +) +from switchyard.lib.processors.reasoning_effort_normalizer import ReasoningEffortNormalizer +from switchyard.lib.profiles import EscalationRouterConfig, EscalationRouterProfileConfig + + +def _target(tier: str) -> LlmTarget: + return LlmTarget( + id=tier, + model=f"{tier}-model", + base_url=f"https://{tier}.invalid/v1", + api_key=f"sk-{tier}", + ) + + +def _config(**overrides: Any) -> EscalationRouterConfig: + data: dict[str, Any] = { + "strong": _target("strong"), + "weak": _target("weak"), + "judge": _target("judge"), + "fallback_target_on_evict": "strong", + } + data.update(overrides) + return EscalationRouterConfig.model_validate(data) + + +def test_fallback_must_match_a_tier_id() -> None: + with pytest.raises(ValidationError) as exc: + _config(fallback_target_on_evict="judge") + assert "fallback_target_on_evict" in str(exc.value) + + +def test_judge_target_is_required() -> None: + with pytest.raises(ValidationError): + EscalationRouterConfig.model_validate({ + "strong": _target("strong"), + "weak": _target("weak"), + "fallback_target_on_evict": "strong", + }) + + +def test_blank_judge_prompt_is_unset() -> None: + assert _config(judge_system_prompt=" ").judge_system_prompt is None + + +def test_build_composes_judge_chain() -> None: + profile = EscalationRouterProfileConfig.from_config(_config()).build() + + processors = profile._request_processors + assert [type(p) for p in processors] == [ + ReasoningEffortNormalizer, + EscalationJudgeRequestProcessor, + ] + backend = profile._backend + assert isinstance(backend, DeterministicRoutingLLMBackend) + assert set(backend._backends) == {"strong", "weak"} + assert backend._default_tier == "weak" + assert profile._fallback_target_on_evict == "strong" + + +def test_build_threads_judge_settings() -> None: + profile = EscalationRouterProfileConfig.from_config( + _config( + judge_min_turn=5, + judge_recent_turn_window=20, + session_key_depth=2, + ), + ).build() + + judge = profile._request_processors[1] + assert isinstance(judge, EscalationJudgeRequestProcessor) + assert judge._config.min_judge_turn == 5 + assert judge._config.recent_turn_window == 20 + assert judge._config.system_prompt == ESCALATION_JUDGE_SYSTEM_PROMPT + assert judge._session_key_depth == 2 + assert judge._affinity.enabled diff --git a/tests/test_route_bundle.py b/tests/test_route_bundle.py index 5544e97b..1ada519b 100644 --- a/tests/test_route_bundle.py +++ b/tests/test_route_bundle.py @@ -768,6 +768,89 @@ def _fake_serve( assert any(isinstance(component, IntakeResponseProcessor) for component in components) +class TestEscalationRouterRouteType: + """`type: escalation_router` wires the judge-latched chain via YAML.""" + + def _bundle(self) -> dict: + return { + "routes": { + "myrouter/escalation": { + "type": "escalation_router", + "fallback_target_on_evict": "strong", + "judge": { + "model": "google/gemini-3.5-flash", + "api_key": "sk-judge", + "base_url": "https://judge.invalid/v1", + "timeout_secs": 5.0, + "min_turn": 4, + "recent_turn_window": 10, + }, + "strong": { + "model": "openai/gpt-5.2", + "api_key": "sk-strong", + "base_url": "https://strong.invalid/v1", + }, + "weak": { + "model": "deepseek/deepseek-v4-pro", + "api_key": "sk-weak", + "base_url": "https://weak.invalid/v1", + }, + "session_key_depth": 2, + }, + }, + } + + def test_registers_route_key_and_tier_passthroughs(self): + from switchyard.cli.route_bundle import build_route_bundle_table + table = build_route_bundle_table(self._bundle()) + # Route key first, tiers as direct passthroughs; the judge tier is + # internal-only and never registered. + assert table.registered_models() == [ + "myrouter/escalation", + "openai/gpt-5.2", + "deepseek/deepseek-v4-pro", + ] + + def test_judge_settings_thread_into_processor(self): + from switchyard.cli.route_bundle import build_route_bundle_table + from switchyard.lib.processors.escalation_judge_request_processor import ( + EscalationJudgeRequestProcessor, + ) + table = build_route_bundle_table(self._bundle()) + switchyard = table.lookup_switchyard("myrouter/escalation") + judge = next( + c for c in switchyard.iter_components() + if isinstance(c, EscalationJudgeRequestProcessor) + ) + assert judge._config.model == "google/gemini-3.5-flash" + assert judge._config.min_judge_turn == 4 + assert judge._config.recent_turn_window == 10 + assert judge._config.timeout_s == 5.0 + assert judge._session_key_depth == 2 + + def test_rejects_missing_judge_block(self): + from switchyard.cli.route_bundle import ( + RouteBundleConfigError, + build_route_bundle_table, + ) + bundle = self._bundle() + del bundle["routes"]["myrouter/escalation"]["judge"] + with pytest.raises(RouteBundleConfigError) as exc: + build_route_bundle_table(bundle) + assert "judge" in str(exc.value) + + def test_rejects_fallback_not_matching_a_tier(self): + from switchyard.cli.route_bundle import ( + RouteBundleConfigError, + build_route_bundle_table, + ) + bundle = self._bundle() + bundle["routes"]["myrouter/escalation"]["fallback_target_on_evict"] = "judge" + with pytest.raises(RouteBundleConfigError) as exc: + build_route_bundle_table(bundle) + assert "fallback_target_on_evict" in str(exc.value) + + class TestDeterministicRouteType: """`type: deterministic` wires the LLM-classifier chain via YAML.""" From 411620e26261dfdf0edee835330cd8315f8aece3 Mon Sep 17 00:00:00 2001 From: Lin Jia Date: Mon, 13 Jul 2026 14:38:43 -0700 Subject: [PATCH 02/23] feat(router): ESC8 escalation-judge campaign through ESC7 + prompt generalization audit --- .agents/skills/switchyard-lib-core/SKILL.md | 1 + benchmark/compare_runs.py | 126 +++++++ ...-lite-escalation-opus-deepseek-nvidia.yaml | 70 ++++ ...tb-lite-single-deepseek-v4-pro-nvidia.yaml | 34 ++ .../tb-lite-single-opus-4-7-nvidia.yaml | 24 ++ .../tb-lite-single-opus-4-8-nvidia.yaml | 24 ++ .../switchyard-components/src/stats/cost.rs | 20 +- switchyard/cli/route_bundle.py | 19 + switchyard/lib/cost_estimator.py | 37 ++ .../escalation_judge_request_processor.py | 338 +++++++++++++++++- .../lib/profiles/escalation_router_config.py | 14 + .../escalation_router_profile_config.py | 18 +- ...test_escalation_judge_request_processor.py | 144 +++++++- tests/test_escalation_router_profile.py | 30 ++ 14 files changed, 872 insertions(+), 27 deletions(-) create mode 100644 benchmark/compare_runs.py create mode 100644 benchmark/routing-profiles/tb-lite-escalation-opus-deepseek-nvidia.yaml create mode 100644 benchmark/routing-profiles/tb-lite-single-deepseek-v4-pro-nvidia.yaml create mode 100644 benchmark/routing-profiles/tb-lite-single-opus-4-7-nvidia.yaml create mode 100644 benchmark/routing-profiles/tb-lite-single-opus-4-8-nvidia.yaml diff --git a/.agents/skills/switchyard-lib-core/SKILL.md b/.agents/skills/switchyard-lib-core/SKILL.md index 1a513e64..0436029b 100644 --- a/.agents/skills/switchyard-lib-core/SKILL.md +++ b/.agents/skills/switchyard-lib-core/SKILL.md @@ -76,6 +76,7 @@ profiles are the reference set under `switchyard/lib/profiles/`. | A new role-shaped abstraction invented outside the backend role. | Keep pre-call logic in request-side profile code or plain request components, call logic in `LLMBackend`, post-call logic in response-side profile code or plain response components, and final wire conversion in `TranslationEngine`. | | Adding an OpenRouter-specific backend or translator just to point at `https://openrouter.ai/api/v1`. | Route it through the OpenAI-compatible backend/profile and provider configuration. Keep provider-specific code for actual protocol differences. | | Making route YAML declare arbitrary Python processors. | Route YAML is deployment config for supported profile route types. Runtime-only hooks such as intake are injected by the caller through the route-table builder kwargs. | +| Hand-billing token costs from raw stats fields — charging `prompt_tokens` at the fresh-input rate and adding `cached_tokens` on top. | `prompt_tokens` in stats snapshots is **total** input: `uncached + cached_tokens + cache_creation_tokens` (Anthropic's three sibling usage counters are normalized into it). Fresh input = `prompt − cached − creation`. Use the built-in estimators — the snapshot's own `cost_estimate`, `estimate_model_cost()` in `cost_estimator.py`, or Rust `stats/cost.rs` — instead of ad-hoc math; ignoring the subtraction overstates cost ~5× on cache-heavy agentic traffic. | ## When Something Genuinely Doesn't Fit diff --git a/benchmark/compare_runs.py b/benchmark/compare_runs.py new file mode 100644 index 00000000..bfb3eb3a --- /dev/null +++ b/benchmark/compare_runs.py @@ -0,0 +1,126 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Compare accuracy and token cost across run-baseline.sh run directories. + +Usage:: + + uv run --no-sync python benchmark/compare_runs.py \ + label1=benchmark/tb_runs/ label2=benchmark/tb_runs/ ... + +Reads each run's Harbor ``result.json`` (accuracy) and Switchyard +``routing_stats_final.json`` (per-tier tokens, judge/classifier overhead, +cost estimate) and prints a side-by-side table plus per-task outcomes. +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path +from typing import Any + + +def _load_run(run_dir: Path) -> dict[str, Any]: + results = sorted(run_dir.glob("jobs/*/result.json")) + if not results: + raise FileNotFoundError(f"no jobs/*/result.json under {run_dir}") + harbor = json.loads(results[0].read_text()) + stats_path = run_dir / "routing_stats_final.json" + routing = json.loads(stats_path.read_text()) if stats_path.exists() else {} + + ev = next(iter(harbor["stats"]["evals"].values())) + rewards: dict[str, float] = {} + for value, trials in (ev.get("reward_stats", {}).get("reward", {}) or {}).items(): + for trial in trials: + # trial names look like -; strip the trial suffix. + rewards[trial.rsplit("-", 1)[0]] = float(value) + + return { + "harbor": harbor["stats"], + "eval": ev, + "rewards": rewards, + "routing": routing, + } + + +def _fmt_cost(routing: dict[str, Any]) -> tuple[float, float, float]: + ce = routing.get("cost_estimate", {}) or {} + return ( + ce.get("total_cost", 0.0), + ce.get("backend_cost", 0.0), + ce.get("classifier_cost", 0.0), + ) + + +def main(argv: list[str]) -> int: + runs: dict[str, dict[str, Any]] = {} + for arg in argv: + label, _, path = arg.partition("=") + if not path: + print(f"ERROR: expected label=path, got {arg!r}") + return 2 + runs[label] = _load_run(Path(path)) + + header = f"{'metric':<28}" + "".join(f"{label:>18}" for label in runs) + print(header) + print("-" * len(header)) + + def row(name: str, values: list[str]) -> None: + print(f"{name:<28}" + "".join(f"{v:>18}" for v in values)) + + row("completed trials", [str(r["harbor"]["n_completed_trials"]) for r in runs.values()]) + row("errored trials", [str(r["harbor"]["n_errored_trials"]) for r in runs.values()]) + row("mean reward", [f"{r['eval']['metrics'][0].get('mean', 0.0):.3f}" for r in runs.values()]) + row( + "solved / total", + [ + f"{sum(1 for v in r['rewards'].values() if v >= 1.0)}/{len(r['rewards'])}" + for r in runs.values() + ], + ) + row( + "input tokens", + [f"{r['harbor'].get('n_input_tokens', 0):,}" for r in runs.values()], + ) + row( + "output tokens", + [f"{r['harbor'].get('n_output_tokens', 0):,}" for r in runs.values()], + ) + for label_idx, name in ((0, "total cost $"), (1, "backend cost $"), (2, "judge/clf cost $")): + row(name, [f"{_fmt_cost(r['routing'])[label_idx]:.3f}" for r in runs.values()]) + + # Tier split (escalation / routed runs only). + row( + "tier split (calls)", + [ + " ".join( + f"{v.get('tier') or m.rsplit('/', 1)[-1]}:{v.get('calls', 0)}" + for m, v in (r["routing"].get("models", {}) or {}).items() + ) + or "-" + for r in runs.values() + ], + ) + clf_rows = [] + for r in runs.values(): + clf = r["routing"].get("classifier", {}) or {} + n, e = clf.get("total_requests", 0), clf.get("total_errors", 0) + clf_rows.append(f"{n} ({e} err)" if n or e else "-") + row("judge calls", clf_rows) + + # Per-task outcome grid. + tasks = sorted({t for r in runs.values() for t in r["rewards"]}) + print() + print(f"{'task':<44}" + "".join(f"{label:>14}" for label in runs)) + for task in tasks: + marks = [] + for r in runs.values(): + v = r["rewards"].get(task) + marks.append("-" if v is None else ("PASS" if v >= 1.0 else "fail")) + print(f"{task:<44}" + "".join(f"{m:>14}" for m in marks)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/benchmark/routing-profiles/tb-lite-escalation-opus-deepseek-nvidia.yaml b/benchmark/routing-profiles/tb-lite-escalation-opus-deepseek-nvidia.yaml new file mode 100644 index 00000000..87ff86e6 --- /dev/null +++ b/benchmark/routing-profiles/tb-lite-escalation-opus-deepseek-nvidia.yaml @@ -0,0 +1,70 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Escalation-router Opus/DeepSeek profile for Harbor Terminal-Bench Lite on +# the NVIDIA Inference Hub. Every conversation starts on DeepSeek V4 Pro; a +# DeepSeek V4 Flash judge watches the trajectory and latches the session to +# Claude Opus 4.8 on a clear pattern of trouble. +# +# All three tiers use https://inference-api.nvidia.com/v1, which requires a +# LiteLLM virtual key (sk-...), not an nvapi- key. Export the same key for +# all three vars (they are the ones run-baseline.sh forwards into the +# Switchyard container): +# export SWITCHYARD_STRONG_API_KEY=sk-... +# export SWITCHYARD_WEAK_API_KEY=sk-... +# export SWITCHYARD_CLASSIFIER_API_KEY=sk-... +# +# Run with: +# benchmark/run-baseline.sh \ +# --harbor-path benchmark/datasets/openthoughts-tblite-closed-book \ +# --routing-profiles benchmark/routing-profiles/tb-lite-escalation-opus-deepseek-nvidia.yaml \ +# --model switchyard + +routes: + switchyard: + type: escalation_router + enable_stats: true + fallback_target_on_evict: strong + # Distinguish k>1 repeated-task trials against one shared server: the + # session key includes the first 2 post-task messages, so trials diverge + # via early model responses instead of sharing an escalation latch. + # Requires nonzero sampling temperature; keep 0 outside benchmarks. + session_key_depth: 2 + strong: + model: aws/anthropic/bedrock-claude-opus-4-8 + api_key: ${SWITCHYARD_STRONG_API_KEY} + base_url: https://inference-api.nvidia.com/v1 + format: openai + timeout_secs: 600 + weak: + # The evals- batch-gateway variant (nvidia/deepseek-ai/evals-deepseek-v4-pro) + # avoids the regular gateway's 6-min timeout under high concurrency, but + # its upstream host is currently failing TLS verification; switch back + # once it probes clean. + model: nvidia/deepseek-ai/deepseek-v4-pro + api_key: ${SWITCHYARD_WEAK_API_KEY} + base_url: https://inference-api.nvidia.com/v1 + format: openai + timeout_secs: 600 + # Explicit extra_body overrides the DeepSeek default of + # enable_thinking: false; the X-Inference-Priority: batch header is + # still injected (headers merge independently of the body). + extra_body: + chat_template_kwargs: + enable_thinking: true + judge: + model: nvidia/deepseek-ai/deepseek-v4-flash + api_key: ${SWITCHYARD_CLASSIFIER_API_KEY} + base_url: https://inference-api.nvidia.com/v1 + # Hub Flash latency is ~4-8s on small judge calls (and the profile + # build pins DeepSeek judges to the X-Inference-Priority: batch + # gateway); the 5s default used with Gemini Flash would fail-open + # (stay weak) on most turns here. + timeout_secs: 30 + min_turn: 3 + # Require a second consecutive escalate verdict before latching: + # round-2 traces showed one-shot eager verdicts (single failed + # command at turn 4-9) causing regressions on tasks the weak tier + # solves; real loops persist across turns and still latch. + confirmations: 2 + recent_turn_window: 14 diff --git a/benchmark/routing-profiles/tb-lite-single-deepseek-v4-pro-nvidia.yaml b/benchmark/routing-profiles/tb-lite-single-deepseek-v4-pro-nvidia.yaml new file mode 100644 index 00000000..8a2e2bd7 --- /dev/null +++ b/benchmark/routing-profiles/tb-lite-single-deepseek-v4-pro-nvidia.yaml @@ -0,0 +1,34 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Serve-backed single-model DeepSeek V4 Pro profile on the NVIDIA Inference +# Hub — weak-only baseline for the escalation-router comparison. +# Requires a LiteLLM virtual key (sk-...): +# export SWITCHYARD_WEAK_API_KEY=sk-... +# Run with: +# benchmark/run-baseline.sh \ +# --harbor-path benchmark/datasets/openthoughts-tblite-closed-book \ +# --routing-profiles benchmark/routing-profiles/tb-lite-single-deepseek-v4-pro-nvidia.yaml \ +# --model tb-lite-single-deepseek-v4-pro-nvidia + +defaults: + api_key: ${SWITCHYARD_WEAK_API_KEY} + base_url: https://inference-api.nvidia.com/v1 + timeout_secs: 600.0 + +routes: + tb-lite-single-deepseek-v4-pro-nvidia: + type: model + target: + # The evals- batch-gateway variant avoids the regular gateway's 6-min + # timeout under high concurrency but is currently failing TLS + # verification upstream; switch back once it probes clean. + model: nvidia/deepseek-ai/deepseek-v4-pro + format: openai + extra_headers: + X-Inference-Priority: batch + # Thinking on, matching the escalation profile's weak tier so the + # weak-only baseline stays apples-to-apples. + extra_body: + chat_template_kwargs: + enable_thinking: true diff --git a/benchmark/routing-profiles/tb-lite-single-opus-4-7-nvidia.yaml b/benchmark/routing-profiles/tb-lite-single-opus-4-7-nvidia.yaml new file mode 100644 index 00000000..8831e419 --- /dev/null +++ b/benchmark/routing-profiles/tb-lite-single-opus-4-7-nvidia.yaml @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Serve-backed single-model Opus 4.7 profile on the NVIDIA Inference Hub — +# strong-only baseline for the escalation-router comparison. +# Requires a LiteLLM virtual key (sk-...): +# export SWITCHYARD_STRONG_API_KEY=sk-... +# Run with: +# benchmark/run-baseline.sh \ +# --harbor-path benchmark/datasets/openthoughts-tblite-closed-book \ +# --routing-profiles benchmark/routing-profiles/tb-lite-single-opus-4-7-nvidia.yaml \ +# --model tb-lite-single-opus-4-7-nvidia + +defaults: + api_key: ${SWITCHYARD_STRONG_API_KEY} + base_url: https://inference-api.nvidia.com/v1 + timeout_secs: 600.0 + +routes: + tb-lite-single-opus-4-7-nvidia: + type: model + target: + model: aws/anthropic/bedrock-claude-opus-4-7 + format: openai diff --git a/benchmark/routing-profiles/tb-lite-single-opus-4-8-nvidia.yaml b/benchmark/routing-profiles/tb-lite-single-opus-4-8-nvidia.yaml new file mode 100644 index 00000000..dd871169 --- /dev/null +++ b/benchmark/routing-profiles/tb-lite-single-opus-4-8-nvidia.yaml @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Serve-backed single-model Opus 4.8 profile on the NVIDIA Inference Hub — +# strong-only baseline for the escalation-router comparison. +# Requires a LiteLLM virtual key (sk-...): +# export SWITCHYARD_STRONG_API_KEY=sk-... +# Run with: +# benchmark/run-baseline.sh \ +# --harbor-path benchmark/datasets/openthoughts-tblite-closed-book \ +# --routing-profiles benchmark/routing-profiles/tb-lite-single-opus-4-8-nvidia.yaml \ +# --model tb-lite-single-opus-4-8-nvidia + +defaults: + api_key: ${SWITCHYARD_STRONG_API_KEY} + base_url: https://inference-api.nvidia.com/v1 + timeout_secs: 600.0 + +routes: + tb-lite-single-opus-4-8-nvidia: + type: model + target: + model: aws/anthropic/bedrock-claude-opus-4-8 + format: openai diff --git a/crates/switchyard-components/src/stats/cost.rs b/crates/switchyard-components/src/stats/cost.rs index 1e21a636..2b8d8928 100644 --- a/crates/switchyard-components/src/stats/cost.rs +++ b/crates/switchyard-components/src/stats/cost.rs @@ -189,11 +189,28 @@ fn raw_model_price(model: &str) -> Option { cached: 0.15, cache_write: 1.50, }, - "aws/anthropic/bedrock-claude-opus-4-7" + // Nemotron 3 Ultra (550B/55B MoE) — OpenRouter reference list + // price, June 2026. evals-N ids are parallel benchmarking-gateway + // deployments of the same model. + "nvidia/nvidia/nemotron-3-ultra" + | "openai/nvidia/nvidia/nemotron-3-ultra" + | "nvidia/nvidia/evals-nemotron-ultra" + | "nvidia/nvidia/evals-nemotron-ultra-2" + | "nvidia/nvidia/evals-nemotron-ultra-3" + | "nvidia/nvidia/evals-nemotron-ultra-4" => ModelPrice { + input: 0.50, + output: 2.20, + cached: 0.05, + cache_write: 0.50, + }, + "aws/anthropic/bedrock-claude-opus-4-8" + | "aws/anthropic/bedrock-claude-opus-4-7" | "aws/anthropic/bedrock-claude-opus-4-6" | "aws/anthropic/bedrock-claude-opus-4-5" + | "azure/anthropic/claude-opus-4-8" | "azure/anthropic/claude-opus-4-7" | "azure/anthropic/claude-opus-4-6" + | "claude-opus-4-8" | "claude-opus-4-7" | "claude-opus-4-6" | "claude-opus-4-5" => ModelPrice { @@ -204,6 +221,7 @@ fn raw_model_price(model: &str) -> Option { }, "aws/anthropic/bedrock-claude-sonnet-4-6" | "aws/anthropic/bedrock-claude-sonnet-4-5" + | "azure/anthropic/claude-sonnet-4-5" | "claude-sonnet-4-6" | "claude-sonnet-4-5" => ModelPrice { input: 3.00, diff --git a/switchyard/cli/route_bundle.py b/switchyard/cli/route_bundle.py index bb342e97..36705b9c 100644 --- a/switchyard/cli/route_bundle.py +++ b/switchyard/cli/route_bundle.py @@ -287,7 +287,11 @@ def llm_target_to_route_dict(target: LlmTarget) -> dict[str, Any]: "timeout", "timeout_secs", "min_turn", + "confirmations", + "confirmation_window", + "disable_reasoning", "recent_turn_window", + "window_message_chars", "prompt", "max_request_chars", }) @@ -1070,6 +1074,9 @@ def _escalation_router_switchyard( base_url: https://openrouter.ai/api/v1 timeout_secs: 5.0 min_turn: 3 # first turn the judge runs on + confirmations: 1 # consecutive escalate verdicts to latch + confirmation_window: 1 # >1 tolerates declines between fires + disable_reasoning: true # false lets a reasoning judge think recent_turn_window: 14 # trailing messages shown to the judge strong: model: anthropic/claude-opus-4.7 @@ -1135,9 +1142,21 @@ def _escalation_router_switchyard( }, "fallback_target_on_evict": fallback_target_on_evict, "judge_min_turn": _optional_int(judge.get("min_turn"), default=3), + "judge_escalate_confirmations": _optional_int( + judge.get("confirmations"), default=1, + ), + "judge_confirmation_window": _optional_int( + judge.get("confirmation_window"), default=1, + ), + "judge_disable_reasoning": _optional_bool( + judge.get("disable_reasoning"), default=True, + ), "judge_recent_turn_window": _optional_int( judge.get("recent_turn_window"), default=14, ), + "judge_window_message_chars": _optional_int( + judge.get("window_message_chars"), default=300, + ), "judge_system_prompt": _optional_str(judge.get("prompt")), "judge_timeout_s": _optional_float(judge.get("timeout_secs"), default=5.0), "enable_stats": _optional_bool(route.get("enable_stats"), default=True), diff --git a/switchyard/lib/cost_estimator.py b/switchyard/lib/cost_estimator.py index 7afd8c86..f4739749 100644 --- a/switchyard/lib/cost_estimator.py +++ b/switchyard/lib/cost_estimator.py @@ -81,6 +81,29 @@ class ModelPriceData: "openai/nvidia/nvidia/Nemotron-3-Nano-30B-A3B": ModelPriceData( input=0.05, output=0.20, cached=0.005, cache_write=0.05, ), + # Nemotron 3 Ultra — 550B/55B-active MoE, 1M ctx (released 2026-06-04). + # OpenRouter reference list price (June 2026): $0.50 / $2.20 per 1M + # tokens. The ``evals-nemotron-ultra-N`` ids are parallel deployments + # of the same model on the hub's benchmarking gateway (pair with + # ``X-Inference-Priority: batch``); same per-token pricing. + "nvidia/nvidia/nemotron-3-ultra": ModelPriceData( + input=0.50, output=2.20, cached=0.05, cache_write=0.50, + ), + "openai/nvidia/nvidia/nemotron-3-ultra": ModelPriceData( + input=0.50, output=2.20, cached=0.05, cache_write=0.50, + ), + "nvidia/nvidia/evals-nemotron-ultra": ModelPriceData( + input=0.50, output=2.20, cached=0.05, cache_write=0.50, + ), + "nvidia/nvidia/evals-nemotron-ultra-2": ModelPriceData( + input=0.50, output=2.20, cached=0.05, cache_write=0.50, + ), + "nvidia/nvidia/evals-nemotron-ultra-3": ModelPriceData( + input=0.50, output=2.20, cached=0.05, cache_write=0.50, + ), + "nvidia/nvidia/evals-nemotron-ultra-4": ModelPriceData( + input=0.50, output=2.20, cached=0.05, cache_write=0.50, + ), # Moonshot Kimi K2.6 — official platform.kimi.ai pricing (May 2026). # OpenAI wire format, so no cache_write premium (cache_write = input). "nvidia/moonshotai/kimi-k2.6": ModelPriceData( @@ -155,6 +178,11 @@ class ModelPriceData: ), # --- Anthropic Claude on AWS Bedrock (via NVIDIA Inference Hub) --- # 5-minute cache write = 1.25x input; cache read = 0.1x input. + # Opus 4.8 keeps the Opus-tier $5/$25 list price (docs.anthropic.com, + # verified 2026-07-11). + "aws/anthropic/bedrock-claude-opus-4-8": ModelPriceData( + input=5.00, output=25.00, cached=0.50, cache_write=6.25, + ), "aws/anthropic/bedrock-claude-opus-4-7": ModelPriceData( input=5.00, output=25.00, cached=0.50, cache_write=6.25, ), @@ -177,6 +205,12 @@ class ModelPriceData: "azure/anthropic/claude-opus-4-7": ModelPriceData( input=5.00, output=25.00, cached=0.50, cache_write=6.25, ), + "azure/anthropic/claude-opus-4-8": ModelPriceData( + input=5.00, output=25.00, cached=0.50, cache_write=6.25, + ), + "azure/anthropic/claude-sonnet-4-5": ModelPriceData( + input=3.00, output=15.00, cached=0.30, cache_write=3.75, + ), "aws/anthropic/bedrock-claude-sonnet-4-6": ModelPriceData( input=3.00, output=15.00, cached=0.30, cache_write=3.75, ), @@ -187,6 +221,9 @@ class ModelPriceData: input=1.00, output=5.00, cached=0.10, cache_write=1.25, ), # --- Anthropic direct API aliases (no AWS prefix) --- + "claude-opus-4-8": ModelPriceData( + input=5.00, output=25.00, cached=0.50, cache_write=6.25, + ), "claude-opus-4-7": ModelPriceData( input=5.00, output=25.00, cached=0.50, cache_write=6.25, ), diff --git a/switchyard/lib/processors/escalation_judge_request_processor.py b/switchyard/lib/processors/escalation_judge_request_processor.py index 957dfa8f..73d61244 100644 --- a/switchyard/lib/processors/escalation_judge_request_processor.py +++ b/switchyard/lib/processors/escalation_judge_request_processor.py @@ -6,7 +6,11 @@ from __future__ import annotations import hashlib +import json import logging +import sys +import time +from collections import OrderedDict from typing import Any, Literal from pydantic import BaseModel, ConfigDict, Field @@ -24,14 +28,20 @@ ) from switchyard.lib.proxy_context import ProxyContext from switchyard.lib.session_affinity import CTX_SESSION_KEY, SessionAffinity +from switchyard.lib.session_key import session_key_from_body +from switchyard.lib.stats_accumulator import StatsAccumulator from switchyard_rust.core import ChatRequest log = logging.getLogger(__name__) #: ``ProxyContext.metadata`` key holding the audit record of the judge's -#: decision for this turn: ``{"escalate", "reason", "turn", "source"}``. +#: decision for this turn: ``{"escalate", "reason", "turn", "source", +#: "session", "streak", "confirmed"}`` (the last three only on judged turns). CTX_ESCALATION_VERDICT = "_escalation_verdict" +#: LRU cap on the consecutive-escalate streak store. +_MAX_STREAK_SESSIONS = 10_000 + #: Tier labels stamped into ``CTX_DETERMINISTIC_ROUTING_TIER``; must match the #: labels the profile registers on ``DeterministicRoutingLLMBackend``. TIER_STRONG = "strong" @@ -39,21 +49,69 @@ ESCALATION_JUDGE_SYSTEM_PROMPT = """\ You are an escalation judge inside an agentic coding router. The session -started on the EFFICIENT tier (a cheap but top-class 2026 model). Your -job is to detect when the run is genuinely in trouble so the router can -escalate the rest of the task to the STRONG tier (frontier, expensive). +started on the EFFICIENT tier (a cheap but capable general-purpose +model). Your job is to detect when the run is genuinely in trouble so +the router can escalate the rest of the session to the STRONG tier +(frontier, expensive). Sessions range from single autonomous tasks to +long interactive collaborations where the user steers across several +sub-tasks — the same trouble signals apply to both. You see a condensed view of one session: the task framing (system prompt -+ first user message) and the most recent turns of activity (assistant -messages and tool results). Judge the *trajectory* — is the agent making -real progress toward the stated task — not the difficulty of the task -itself. Return exactly one JSON object: ++ first user message) and the most recent turns of activity. Those +recent turns include assistant messages, tool results, and any later +user messages — a user message in the recent window is the user +steering the session (a new instruction, a correction, an answer), and +is the most current statement of what the agent should be doing. Judge +the *trajectory* — is the agent making real progress toward what the +user currently wants — not the difficulty of the task itself. Return +exactly one JSON object: {"escalate": boolean, "reason": "one short sentence naming the pattern"} -Escalation is one-way for the rest of the task and expensive. Escalate -only on a clear PATTERN of trouble, never on a single failed command. -When the evidence is thin or ambiguous, return {"escalate": false}. +Escalation is one-way for the rest of the session and expensive. +Escalate only on a clear PATTERN of trouble, never on a single failed +command. When the evidence is thin or ambiguous, return +{"escalate": false}. + +The bar is not "is there friction" — agentic coding is full of friction +the efficient tier works through on its own. The bar is "is this run +likely DOOMED without intervention": the agent is stuck in place and +its recent behavior shows no mechanism by which the next few turns +would look different. + +# Is the stuck point beyond the efficient tier? + +Escalation pays only when the trouble is the KIND the strong tier is +better at. The efficient tier handles routine coding, file +exploration, single-file edits, normal debugging, dependency and +environment setup, and most refactors on its own — being stuck on +those is usually temporary. Weigh the kind of stuck point: + +Escalate sooner — the stuck point exceeds the efficient tier's +capability, and a stronger model would likely break the loop: +- Cross-module or cross-codebase synthesis: the fix requires learning + a convention, contract, or invariant from elsewhere in the codebase + and applying it consistently — even when the edit itself is + single-file. +- Subtle invariants: plausible-looking fixes keep failing the same + test because the root cause hinges on a behavior contract none of + the attempts have touched. +- Root causes genuinely spanning modules, or multi-step algorithmic / + formal reasoning the agent keeps getting almost-right. + +Hold weak — the efficient tier resolves these with iteration: +- Procedural or mechanical friction: tool availability, installs, + service startup, recipe-following scaffolding, localized one-file + test fixes. + +Hold weak — no model can fix these, so escalation is pure waste: +- The blocker is external: required data or files that simply do not + exist in the environment, a permanently broken or missing service, + or requirements the environment contradicts. A stronger model + changes nothing about a missing resource. One boundary to respect: + when producing, recovering, or decoding that very artifact IS the + stated task, its absence is the work itself, not a blocker — judge + the trajectory on it like any other work. # Trouble patterns — escalate when you see these @@ -71,15 +129,26 @@ (test output, exit code, error text) shows failure. - Finishing without running the verification the task specifies, when the task states how success is checked (e.g. "make the provided - tests pass") and running it was possible. + tests pass") and running it was possible. (Handing a result back to + the user to review, when no verification was specified, is normal — + not false progress.) - A reproduction or test the agent wrote that passes trivially without exercising the actual issue, then building on that false signal. - The agent's stated reading of a tool result contradicts what the result actually says (treating an error or empty output as success). Drift and dead ends: -- Recent activity no longer serves the task in the first user message - (e.g. polishing style while the required feature is unstarted). +- Recent activity no longer serves the goal the user has actually + asked for (e.g. polishing style while the required feature is + unstarted). Judge against the LATEST instruction the user has given, + not only the opening request: in an interactive session the user may + have redirected, narrowed, or added a new sub-task, and following + that redirection is correct behavior, not drift. A debugging detour + that plausibly unblocks the goal — fixing the environment, starting a + required service, investigating an error in a dependency — is NOT + drift; call drift only when the detour has produced nothing useful + for many turns AND the real work of the current goal remains + untouched. - Violating an explicit task constraint (modifying files the task says not to touch, changing the tests instead of the code under test). - Editing or reasoning about code without ever having opened the files @@ -88,11 +157,13 @@ the session (forgetting its own findings). - Many turns elapsed with nothing durable produced (no successful writes, no passing checks) and no visible narrowing of the problem — - the run is on pace to exhaust its turn budget. + effort continues but the problem is not getting smaller. Desperation: -- Giving up: declaring the task impossible, asking to stop, or drifting - into restating the problem instead of acting on it. +- Giving up: declaring the task impossible, or drifting into restating + the problem instead of acting on it. (A genuine clarifying question to + the user, or handing back a decision only the user can make, is normal + collaboration in an interactive session — not desperation.) - Destructive flailing: rm -rf, wholesale reinstalls, chmod -R, or reverting everything as a reaction to being stuck rather than a reasoned step. @@ -108,12 +179,29 @@ reading a file that turns out to be irrelevant) while the agent is still orienting. - A missing tool handled adaptively (tries `rg`, falls back to `grep`). +- Sequential alternatives: trying a DIFFERENT library, tool, or + approach after one fails is adaptation, not a loop — even when + several alternatives fail in a row. The loop pattern requires the + SAME approach retried without material change. +- A service that is unreachable or not yet running (server not + started, port closed, connection refused) while the agent is still + actively working to start, configure, or replace it. +- Planning activity: todo-list and plan updates (TodoWrite, + update_plan) are routine agent workflow — planning is neither drift + nor struggle, and harness-injected skill/instruction reading early + in a session is orientation, not off-task work. +- Zero-count summaries: "0 failed", "0 errors", "0 warnings" are + CLEAN results. Read failure keywords together with their counts — + only a nonzero count is a failure. - A long-running command (build, install, test suite) that simply has not finished, or the agent waiting on information it asked for. The distinguishing question: is each failure producing new information that changes the next action? Failing forward is fine; failing in place -is trouble. +is trouble. Also weigh the session's own recovery record: if this same +session already shows friction the agent subsequently cleared (a failure +followed by a verified fix or passing check), lean toward holding — a +session that has recovered before will usually recover again. # Worked examples (none drawn from any benchmark task set) @@ -143,6 +231,14 @@ passing test. -> {"escalate": false} * `npm install` has been running for one turn with no output yet. -> {"escalate": false} — slow command, not a stall. +* Four different serialization libraries failed to import; the agent + is now writing the converter with a fifth approach it has not tried + before. -> {"escalate": false} — sequential alternatives are + adaptation, even when none has succeeded yet. +* Task: tune a slow batch pipeline. The agent is investigating why + the message broker fails to start, since the pipeline cannot be + measured without it. -> {"escalate": false} — unblocking + verification serves the task. Do not emit markdown, commentary, or chain-of-thought — only the JSON object. @@ -181,10 +277,41 @@ class EscalationJudgeConfig(BaseModel): disable_reasoning: bool = True extra_headers: dict[str, str] | None = None + dump_verdicts_to_stderr: bool = True + """Emit one ``escalation_verdict={...}`` JSON line to ``sys.stderr`` per + judge call (verdict or fail-open). + + Written directly (not via the logging module) so it lands in the + benchmark server's captured log regardless of uvicorn's logger config — + mirrors :attr:`LLMClassifierConfig.dump_signals_to_stderr`. Grep + ``escalation_verdict=`` to reconstruct per-turn judge decisions and + escalation timing for a run. Disable for callers that share stderr + with an interactive TUI.""" + min_judge_turn: int = Field(default=3, ge=1) """First conversation turn on which the judge runs. Earlier turns have no trajectory to judge and always route weak.""" + escalate_confirmations: int = Field(default=1, ge=1) + """Consecutive ``escalate`` verdicts required before the latch fires. + + ``1`` (default) pins on the first escalate verdict. ``2`` requires the + judge to confirm on the next judged turn, filtering one-shot eager + verdicts (a single failed command misread as a pattern) while keeping + recall on real trouble, which by definition persists across turns. A + non-escalate verdict resets the streak; a fail-open leaves it + unchanged (no evidence either way).""" + + confirmation_window: int = Field(default=1, ge=1) + """How many judged turns an escalate verdict stays live for + confirmation purposes. + + ``1`` (default) keeps the strict-consecutive behaviour: any decline + resets the streak. ``N > 1`` lets a later escalate verdict confirm an + earlier one across up to ``N - 1`` intervening declines — trouble that + keeps *recurring* is trouble, even when a quiet turn separates the + flare-ups.""" + recent_turn_window: int = Field(default=14, ge=1) """Trailing messages shown to the judge on top of the anchors. Wider than the classifier's default because loop detection needs to see the repeats: @@ -227,12 +354,20 @@ def __init__( affinity: SessionAffinity, client: LLMClassifierClient | None = None, session_key_depth: int = 0, + stats_accumulator: StatsAccumulator | None = None, ) -> None: if session_key_depth < 0: raise ValueError("session_key_depth must be >= 0") self._config = config self._affinity = affinity self._session_key_depth = session_key_depth + # Also attached post-construction by the profile chain's + # ``with_runtime_components`` (same hook as the LLM classifier). + self._stats_accumulator = stats_accumulator + # Per-conversation (streak, declines_since_last_fire), only consulted + # when ``escalate_confirmations > 1``. LRU-capped alongside the + # affinity store so a long-lived server cannot grow it unbounded. + self._escalate_streaks: OrderedDict[str, tuple[int, int]] = OrderedDict() self._client = client or OpenAIChatLLMClassifierClient( OpenAILLMClient( api_key=config.api_key, @@ -272,7 +407,11 @@ async def process(self, ctx: ProxyContext, request: ChatRequest) -> ChatRequest: if not affinity_ready or turn < self._config.min_judge_turn: return request + # Present when session_key_depth > 0; lets verdict-dump consumers + # group per-turn judge decisions by conversation. + session = ctx.metadata.get(CTX_SESSION_KEY) summary = _summarize_for_judge(request, turn=turn, config=self._config) + started_at = time.perf_counter() try: completion = await self._client.classify( model=self._config.model, @@ -289,21 +428,55 @@ async def process(self, ctx: ProxyContext, request: ChatRequest) -> ChatRequest: "EscalationJudgeRequestProcessor: judge failed; staying on weak tier: %s", exc, ) + if self._stats_accumulator is not None: + # Record the failure into the classifier bucket so + # ``/v1/routing/stats`` exposes the judge fail-open rate; + # a silently failing judge reads as "router never + # escalates" to the benchmark observer. + await self._stats_accumulator.record_classifier_error(self._config.model) ctx.metadata[CTX_ESCALATION_VERDICT] = { "escalate": False, "reason": f"fail_open: {str(exc)[:200]}", "turn": turn, "source": "fail_open", + "session": session, } + self._dump_verdict( + ctx.metadata[CTX_ESCALATION_VERDICT], + latency_ms=(time.perf_counter() - started_at) * 1000, + request=request, + ) return request + if self._stats_accumulator is not None: + await self._record_judge_call( + usage=completion.usage, + latency_ms=(time.perf_counter() - started_at) * 1000, + ) + + if verdict.escalate: + streak = self._bump_streak(ctx, request) + confirmed = streak >= self._config.escalate_confirmations + else: + streak = 0 + confirmed = False + self._reset_streak(ctx, request) + ctx.metadata[CTX_ESCALATION_VERDICT] = { "escalate": verdict.escalate, "reason": verdict.reason, "turn": turn, "source": "judge", + "session": session, + "streak": streak, + "confirmed": confirmed, } - if verdict.escalate: + self._dump_verdict( + ctx.metadata[CTX_ESCALATION_VERDICT], + latency_ms=(time.perf_counter() - started_at) * 1000, + request=request, + ) + if confirmed: log.info( "EscalationJudgeRequestProcessor: escalating to strong tier " "(turn %d): %s", @@ -314,6 +487,102 @@ async def process(self, ctx: ProxyContext, request: ChatRequest) -> ChatRequest: ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] = TIER_STRONG return request + def _streak_key(self, ctx: ProxyContext, request: ChatRequest) -> str: + """Conversation key for streak bookkeeping (seeded deep key or Rust default).""" + cached = ctx.metadata.get(CTX_SESSION_KEY) + if isinstance(cached, str): + return cached + return session_key_from_body(request.body) + + def _bump_streak(self, ctx: ProxyContext, request: ChatRequest) -> int: + """Record one escalate verdict for this conversation; return the streak.""" + if self._config.escalate_confirmations <= 1: + # Single-verdict latch: no bookkeeping needed, streak is per-call. + return 1 + key = self._streak_key(ctx, request) + streak, _ = self._escalate_streaks.get(key, (0, 0)) + streak += 1 + self._escalate_streaks[key] = (streak, 0) + self._escalate_streaks.move_to_end(key) + while len(self._escalate_streaks) > _MAX_STREAK_SESSIONS: + self._escalate_streaks.popitem(last=False) + return streak + + def _reset_streak(self, ctx: ProxyContext, request: ChatRequest) -> None: + """Register a non-escalate verdict against the streak. + + With ``confirmation_window == 1`` any decline clears the streak + (strict-consecutive). A larger window tolerates up to + ``window - 1`` intervening declines before the streak expires, so + recurring intermittent trouble can still confirm. + """ + if self._config.escalate_confirmations <= 1: + return + key = self._streak_key(ctx, request) + entry = self._escalate_streaks.get(key) + if entry is None: + return + streak, declines = entry + declines += 1 + if declines >= self._config.confirmation_window: + self._escalate_streaks.pop(key, None) + else: + self._escalate_streaks[key] = (streak, declines) + self._escalate_streaks.move_to_end(key) + + def _dump_verdict( + self, + verdict: dict[str, Any], + *, + latency_ms: float, + request: ChatRequest | None = None, + ) -> None: + """Write one ``escalation_verdict={...}`` JSON line to stderr. + + Direct stderr (not the logging module) so benchmark server logs + capture it — see :attr:`EscalationJudgeConfig.dump_verdicts_to_stderr`. + Includes a short first-user-message snippet (``task_hint``) so + offline analysis can attribute per-turn verdicts to tasks exactly, + instead of guessing from overlapping trial time windows. + """ + if not self._config.dump_verdicts_to_stderr: + return + payload = dict(verdict) + payload["latency_ms"] = round(latency_ms, 1) + if request is not None: + hint = _first_user_text(request) + if hint: + payload["task_hint"] = hint[:48] + sys.stderr.write(f"escalation_verdict={json.dumps(payload, sort_keys=True)}\n") + sys.stderr.flush() + + async def _record_judge_call(self, *, usage: Any, latency_ms: float) -> None: + """Record judge token spend and latency into the classifier stats bucket. + + The judge is a routing-overhead call, exactly like the LLM + classifier: keeping it in the classifier bucket stops its spend + from merging with a same-named tier model and keeps + ``routing_stats_final.json`` cost math honest for the + escalation-router benchmark. + """ + assert self._stats_accumulator is not None + prompt = 0 + completion = 0 + cached = 0 + if usage is not None: + prompt = getattr(usage, "prompt_tokens", 0) or 0 + completion = getattr(usage, "completion_tokens", 0) or 0 + ptd = getattr(usage, "prompt_tokens_details", None) + if ptd is not None: + cached = getattr(ptd, "cached_tokens", 0) or 0 + await self._stats_accumulator.record_classifier_usage( + model=self._config.model, + prompt_tokens=prompt, + completion_tokens=completion, + cached_tokens=cached, + latency_ms=latency_ms, + ) + def _seed_session_key(self, ctx: ProxyContext, request: ChatRequest) -> bool: """Seed the deep session key when configured; return whether affinity is usable. @@ -332,6 +601,37 @@ def _seed_session_key(self, ctx: ProxyContext, request: ChatRequest) -> bool: return True +def _first_user_text(request: ChatRequest) -> str: + """Flatten the first user message's task text for verdict dumps. + + Harness wrappers (````/```` + blocks injected ahead of the actual task statement) are skipped so the + snippet identifies the task rather than repeating boilerplate shared + by every conversation. + """ + body = getattr(request, "body", None) + if not isinstance(body, dict): + return "" + messages = body.get("messages") + if not isinstance(messages, list): + messages = body.get("input") + if not isinstance(messages, list): + return "" + for m in messages: + if isinstance(m, dict) and m.get("role") == "user": + text = _message_text(m).strip() + while True: + for tag in ("system-reminder", "environment_context"): + if text.startswith(f"<{tag}>"): + end = text.find(f"") + if end >= 0: + text = text[end + len(tag) + 3 :].lstrip() + break + else: + return text + return "" + + def _deep_session_key(request: ChatRequest, depth: int) -> str | None: """Hash system + first user + first ``depth`` later messages, or ``None`` if short. diff --git a/switchyard/lib/profiles/escalation_router_config.py b/switchyard/lib/profiles/escalation_router_config.py index 1dbfd494..37662994 100644 --- a/switchyard/lib/profiles/escalation_router_config.py +++ b/switchyard/lib/profiles/escalation_router_config.py @@ -37,8 +37,18 @@ class EscalationRouterConfig(BaseModel): eviction (context-window overflow). Must match ``strong.id`` or ``weak.id``; the judge target is not a routing candidate. judge_min_turn: First conversation turn on which the judge runs. + judge_escalate_confirmations: Consecutive escalate verdicts required + before the strong latch fires (``1`` pins on the first verdict). + judge_confirmation_window: Judged turns an escalate verdict stays + live for confirmation; ``N > 1`` tolerates up to ``N - 1`` + intervening declines (recurring intermittent trouble confirms). + judge_disable_reasoning: Send the thinking-off template hint on judge + calls (default). ``False`` lets a reasoning judge model think, + trading per-turn latency for rubric adherence. judge_recent_turn_window: Trailing messages shown to the judge on top of the system + first-user anchors. + judge_window_message_chars: Per-message truncation cap inside the + trailing window (larger keeps more tool-output detail per turn). judge_max_request_chars: Cap on the assembled judge transcript. judge_system_prompt: Optional judge prompt override. ``None`` uses the built-in prompt. @@ -64,7 +74,11 @@ class EscalationRouterConfig(BaseModel): judge: LlmTarget fallback_target_on_evict: str judge_min_turn: int = Field(default=3, ge=1) + judge_escalate_confirmations: int = Field(default=1, ge=1) + judge_confirmation_window: int = Field(default=1, ge=1) + judge_disable_reasoning: bool = True judge_recent_turn_window: int = Field(default=14, ge=1) + judge_window_message_chars: int = Field(default=300, ge=50) judge_max_request_chars: int = Field(default=12_000, ge=1_000) judge_system_prompt: str | None = Field(default=None, min_length=1) judge_timeout_s: float = Field(default=5.0, gt=0.0) diff --git a/switchyard/lib/profiles/escalation_router_profile_config.py b/switchyard/lib/profiles/escalation_router_profile_config.py index 40945806..d3d42dd9 100644 --- a/switchyard/lib/profiles/escalation_router_profile_config.py +++ b/switchyard/lib/profiles/escalation_router_profile_config.py @@ -64,16 +64,24 @@ def build(self) -> ComponentChainProfile: warmup_turns=0, ) + # DeepSeek judges get the same benchmark-gateway defaults as DeepSeek + # tiers (``X-Inference-Priority: batch``) so their calls land on the + # relaxed-timeout gateway alongside the routed tier traffic. + judge_target = _apply_deepseek_overrides(config.judge) judge_config = EscalationJudgeConfig( - model=config.judge.model, - api_key=config.judge.api_key, - base_url=config.judge.base_url, - timeout_s=config.judge.endpoint.timeout_secs or 5.0, + model=judge_target.model, + api_key=judge_target.api_key, + base_url=judge_target.base_url, + timeout_s=judge_target.endpoint.timeout_secs or 5.0, system_prompt=config.judge_system_prompt or ESCALATION_JUDGE_SYSTEM_PROMPT, min_judge_turn=config.judge_min_turn, + escalate_confirmations=config.judge_escalate_confirmations, + confirmation_window=config.judge_confirmation_window, + disable_reasoning=config.judge_disable_reasoning, recent_turn_window=config.judge_recent_turn_window, + window_message_chars=config.judge_window_message_chars, max_request_chars=config.judge_max_request_chars, - extra_headers=config.judge.extra_headers or None, + extra_headers=judge_target.extra_headers or None, ) request_processors: list[Any] = [ ReasoningEffortNormalizer(), diff --git a/tests/test_escalation_judge_request_processor.py b/tests/test_escalation_judge_request_processor.py index 56a56779..a8e3968d 100644 --- a/tests/test_escalation_judge_request_processor.py +++ b/tests/test_escalation_judge_request_processor.py @@ -6,6 +6,7 @@ from __future__ import annotations import json +from types import SimpleNamespace from typing import Any, cast from switchyard.lib.backends.deterministic_routing_llm_backend import ( @@ -19,12 +20,14 @@ from switchyard.lib.processors.llm_classifier import ClassifierCompletion from switchyard.lib.proxy_context import ProxyContext from switchyard.lib.session_affinity import CTX_SESSION_KEY, SessionAffinity +from switchyard.lib.stats_accumulator import StatsAccumulator from switchyard_rust.core import ChatRequest class _FakeJudgeClient: - def __init__(self, response: str | Exception) -> None: + def __init__(self, response: str | Exception, usage: Any | None = None) -> None: self.response = response + self.usage = usage self.calls: list[dict[str, str]] = [] async def classify( @@ -41,7 +44,7 @@ async def classify( }) if isinstance(self.response, Exception): raise self.response - return ClassifierCompletion(content=self.response) + return ClassifierCompletion(content=self.response, usage=self.usage) def _verdict_json(escalate: bool, reason: str = "same error repeating") -> str: @@ -136,6 +139,143 @@ async def test_judge_failure_fails_open_to_weak_without_pin() -> None: assert await affinity.pinned(ProxyContext(), _request()) is None +async def test_confirmations_require_consecutive_escalates() -> None: + """With confirmations=2, one escalate verdict stays weak; the second latches.""" + processor, fake, affinity = _processor(_verdict_json(True), escalate_confirmations=2) + + ctx = ProxyContext() + await processor.process(ctx, _request(n_assistant_turns=3)) + assert ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "weak" + assert ctx.metadata[CTX_ESCALATION_VERDICT]["escalate"] is True + assert ctx.metadata[CTX_ESCALATION_VERDICT]["confirmed"] is False + assert ctx.metadata[CTX_ESCALATION_VERDICT]["streak"] == 1 + assert await affinity.pinned(ProxyContext(), _request()) is None + + ctx = ProxyContext() + await processor.process(ctx, _request(n_assistant_turns=4)) + assert ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "strong" + assert ctx.metadata[CTX_ESCALATION_VERDICT]["confirmed"] is True + assert ctx.metadata[CTX_ESCALATION_VERDICT]["streak"] == 2 + + +async def test_confirmation_window_tolerates_intervening_decline() -> None: + """window=3: escalate -> decline -> escalate confirms (recurring trouble).""" + affinity = SessionAffinity(enabled=True) + fake = _FakeJudgeClient(_verdict_json(True)) + processor = EscalationJudgeRequestProcessor( + EscalationJudgeConfig( + model="judge-model", escalate_confirmations=2, confirmation_window=3, + ), + affinity=affinity, + client=fake, + ) + + await processor.process(ProxyContext(), _request(n_assistant_turns=3)) + fake.response = _verdict_json(False, "") + await processor.process(ProxyContext(), _request(n_assistant_turns=4)) + fake.response = _verdict_json(True) + ctx = ProxyContext() + await processor.process(ctx, _request(n_assistant_turns=5)) + + assert ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "strong" + assert ctx.metadata[CTX_ESCALATION_VERDICT]["confirmed"] is True + assert ctx.metadata[CTX_ESCALATION_VERDICT]["streak"] == 2 + + +async def test_confirmation_window_expires_after_enough_declines() -> None: + """window=2: two declines between fires expire the streak.""" + affinity = SessionAffinity(enabled=True) + fake = _FakeJudgeClient(_verdict_json(True)) + processor = EscalationJudgeRequestProcessor( + EscalationJudgeConfig( + model="judge-model", escalate_confirmations=2, confirmation_window=2, + ), + affinity=affinity, + client=fake, + ) + + await processor.process(ProxyContext(), _request(n_assistant_turns=3)) + fake.response = _verdict_json(False, "") + await processor.process(ProxyContext(), _request(n_assistant_turns=4)) + await processor.process(ProxyContext(), _request(n_assistant_turns=5)) + fake.response = _verdict_json(True) + ctx = ProxyContext() + await processor.process(ctx, _request(n_assistant_turns=6)) + + assert ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "weak" + assert ctx.metadata[CTX_ESCALATION_VERDICT]["streak"] == 1 + assert ( + ctx.metadata[CTX_ESCALATION_VERDICT]["confirmed"] is False + ) + + +async def test_confirmations_streak_resets_on_decline() -> None: + """escalate -> decline -> escalate never latches with confirmations=2.""" + affinity = SessionAffinity(enabled=True) + fake = _FakeJudgeClient(_verdict_json(True)) + processor = EscalationJudgeRequestProcessor( + EscalationJudgeConfig(model="judge-model", escalate_confirmations=2), + affinity=affinity, + client=fake, + ) + + await processor.process(ProxyContext(), _request(n_assistant_turns=3)) + fake.response = _verdict_json(False, "") + await processor.process(ProxyContext(), _request(n_assistant_turns=4)) + fake.response = _verdict_json(True) + ctx = ProxyContext() + await processor.process(ctx, _request(n_assistant_turns=5)) + + assert ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "weak" + assert ctx.metadata[CTX_ESCALATION_VERDICT]["streak"] == 1 + assert await affinity.pinned(ProxyContext(), _request()) is None + + +async def test_judge_usage_recorded_into_classifier_stats() -> None: + """Judge token spend lands in the classifier bucket of the stats snapshot.""" + stats = StatsAccumulator() + affinity = SessionAffinity(enabled=True) + fake = _FakeJudgeClient( + _verdict_json(False, ""), + usage=SimpleNamespace( + prompt_tokens=321, + completion_tokens=17, + prompt_tokens_details=SimpleNamespace(cached_tokens=100), + ), + ) + processor = EscalationJudgeRequestProcessor( + EscalationJudgeConfig(model="judge-model"), + affinity=affinity, + client=fake, + stats_accumulator=stats, + ) + + await processor.process(ProxyContext(), _request()) + + snapshot = await stats.snapshot() + judge = snapshot["classifier"]["models"]["judge-model"] + assert judge["calls"] == 1 + assert judge["prompt_tokens"] == 321 + assert judge["completion_tokens"] == 17 + assert judge["model_call_latency"]["count"] == 1 + + +async def test_judge_failure_recorded_as_classifier_error() -> None: + stats = StatsAccumulator() + fake = _FakeJudgeClient(RuntimeError("judge down")) + processor = EscalationJudgeRequestProcessor( + EscalationJudgeConfig(model="judge-model"), + affinity=SessionAffinity(enabled=True), + client=fake, + stats_accumulator=stats, + ) + + await processor.process(ProxyContext(), _request()) + + snapshot = await stats.snapshot() + assert snapshot["classifier"]["total_errors"] == 1 + + async def test_markdown_fenced_verdict_is_parsed() -> None: processor, _, _ = _processor("```json\n" + _verdict_json(True) + "\n```") ctx = ProxyContext() diff --git a/tests/test_escalation_router_profile.py b/tests/test_escalation_router_profile.py index b88478bb..afc29190 100644 --- a/tests/test_escalation_router_profile.py +++ b/tests/test_escalation_router_profile.py @@ -76,11 +76,39 @@ def test_build_composes_judge_chain() -> None: assert profile._fallback_target_on_evict == "strong" +def test_build_pins_deepseek_judge_to_batch_gateway() -> None: + """A DeepSeek judge inherits the benchmark-gateway header like DeepSeek tiers.""" + profile = EscalationRouterProfileConfig.from_config( + _config( + judge=LlmTarget( + id="judge", + model="nvidia/deepseek-ai/deepseek-v4-flash", + base_url="https://judge.invalid/v1", + api_key="sk-judge", + ), + ), + ).build() + + judge = profile._request_processors[1] + assert isinstance(judge, EscalationJudgeRequestProcessor) + assert judge._config.extra_headers == {"X-Inference-Priority": "batch"} + + +def test_build_non_deepseek_judge_has_no_default_headers() -> None: + profile = EscalationRouterProfileConfig.from_config(_config()).build() + + judge = profile._request_processors[1] + assert isinstance(judge, EscalationJudgeRequestProcessor) + assert judge._config.extra_headers is None + + def test_build_threads_judge_settings() -> None: profile = EscalationRouterProfileConfig.from_config( _config( judge_min_turn=5, judge_recent_turn_window=20, + judge_window_message_chars=500, + judge_disable_reasoning=False, session_key_depth=2, ), ).build() @@ -89,6 +117,8 @@ def test_build_threads_judge_settings() -> None: assert isinstance(judge, EscalationJudgeRequestProcessor) assert judge._config.min_judge_turn == 5 assert judge._config.recent_turn_window == 20 + assert judge._config.window_message_chars == 500 + assert judge._config.disable_reasoning is False assert judge._config.system_prompt == ESCALATION_JUDGE_SYSTEM_PROMPT assert judge._session_key_depth == 2 assert judge._affinity.enabled From ee550bd7aa5faac2db7cb965d2ea2af6aa745b36 Mon Sep 17 00:00:00 2001 From: Lin Jia Date: Mon, 13 Jul 2026 23:41:37 -0700 Subject: [PATCH 03/23] =?UTF-8?q?feat(router):=20ESC9=20judge=20prompt=20?= =?UTF-8?q?=E2=80=94=20impossibility-holds=20gate,=20setup-grind=20bar,=20?= =?UTF-8?q?varied-exploration=20carve-out?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../escalation_judge_request_processor.py | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/switchyard/lib/processors/escalation_judge_request_processor.py b/switchyard/lib/processors/escalation_judge_request_processor.py index 73d61244..8320661a 100644 --- a/switchyard/lib/processors/escalation_judge_request_processor.py +++ b/switchyard/lib/processors/escalation_judge_request_processor.py @@ -112,6 +112,14 @@ when producing, recovering, or decoding that very artifact IS the stated task, its absence is the work itself, not a blocker — judge the trajectory on it like any other work. + Impossibility is grounds for HOLDING, never for escalating: if the + honest summary of the stuck point is "no model could fix this", the + verdict is {"escalate": false}. An escalate reason must name + something a stronger model could plausibly do differently — never + cite a model-independent blocker as the reason to escalate. With an + external blocker, the pattern to watch is whether the agent adapts + AROUND it (an alternative source, tool, or route); escalate only if + that adaptation itself shows a trouble pattern. # Trouble patterns — escalate when you see these @@ -123,6 +131,12 @@ - Fighting the environment: repeatedly invoking a missing executable, retrying installs that fail the same way, or trying variations of a command the environment has already rejected, instead of adapting. + Early setup grind is not yet this pattern: a few failed install or + setup attempts in a row are normal, and it becomes fighting only + when the same blocker persists across multiple DISTINCT remedies + (a different installer or channel, an isolated environment, building + from source, switching tools) — not merely across retries of one + remedy the agent is still varying. False progress (looks like progress, is not): - Declaring success or moving on while the latest visible evidence @@ -157,7 +171,8 @@ the session (forgetting its own findings). - Many turns elapsed with nothing durable produced (no successful writes, no passing checks) and no visible narrowing of the problem — - effort continues but the problem is not getting smaller. + effort continues but the problem is not getting smaller, and the + recent attempts repeat prior probes rather than vary them. Desperation: - Giving up: declaring the task impossible, or drifting into restating @@ -195,6 +210,12 @@ only a nonzero count is a failure. - A long-running command (build, install, test suite) that simply has not finished, or the agent waiting on information it asked for. +- Extended exploration or experimentation in analysis-heavy work: + probing data, running varied experiments, or reading widely IS the + work while the probes vary and keep returning new information — + absence of a written artifact is not by itself a stall. A genuine + stall repeats the SAME probe and gets the SAME result; when calling + a loop, name the unchanged error or result. The distinguishing question: is each failure producing new information that changes the next action? Failing forward is fine; failing in place From 5a074188940b3aa0b2a668c2bb89435942de78ab Mon Sep 17 00:00:00 2001 From: Lin Jia Date: Tue, 14 Jul 2026 09:50:08 -0700 Subject: [PATCH 04/23] revert(router): restore ESC7 judge prompt as benchmark line of record --- .../escalation_judge_request_processor.py | 79 +++++-------------- 1 file changed, 21 insertions(+), 58 deletions(-) diff --git a/switchyard/lib/processors/escalation_judge_request_processor.py b/switchyard/lib/processors/escalation_judge_request_processor.py index 8320661a..fab1a9a8 100644 --- a/switchyard/lib/processors/escalation_judge_request_processor.py +++ b/switchyard/lib/processors/escalation_judge_request_processor.py @@ -49,29 +49,21 @@ ESCALATION_JUDGE_SYSTEM_PROMPT = """\ You are an escalation judge inside an agentic coding router. The session -started on the EFFICIENT tier (a cheap but capable general-purpose -model). Your job is to detect when the run is genuinely in trouble so -the router can escalate the rest of the session to the STRONG tier -(frontier, expensive). Sessions range from single autonomous tasks to -long interactive collaborations where the user steers across several -sub-tasks — the same trouble signals apply to both. +started on the EFFICIENT tier (a cheap but top-class 2026 model). Your +job is to detect when the run is genuinely in trouble so the router can +escalate the rest of the task to the STRONG tier (frontier, expensive). You see a condensed view of one session: the task framing (system prompt -+ first user message) and the most recent turns of activity. Those -recent turns include assistant messages, tool results, and any later -user messages — a user message in the recent window is the user -steering the session (a new instruction, a correction, an answer), and -is the most current statement of what the agent should be doing. Judge -the *trajectory* — is the agent making real progress toward what the -user currently wants — not the difficulty of the task itself. Return -exactly one JSON object: ++ first user message) and the most recent turns of activity (assistant +messages and tool results). Judge the *trajectory* — is the agent making +real progress toward the stated task — not the difficulty of the task +itself. Return exactly one JSON object: {"escalate": boolean, "reason": "one short sentence naming the pattern"} -Escalation is one-way for the rest of the session and expensive. -Escalate only on a clear PATTERN of trouble, never on a single failed -command. When the evidence is thin or ambiguous, return -{"escalate": false}. +Escalation is one-way for the rest of the task and expensive. Escalate +only on a clear PATTERN of trouble, never on a single failed command. +When the evidence is thin or ambiguous, return {"escalate": false}. The bar is not "is there friction" — agentic coding is full of friction the efficient tier works through on its own. The bar is "is this run @@ -112,14 +104,6 @@ when producing, recovering, or decoding that very artifact IS the stated task, its absence is the work itself, not a blocker — judge the trajectory on it like any other work. - Impossibility is grounds for HOLDING, never for escalating: if the - honest summary of the stuck point is "no model could fix this", the - verdict is {"escalate": false}. An escalate reason must name - something a stronger model could plausibly do differently — never - cite a model-independent blocker as the reason to escalate. With an - external blocker, the pattern to watch is whether the agent adapts - AROUND it (an alternative source, tool, or route); escalate only if - that adaptation itself shows a trouble pattern. # Trouble patterns — escalate when you see these @@ -131,38 +115,26 @@ - Fighting the environment: repeatedly invoking a missing executable, retrying installs that fail the same way, or trying variations of a command the environment has already rejected, instead of adapting. - Early setup grind is not yet this pattern: a few failed install or - setup attempts in a row are normal, and it becomes fighting only - when the same blocker persists across multiple DISTINCT remedies - (a different installer or channel, an isolated environment, building - from source, switching tools) — not merely across retries of one - remedy the agent is still varying. False progress (looks like progress, is not): - Declaring success or moving on while the latest visible evidence (test output, exit code, error text) shows failure. - Finishing without running the verification the task specifies, when the task states how success is checked (e.g. "make the provided - tests pass") and running it was possible. (Handing a result back to - the user to review, when no verification was specified, is normal — - not false progress.) + tests pass") and running it was possible. - A reproduction or test the agent wrote that passes trivially without exercising the actual issue, then building on that false signal. - The agent's stated reading of a tool result contradicts what the result actually says (treating an error or empty output as success). Drift and dead ends: -- Recent activity no longer serves the goal the user has actually - asked for (e.g. polishing style while the required feature is - unstarted). Judge against the LATEST instruction the user has given, - not only the opening request: in an interactive session the user may - have redirected, narrowed, or added a new sub-task, and following - that redirection is correct behavior, not drift. A debugging detour - that plausibly unblocks the goal — fixing the environment, starting a - required service, investigating an error in a dependency — is NOT - drift; call drift only when the detour has produced nothing useful - for many turns AND the real work of the current goal remains - untouched. +- Recent activity no longer serves the task in the first user message + (e.g. polishing style while the required feature is unstarted). + A debugging detour that plausibly unblocks the task — fixing the + environment, starting a required service, investigating an error in + a dependency — is NOT drift; call drift only when the detour has + produced nothing useful for many turns AND the task's real + verification remains untouched. - Violating an explicit task constraint (modifying files the task says not to touch, changing the tests instead of the code under test). - Editing or reasoning about code without ever having opened the files @@ -171,14 +143,11 @@ the session (forgetting its own findings). - Many turns elapsed with nothing durable produced (no successful writes, no passing checks) and no visible narrowing of the problem — - effort continues but the problem is not getting smaller, and the - recent attempts repeat prior probes rather than vary them. + the run is on pace to exhaust its turn budget. Desperation: -- Giving up: declaring the task impossible, or drifting into restating - the problem instead of acting on it. (A genuine clarifying question to - the user, or handing back a decision only the user can make, is normal - collaboration in an interactive session — not desperation.) +- Giving up: declaring the task impossible, asking to stop, or drifting + into restating the problem instead of acting on it. - Destructive flailing: rm -rf, wholesale reinstalls, chmod -R, or reverting everything as a reaction to being stuck rather than a reasoned step. @@ -210,12 +179,6 @@ only a nonzero count is a failure. - A long-running command (build, install, test suite) that simply has not finished, or the agent waiting on information it asked for. -- Extended exploration or experimentation in analysis-heavy work: - probing data, running varied experiments, or reading widely IS the - work while the probes vary and keep returning new information — - absence of a written artifact is not by itself a stall. A genuine - stall repeats the SAME probe and gets the SAME result; when calling - a loop, name the unchanged error or result. The distinguishing question: is each failure producing new information that changes the next action? Failing forward is fine; failing in place From e85396e95772139a52bf9b60e153f9e32f6e87a5 Mon Sep 17 00:00:00 2001 From: Lin Jia Date: Wed, 15 Jul 2026 14:49:34 -0700 Subject: [PATCH 05/23] refactor(router): load judge prompt from package data; add judge.prompt_path knob --- .../escalation_router_routing.md | 3 +- pyproject.toml | 6 +- switchyard/cli/route_bundle.py | 28 ++- .../escalation_judge_request_processor.py | 196 ++---------------- .../processors/prompts/escalation_judge.md | 178 ++++++++++++++++ ...test_escalation_judge_request_processor.py | 21 ++ tests/test_route_bundle.py | 47 +++++ 7 files changed, 296 insertions(+), 183 deletions(-) create mode 100644 switchyard/lib/processors/prompts/escalation_judge.md diff --git a/docs/routing_algorithms/escalation_router_routing.md b/docs/routing_algorithms/escalation_router_routing.md index 2c3548fd..876028fb 100644 --- a/docs/routing_algorithms/escalation_router_routing.md +++ b/docs/routing_algorithms/escalation_router_routing.md @@ -67,7 +67,8 @@ routes: | `judge.timeout_secs` | `5.0` | Judge wall-clock ceiling; fails open to weak. | | `judge.min_turn` | `3` | First conversation turn the judge runs on. Earlier turns have no trajectory to judge. | | `judge.recent_turn_window` | `14` | Trailing messages shown to the judge. Loops longer than the window are invisible — widen before concluding the judge misses them. | -| `judge.prompt` | built-in | Judge system-prompt override. | +| `judge.prompt` | built-in | Judge system-prompt override, inlined in the YAML. | +| `judge.prompt_path` | built-in | Judge system-prompt override read from a file; mutually exclusive with `judge.prompt`. Relative paths resolve against the server's working directory. | | `judge.max_request_chars` | `12000` | Cap on the judge transcript; oldest window messages are dropped first. | | `fallback_target_on_evict` | required | `strong` or `weak`; context-window-eviction reroute target. | | `session_key_depth` | `0` | See [Repeated-trial benchmarking](#repeated-trial-benchmarking) below. | diff --git a/pyproject.toml b/pyproject.toml index 14e3aee6..91eb2a72 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -147,7 +147,11 @@ where = ["."] include = ["switchyard*"] [tool.setuptools.package-data] -switchyard = ["py.typed", "lib/processors/stage_router/prompts/*.md"] +switchyard = [ + "py.typed", + "lib/processors/prompts/*.md", + "lib/processors/stage_router/prompts/*.md", +] switchyard_rust = ["py.typed", "*.pyi"] [tool.maturin] diff --git a/switchyard/cli/route_bundle.py b/switchyard/cli/route_bundle.py index 36705b9c..4f8dfd3c 100644 --- a/switchyard/cli/route_bundle.py +++ b/switchyard/cli/route_bundle.py @@ -293,6 +293,7 @@ def llm_target_to_route_dict(target: LlmTarget) -> dict[str, Any]: "recent_turn_window", "window_message_chars", "prompt", + "prompt_path", "max_request_chars", }) _DETERMINISTIC_CLASSIFIER_KEYS = frozenset({ @@ -1051,6 +1052,31 @@ def _deterministic_switchyard( ) +def _judge_prompt_value(judge: Mapping[str, object], model_id: str) -> str | None: + """Resolve the judge prompt override from ``prompt`` or ``prompt_path``. + + ``prompt`` inlines the text; ``prompt_path`` reads it from a file + (relative paths resolve against the server's working directory). + ``None`` keeps the built-in default. + """ + inline = _optional_str(judge.get("prompt")) + path = _optional_str(judge.get("prompt_path")) + if inline is not None and path is not None: + raise RouteBundleConfigError( + f"route {model_id!r}: judge.prompt and judge.prompt_path are " + "mutually exclusive", + ) + if path is None: + return inline + try: + return Path(path).read_text(encoding="utf-8") + except (OSError, UnicodeError) as exc: + raise RouteBundleConfigError( + f"route {model_id!r}: judge.prompt_path {path!r}: cannot read: " + f"{_format_exception_one_line(exc)}" + ) from exc + + def _escalation_router_switchyard( model_id: str, route: Mapping[str, object], @@ -1157,7 +1183,7 @@ def _escalation_router_switchyard( "judge_window_message_chars": _optional_int( judge.get("window_message_chars"), default=300, ), - "judge_system_prompt": _optional_str(judge.get("prompt")), + "judge_system_prompt": _judge_prompt_value(judge, model_id), "judge_timeout_s": _optional_float(judge.get("timeout_secs"), default=5.0), "enable_stats": _optional_bool(route.get("enable_stats"), default=True), } diff --git a/switchyard/lib/processors/escalation_judge_request_processor.py b/switchyard/lib/processors/escalation_judge_request_processor.py index fab1a9a8..0b26fcd5 100644 --- a/switchyard/lib/processors/escalation_judge_request_processor.py +++ b/switchyard/lib/processors/escalation_judge_request_processor.py @@ -11,6 +11,7 @@ import sys import time from collections import OrderedDict +from importlib.resources import files from typing import Any, Literal from pydantic import BaseModel, ConfigDict, Field @@ -47,186 +48,21 @@ TIER_STRONG = "strong" TIER_WEAK = "weak" -ESCALATION_JUDGE_SYSTEM_PROMPT = """\ -You are an escalation judge inside an agentic coding router. The session -started on the EFFICIENT tier (a cheap but top-class 2026 model). Your -job is to detect when the run is genuinely in trouble so the router can -escalate the rest of the task to the STRONG tier (frontier, expensive). - -You see a condensed view of one session: the task framing (system prompt -+ first user message) and the most recent turns of activity (assistant -messages and tool results). Judge the *trajectory* — is the agent making -real progress toward the stated task — not the difficulty of the task -itself. Return exactly one JSON object: - -{"escalate": boolean, "reason": "one short sentence naming the pattern"} - -Escalation is one-way for the rest of the task and expensive. Escalate -only on a clear PATTERN of trouble, never on a single failed command. -When the evidence is thin or ambiguous, return {"escalate": false}. - -The bar is not "is there friction" — agentic coding is full of friction -the efficient tier works through on its own. The bar is "is this run -likely DOOMED without intervention": the agent is stuck in place and -its recent behavior shows no mechanism by which the next few turns -would look different. - -# Is the stuck point beyond the efficient tier? - -Escalation pays only when the trouble is the KIND the strong tier is -better at. The efficient tier handles routine coding, file -exploration, single-file edits, normal debugging, dependency and -environment setup, and most refactors on its own — being stuck on -those is usually temporary. Weigh the kind of stuck point: - -Escalate sooner — the stuck point exceeds the efficient tier's -capability, and a stronger model would likely break the loop: -- Cross-module or cross-codebase synthesis: the fix requires learning - a convention, contract, or invariant from elsewhere in the codebase - and applying it consistently — even when the edit itself is - single-file. -- Subtle invariants: plausible-looking fixes keep failing the same - test because the root cause hinges on a behavior contract none of - the attempts have touched. -- Root causes genuinely spanning modules, or multi-step algorithmic / - formal reasoning the agent keeps getting almost-right. - -Hold weak — the efficient tier resolves these with iteration: -- Procedural or mechanical friction: tool availability, installs, - service startup, recipe-following scaffolding, localized one-file - test fixes. - -Hold weak — no model can fix these, so escalation is pure waste: -- The blocker is external: required data or files that simply do not - exist in the environment, a permanently broken or missing service, - or requirements the environment contradicts. A stronger model - changes nothing about a missing resource. One boundary to respect: - when producing, recovering, or decoding that very artifact IS the - stated task, its absence is the work itself, not a blocker — judge - the trajectory on it like any other work. - -# Trouble patterns — escalate when you see these - -Repetition and loops (the most common way agent runs die): -- The same command or edit failing 2+ times with materially the same - error, especially with unrelated changes in between. -- Near-identical tool calls repeated, or the same files re-read, without - new information gained — including longer cycles (A -> B -> C -> A). -- Fighting the environment: repeatedly invoking a missing executable, - retrying installs that fail the same way, or trying variations of a - command the environment has already rejected, instead of adapting. - -False progress (looks like progress, is not): -- Declaring success or moving on while the latest visible evidence - (test output, exit code, error text) shows failure. -- Finishing without running the verification the task specifies, when - the task states how success is checked (e.g. "make the provided - tests pass") and running it was possible. -- A reproduction or test the agent wrote that passes trivially without - exercising the actual issue, then building on that false signal. -- The agent's stated reading of a tool result contradicts what the - result actually says (treating an error or empty output as success). - -Drift and dead ends: -- Recent activity no longer serves the task in the first user message - (e.g. polishing style while the required feature is unstarted). - A debugging detour that plausibly unblocks the task — fixing the - environment, starting a required service, investigating an error in - a dependency — is NOT drift; call drift only when the detour has - produced nothing useful for many turns AND the task's real - verification remains untouched. -- Violating an explicit task constraint (modifying files the task says - not to touch, changing the tests instead of the code under test). -- Editing or reasoning about code without ever having opened the files - the errors point to — acting on guessed file contents. -- Contradicting or re-deriving something already established earlier in - the session (forgetting its own findings). -- Many turns elapsed with nothing durable produced (no successful - writes, no passing checks) and no visible narrowing of the problem — - the run is on pace to exhaust its turn budget. - -Desperation: -- Giving up: declaring the task impossible, asking to stop, or drifting - into restating the problem instead of acting on it. -- Destructive flailing: rm -rf, wholesale reinstalls, chmod -R, or - reverting everything as a reaction to being stuck rather than a - reasoned step. - -# Expected friction — do NOT escalate on these - -Agentic coding is full of failures that are part of healthy work: -- A test written to fail first (TDD) or a bug being reproduced on - purpose. -- A compile, lint, or test error fixed or meaningfully acted on in the - immediately following turn. -- Exploration dead-ends early in a session (grep with no matches, - reading a file that turns out to be irrelevant) while the agent is - still orienting. -- A missing tool handled adaptively (tries `rg`, falls back to `grep`). -- Sequential alternatives: trying a DIFFERENT library, tool, or - approach after one fails is adaptation, not a loop — even when - several alternatives fail in a row. The loop pattern requires the - SAME approach retried without material change. -- A service that is unreachable or not yet running (server not - started, port closed, connection refused) while the agent is still - actively working to start, configure, or replace it. -- Planning activity: todo-list and plan updates (TodoWrite, - update_plan) are routine agent workflow — planning is neither drift - nor struggle, and harness-injected skill/instruction reading early - in a session is orientation, not off-task work. -- Zero-count summaries: "0 failed", "0 errors", "0 warnings" are - CLEAN results. Read failure keywords together with their counts — - only a nonzero count is a failure. -- A long-running command (build, install, test suite) that simply has - not finished, or the agent waiting on information it asked for. - -The distinguishing question: is each failure producing new information -that changes the next action? Failing forward is fine; failing in place -is trouble. Also weigh the session's own recovery record: if this same -session already shows friction the agent subsequently cleared (a failure -followed by a verified fix or passing check), lean toward holding — a -session that has recovered before will usually recover again. - -# Worked examples (none drawn from any benchmark task set) - -* Turn 3; the agent ran the test suite, 4 tests fail, and it is now - reading the first failing test. -> {"escalate": false} — reproducing - failures is the job. -* The agent has run `pytest tests/test_api.py` 4 times with the same - ImportError, editing an unrelated config file between attempts. -> - {"escalate": true, "reason": "same ImportError 4 times while editing - unrelated files"} -* `conda` is not installed; the agent has tried `conda install` five - ways instead of using the `pip` that earlier output showed present. - -> {"escalate": true, "reason": "fighting missing executable instead - of adapting"} -* Task: "make the provided integration tests pass." Recent turns: - renaming variables and reformatting docstrings; tests not run in 8 - turns. -> {"escalate": true, "reason": "drifted to cosmetic edits, - verification abandoned"} -* The agent says "All tests pass, task complete" but the last visible - test output shows "2 failed, 11 passed". -> {"escalate": true, - "reason": "claims success contradicted by latest test output"} -* The agent wrote a reproduction script that exits 0 without invoking - the code path the issue describes, concluded "bug not reproducible", - and is wrapping up. -> {"escalate": true, "reason": "reproduction - never exercised the reported code path"} -* Two turns of edits, one failed build, then a fixed build and a - passing test. -> {"escalate": false} -* `npm install` has been running for one turn with no output yet. -> - {"escalate": false} — slow command, not a stall. -* Four different serialization libraries failed to import; the agent - is now writing the converter with a fifth approach it has not tried - before. -> {"escalate": false} — sequential alternatives are - adaptation, even when none has succeeded yet. -* Task: tune a slow batch pipeline. The agent is investigating why - the message broker fails to start, since the pipeline cannot be - measured without it. -> {"escalate": false} — unblocking - verification serves the task. - -Do not emit markdown, commentary, or chain-of-thought — only the JSON -object. -""" + +def _load_system_prompt() -> str: + """Read the default judge system prompt from the prompts/ package-data file. + + The file carries the benchmarked line-of-record prompt byte-exact; its + content is hash-pinned in tests, so edits require deliberate re-validation. + """ + return ( + files("switchyard.lib.processors.prompts") + .joinpath("escalation_judge.md") + .read_text(encoding="utf-8") + ) + + +ESCALATION_JUDGE_SYSTEM_PROMPT: str = _load_system_prompt() class EscalationVerdict(BaseModel): diff --git a/switchyard/lib/processors/prompts/escalation_judge.md b/switchyard/lib/processors/prompts/escalation_judge.md new file mode 100644 index 00000000..6d57369b --- /dev/null +++ b/switchyard/lib/processors/prompts/escalation_judge.md @@ -0,0 +1,178 @@ +You are an escalation judge inside an agentic coding router. The session +started on the EFFICIENT tier (a cheap but top-class 2026 model). Your +job is to detect when the run is genuinely in trouble so the router can +escalate the rest of the task to the STRONG tier (frontier, expensive). + +You see a condensed view of one session: the task framing (system prompt ++ first user message) and the most recent turns of activity (assistant +messages and tool results). Judge the *trajectory* — is the agent making +real progress toward the stated task — not the difficulty of the task +itself. Return exactly one JSON object: + +{"escalate": boolean, "reason": "one short sentence naming the pattern"} + +Escalation is one-way for the rest of the task and expensive. Escalate +only on a clear PATTERN of trouble, never on a single failed command. +When the evidence is thin or ambiguous, return {"escalate": false}. + +The bar is not "is there friction" — agentic coding is full of friction +the efficient tier works through on its own. The bar is "is this run +likely DOOMED without intervention": the agent is stuck in place and +its recent behavior shows no mechanism by which the next few turns +would look different. + +# Is the stuck point beyond the efficient tier? + +Escalation pays only when the trouble is the KIND the strong tier is +better at. The efficient tier handles routine coding, file +exploration, single-file edits, normal debugging, dependency and +environment setup, and most refactors on its own — being stuck on +those is usually temporary. Weigh the kind of stuck point: + +Escalate sooner — the stuck point exceeds the efficient tier's +capability, and a stronger model would likely break the loop: +- Cross-module or cross-codebase synthesis: the fix requires learning + a convention, contract, or invariant from elsewhere in the codebase + and applying it consistently — even when the edit itself is + single-file. +- Subtle invariants: plausible-looking fixes keep failing the same + test because the root cause hinges on a behavior contract none of + the attempts have touched. +- Root causes genuinely spanning modules, or multi-step algorithmic / + formal reasoning the agent keeps getting almost-right. + +Hold weak — the efficient tier resolves these with iteration: +- Procedural or mechanical friction: tool availability, installs, + service startup, recipe-following scaffolding, localized one-file + test fixes. + +Hold weak — no model can fix these, so escalation is pure waste: +- The blocker is external: required data or files that simply do not + exist in the environment, a permanently broken or missing service, + or requirements the environment contradicts. A stronger model + changes nothing about a missing resource. One boundary to respect: + when producing, recovering, or decoding that very artifact IS the + stated task, its absence is the work itself, not a blocker — judge + the trajectory on it like any other work. + +# Trouble patterns — escalate when you see these + +Repetition and loops (the most common way agent runs die): +- The same command or edit failing 2+ times with materially the same + error, especially with unrelated changes in between. +- Near-identical tool calls repeated, or the same files re-read, without + new information gained — including longer cycles (A -> B -> C -> A). +- Fighting the environment: repeatedly invoking a missing executable, + retrying installs that fail the same way, or trying variations of a + command the environment has already rejected, instead of adapting. + +False progress (looks like progress, is not): +- Declaring success or moving on while the latest visible evidence + (test output, exit code, error text) shows failure. +- Finishing without running the verification the task specifies, when + the task states how success is checked (e.g. "make the provided + tests pass") and running it was possible. +- A reproduction or test the agent wrote that passes trivially without + exercising the actual issue, then building on that false signal. +- The agent's stated reading of a tool result contradicts what the + result actually says (treating an error or empty output as success). + +Drift and dead ends: +- Recent activity no longer serves the task in the first user message + (e.g. polishing style while the required feature is unstarted). + A debugging detour that plausibly unblocks the task — fixing the + environment, starting a required service, investigating an error in + a dependency — is NOT drift; call drift only when the detour has + produced nothing useful for many turns AND the task's real + verification remains untouched. +- Violating an explicit task constraint (modifying files the task says + not to touch, changing the tests instead of the code under test). +- Editing or reasoning about code without ever having opened the files + the errors point to — acting on guessed file contents. +- Contradicting or re-deriving something already established earlier in + the session (forgetting its own findings). +- Many turns elapsed with nothing durable produced (no successful + writes, no passing checks) and no visible narrowing of the problem — + the run is on pace to exhaust its turn budget. + +Desperation: +- Giving up: declaring the task impossible, asking to stop, or drifting + into restating the problem instead of acting on it. +- Destructive flailing: rm -rf, wholesale reinstalls, chmod -R, or + reverting everything as a reaction to being stuck rather than a + reasoned step. + +# Expected friction — do NOT escalate on these + +Agentic coding is full of failures that are part of healthy work: +- A test written to fail first (TDD) or a bug being reproduced on + purpose. +- A compile, lint, or test error fixed or meaningfully acted on in the + immediately following turn. +- Exploration dead-ends early in a session (grep with no matches, + reading a file that turns out to be irrelevant) while the agent is + still orienting. +- A missing tool handled adaptively (tries `rg`, falls back to `grep`). +- Sequential alternatives: trying a DIFFERENT library, tool, or + approach after one fails is adaptation, not a loop — even when + several alternatives fail in a row. The loop pattern requires the + SAME approach retried without material change. +- A service that is unreachable or not yet running (server not + started, port closed, connection refused) while the agent is still + actively working to start, configure, or replace it. +- Planning activity: todo-list and plan updates (TodoWrite, + update_plan) are routine agent workflow — planning is neither drift + nor struggle, and harness-injected skill/instruction reading early + in a session is orientation, not off-task work. +- Zero-count summaries: "0 failed", "0 errors", "0 warnings" are + CLEAN results. Read failure keywords together with their counts — + only a nonzero count is a failure. +- A long-running command (build, install, test suite) that simply has + not finished, or the agent waiting on information it asked for. + +The distinguishing question: is each failure producing new information +that changes the next action? Failing forward is fine; failing in place +is trouble. Also weigh the session's own recovery record: if this same +session already shows friction the agent subsequently cleared (a failure +followed by a verified fix or passing check), lean toward holding — a +session that has recovered before will usually recover again. + +# Worked examples (none drawn from any benchmark task set) + +* Turn 3; the agent ran the test suite, 4 tests fail, and it is now + reading the first failing test. -> {"escalate": false} — reproducing + failures is the job. +* The agent has run `pytest tests/test_api.py` 4 times with the same + ImportError, editing an unrelated config file between attempts. -> + {"escalate": true, "reason": "same ImportError 4 times while editing + unrelated files"} +* `conda` is not installed; the agent has tried `conda install` five + ways instead of using the `pip` that earlier output showed present. + -> {"escalate": true, "reason": "fighting missing executable instead + of adapting"} +* Task: "make the provided integration tests pass." Recent turns: + renaming variables and reformatting docstrings; tests not run in 8 + turns. -> {"escalate": true, "reason": "drifted to cosmetic edits, + verification abandoned"} +* The agent says "All tests pass, task complete" but the last visible + test output shows "2 failed, 11 passed". -> {"escalate": true, + "reason": "claims success contradicted by latest test output"} +* The agent wrote a reproduction script that exits 0 without invoking + the code path the issue describes, concluded "bug not reproducible", + and is wrapping up. -> {"escalate": true, "reason": "reproduction + never exercised the reported code path"} +* Two turns of edits, one failed build, then a fixed build and a + passing test. -> {"escalate": false} +* `npm install` has been running for one turn with no output yet. -> + {"escalate": false} — slow command, not a stall. +* Four different serialization libraries failed to import; the agent + is now writing the converter with a fifth approach it has not tried + before. -> {"escalate": false} — sequential alternatives are + adaptation, even when none has succeeded yet. +* Task: tune a slow batch pipeline. The agent is investigating why + the message broker fails to start, since the pipeline cannot be + measured without it. -> {"escalate": false} — unblocking + verification serves the task. + +Do not emit markdown, commentary, or chain-of-thought — only the JSON +object. diff --git a/tests/test_escalation_judge_request_processor.py b/tests/test_escalation_judge_request_processor.py index a8e3968d..7c1cc3e6 100644 --- a/tests/test_escalation_judge_request_processor.py +++ b/tests/test_escalation_judge_request_processor.py @@ -5,6 +5,7 @@ from __future__ import annotations +import hashlib import json from types import SimpleNamespace from typing import Any, cast @@ -14,6 +15,7 @@ ) from switchyard.lib.processors.escalation_judge_request_processor import ( CTX_ESCALATION_VERDICT, + ESCALATION_JUDGE_SYSTEM_PROMPT, EscalationJudgeConfig, EscalationJudgeRequestProcessor, ) @@ -412,3 +414,22 @@ def _trial(first_response: str, extra_turns: int = 2) -> ChatRequest: await processor.process(ctx2, _trial("trial two path")) assert ctx2.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "weak" assert len(fake.calls) == 2 + + +def test_default_system_prompt_is_the_line_of_record() -> None: + """Pin the packaged default prompt byte-exact. + + The prompt is loaded from ``prompts/escalation_judge.md`` package data + and is the benchmark-validated line of record. If this test fails, the + file changed: either revert, or re-validate the new prompt on a full + benchmark run and update the digest deliberately. + """ + digest = hashlib.sha256( + ESCALATION_JUDGE_SYSTEM_PROMPT.encode("utf-8") + ).hexdigest() + assert digest == ( + "69610eeecbac59fc20c7933ef57fa21029fa80c374b3a399b1f1a9557a6de234" + ) + # Loader must not normalize: the wire bytes are part of the record. + assert ESCALATION_JUDGE_SYSTEM_PROMPT.endswith("\n") + assert not ESCALATION_JUDGE_SYSTEM_PROMPT.startswith("\n") diff --git a/tests/test_route_bundle.py b/tests/test_route_bundle.py index 1ada519b..3d2a7348 100644 --- a/tests/test_route_bundle.py +++ b/tests/test_route_bundle.py @@ -828,6 +828,53 @@ def test_judge_settings_thread_into_processor(self): assert judge._config.timeout_s == 5.0 assert judge._session_key_depth == 2 + def test_judge_prompt_path_reads_file(self, tmp_path): + from switchyard.cli.route_bundle import build_route_bundle_table + from switchyard.lib.processors.escalation_judge_request_processor import ( + EscalationJudgeRequestProcessor, + ) + prompt_file = tmp_path / "judge_prompt.md" + prompt_file.write_text("file-based judge prompt\n", encoding="utf-8") + bundle = self._bundle() + bundle["routes"]["myrouter/escalation"]["judge"]["prompt_path"] = ( + str(prompt_file) + ) + table = build_route_bundle_table(bundle) + switchyard = table.lookup_switchyard("myrouter/escalation") + judge = next( + c for c in switchyard.iter_components() + if isinstance(c, EscalationJudgeRequestProcessor) + ) + assert judge._config.system_prompt == "file-based judge prompt\n" + + def test_judge_prompt_and_prompt_path_are_mutually_exclusive(self, tmp_path): + from switchyard.cli.route_bundle import ( + RouteBundleConfigError, + build_route_bundle_table, + ) + prompt_file = tmp_path / "judge_prompt.md" + prompt_file.write_text("file prompt", encoding="utf-8") + bundle = self._bundle() + judge = bundle["routes"]["myrouter/escalation"]["judge"] + judge["prompt"] = "inline prompt" + judge["prompt_path"] = str(prompt_file) + with pytest.raises(RouteBundleConfigError) as exc: + build_route_bundle_table(bundle) + assert "mutually exclusive" in str(exc.value) + + def test_judge_prompt_path_missing_file_is_a_config_error(self, tmp_path): + from switchyard.cli.route_bundle import ( + RouteBundleConfigError, + build_route_bundle_table, + ) + bundle = self._bundle() + bundle["routes"]["myrouter/escalation"]["judge"]["prompt_path"] = ( + str(tmp_path / "nope.md") + ) + with pytest.raises(RouteBundleConfigError) as exc: + build_route_bundle_table(bundle) + assert "prompt_path" in str(exc.value) + def test_rejects_missing_judge_block(self): from switchyard.cli.route_bundle import ( RouteBundleConfigError, From 7495a39b799f1ae50c656753fabc6ba98ee86b23 Mon Sep 17 00:00:00 2001 From: Lin Jia Date: Wed, 15 Jul 2026 15:06:45 -0700 Subject: [PATCH 06/23] refactor(router): reuse session-affinity key resolution for judge streak bookkeeping --- .../escalation_judge_request_processor.py | 12 +++++----- switchyard/lib/session_affinity.py | 24 +++++++++++++------ 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/switchyard/lib/processors/escalation_judge_request_processor.py b/switchyard/lib/processors/escalation_judge_request_processor.py index 0b26fcd5..0f7b4f97 100644 --- a/switchyard/lib/processors/escalation_judge_request_processor.py +++ b/switchyard/lib/processors/escalation_judge_request_processor.py @@ -28,8 +28,11 @@ _trim_messages, ) from switchyard.lib.proxy_context import ProxyContext -from switchyard.lib.session_affinity import CTX_SESSION_KEY, SessionAffinity -from switchyard.lib.session_key import session_key_from_body +from switchyard.lib.session_affinity import ( + CTX_SESSION_KEY, + SessionAffinity, + resolve_session_key, +) from switchyard.lib.stats_accumulator import StatsAccumulator from switchyard_rust.core import ChatRequest @@ -309,10 +312,7 @@ async def process(self, ctx: ProxyContext, request: ChatRequest) -> ChatRequest: def _streak_key(self, ctx: ProxyContext, request: ChatRequest) -> str: """Conversation key for streak bookkeeping (seeded deep key or Rust default).""" - cached = ctx.metadata.get(CTX_SESSION_KEY) - if isinstance(cached, str): - return cached - return session_key_from_body(request.body) + return resolve_session_key(ctx, request) def _bump_streak(self, ctx: ProxyContext, request: ChatRequest) -> int: """Record one escalate verdict for this conversation; return the streak.""" diff --git a/switchyard/lib/session_affinity.py b/switchyard/lib/session_affinity.py index eca46d2e..15139bf6 100644 --- a/switchyard/lib/session_affinity.py +++ b/switchyard/lib/session_affinity.py @@ -35,6 +35,21 @@ CTX_SESSION_KEY = "_session_affinity_key" +def resolve_session_key(ctx: ProxyContext, request: ChatRequest) -> str: + """Derive the conversation key once per request, memoized on ``ctx``. + + Components that pre-seed a custom key into ``CTX_SESSION_KEY`` (e.g. the + escalation judge's deep benchmark key) are respected: the seeded value + wins over re-derivation. + """ + cached = ctx.metadata.get(CTX_SESSION_KEY) + if isinstance(cached, str): + return cached + key = session_key_from_body(request.body) + ctx.metadata[CTX_SESSION_KEY] = key + return key + + class SessionAffinity: """Pins a routing decision per conversation and reuses it on later turns. @@ -238,16 +253,11 @@ def __len__(self) -> int: def _session_key(self, ctx: ProxyContext, request: ChatRequest) -> str: """Derive the conversation key once per request, memoized on ``ctx``.""" - cached = ctx.metadata.get(CTX_SESSION_KEY) - if isinstance(cached, str): - return cached - key = session_key_from_body(request.body) - ctx.metadata[CTX_SESSION_KEY] = key - return key + return resolve_session_key(ctx, request) def _is_warmup_turn(self, request: ChatRequest) -> bool: """Return whether this request is still inside the no-stick warmup.""" return conversation_turn_number(request) <= self._warmup_turns -__all__ = ["CTX_SESSION_KEY", "SessionAffinity"] +__all__ = ["CTX_SESSION_KEY", "SessionAffinity", "resolve_session_key"] From 04b0f6eff3fabc6e27aef94987b0fdab58ff7755 Mon Sep 17 00:00:00 2001 From: Lin Jia Date: Wed, 15 Jul 2026 15:06:45 -0700 Subject: [PATCH 07/23] docs(cli): tighten the escalation route-bundle docstring, point at the routing guide --- switchyard/cli/route_bundle.py | 31 +++++-------------------------- 1 file changed, 5 insertions(+), 26 deletions(-) diff --git a/switchyard/cli/route_bundle.py b/switchyard/cli/route_bundle.py index 4f8dfd3c..cd108ee3 100644 --- a/switchyard/cli/route_bundle.py +++ b/switchyard/cli/route_bundle.py @@ -1088,32 +1088,11 @@ def _escalation_router_switchyard( """Build the judge-latched escalation-routing chain for a route. Every conversation starts on ``weak``; an LLM judge watches the - trajectory each turn and, on a clear pattern of trouble, latches the - conversation to ``strong`` for the rest of the task. - - YAML schema:: - - type: escalation_router - judge: - model: google/gemini-3.5-flash - api_key: ${OPENROUTER_API_KEY} - base_url: https://openrouter.ai/api/v1 - timeout_secs: 5.0 - min_turn: 3 # first turn the judge runs on - confirmations: 1 # consecutive escalate verdicts to latch - confirmation_window: 1 # >1 tolerates declines between fires - disable_reasoning: true # false lets a reasoning judge think - recent_turn_window: 14 # trailing messages shown to the judge - strong: - model: anthropic/claude-opus-4.7 - api_key: ${OPENROUTER_API_KEY} - base_url: https://openrouter.ai/api/v1 - weak: - model: deepseek/deepseek-v4-pro - api_key: ${OPENROUTER_API_KEY} - base_url: https://openrouter.ai/api/v1 - fallback_target_on_evict: strong - session_key_depth: 0 # >0 only for k>1 benchmark trials + trajectory and, on a clear pattern of trouble, latches the conversation + to ``strong`` for the rest of the task. Requires ``judge``/``strong``/ + ``weak`` targets plus ``fallback_target_on_evict``; the full YAML schema + and knob reference live in ``docs/routing_algorithms/ + escalation_router_routing.md``. """ judge_raw = route.get("judge") if not isinstance(judge_raw, Mapping): From 92072b50111af8e6434861e54124008c0f181792 Mon Sep 17 00:00:00 2001 From: Lin Jia Date: Wed, 15 Jul 2026 15:06:45 -0700 Subject: [PATCH 08/23] fix(router): wire judge_timeout_s through the escalation profile build; hoist deferred imports --- .../escalation_router_profile_config.py | 49 +++++++++---------- tests/test_escalation_router_profile.py | 2 + 2 files changed, 26 insertions(+), 25 deletions(-) diff --git a/switchyard/lib/profiles/escalation_router_profile_config.py b/switchyard/lib/profiles/escalation_router_profile_config.py index d3d42dd9..1f9d9cc2 100644 --- a/switchyard/lib/profiles/escalation_router_profile_config.py +++ b/switchyard/lib/profiles/escalation_router_profile_config.py @@ -7,9 +7,32 @@ from typing import Any, Self +from switchyard.lib.backends.anthropic_cache_breakpoint_backend import ( + maybe_wrap_anthropic_cache, +) +from switchyard.lib.backends.deterministic_routing_llm_backend import ( + DeterministicRoutingLLMBackend, +) +from switchyard.lib.backends.multi_llm_backend import ( + build_native_backend, + resolve_llm_target, +) +from switchyard.lib.processors.escalation_judge_request_processor import ( + ESCALATION_JUDGE_SYSTEM_PROMPT, + EscalationJudgeConfig, + EscalationJudgeRequestProcessor, +) +from switchyard.lib.processors.reasoning_effort_normalizer import ( + ReasoningEffortNormalizer, +) from switchyard.lib.profiles.chain import ComponentChainProfile +from switchyard.lib.profiles.deterministic_routing_profile_config import ( + _apply_deepseek_overrides, + _apply_default_tier_timeout, +) from switchyard.lib.profiles.escalation_router_config import EscalationRouterConfig from switchyard.lib.profiles.table import profile_config +from switchyard.lib.session_affinity import SessionAffinity _TIER_STRONG = "strong" _TIER_WEAK = "weak" @@ -28,30 +51,6 @@ def from_config(cls, config: EscalationRouterConfig) -> Self: def build(self) -> ComponentChainProfile: """Build the escalation-router profile runtime.""" - from switchyard.lib.backends.anthropic_cache_breakpoint_backend import ( - maybe_wrap_anthropic_cache, - ) - from switchyard.lib.backends.deterministic_routing_llm_backend import ( - DeterministicRoutingLLMBackend, - ) - from switchyard.lib.backends.multi_llm_backend import ( - build_native_backend, - resolve_llm_target, - ) - from switchyard.lib.processors.escalation_judge_request_processor import ( - ESCALATION_JUDGE_SYSTEM_PROMPT, - EscalationJudgeConfig, - EscalationJudgeRequestProcessor, - ) - from switchyard.lib.processors.reasoning_effort_normalizer import ( - ReasoningEffortNormalizer, - ) - from switchyard.lib.profiles.deterministic_routing_profile_config import ( - _apply_deepseek_overrides, - _apply_default_tier_timeout, - ) - from switchyard.lib.session_affinity import SessionAffinity - config = self.config # The affinity store IS the escalation latch, so it is always on. @@ -72,7 +71,7 @@ def build(self) -> ComponentChainProfile: model=judge_target.model, api_key=judge_target.api_key, base_url=judge_target.base_url, - timeout_s=judge_target.endpoint.timeout_secs or 5.0, + timeout_s=judge_target.endpoint.timeout_secs or config.judge_timeout_s, system_prompt=config.judge_system_prompt or ESCALATION_JUDGE_SYSTEM_PROMPT, min_judge_turn=config.judge_min_turn, escalate_confirmations=config.judge_escalate_confirmations, diff --git a/tests/test_escalation_router_profile.py b/tests/test_escalation_router_profile.py index afc29190..772d7b6e 100644 --- a/tests/test_escalation_router_profile.py +++ b/tests/test_escalation_router_profile.py @@ -109,6 +109,7 @@ def test_build_threads_judge_settings() -> None: judge_recent_turn_window=20, judge_window_message_chars=500, judge_disable_reasoning=False, + judge_timeout_s=12.0, session_key_depth=2, ), ).build() @@ -119,6 +120,7 @@ def test_build_threads_judge_settings() -> None: assert judge._config.recent_turn_window == 20 assert judge._config.window_message_chars == 500 assert judge._config.disable_reasoning is False + assert judge._config.timeout_s == 12.0 assert judge._config.system_prompt == ESCALATION_JUDGE_SYSTEM_PROMPT assert judge._session_key_depth == 2 assert judge._affinity.enabled From 0e3bc56bda1f0eeb7710bf820682f3303693880d Mon Sep 17 00:00:00 2001 From: Lin Jia Date: Wed, 15 Jul 2026 15:06:45 -0700 Subject: [PATCH 09/23] docs(router): clarify the judge starts at a configurable minimum turn --- docs/routing_algorithms/overview.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/routing_algorithms/overview.md b/docs/routing_algorithms/overview.md index 2a6a3bb1..386c1d6a 100644 --- a/docs/routing_algorithms/overview.md +++ b/docs/routing_algorithms/overview.md @@ -155,8 +155,8 @@ continue to make a routing decision for each request. The escalation router uses the same affinity store with the opposite policy: affinity is always on (the pin *is* the escalation latch), only the strong -tier is ever pinned, and the judge keeps running every turn until the latch -fires. See +tier is ever pinned, and once the judge starts (a configurable minimum turn), +it keeps running each judged turn until the latch fires. See [Escalation-Router Routing](escalation_router_routing.md). !!! note "CLI schema availability" From 5ece2a5eca6bd4b4e19f61f81ec196402f5d5de4 Mon Sep 17 00:00:00 2001 From: Lin Jia Date: Wed, 15 Jul 2026 15:07:05 -0700 Subject: [PATCH 10/23] docs(router): fix repeated-trial benchmarking anchor (MD051) --- docs/routing_algorithms/escalation_router_routing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/routing_algorithms/escalation_router_routing.md b/docs/routing_algorithms/escalation_router_routing.md index 876028fb..17137ce9 100644 --- a/docs/routing_algorithms/escalation_router_routing.md +++ b/docs/routing_algorithms/escalation_router_routing.md @@ -71,7 +71,7 @@ routes: | `judge.prompt_path` | built-in | Judge system-prompt override read from a file; mutually exclusive with `judge.prompt`. Relative paths resolve against the server's working directory. | | `judge.max_request_chars` | `12000` | Cap on the judge transcript; oldest window messages are dropped first. | | `fallback_target_on_evict` | required | `strong` or `weak`; context-window-eviction reroute target. | -| `session_key_depth` | `0` | See [Repeated-trial benchmarking](#repeated-trial-benchmarking) below. | +| `session_key_depth` | `0` | See [Repeated-trial benchmarking](#repeated-trial-benchmarking-k1) below. | | `tier_timeout_s` | `600` | Default per-call timeout for strong/weak targets without their own `timeout_secs`. | | `affinity_max_sessions` | `10000` | LRU capacity of the escalation latch. | From 3caf3fee0d7ab18930c568a495a2da607eacb111 Mon Sep 17 00:00:00 2001 From: Lin Jia Date: Mon, 13 Jul 2026 16:31:43 -0700 Subject: [PATCH 11/23] docs(router): align escalation routing guide Signed-off-by: Lin Jia --- .../escalation_router_routing.md | 263 +++++++++++------- 1 file changed, 170 insertions(+), 93 deletions(-) diff --git a/docs/routing_algorithms/escalation_router_routing.md b/docs/routing_algorithms/escalation_router_routing.md index 17137ce9..16cd98b5 100644 --- a/docs/routing_algorithms/escalation_router_routing.md +++ b/docs/routing_algorithms/escalation_router_routing.md @@ -1,47 +1,84 @@ # Escalation-Router Routing -The escalation router starts every conversation on the cheap **weak** tier. An -LLM **judge** watches the trajectory each turn and, when the run shows a clear -pattern of trouble (repeated errors, loops, false progress, drift, giving up), -escalates the conversation to the **strong** tier — one-way for the rest of -the task. A new conversation starts weak again. - -Unlike [LLM Classifier Routing](llm_classifier_routing.md), which predicts -per-turn difficulty from request content, the escalation router judges whether -the work is *going well*. Hard-but-smooth runs stay on the weak tier; runs the -weak tier is fumbling get rescued by the strong tier. +Escalation-router routing starts each conversation on a cheaper `weak` model. +An LLM judge watches how the work progresses and moves the conversation to a +more capable `strong` model when it detects sustained, recoverable trouble. +After escalation, the conversation stays on the strong tier for the rest of +the task. + +Use it for multi-turn agent workloads where a weak model can handle routine +work but may need rescue after repeated errors, loops, drift, false progress, +or premature completion. Unlike +[LLM Classifier Routing](llm_classifier_routing.md), which predicts how +difficult a request looks, escalation routing judges whether the run is +actually going well. ## How it works -Per turn, the `EscalationJudgeRequestProcessor`: +The router keeps one state value per conversation: whether that conversation +has escalated. Conversations are identified from the stable system prompt and +first user message, using the same bounded in-memory store described in +[Sticky Routing](sticky_routing.md). + +For each turn, Switchyard: + +1. Checks the conversation latch. A latched conversation routes to `strong` + without calling the judge again. +2. Routes an unlatched conversation to `weak`. Before `judge.min_turn` + (default `3`), it skips the judge because there is too little trajectory to + assess. +3. Gives the judge a bounded summary containing the system and first-user + anchors, recent messages, and a coverage note for omitted history. +4. Parses the judge's structured `escalate` decision. When the configured + confirmation policy is satisfied, Switchyard pins the conversation to + `strong` and uses the strong model for the current turn. +5. Fails open to `weak` when the judge times out, errors, or returns invalid + output. A judge failure never creates a strong-tier pin. + +The routing decision for one turn is: + +```mermaid +%%{init: {"flowchart": {"nodeSpacing": 18, "rankSpacing": 26}}}%% +flowchart LR + t["turn"] --> p{"strong tier pinned?"} + p -->|yes| s["route strong; skip judge"] + p -->|no| m{"turn >= min_turn?"} + m -->|no| w["route weak"] + m -->|yes| j["judge recent trajectory"] + j -->|stay / fail open| w + j -->|not yet confirmed| w + j -->|confirmed escalation| l["pin and route strong"] + + classDef box font-family:monospace,fill:none,stroke:#9aa0a6,stroke-width:1px; + class t,p,s,m,w,j,l box; +``` + +## Confirmation policy -1. Checks the session-affinity latch. A pinned conversation routes strong with - no judge call — the latch is one-way per task. -2. Otherwise routes weak. From `judge.min_turn` on (default 3), it sends the - judge a condensed transcript: the system + first-user anchors (individually - capped so harness boilerplate cannot crowd out the evidence), the last - `judge.recent_turn_window` messages (head/tail-truncated), and a coverage - header stating how much history is not shown. -3. The judge returns `{"escalate": bool, "reason": ""}`. On - `escalate: true` the conversation is pinned to strong, effective the same - turn. The reason is logged and stamped into the request metadata - (`_escalation_verdict`) as the audit record. -4. Any judge failure — timeout, error, invalid JSON — fails open to the weak - tier and never pins. A judge outage costs quality risk, never money. +`judge.confirmations` controls how many positive verdicts are required before +the strong-tier latch fires: -The latch reuses the [session-affinity](sticky_routing.md) store: conversations -are keyed on the stable prefix (system prompt + first user message), so a new -task gets a fresh key and resets to weak automatically. The weak tier is never -pinned — "not pinned" means weak-and-still-watching. +- `confirmations: 1` is the default and escalates on the first positive + verdict. +- `confirmations: 2` with `confirmation_window: 1` requires positive verdicts + on consecutive judged turns. +- A larger `confirmation_window` allows recurring trouble to confirm even when + negative verdicts occur between positive ones. A positive verdict remains + live across at most `confirmation_window - 1` intervening negative verdicts. -## Configuration +A judge failure provides no evidence either way: it routes the turn to weak +without clearing an existing confirmation streak. -The CLI accepts `type: escalation_router` in a `routes:` bundle loaded with -`--routing-profiles` (same path the `deterministic` router uses): +## Configure an escalation route + +Escalation routing is currently available through the legacy `routes:` bundle +used by `--routing-profiles`. This path remains necessary for launcher-owned +routing; route bundles and `--routing-profiles` are otherwise deprecated in +favor of `switchyard serve --config` profiles. ```yaml routes: - switchyard: + agent-escalation: type: escalation_router fallback_target_on_evict: strong judge: @@ -49,73 +86,113 @@ routes: api_key: ${OPENROUTER_API_KEY} base_url: https://openrouter.ai/api/v1 timeout_secs: 5.0 - min_turn: 3 # first turn the judge runs on - recent_turn_window: 14 # trailing messages shown to the judge - strong: - model: anthropic/claude-opus-4.7 + min_turn: 3 + confirmations: 1 + confirmation_window: 1 + recent_turn_window: 14 + weak: + model: moonshotai/kimi-k2.6 api_key: ${OPENROUTER_API_KEY} base_url: https://openrouter.ai/api/v1 - weak: - model: deepseek/deepseek-v4-pro + strong: + model: anthropic/claude-opus-4.7 api_key: ${OPENROUTER_API_KEY} base_url: https://openrouter.ai/api/v1 ``` -| Key | Default | Meaning | +Run the route as a standalone proxy: + +```bash +switchyard --routing-profiles routes.yaml -- serve --port 4000 +``` + +Or use the same bundle with an agent launcher: + +```bash +switchyard --routing-profiles routes.yaml -- launch claude +``` + +The route ID (`agent-escalation`) is the model ID clients select to use the +router. The strong and weak model IDs are also registered as direct +passthrough choices. The judge is internal to the route and is not exposed as +a client-selectable model. + +If the selected tier exceeds its context window, Switchyard retries once on +`fallback_target_on_evict`, which must be `strong` or `weak`. See +[Context-Window Handling](../operations/context_window.md). + +## Useful options + +| Option | Default | Use it when | |---|---|---| -| `judge.model` / `api_key` / `base_url` | required | Judge LLM target. Pick something small and fast — it sits on the request path of every pre-escalation turn. | -| `judge.timeout_secs` | `5.0` | Judge wall-clock ceiling; fails open to weak. | -| `judge.min_turn` | `3` | First conversation turn the judge runs on. Earlier turns have no trajectory to judge. | -| `judge.recent_turn_window` | `14` | Trailing messages shown to the judge. Loops longer than the window are invisible — widen before concluding the judge misses them. | -| `judge.prompt` | built-in | Judge system-prompt override, inlined in the YAML. | -| `judge.prompt_path` | built-in | Judge system-prompt override read from a file; mutually exclusive with `judge.prompt`. Relative paths resolve against the server's working directory. | -| `judge.max_request_chars` | `12000` | Cap on the judge transcript; oldest window messages are dropped first. | -| `fallback_target_on_evict` | required | `strong` or `weak`; context-window-eviction reroute target. | -| `session_key_depth` | `0` | See [Repeated-trial benchmarking](#repeated-trial-benchmarking-k1) below. | -| `tier_timeout_s` | `600` | Default per-call timeout for strong/weak targets without their own `timeout_secs`. | -| `affinity_max_sessions` | `10000` | LRU capacity of the escalation latch. | - -In Python, build it from the exported config pair: - -```python -from switchyard import EscalationRouterConfig, EscalationRouterProfileConfig - -profile = EscalationRouterProfileConfig.from_config( - EscalationRouterConfig.model_validate({...}) -).build() +| `judge.timeout_secs` | `5.0` | The judge needs a different wall-clock limit. Timeouts fail open to weak. | +| `judge.min_turn` | `3` | The judge should start earlier or wait for more trajectory evidence. | +| `judge.confirmations` | `1` | One positive verdict is too eager and escalation should require repeated evidence. | +| `judge.confirmation_window` | `1` | Intermittent trouble should remain eligible for confirmation across negative verdicts. | +| `judge.disable_reasoning` | `true` | Set to `false` when a reasoning judge benefits from thinking despite the added latency. | +| `judge.recent_turn_window` | `14` | The judge needs a wider or narrower trailing-message window. | +| `judge.window_message_chars` | `300` | More tool-output detail should survive per-message truncation. | +| `judge.max_request_chars` | `12000` | The complete judge request needs a different character budget. Oldest recent messages are dropped first. | +| `judge.prompt` | built-in | A deployment needs a custom escalation rubric inlined in the YAML. | +| `judge.prompt_path` | built-in | The custom rubric lives in a file (mutually exclusive with `judge.prompt`; relative paths resolve against the server's working directory). | +| `tier_timeout_s` | `600` | Strong or weak targets without their own `timeout_secs` need a different call timeout. | +| `enable_stats` | `true` | Set to `false` only when route-level usage statistics are not needed. | +| `affinity_max_sessions` | `10000` | A long-lived process needs a different in-memory latch capacity. | +| `session_key_depth` | `0` | Repeated benchmark trials with identical task prefixes need separate latches; keep `0` for normal traffic. | + +## Observability + +Read the standard routing stats endpoint: + +```bash +curl -s http://localhost:4000/v1/routing/stats ``` -## Repeated-trial benchmarking (k>1) - -The session key is a content hash of the system prompt + first user message, -so **k repeated trials of the same task against one long-lived server share -one latch**: trial 1's escalation makes trials 2..k start on strong from turn -1, silently corrupting pass@k and cost numbers. Two ways to keep trials -independent: - -- **Run-level isolation (zero config):** run each trial set as a separate - `benchmark/run-baseline.sh` invocation. Each run gets a fresh Switchyard - container; latch state cannot survive between runs. -- **`session_key_depth: N` (within-run k>1):** extends the session key with - the first `N` post-first-user messages. With nonzero sampling temperature, - trials diverge in their early model responses and get distinct keys. The - prefix of a conversation never changes as it grows, so the key stays stable - within a trial; until the prefix is complete, affinity is untouched. - Caveats: useless at temperature 0 (identical trajectories still collide), - and mid-session history rewrites (context compaction) change the key and - silently drop the latch — keep `0` outside benchmarks. - -Also set Harbor `--max-retries 0` for escalation-router runs: a retry re-runs -the failed task against the same warm server, and failed attempts are exactly -the ones likely to have escalated, so the retry would warm-start on strong. - -## Known limits and v2 levers - -- **Blocking judge:** the judge call adds ~100–300 ms to each pre-escalation - turn. If benchmarks show latency pain, the designed successor is running the - judge concurrently with the weak call and applying the verdict next turn. -- **Window-bounded loop detection:** a repeat cycle longer than - `recent_turn_window` is invisible. The designed successor is a per-turn - trajectory digest distilled from the whole history. -- **No difficulty prediction:** this router rescues struggling runs; it does - not predict hard ones. Composing it with the LLM classifier is future work. +The snapshot reports per-model calls, tokens, latency, and cost for the strong +and weak tiers. Judge calls are recorded in the classifier stats bucket so +their token cost, latency, and errors remain visible as routing overhead. + +Each judged turn also writes an `escalation_verdict={...}` JSON line to server +stderr. The record includes the decision, reason, turn, confirmation state, +and judge latency. After the latch fires, later turns skip the judge and report +the pinned routing source in request metadata. + +## Repeated benchmark trials + +By default, the session key is derived from the system prompt and first user +message. Repeated trials of the same task against one long-lived server can +therefore share a latch: if the first trial escalates, later trials may start +on strong. + +Use one of these isolation strategies: + +- Start a fresh Switchyard process for each trial set. +- Set `session_key_depth: N` to extend the key with the first `N` messages + after the initial user message. This only separates trials when those early + trajectories differ, so it requires nonzero sampling temperature. + +Keep `session_key_depth: 0` for normal traffic. Context compaction or other +mid-session prefix rewrites can change a deep key and lose the existing latch. + +## When not to use escalation routing + +- **One-shot requests.** There is no trajectory to judge. Use + [LLM Classifier Routing](llm_classifier_routing.md) when the initial request + should determine the tier. +- **Fixed traffic experiments.** Use + [Random Routing](random_routing.md) for A/B splits and gradual traffic ramps. +- **Per-turn stage optimization.** Use + [Stage-Router Routing](stage_router_routing.md) when tool-result signals + should move individual turns in both directions. +- **Latency-critical traffic.** Eligible unlatched turns wait for the judge + before the selected backend call. +- **Long-range failure cycles.** The judge only sees a bounded recent window; + cycles longer than that window may be missed. + +## Related + +- [Routing Overview](overview.md): compare all supported routing strategies. +- [Sticky Routing](sticky_routing.md): session-key derivation and affinity + behavior. +- [Architecture](../architecture.md): the end-to-end request lifecycle and + system boundaries. From f6824de22ce2b7b6876bdc2e48b110b0cd55cc0e Mon Sep 17 00:00:00 2001 From: Lin Jia Date: Wed, 15 Jul 2026 15:54:48 -0700 Subject: [PATCH 12/23] refactor(session): fold the judge's deep benchmark key into session_key_from_body via a depth param Signed-off-by: Lin Jia --- crates/switchyard-core/src/session.rs | 173 +++++++++++++++++- .../src/core_bindings/session.rs | 17 +- .../escalation_judge_request_processor.py | 82 ++------- switchyard/lib/session_affinity.py | 26 ++- switchyard/lib/session_key.py | 25 ++- switchyard_rust/core.py | 4 +- 6 files changed, 235 insertions(+), 92 deletions(-) diff --git a/crates/switchyard-core/src/session.rs b/crates/switchyard-core/src/session.rs index c5c13698..6a8c0c1d 100644 --- a/crates/switchyard-core/src/session.rs +++ b/crates/switchyard-core/src/session.rs @@ -17,6 +17,28 @@ use serde_json::Value; /// message — so every turn of a conversation shares a key while distinct /// conversations differ. Returns a 16-char lowercase hex string. pub fn session_key_from_body(body: &Value) -> String { + hash_conversation_prefix(body, 0).0 +} + +/// Deep variant of [`session_key_from_body`]: extend the hashed prefix with +/// the first `depth` messages after the first user message, so repeated runs +/// of an identical task diverge via early model responses (repeated-trial +/// benchmarking). +/// +/// Returns `None` until the conversation actually contains a first user +/// message plus `depth` later messages — a shorter prefix would hash to a key +/// that later turns of the same conversation no longer match. `depth == 0` is +/// exactly the base key and always yields `Some`. +pub fn session_key_from_body_with_depth(body: &Value, depth: usize) -> Option { + let (key, complete) = hash_conversation_prefix(body, depth); + complete.then_some(key) +} + +/// Shared hashing core for both key variants: anchors (top-level system, +/// in-list system/developer messages, first user message) plus the first +/// `depth` non-system messages after the first user. Returns the key and +/// whether the requested prefix was complete (always true at depth 0). +fn hash_conversation_prefix(body: &Value, depth: usize) -> (String, bool) { let mut hasher = DefaultHasher::new(); // Anthropic carries the system prompt at the top level. @@ -25,23 +47,31 @@ pub fn session_key_from_body(body: &Value) -> String { // OpenAI uses "messages"; the Responses API uses "input". A messages list // with no user message falls through to "input". let mut anchored = false; + let mut tail_taken = 0usize; for seq_key in ["messages", "input"] { if let Some(Value::Array(items)) = body.get(seq_key) { for item in items { - let Some(role) = item.get("role").and_then(Value::as_str) else { - continue; - }; - match role { - "system" | "developer" => { + match item.get("role").and_then(Value::as_str) { + Some("system") | Some("developer") => { flatten_text(item.get("content")).hash(&mut hasher); } - "user" => { + Some("user") if !anchored => { flatten_text(item.get("content")).hash(&mut hasher); anchored = true; - break; + } + // Post-anchor tail (any non-system item, roleless + // Responses items included): hash the full message — + // tool calls too, so assistant turns that differ only + // in tool calls still diverge the deep key. + _ if anchored && tail_taken < depth => { + flatten_message_text(item).hash(&mut hasher); + tail_taken += 1; } _ => {} } + if anchored && tail_taken >= depth { + break; + } } } if anchored { @@ -49,7 +79,43 @@ pub fn session_key_from_body(body: &Value) -> String { } } - format!("{:016x}", hasher.finish()) + let complete = depth == 0 || (anchored && tail_taken >= depth); + (format!("{:016x}", hasher.finish()), complete) +} + +/// Flatten a whole message for deep-key hashing: `content` text plus tool-call +/// payloads (OpenAI chat `tool_calls`, Responses `function_call` items) and +/// tool `output`. Anchors keep hashing `content` only — this richer form is +/// applied to post-anchor tail messages, where the divergence signal between +/// repeated trials often lives entirely in the tool calls. +fn flatten_message_text(item: &Value) -> String { + let mut parts: Vec = Vec::new(); + let content = flatten_text(item.get("content")); + if !content.is_empty() { + parts.push(content); + } + if let Some(Value::Array(calls)) = item.get("tool_calls") { + for call in calls { + if let Some(function) = call.get("function").and_then(Value::as_object) { + let name = function.get("name").and_then(Value::as_str).unwrap_or(""); + let arguments = function + .get("arguments") + .and_then(Value::as_str) + .unwrap_or(""); + parts.push(format!("tool_call {name}({arguments})")); + } + } + } + if item.get("type").and_then(Value::as_str) == Some("function_call") { + let name = item.get("name").and_then(Value::as_str).unwrap_or(""); + let arguments = item.get("arguments").and_then(Value::as_str).unwrap_or(""); + parts.push(format!("tool_call {name}({arguments})")); + } + let output = flatten_text(item.get("output")); + if !output.is_empty() { + parts.push(output); + } + parts.join(" ") } /// Flatten a message-content value into a single string for hashing: strings @@ -244,6 +310,97 @@ mod tests { assert!(key.chars().all(|c| c.is_ascii_hexdigit())); } + #[test] + fn deep_key_depth_zero_matches_base() { + let body = json!({ + "system": "you are helpful", + "messages": [{"role": "user", "content": "first question"}], + }); + assert_eq!( + session_key_from_body_with_depth(&body, 0), + Some(session_key_from_body(&body)) + ); + } + + #[test] + fn deep_key_diverges_on_early_responses() { + // Identical anchors, different first assistant response: the deep key + // separates the trials while the base key does not. + let trial = |first_response: &str| { + json!({ + "messages": [ + {"role": "user", "content": "identical task text"}, + {"role": "assistant", "content": first_response}, + ], + }) + }; + let a = trial("read the tests first"); + let b = trial("look at the repo layout"); + assert_eq!(session_key_from_body(&a), session_key_from_body(&b)); + assert_ne!( + session_key_from_body_with_depth(&a, 1), + session_key_from_body_with_depth(&b, 1) + ); + } + + #[test] + fn deep_key_stable_as_conversation_grows() { + // The deep key hashes a fixed prefix, so it survives appended turns. + let base = json!({ + "messages": [ + {"role": "user", "content": "task"}, + {"role": "assistant", "content": "first response"}, + ], + }); + let grown = json!({ + "messages": [ + {"role": "user", "content": "task"}, + {"role": "assistant", "content": "first response"}, + {"role": "user", "content": "tool result"}, + {"role": "assistant", "content": "much later turn"}, + ], + }); + assert_eq!( + session_key_from_body_with_depth(&base, 1), + session_key_from_body_with_depth(&grown, 1) + ); + } + + #[test] + fn deep_key_none_until_prefix_complete() { + // One post-user message can't satisfy depth 2; no user at all can't + // satisfy any positive depth. + let short = json!({ + "messages": [ + {"role": "user", "content": "task"}, + {"role": "assistant", "content": "first response"}, + ], + }); + assert_eq!(session_key_from_body_with_depth(&short, 2), None); + let unanchored = json!({"messages": [{"role": "system", "content": "sys"}]}); + assert_eq!(session_key_from_body_with_depth(&unanchored, 1), None); + } + + #[test] + fn deep_key_sees_tool_calls() { + // Assistant turns whose divergence lives entirely in tool calls + // (empty content) must still separate the trials. + let trial = |arguments: &str| { + json!({ + "messages": [ + {"role": "user", "content": "task"}, + {"role": "assistant", "content": null, "tool_calls": [ + {"function": {"name": "read_file", "arguments": arguments}}, + ]}, + ], + }) + }; + assert_ne!( + session_key_from_body_with_depth(&trial("{\"path\": \"a.rs\"}"), 1), + session_key_from_body_with_depth(&trial("{\"path\": \"b.rs\"}"), 1) + ); + } + #[test] fn session_key_uses_input_when_messages_has_no_user() { // A messages list with no user message falls through to `input`; the diff --git a/crates/switchyard-py/src/core_bindings/session.rs b/crates/switchyard-py/src/core_bindings/session.rs index a1c5d58a..f5114ee3 100644 --- a/crates/switchyard-py/src/core_bindings/session.rs +++ b/crates/switchyard-py/src/core_bindings/session.rs @@ -11,12 +11,19 @@ use crate::py_serde::value_from_python; /// /// Accepts any Python mapping (e.g. a request body dict) and returns the key /// the core derivation produces for it, used to pin requests of one session to -/// a previously selected model. +/// a previously selected model. `depth == 0` (default) always yields a key; +/// `depth > 0` extends the hashed prefix with the first `depth` post-first-user +/// messages and returns `None` until that prefix exists. #[pyfunction] -pub(crate) fn session_key_from_body(body: &Bound<'_, PyAny>) -> PyResult { - Ok(switchyard_core::session_key_from_body(&value_from_python( - body, - )?)) +#[pyo3(signature = (body, depth = 0))] +pub(crate) fn session_key_from_body( + body: &Bound<'_, PyAny>, + depth: usize, +) -> PyResult> { + Ok(switchyard_core::session_key_from_body_with_depth( + &value_from_python(body)?, + depth, + )) } /// LRU-bounded cache mapping session keys to arbitrary Python values. diff --git a/switchyard/lib/processors/escalation_judge_request_processor.py b/switchyard/lib/processors/escalation_judge_request_processor.py index 0f7b4f97..5ee8ef43 100644 --- a/switchyard/lib/processors/escalation_judge_request_processor.py +++ b/switchyard/lib/processors/escalation_judge_request_processor.py @@ -5,7 +5,6 @@ from __future__ import annotations -import hashlib import json import logging import sys @@ -209,10 +208,15 @@ def __init__( async def process(self, ctx: ProxyContext, request: ChatRequest) -> ChatRequest: """Stamp the tier for this turn and maybe escalate. Leaves the request unchanged.""" turn = conversation_turn_number(request) - # With a deep key, affinity is untouchable until the key prefix is - # complete — hashing a shorter prefix would produce a key that later - # turns of the same conversation no longer match. - affinity_ready = self._seed_session_key(ctx, request) + # With a deep key (session_key_depth > 0), affinity is untouchable + # until the key prefix is complete — hashing a shorter prefix would + # produce a key that later turns of the same conversation no longer + # match. resolve_session_key returns None (memoizing nothing) until + # then; once resolved, the memoized key is what affinity and streak + # bookkeeping see for the rest of the request. + affinity_ready = ( + resolve_session_key(ctx, request, depth=self._session_key_depth) is not None + ) if affinity_ready: pinned = await self._affinity.pinned(ctx, request) @@ -230,8 +234,8 @@ async def process(self, ctx: ProxyContext, request: ChatRequest) -> ChatRequest: if not affinity_ready or turn < self._config.min_judge_turn: return request - # Present when session_key_depth > 0; lets verdict-dump consumers - # group per-turn judge decisions by conversation. + # Lets verdict-dump consumers group per-turn judge decisions by + # conversation. session = ctx.metadata.get(CTX_SESSION_KEY) summary = _summarize_for_judge(request, turn=turn, config=self._config) started_at = time.perf_counter() @@ -403,24 +407,6 @@ async def _record_judge_call(self, *, usage: Any, latency_ms: float) -> None: latency_ms=latency_ms, ) - def _seed_session_key(self, ctx: ProxyContext, request: ChatRequest) -> bool: - """Seed the deep session key when configured; return whether affinity is usable. - - ``session_key_depth == 0`` keeps the default Rust key (system + first - user message) — always usable. Depth ``N`` extends the hash with the - first ``N`` post-first-user messages so repeated runs of the same task - diverge via early model responses; until those messages exist, the key - is not yet stable and affinity must not be read or written. - """ - if self._session_key_depth == 0: - return True - key = _deep_session_key(request, self._session_key_depth) - if key is None: - return False - ctx.metadata[CTX_SESSION_KEY] = key - return True - - def _first_user_text(request: ChatRequest) -> str: """Flatten the first user message's task text for verdict dumps. @@ -452,52 +438,6 @@ def _first_user_text(request: ChatRequest) -> str: return "" -def _deep_session_key(request: ChatRequest, depth: int) -> str | None: - """Hash system + first user + first ``depth`` later messages, or ``None`` if short. - - Mirrors the anchor semantics of the Rust ``session_key_from_body`` and - extends the prefix by ``depth`` messages. The prefix of a conversation - never changes as it grows, so the key is stable from the moment it exists. - """ - body = getattr(request, "body", None) - if not isinstance(body, dict): - return None - hasher = hashlib.sha256() - top_system = body.get("system") - if isinstance(top_system, str | list): - hasher.update(_content_text(top_system).encode("utf-8")) - hasher.update(b"\x00") - - messages = body.get("messages") - if not isinstance(messages, list): - messages = body.get("input") - if not isinstance(messages, list): - return None - - first_user_seen = False - tail_taken = 0 - for m in messages: - if not isinstance(m, dict): - continue - role = m.get("role") - if role in ("system", "developer"): - hasher.update(_message_text(m).encode("utf-8")) - hasher.update(b"\x00") - elif not first_user_seen and role == "user": - first_user_seen = True - hasher.update(_message_text(m).encode("utf-8")) - hasher.update(b"\x00") - elif first_user_seen and tail_taken < depth: - tail_taken += 1 - hasher.update(_message_text(m).encode("utf-8")) - hasher.update(b"\x00") - if first_user_seen and tail_taken >= depth: - break - if not first_user_seen or tail_taken < depth: - return None - return hasher.hexdigest()[:16] - - def _summarize_for_judge( request: ChatRequest, *, diff --git a/switchyard/lib/session_affinity.py b/switchyard/lib/session_affinity.py index 15139bf6..a4ef6e57 100644 --- a/switchyard/lib/session_affinity.py +++ b/switchyard/lib/session_affinity.py @@ -8,7 +8,7 @@ import inspect import logging import time -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, overload from switchyard.lib.conversation_turn import conversation_turn_number from switchyard.lib.proxy_context import ProxyContext @@ -35,17 +35,31 @@ CTX_SESSION_KEY = "_session_affinity_key" -def resolve_session_key(ctx: ProxyContext, request: ChatRequest) -> str: +@overload +def resolve_session_key(ctx: ProxyContext, request: ChatRequest) -> str: ... + + +@overload +def resolve_session_key(ctx: ProxyContext, request: ChatRequest, *, depth: int) -> str | None: ... + + +def resolve_session_key(ctx: ProxyContext, request: ChatRequest, *, depth: int = 0) -> str | None: """Derive the conversation key once per request, memoized on ``ctx``. - Components that pre-seed a custom key into ``CTX_SESSION_KEY`` (e.g. the - escalation judge's deep benchmark key) are respected: the seeded value - wins over re-derivation. + ``depth > 0`` derives the deep key (see + :func:`~switchyard.lib.session_key.session_key_from_body`) the escalation + judge uses for repeated-trial benchmarking, and returns ``None`` — without + memoizing — while the conversation is still shorter than the requested + prefix. A key already memoized on ``ctx`` (by an earlier component this + request, whatever its depth) wins over re-derivation, so every consumer + sees one consistent key per request. """ cached = ctx.metadata.get(CTX_SESSION_KEY) if isinstance(cached, str): return cached - key = session_key_from_body(request.body) + key = session_key_from_body(request.body, depth) + if key is None: + return None ctx.metadata[CTX_SESSION_KEY] = key return key diff --git a/switchyard/lib/session_key.py b/switchyard/lib/session_key.py index 8b63b0c1..2026a0ab 100644 --- a/switchyard/lib/session_key.py +++ b/switchyard/lib/session_key.py @@ -9,6 +9,29 @@ from __future__ import annotations -from switchyard_rust.core import session_key_from_body +from typing import Any, Literal, overload + +from switchyard_rust.core import session_key_from_body as _native_session_key_from_body + + +@overload +def session_key_from_body(body: Any, depth: Literal[0] = 0) -> str: ... + + +@overload +def session_key_from_body(body: Any, depth: int) -> str | None: ... + + +def session_key_from_body(body: Any, depth: int = 0) -> str | None: + """Derive the per-conversation key from a request body. + + ``depth == 0`` (default) hashes the stable anchors only and always returns + a key. ``depth > 0`` extends the hashed prefix with the first ``depth`` + post-first-user messages — so repeated trials of an identical task diverge + via early model responses — and returns ``None`` until the conversation is + long enough for that prefix to exist. + """ + return _native_session_key_from_body(body, depth) + __all__ = ["session_key_from_body"] diff --git a/switchyard_rust/core.py b/switchyard_rust/core.py index c1497b83..678ba8fd 100644 --- a/switchyard_rust/core.py +++ b/switchyard_rust/core.py @@ -616,7 +616,9 @@ def values(self) -> list[Any]: ... def max_sessions(self) -> int: ... def __len__(self) -> int: ... - def session_key_from_body(body: Mapping[str, Any] | JsonValue) -> str: ... + def session_key_from_body( + body: Mapping[str, Any] | JsonValue, depth: int = 0 + ) -> str | None: ... def __getattr__(name: str) -> object: From ef6f47aad0af6aa7b1a05787330ec10ffdda74ee Mon Sep 17 00:00:00 2001 From: Lin Jia Date: Wed, 15 Jul 2026 15:56:35 -0700 Subject: [PATCH 13/23] refactor(processors): share message-condensing helpers across classifier, judge, and planner Signed-off-by: Lin Jia --- .../escalation_judge_request_processor.py | 89 ++------- .../llm_classifier/request_processor.py | 92 +--------- .../lib/processors/message_condensing.py | 171 ++++++++++++++++++ .../plan_execute/request_processor.py | 16 +- 4 files changed, 196 insertions(+), 172 deletions(-) create mode 100644 switchyard/lib/processors/message_condensing.py diff --git a/switchyard/lib/processors/escalation_judge_request_processor.py b/switchyard/lib/processors/escalation_judge_request_processor.py index 5ee8ef43..2cd39c7f 100644 --- a/switchyard/lib/processors/escalation_judge_request_processor.py +++ b/switchyard/lib/processors/escalation_judge_request_processor.py @@ -23,8 +23,13 @@ from switchyard.lib.processors.llm_classifier.request_processor import ( LLMClassifierClient, OpenAIChatLLMClassifierClient, - _strip_markdown_fence, - _trim_messages, +) +from switchyard.lib.processors.message_condensing import ( + content_text, + message_text, + strip_markdown_fence, + trim_messages, + truncate_middle, ) from switchyard.lib.proxy_context import ProxyContext from switchyard.lib.session_affinity import ( @@ -246,7 +251,7 @@ async def process(self, ctx: ProxyContext, request: ChatRequest) -> ChatRequest: request_summary=summary, ) verdict = EscalationVerdict.model_validate_json( - _strip_markdown_fence(completion.content), + strip_markdown_fence(completion.content), ) except Exception as exc: # Fail open to the current (weak) tier and never pin: a judge @@ -425,7 +430,7 @@ def _first_user_text(request: ChatRequest) -> str: return "" for m in messages: if isinstance(m, dict) and m.get("role") == "user": - text = _message_text(m).strip() + text = message_text(m).strip() while True: for tag in ("system-reminder", "environment_context"): if text.startswith(f"<{tag}>"): @@ -461,7 +466,7 @@ def _summarize_for_judge( if isinstance(raw, list): messages = raw - trimmed = _trim_messages(messages, recent_turn_window=config.recent_turn_window) + trimmed = trim_messages(messages, recent_turn_window=config.recent_turn_window) anchor_lines: list[str] = [] window_lines: list[str] = [] @@ -469,22 +474,26 @@ def _summarize_for_judge( top_system = body.get("system") if isinstance(top_system, str | list): anchor_lines.append( - "[system] " + _truncate(_content_text(top_system), config.system_chars) + "[system] " + truncate_middle(content_text(top_system), config.system_chars) ) first_user_seen = False for m in trimmed: if not isinstance(m, dict): continue role = m.get("role") - text = _message_text(m) + text = message_text(m) if role in ("system", "developer"): - anchor_lines.append(f"[{role}] " + _truncate(text, config.system_chars)) + anchor_lines.append(f"[{role}] " + truncate_middle(text, config.system_chars)) elif role == "user" and not first_user_seen: first_user_seen = True - anchor_lines.append("[user (task)] " + _truncate(text, config.first_user_chars)) + anchor_lines.append( + "[user (task)] " + truncate_middle(text, config.first_user_chars) + ) else: label = role or m.get("type") or "message" - window_lines.append(f"[{label}] " + _truncate(text, config.window_message_chars)) + window_lines.append( + f"[{label}] " + truncate_middle(text, config.window_message_chars) + ) n_shown = len(window_lines) header = ( @@ -506,66 +515,6 @@ def _assemble() -> str: return text -def _message_text(message: dict[str, Any]) -> str: - """Flatten a chat message (or Responses item) to plain text, tool calls included.""" - parts: list[str] = [] - content = message.get("content") - if isinstance(content, str | list): - text = _content_text(content) - if text: - parts.append(text) - # OpenAI chat tool calls: the command shapes the judge needs for loop detection. - tool_calls = message.get("tool_calls") - if isinstance(tool_calls, list): - for call in tool_calls: - if not isinstance(call, dict): - continue - fn = call.get("function") - if isinstance(fn, dict): - parts.append(f"tool_call {fn.get('name')}({fn.get('arguments', '')})") - # OpenAI Responses items carry their payloads in type-specific fields. - if message.get("type") == "function_call": - parts.append(f"tool_call {message.get('name')}({message.get('arguments', '')})") - output = message.get("output") - if isinstance(output, str | list): - text = _content_text(output) - if text: - parts.append(text) - return " ".join(parts) - - -def _content_text(content: str | list[Any]) -> str: - """Flatten string-or-content-block message content to text.""" - if isinstance(content, str): - return content - parts: list[str] = [] - for block in content: - if isinstance(block, str): - parts.append(block) - continue - if not isinstance(block, dict): - continue - text = block.get("text") - if isinstance(text, str): - parts.append(text) - continue - inner = block.get("content") - if isinstance(inner, str | list): - parts.append(_content_text(inner)) - return " ".join(p for p in parts if p) - - -def _truncate(text: str, limit: int) -> str: - """Keep the head and tail of ``text`` within ``limit`` chars.""" - if len(text) <= limit: - return text - marker = " ...[trimmed] " - keep = max(limit - len(marker), 20) - head = (keep * 2) // 3 - tail = keep - head - return text[:head] + marker + text[-tail:] - - __all__ = [ "CTX_ESCALATION_VERDICT", "ESCALATION_JUDGE_SYSTEM_PROMPT", diff --git a/switchyard/lib/processors/llm_classifier/request_processor.py b/switchyard/lib/processors/llm_classifier/request_processor.py index 6196fded..8c6f73d5 100644 --- a/switchyard/lib/processors/llm_classifier/request_processor.py +++ b/switchyard/lib/processors/llm_classifier/request_processor.py @@ -22,6 +22,7 @@ RouteSignals, RouteTier, ) +from switchyard.lib.processors.message_condensing import strip_markdown_fence, trim_messages from switchyard.lib.proxy_context import ProxyContext from switchyard.lib.session_affinity import SessionAffinity from switchyard.lib.stats_accumulator import StatsAccumulator @@ -453,7 +454,7 @@ def parse_route_decision( schema: type[RouteDecision] = RouteSignals, ) -> RouteDecision: """Parse classifier JSON output against the given decision schema.""" - stripped = _strip_markdown_fence(raw) + stripped = strip_markdown_fence(raw) try: return schema.model_validate_json(stripped) except ValidationError as exc: @@ -524,11 +525,11 @@ def _condense_body( messages = body.get("messages") if isinstance(messages, list): - out["messages"] = _trim_messages(messages, recent_turn_window=recent_turn_window) + out["messages"] = trim_messages(messages, recent_turn_window=recent_turn_window) raw_input = body.get("input") if isinstance(raw_input, list): - out["input"] = _trim_messages(raw_input, recent_turn_window=recent_turn_window) + out["input"] = trim_messages(raw_input, recent_turn_window=recent_turn_window) elif isinstance(raw_input, str): out["input"] = raw_input @@ -550,78 +551,6 @@ def _condense_tool(tool: Any) -> Any: return {k: v for k, v in tool.items() if k != "input_schema"} -def _trim_messages(messages: list[Any], *, recent_turn_window: int = 0) -> list[Any]: - """Keep system + first-user anchor + a trailing window of messages. - - Anchors retained unconditionally: - - * system / developer messages — global framing the classifier - always needs. - * the **first** user message — agent frameworks like terminus-2 - bundle task framing into ``role="user"`` rather than ``system``; - losing it leaves the classifier blind to what the agent is - working on. - - Trailing window controlled by ``recent_turn_window``: - - * ``0`` (default) — keep only the last user message. Smallest - classifier prompt, but blind to recent assistant tool calls and - tool results, so signal estimation (``tool_call_count_estimate``, - DEBUG-vs-EXPLORATION turn type) must guess from a terse - "Continue" echo. Tends to over-escalate on pessimistic - classifiers. - * ``N >= 1`` — keep the last ``N`` non-anchor messages - (assistant / tool / non-first user) in original order. Gives - the classifier visibility into recent agent activity. Each new - turn appends to a stable prefix the upstream can prompt-cache, - so the extra tokens are nearly free on cache-discounted backends - (DeepSeek V4 Flash ~98% cache discount). - """ - system_msgs: list[Any] = [] - first_user: Any = None - first_user_idx: int | None = None - for idx, m in enumerate(messages): - if not isinstance(m, dict): - continue - role = m.get("role") - if role in ("system", "developer"): - system_msgs.append(m) - elif role == "user" and first_user is None: - first_user = m - first_user_idx = idx - - if first_user is None: - return system_msgs - - # Candidate tail = everything after the first user message that - # isn't a system/developer anchor (those are already included). - tail_candidates = [ - m - for idx, m in enumerate(messages) - if idx > (first_user_idx or 0) - and isinstance(m, dict) - and m.get("role") not in ("system", "developer") - ] - - if recent_turn_window <= 0: - # Historical behavior: keep only the last user message. - last_user: Any = None - for m in tail_candidates: - if m.get("role") == "user": - last_user = m - if last_user is None: - return [*system_msgs, first_user] - if last_user is first_user: - return [*system_msgs, first_user] - return [*system_msgs, first_user, last_user] - - window = tail_candidates[-recent_turn_window:] - # Filter out the first user (already pinned) to avoid duplicating - # it if the window reaches back that far on short conversations. - window = [m for m in window if m is not first_user] - return [*system_msgs, first_user, *window] - - def _completion_content(result: Any) -> str: choices = getattr(result, "choices", None) if not choices: @@ -634,19 +563,6 @@ def _completion_content(result: Any) -> str: raise LLMClassifierError("classifier completion had empty content") -def _strip_markdown_fence(raw: str) -> str: - stripped = raw.strip() - if not stripped.startswith("```"): - return stripped - - lines = stripped.splitlines() - if lines and lines[0].startswith("```"): - lines = lines[1:] - if lines and lines[-1].startswith("```"): - lines = lines[:-1] - return "\n".join(lines).strip() - - __all__ = [ "DEFAULT_CLASSIFIER_SYSTEM_PROMPT", "ClassifierCompletion", diff --git a/switchyard/lib/processors/message_condensing.py b/switchyard/lib/processors/message_condensing.py new file mode 100644 index 00000000..f6f50ba6 --- /dev/null +++ b/switchyard/lib/processors/message_condensing.py @@ -0,0 +1,171 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Shared helpers for condensing chat requests into routing-side LLM prompts. + +The LLM classifier, the escalation judge, and the plan-execute planner all +show an inner LLM a compact view of the live request: anchors (system + first +user message) plus a recent window, with bulk trimmed. These helpers are the +common plumbing — message trimming, content flattening, truncation, and +output-fence stripping — shared so the routing processors condense requests +consistently instead of each keeping a private copy. +""" + +from __future__ import annotations + +from typing import Any + + +def trim_messages(messages: list[Any], *, recent_turn_window: int = 0) -> list[Any]: + """Keep system + first-user anchor + a trailing window of messages. + + Anchors retained unconditionally: + + * system / developer messages — global framing the inner LLM + always needs. + * the **first** user message — agent frameworks like terminus-2 + bundle task framing into ``role="user"`` rather than ``system``; + losing it leaves the inner LLM blind to what the agent is + working on. + + Trailing window controlled by ``recent_turn_window``: + + * ``0`` (default) — keep only the last user message. Smallest + prompt, but blind to recent assistant tool calls and + tool results, so signal estimation (``tool_call_count_estimate``, + DEBUG-vs-EXPLORATION turn type) must guess from a terse + "Continue" echo. Tends to over-escalate on pessimistic + classifiers. + * ``N >= 1`` — keep the last ``N`` non-anchor messages + (assistant / tool / non-first user) in original order. Gives + the inner LLM visibility into recent agent activity. Each new + turn appends to a stable prefix the upstream can prompt-cache, + so the extra tokens are nearly free on cache-discounted backends + (DeepSeek V4 Flash ~98% cache discount). + """ + system_msgs: list[Any] = [] + first_user: Any = None + first_user_idx: int | None = None + for idx, m in enumerate(messages): + if not isinstance(m, dict): + continue + role = m.get("role") + if role in ("system", "developer"): + system_msgs.append(m) + elif role == "user" and first_user is None: + first_user = m + first_user_idx = idx + + if first_user is None: + return system_msgs + + # Candidate tail = everything after the first user message that + # isn't a system/developer anchor (those are already included). + tail_candidates = [ + m + for idx, m in enumerate(messages) + if idx > (first_user_idx or 0) + and isinstance(m, dict) + and m.get("role") not in ("system", "developer") + ] + + if recent_turn_window <= 0: + # Historical behavior: keep only the last user message. + last_user: Any = None + for m in tail_candidates: + if m.get("role") == "user": + last_user = m + if last_user is None: + return [*system_msgs, first_user] + if last_user is first_user: + return [*system_msgs, first_user] + return [*system_msgs, first_user, last_user] + + window = tail_candidates[-recent_turn_window:] + # Filter out the first user (already pinned) to avoid duplicating + # it if the window reaches back that far on short conversations. + window = [m for m in window if m is not first_user] + return [*system_msgs, first_user, *window] + + +def message_text(message: dict[str, Any]) -> str: + """Flatten a chat message (or Responses item) to plain text, tool calls included.""" + parts: list[str] = [] + content = message.get("content") + if isinstance(content, str | list): + text = content_text(content) + if text: + parts.append(text) + # OpenAI chat tool calls: the command shapes an inner LLM needs for loop detection. + tool_calls = message.get("tool_calls") + if isinstance(tool_calls, list): + for call in tool_calls: + if not isinstance(call, dict): + continue + fn = call.get("function") + if isinstance(fn, dict): + parts.append(f"tool_call {fn.get('name')}({fn.get('arguments', '')})") + # OpenAI Responses items carry their payloads in type-specific fields. + if message.get("type") == "function_call": + parts.append(f"tool_call {message.get('name')}({message.get('arguments', '')})") + output = message.get("output") + if isinstance(output, str | list): + text = content_text(output) + if text: + parts.append(text) + return " ".join(parts) + + +def content_text(content: str | list[Any]) -> str: + """Flatten string-or-content-block message content to text.""" + if isinstance(content, str): + return content + parts: list[str] = [] + for block in content: + if isinstance(block, str): + parts.append(block) + continue + if not isinstance(block, dict): + continue + text = block.get("text") + if isinstance(text, str): + parts.append(text) + continue + inner = block.get("content") + if isinstance(inner, str | list): + parts.append(content_text(inner)) + return " ".join(p for p in parts if p) + + +def truncate_middle(text: str, limit: int) -> str: + """Keep the head and tail of ``text`` within ``limit`` chars.""" + if len(text) <= limit: + return text + marker = " ...[trimmed] " + keep = max(limit - len(marker), 20) + head = (keep * 2) // 3 + tail = keep - head + return text[:head] + marker + text[-tail:] + + +def strip_markdown_fence(raw: str) -> str: + """Drop a wrapping ``` fence from an inner LLM's JSON output, if present.""" + stripped = raw.strip() + if not stripped.startswith("```"): + return stripped + + lines = stripped.splitlines() + if lines and lines[0].startswith("```"): + lines = lines[1:] + if lines and lines[-1].startswith("```"): + lines = lines[:-1] + return "\n".join(lines).strip() + + +__all__ = [ + "content_text", + "message_text", + "strip_markdown_fence", + "trim_messages", + "truncate_middle", +] diff --git a/switchyard/lib/processors/plan_execute/request_processor.py b/switchyard/lib/processors/plan_execute/request_processor.py index 39545124..25e63c23 100644 --- a/switchyard/lib/processors/plan_execute/request_processor.py +++ b/switchyard/lib/processors/plan_execute/request_processor.py @@ -16,6 +16,7 @@ from pydantic import BaseModel, ConfigDict, Field, ValidationError from switchyard.lib.llm_client import OpenAILLMClient +from switchyard.lib.processors.message_condensing import strip_markdown_fence from switchyard.lib.processors.plan_execute.plan import ( CTX_PLANNER_DECISION, PlannerDecision, @@ -759,7 +760,7 @@ def parse_planner_decision(raw: str) -> PlannerDecision: emit one despite instructions). Any other deviation from the schema raises :class:`PlanningError`. """ - stripped = _strip_markdown_fence(raw) + stripped = strip_markdown_fence(raw) try: return PlannerDecision.model_validate_json(stripped) except ValidationError as exc: @@ -977,19 +978,6 @@ def _tool_call_arguments(message: Any) -> str | None: return None -def _strip_markdown_fence(raw: str) -> str: - stripped = raw.strip() - if not stripped.startswith("```"): - return stripped - - lines = stripped.splitlines() - if lines and lines[0].startswith("```"): - lines = lines[1:] - if lines and lines[-1].startswith("```"): - lines = lines[:-1] - return "\n".join(lines).strip() - - __all__ = [ "DEFAULT_INITIAL_PLANNER_SYSTEM_PROMPT", "DEFAULT_REVISION_PLANNER_SYSTEM_PROMPT", From 6e588a6a31d9ba1e1728bd1a3c3db7c354434c5b Mon Sep 17 00:00:00 2001 From: Lin Jia Date: Wed, 15 Jul 2026 16:06:38 -0700 Subject: [PATCH 14/23] fix(router): key escalation tiers by the configured target ids so overflow reroute lands on a registered tier Signed-off-by: Lin Jia --- .../escalation_judge_request_processor.py | 25 ++-- .../lib/profiles/escalation_router_config.py | 11 ++ .../escalation_router_profile_config.py | 18 ++- tests/test_escalation_router_profile.py | 109 ++++++++++++++++++ 4 files changed, 150 insertions(+), 13 deletions(-) diff --git a/switchyard/lib/processors/escalation_judge_request_processor.py b/switchyard/lib/processors/escalation_judge_request_processor.py index 2cd39c7f..93a13d9c 100644 --- a/switchyard/lib/processors/escalation_judge_request_processor.py +++ b/switchyard/lib/processors/escalation_judge_request_processor.py @@ -50,8 +50,10 @@ #: LRU cap on the consecutive-escalate streak store. _MAX_STREAK_SESSIONS = 10_000 -#: Tier labels stamped into ``CTX_DETERMINISTIC_ROUTING_TIER``; must match the -#: labels the profile registers on ``DeterministicRoutingLLMBackend``. +#: Default tier labels stamped into ``CTX_DETERMINISTIC_ROUTING_TIER``. The +#: labels must match what the profile registers on +#: ``DeterministicRoutingLLMBackend`` — profiles with custom target ids pass +#: their own via ``strong_tier`` / ``weak_tier``. TIER_STRONG = "strong" TIER_WEAK = "weak" @@ -182,12 +184,21 @@ def __init__( client: LLMClassifierClient | None = None, session_key_depth: int = 0, stats_accumulator: StatsAccumulator | None = None, + strong_tier: str = TIER_STRONG, + weak_tier: str = TIER_WEAK, ) -> None: if session_key_depth < 0: raise ValueError("session_key_depth must be >= 0") + if strong_tier == weak_tier: + raise ValueError("strong_tier and weak_tier must differ") self._config = config self._affinity = affinity self._session_key_depth = session_key_depth + # The labels this processor stamps and pins; they key the tier dict on + # the downstream DeterministicRoutingLLMBackend, so both must match + # what the owning profile registered there. + self._strong_tier = strong_tier + self._weak_tier = weak_tier # Also attached post-construction by the profile chain's # ``with_runtime_components`` (same hook as the LLM classifier). self._stats_accumulator = stats_accumulator @@ -225,8 +236,8 @@ async def process(self, ctx: ProxyContext, request: ChatRequest) -> ChatRequest: if affinity_ready: pinned = await self._affinity.pinned(ctx, request) - if pinned == TIER_STRONG: - ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] = TIER_STRONG + if pinned == self._strong_tier: + ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] = self._strong_tier ctx.metadata[CTX_ESCALATION_VERDICT] = { "escalate": True, "reason": "", @@ -235,7 +246,7 @@ async def process(self, ctx: ProxyContext, request: ChatRequest) -> ChatRequest: } return request - ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] = TIER_WEAK + ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] = self._weak_tier if not affinity_ready or turn < self._config.min_judge_turn: return request @@ -315,8 +326,8 @@ async def process(self, ctx: ProxyContext, request: ChatRequest) -> ChatRequest: turn, verdict.reason, ) - await self._affinity.pin(ctx, request, TIER_STRONG) - ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] = TIER_STRONG + await self._affinity.pin(ctx, request, self._strong_tier) + ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] = self._strong_tier return request def _streak_key(self, ctx: ProxyContext, request: ChatRequest) -> str: diff --git a/switchyard/lib/profiles/escalation_router_config.py b/switchyard/lib/profiles/escalation_router_config.py index 37662994..6769b324 100644 --- a/switchyard/lib/profiles/escalation_router_config.py +++ b/switchyard/lib/profiles/escalation_router_config.py @@ -102,6 +102,17 @@ def _target_model_non_empty(cls, tier: LlmTarget) -> LlmTarget: raise ValueError("target.model must be a non-empty string") return tier + @field_validator("weak") + @classmethod + def _tier_ids_distinct(cls, value: LlmTarget, info: ValidationInfo) -> LlmTarget: + strong = info.data.get("strong") + if isinstance(strong, LlmTarget) and strong.id == value.id: + raise ValueError( + f"strong.id and weak.id must differ (both are {value.id!r}); " + "they key the routing tiers" + ) + return value + @field_validator("judge_system_prompt", mode="before") @classmethod def _blank_judge_prompt_is_unset(cls, value: object) -> object: diff --git a/switchyard/lib/profiles/escalation_router_profile_config.py b/switchyard/lib/profiles/escalation_router_profile_config.py index 1f9d9cc2..bc160e15 100644 --- a/switchyard/lib/profiles/escalation_router_profile_config.py +++ b/switchyard/lib/profiles/escalation_router_profile_config.py @@ -34,9 +34,6 @@ from switchyard.lib.profiles.table import profile_config from switchyard.lib.session_affinity import SessionAffinity -_TIER_STRONG = "strong" -_TIER_WEAK = "weak" - @profile_config("escalation_router") class EscalationRouterProfileConfig: @@ -82,12 +79,21 @@ def build(self) -> ComponentChainProfile: max_request_chars=config.judge_max_request_chars, extra_headers=judge_target.extra_headers or None, ) + # Tier labels are the configured target ids ("strong"/"weak" unless + # overridden). The judge stamps them, the backend keys its tier dict + # by them, and the chain's evict-and-retry rewrites selected_target to + # fallback_target_on_evict — which the config validates against these + # same ids — so an overflow reroute always lands on a registered tier. + strong_id = config.strong.id + weak_id = config.weak.id request_processors: list[Any] = [ ReasoningEffortNormalizer(), EscalationJudgeRequestProcessor( judge_config, affinity=affinity, session_key_depth=config.session_key_depth, + strong_tier=strong_id, + weak_tier=weak_id, ), ] @@ -115,12 +121,12 @@ def build(self) -> ComponentChainProfile: backend = DeterministicRoutingLLMBackend( tiers={ - _TIER_STRONG: (strong_backend, strong_target.model), - _TIER_WEAK: (weak_backend, weak_target.model), + strong_id: (strong_backend, strong_target.model), + weak_id: (weak_backend, weak_target.model), }, # Weak is the resting state; the judge processor stamps a tier on # every turn, so the default only covers malformed metadata. - default_tier=_TIER_WEAK, + default_tier=weak_id, ) return ComponentChainProfile( request_processors=request_processors, diff --git a/tests/test_escalation_router_profile.py b/tests/test_escalation_router_profile.py index 772d7b6e..06262f46 100644 --- a/tests/test_escalation_router_profile.py +++ b/tests/test_escalation_router_profile.py @@ -20,6 +20,15 @@ ) from switchyard.lib.processors.reasoning_effort_normalizer import ReasoningEffortNormalizer from switchyard.lib.profiles import EscalationRouterConfig, EscalationRouterProfileConfig +from switchyard.lib.proxy_context import ProxyContext +from switchyard.lib.roles import LLMBackend +from switchyard_rust.core import ( + ChatRequest, + ChatRequestType, + ChatResponse, + SwitchyardContextWindowExceededError, +) +from switchyard_rust.profiles import ProfileInput def _target(tier: str) -> LlmTarget: @@ -124,3 +133,103 @@ def test_build_threads_judge_settings() -> None: assert judge._config.system_prompt == ESCALATION_JUDGE_SYSTEM_PROMPT assert judge._session_key_depth == 2 assert judge._affinity.enabled + + +def _custom_id_config() -> EscalationRouterConfig: + return _config( + strong=LlmTarget( + id="frontier", + model="strong-model", + base_url="https://strong.invalid/v1", + api_key="sk-strong", + ), + weak=LlmTarget( + id="cheap", + model="weak-model", + base_url="https://weak.invalid/v1", + api_key="sk-weak", + ), + fallback_target_on_evict="frontier", + ) + + +def test_duplicate_tier_ids_rejected() -> None: + with pytest.raises(ValidationError) as exc: + _config( + strong=LlmTarget(id="tier", model="strong-model"), + weak=LlmTarget(id="tier", model="weak-model"), + fallback_target_on_evict="tier", + ) + assert "must differ" in str(exc.value) + + +def test_custom_tier_ids_key_backend_and_judge() -> None: + """Backend tiers and judge labels follow the configured target ids.""" + profile = EscalationRouterProfileConfig.from_config(_custom_id_config()).build() + + backend = profile._backend + assert isinstance(backend, DeterministicRoutingLLMBackend) + assert set(backend._backends) == {"frontier", "cheap"} + assert backend._default_tier == "cheap" + judge = profile._request_processors[1] + assert isinstance(judge, EscalationJudgeRequestProcessor) + assert judge._strong_tier == "frontier" + assert judge._weak_tier == "cheap" + + +class _TierFake(LLMBackend): + """Stub tier backend: overflows when told to, else returns a completion.""" + + def __init__(self, *, overflow: bool, target_id: str) -> None: + self.calls = 0 + self._overflow = overflow + self._target_id = target_id + + @property + def supported_request_types(self) -> list[ChatRequestType]: + return [ChatRequestType.OPENAI_CHAT] + + async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse: + self.calls += 1 + if self._overflow: + error = SwitchyardContextWindowExceededError(f"{self._target_id} overflowed") + error.target_id = self._target_id # type: ignore[attr-defined] + raise error + return ChatResponse.openai_completion({ + "id": "escalation-overflow-test", + "object": "chat.completion", + "model": request.model, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "ok"}, + "finish_reason": "stop", + } + ], + }) + + +async def test_overflow_reroutes_to_custom_strong_id_through_full_profile() -> None: + """Weak overflow retries onto the configured strong id, not an unknown label. + + Regression test for tiers keyed as literal strong/weak while the chain's + evict-and-retry rewrote ``selected_target`` to the *configured* fallback + id: with ids ``cheap``/``frontier``, the rewritten pick was unrecognised, + the retry hit weak again, and the pool exhausted. + """ + profile = EscalationRouterProfileConfig.from_config(_custom_id_config()).build() + backend = profile._backend + assert isinstance(backend, DeterministicRoutingLLMBackend) + cheap = _TierFake(overflow=True, target_id="cheap") + frontier = _TierFake(overflow=False, target_id="frontier") + backend._backends = {"cheap": cheap, "frontier": frontier} + + request = ChatRequest.openai_chat({ + "model": "client/model", + "messages": [{"role": "user", "content": "hi"}], + }) + response = await profile.run(ProfileInput(request)) + + assert cheap.calls == 1 + assert frontier.calls == 1 + assert response.body["choices"][0]["message"]["content"] == "ok" From 084edbd833ce201c94b1e07f595bebb2d4a9b643 Mon Sep 17 00:00:00 2001 From: Lin Jia Date: Wed, 15 Jul 2026 16:06:49 -0700 Subject: [PATCH 15/23] fix(router): only send the vLLM thinking-off hint to judge models that accept it Signed-off-by: Lin Jia --- .../escalation_router_profile_config.py | 10 ++++++- tests/test_escalation_router_profile.py | 27 +++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/switchyard/lib/profiles/escalation_router_profile_config.py b/switchyard/lib/profiles/escalation_router_profile_config.py index bc160e15..4eaf30fa 100644 --- a/switchyard/lib/profiles/escalation_router_profile_config.py +++ b/switchyard/lib/profiles/escalation_router_profile_config.py @@ -25,6 +25,7 @@ from switchyard.lib.processors.reasoning_effort_normalizer import ( ReasoningEffortNormalizer, ) +from switchyard.lib.processors.reasoning_hint import model_accepts_reasoning_hint from switchyard.lib.profiles.chain import ComponentChainProfile from switchyard.lib.profiles.deterministic_routing_profile_config import ( _apply_deepseek_overrides, @@ -73,7 +74,14 @@ def build(self) -> ComponentChainProfile: min_judge_turn=config.judge_min_turn, escalate_confirmations=config.judge_escalate_confirmations, confirmation_window=config.judge_confirmation_window, - disable_reasoning=config.judge_disable_reasoning, + # Claude/Bedrock judges reject the vLLM-only chat_template_kwargs + # hint outright — every judged turn would fail open to weak — so + # the hint is only sent to models that tolerate it (same gate as + # the classifier presets and the stage router). + disable_reasoning=( + config.judge_disable_reasoning + and model_accepts_reasoning_hint(judge_target.model) + ), recent_turn_window=config.judge_recent_turn_window, window_message_chars=config.judge_window_message_chars, max_request_chars=config.judge_max_request_chars, diff --git a/tests/test_escalation_router_profile.py b/tests/test_escalation_router_profile.py index 06262f46..358e283d 100644 --- a/tests/test_escalation_router_profile.py +++ b/tests/test_escalation_router_profile.py @@ -233,3 +233,30 @@ async def test_overflow_reroutes_to_custom_strong_id_through_full_profile() -> N assert cheap.calls == 1 assert frontier.calls == 1 assert response.body["choices"][0]["message"]["content"] == "ok" + + +def test_claude_judge_skips_reasoning_hint() -> None: + """Claude/Bedrock judges must not receive the vLLM-only thinking hint.""" + profile = EscalationRouterProfileConfig.from_config( + _config( + judge=LlmTarget( + id="judge", + model="anthropic/claude-sonnet-5", + base_url="https://judge.invalid/v1", + api_key="sk-judge", + ), + ), + ).build() + + judge = profile._request_processors[1] + assert isinstance(judge, EscalationJudgeRequestProcessor) + assert judge._config.disable_reasoning is False + + +def test_vllm_judge_keeps_reasoning_hint() -> None: + """Non-Anthropic judge models keep the default thinking-off hint.""" + profile = EscalationRouterProfileConfig.from_config(_config()).build() + + judge = profile._request_processors[1] + assert isinstance(judge, EscalationJudgeRequestProcessor) + assert judge._config.disable_reasoning is True From ff71fbacb8d23034e6a82a41c304e8c25117eede Mon Sep 17 00:00:00 2001 From: Lin Jia Date: Wed, 15 Jul 2026 16:09:39 -0700 Subject: [PATCH 16/23] fix(processors): render Anthropic tool_use blocks in condensed transcripts and deep-key hashing Signed-off-by: Lin Jia --- crates/switchyard-core/src/session.rs | 31 ++++++++ .../lib/processors/message_condensing.py | 16 ++++- ...test_escalation_judge_request_processor.py | 33 +++++++++ tests/test_message_condensing.py | 70 +++++++++++++++++++ 4 files changed, 149 insertions(+), 1 deletion(-) create mode 100644 tests/test_message_condensing.py diff --git a/crates/switchyard-core/src/session.rs b/crates/switchyard-core/src/session.rs index 6a8c0c1d..0e80b8d8 100644 --- a/crates/switchyard-core/src/session.rs +++ b/crates/switchyard-core/src/session.rs @@ -94,6 +94,17 @@ fn flatten_message_text(item: &Value) -> String { if !content.is_empty() { parts.push(content); } + // Anthropic tool_use blocks carry their payload in name/input rather than + // text/content, so the plain content flatten yields nothing for them. + if let Some(Value::Array(blocks)) = item.get("content") { + for block in blocks { + if block.get("type").and_then(Value::as_str) == Some("tool_use") { + let name = block.get("name").and_then(Value::as_str).unwrap_or(""); + let input = block.get("input").map(Value::to_string).unwrap_or_default(); + parts.push(format!("tool_call {name}({input})")); + } + } + } if let Some(Value::Array(calls)) = item.get("tool_calls") { for call in calls { if let Some(function) = call.get("function").and_then(Value::as_object) { @@ -401,6 +412,26 @@ mod tests { ); } + #[test] + fn deep_key_sees_anthropic_tool_use() { + // Anthropic assistant turns whose divergence lives in tool_use blocks + // (no text content) must still separate the trials. + let trial = |command: &str| { + json!({ + "messages": [ + {"role": "user", "content": "task"}, + {"role": "assistant", "content": [ + {"type": "tool_use", "name": "bash", "input": {"command": command}}, + ]}, + ], + }) + }; + assert_ne!( + session_key_from_body_with_depth(&trial("ls tests/"), 1), + session_key_from_body_with_depth(&trial("cat README.md"), 1) + ); + } + #[test] fn session_key_uses_input_when_messages_has_no_user() { // A messages list with no user message falls through to `input`; the diff --git a/switchyard/lib/processors/message_condensing.py b/switchyard/lib/processors/message_condensing.py index f6f50ba6..7b74832b 100644 --- a/switchyard/lib/processors/message_condensing.py +++ b/switchyard/lib/processors/message_condensing.py @@ -13,6 +13,7 @@ from __future__ import annotations +import json from typing import Any @@ -117,7 +118,14 @@ def message_text(message: dict[str, Any]) -> str: def content_text(content: str | list[Any]) -> str: - """Flatten string-or-content-block message content to text.""" + """Flatten string-or-content-block message content to text. + + Handles plain strings, OpenAI/Anthropic text blocks, nested + ``tool_result`` content, and Anthropic ``tool_use`` blocks — the last + carry their payload in ``name``/``input`` rather than ``text``/``content`` + and would otherwise flatten to nothing, erasing exactly the + repeated-command signal a trajectory judge relies on. + """ if isinstance(content, str): return content parts: list[str] = [] @@ -127,6 +135,12 @@ def content_text(content: str | list[Any]) -> str: continue if not isinstance(block, dict): continue + if block.get("type") == "tool_use": + arguments = json.dumps( + block.get("input", {}), ensure_ascii=False, sort_keys=True, default=str + ) + parts.append(f"tool_call {block.get('name')}({arguments})") + continue text = block.get("text") if isinstance(text, str): parts.append(text) diff --git a/tests/test_escalation_judge_request_processor.py b/tests/test_escalation_judge_request_processor.py index 7c1cc3e6..d671bd93 100644 --- a/tests/test_escalation_judge_request_processor.py +++ b/tests/test_escalation_judge_request_processor.py @@ -338,6 +338,39 @@ async def test_global_cap_drops_oldest_window_messages_first() -> None: assert "attempt 5" not in summary +async def test_judge_transcript_renders_anthropic_tool_use() -> None: + """Anthropic tool_use turns must not flatten to empty transcript lines.""" + processor, fake, _ = _processor(_verdict_json(False, "")) + messages: list[dict[str, Any]] = [ + {"role": "user", "content": [{"type": "text", "text": "fix the failing tests"}]}, + ] + for i in range(3): + messages.append({ + "role": "assistant", + "content": [ + {"type": "tool_use", "id": f"tu_{i}", "name": "bash", + "input": {"command": "pytest -x"}}, + ], + }) + messages.append({ + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": f"tu_{i}", "content": "1 failed"}, + ], + }) + request = ChatRequest.anthropic(cast(Any, { + "model": "claude-test", + "max_tokens": 128, + "messages": messages, + })) + + await processor.process(ProxyContext(), request) + + summary = fake.calls[0]["request_summary"] + assert 'tool_call bash({"command": "pytest -x"})' in summary + assert "1 failed" in summary + + async def test_deep_session_key_diverges_on_early_responses() -> None: processor, _, _ = _processor(_verdict_json(False, ""), session_key_depth=2) diff --git a/tests/test_message_condensing.py b/tests/test_message_condensing.py new file mode 100644 index 00000000..7ab74f66 --- /dev/null +++ b/tests/test_message_condensing.py @@ -0,0 +1,70 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the shared message-condensing helpers.""" + +from __future__ import annotations + +from switchyard.lib.processors.message_condensing import ( + content_text, + message_text, + strip_markdown_fence, + truncate_middle, +) + + +def test_content_text_renders_anthropic_tool_use() -> None: + """tool_use blocks have neither text nor nested content; render name+input.""" + blocks = [ + {"type": "text", "text": "let me check"}, + {"type": "tool_use", "id": "tu_1", "name": "bash", "input": {"command": "pytest -x"}}, + ] + + text = content_text(blocks) + + assert "let me check" in text + assert 'tool_call bash({"command": "pytest -x"})' in text + + +def test_content_text_renders_tool_result_content() -> None: + blocks = [ + { + "type": "tool_result", + "tool_use_id": "tu_1", + "content": [{"type": "text", "text": "1 failed"}], + }, + ] + assert content_text(blocks) == "1 failed" + + +def test_message_text_covers_all_three_tool_call_shapes() -> None: + chat = { + "role": "assistant", + "content": None, + "tool_calls": [{"function": {"name": "read_file", "arguments": '{"path": "a.py"}'}}], + } + assert message_text(chat) == 'tool_call read_file({"path": "a.py"})' + + responses_item = {"type": "function_call", "name": "bash", "arguments": '{"command": "ls"}'} + assert message_text(responses_item) == 'tool_call bash({"command": "ls"})' + + anthropic = { + "role": "assistant", + "content": [{"type": "tool_use", "name": "bash", "input": {"command": "ls"}}], + } + assert message_text(anthropic) == 'tool_call bash({"command": "ls"})' + + +def test_truncate_middle_keeps_head_and_tail() -> None: + text = "H" * 100 + "T" * 100 + out = truncate_middle(text, 60) + assert len(out) <= 60 + assert out.startswith("H") + assert out.endswith("T") + assert "...[trimmed]" in out + + +def test_strip_markdown_fence_unwraps_json() -> None: + fenced = '```json\n{"escalate": true}\n```' + assert strip_markdown_fence(fenced) == '{"escalate": true}' + assert strip_markdown_fence('{"escalate": true}') == '{"escalate": true}' From 11ac385b9a6c93533fca4ab3c6c204b43969b779 Mon Sep 17 00:00:00 2001 From: Lin Jia Date: Wed, 15 Jul 2026 16:19:13 -0700 Subject: [PATCH 17/23] feat(router): add judge completion-budget and opt-in verdict-dump knobs; support a shared Redis latch store Signed-off-by: Lin Jia --- .../escalation_router_routing.md | 8 ++- .../escalation_judge_request_processor.py | 16 +++--- .../lib/profiles/escalation_router_config.py | 45 ++++++++++++++- .../escalation_router_profile_config.py | 49 +++++++++++++--- tests/test_escalation_router_profile.py | 56 +++++++++++++++++++ 5 files changed, 157 insertions(+), 17 deletions(-) diff --git a/docs/routing_algorithms/escalation_router_routing.md b/docs/routing_algorithms/escalation_router_routing.md index 16cd98b5..ba45fe69 100644 --- a/docs/routing_algorithms/escalation_router_routing.md +++ b/docs/routing_algorithms/escalation_router_routing.md @@ -129,7 +129,9 @@ If the selected tier exceeds its context window, Switchyard retries once on | `judge.min_turn` | `3` | The judge should start earlier or wait for more trajectory evidence. | | `judge.confirmations` | `1` | One positive verdict is too eager and escalation should require repeated evidence. | | `judge.confirmation_window` | `1` | Intermittent trouble should remain eligible for confirmation across negative verdicts. | -| `judge.disable_reasoning` | `true` | Set to `false` when a reasoning judge benefits from thinking despite the added latency. | +| `judge.disable_reasoning` | `true` | Set to `false` when a reasoning judge benefits from thinking despite the added latency. The thinking-off hint is only sent to models that accept it (Claude/Bedrock judges never receive it). | +| `judge.max_completion_tokens` | auto | The judge's completion budget needs an explicit value. The default follows the reasoning mode: `128` with thinking off, `4096` with thinking on (reasoning tokens and the JSON verdict share the budget; a too-small cap silently fails open every turn). | +| `judge.dump_verdicts` | `false` | Benchmark runs need per-turn `escalation_verdict=` stderr lines (includes a first-user `task_hint` snippet). Keep off in production; routing telemetry flows through the stats endpoints. | | `judge.recent_turn_window` | `14` | The judge needs a wider or narrower trailing-message window. | | `judge.window_message_chars` | `300` | More tool-output detail should survive per-message truncation. | | `judge.max_request_chars` | `12000` | The complete judge request needs a different character budget. Oldest recent messages are dropped first. | @@ -138,6 +140,10 @@ If the selected tier exceeds its context window, Switchyard retries once on | `tier_timeout_s` | `600` | Strong or weak targets without their own `timeout_secs` need a different call timeout. | | `enable_stats` | `true` | Set to `false` only when route-level usage statistics are not needed. | | `affinity_max_sessions` | `10000` | A long-lived process needs a different in-memory latch capacity. | +| `affinity_store` | `memory` | Set to `redis` to share the escalation latch across workers and keep it across pod churn; with `memory`, a worker change silently restarts an escalated conversation on weak. Best-effort — a store error never fails a request. | +| `affinity_store_url` | — | Connection URL for the shared latch store (e.g. `redis://host:6379/0`); required with `affinity_store: redis`. | +| `affinity_store_ttl_seconds` | `3600` | A shared latch entry should expire on a different horizon. | +| `affinity_key_prefix` | `swyd:esc:` | Multiple deployments share one Redis and need namespaced keys. | | `session_key_depth` | `0` | Repeated benchmark trials with identical task prefixes need separate latches; keep `0` for normal traffic. | ## Observability diff --git a/switchyard/lib/processors/escalation_judge_request_processor.py b/switchyard/lib/processors/escalation_judge_request_processor.py index 93a13d9c..a271e486 100644 --- a/switchyard/lib/processors/escalation_judge_request_processor.py +++ b/switchyard/lib/processors/escalation_judge_request_processor.py @@ -106,16 +106,18 @@ class EscalationJudgeConfig(BaseModel): disable_reasoning: bool = True extra_headers: dict[str, str] | None = None - dump_verdicts_to_stderr: bool = True + dump_verdicts_to_stderr: bool = False """Emit one ``escalation_verdict={...}`` JSON line to ``sys.stderr`` per judge call (verdict or fail-open). - Written directly (not via the logging module) so it lands in the - benchmark server's captured log regardless of uvicorn's logger config — - mirrors :attr:`LLMClassifierConfig.dump_signals_to_stderr`. Grep - ``escalation_verdict=`` to reconstruct per-turn judge decisions and - escalation timing for a run. Disable for callers that share stderr - with an interactive TUI.""" + Benchmark-only diagnostics, **opt-in**: the payload includes a short + first-user snippet (``task_hint``) and the line interferes with agent + TUIs sharing stderr, so shipping profiles keep it off and production + routing telemetry flows through the normal stats path. When enabled it + is written directly (not via the logging module) so it lands in the + benchmark server's captured log regardless of uvicorn's logger config. + Grep ``escalation_verdict=`` to reconstruct per-turn judge decisions + and escalation timing for a run.""" min_judge_turn: int = Field(default=3, ge=1) """First conversation turn on which the judge runs. Earlier turns have no diff --git a/switchyard/lib/profiles/escalation_router_config.py b/switchyard/lib/profiles/escalation_router_config.py index 6769b324..1e63b95f 100644 --- a/switchyard/lib/profiles/escalation_router_config.py +++ b/switchyard/lib/profiles/escalation_router_config.py @@ -5,12 +5,15 @@ from __future__ import annotations +from typing import Literal, Self + from pydantic import ( BaseModel, ConfigDict, Field, ValidationInfo, field_validator, + model_validator, ) from switchyard.lib.backends.llm_target import LlmTarget, coerce_llm_target @@ -39,12 +42,27 @@ class EscalationRouterConfig(BaseModel): judge_min_turn: First conversation turn on which the judge runs. judge_escalate_confirmations: Consecutive escalate verdicts required before the strong latch fires (``1`` pins on the first verdict). + The confirmation streak is tracked per process: with multiple + workers and no session-sticky load balancing, verdicts for one + conversation can land on different workers and take longer to + accumulate. Keep ``1`` (the default) on multi-worker deployments, + or route conversations sticky to a worker. judge_confirmation_window: Judged turns an escalate verdict stays live for confirmation; ``N > 1`` tolerates up to ``N - 1`` intervening declines (recurring intermittent trouble confirms). judge_disable_reasoning: Send the thinking-off template hint on judge calls (default). ``False`` lets a reasoning judge model think, - trading per-turn latency for rubric adherence. + trading per-turn latency for rubric adherence. The hint is only + sent to models that accept it (Claude/Bedrock reject it). + judge_max_completion_tokens: Completion-token budget for the judge + call. ``None`` (default) picks 128 with the thinking-off hint in + effect and 4096 otherwise — reasoning tokens and the JSON verdict + share this budget, and a too-small cap truncates mid-reasoning so + the judge silently fails open every turn. + judge_dump_verdicts: Emit one ``escalation_verdict={...}`` line to + stderr per judge call (includes a first-user ``task_hint`` + snippet). Benchmark-only diagnostics, off by default; production + routing telemetry flows through the normal stats path. judge_recent_turn_window: Trailing messages shown to the judge on top of the system + first-user anchors. judge_window_message_chars: Per-message truncation cap inside the @@ -65,6 +83,18 @@ class EscalationRouterConfig(BaseModel): when a target does not set its own ``timeout_secs``. enable_stats: Wire stats processors and per-tier stats wrappers. affinity_max_sessions: LRU capacity of the escalation latch store. + affinity_store: Shared L2 behind the in-process latch LRU. ``"memory"`` + (default) keeps the escalation latch per process — a worker or pod + change can lose it. ``"redis"`` shares the latch across workers + and persists it across pod churn (best-effort: an L2 error never + fails a request). + affinity_store_url: Connection URL for the shared store (e.g. + ``"redis://host:6379/0"``). Required when ``affinity_store`` is + ``"redis"``. + affinity_store_ttl_seconds: Expiry for a shared latch entry. The latch + is re-read every turn; an escalated conversation older than the + TTL restarts on weak. + affinity_key_prefix: Namespace prefix for shared-store keys. """ model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) @@ -77,6 +107,8 @@ class EscalationRouterConfig(BaseModel): judge_escalate_confirmations: int = Field(default=1, ge=1) judge_confirmation_window: int = Field(default=1, ge=1) judge_disable_reasoning: bool = True + judge_max_completion_tokens: int | None = Field(default=None, ge=16) + judge_dump_verdicts: bool = False judge_recent_turn_window: int = Field(default=14, ge=1) judge_window_message_chars: int = Field(default=300, ge=50) judge_max_request_chars: int = Field(default=12_000, ge=1_000) @@ -89,6 +121,17 @@ class EscalationRouterConfig(BaseModel): ) enable_stats: bool = True affinity_max_sessions: int = Field(default=10_000, gt=0) + affinity_store: Literal["memory", "redis"] = "memory" + affinity_store_url: str | None = None + affinity_store_ttl_seconds: int = Field(default=3_600, gt=0) + affinity_key_prefix: str = "swyd:esc:" + + @model_validator(mode="after") + def _redis_store_requires_url(self) -> Self: + # A shared store is dead config unless it is reachable. + if self.affinity_store == "redis" and not self.affinity_store_url: + raise ValueError('affinity_store="redis" requires affinity_store_url to be set') + return self @field_validator("strong", "weak", "judge", mode="before") @classmethod diff --git a/switchyard/lib/profiles/escalation_router_profile_config.py b/switchyard/lib/profiles/escalation_router_profile_config.py index 4eaf30fa..cfeba452 100644 --- a/switchyard/lib/profiles/escalation_router_profile_config.py +++ b/switchyard/lib/profiles/escalation_router_profile_config.py @@ -7,6 +7,7 @@ from typing import Any, Self +from switchyard.lib.affinity_pin_store import AffinityPinStore from switchyard.lib.backends.anthropic_cache_breakpoint_backend import ( maybe_wrap_anthropic_cache, ) @@ -54,17 +55,28 @@ def build(self) -> ComponentChainProfile: # The affinity store IS the escalation latch, so it is always on. # Warmup gating lives in the judge processor (min_judge_turn), not # here — affinity warmup would delay a decided escalation from - # taking effect, not just from being decided. + # taking effect, not just from being decided. The optional Redis L2 + # keeps the latch across workers and pod churn; without it a worker + # change silently restarts an escalated conversation on weak. affinity = SessionAffinity( enabled=True, max_sessions=config.affinity_max_sessions, warmup_turns=0, + l2=_build_affinity_l2(config), ) # DeepSeek judges get the same benchmark-gateway defaults as DeepSeek # tiers (``X-Inference-Priority: batch``) so their calls land on the # relaxed-timeout gateway alongside the routed tier traffic. judge_target = _apply_deepseek_overrides(config.judge) + # Claude/Bedrock judges reject the vLLM-only chat_template_kwargs + # hint outright — every judged turn would fail open to weak — so + # the hint is only sent to models that tolerate it (same gate as + # the classifier presets and the stage router). + judge_disable_reasoning = ( + config.judge_disable_reasoning + and model_accepts_reasoning_hint(judge_target.model) + ) judge_config = EscalationJudgeConfig( model=judge_target.model, api_key=judge_target.api_key, @@ -74,14 +86,16 @@ def build(self) -> ComponentChainProfile: min_judge_turn=config.judge_min_turn, escalate_confirmations=config.judge_escalate_confirmations, confirmation_window=config.judge_confirmation_window, - # Claude/Bedrock judges reject the vLLM-only chat_template_kwargs - # hint outright — every judged turn would fail open to weak — so - # the hint is only sent to models that tolerate it (same gate as - # the classifier presets and the stage router). - disable_reasoning=( - config.judge_disable_reasoning - and model_accepts_reasoning_hint(judge_target.model) + disable_reasoning=judge_disable_reasoning, + # Reasoning tokens and the JSON verdict share one budget: a + # thinking judge needs the classifier's larger ceiling or it + # truncates mid-reasoning and silently fails open every turn. + max_completion_tokens=( + config.judge_max_completion_tokens + if config.judge_max_completion_tokens is not None + else (128 if judge_disable_reasoning else 4096) ), + dump_verdicts_to_stderr=config.judge_dump_verdicts, recent_turn_window=config.judge_recent_turn_window, window_message_chars=config.judge_window_message_chars, max_request_chars=config.judge_max_request_chars, @@ -143,4 +157,23 @@ def build(self) -> ComponentChainProfile: ) +def _build_affinity_l2(config: EscalationRouterConfig) -> AffinityPinStore | None: + """Build the optional shared L2 latch store (``None`` = L1-only). + + Mirrors the latency route's ``affinity_store`` semantics; ``"redis"`` + imports :class:`RedisPinStore` lazily so the ``redis`` dependency stays + optional. Config validation guarantees a URL when the store is Redis. + """ + if config.affinity_store != "redis": + return None + from switchyard.lib.redis_pin_store import RedisPinStore + + assert config.affinity_store_url is not None # enforced by config validator + return RedisPinStore( + config.affinity_store_url, + ttl_seconds=config.affinity_store_ttl_seconds, + key_prefix=config.affinity_key_prefix, + ) + + __all__ = ["EscalationRouterProfileConfig"] diff --git a/tests/test_escalation_router_profile.py b/tests/test_escalation_router_profile.py index 358e283d..aedd44f8 100644 --- a/tests/test_escalation_router_profile.py +++ b/tests/test_escalation_router_profile.py @@ -260,3 +260,59 @@ def test_vllm_judge_keeps_reasoning_hint() -> None: judge = profile._request_processors[1] assert isinstance(judge, EscalationJudgeRequestProcessor) assert judge._config.disable_reasoning is True + + +def test_judge_completion_budget_follows_reasoning_mode() -> None: + """A thinking judge gets the classifier-sized budget; explicit value wins.""" + default = EscalationRouterProfileConfig.from_config(_config()).build() + assert default._request_processors[1]._config.max_completion_tokens == 128 + + thinking = EscalationRouterProfileConfig.from_config( + _config(judge_disable_reasoning=False), + ).build() + assert thinking._request_processors[1]._config.max_completion_tokens == 4096 + + explicit = EscalationRouterProfileConfig.from_config( + _config(judge_disable_reasoning=False, judge_max_completion_tokens=512), + ).build() + assert explicit._request_processors[1]._config.max_completion_tokens == 512 + + +def test_verdict_dump_is_opt_in() -> None: + default = EscalationRouterProfileConfig.from_config(_config()).build() + assert default._request_processors[1]._config.dump_verdicts_to_stderr is False + + opted_in = EscalationRouterProfileConfig.from_config( + _config(judge_dump_verdicts=True), + ).build() + assert opted_in._request_processors[1]._config.dump_verdicts_to_stderr is True + + +def test_redis_affinity_store_requires_url() -> None: + with pytest.raises(ValidationError) as exc: + _config(affinity_store="redis") + assert "affinity_store_url" in str(exc.value) + + +def test_redis_affinity_store_wires_l2_latch() -> None: + from switchyard.lib.redis_pin_store import RedisPinStore + + profile = EscalationRouterProfileConfig.from_config( + _config( + affinity_store="redis", + affinity_store_url="redis://cache:6379/0", + affinity_store_ttl_seconds=120, + ), + ).build() + + judge = profile._request_processors[1] + assert isinstance(judge, EscalationJudgeRequestProcessor) + assert isinstance(judge._affinity._l2, RedisPinStore) + + +def test_default_affinity_store_is_l1_only() -> None: + profile = EscalationRouterProfileConfig.from_config(_config()).build() + + judge = profile._request_processors[1] + assert isinstance(judge, EscalationJudgeRequestProcessor) + assert judge._affinity._l2 is None From d556ffd71170faed74421e82639a97daf92384a7 Mon Sep 17 00:00:00 2001 From: Lin Jia Date: Wed, 15 Jul 2026 16:19:14 -0700 Subject: [PATCH 18/23] refactor(cli): forward escalation route options verbatim so the config model owns every default Signed-off-by: Lin Jia --- switchyard/cli/route_bundle.py | 70 +++++++++++++++++----------------- tests/test_route_bundle.py | 40 +++++++++++++++++++ 2 files changed, 76 insertions(+), 34 deletions(-) diff --git a/switchyard/cli/route_bundle.py b/switchyard/cli/route_bundle.py index cd108ee3..5acceaca 100644 --- a/switchyard/cli/route_bundle.py +++ b/switchyard/cli/route_bundle.py @@ -278,6 +278,10 @@ def llm_target_to_route_dict(target: LlmTarget) -> dict[str, Any]: "tier_timeout_s", "session_key_depth", "affinity_max_sessions", + "affinity_store", + "affinity_store_url", + "affinity_store_ttl_seconds", + "affinity_key_prefix", }) ) _ESCALATION_JUDGE_KEYS = frozenset({ @@ -290,6 +294,8 @@ def llm_target_to_route_dict(target: LlmTarget) -> dict[str, Any]: "confirmations", "confirmation_window", "disable_reasoning", + "max_completion_tokens", + "dump_verdicts", "recent_turn_window", "window_message_chars", "prompt", @@ -1146,43 +1152,39 @@ def _escalation_router_switchyard( ), }, "fallback_target_on_evict": fallback_target_on_evict, - "judge_min_turn": _optional_int(judge.get("min_turn"), default=3), - "judge_escalate_confirmations": _optional_int( - judge.get("confirmations"), default=1, - ), - "judge_confirmation_window": _optional_int( - judge.get("confirmation_window"), default=1, - ), - "judge_disable_reasoning": _optional_bool( - judge.get("disable_reasoning"), default=True, - ), - "judge_recent_turn_window": _optional_int( - judge.get("recent_turn_window"), default=14, - ), - "judge_window_message_chars": _optional_int( - judge.get("window_message_chars"), default=300, - ), "judge_system_prompt": _judge_prompt_value(judge, model_id), "judge_timeout_s": _optional_float(judge.get("timeout_secs"), default=5.0), - "enable_stats": _optional_bool(route.get("enable_stats"), default=True), } - if "max_request_chars" in judge: - config_data["judge_max_request_chars"] = _optional_int( - judge.get("max_request_chars"), default=12_000, - ) - if "tier_timeout_s" in route: - config_data["tier_timeout_s"] = _optional_float( - route.get("tier_timeout_s"), - default=None, - ) - if "session_key_depth" in route: - config_data["session_key_depth"] = _optional_int( - route.get("session_key_depth"), default=0 - ) - if "affinity_max_sessions" in route: - config_data["affinity_max_sessions"] = _optional_int( - route.get("affinity_max_sessions"), default=10_000 - ) + # Optional knobs are forwarded verbatim and only when present, so + # ``EscalationRouterConfig`` stays the single owner of every default and + # of value validation; re-declaring defaults here is exactly the config + # drift this compatibility path must not accumulate. + judge_key_map = { + "min_turn": "judge_min_turn", + "confirmations": "judge_escalate_confirmations", + "confirmation_window": "judge_confirmation_window", + "disable_reasoning": "judge_disable_reasoning", + "max_completion_tokens": "judge_max_completion_tokens", + "dump_verdicts": "judge_dump_verdicts", + "recent_turn_window": "judge_recent_turn_window", + "window_message_chars": "judge_window_message_chars", + "max_request_chars": "judge_max_request_chars", + } + for source_key, config_key in judge_key_map.items(): + if judge.get(source_key) is not None: + config_data[config_key] = judge[source_key] + for route_key in ( + "enable_stats", + "tier_timeout_s", + "session_key_depth", + "affinity_max_sessions", + "affinity_store", + "affinity_store_url", + "affinity_store_ttl_seconds", + "affinity_key_prefix", + ): + if route.get(route_key) is not None: + config_data[route_key] = route[route_key] config = EscalationRouterConfig.model_validate(config_data) return ProfileSwitchyard( diff --git a/tests/test_route_bundle.py b/tests/test_route_bundle.py index 3d2a7348..950ae7a9 100644 --- a/tests/test_route_bundle.py +++ b/tests/test_route_bundle.py @@ -828,6 +828,46 @@ def test_judge_settings_thread_into_processor(self): assert judge._config.timeout_s == 5.0 assert judge._session_key_depth == 2 + def test_defaults_come_from_the_config_model(self): + """Omitted keys inherit EscalationRouterConfig defaults, owned once.""" + from switchyard.cli.route_bundle import build_route_bundle_table + from switchyard.lib.processors.escalation_judge_request_processor import ( + EscalationJudgeRequestProcessor, + ) + table = build_route_bundle_table(self._bundle()) + switchyard = table.lookup_switchyard("myrouter/escalation") + judge = next( + c for c in switchyard.iter_components() + if isinstance(c, EscalationJudgeRequestProcessor) + ) + assert judge._config.escalate_confirmations == 1 + assert judge._config.window_message_chars == 300 + assert judge._config.dump_verdicts_to_stderr is False + assert judge._config.max_completion_tokens == 128 + assert judge._affinity._l2 is None + + def test_benchmark_knobs_and_redis_latch_thread_through(self): + from switchyard.cli.route_bundle import build_route_bundle_table + from switchyard.lib.processors.escalation_judge_request_processor import ( + EscalationJudgeRequestProcessor, + ) + from switchyard.lib.redis_pin_store import RedisPinStore + bundle = self._bundle() + route = bundle["routes"]["myrouter/escalation"] + route["judge"]["dump_verdicts"] = True + route["judge"]["max_completion_tokens"] = 2048 + route["affinity_store"] = "redis" + route["affinity_store_url"] = "redis://cache:6379/0" + table = build_route_bundle_table(bundle) + switchyard = table.lookup_switchyard("myrouter/escalation") + judge = next( + c for c in switchyard.iter_components() + if isinstance(c, EscalationJudgeRequestProcessor) + ) + assert judge._config.dump_verdicts_to_stderr is True + assert judge._config.max_completion_tokens == 2048 + assert isinstance(judge._affinity._l2, RedisPinStore) + def test_judge_prompt_path_reads_file(self, tmp_path): from switchyard.cli.route_bundle import build_route_bundle_table from switchyard.lib.processors.escalation_judge_request_processor import ( From 7cf94522c930d93ee514155f2b6b1c2ab8d5c704 Mon Sep 17 00:00:00 2001 From: Lin Jia Date: Wed, 15 Jul 2026 16:22:40 -0700 Subject: [PATCH 19/23] refactor(profiles): share one two-tier target and backend builder between the deterministic and escalation routers Signed-off-by: Lin Jia --- .../deterministic_routing_profile_config.py | 95 +--------------- .../escalation_router_profile_config.py | 40 ++----- .../lib/profiles/tier_target_builders.py | 106 ++++++++++++++++++ tests/test_deterministic_routing_profile.py | 22 ++-- tests/test_route_bundle.py | 2 +- 5 files changed, 132 insertions(+), 133 deletions(-) create mode 100644 switchyard/lib/profiles/tier_target_builders.py diff --git a/switchyard/lib/profiles/deterministic_routing_profile_config.py b/switchyard/lib/profiles/deterministic_routing_profile_config.py index fc32ecaa..b7ece08e 100644 --- a/switchyard/lib/profiles/deterministic_routing_profile_config.py +++ b/switchyard/lib/profiles/deterministic_routing_profile_config.py @@ -7,17 +7,16 @@ from typing import Any, Self -from switchyard.lib.backends.llm_target import LlmTarget from switchyard.lib.processors.llm_classifier.presets import ( PROFILE_FACTORIES, resolve_classifier_prompt, ) from switchyard.lib.profiles.chain import ComponentChainProfile from switchyard.lib.profiles.deterministic_routing_config import ( - DEFAULT_DETERMINISTIC_TIER_TIMEOUT_S, DeterministicRoutingConfig, ) from switchyard.lib.profiles.table import profile_config +from switchyard.lib.profiles.tier_target_builders import build_tier_backend _TIER_STRONG = "strong" _TIER_WEAK = "weak" @@ -36,16 +35,9 @@ def from_config(cls, config: DeterministicRoutingConfig) -> Self: def build(self) -> ComponentChainProfile: """Build the deterministic routing profile runtime.""" - from switchyard.lib.backends.anthropic_cache_breakpoint_backend import ( - maybe_wrap_anthropic_cache, - ) from switchyard.lib.backends.deterministic_routing_llm_backend import ( DeterministicRoutingLLMBackend, ) - from switchyard.lib.backends.multi_llm_backend import ( - build_native_backend, - resolve_llm_target, - ) from switchyard.lib.processors.llm_classifier import ( LLMClassifierRequestProcessor, SignalTierSelectorRequestProcessor, @@ -101,32 +93,11 @@ def build(self) -> ComponentChainProfile: ) ) - # Resolve format='auto' once after deterministic tier defaults are - # applied so backend selection and Anthropic cache wrapping see the - # same concrete target. - strong_target = resolve_llm_target( - _apply_deepseek_overrides( - _apply_default_tier_timeout( - config.strong, - config.tier_timeout_s, - ), - ), - ) - weak_target = resolve_llm_target( - _apply_deepseek_overrides( - _apply_default_tier_timeout( - config.weak, - config.tier_timeout_s, - ), - ), - ) - strong_backend = maybe_wrap_anthropic_cache( - build_native_backend(strong_target), - strong_target, + strong_target, strong_backend = build_tier_backend( + config.strong, config.tier_timeout_s, ) - weak_backend = maybe_wrap_anthropic_cache( - build_native_backend(weak_target), - weak_target, + weak_target, weak_backend = build_tier_backend( + config.weak, config.tier_timeout_s, ) backend = DeterministicRoutingLLMBackend( @@ -143,60 +114,4 @@ def build(self) -> ComponentChainProfile: ) -def _apply_deepseek_overrides(target: LlmTarget) -> LlmTarget: - """Apply benchmark-specific DeepSeek extras without clobbering callers.""" - default_body = ( - {"chat_template_kwargs": {"enable_thinking": False}} - if "deepseek-v4" in target.model - else None - ) - default_headers = ( - {"X-Inference-Priority": "batch"} - if "deepseek" in target.model - else None - ) - if default_body is None and default_headers is None: - return target - - existing_body = target.extra_body - # LlmTarget normalizes omitted and explicit empty headers to the same - # empty dict, so keep current defaulting behavior for normal DeepSeek - # targets until the target type preserves "headers were provided" state. - existing_headers = target.extra_headers or None - merged_body = existing_body if existing_body is not None else default_body - merged_headers = existing_headers if existing_headers is not None else default_headers - if merged_body == existing_body and merged_headers == existing_headers: - return target - - return LlmTarget( - id=target.id, - model=target.model, - format=target.format, - base_url=target.base_url, - api_key=target.api_key, - timeout_secs=target.endpoint.timeout_secs, - extra_body=merged_body, - extra_headers=merged_headers, - ) - - -def _apply_default_tier_timeout( - target: LlmTarget, - timeout_s: float | None = DEFAULT_DETERMINISTIC_TIER_TIMEOUT_S, -) -> LlmTarget: - """Apply deterministic tier timeout when a target has no explicit timeout.""" - if timeout_s is None or target.endpoint.timeout_secs is not None: - return target - return LlmTarget( - id=target.id, - model=target.model, - format=target.format, - base_url=target.base_url, - api_key=target.api_key, - timeout_secs=timeout_s, - extra_body=target.extra_body, - extra_headers=target.extra_headers, - ) - - __all__ = ["DeterministicRoutingProfileConfig"] diff --git a/switchyard/lib/profiles/escalation_router_profile_config.py b/switchyard/lib/profiles/escalation_router_profile_config.py index cfeba452..96c59db2 100644 --- a/switchyard/lib/profiles/escalation_router_profile_config.py +++ b/switchyard/lib/profiles/escalation_router_profile_config.py @@ -8,16 +8,9 @@ from typing import Any, Self from switchyard.lib.affinity_pin_store import AffinityPinStore -from switchyard.lib.backends.anthropic_cache_breakpoint_backend import ( - maybe_wrap_anthropic_cache, -) from switchyard.lib.backends.deterministic_routing_llm_backend import ( DeterministicRoutingLLMBackend, ) -from switchyard.lib.backends.multi_llm_backend import ( - build_native_backend, - resolve_llm_target, -) from switchyard.lib.processors.escalation_judge_request_processor import ( ESCALATION_JUDGE_SYSTEM_PROMPT, EscalationJudgeConfig, @@ -28,12 +21,12 @@ ) from switchyard.lib.processors.reasoning_hint import model_accepts_reasoning_hint from switchyard.lib.profiles.chain import ComponentChainProfile -from switchyard.lib.profiles.deterministic_routing_profile_config import ( - _apply_deepseek_overrides, - _apply_default_tier_timeout, -) from switchyard.lib.profiles.escalation_router_config import EscalationRouterConfig from switchyard.lib.profiles.table import profile_config +from switchyard.lib.profiles.tier_target_builders import ( + apply_deepseek_overrides, + build_tier_backend, +) from switchyard.lib.session_affinity import SessionAffinity @@ -68,7 +61,7 @@ def build(self) -> ComponentChainProfile: # DeepSeek judges get the same benchmark-gateway defaults as DeepSeek # tiers (``X-Inference-Priority: batch``) so their calls land on the # relaxed-timeout gateway alongside the routed tier traffic. - judge_target = _apply_deepseek_overrides(config.judge) + judge_target = apply_deepseek_overrides(config.judge) # Claude/Bedrock judges reject the vLLM-only chat_template_kwargs # hint outright — every judged turn would fail open to weak — so # the hint is only sent to models that tolerate it (same gate as @@ -119,26 +112,11 @@ def build(self) -> ComponentChainProfile: ), ] - # Resolve format='auto' once after tier defaults are applied so - # backend selection and Anthropic cache wrapping see the same - # concrete target (mirrors the deterministic profile). - strong_target = resolve_llm_target( - _apply_deepseek_overrides( - _apply_default_tier_timeout(config.strong, config.tier_timeout_s), - ), - ) - weak_target = resolve_llm_target( - _apply_deepseek_overrides( - _apply_default_tier_timeout(config.weak, config.tier_timeout_s), - ), - ) - strong_backend = maybe_wrap_anthropic_cache( - build_native_backend(strong_target), - strong_target, + strong_target, strong_backend = build_tier_backend( + config.strong, config.tier_timeout_s, ) - weak_backend = maybe_wrap_anthropic_cache( - build_native_backend(weak_target), - weak_target, + weak_target, weak_backend = build_tier_backend( + config.weak, config.tier_timeout_s, ) backend = DeterministicRoutingLLMBackend( diff --git a/switchyard/lib/profiles/tier_target_builders.py b/switchyard/lib/profiles/tier_target_builders.py new file mode 100644 index 00000000..4b87c56a --- /dev/null +++ b/switchyard/lib/profiles/tier_target_builders.py @@ -0,0 +1,106 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Shared strong/weak tier target preparation and backend construction. + +The deterministic (classifier) router and the escalation router build the same +two-tier shape: apply the default tier timeout, apply DeepSeek +benchmark-gateway overrides, resolve ``format='auto'`` once, build the native +backend, and wrap Anthropic targets with cache breakpoints. Owning those rules +here keeps the two profiles from drifting as the rules change. +""" + +from __future__ import annotations + +from switchyard.lib.backends.anthropic_cache_breakpoint_backend import ( + maybe_wrap_anthropic_cache, +) +from switchyard.lib.backends.llm_target import LlmTarget +from switchyard.lib.backends.multi_llm_backend import ( + build_native_backend, + resolve_llm_target, +) +from switchyard.lib.profiles.deterministic_routing_config import ( + DEFAULT_DETERMINISTIC_TIER_TIMEOUT_S, +) +from switchyard.lib.roles import LLMBackend + + +def apply_deepseek_overrides(target: LlmTarget) -> LlmTarget: + """Apply benchmark-specific DeepSeek extras without clobbering callers.""" + default_body = ( + {"chat_template_kwargs": {"enable_thinking": False}} + if "deepseek-v4" in target.model + else None + ) + default_headers = ( + {"X-Inference-Priority": "batch"} + if "deepseek" in target.model + else None + ) + if default_body is None and default_headers is None: + return target + + existing_body = target.extra_body + # LlmTarget normalizes omitted and explicit empty headers to the same + # empty dict, so keep current defaulting behavior for normal DeepSeek + # targets until the target type preserves "headers were provided" state. + existing_headers = target.extra_headers or None + merged_body = existing_body if existing_body is not None else default_body + merged_headers = existing_headers if existing_headers is not None else default_headers + if merged_body == existing_body and merged_headers == existing_headers: + return target + + return LlmTarget( + id=target.id, + model=target.model, + format=target.format, + base_url=target.base_url, + api_key=target.api_key, + timeout_secs=target.endpoint.timeout_secs, + extra_body=merged_body, + extra_headers=merged_headers, + ) + + +def apply_default_tier_timeout( + target: LlmTarget, + timeout_s: float | None = DEFAULT_DETERMINISTIC_TIER_TIMEOUT_S, +) -> LlmTarget: + """Apply the default tier timeout when a target has no explicit timeout.""" + if timeout_s is None or target.endpoint.timeout_secs is not None: + return target + return LlmTarget( + id=target.id, + model=target.model, + format=target.format, + base_url=target.base_url, + api_key=target.api_key, + timeout_secs=timeout_s, + extra_body=target.extra_body, + extra_headers=target.extra_headers, + ) + + +def build_tier_backend( + target: LlmTarget, + tier_timeout_s: float | None, +) -> tuple[LlmTarget, LLMBackend]: + """Prepare one tier target and build its wrapped backend. + + Resolves ``format='auto'`` once after the tier defaults are applied so + backend selection and Anthropic cache wrapping see the same concrete + target. Returns ``(resolved_target, backend)`` — profiles need the + resolved target for the tier's model id. + """ + resolved = resolve_llm_target( + apply_deepseek_overrides(apply_default_tier_timeout(target, tier_timeout_s)), + ) + return resolved, maybe_wrap_anthropic_cache(build_native_backend(resolved), resolved) + + +__all__ = [ + "apply_deepseek_overrides", + "apply_default_tier_timeout", + "build_tier_backend", +] diff --git a/tests/test_deterministic_routing_profile.py b/tests/test_deterministic_routing_profile.py index 0d031399..4f4422dd 100644 --- a/tests/test_deterministic_routing_profile.py +++ b/tests/test_deterministic_routing_profile.py @@ -28,9 +28,9 @@ DeterministicRoutingProfileConfig, ProfileSwitchyard, ) -from switchyard.lib.profiles.deterministic_routing_profile_config import ( - _apply_deepseek_overrides, - _apply_default_tier_timeout, +from switchyard.lib.profiles.tier_target_builders import ( + apply_deepseek_overrides, + apply_default_tier_timeout, ) from switchyard.lib.proxy_context import ProxyContext from switchyard.lib.route_table_builders import deterministic_routing_virtual_model_id @@ -332,7 +332,7 @@ def test_deepseek_v4_pro_gets_thinking_off(self) -> None: api_key="k", base_url="https://e/v1", ) - out = _apply_deepseek_overrides(target) + out = apply_deepseek_overrides(target) assert out.extra_body == {"chat_template_kwargs": {"enable_thinking": False}} def test_deepseek_gets_batch_priority_header(self) -> None: @@ -343,7 +343,7 @@ def test_deepseek_gets_batch_priority_header(self) -> None: api_key="k", base_url="https://e/v1", ) - out = _apply_deepseek_overrides(target) + out = apply_deepseek_overrides(target) assert out.extra_headers == {"X-Inference-Priority": "batch"} def test_non_deepseek_passes_through_unchanged(self) -> None: @@ -354,7 +354,7 @@ def test_non_deepseek_passes_through_unchanged(self) -> None: api_key="k", base_url="https://e/v1", ) - out = _apply_deepseek_overrides(target) + out = apply_deepseek_overrides(target) assert out is target # no rebuild needed def test_caller_supplied_extras_win(self) -> None: @@ -367,7 +367,7 @@ def test_caller_supplied_extras_win(self) -> None: extra_body={"chat_template_kwargs": {"enable_thinking": True}}, extra_headers={"X-Inference-Priority": "interactive"}, ) - out = _apply_deepseek_overrides(target) + out = apply_deepseek_overrides(target) assert out.extra_body == {"chat_template_kwargs": {"enable_thinking": True}} assert out.extra_headers == {"X-Inference-Priority": "interactive"} @@ -380,7 +380,7 @@ def test_caller_supplied_empty_body_wins(self) -> None: base_url="https://e/v1", extra_body={}, ) - out = _apply_deepseek_overrides(target) + out = apply_deepseek_overrides(target) assert out.extra_body == {} @@ -396,7 +396,7 @@ def test_default_timeout_applies_when_target_has_no_timeout(self) -> None: base_url="https://e/v1", ) - out = _apply_default_tier_timeout(target, 123.0) + out = apply_default_tier_timeout(target, 123.0) assert out.endpoint.timeout_secs == 123.0 @@ -410,7 +410,7 @@ def test_existing_timeout_wins(self) -> None: timeout_secs=45.0, ) - out = _apply_default_tier_timeout(target, 123.0) + out = apply_default_tier_timeout(target, 123.0) assert out is target assert out.endpoint.timeout_secs == 45.0 @@ -424,7 +424,7 @@ def test_none_disables_default_timeout(self) -> None: base_url="https://e/v1", ) - out = _apply_default_tier_timeout(target, None) + out = apply_default_tier_timeout(target, None) assert out is target assert out.endpoint.timeout_secs is None diff --git a/tests/test_route_bundle.py b/tests/test_route_bundle.py index 950ae7a9..89d757d4 100644 --- a/tests/test_route_bundle.py +++ b/tests/test_route_bundle.py @@ -1101,7 +1101,7 @@ def fake_apply_default_tier_timeout(target, timeout_s): return target monkeypatch.setattr( - "switchyard.lib.profiles.deterministic_routing_profile_config._apply_default_tier_timeout", + "switchyard.lib.profiles.tier_target_builders.apply_default_tier_timeout", fake_apply_default_tier_timeout, ) bundle = self._bundle() From 6e3f5f0fe496bb8ed73c7bcac500204fd53d1555 Mon Sep 17 00:00:00 2001 From: Lin Jia Date: Wed, 15 Jul 2026 16:48:19 -0700 Subject: [PATCH 20/23] fix(router): key deterministic tiers by the configured target ids so overflow reroute lands on a registered tier Signed-off-by: Lin Jia --- .../profiles/deterministic_routing_config.py | 11 ++ .../deterministic_routing_profile_config.py | 21 +-- tests/test_deterministic_routing_profile.py | 130 +++++++++++++++++- 3 files changed, 153 insertions(+), 9 deletions(-) diff --git a/switchyard/lib/profiles/deterministic_routing_config.py b/switchyard/lib/profiles/deterministic_routing_config.py index 08276204..8a0b893b 100644 --- a/switchyard/lib/profiles/deterministic_routing_config.py +++ b/switchyard/lib/profiles/deterministic_routing_config.py @@ -124,6 +124,17 @@ def _target_model_non_empty(cls, tier: LlmTarget) -> LlmTarget: raise ValueError("target.model must be a non-empty string") return tier + @field_validator("weak") + @classmethod + def _tier_ids_distinct(cls, value: LlmTarget, info: ValidationInfo) -> LlmTarget: + strong = info.data.get("strong") + if isinstance(strong, LlmTarget) and strong.id == value.id: + raise ValueError( + f"strong.id and weak.id must differ (both are {value.id!r}); " + "they key the routing tiers" + ) + return value + @field_validator("classifier_system_prompt", mode="before") @classmethod def _blank_classifier_prompt_is_unset(cls, value: object) -> object: diff --git a/switchyard/lib/profiles/deterministic_routing_profile_config.py b/switchyard/lib/profiles/deterministic_routing_profile_config.py index b7ece08e..3be3eb9c 100644 --- a/switchyard/lib/profiles/deterministic_routing_profile_config.py +++ b/switchyard/lib/profiles/deterministic_routing_profile_config.py @@ -18,9 +18,6 @@ from switchyard.lib.profiles.table import profile_config from switchyard.lib.profiles.tier_target_builders import build_tier_backend -_TIER_STRONG = "strong" -_TIER_WEAK = "weak" - @profile_config("deterministic") class DeterministicRoutingProfileConfig: @@ -48,9 +45,17 @@ def build(self) -> ComponentChainProfile: from switchyard.lib.session_affinity import SessionAffinity config = self.config + # Tier labels are the configured target ids ("strong"/"weak" unless + # overridden). The tier selector stamps them, the backend keys its + # tier dict by them, and the chain's evict-and-retry rewrites + # selected_target to fallback_target_on_evict — which the config + # validates against these same ids — so an overflow reroute always + # lands on a registered tier. + strong_id = config.strong.id + weak_id = config.weak.id profile = PROFILE_FACTORIES[config.profile_name]( - weak=_TIER_WEAK, - strong=_TIER_STRONG, + weak=weak_id, + strong=strong_id, ) request_processors: list[Any] = [ReasoningEffortNormalizer()] @@ -102,10 +107,10 @@ def build(self) -> ComponentChainProfile: backend = DeterministicRoutingLLMBackend( tiers={ - _TIER_STRONG: (strong_backend, strong_target.model), - _TIER_WEAK: (weak_backend, weak_target.model), + strong_id: (strong_backend, strong_target.model), + weak_id: (weak_backend, weak_target.model), }, - default_tier=_TIER_STRONG, + default_tier=strong_id, ) return ComponentChainProfile( request_processors=request_processors, diff --git a/tests/test_deterministic_routing_profile.py b/tests/test_deterministic_routing_profile.py index 4f4422dd..331b3bb2 100644 --- a/tests/test_deterministic_routing_profile.py +++ b/tests/test_deterministic_routing_profile.py @@ -11,6 +11,7 @@ from pydantic import ValidationError from switchyard.lib.backends.deterministic_routing_llm_backend import ( + CTX_DETERMINISTIC_ROUTING_TIER, DeterministicRoutingLLMBackend, ) from switchyard.lib.backends.llm_target import BackendFormat, LlmTarget @@ -33,9 +34,16 @@ apply_default_tier_timeout, ) from switchyard.lib.proxy_context import ProxyContext +from switchyard.lib.roles import LLMBackend from switchyard.lib.route_table_builders import deterministic_routing_virtual_model_id from switchyard.lib.stats_accumulator import StatsAccumulator -from switchyard_rust.core import ChatRequest, ChatResponse +from switchyard_rust.core import ( + ChatRequest, + ChatRequestType, + ChatResponse, + SwitchyardContextWindowExceededError, +) +from switchyard_rust.profiles import ProfileInput from switchyard_rust.translation import TranslationEngine @@ -489,3 +497,123 @@ def test_virtual_model_id_changes_with_prompt_and_context() -> None: assert prompt_id != base_id assert max_chars_id != base_id + + +def _custom_id_config() -> DeterministicRoutingConfig: + return DeterministicRoutingConfig( + strong=LlmTarget( + id="frontier", + model="strong-model", + format=BackendFormat.OPENAI, + api_key="sk-test", + base_url="https://strong.invalid/v1", + ), + weak=LlmTarget( + id="cheap", + model="weak-model", + format=BackendFormat.OPENAI, + api_key="sk-test", + base_url="https://weak.invalid/v1", + ), + classifier=LlmTarget( + id="classifier", + model="classifier-model", + format=BackendFormat.OPENAI, + api_key="sk-test", + base_url="https://classifier.invalid/v1", + ), + profile_name="coding_agent", + fallback_target_on_evict="frontier", + ) + + +def test_duplicate_tier_ids_rejected() -> None: + config = _custom_id_config() + with pytest.raises(ValidationError) as exc: + DeterministicRoutingConfig.model_validate({ + **config.model_dump(), + "strong": LlmTarget(id="tier", model="strong-model"), + "weak": LlmTarget(id="tier", model="weak-model"), + "fallback_target_on_evict": "tier", + }) + assert "must differ" in str(exc.value) + + +def test_custom_tier_ids_key_backend_and_selector() -> None: + """Backend tiers and the selector's labels follow the configured ids.""" + profile = DeterministicRoutingProfileConfig.from_config(_custom_id_config()).build() + + backend = profile._backend + assert isinstance(backend, DeterministicRoutingLLMBackend) + assert set(backend._backends) == {"frontier", "cheap"} + assert backend._default_tier == "frontier" + + +class _OverflowableTier(LLMBackend): + """Stub tier backend: overflows when told to, else returns a completion.""" + + def __init__(self, *, overflow: bool, target_id: str) -> None: + self.calls = 0 + self._overflow = overflow + self._target_id = target_id + + @property + def supported_request_types(self) -> list[ChatRequestType]: + return [ChatRequestType.OPENAI_CHAT] + + async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse: + self.calls += 1 + if self._overflow: + error = SwitchyardContextWindowExceededError(f"{self._target_id} overflowed") + error.target_id = self._target_id # type: ignore[attr-defined] + raise error + return ChatResponse.openai_completion({ + "id": "deterministic-overflow-test", + "object": "chat.completion", + "model": request.model, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "ok"}, + "finish_reason": "stop", + } + ], + }) + + +class _StampTier: + """Stand-in for the classifier+selector pair: stamp a fixed tier label.""" + + def __init__(self, tier: str) -> None: + self._tier = tier + + async def process(self, ctx: ProxyContext, request: ChatRequest) -> ChatRequest: + ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] = self._tier + return request + + +async def test_overflow_reroutes_to_custom_strong_id_through_full_profile() -> None: + """Weak overflow retries onto the configured strong id, not an unknown label. + + Regression test for tiers keyed as literal strong/weak while the chain's + evict-and-retry rewrote ``selected_target`` to the *configured* fallback + id: with ids ``cheap``/``frontier``, the rewritten pick was unrecognised, + the retry hit weak again, and the pool exhausted. + """ + profile = DeterministicRoutingProfileConfig.from_config(_custom_id_config()).build() + backend = profile._backend + assert isinstance(backend, DeterministicRoutingLLMBackend) + cheap = _OverflowableTier(overflow=True, target_id="cheap") + frontier = _OverflowableTier(overflow=False, target_id="frontier") + backend._backends = {"cheap": cheap, "frontier": frontier} + profile._request_processors = (_StampTier("cheap"),) + + request = ChatRequest.openai_chat({ + "model": "client/model", + "messages": [{"role": "user", "content": "hi"}], + }) + response = await profile.run(ProfileInput(request)) + + assert cheap.calls == 1 + assert frontier.calls == 1 + assert response.body["choices"][0]["message"]["content"] == "ok" From 34d5ba9b0c4623cad5dea65250d881178de534f5 Mon Sep 17 00:00:00 2001 From: Lin Jia Date: Wed, 15 Jul 2026 16:48:19 -0700 Subject: [PATCH 21/23] docs(router): correct fallback-id and verdict-dump wording; cover the escalation affinity store in the lib-core skill Signed-off-by: Lin Jia --- .agents/skills/switchyard-lib-core/SKILL.md | 2 +- .../escalation_router_routing.md | 14 ++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.agents/skills/switchyard-lib-core/SKILL.md b/.agents/skills/switchyard-lib-core/SKILL.md index 0436029b..c2812eba 100644 --- a/.agents/skills/switchyard-lib-core/SKILL.md +++ b/.agents/skills/switchyard-lib-core/SKILL.md @@ -39,7 +39,7 @@ right validation set. If the change is driven by a launcher need, also read | An OpenAI-compatible provider target such as NVIDIA Inference Hub or OpenRouter | Use the existing OpenAI-compatible backend/profile with `base_url`, `api_key`, and model id wiring. Add a new backend only when the provider has a real wire-format, auth, retry, or health contract that cannot fit that path. | | Direct Rust component bindings | Add concrete PyO3 classes under `crates/switchyard-py/src/component_bindings/`, keep config bindings near the component binding that consumes them, and expose them lazily from `switchyard_rust/components.py`. Do not keep growing `core_bindings.rs` or `switchyard_rust/core.py` with concrete component classes. | | Route YAML / model dispatch | Use `switchyard/cli/route_bundle.py` and `switchyard/lib/route_table_builders.py`. They build `RouteTable` entries from profile-backed runtimes and keep launchers plus `switchyard serve --routing-profiles` on one path. | -| Shared/persistent session-affinity pins across workers or pod churn | Configure the latency route with `session_affinity: true` + `affinity_store: redis` + `affinity_store_url` (optional `affinity_store_ttl_seconds`, `affinity_key_prefix`). `SessionAffinity` keeps the Rust `SessionCache` as L1 and reads/writes through the `AffinityPinStore` L2 (`switchyard/lib/redis_pin_store.py`), fail-open behind a 0.1s socket timeout and a 3-failure/10s-cooldown circuit breaker (`switchyard_affinity_l2_breaker_open` gauge). Requires the `switchyard[affinity-redis]` extra. | +| Shared/persistent session-affinity pins across workers or pod churn | Configure the latency route with `session_affinity: true` + `affinity_store: redis` + `affinity_store_url` (optional `affinity_store_ttl_seconds`, `affinity_key_prefix`); the escalation_router route takes the same `affinity_store*` keys (no `session_affinity` flag — its latch is always on; default prefix `swyd:esc:`). `SessionAffinity` keeps the Rust `SessionCache` as L1 and reads/writes through the `AffinityPinStore` L2 (`switchyard/lib/redis_pin_store.py`), fail-open behind a 0.1s socket timeout and a 3-failure/10s-cooldown circuit breaker (`switchyard_affinity_l2_breaker_open` gauge). Requires the `switchyard[affinity-redis]` extra. | | Stats / telemetry | Reuse `StatsRequestProcessor`, `StatsResponseProcessor`, `StatsLlmBackend`, and `StatsAccumulator`. A profile config should thread one accumulator through all three when stats are enabled. Do not write a parallel collector. | | A fixed-path endpoint contributed by per-route components | Set `Endpoint.register_once = True`; `build_switchyard_app(...)` mounts the first instance while still running every component's lifecycle. Leave the default `False` for configurable endpoint classes that may mount distinct instances. | | Per-endpoint attribution on `/metrics` for a Python backend that can't be wrapped by `StatsLlmBackend` | Set `ctx.selected_model = endpoint_id` before returning the response. Also set `ctx.backend_call_latency_ms = upstream_call_ms` so the response processor can compute routing overhead. `LatencyServiceLLMBackend.call` is the reference. | diff --git a/docs/routing_algorithms/escalation_router_routing.md b/docs/routing_algorithms/escalation_router_routing.md index ba45fe69..d92fd36a 100644 --- a/docs/routing_algorithms/escalation_router_routing.md +++ b/docs/routing_algorithms/escalation_router_routing.md @@ -118,7 +118,8 @@ passthrough choices. The judge is internal to the route and is not exposed as a client-selectable model. If the selected tier exceeds its context window, Switchyard retries once on -`fallback_target_on_evict`, which must be `strong` or `weak`. See +`fallback_target_on_evict`, which must match one of the configured tier ids +(`strong` / `weak` unless the targets set their own `id`). See [Context-Window Handling](../operations/context_window.md). ## Useful options @@ -130,7 +131,7 @@ If the selected tier exceeds its context window, Switchyard retries once on | `judge.confirmations` | `1` | One positive verdict is too eager and escalation should require repeated evidence. | | `judge.confirmation_window` | `1` | Intermittent trouble should remain eligible for confirmation across negative verdicts. | | `judge.disable_reasoning` | `true` | Set to `false` when a reasoning judge benefits from thinking despite the added latency. The thinking-off hint is only sent to models that accept it (Claude/Bedrock judges never receive it). | -| `judge.max_completion_tokens` | auto | The judge's completion budget needs an explicit value. The default follows the reasoning mode: `128` with thinking off, `4096` with thinking on (reasoning tokens and the JSON verdict share the budget; a too-small cap silently fails open every turn). | +| `judge.max_completion_tokens` | auto | The judge's completion budget needs an explicit value. The default follows the reasoning mode: `128` when the thinking-off hint is sent, `4096` otherwise (reasoning tokens and the JSON verdict share the budget; a too-small cap silently fails open every turn). Judges that never accept the hint (Claude/Bedrock) also get the larger ceiling — it is a cap, not spend. | | `judge.dump_verdicts` | `false` | Benchmark runs need per-turn `escalation_verdict=` stderr lines (includes a first-user `task_hint` snippet). Keep off in production; routing telemetry flows through the stats endpoints. | | `judge.recent_turn_window` | `14` | The judge needs a wider or narrower trailing-message window. | | `judge.window_message_chars` | `300` | More tool-output detail should survive per-message truncation. | @@ -158,10 +159,11 @@ The snapshot reports per-model calls, tokens, latency, and cost for the strong and weak tiers. Judge calls are recorded in the classifier stats bucket so their token cost, latency, and errors remain visible as routing overhead. -Each judged turn also writes an `escalation_verdict={...}` JSON line to server -stderr. The record includes the decision, reason, turn, confirmation state, -and judge latency. After the latch fires, later turns skip the judge and report -the pinned routing source in request metadata. +With `judge.dump_verdicts: true` (off by default), each judged turn also +writes an `escalation_verdict={...}` JSON line to server stderr. The record +includes the decision, reason, turn, confirmation state, and judge latency. +After the latch fires, later turns skip the judge and report the pinned +routing source in request metadata. ## Repeated benchmark trials From 48af4ba696b1c18248059043c3e2e6799ed6e444 Mon Sep 17 00:00:00 2001 From: Lin Jia Date: Wed, 15 Jul 2026 16:48:19 -0700 Subject: [PATCH 22/23] refactor(cli): surface escalation config errors as route-bundle diagnostics; single-source the judge timeout default Signed-off-by: Lin Jia --- switchyard/cli/route_bundle.py | 18 ++++++++++++------ tests/test_route_bundle.py | 17 ++++++++++++++++- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/switchyard/cli/route_bundle.py b/switchyard/cli/route_bundle.py index 5acceaca..b1df1d4f 100644 --- a/switchyard/cli/route_bundle.py +++ b/switchyard/cli/route_bundle.py @@ -30,6 +30,8 @@ from pathlib import Path from typing import Any, Protocol, cast, overload +from pydantic import ValidationError + from switchyard.cli.model_catalog.model_discovery import fetch_model_ids from switchyard.lib.backends.llm_target import LlmTarget, coerce_llm_target from switchyard.lib.config import LatencyServiceBackendConfig, LatencyServiceEndpoint @@ -1146,20 +1148,16 @@ def _escalation_router_switchyard( "base_url": _required_str( judge.get("base_url"), f"{model_id}.judge.base_url" ), - "timeout_secs": _optional_float( - judge.get("timeout_secs"), - default=5.0, - ), }, "fallback_target_on_evict": fallback_target_on_evict, "judge_system_prompt": _judge_prompt_value(judge, model_id), - "judge_timeout_s": _optional_float(judge.get("timeout_secs"), default=5.0), } # Optional knobs are forwarded verbatim and only when present, so # ``EscalationRouterConfig`` stays the single owner of every default and # of value validation; re-declaring defaults here is exactly the config # drift this compatibility path must not accumulate. judge_key_map = { + "timeout_secs": "judge_timeout_s", "min_turn": "judge_min_turn", "confirmations": "judge_escalate_confirmations", "confirmation_window": "judge_confirmation_window", @@ -1186,7 +1184,15 @@ def _escalation_router_switchyard( if route.get(route_key) is not None: config_data[route_key] = route[route_key] - config = EscalationRouterConfig.model_validate(config_data) + try: + config = EscalationRouterConfig.model_validate(config_data) + except ValidationError as exc: + # Surface config mistakes as the CLI's one-line diagnostic instead of + # a raw pydantic traceback (see the RouteBundleConfigError handler in + # switchyard_cli.main). + raise RouteBundleConfigError( + f"route {model_id!r}: invalid escalation_router config: {exc}" + ) from exc return ProfileSwitchyard( EscalationRouterProfileConfig.from_config(config) .build() diff --git a/tests/test_route_bundle.py b/tests/test_route_bundle.py index 89d757d4..320fbea2 100644 --- a/tests/test_route_bundle.py +++ b/tests/test_route_bundle.py @@ -834,7 +834,9 @@ def test_defaults_come_from_the_config_model(self): from switchyard.lib.processors.escalation_judge_request_processor import ( EscalationJudgeRequestProcessor, ) - table = build_route_bundle_table(self._bundle()) + bundle = self._bundle() + del bundle["routes"]["myrouter/escalation"]["judge"]["timeout_secs"] + table = build_route_bundle_table(bundle) switchyard = table.lookup_switchyard("myrouter/escalation") judge = next( c for c in switchyard.iter_components() @@ -844,8 +846,21 @@ def test_defaults_come_from_the_config_model(self): assert judge._config.window_message_chars == 300 assert judge._config.dump_verdicts_to_stderr is False assert judge._config.max_completion_tokens == 128 + assert judge._config.timeout_s == 5.0 assert judge._affinity._l2 is None + def test_invalid_value_is_a_one_line_config_error(self): + """Pydantic rejections surface as RouteBundleConfigError, not a traceback.""" + from switchyard.cli.route_bundle import ( + RouteBundleConfigError, + build_route_bundle_table, + ) + bundle = self._bundle() + bundle["routes"]["myrouter/escalation"]["session_key_depth"] = "two" + with pytest.raises(RouteBundleConfigError) as exc: + build_route_bundle_table(bundle) + assert "session_key_depth" in str(exc.value) + def test_benchmark_knobs_and_redis_latch_thread_through(self): from switchyard.cli.route_bundle import build_route_bundle_table from switchyard.lib.processors.escalation_judge_request_processor import ( From f3f58aa9b672218d7ff583ceceefc0921e340790 Mon Sep 17 00:00:00 2001 From: Lin Jia Date: Wed, 15 Jul 2026 17:12:36 -0700 Subject: [PATCH 23/23] chore(benchmark): opt escalation profiles into judge verdict dumps Signed-off-by: Lin Jia --- .../tb-lite-escalation-opus-deepseek-gemini.yaml | 3 +++ .../tb-lite-escalation-opus-deepseek-nvidia.yaml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/benchmark/routing-profiles/tb-lite-escalation-opus-deepseek-gemini.yaml b/benchmark/routing-profiles/tb-lite-escalation-opus-deepseek-gemini.yaml index 3336ec2e..52389615 100644 --- a/benchmark/routing-profiles/tb-lite-escalation-opus-deepseek-gemini.yaml +++ b/benchmark/routing-profiles/tb-lite-escalation-opus-deepseek-gemini.yaml @@ -30,3 +30,6 @@ routes: timeout_secs: 5 min_turn: 3 recent_turn_window: 14 + # Verdict dumps became opt-in upstream; the bench harness reads + # escalation_verdict= lines from the captured server log. + dump_verdicts: true diff --git a/benchmark/routing-profiles/tb-lite-escalation-opus-deepseek-nvidia.yaml b/benchmark/routing-profiles/tb-lite-escalation-opus-deepseek-nvidia.yaml index 87ff86e6..d213bb90 100644 --- a/benchmark/routing-profiles/tb-lite-escalation-opus-deepseek-nvidia.yaml +++ b/benchmark/routing-profiles/tb-lite-escalation-opus-deepseek-nvidia.yaml @@ -68,3 +68,6 @@ routes: # solves; real loops persist across turns and still latch. confirmations: 2 recent_turn_window: 14 + # Verdict dumps became opt-in upstream; the bench harness reads + # escalation_verdict= lines from the captured server log. + dump_verdicts: true