From d7da8a6ba34875bc7ff1c194c5bdae8801e6ef5e Mon Sep 17 00:00:00 2001 From: Ryan Lempka Date: Wed, 15 Jul 2026 11:21:50 -0500 Subject: [PATCH 01/12] docs: add NeMo Relay recipe for library-mode (libsy) integration Signed-off-by: Ryan Lempka --- docs/guides/nemo_relay.md | 101 ++++++++++++++++++++++++++++++++++++++ mkdocs.yml | 1 + 2 files changed, 102 insertions(+) create mode 100644 docs/guides/nemo_relay.md diff --git a/docs/guides/nemo_relay.md b/docs/guides/nemo_relay.md new file mode 100644 index 00000000..86bfdac7 --- /dev/null +++ b/docs/guides/nemo_relay.md @@ -0,0 +1,101 @@ +# NeMo Relay Recipe + +This recipe shows how to run Switchyard **as a library** inside +[NeMo Relay](https://github.com/NVIDIA/NeMo-Relay), so routing decisions are +made in-process by [libsy](https://github.com/NVIDIA-NeMo/Switchyard/tree/main/crates/libsy) +while Relay keeps ownership of the actual LLM calls. No `switchyard serve` +process is involved. + +> **Status:** work in progress. Steps marked *validated* have been run end to +> end; everything else is the intended shape and may change. + +## Two ways to pair Switchyard with Relay + +| Mode | Decision path | Switchyard process | Where it lives | +|---|---|---|---| +| Server | Relay POSTs to Switchyard's HTTP Decision API | Required (`switchyard serve`) | Relay `crates/switchyard` plugin, Switchyard `topic/nemo-relay-integration` | +| Library (this recipe) | Relay calls libsy in-process | None | libsy embedded in Relay's Switchyard plugin | + +Relay's Switchyard plugin already anticipates this: its README notes that a +future in-process decision provider may replace the HTTP Decision API call +without changing Relay-owned dispatch and observability. + +## How the pieces fit + +```text +agent (e.g. Claude Code) + │ Anthropic / OpenAI wire protocol + ▼ +Relay gateway (local proxy) + │ managed LLM call + ▼ +LLM execution intercept (Relay's Switchyard plugin) + │ + ├─ libsy Algorithm decides which model to call + │ Step::CallLlm ──▶ Relay fulfills via next(request) ──▶ provider + │ Step::Decision ──▶ routing trace / observability + │ Step::ReturnToAgent ──▶ final response + ▼ +response translated back to the inbound protocol +``` + +libsy never performs a network call. Each offloaded `CallLlm` promise is +fulfilled by Relay's own dispatch chain (`next(request)`), and the result is +handed back with `CallLlmRequest::respond(Ok/Err)`. This keeps provider +credentials, retries, and telemetry entirely on the Relay side. + +## Prerequisites + +- Rust 1.96.1 (both repos pin their toolchain) +- A Switchyard checkout on `main` (libsy landed in + [PR #17](https://github.com/NVIDIA-NeMo/Switchyard/pull/17)) +- A NeMo Relay checkout +- Provider credentials for at least two models (a strong and a weak target) + +## Quick start + +*(to be validated; commands will firm up as the integration lands)* + +1. Build Relay with the Switchyard feature: + + ```bash + cargo build -p nemo-relay-cli --features switchyard + ``` + +2. Configure the Switchyard component for library-mode decisions in + `plugins.toml`, selecting a libsy algorithm (the LLM classifier routes + strong/weak based on a classifier model's difficulty score): + + ```toml + [[components]] + kind = "switchyard" + enabled = true + + [components.config] + # decision_backend = "libsy" # in-process, no decision_api_url needed + # algorithm = "llm_classifier" + ``` + +3. Launch an agent through Relay: + + ```bash + nemo-relay claude -- "Summarize this repository." + ``` + +4. Inspect routing decisions in Relay's ATOF output + (`.nemo-relay/atof/events.jsonl`) and mark events + (`switchyard.routing.decision`). + +## The embedding contract + +The host (Relay) needs exactly three things from libsy: + +- `Algorithm` trait object: one shared `Arc` serves concurrent + requests with no lock. +- `run_stream(ctx, request)`: returns a stream of `Step`s. The host serves + `Step::CallLlm` items itself and fulfills each with `respond(...)`. +- The conversation IR (`LlmRequest` / `LlmResponse` from `libsy-protocol`), + which Relay's plugin already speaks through `switchyard-translation`. + +See the [libsy README](https://github.com/NVIDIA-NeMo/Switchyard/tree/main/crates/libsy) +for the full API walkthrough. diff --git a/mkdocs.yml b/mkdocs.yml index 537c1af6..83f29cb3 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -22,6 +22,7 @@ nav: - Known Issues in 0.1.0: known_issues.md - Guides: - Agent Launchers: guides/agent_launchers.md + - NeMo Relay: guides/nemo_relay.md - Skill Distillation: skill_distillation.md - Concepts: - Core Concepts: core_concepts.md From 86eeaa3eb3439856eb4e7d97c951f88a359df484 Mon Sep 17 00:00:00 2001 From: Ryan Lempka Date: Wed, 15 Jul 2026 11:42:38 -0500 Subject: [PATCH 02/12] docs: validate NeMo Relay recipe buffered path, document streaming gap Signed-off-by: Ryan Lempka --- docs/guides/nemo_relay.md | 118 ++++++++++++++++++++++++-------------- 1 file changed, 74 insertions(+), 44 deletions(-) diff --git a/docs/guides/nemo_relay.md b/docs/guides/nemo_relay.md index 86bfdac7..f2b605fb 100644 --- a/docs/guides/nemo_relay.md +++ b/docs/guides/nemo_relay.md @@ -2,23 +2,26 @@ This recipe shows how to run Switchyard **as a library** inside [NeMo Relay](https://github.com/NVIDIA/NeMo-Relay), so routing decisions are -made in-process by [libsy](https://github.com/NVIDIA-NeMo/Switchyard/tree/main/crates/libsy) +made in-process by [libsy](../../crates/libsy/README.md) while Relay keeps ownership of the actual LLM calls. No `switchyard serve` process is involved. -> **Status:** work in progress. Steps marked *validated* have been run end to -> end; everything else is the intended shape and may change. +> **Status:** the buffered request path is validated end to end against +> NeMo Relay's `feat/libsy-decision-backend` branch (classifier call and +> routed call both fulfilled by Relay dispatch). Streaming requests are not +> yet routed by libsy; they dispatch Relay's trusted per-protocol fallback. ## Two ways to pair Switchyard with Relay | Mode | Decision path | Switchyard process | Where it lives | |---|---|---|---| | Server | Relay POSTs to Switchyard's HTTP Decision API | Required (`switchyard serve`) | Relay `crates/switchyard` plugin, Switchyard `topic/nemo-relay-integration` | -| Library (this recipe) | Relay calls libsy in-process | None | libsy embedded in Relay's Switchyard plugin | +| Library (this recipe) | Relay calls libsy in-process | None | `decision_backend = "libsy"` in Relay's Switchyard plugin | -Relay's Switchyard plugin already anticipates this: its README notes that a -future in-process decision provider may replace the HTTP Decision API call -without changing Relay-owned dispatch and observability. +Relay's Switchyard plugin README anticipated this: a future in-process +decision provider replacing the HTTP Decision API call without changing +Relay-owned dispatch and observability. `decision_backend = "libsy"` is that +provider. ## How the pieces fit @@ -31,9 +34,9 @@ Relay gateway (local proxy) ▼ LLM execution intercept (Relay's Switchyard plugin) │ - ├─ libsy Algorithm decides which model to call + ├─ libsy Algorithm decides which target to call │ Step::CallLlm ──▶ Relay fulfills via next(request) ──▶ provider - │ Step::Decision ──▶ routing trace / observability + │ Step::Decision ──▶ switchyard.routing.decision mark events │ Step::ReturnToAgent ──▶ final response ▼ response translated back to the inbound protocol @@ -41,50 +44,77 @@ response translated back to the inbound protocol libsy never performs a network call. Each offloaded `CallLlm` promise is fulfilled by Relay's own dispatch chain (`next(request)`), and the result is -handed back with `CallLlmRequest::respond(Ok/Err)`. This keeps provider -credentials, retries, and telemetry entirely on the Relay side. +handed back with `CallLlmRequest::respond(Ok/Err)`. The classifier's scoring +call is a Relay-managed provider call like any other, so it shows up in +Relay's observability. libsy's semantic target names (`"strong"`, `"weak"`, +`"classifier"`) map onto the plugin's existing `TargetBinding` table, which +binds each name to a Relay-owned backend URL, model, and protocol. ## Prerequisites -- Rust 1.96.1 (both repos pin their toolchain) -- A Switchyard checkout on `main` (libsy landed in - [PR #17](https://github.com/NVIDIA-NeMo/Switchyard/pull/17)) -- A NeMo Relay checkout -- Provider credentials for at least two models (a strong and a weak target) +- Rust 1.96.1 (Switchyard's MSRV; note Relay pins 1.93.0, so build Relay with + `RUSTUP_TOOLCHAIN=1.96.1` until the toolchains converge) +- NeMo Relay checkout on the `feat/libsy-decision-backend` branch +- Provider credentials for your targets (the demo below needs none) -## Quick start +## Quick start (validated) -*(to be validated; commands will firm up as the integration lands)* +Run the self-contained demo from a NeMo Relay checkout: -1. Build Relay with the Switchyard feature: - - ```bash - cargo build -p nemo-relay-cli --features switchyard - ``` - -2. Configure the Switchyard component for library-mode decisions in - `plugins.toml`, selecting a libsy algorithm (the LLM classifier routes - strong/weak based on a classifier model's difficulty score): - - ```toml - [[components]] - kind = "switchyard" - enabled = true +```bash +RUSTUP_TOOLCHAIN=1.96.1 ./examples/switchyard/run-libsy-e2e.sh +``` - [components.config] - # decision_backend = "libsy" # in-process, no decision_api_url needed - # algorithm = "llm_classifier" - ``` +This starts a deterministic fake provider and Relay with the libsy backend, +then verifies an easy prompt routes to the weak model and a hard prompt +routes to the strong model, with the classifier and routed calls both flowing +through Relay dispatch. + +The plugin configuration (`examples/switchyard/libsy-plugins.toml`): + +```toml +[[components]] +kind = "switchyard" +enabled = true + +[components.config] +mode = "enforce" +decision_backend = "libsy" +request_materialization = "summary_only" +context_mode = "payload_only" +enabled_inbound_profiles = ["openai_chat"] + +[components.config.libsy] +algorithm = "llm_classifier" +classifier_target = "classifier" +strong_target = "strong" +weak_target = "weak" +threshold = 0.5 + +[components.config.default_targets] +openai_chat = "weak" + +[components.config.targets.classifier] +model = "classifier-model" +protocol = "openai_chat" +endpoint = "/v1/chat/completions" +base_url = "http://127.0.0.1:4102" + +# targets.strong and targets.weak follow the same shape +``` -3. Launch an agent through Relay: +To point at real providers, replace the target `base_url` / `model` values and +supply credentials with per-target `headers` / `header_env`. - ```bash - nemo-relay claude -- "Summarize this repository." - ``` +## Launching an agent (partial) -4. Inspect routing decisions in Relay's ATOF output - (`.nemo-relay/atof/events.jsonl`) and mark events - (`switchyard.routing.decision`). +`nemo-relay claude -- "..."` launches Claude Code through the Relay gateway +with this plugin active. Buffered requests are routed by libsy; streaming +requests (most of Claude Code's interactive traffic) currently dispatch the +trusted per-protocol fallback instead of a libsy decision. Closing the +streaming gap is the top open item: libsy's `CallLlmRequest::respond` accepts +a buffered `Response`, so the final routed hop has no way to stream back to +the agent yet. ## The embedding contract @@ -97,5 +127,5 @@ The host (Relay) needs exactly three things from libsy: - The conversation IR (`LlmRequest` / `LlmResponse` from `libsy-protocol`), which Relay's plugin already speaks through `switchyard-translation`. -See the [libsy README](https://github.com/NVIDIA-NeMo/Switchyard/tree/main/crates/libsy) +See the [libsy README](../../crates/libsy/README.md) for the full API walkthrough. From 82da3b01efdda8c8381c10f95ab31f34e6b859e8 Mon Sep 17 00:00:00 2001 From: Ryan Lempka Date: Wed, 15 Jul 2026 13:15:58 -0500 Subject: [PATCH 03/12] docs: tested Claude Code TUI quickstart and the libsy streaming break point Signed-off-by: Ryan Lempka --- docs/guides/nemo_relay.md | 47 +++++++++++++++++++++++++++++++-------- 1 file changed, 38 insertions(+), 9 deletions(-) diff --git a/docs/guides/nemo_relay.md b/docs/guides/nemo_relay.md index f2b605fb..5472375a 100644 --- a/docs/guides/nemo_relay.md +++ b/docs/guides/nemo_relay.md @@ -106,15 +106,44 @@ base_url = "http://127.0.0.1:4102" To point at real providers, replace the target `base_url` / `model` values and supply credentials with per-target `headers` / `header_env`. -## Launching an agent (partial) - -`nemo-relay claude -- "..."` launches Claude Code through the Relay gateway -with this plugin active. Buffered requests are routed by libsy; streaming -requests (most of Claude Code's interactive traffic) currently dispatch the -trusted per-protocol fallback instead of a libsy decision. Closing the -streaming gap is the top open item: libsy's `CallLlmRequest::respond` accepts -a buffered `Response`, so the final routed hop has no way to stream back to -the agent yet. +## Launching Claude Code (tested), and where it breaks + +From a project directory containing the two config files below, +`nemo-relay claude` opens the normal interactive Claude Code TUI with the +plugin active. Verified end to end on 2026-07-15 with Relay branch +`feat/libsy-decision-backend` ([fork](https://github.com/ryan-lempka/NeMo-Relay/tree/feat/libsy-decision-backend)). + +`.nemo-relay/config.toml`: + +```toml +[agents.claude] +command = "claude" +``` + +`.nemo-relay/plugins.toml`: the libsy component config above, with +Anthropic-protocol targets (e.g. a Haiku classifier routing between Opus and +Sonnet) plus an `observability` component with a `file` ATOF sink to see +routing events. + +What works today, against libsy `main`: + +| Step | Status | +|---|---| +| TUI launches; all traffic flows through the gateway and plugin | works | +| Requests carrying `cache_control` reach the router (same-protocol passthrough) | works | +| Buffered requests routed by the libsy classifier | works | +| Streamed requests routed by libsy | **breaks** | + +The break is in libsy's response contract, not in Relay: +`CallLlmRequest::respond` and `Step::ReturnToAgent` carry only buffered +responses, so no host can pass a live token stream through an algorithm. +Because Claude Code streams its interactive calls, those requests emit +`switchyard.routing.error` (`libsy_stream`) and dispatch the trusted +per-protocol fallback (`switchyard.routing.fallback`, +`libsy_streaming_unsupported`) instead of a libsy decision. Streaming support +exists on the unmerged `grclark/simple-proxy` branch (`LlmResponse` becomes +buffered-or-stream); once it lands on `main`, the Relay plugin can fulfill +promises with live streams and this table's last row flips. ## The embedding contract From 9af402f11f443fe7414c1901bba0c39f8ab4c6a7 Mon Sep 17 00:00:00 2001 From: Ryan Lempka Date: Wed, 15 Jul 2026 13:19:25 -0500 Subject: [PATCH 04/12] docs: complete Claude Code e2e target with full configs and verified break demo Signed-off-by: Ryan Lempka --- docs/guides/nemo_relay.md | 115 ++++++++++++++++++++++++++++++++------ 1 file changed, 98 insertions(+), 17 deletions(-) diff --git a/docs/guides/nemo_relay.md b/docs/guides/nemo_relay.md index 5472375a..708dc389 100644 --- a/docs/guides/nemo_relay.md +++ b/docs/guides/nemo_relay.md @@ -106,12 +106,23 @@ base_url = "http://127.0.0.1:4102" To point at real providers, replace the target `base_url` / `model` values and supply credentials with per-target `headers` / `header_env`. -## Launching Claude Code (tested), and where it breaks +## Launching Claude Code: the end-to-end we want -From a project directory containing the two config files below, -`nemo-relay claude` opens the normal interactive Claude Code TUI with the -plugin active. Verified end to end on 2026-07-15 with Relay branch -`feat/libsy-decision-backend` ([fork](https://github.com/ryan-lempka/NeMo-Relay/tree/feat/libsy-decision-backend)). +The target experience: from a project directory with the two files below, +`nemo-relay claude` opens the normal interactive Claude Code TUI, and every +LLM call the agent makes is scored by the classifier and routed strong/weak +in-process, with the full decision trail in ATOF events. Everything here is +copy-pasteable; the only unmet dependency is libsy streaming (see the status +table). + +Build Relay with the plugin (reference branch: +[feat/libsy-decision-backend](https://github.com/ryan-lempka/NeMo-Relay/tree/feat/libsy-decision-backend)): + +```bash +git clone -b feat/libsy-decision-backend https://github.com/ryan-lempka/NeMo-Relay.git +cd NeMo-Relay +RUSTUP_TOOLCHAIN=1.96.1 cargo build -p nemo-relay-cli --features switchyard +``` `.nemo-relay/config.toml`: @@ -120,30 +131,100 @@ plugin active. Verified end to end on 2026-07-15 with Relay branch command = "claude" ``` -`.nemo-relay/plugins.toml`: the libsy component config above, with -Anthropic-protocol targets (e.g. a Haiku classifier routing between Opus and -Sonnet) plus an `observability` component with a `file` ATOF sink to see -routing events. +`.nemo-relay/plugins.toml` (complete; Haiku scores each request, at or above +0.5 routes to Opus, below routes to Sonnet; Claude Code's own auth headers +pass through to Anthropic): + +```toml +version = 1 + +[[components]] +kind = "switchyard" +enabled = true + +[components.config] +mode = "enforce" +decision_backend = "libsy" +request_materialization = "summary_only" +context_mode = "payload_only" +enabled_inbound_profiles = ["anthropic_messages"] + +[components.config.libsy] +algorithm = "llm_classifier" +classifier_target = "classifier" +strong_target = "strong" +weak_target = "weak" +threshold = 0.5 + +[components.config.default_targets] +anthropic_messages = "strong" + +[components.config.targets.classifier] +model = "claude-haiku-4-5-20251001" +protocol = "anthropic_messages" +endpoint = "/v1/messages" +base_url = "https://api.anthropic.com" + +[components.config.targets.strong] +model = "claude-opus-4-8" +protocol = "anthropic_messages" +endpoint = "/v1/messages" +base_url = "https://api.anthropic.com" + +[components.config.targets.weak] +model = "claude-sonnet-5" +protocol = "anthropic_messages" +endpoint = "/v1/messages" +base_url = "https://api.anthropic.com" + +[[components]] +kind = "observability" +enabled = true + +[components.config.atof] +enabled = true + +[[components.config.atof.sinks]] +type = "file" +mode = "append" +``` + +Launch and inspect: + +```bash +./target/debug/nemo-relay claude # interactive TUI +grep switchyard.routing nemo-relay-events-*.jsonl +``` -What works today, against libsy `main`: +## Status against libsy `main` (verified 2026-07-15, fresh public checkout) | Step | Status | |---|---| +| Fresh clone builds hermetically; `run-libsy-e2e.sh` passes | works | | TUI launches; all traffic flows through the gateway and plugin | works | | Requests carrying `cache_control` reach the router (same-protocol passthrough) | works | | Buffered requests routed by the libsy classifier | works | | Streamed requests routed by libsy | **breaks** | -The break is in libsy's response contract, not in Relay: +The break, demonstrated with the same prompt sent both ways through the demo +stack: + +```text +buffered "This is a hard problem: ..." -> classifier scored 0.9 -> served by strong-model +streamed "This is a hard problem: ..." -> no classifier call -> served by weak-model (trusted fallback) +``` + +The wrong model answers the streamed request purely because of transport +framing. The cause is libsy's response contract on `main`: `CallLlmRequest::respond` and `Step::ReturnToAgent` carry only buffered responses, so no host can pass a live token stream through an algorithm. -Because Claude Code streams its interactive calls, those requests emit +Claude Code streams its interactive calls, so those requests emit `switchyard.routing.error` (`libsy_stream`) and dispatch the trusted -per-protocol fallback (`switchyard.routing.fallback`, -`libsy_streaming_unsupported`) instead of a libsy decision. Streaming support -exists on the unmerged `grclark/simple-proxy` branch (`LlmResponse` becomes -buffered-or-stream); once it lands on `main`, the Relay plugin can fulfill -promises with live streams and this table's last row flips. +per-protocol fallback (`libsy_streaming_unsupported`) instead of a libsy +decision. Streaming support exists on the unmerged `grclark/simple-proxy` +branch (`LlmResponse` becomes buffered-or-stream); when it lands on `main`, +the Relay plugin can fulfill promises with live streams and the last table +row flips without config changes. ## The embedding contract From 363880e7e985e4280326d9bfc3ffc34d82c36f73 Mon Sep 17 00:00:00 2001 From: Ryan Lempka Date: Wed, 15 Jul 2026 13:26:24 -0500 Subject: [PATCH 05/12] docs: align NeMo Relay recipe with the quickstart shape, move configs to examples Signed-off-by: Ryan Lempka --- docs/guides/nemo_relay.md | 259 ++++++++----------------------- examples/nemo_relay/config.toml | 5 + examples/nemo_relay/plugins.toml | 58 +++++++ 3 files changed, 130 insertions(+), 192 deletions(-) create mode 100644 examples/nemo_relay/config.toml create mode 100644 examples/nemo_relay/plugins.toml diff --git a/docs/guides/nemo_relay.md b/docs/guides/nemo_relay.md index 708dc389..d47d88e4 100644 --- a/docs/guides/nemo_relay.md +++ b/docs/guides/nemo_relay.md @@ -1,241 +1,116 @@ # NeMo Relay Recipe -This recipe shows how to run Switchyard **as a library** inside -[NeMo Relay](https://github.com/NVIDIA/NeMo-Relay), so routing decisions are -made in-process by [libsy](../../crates/libsy/README.md) -while Relay keeps ownership of the actual LLM calls. No `switchyard serve` -process is involved. +Run Switchyard **as a library** inside [NeMo Relay](https://github.com/NVIDIA/NeMo-Relay): +routing decisions are made in-process by [libsy](../../crates/libsy/README.md) +while Relay owns the proxy, dispatch, credentials, and observability. No +`switchyard serve` process is involved. + +| Mode | Decision path | Switchyard process | +|---|---|---| +| Server (`topic/nemo-relay-integration`) | Relay POSTs to the HTTP Decision API | required | +| Library (this recipe) | Relay calls libsy in-process | none | + +libsy never performs a network call: each offloaded `CallLlm` promise is +fulfilled by Relay's dispatch chain and answered with +`CallLlmRequest::respond(Ok/Err)`. libsy's semantic target names map onto the +Relay plugin's `TargetBinding` table, which binds each name to a backend URL, +model, and protocol. -> **Status:** the buffered request path is validated end to end against -> NeMo Relay's `feat/libsy-decision-backend` branch (classifier call and -> routed call both fulfilled by Relay dispatch). Streaming requests are not -> yet routed by libsy; they dispatch Relay's trusted per-protocol fallback. - -## Two ways to pair Switchyard with Relay +## Prerequisites -| Mode | Decision path | Switchyard process | Where it lives | -|---|---|---|---| -| Server | Relay POSTs to Switchyard's HTTP Decision API | Required (`switchyard serve`) | Relay `crates/switchyard` plugin, Switchyard `topic/nemo-relay-integration` | -| Library (this recipe) | Relay calls libsy in-process | None | `decision_backend = "libsy"` in Relay's Switchyard plugin | +- Rust 1.96.1 (Switchyard's MSRV; Relay pins 1.93.0, so prefix builds with + `RUSTUP_TOOLCHAIN=1.96.1` until the toolchains converge) +- Claude Code, for the agent launcher path -Relay's Switchyard plugin README anticipated this: a future in-process -decision provider replacing the HTTP Decision API call without changing -Relay-owned dispatch and observability. `decision_backend = "libsy"` is that -provider. +## Install -## How the pieces fit +Build the Relay CLI with the Switchyard plugin (reference branch: +[feat/libsy-decision-backend](https://github.com/ryan-lempka/NeMo-Relay/tree/feat/libsy-decision-backend)): -```text -agent (e.g. Claude Code) - │ Anthropic / OpenAI wire protocol - ▼ -Relay gateway (local proxy) - │ managed LLM call - ▼ -LLM execution intercept (Relay's Switchyard plugin) - │ - ├─ libsy Algorithm decides which target to call - │ Step::CallLlm ──▶ Relay fulfills via next(request) ──▶ provider - │ Step::Decision ──▶ switchyard.routing.decision mark events - │ Step::ReturnToAgent ──▶ final response - ▼ -response translated back to the inbound protocol +```bash +git clone -b feat/libsy-decision-backend https://github.com/ryan-lempka/NeMo-Relay.git +cd NeMo-Relay +RUSTUP_TOOLCHAIN=1.96.1 cargo build -p nemo-relay-cli --features switchyard ``` -libsy never performs a network call. Each offloaded `CallLlm` promise is -fulfilled by Relay's own dispatch chain (`next(request)`), and the result is -handed back with `CallLlmRequest::respond(Ok/Err)`. The classifier's scoring -call is a Relay-managed provider call like any other, so it shows up in -Relay's observability. libsy's semantic target names (`"strong"`, `"weak"`, -`"classifier"`) map onto the plugin's existing `TargetBinding` table, which -binds each name to a Relay-owned backend URL, model, and protocol. - -## Prerequisites +## Configure -- Rust 1.96.1 (Switchyard's MSRV; note Relay pins 1.93.0, so build Relay with - `RUSTUP_TOOLCHAIN=1.96.1` until the toolchains converge) -- NeMo Relay checkout on the `feat/libsy-decision-backend` branch -- Provider credentials for your targets (the demo below needs none) +Copy the two example configs into the project you will launch from: -## Quick start (validated) +- [examples/nemo_relay/config.toml](../../examples/nemo_relay/config.toml) → `.nemo-relay/config.toml` +- [examples/nemo_relay/plugins.toml](../../examples/nemo_relay/plugins.toml) → `.nemo-relay/plugins.toml` -Run the self-contained demo from a NeMo Relay checkout: +The plugin config selects `decision_backend = "libsy"` with the LLM-classifier +algorithm: a Haiku target scores each request, and the score routes to an Opus +(strong) or Sonnet (weak) target. Claude Code's own auth headers pass through +to Anthropic. Verify with: ```bash -RUSTUP_TOOLCHAIN=1.96.1 ./examples/switchyard/run-libsy-e2e.sh +./target/debug/nemo-relay doctor ``` -This starts a deterministic fake provider and Relay with the libsy backend, -then verifies an easy prompt routes to the weak model and a hard prompt -routes to the strong model, with the classifier and routed calls both flowing -through Relay dispatch. - -The plugin configuration (`examples/switchyard/libsy-plugins.toml`): - -```toml -[[components]] -kind = "switchyard" -enabled = true - -[components.config] -mode = "enforce" -decision_backend = "libsy" -request_materialization = "summary_only" -context_mode = "payload_only" -enabled_inbound_profiles = ["openai_chat"] - -[components.config.libsy] -algorithm = "llm_classifier" -classifier_target = "classifier" -strong_target = "strong" -weak_target = "weak" -threshold = 0.5 - -[components.config.default_targets] -openai_chat = "weak" - -[components.config.targets.classifier] -model = "classifier-model" -protocol = "openai_chat" -endpoint = "/v1/chat/completions" -base_url = "http://127.0.0.1:4102" - -# targets.strong and targets.weak follow the same shape -``` - -To point at real providers, replace the target `base_url` / `model` values and -supply credentials with per-target `headers` / `header_env`. - -## Launching Claude Code: the end-to-end we want +## Path A: Gateway mode -The target experience: from a project directory with the two files below, -`nemo-relay claude` opens the normal interactive Claude Code TUI, and every -LLM call the agent makes is scored by the classifier and routed strong/weak -in-process, with the full decision trail in ATOF events. Everything here is -copy-pasteable; the only unmet dependency is libsy streaming (see the status -table). - -Build Relay with the plugin (reference branch: -[feat/libsy-decision-backend](https://github.com/ryan-lempka/NeMo-Relay/tree/feat/libsy-decision-backend)): +Runs Relay's gateway with libsy routing and a deterministic fake provider; no +credentials needed. This is the self-contained e2e shipped on the Relay branch: ```bash -git clone -b feat/libsy-decision-backend https://github.com/ryan-lempka/NeMo-Relay.git -cd NeMo-Relay -RUSTUP_TOOLCHAIN=1.96.1 cargo build -p nemo-relay-cli --features switchyard +RUSTUP_TOOLCHAIN=1.96.1 ./examples/switchyard/run-libsy-e2e.sh ``` -`.nemo-relay/config.toml`: +It verifies an easy prompt routes weak, a hard prompt routes strong, and the +classifier and routed calls both flow through Relay dispatch. Test by hand +with curl against `http://127.0.0.1:4042/v1/chat/completions`. -```toml -[agents.claude] -command = "claude" -``` +## Path B: Agent launcher + +From the directory containing `.nemo-relay/`: -`.nemo-relay/plugins.toml` (complete; Haiku scores each request, at or above -0.5 routes to Opus, below routes to Sonnet; Claude Code's own auth headers -pass through to Anthropic): - -```toml -version = 1 - -[[components]] -kind = "switchyard" -enabled = true - -[components.config] -mode = "enforce" -decision_backend = "libsy" -request_materialization = "summary_only" -context_mode = "payload_only" -enabled_inbound_profiles = ["anthropic_messages"] - -[components.config.libsy] -algorithm = "llm_classifier" -classifier_target = "classifier" -strong_target = "strong" -weak_target = "weak" -threshold = 0.5 - -[components.config.default_targets] -anthropic_messages = "strong" - -[components.config.targets.classifier] -model = "claude-haiku-4-5-20251001" -protocol = "anthropic_messages" -endpoint = "/v1/messages" -base_url = "https://api.anthropic.com" - -[components.config.targets.strong] -model = "claude-opus-4-8" -protocol = "anthropic_messages" -endpoint = "/v1/messages" -base_url = "https://api.anthropic.com" - -[components.config.targets.weak] -model = "claude-sonnet-5" -protocol = "anthropic_messages" -endpoint = "/v1/messages" -base_url = "https://api.anthropic.com" - -[[components]] -kind = "observability" -enabled = true - -[components.config.atof] -enabled = true - -[[components.config.atof.sinks]] -type = "file" -mode = "append" +```bash +./target/debug/nemo-relay claude ``` -Launch and inspect: +This opens the normal interactive Claude Code TUI through the Relay gateway. +Inspect routing decisions afterwards: ```bash -./target/debug/nemo-relay claude # interactive TUI grep switchyard.routing nemo-relay-events-*.jsonl ``` -## Status against libsy `main` (verified 2026-07-15, fresh public checkout) +## Status against libsy `main` (verified 2026-07-15) | Step | Status | |---|---| -| Fresh clone builds hermetically; `run-libsy-e2e.sh` passes | works | +| Fresh clone builds hermetically; Path A e2e passes | works | | TUI launches; all traffic flows through the gateway and plugin | works | -| Requests carrying `cache_control` reach the router (same-protocol passthrough) | works | +| Requests carrying `cache_control` reach the router | works | | Buffered requests routed by the libsy classifier | works | | Streamed requests routed by libsy | **breaks** | -The break, demonstrated with the same prompt sent both ways through the demo -stack: +The break, demonstrated with the same prompt sent both ways through Path A: ```text buffered "This is a hard problem: ..." -> classifier scored 0.9 -> served by strong-model streamed "This is a hard problem: ..." -> no classifier call -> served by weak-model (trusted fallback) ``` -The wrong model answers the streamed request purely because of transport -framing. The cause is libsy's response contract on `main`: -`CallLlmRequest::respond` and `Step::ReturnToAgent` carry only buffered -responses, so no host can pass a live token stream through an algorithm. -Claude Code streams its interactive calls, so those requests emit -`switchyard.routing.error` (`libsy_stream`) and dispatch the trusted -per-protocol fallback (`libsy_streaming_unsupported`) instead of a libsy -decision. Streaming support exists on the unmerged `grclark/simple-proxy` -branch (`LlmResponse` becomes buffered-or-stream); when it lands on `main`, -the Relay plugin can fulfill promises with live streams and the last table -row flips without config changes. +The cause is libsy's response contract on `main`: `CallLlmRequest::respond` +and `Step::ReturnToAgent` carry only buffered responses, so no host can pass +a live token stream through an algorithm. Claude Code streams its interactive +calls, so those requests emit `switchyard.routing.error` (`libsy_stream`) and +dispatch the trusted per-protocol fallback (`libsy_streaming_unsupported`) +instead of a libsy decision. Streaming support exists on the unmerged +`grclark/simple-proxy` branch; when it lands, the last table row flips with no +config changes. ## The embedding contract -The host (Relay) needs exactly three things from libsy: +A host needs three things from libsy: -- `Algorithm` trait object: one shared `Arc` serves concurrent - requests with no lock. -- `run_stream(ctx, request)`: returns a stream of `Step`s. The host serves - `Step::CallLlm` items itself and fulfills each with `respond(...)`. +- `Algorithm`: one shared `Arc` serves concurrent requests. +- `run_stream(ctx, request)`: a stream of `Step`s; the host serves each + `Step::CallLlm` and fulfills it with `respond(...)`. - The conversation IR (`LlmRequest` / `LlmResponse` from `libsy-protocol`), - which Relay's plugin already speaks through `switchyard-translation`. + which Relay already speaks through `switchyard-translation`. -See the [libsy README](../../crates/libsy/README.md) -for the full API walkthrough. +See the [libsy README](../../crates/libsy/README.md) for the full API. diff --git a/examples/nemo_relay/config.toml b/examples/nemo_relay/config.toml new file mode 100644 index 00000000..395adc6e --- /dev/null +++ b/examples/nemo_relay/config.toml @@ -0,0 +1,5 @@ +# NeMo Relay agent config for the Switchyard library-mode quickstart. +# Place at .nemo-relay/config.toml in the project you launch from. + +[agents.claude] +command = "claude" diff --git a/examples/nemo_relay/plugins.toml b/examples/nemo_relay/plugins.toml new file mode 100644 index 00000000..c84d0776 --- /dev/null +++ b/examples/nemo_relay/plugins.toml @@ -0,0 +1,58 @@ +# NeMo Relay plugin config: in-process libsy LLM-classifier routing. +# Place at .nemo-relay/plugins.toml in the project you launch from. +# +# Haiku scores each request; at or above the threshold routes to Opus, +# below it routes to Sonnet. Claude Code's own auth headers pass through +# to Anthropic, so no keys are needed here. The observability component +# writes routing events to nemo-relay-events-*.jsonl in the working dir. +version = 1 + +[[components]] +kind = "switchyard" +enabled = true + +[components.config] +mode = "enforce" +decision_backend = "libsy" +request_materialization = "summary_only" +context_mode = "payload_only" +enabled_inbound_profiles = ["anthropic_messages"] + +[components.config.libsy] +algorithm = "llm_classifier" +classifier_target = "classifier" +strong_target = "strong" +weak_target = "weak" +threshold = 0.5 + +[components.config.default_targets] +anthropic_messages = "strong" + +[components.config.targets.classifier] +model = "claude-haiku-4-5-20251001" +protocol = "anthropic_messages" +endpoint = "/v1/messages" +base_url = "https://api.anthropic.com" + +[components.config.targets.strong] +model = "claude-opus-4-8" +protocol = "anthropic_messages" +endpoint = "/v1/messages" +base_url = "https://api.anthropic.com" + +[components.config.targets.weak] +model = "claude-sonnet-5" +protocol = "anthropic_messages" +endpoint = "/v1/messages" +base_url = "https://api.anthropic.com" + +[[components]] +kind = "observability" +enabled = true + +[components.config.atof] +enabled = true + +[[components.config.atof.sinks]] +type = "file" +mode = "append" From 9e48ab042c4049ba90f7d9b2617f01bf2aae3788 Mon Sep 17 00:00:00 2001 From: Ryan Lempka Date: Wed, 15 Jul 2026 13:28:08 -0500 Subject: [PATCH 06/12] docs: split NeMo Relay recipe into clean quickstart and known-issues tracker Signed-off-by: Ryan Lempka --- docs/guides/nemo_relay.md | 61 ++++++++++-------------- docs/guides/nemo_relay_known_issues.md | 66 ++++++++++++++++++++++++++ mkdocs.yml | 1 + 3 files changed, 93 insertions(+), 35 deletions(-) create mode 100644 docs/guides/nemo_relay_known_issues.md diff --git a/docs/guides/nemo_relay.md b/docs/guides/nemo_relay.md index d47d88e4..d720ec7f 100644 --- a/docs/guides/nemo_relay.md +++ b/docs/guides/nemo_relay.md @@ -5,21 +5,18 @@ routing decisions are made in-process by [libsy](../../crates/libsy/README.md) while Relay owns the proxy, dispatch, credentials, and observability. No `switchyard serve` process is involved. -| Mode | Decision path | Switchyard process | -|---|---|---| -| Server (`topic/nemo-relay-integration`) | Relay POSTs to the HTTP Decision API | required | -| Library (this recipe) | Relay calls libsy in-process | none | - libsy never performs a network call: each offloaded `CallLlm` promise is fulfilled by Relay's dispatch chain and answered with `CallLlmRequest::respond(Ok/Err)`. libsy's semantic target names map onto the Relay plugin's `TargetBinding` table, which binds each name to a backend URL, model, and protocol. +Current gaps are tracked separately in +[NeMo Relay: Known Issues](nemo_relay_known_issues.md). + ## Prerequisites -- Rust 1.96.1 (Switchyard's MSRV; Relay pins 1.93.0, so prefix builds with - `RUSTUP_TOOLCHAIN=1.96.1` until the toolchains converge) +- Rust 1.96.1 - Claude Code, for the agent launcher path ## Install @@ -52,15 +49,20 @@ to Anthropic. Verify with: ## Path A: Gateway mode Runs Relay's gateway with libsy routing and a deterministic fake provider; no -credentials needed. This is the self-contained e2e shipped on the Relay branch: +credentials needed: ```bash RUSTUP_TOOLCHAIN=1.96.1 ./examples/switchyard/run-libsy-e2e.sh ``` -It verifies an easy prompt routes weak, a hard prompt routes strong, and the -classifier and routed calls both flow through Relay dispatch. Test by hand -with curl against `http://127.0.0.1:4042/v1/chat/completions`. +Expected receipt: + +```text +ok: easy prompt routes weak +ok: hard prompt routes strong +ok: classifier and routed calls both flowed through Relay dispatch (4 upstream calls) +libsy in-process routing e2e passed +``` ## Path B: Agent launcher @@ -71,37 +73,26 @@ From the directory containing `.nemo-relay/`: ``` This opens the normal interactive Claude Code TUI through the Relay gateway. -Inspect routing decisions afterwards: -```bash -grep switchyard.routing nemo-relay-events-*.jsonl -``` +## Validate -## Status against libsy `main` (verified 2026-07-15) +Routing decisions land in `nemo-relay-events-*.jsonl` in the working +directory. One line per decision: -| Step | Status | -|---|---| -| Fresh clone builds hermetically; Path A e2e passes | works | -| TUI launches; all traffic flows through the gateway and plugin | works | -| Requests carrying `cache_control` reach the router | works | -| Buffered requests routed by the libsy classifier | works | -| Streamed requests routed by libsy | **breaks** | +```bash +grep -h switchyard.routing.decision nemo-relay-events-*.jsonl +``` -The break, demonstrated with the same prompt sent both ways through Path A: +Expected receipt: `switchyard.routing.decision` marks naming the classifier +step and the routed target, e.g. -```text -buffered "This is a hard problem: ..." -> classifier scored 0.9 -> served by strong-model -streamed "This is a hard problem: ..." -> no classifier call -> served by weak-model (trusted fallback) +```json +{"backend_id": "classifier", "reason_summary": "classifying request via classifier", ...} +{"backend_id": "weak", "target_model": "claude-sonnet-5", ...} ``` -The cause is libsy's response contract on `main`: `CallLlmRequest::respond` -and `Step::ReturnToAgent` carry only buffered responses, so no host can pass -a live token stream through an algorithm. Claude Code streams its interactive -calls, so those requests emit `switchyard.routing.error` (`libsy_stream`) and -dispatch the trusted per-protocol fallback (`libsy_streaming_unsupported`) -instead of a libsy decision. Streaming support exists on the unmerged -`grclark/simple-proxy` branch; when it lands, the last table row flips with no -config changes. +A decision naming `weak` or `strong` is the proof: libsy scored the request +in-process and Relay dispatched the routed call. ## The embedding contract diff --git a/docs/guides/nemo_relay_known_issues.md b/docs/guides/nemo_relay_known_issues.md new file mode 100644 index 00000000..48d67955 --- /dev/null +++ b/docs/guides/nemo_relay_known_issues.md @@ -0,0 +1,66 @@ +# NeMo Relay: Known Issues + +What stands between the [NeMo Relay Recipe](nemo_relay.md) and a fully routed +interactive Claude Code session. Status verified 2026-07-15 against libsy +`main` from a fresh public checkout. + +| Step | Status | +|---|---| +| Fresh clone builds hermetically; Path A e2e passes | works | +| Claude Code TUI launches; all traffic flows through the gateway and plugin | works | +| Requests carrying `cache_control` reach the router | works | +| Buffered requests routed by the libsy classifier | works | +| Streamed requests routed by libsy | **breaks** | +| Classifier scoring call accepted by Anthropic | **breaks** | + +## 1. libsy has no streaming response path + +`CallLlmRequest::respond` and `Step::ReturnToAgent` carry only buffered +responses, so no host can pass a live token stream through an algorithm. +Claude Code streams its interactive calls, so those requests emit +`switchyard.routing.error` (`libsy_stream`) and dispatch the trusted +per-protocol fallback (`libsy_streaming_unsupported`) instead of a libsy +decision. Demonstrated with the same prompt sent both ways through Path A: + +```text +buffered "This is a hard problem: ..." -> classifier scored 0.9 -> served by strong-model +streamed "This is a hard problem: ..." -> no classifier call -> served by weak-model (trusted fallback) +``` + +**Owner:** libsy. Streaming support exists on the unmerged +`grclark/simple-proxy` branch (`LlmResponse` becomes buffered-or-stream); when +it lands on `main`, the Relay plugin can fulfill promises with live streams +and no recipe config changes are needed. + +## 2. Classifier scoring call rejected upstream (HTTP 400) + +In a live TUI session, Claude Code's buffered utility calls reached libsy and +produced classifier decisions, but the scoring call to Anthropic failed with +`invalid_request:http_400`. Leading suspect: `LlmClassifierOrchAlgo` builds +its request with `..LlmRequest::default()` (no output params) and the +Anthropic encoder defaults `max_tokens` to 128,000 when unset, above Haiku's +output ceiling. + +**Owner:** switchyard-translation (encoder default) and/or libsy reference +algorithms (set explicit output params on synthesized calls). + +## 3. No fail-open on classifier call errors + +`LlmClassifierOrchAlgo` fails open only on an unparseable score. A classifier +call *error* propagates and kills the run, so the whole request falls back +(`decision_error`) instead of routing strong. + +**Owner:** libsy reference algorithms. + +## 4. Provider errors are opaque in routing events + +The Relay plugin condenses upstream failures to summaries like +`invalid_request:http_400`, discarding the response body. Issue 2 had to be +root-caused from source instead of logs. + +**Owner:** Relay plugin (include a bounded body snippet in error marks). + +## Toolchain note + +Relay pins rustc 1.93.0; Switchyard's MSRV is 1.96.1. Until the toolchains +converge, prefix Relay builds with `RUSTUP_TOOLCHAIN=1.96.1`. diff --git a/mkdocs.yml b/mkdocs.yml index 83f29cb3..dd04ba8c 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -23,6 +23,7 @@ nav: - Guides: - Agent Launchers: guides/agent_launchers.md - NeMo Relay: guides/nemo_relay.md + - "NeMo Relay: Known Issues": guides/nemo_relay_known_issues.md - Skill Distillation: skill_distillation.md - Concepts: - Core Concepts: core_concepts.md From c787326dd9a10263c3065ba1b8e29684d4dbae0f Mon Sep 17 00:00:00 2001 From: Ryan Lempka Date: Wed, 15 Jul 2026 13:34:18 -0500 Subject: [PATCH 07/12] docs: tighten NeMo Relay known issues Signed-off-by: Ryan Lempka --- docs/guides/nemo_relay_known_issues.md | 46 ++++++++++++-------------- 1 file changed, 21 insertions(+), 25 deletions(-) diff --git a/docs/guides/nemo_relay_known_issues.md b/docs/guides/nemo_relay_known_issues.md index 48d67955..2ecf8408 100644 --- a/docs/guides/nemo_relay_known_issues.md +++ b/docs/guides/nemo_relay_known_issues.md @@ -1,8 +1,8 @@ # NeMo Relay: Known Issues -What stands between the [NeMo Relay Recipe](nemo_relay.md) and a fully routed -interactive Claude Code session. Status verified 2026-07-15 against libsy -`main` from a fresh public checkout. +Gaps between the [NeMo Relay Recipe](nemo_relay.md) and a fully routed +interactive Claude Code session. Verified 2026-07-15 against libsy `main` +from a fresh public checkout. | Step | Status | |---|---| @@ -16,39 +16,35 @@ interactive Claude Code session. Status verified 2026-07-15 against libsy ## 1. libsy has no streaming response path `CallLlmRequest::respond` and `Step::ReturnToAgent` carry only buffered -responses, so no host can pass a live token stream through an algorithm. -Claude Code streams its interactive calls, so those requests emit -`switchyard.routing.error` (`libsy_stream`) and dispatch the trusted -per-protocol fallback (`libsy_streaming_unsupported`) instead of a libsy -decision. Demonstrated with the same prompt sent both ways through Path A: +responses, so a host cannot pass a live token stream through an algorithm. +Claude Code streams its interactive calls, so they dispatch the trusted +fallback (`libsy_streaming_unsupported`) instead of a libsy decision. Same +prompt, sent both ways through Path A: ```text -buffered "This is a hard problem: ..." -> classifier scored 0.9 -> served by strong-model -streamed "This is a hard problem: ..." -> no classifier call -> served by weak-model (trusted fallback) +buffered -> classifier scored 0.9 -> served by strong-model +streamed -> no classifier call -> served by weak-model (fallback) ``` -**Owner:** libsy. Streaming support exists on the unmerged -`grclark/simple-proxy` branch (`LlmResponse` becomes buffered-or-stream); when -it lands on `main`, the Relay plugin can fulfill promises with live streams -and no recipe config changes are needed. +**Owner:** libsy. Streaming exists on the unmerged `grclark/simple-proxy` +branch; when it lands, this fixes with no recipe changes. ## 2. Classifier scoring call rejected upstream (HTTP 400) -In a live TUI session, Claude Code's buffered utility calls reached libsy and -produced classifier decisions, but the scoring call to Anthropic failed with -`invalid_request:http_400`. Leading suspect: `LlmClassifierOrchAlgo` builds -its request with `..LlmRequest::default()` (no output params) and the -Anthropic encoder defaults `max_tokens` to 128,000 when unset, above Haiku's -output ceiling. +In a live TUI session, buffered calls reached libsy and produced classifier +decisions, but the scoring call to Anthropic failed with +`invalid_request:http_400`. Leading suspect: `LlmClassifierOrchAlgo` sets no +output params and the Anthropic encoder defaults `max_tokens` to 128,000, +above Haiku's output ceiling. Not yet proven; see issue 4. -**Owner:** switchyard-translation (encoder default) and/or libsy reference +**Owner:** switchyard-translation (encoder default) and libsy reference algorithms (set explicit output params on synthesized calls). ## 3. No fail-open on classifier call errors `LlmClassifierOrchAlgo` fails open only on an unparseable score. A classifier call *error* propagates and kills the run, so the whole request falls back -(`decision_error`) instead of routing strong. +instead of routing strong. **Owner:** libsy reference algorithms. @@ -58,9 +54,9 @@ The Relay plugin condenses upstream failures to summaries like `invalid_request:http_400`, discarding the response body. Issue 2 had to be root-caused from source instead of logs. -**Owner:** Relay plugin (include a bounded body snippet in error marks). +**Owner:** Relay plugin. ## Toolchain note -Relay pins rustc 1.93.0; Switchyard's MSRV is 1.96.1. Until the toolchains -converge, prefix Relay builds with `RUSTUP_TOOLCHAIN=1.96.1`. +Relay pins rustc 1.93.0; Switchyard's MSRV is 1.96.1. Prefix Relay builds with +`RUSTUP_TOOLCHAIN=1.96.1` until the toolchains converge. From 363360d72f6909e688243921d98c89fa26d3fb90 Mon Sep 17 00:00:00 2001 From: Ryan Lempka Date: Wed, 15 Jul 2026 13:53:39 -0500 Subject: [PATCH 08/12] docs: proven root cause for classifier 400, confirm issue ownership Signed-off-by: Ryan Lempka --- docs/guides/nemo_relay_known_issues.md | 36 ++++++++++++++++++-------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/docs/guides/nemo_relay_known_issues.md b/docs/guides/nemo_relay_known_issues.md index 2ecf8408..f20fdd43 100644 --- a/docs/guides/nemo_relay_known_issues.md +++ b/docs/guides/nemo_relay_known_issues.md @@ -31,14 +31,20 @@ branch; when it lands, this fixes with no recipe changes. ## 2. Classifier scoring call rejected upstream (HTTP 400) -In a live TUI session, buffered calls reached libsy and produced classifier -decisions, but the scoring call to Anthropic failed with -`invalid_request:http_400`. Leading suspect: `LlmClassifierOrchAlgo` sets no -output params and the Anthropic encoder defaults `max_tokens` to 128,000, -above Haiku's output ceiling. Not yet proven; see issue 4. +Root cause proven by capturing the request the plugin sends: +`LlmClassifierOrchAlgo` sets no output params (`..LlmRequest::default()`, +`llm_class.rs`), and the Anthropic encoder injects a hardcoded default of +`max_tokens: 128000` when unset (`codecs/anthropic/buffered.rs:207`). Haiku +4.5's output ceiling is 64,000, so Anthropic rejects the scoring call with +HTTP 400 and the whole request falls back (via issue 3). + +```json +{"model": "claude-haiku-4-5-20251001", "max_tokens": 128000, "messages": [...]} +``` -**Owner:** switchyard-translation (encoder default) and libsy reference -algorithms (set explicit output params on synthesized calls). +**Owner:** switchyard-translation (remove or clamp the 128k default) and libsy +reference algorithms (set explicit small output params; a score needs a few +tokens). Not a Relay issue. ## 3. No fail-open on classifier call errors @@ -50,11 +56,19 @@ instead of routing strong. ## 4. Provider errors are opaque in routing events -The Relay plugin condenses upstream failures to summaries like -`invalid_request:http_400`, discarding the response body. Issue 2 had to be -root-caused from source instead of logs. +The Relay plugin's `provider_error_summary` formats upstream failures as +`class:http_status` and discards the response body. Issue 2 had to be +root-caused by capturing the request instead of reading the logs. + +**Owner:** Relay plugin. Not a libsy issue. + +## 5. Classifier prompt includes system boilerplate (minor) -**Owner:** Relay plugin. +`LlmClassifierOrchAlgo` builds its scoring prompt by flattening all text in +the request, so agent system content (billing headers, system reminders) +precedes the user's question. Does not fail, but skews scores. + +**Owner:** libsy reference algorithms. ## Toolchain note From 2b12521c2fbdfbed97ca8863a2fb1094ad11bd4f Mon Sep 17 00:00:00 2001 From: Ryan Lempka Date: Wed, 15 Jul 2026 13:54:55 -0500 Subject: [PATCH 09/12] docs: split known issues into Switchyard and Relay sections Signed-off-by: Ryan Lempka --- docs/guides/nemo_relay_known_issues.md | 59 +++++++++++++------------- 1 file changed, 29 insertions(+), 30 deletions(-) diff --git a/docs/guides/nemo_relay_known_issues.md b/docs/guides/nemo_relay_known_issues.md index f20fdd43..c55cf9b1 100644 --- a/docs/guides/nemo_relay_known_issues.md +++ b/docs/guides/nemo_relay_known_issues.md @@ -13,7 +13,9 @@ from a fresh public checkout. | Streamed requests routed by libsy | **breaks** | | Classifier scoring call accepted by Anthropic | **breaks** | -## 1. libsy has no streaming response path +## Switchyard / libsy improvements + +### S1. No streaming response path (libsy core) `CallLlmRequest::respond` and `Step::ReturnToAgent` carry only buffered responses, so a host cannot pass a live token stream through an algorithm. @@ -26,51 +28,48 @@ buffered -> classifier scored 0.9 -> served by strong-model streamed -> no classifier call -> served by weak-model (fallback) ``` -**Owner:** libsy. Streaming exists on the unmerged `grclark/simple-proxy` -branch; when it lands, this fixes with no recipe changes. +Streaming exists on the unmerged `grclark/simple-proxy` branch; when it +lands, this fixes with no recipe changes. -## 2. Classifier scoring call rejected upstream (HTTP 400) +### S2. Anthropic encoder defaults `max_tokens` to 128,000 (switchyard-translation) -Root cause proven by capturing the request the plugin sends: -`LlmClassifierOrchAlgo` sets no output params (`..LlmRequest::default()`, -`llm_class.rs`), and the Anthropic encoder injects a hardcoded default of -`max_tokens: 128000` when unset (`codecs/anthropic/buffered.rs:207`). Haiku -4.5's output ceiling is 64,000, so Anthropic rejects the scoring call with -HTTP 400 and the whole request falls back (via issue 3). +When a request sets no output params, the encoder injects +`max_tokens: 128000` (`codecs/anthropic/buffered.rs:207`). Haiku 4.5's +ceiling is 64,000, so Anthropic rejects the classifier's scoring call with +HTTP 400 and the whole request falls back (via S3). Proven by capturing the +sent request: ```json {"model": "claude-haiku-4-5-20251001", "max_tokens": 128000, "messages": [...]} ``` -**Owner:** switchyard-translation (remove or clamp the 128k default) and libsy -reference algorithms (set explicit small output params; a score needs a few -tokens). Not a Relay issue. +Fix pair: clamp or remove the encoder default, and have +`LlmClassifierOrchAlgo` set explicit small output params (a score needs a +few tokens). -## 3. No fail-open on classifier call errors +### S3. No fail-open on classifier call errors (libsy-examples) `LlmClassifierOrchAlgo` fails open only on an unparseable score. A classifier call *error* propagates and kills the run, so the whole request falls back instead of routing strong. -**Owner:** libsy reference algorithms. - -## 4. Provider errors are opaque in routing events - -The Relay plugin's `provider_error_summary` formats upstream failures as -`class:http_status` and discards the response body. Issue 2 had to be -root-caused by capturing the request instead of reading the logs. +### S4. Classifier prompt includes system boilerplate (libsy-examples, minor) -**Owner:** Relay plugin. Not a libsy issue. +The scoring prompt flattens all text in the request, so agent system content +(billing headers, system reminders) precedes the user's question. Does not +fail, but skews scores. -## 5. Classifier prompt includes system boilerplate (minor) +## NeMo Relay improvements -`LlmClassifierOrchAlgo` builds its scoring prompt by flattening all text in -the request, so agent system content (billing headers, system reminders) -precedes the user's question. Does not fail, but skews scores. +### R1. Provider errors are opaque in routing events -**Owner:** libsy reference algorithms. +The plugin's `provider_error_summary` formats upstream failures as +`class:http_status` and discards the response body. S2 had to be root-caused +by capturing the request instead of reading the logs. Fix: include a bounded +body snippet in error marks. -## Toolchain note +### R2. Toolchain pin below Switchyard's MSRV -Relay pins rustc 1.93.0; Switchyard's MSRV is 1.96.1. Prefix Relay builds with -`RUSTUP_TOOLCHAIN=1.96.1` until the toolchains converge. +Relay pins rustc 1.93.0; Switchyard's MSRV is 1.96.1. Prefix Relay builds +with `RUSTUP_TOOLCHAIN=1.96.1` until Relay bumps its pin (or Switchyard +lowers its MSRV). From e1ba8f78acbe074d42201ee98d922f441a5fadd6 Mon Sep 17 00:00:00 2001 From: Ryan Lempka Date: Wed, 15 Jul 2026 13:59:59 -0500 Subject: [PATCH 10/12] docs: S4 is a behavior change from the rewrite, scores wrong text Signed-off-by: Ryan Lempka --- docs/guides/nemo_relay_known_issues.md | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/docs/guides/nemo_relay_known_issues.md b/docs/guides/nemo_relay_known_issues.md index c55cf9b1..1c1c5aa8 100644 --- a/docs/guides/nemo_relay_known_issues.md +++ b/docs/guides/nemo_relay_known_issues.md @@ -53,11 +53,16 @@ few tokens). call *error* propagates and kills the run, so the whole request falls back instead of routing strong. -### S4. Classifier prompt includes system boilerplate (libsy-examples, minor) - -The scoring prompt flattens all text in the request, so agent system content -(billing headers, system reminders) precedes the user's question. Does not -fail, but skews scores. +### S4. Classifier scores system boilerplate, not the user's question (libsy-examples) + +`LlmClassifierOrchAlgo` builds its scoring prompt by flattening instructions +and all messages of every role, so agent system content (billing headers, +system reminders) precedes the user's question. Captured in a live session: +the prompt opens with Claude Code's billing header rather than the request +being scored. The preexisting Python classifier +(`routellm_request_processor.py`) scores only the latest user message; this +is a behavior change introduced with the rewrite. Requests do not fail, but +every routing score is computed over the wrong text. ## NeMo Relay improvements From 34012bd51af32fc59deb72dbf3b97036e80e9fde Mon Sep 17 00:00:00 2001 From: Ryan Lempka Date: Wed, 15 Jul 2026 14:03:06 -0500 Subject: [PATCH 11/12] docs: correct S4 against the llm-routing predecessor, note S3 fail-open precedent Signed-off-by: Ryan Lempka --- docs/guides/nemo_relay_known_issues.md | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/docs/guides/nemo_relay_known_issues.md b/docs/guides/nemo_relay_known_issues.md index 1c1c5aa8..2bbe5be2 100644 --- a/docs/guides/nemo_relay_known_issues.md +++ b/docs/guides/nemo_relay_known_issues.md @@ -51,18 +51,17 @@ few tokens). `LlmClassifierOrchAlgo` fails open only on an unparseable score. A classifier call *error* propagates and kills the run, so the whole request falls back -instead of routing strong. - -### S4. Classifier scores system boilerplate, not the user's question (libsy-examples) - -`LlmClassifierOrchAlgo` builds its scoring prompt by flattening instructions -and all messages of every role, so agent system content (billing headers, -system reminders) precedes the user's question. Captured in a live session: -the prompt opens with Claude Code's billing header rather than the request -being scored. The preexisting Python classifier -(`routellm_request_processor.py`) scores only the latest user message; this -is a behavior change introduced with the rewrite. Requests do not fail, but -every routing score is computed over the wrong text. +instead of routing strong. The previous `llm-routing` profile failed open on +classifier errors by default (`classifier_fail_open`). + +### S4. Classifier input is flattened and untrimmed (libsy-examples) + +`LlmClassifierOrchAlgo` flattens instructions and all messages of every role +into one string, and includes the full conversation on every score. The +previous `llm-routing` classifier sent a structured, trimmed selection +(system messages, first user message, recent turns) with a task-specific +prompt and structured output. Requests do not fail, but scores are computed +over unstructured, unbounded input. ## NeMo Relay improvements From ee3d051ee9e51acdba89f17c838c2241877adfcd Mon Sep 17 00:00:00 2001 From: Ryan Lempka Date: Fri, 17 Jul 2026 14:22:29 -0500 Subject: [PATCH 12/12] docs: reshape NeMo Relay recipe as customer quickstart with OpenRouter Signed-off-by: Ryan Lempka --- docs/guides/nemo_relay.md | 59 +++++++++---------- docs/guides/nemo_relay_known_issues.md | 79 -------------------------- examples/nemo_relay/plugins.toml | 37 ++++++------ mkdocs.yml | 1 - 4 files changed, 47 insertions(+), 129 deletions(-) delete mode 100644 docs/guides/nemo_relay_known_issues.md diff --git a/docs/guides/nemo_relay.md b/docs/guides/nemo_relay.md index d720ec7f..2fb49281 100644 --- a/docs/guides/nemo_relay.md +++ b/docs/guides/nemo_relay.md @@ -9,15 +9,16 @@ libsy never performs a network call: each offloaded `CallLlm` promise is fulfilled by Relay's dispatch chain and answered with `CallLlmRequest::respond(Ok/Err)`. libsy's semantic target names map onto the Relay plugin's `TargetBinding` table, which binds each name to a backend URL, -model, and protocol. - -Current gaps are tracked separately in -[NeMo Relay: Known Issues](nemo_relay_known_issues.md). +model, protocol, and credentials. Buffered and streamed requests are both +routed; keep every target on the agent's protocol (as below) so streams pass +through without translation. ## Prerequisites -- Rust 1.96.1 +- Rust 1.96.1 or later - Claude Code, for the agent launcher path +- An OpenRouter API key, from the + [OpenRouter keys page](https://openrouter.ai/keys) ## Install @@ -27,32 +28,35 @@ Build the Relay CLI with the Switchyard plugin (reference branch: ```bash git clone -b feat/libsy-decision-backend https://github.com/ryan-lempka/NeMo-Relay.git cd NeMo-Relay -RUSTUP_TOOLCHAIN=1.96.1 cargo build -p nemo-relay-cli --features switchyard +cargo build -p nemo-relay-cli --features switchyard ``` ## Configure -Copy the two example configs into the project you will launch from: +Copy the two example configs into the project you will launch from, and +provide your OpenRouter key: - [examples/nemo_relay/config.toml](../../examples/nemo_relay/config.toml) → `.nemo-relay/config.toml` - [examples/nemo_relay/plugins.toml](../../examples/nemo_relay/plugins.toml) → `.nemo-relay/plugins.toml` -The plugin config selects `decision_backend = "libsy"` with the LLM-classifier -algorithm: a Haiku target scores each request, and the score routes to an Opus -(strong) or Sonnet (weak) target. Claude Code's own auth headers pass through -to Anthropic. Verify with: - ```bash +export OPENROUTER_AUTHORIZATION="Bearer $OPENROUTER_API_KEY" ./target/debug/nemo-relay doctor ``` +The plugin config selects `decision_backend = "libsy"` with random routing +between a strong target (Opus) and a weak target (Sonnet), both served by +OpenRouter's Anthropic-compatible endpoint. Since Relay v0.7, target bindings +own their credentials: the agent's own login is never forwarded to a routed +target. + ## Path A: Gateway mode Runs Relay's gateway with libsy routing and a deterministic fake provider; no credentials needed: ```bash -RUSTUP_TOOLCHAIN=1.96.1 ./examples/switchyard/run-libsy-e2e.sh +./examples/switchyard/run-libsy-e2e.sh ``` Expected receipt: @@ -60,7 +64,8 @@ Expected receipt: ```text ok: easy prompt routes weak ok: hard prompt routes strong -ok: classifier and routed calls both flowed through Relay dispatch (4 upstream calls) +ok: streamed hard prompt routes strong +ok: classifier and routed calls all flowed through Relay dispatch (6 upstream calls) libsy in-process routing e2e passed ``` @@ -77,22 +82,16 @@ This opens the normal interactive Claude Code TUI through the Relay gateway. ## Validate Routing decisions land in `nemo-relay-events-*.jsonl` in the working -directory. One line per decision: +directory, one mark per decision: ```bash -grep -h switchyard.routing.decision nemo-relay-events-*.jsonl -``` - -Expected receipt: `switchyard.routing.decision` marks naming the classifier -step and the routed target, e.g. - -```json -{"backend_id": "classifier", "reason_summary": "classifying request via classifier", ...} -{"backend_id": "weak", "target_model": "claude-sonnet-5", ...} +grep -h -o '"backend_id":"[a-z]*"' nemo-relay-events-*.jsonl | sort | uniq -c ``` -A decision naming `weak` or `strong` is the proof: libsy scored the request -in-process and Relay dispatched the routed call. +Expected receipt: `switchyard.routing.decision` marks naming `strong` and +`weak`, with responses served by both models across the session. A decision +naming a target is the proof: libsy selected it in-process and Relay +dispatched the routed call. ## The embedding contract @@ -100,8 +99,10 @@ A host needs three things from libsy: - `Algorithm`: one shared `Arc` serves concurrent requests. - `run_stream(ctx, request)`: a stream of `Step`s; the host serves each - `Step::CallLlm` and fulfills it with `respond(...)`. -- The conversation IR (`LlmRequest` / `LlmResponse` from `libsy-protocol`), - which Relay already speaks through `switchyard-translation`. + `Step::CallLlm` and fulfills it with `respond(...)` — a buffered response + or a live chunk stream. +- The conversation IR (`LlmRequest`, `AggLlmResponse`, `LlmResponseChunk` + from `switchyard-protocol`), which Relay already speaks through + `switchyard-translation`. See the [libsy README](../../crates/libsy/README.md) for the full API. diff --git a/docs/guides/nemo_relay_known_issues.md b/docs/guides/nemo_relay_known_issues.md deleted file mode 100644 index 2bbe5be2..00000000 --- a/docs/guides/nemo_relay_known_issues.md +++ /dev/null @@ -1,79 +0,0 @@ -# NeMo Relay: Known Issues - -Gaps between the [NeMo Relay Recipe](nemo_relay.md) and a fully routed -interactive Claude Code session. Verified 2026-07-15 against libsy `main` -from a fresh public checkout. - -| Step | Status | -|---|---| -| Fresh clone builds hermetically; Path A e2e passes | works | -| Claude Code TUI launches; all traffic flows through the gateway and plugin | works | -| Requests carrying `cache_control` reach the router | works | -| Buffered requests routed by the libsy classifier | works | -| Streamed requests routed by libsy | **breaks** | -| Classifier scoring call accepted by Anthropic | **breaks** | - -## Switchyard / libsy improvements - -### S1. No streaming response path (libsy core) - -`CallLlmRequest::respond` and `Step::ReturnToAgent` carry only buffered -responses, so a host cannot pass a live token stream through an algorithm. -Claude Code streams its interactive calls, so they dispatch the trusted -fallback (`libsy_streaming_unsupported`) instead of a libsy decision. Same -prompt, sent both ways through Path A: - -```text -buffered -> classifier scored 0.9 -> served by strong-model -streamed -> no classifier call -> served by weak-model (fallback) -``` - -Streaming exists on the unmerged `grclark/simple-proxy` branch; when it -lands, this fixes with no recipe changes. - -### S2. Anthropic encoder defaults `max_tokens` to 128,000 (switchyard-translation) - -When a request sets no output params, the encoder injects -`max_tokens: 128000` (`codecs/anthropic/buffered.rs:207`). Haiku 4.5's -ceiling is 64,000, so Anthropic rejects the classifier's scoring call with -HTTP 400 and the whole request falls back (via S3). Proven by capturing the -sent request: - -```json -{"model": "claude-haiku-4-5-20251001", "max_tokens": 128000, "messages": [...]} -``` - -Fix pair: clamp or remove the encoder default, and have -`LlmClassifierOrchAlgo` set explicit small output params (a score needs a -few tokens). - -### S3. No fail-open on classifier call errors (libsy-examples) - -`LlmClassifierOrchAlgo` fails open only on an unparseable score. A classifier -call *error* propagates and kills the run, so the whole request falls back -instead of routing strong. The previous `llm-routing` profile failed open on -classifier errors by default (`classifier_fail_open`). - -### S4. Classifier input is flattened and untrimmed (libsy-examples) - -`LlmClassifierOrchAlgo` flattens instructions and all messages of every role -into one string, and includes the full conversation on every score. The -previous `llm-routing` classifier sent a structured, trimmed selection -(system messages, first user message, recent turns) with a task-specific -prompt and structured output. Requests do not fail, but scores are computed -over unstructured, unbounded input. - -## NeMo Relay improvements - -### R1. Provider errors are opaque in routing events - -The plugin's `provider_error_summary` formats upstream failures as -`class:http_status` and discards the response body. S2 had to be root-caused -by capturing the request instead of reading the logs. Fix: include a bounded -body snippet in error marks. - -### R2. Toolchain pin below Switchyard's MSRV - -Relay pins rustc 1.93.0; Switchyard's MSRV is 1.96.1. Prefix Relay builds -with `RUSTUP_TOOLCHAIN=1.96.1` until Relay bumps its pin (or Switchyard -lowers its MSRV). diff --git a/examples/nemo_relay/plugins.toml b/examples/nemo_relay/plugins.toml index c84d0776..e691b933 100644 --- a/examples/nemo_relay/plugins.toml +++ b/examples/nemo_relay/plugins.toml @@ -1,10 +1,11 @@ -# NeMo Relay plugin config: in-process libsy LLM-classifier routing. +# NeMo Relay plugin config: in-process libsy routing. # Place at .nemo-relay/plugins.toml in the project you launch from. # -# Haiku scores each request; at or above the threshold routes to Opus, -# below it routes to Sonnet. Claude Code's own auth headers pass through -# to Anthropic, so no keys are needed here. The observability component -# writes routing events to nemo-relay-events-*.jsonl in the working dir. +# Routes between a strong and a weak model served by OpenRouter's +# Anthropic-compatible endpoint. Set OPENROUTER_AUTHORIZATION to +# "Bearer " before launching. The observability +# component writes routing events to nemo-relay-events-*.jsonl in the +# working directory. version = 1 [[components]] @@ -19,32 +20,28 @@ context_mode = "payload_only" enabled_inbound_profiles = ["anthropic_messages"] [components.config.libsy] -algorithm = "llm_classifier" -classifier_target = "classifier" -strong_target = "strong" -weak_target = "weak" -threshold = 0.5 +algorithm = "random" [components.config.default_targets] anthropic_messages = "strong" -[components.config.targets.classifier] -model = "claude-haiku-4-5-20251001" -protocol = "anthropic_messages" -endpoint = "/v1/messages" -base_url = "https://api.anthropic.com" - [components.config.targets.strong] -model = "claude-opus-4-8" +model = "anthropic/claude-opus-4.8" protocol = "anthropic_messages" endpoint = "/v1/messages" -base_url = "https://api.anthropic.com" +base_url = "https://openrouter.ai/api" + +[components.config.targets.strong.header_env] +authorization = "OPENROUTER_AUTHORIZATION" [components.config.targets.weak] -model = "claude-sonnet-5" +model = "anthropic/claude-sonnet-5" protocol = "anthropic_messages" endpoint = "/v1/messages" -base_url = "https://api.anthropic.com" +base_url = "https://openrouter.ai/api" + +[components.config.targets.weak.header_env] +authorization = "OPENROUTER_AUTHORIZATION" [[components]] kind = "observability" diff --git a/mkdocs.yml b/mkdocs.yml index dd04ba8c..83f29cb3 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -23,7 +23,6 @@ nav: - Guides: - Agent Launchers: guides/agent_launchers.md - NeMo Relay: guides/nemo_relay.md - - "NeMo Relay: Known Issues": guides/nemo_relay_known_issues.md - Skill Distillation: skill_distillation.md - Concepts: - Core Concepts: core_concepts.md