diff --git a/benchmark/compare_runs.py b/benchmark/compare_runs.py new file mode 100644 index 00000000..bfb3eb3a --- /dev/null +++ b/benchmark/compare_runs.py @@ -0,0 +1,126 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Compare accuracy and token cost across run-baseline.sh run directories. + +Usage:: + + uv run --no-sync python benchmark/compare_runs.py \ + label1=benchmark/tb_runs/ label2=benchmark/tb_runs/ ... + +Reads each run's Harbor ``result.json`` (accuracy) and Switchyard +``routing_stats_final.json`` (per-tier tokens, judge/classifier overhead, +cost estimate) and prints a side-by-side table plus per-task outcomes. +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path +from typing import Any + + +def _load_run(run_dir: Path) -> dict[str, Any]: + results = sorted(run_dir.glob("jobs/*/result.json")) + if not results: + raise FileNotFoundError(f"no jobs/*/result.json under {run_dir}") + harbor = json.loads(results[0].read_text()) + stats_path = run_dir / "routing_stats_final.json" + routing = json.loads(stats_path.read_text()) if stats_path.exists() else {} + + ev = next(iter(harbor["stats"]["evals"].values())) + rewards: dict[str, float] = {} + for value, trials in (ev.get("reward_stats", {}).get("reward", {}) or {}).items(): + for trial in trials: + # trial names look like -; strip the trial suffix. + rewards[trial.rsplit("-", 1)[0]] = float(value) + + return { + "harbor": harbor["stats"], + "eval": ev, + "rewards": rewards, + "routing": routing, + } + + +def _fmt_cost(routing: dict[str, Any]) -> tuple[float, float, float]: + ce = routing.get("cost_estimate", {}) or {} + return ( + ce.get("total_cost", 0.0), + ce.get("backend_cost", 0.0), + ce.get("classifier_cost", 0.0), + ) + + +def main(argv: list[str]) -> int: + runs: dict[str, dict[str, Any]] = {} + for arg in argv: + label, _, path = arg.partition("=") + if not path: + print(f"ERROR: expected label=path, got {arg!r}") + return 2 + runs[label] = _load_run(Path(path)) + + header = f"{'metric':<28}" + "".join(f"{label:>18}" for label in runs) + print(header) + print("-" * len(header)) + + def row(name: str, values: list[str]) -> None: + print(f"{name:<28}" + "".join(f"{v:>18}" for v in values)) + + row("completed trials", [str(r["harbor"]["n_completed_trials"]) for r in runs.values()]) + row("errored trials", [str(r["harbor"]["n_errored_trials"]) for r in runs.values()]) + row("mean reward", [f"{r['eval']['metrics'][0].get('mean', 0.0):.3f}" for r in runs.values()]) + row( + "solved / total", + [ + f"{sum(1 for v in r['rewards'].values() if v >= 1.0)}/{len(r['rewards'])}" + for r in runs.values() + ], + ) + row( + "input tokens", + [f"{r['harbor'].get('n_input_tokens', 0):,}" for r in runs.values()], + ) + row( + "output tokens", + [f"{r['harbor'].get('n_output_tokens', 0):,}" for r in runs.values()], + ) + for label_idx, name in ((0, "total cost $"), (1, "backend cost $"), (2, "judge/clf cost $")): + row(name, [f"{_fmt_cost(r['routing'])[label_idx]:.3f}" for r in runs.values()]) + + # Tier split (escalation / routed runs only). + row( + "tier split (calls)", + [ + " ".join( + f"{v.get('tier') or m.rsplit('/', 1)[-1]}:{v.get('calls', 0)}" + for m, v in (r["routing"].get("models", {}) or {}).items() + ) + or "-" + for r in runs.values() + ], + ) + clf_rows = [] + for r in runs.values(): + clf = r["routing"].get("classifier", {}) or {} + n, e = clf.get("total_requests", 0), clf.get("total_errors", 0) + clf_rows.append(f"{n} ({e} err)" if n or e else "-") + row("judge calls", clf_rows) + + # Per-task outcome grid. + tasks = sorted({t for r in runs.values() for t in r["rewards"]}) + print() + print(f"{'task':<44}" + "".join(f"{label:>14}" for label in runs)) + for task in tasks: + marks = [] + for r in runs.values(): + v = r["rewards"].get(task) + marks.append("-" if v is None else ("PASS" if v >= 1.0 else "fail")) + print(f"{task:<44}" + "".join(f"{m:>14}" for m in marks)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/benchmark/routing-profiles/tb-lite-escalation-opus-deepseek-gemini.yaml b/benchmark/routing-profiles/tb-lite-escalation-opus-deepseek-gemini.yaml new file mode 100644 index 00000000..52389615 --- /dev/null +++ b/benchmark/routing-profiles/tb-lite-escalation-opus-deepseek-gemini.yaml @@ -0,0 +1,35 @@ +routes: + switchyard: + type: escalation_router + enable_stats: true + fallback_target_on_evict: strong + # Distinguish k>1 repeated-task trials against one shared server: the + # session key includes the first 2 post-task messages, so trials diverge + # via early model responses instead of sharing an escalation latch. + # Requires nonzero sampling temperature; keep 0 outside benchmarks. + session_key_depth: 2 + strong: + model: anthropic/claude-opus-4.7 + api_key: ${OPENROUTER_API_KEY} + base_url: https://openrouter.ai/api/v1 + format: openai + timeout_secs: 600 + extra_body: + reasoning: + effort: high + weak: + model: deepseek/deepseek-v4-pro + api_key: ${OPENROUTER_API_KEY} + base_url: https://openrouter.ai/api/v1 + format: openai + timeout_secs: 600 + judge: + model: google/gemini-3.5-flash + api_key: ${OPENROUTER_API_KEY} + base_url: https://openrouter.ai/api/v1 + timeout_secs: 5 + min_turn: 3 + recent_turn_window: 14 + # Verdict dumps became opt-in upstream; the bench harness reads + # escalation_verdict= lines from the captured server log. + dump_verdicts: true diff --git a/benchmark/routing-profiles/tb-lite-escalation-opus-deepseek-nvidia.yaml b/benchmark/routing-profiles/tb-lite-escalation-opus-deepseek-nvidia.yaml new file mode 100644 index 00000000..d213bb90 --- /dev/null +++ b/benchmark/routing-profiles/tb-lite-escalation-opus-deepseek-nvidia.yaml @@ -0,0 +1,73 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Escalation-router Opus/DeepSeek profile for Harbor Terminal-Bench Lite on +# the NVIDIA Inference Hub. Every conversation starts on DeepSeek V4 Pro; a +# DeepSeek V4 Flash judge watches the trajectory and latches the session to +# Claude Opus 4.8 on a clear pattern of trouble. +# +# All three tiers use https://inference-api.nvidia.com/v1, which requires a +# LiteLLM virtual key (sk-...), not an nvapi- key. Export the same key for +# all three vars (they are the ones run-baseline.sh forwards into the +# Switchyard container): +# export SWITCHYARD_STRONG_API_KEY=sk-... +# export SWITCHYARD_WEAK_API_KEY=sk-... +# export SWITCHYARD_CLASSIFIER_API_KEY=sk-... +# +# Run with: +# benchmark/run-baseline.sh \ +# --harbor-path benchmark/datasets/openthoughts-tblite-closed-book \ +# --routing-profiles benchmark/routing-profiles/tb-lite-escalation-opus-deepseek-nvidia.yaml \ +# --model switchyard + +routes: + switchyard: + type: escalation_router + enable_stats: true + fallback_target_on_evict: strong + # Distinguish k>1 repeated-task trials against one shared server: the + # session key includes the first 2 post-task messages, so trials diverge + # via early model responses instead of sharing an escalation latch. + # Requires nonzero sampling temperature; keep 0 outside benchmarks. + session_key_depth: 2 + strong: + model: aws/anthropic/bedrock-claude-opus-4-8 + api_key: ${SWITCHYARD_STRONG_API_KEY} + base_url: https://inference-api.nvidia.com/v1 + format: openai + timeout_secs: 600 + weak: + # The evals- batch-gateway variant (nvidia/deepseek-ai/evals-deepseek-v4-pro) + # avoids the regular gateway's 6-min timeout under high concurrency, but + # its upstream host is currently failing TLS verification; switch back + # once it probes clean. + model: nvidia/deepseek-ai/deepseek-v4-pro + api_key: ${SWITCHYARD_WEAK_API_KEY} + base_url: https://inference-api.nvidia.com/v1 + format: openai + timeout_secs: 600 + # Explicit extra_body overrides the DeepSeek default of + # enable_thinking: false; the X-Inference-Priority: batch header is + # still injected (headers merge independently of the body). + extra_body: + chat_template_kwargs: + enable_thinking: true + judge: + model: nvidia/deepseek-ai/deepseek-v4-flash + api_key: ${SWITCHYARD_CLASSIFIER_API_KEY} + base_url: https://inference-api.nvidia.com/v1 + # Hub Flash latency is ~4-8s on small judge calls (and the profile + # build pins DeepSeek judges to the X-Inference-Priority: batch + # gateway); the 5s default used with Gemini Flash would fail-open + # (stay weak) on most turns here. + timeout_secs: 30 + min_turn: 3 + # Require a second consecutive escalate verdict before latching: + # round-2 traces showed one-shot eager verdicts (single failed + # command at turn 4-9) causing regressions on tasks the weak tier + # solves; real loops persist across turns and still latch. + confirmations: 2 + recent_turn_window: 14 + # Verdict dumps became opt-in upstream; the bench harness reads + # escalation_verdict= lines from the captured server log. + dump_verdicts: true diff --git a/benchmark/routing-profiles/tb-lite-single-deepseek-v4-pro-nvidia.yaml b/benchmark/routing-profiles/tb-lite-single-deepseek-v4-pro-nvidia.yaml new file mode 100644 index 00000000..8a2e2bd7 --- /dev/null +++ b/benchmark/routing-profiles/tb-lite-single-deepseek-v4-pro-nvidia.yaml @@ -0,0 +1,34 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Serve-backed single-model DeepSeek V4 Pro profile on the NVIDIA Inference +# Hub — weak-only baseline for the escalation-router comparison. +# Requires a LiteLLM virtual key (sk-...): +# export SWITCHYARD_WEAK_API_KEY=sk-... +# Run with: +# benchmark/run-baseline.sh \ +# --harbor-path benchmark/datasets/openthoughts-tblite-closed-book \ +# --routing-profiles benchmark/routing-profiles/tb-lite-single-deepseek-v4-pro-nvidia.yaml \ +# --model tb-lite-single-deepseek-v4-pro-nvidia + +defaults: + api_key: ${SWITCHYARD_WEAK_API_KEY} + base_url: https://inference-api.nvidia.com/v1 + timeout_secs: 600.0 + +routes: + tb-lite-single-deepseek-v4-pro-nvidia: + type: model + target: + # The evals- batch-gateway variant avoids the regular gateway's 6-min + # timeout under high concurrency but is currently failing TLS + # verification upstream; switch back once it probes clean. + model: nvidia/deepseek-ai/deepseek-v4-pro + format: openai + extra_headers: + X-Inference-Priority: batch + # Thinking on, matching the escalation profile's weak tier so the + # weak-only baseline stays apples-to-apples. + extra_body: + chat_template_kwargs: + enable_thinking: true diff --git a/benchmark/routing-profiles/tb-lite-single-opus-4-7-nvidia.yaml b/benchmark/routing-profiles/tb-lite-single-opus-4-7-nvidia.yaml new file mode 100644 index 00000000..8831e419 --- /dev/null +++ b/benchmark/routing-profiles/tb-lite-single-opus-4-7-nvidia.yaml @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Serve-backed single-model Opus 4.7 profile on the NVIDIA Inference Hub — +# strong-only baseline for the escalation-router comparison. +# Requires a LiteLLM virtual key (sk-...): +# export SWITCHYARD_STRONG_API_KEY=sk-... +# Run with: +# benchmark/run-baseline.sh \ +# --harbor-path benchmark/datasets/openthoughts-tblite-closed-book \ +# --routing-profiles benchmark/routing-profiles/tb-lite-single-opus-4-7-nvidia.yaml \ +# --model tb-lite-single-opus-4-7-nvidia + +defaults: + api_key: ${SWITCHYARD_STRONG_API_KEY} + base_url: https://inference-api.nvidia.com/v1 + timeout_secs: 600.0 + +routes: + tb-lite-single-opus-4-7-nvidia: + type: model + target: + model: aws/anthropic/bedrock-claude-opus-4-7 + format: openai diff --git a/benchmark/routing-profiles/tb-lite-single-opus-4-8-nvidia.yaml b/benchmark/routing-profiles/tb-lite-single-opus-4-8-nvidia.yaml new file mode 100644 index 00000000..dd871169 --- /dev/null +++ b/benchmark/routing-profiles/tb-lite-single-opus-4-8-nvidia.yaml @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Serve-backed single-model Opus 4.8 profile on the NVIDIA Inference Hub — +# strong-only baseline for the escalation-router comparison. +# Requires a LiteLLM virtual key (sk-...): +# export SWITCHYARD_STRONG_API_KEY=sk-... +# Run with: +# benchmark/run-baseline.sh \ +# --harbor-path benchmark/datasets/openthoughts-tblite-closed-book \ +# --routing-profiles benchmark/routing-profiles/tb-lite-single-opus-4-8-nvidia.yaml \ +# --model tb-lite-single-opus-4-8-nvidia + +defaults: + api_key: ${SWITCHYARD_STRONG_API_KEY} + base_url: https://inference-api.nvidia.com/v1 + timeout_secs: 600.0 + +routes: + tb-lite-single-opus-4-8-nvidia: + type: model + target: + model: aws/anthropic/bedrock-claude-opus-4-8 + format: openai