From 7d3bdbbe3047a83bafad226d08c2bbe7f5971384 Mon Sep 17 00:00:00 2001 From: Hung-Yueh Chiang Date: Sat, 27 Jun 2026 18:37:31 -0700 Subject: [PATCH 01/15] docs(eval): add NEL v0.3.0 migration guide + example configs Add a guide for migrating checkpoint evaluation from the NEL 0.2.x launcher (nemo_evaluator_launcher) to the 0.3.x engine (nemo_evaluator / `nel eval run`): - references/nel-v0.3.0-migration.md: 0.2.x->0.3.x config mapping, backend support matrix (built-in / skills:// / gym://), per-backend launch patterns, single-node vs sharded runs, and the MLflow export flow. - recipes/examples/r030_example_eval.yaml: built-in + skills:// native example. - recipes/examples/r030_gym.yaml: gym:// (NeMo-Gym resource server + reward). Signed-off-by: Hung-Yueh Chiang Co-Authored-By: Claude Opus 4.8 (1M context) --- .../recipes/examples/r030_example_eval.yaml | 111 ++++ .../evaluation/recipes/examples/r030_gym.yaml | 56 ++ .../references/nel-v0.3.0-migration.md | 592 ++++++++++++++++++ 3 files changed, 759 insertions(+) create mode 100644 .agents/skills/evaluation/recipes/examples/r030_example_eval.yaml create mode 100644 .agents/skills/evaluation/recipes/examples/r030_gym.yaml create mode 100644 .agents/skills/evaluation/references/nel-v0.3.0-migration.md diff --git a/.agents/skills/evaluation/recipes/examples/r030_example_eval.yaml b/.agents/skills/evaluation/recipes/examples/r030_example_eval.yaml new file mode 100644 index 00000000000..78b27fb60e8 --- /dev/null +++ b/.agents/skills/evaluation/recipes/examples/r030_example_eval.yaml @@ -0,0 +1,111 @@ +# NEL 0.3.x (engine) — SIMPLE / no-tools, SLURM-sharded. +# Built-in + skills math/multichoice benchmarks for a reasoning checkpoint. +# Fill every <...> placeholder. See references/nel-v0.3.0-migration.md. +# +# nel eval run r030_example_eval.yaml --dry-run +# nel eval run r030_example_eval.yaml +# # after all shards finish: +# nel eval merge / +# # then export from the LOGIN node (SLURM does NOT auto-export — see migration doc "Export results to MLFlow"): +# nel export // --dest mlflow -o tracking_uri= -o experiment_name= + +services: + model: + type: vllm + # HOST checkpoint path; the engine auto-mounts it :/model:ro and serves `vllm serve /model`. + # Do NOT also hand-mount it. For an HF handle, set `model: ` (no mount needed). + model: /path/to/checkpoint + served_model_name: my-model # required; referenced by benchmarks + protocol: chat_completions + port: 8000 + node_pool: gpu + # Single-instance serving. Pick TP/PP from model size & GPU count (references/parallelism.md). + # Scale THROUGHPUT with cluster.shards below, not data_parallel (DP can be fragile on stock images). + tensor_parallel_size: 4 + pipeline_parallel_size: 1 + data_parallel_size: 1 + num_nodes: 1 + image: # pin a version; never :latest. Cross-check recipes.vllm.ai. + startup_timeout: 3600.0 + extra_args: + - "--gpu-memory-utilization=0.9" + - "--max-model-len=262144" # must exceed prompt + generation.max_tokens + - "--trust-remote-code" # if config.json has auto_map + # Reasoning checkpoint: use YOUR model's parser (and plugin path if it ships a custom one). + - "--reasoning-parser=" + # - "--reasoning-parser-plugin=/model/.py" # only if the checkpoint bundles one + - "--enable-expert-parallel" # MoE only; no-op for dense + - "--max-num-seqs=512" # serving-side concurrency knob — raise to use the GPU + extra_env: + HF_TOKEN: ${HF_TOKEN} + HF_HOME: /cache/huggingface + container_mounts: + - /path/to/hf-cache:/cache/huggingface + proxy: + verbose: true # REQUIRED for request_timeout to apply + request_timeout: 36000.0 # LARGE for reasoning models (120s default kills long gens) + # max_concurrent_upstream: 512 # default 64 is the hidden throttle — raise to use the GPU + generation: + temperature: 1.0 # from model card + top_p: 0.95 # from model card + # PER model-call cap. Leave headroom under --max-model-len AND bound one call under the SLURM wall + # so a single long sample can't loop auto_resume forever. + max_tokens: 196608 + +benchmarks: + - name: gpqa # built-in + repeats: 8 + max_concurrent: 64 + solver: {type: simple, service: model} + - name: mmlu_pro # built-in + repeats: 1 + max_concurrent: 64 + solver: {type: simple, service: model} + - name: skills://aime25 # nemo-skills math (per-sample) — needs nemo-skills baked in + repeats: 64 + max_concurrent: 64 + solver: {type: simple, service: model} + # WITH-TOOLS variant of a native benchmark: SAME backend/name, just a tool_calling solver + a local + # sandbox — NO gym server involved. Use in place of the simple `gpqa` entry above to give it tools. + # NOTE: the served model must be launched with --enable-auto-tool-choice + --tool-call-parser= + # (add them to services.model.extra_args) or the tool calls won't be parsed. + # - name: gpqa # still BUILT-IN + # repeats: 8 + # max_concurrent: 64 + # solver: {type: tool_calling, service: model, sandbox_tools: true, max_turns: 100} + # sandbox: {type: local, concurrency: 64} + +cluster: + type: slurm + account: + walltime: "04:00:00" # cluster wall; long evals continue via auto_resume / shards + auto_resume: true # safe here (no-tools can't loop a single call past the wall) + shards: 8 # N independent single-node workers + `nel eval merge` + eval_image: # the one eval container, with nemo-skills + lm-eval pip-installed + mount_home: false # avoid a stale host ~/.local/bin/nel shadowing the CLI + container_mounts: + - /path/to/hf-cache:/cache/huggingface + container_env: + HF_HOME: /cache/huggingface + HF_TOKEN: ${HF_TOKEN} + node_pools: + gpu: + partition: + nodes: 1 + ntasks_per_node: 1 + gres: "gpu:4" # match the node's GPU count + TP×DP + sbatch_extra_flags: + exclusive: true + +output: + dir: /abs/path/to/rundir # MUST be absolute + export: + - mlflow # NOTE: NOT auto-exported on SLURM — run `nel export` manually + export_config: + mlflow: + tracking_uri: + experiment_name: + tags: + model: my-model + precision: + pipeline: r0.3.0 diff --git a/.agents/skills/evaluation/recipes/examples/r030_gym.yaml b/.agents/skills/evaluation/recipes/examples/r030_gym.yaml new file mode 100644 index 00000000000..c854e2665cc --- /dev/null +++ b/.agents/skills/evaluation/recipes/examples/r030_gym.yaml @@ -0,0 +1,56 @@ +# NEL 0.3.x (engine) — gym:// (NeMo-Gym resource server + reward). +# The FAITHFUL path for agentic / judge / batch benchmarks that skills:// only exact-match-scores +# (the † rows in the matrix). The engine connects to a RUNNING gym resource server, drives the +# rollout against your model, and reads the reward back from the server's /verify. +# Validated with the self-contained `mcqa` + `ifbench` servers (native protocol). Fill every <...>. +# Full setup (install gym, start the server, find its port, dataset rcp format): github.com/NVIDIA-NeMo/Gym +# +# PREREQUISITE — start ONE resource server first (separate process), then find its dynamic port: +# export RAY_TMPDIR=/tmp # Lustre AF_UNIX socket paths exceed 107 bytes → Ray fails without this +# gym env start --resources-server & # background it; loads ONLY the grader (not the agent) +# # the head server is at :11000; each resource server registers there on a DYNAMIC port. Read it back: +# curl -s http://127.0.0.1:11000/server_instances | python3 -m json.tool # → [{"name": .., "url": ".../"}] +# # running >1 server (e.g. ifbench + ifeval) lists them all — match the row by "name" to get each port. +# +# nel eval run r030_gym.yaml --dry-run +# nel eval run r030_gym.yaml + +services: + model: # the POLICY the engine rolls out against the gym server + # Hosted OpenAI-compatible endpoint = no GPU needed (ideal for a smoke). To self-serve a checkpoint, + # serve it on a GPU as a SEPARATE `vllm serve` job and point url: at it (migration doc §4/§5) — keep + # type: api here. (type: vllm + cluster: local does NOT work: NEL would run `python -m vllm` from the + # nel venv, which has no vLLM; serving must be its own GPU job reached via type: api.) + type: api + url: + protocol: chat_completions + model: + api_key: ${YOUR_API_KEY} # resolved from env at run time (set -a; source <.env>; set +a) + generation: + temperature: 0.0 + max_tokens: 512 + +benchmarks: + # The benchmark lives in the running server + the data file, NOT after the prefix. + # gym://?protocol=native&data= + # native = evaluator holds the dataset, seeds locally, posts the answer to the server's /verify. + # = the dynamic resource-server port from /server_instances above (NOT the 11000 head port). + - name: "gym://127.0.0.1:?protocol=native&data=/abs/path/to/tasks.jsonl" + repeats: 1 + max_concurrent: 5 + # simple solver = single-shot policy answer scored by the server (mcqa/ifbench-style). For agentic + # gym envs that need tool use, switch to `type: tool_calling` + a `sandbox:`. + solver: {type: simple, service: model} + +cluster: + # `local` = NEL runs the evaluator in-process on the node you launch it from — use a CPU compute node + # (sbatch --partition=cpu), NOT the login node. The gym scorer is CPU; only the policy needs a GPU + # (and here the policy is a hosted API, so no GPU at all). This single-node path is the validated one. + # NOTE: sharding gym is NOT a free swap — the one shared resource server would have to be at a routable + # host (not 127.0.0.1) reachable from every shard node; loopback + shards silently fails to score. + type: local + +output: + dir: /abs/path/to/rundir # MUST be absolute + # export is run manually after the job (see r030_example_eval.yaml / migration doc "Export to MLFlow"): + # nel export / --dest mlflow -o tracking_uri= -o experiment_name= diff --git a/.agents/skills/evaluation/references/nel-v0.3.0-migration.md b/.agents/skills/evaluation/references/nel-v0.3.0-migration.md new file mode 100644 index 00000000000..5e503b4c3f6 --- /dev/null +++ b/.agents/skills/evaluation/references/nel-v0.3.0-migration.md @@ -0,0 +1,592 @@ +# NEL v0.3.0 Migration + +## NEL 0.2.x (launcher) vs 0.3.x (engine) + +| | **0.2.x launcher** (`nemo_evaluator_launcher`) | **0.3.x engine** (`nemo_evaluator`) | +| --- | --- | --- | +| Benchmark source | each task runs in its **own published eval-factory container** (`nvcr.io/nvidia/eval-factory/*`); you never build an eval image | **one** `cluster.eval_image`; benchmarks resolve from Python `@register()` modules | +| Where harnesses live | baked into per-task containers upstream | **you pip-install** external harnesses (`nemo-skills`, `lm-eval`) **into the one eval image** | +| Benchmark families | eval-factory `container/` adapters, `ns_*` nemo-skills | **built-in** (17, `nel list -s builtin`), **`skills://`** (nemo-skills), **`lm-eval://`**, + legacy **`container://`** (BC) | +| CLI | `nel run` / `ls` / `status` / `info` | `nel eval run` / `list` / `export` | +| Config schema | Hydra `deployment:` / `evaluation:` / `execution:` | `services:` / `benchmarks:` / `cluster:` / `output:` (one file) | +| Serving | a `deployment:` block | integrated under `services.model` | +| "with tools" | task-specific | `solver: {type: tool_calling, sandbox_tools: true}` + a `sandbox:` | + +### Comparing the Config File Structure + +**0.2.x launcher** — Hydra, several top-level blocks (+ a `defaults:` preset chain), typically one file per task: + +```yaml +defaults: [...] # Hydra preset chain (resolved at launch) +execution: # SLURM: account, partition, walltime, num_nodes, output_dir, sbatch flags +deployment: # serving: image, hf_model_handle, served_model_name, TP/PP/DP, extra_args +target: # how to reach the served model + api_endpoint: {...} +evaluation: # the benchmark(s) + tasks: [...] # task names live here +export: # mlflow export + mlflow: {...} +``` + +**0.3.x engine** — one self-contained file, four blocks, all tasks in a single `benchmarks:` list: + +```yaml +services: # serving (replaces `deployment:` + `target:`) + model: {type: vllm, model: ..., tensor_parallel_size: 4, extra_args: [...]} +benchmarks: # the tasks (replaces `evaluation.tasks:`), one `- name:` each + - name: gpqa # prefix picks the backend (see matrix below) +cluster: # SLURM (replaces `execution:`): type, account, walltime, shards, eval_image +output: # results + export (replaces `export:`) + dir: ... + export: [mlflow] + export_config: {mlflow: {...}} +``` + +Key moves: +- `deployment:` + `target:` → **`services.model`** (serving + endpoint unified; the engine wires it to a benchmark via `solver.service`) +- `evaluation.tasks:` → **`benchmarks:`** (a list of `- name:`; the prefix selects the backend) +- `execution:` → **`cluster:`** (adds `shards:` for multi-node sharding and `eval_image:` for the one eval container) +- `export:` → **`output.export` / `output.export_config`** +- `defaults:` (Hydra presets) → **gone** — one explicit file, no preset resolution step + + +### Backward supporting v0.2.x + +0.3.x keeps a backward-compat path for v0.2.x's eval-factory `container/` task variants (e.g. +`mmlu_pro_aa_v3`, `gpqa_diamond_aa_v3`) — though they have **no _native_ 0.3.x benchmark of the same +name**. Two options: + +1. run them via the **legacy `container://` backend** (`solver: {type: container}`, which injects a v1-format `run_config.yaml` so v1 configs port with minimal changes — but **`cluster.shards` is unsupported** for legacy container runs) + +2. **recommended** — switch to the native built-in (`mmlu_pro`, `gpqa`) or `skills://` equivalents (sharding + one eval image). Note the launcher's AIME entry was already `ns_aime2025` (nemo-skills, *not* a container variant), so it ports directly to `skills://aime25`. + + +## NEL v0.3.0 Backend Support Matrix + +### Supported Backends + +NEL 0.3.x resolves a benchmark through one of several **backends**, chosen by the `name:` prefix you +put in `benchmarks:`. The same benchmark is often reachable from **more than one** backend, and the +prefix picks which harness actually runs — which also affects scoring fidelity (whether a benchmark is +scored correctly vs silently mis-scored). That's detailed per-benchmark in **Supported Benchmarks** below. + +```text +backend name: in the config find available names +---------- ------------------------------ ------------------------------------------------------- +built-in (e.g. gpqa) `nel list -s builtin` +skills:// skills:// `nel list -s skills` (after prep) / nemo_skills/dataset/ +lm-eval:// lm-eval:// `lm_eval --tasks list` +gym:// gym://?protocol= Gym repo benchmarks/ + resources_servers/ + native&data= (needs a running server → github.com/NVIDIA-NeMo/Gym) +``` + +Example — a `benchmarks:` block, one entry per benchmark. The `name:` prefix selects the backend (it is **not** uniformly `{backend}://{benchmark}`): built-in has **no prefix** (just `{benchmark}`); `skills://` and `lm-eval://` are `{backend}://{benchmark}`; `gym://` points at a running server, `gym://{host:port}?...&data=` (the benchmark lives in the data file, not after the prefix): + +```yaml +benchmarks: # the benchmarks block + - name: gpqa # built-in + - name: skills://aime25 # NeMo-Skills + - name: lm-eval://hellaswag # lm-eval + - name: gym://127.0.0.1:8000?protocol=native&data=tasks.jsonl # NeMo-Gym (running server) +``` + +### Supported Benchmarks + +The list of supported benchmarks are from three NVIDIA-NeMo repositories: +- `NVIDIA-NeMo/Evaluator`: built-in engine [`@register`](https://github.com/NVIDIA-NeMo/Evaluator/tree/main/src/nemo_evaluator/benchmarks) +- `NVIDIA-NeMo/Skills`: **`skills://`** from [`nemo_skills/dataset/`](https://github.com/NVIDIA-NeMo/Skills/tree/main/nemo_skills/dataset), +- `NVIDIA-NeMo/Gym`: **`gym://`** from Gym [`benchmarks/`](https://github.com/NVIDIA-NeMo/Gym/tree/main/benchmarks)+[`resources_servers/`](https://github.com/NVIDIA-NeMo/Gym/tree/main/resources_servers). + +The matrix below covers only these three backends. **`lm-eval://`** (e.g. `lm-eval://hellaswag`) is also +supported but not enumerated here — discover its tasks with `lm_eval --tasks list`. Only the **built-in** +column is verified against this engine (`nel list -s builtin`); the **`skills://`** and **`gym://`** +columns are compiled from the upstream repos and have **not** all been run — treat them as a routing +guide, not a guarantee (see the per-row notes). + + +> **`†`** = `skills://` **may mis-score** this **judge / batch / code** benchmark: its bridge ([`skills.py`](https://github.com/NVIDIA-NeMo/Evaluator/blob/main/src/nemo_evaluator/environments/skills.py)) only scores math/multichoice; other types fall back to exact-match → ~0. Use `gym://` (or the nemo-skills container / 0.2.x launcher) instead. + +```text +benchmark built-in skills:// gym:// notes +------------------------- -------- ---------- ------ ---------------------------------------- +KNOWLEDGE / QA + gpqa yes yes yes gym: gpqa (+ gpqa_diamond server) + mmlu yes yes yes + mmlu_pro yes yes yes + simpleqa yes yes yes + triviaqa yes — — built-in only + drop yes — — built-in only + healthbench yes — — built-in only + hotpotqa — yes yes gym: hotpotqa_qa / hotpotqa_closedbook + supergpqa — yes yes + hle — yes†(judge) yes gym recommended (skills may mis-score) + omniscience — yes†(judge) yes gym recommended (skills may mis-score) + critpt — yes†(judge) yes gym recommended (skills may mis-score) +MATH + math500 yes yes yes skills/gym: math-500 + gsm8k yes yes yes + aime25 — yes yes + aime24 — yes yes + hmmt_feb25 — yes yes + polymath — yes yes + putnam_bench — yes yes + minif2f — yes yes + proofnet — yes yes + ugphysics — yes yes + physics — yes yes + minerva_math — yes — skills only + olympiadbench — yes — skills only + omni_math — yes — skills only +CODE / SWE + humaneval yes yes†(code) yes built-in faithful; skills code-exec=batch + mbpp — yes†(code) yes + livecodebench — yes†(code) yes gym recommended (skills may mis-score) + bigcodebench — yes†(code) yes gym recommended (skills may mis-score) + scicode — yes†(code) yes gym recommended (skills may mis-score) + ioi — yes†(code) yes + cvdp — yes†(code) yes + swe-bench — yes — gym has only SWE-RL servers (swe_pivot/swerl_*) + bird_sql — — yes gym only + spider2_lite — — yes gym only + code_gen — — yes gym only + evalplus — — yes gym only +INSTRUCTION / FORMAT + ifbench — yes†(batch) yes gym recommended (skills may mis-score) + ifeval — yes†(batch) yes gym recommended (skills may mis-score) + instruction_following — — yes gym only + structured_outputs — — yes gym only + multichallenge — — yes gym only +LONG-CONTEXT + ruler — yes yes + mrcr — yes yes + longbench_v2 — yes yes + longcodebench — yes yes + aalcr — yes†(judge) yes gym recommended (skills may mis-score) +AGENTIC / TOOL / RL (mostly gym-only; the few built-ins are noted) + terminal-bench-v1 yes — — built-in; agentic terminal tasks + terminal-bench-hard yes — — built-in (+ -aa-split variant) + nmp_harbor yes — — built-in (harbor packaging) + tau2 — — yes + gdpval — — yes + browsecomp — — yes + mcqa — — yes + blackjack — — yes + gymnasium — — yes + reasoning_gym — — yes + arc_agi — — yes + aviary — — yes + google_search — — yes + tavily_search — — yes + xlam_fc — — yes + ns_tools — — yes + swe_pivot — — yes + pinchbench yes — — built-in only (NOT gym) +ARENA / JUDGE + arena_hard — yes†(judge) yes the "LM Arena" family + arena_hard_v2 — yes†(judge) yes + arena_judge — — yes gym only + genrm_compare — — yes gym only +SAFETY + xstest yes — yes + abstention — — yes gym only + over_refusal_detection — — yes gym only + jailbreak_detection — — yes gym only + indirect_prompt_injection — — yes gym only +MULTILINGUAL / MULTIMODAL / AUDIO + mgsm yes yes — built-in (multilingual GSM8K) + mmmlu — yes yes + flores200 — yes yes + wmt24pp — yes yes + mmmu_pro — yes — skills only (multimodal) + covost2 — yes — skills only (audio) + fleurs — yes — skills only (audio) + labbench2_vlm — — yes gym only (multimodal) + vlm_eval_kit — — yes gym only (multimodal) +``` + + +## Launch NEL v0.3.0 + +The launch pattern follows the **backend**, not the category. Map each matrix category to the matching +section below: + +```text +section (the ### headings below) categories / when +--------------------------------- -------------------------------------------------------------- +built-in and skills:// (native) KNOWLEDGE/QA · MATH · LONG-CONTEXT · MULTILINGUAL · SAFETY + (every row `yes` under built-in or skills with NO †) +gym:// (server + reward) AGENTIC/TOOL/RL (gym-only) + every † row (CODE/SWE, + INSTRUCTION/FORMAT, ARENA/JUDGE, LONG-CONTEXT aalcr) +legacy container:// v0.2.x `*_aa_v3` eval-factory container tasks +``` + +> **Multimodal / audio — not validated here.** Our migration only covered text benchmarks. The engine +> registers a `vlmevalkit://` backend (`environments/registry.py` → `VLMEvalKitEnvironment`), and +> multimodal/audio datasets (e.g. `mmmu_pro`, `covost2`, `fleurs`) may be reachable via `skills://` or +> `gym://` — but we ran **none** of them, so the exact name→backend routing is unconfirmed. Treat as a +> starting pointer, not a tested recipe. + +### Basic Usage + +Core 0.3.x CLI — the same for any benchmark; the backend (the three subsections below) only changes what goes in `benchmarks:`. + +```bash +# Check the version +nel --version # v0.3.x +# Config mode (recommended): one file with services + benchmarks + cluster + output +nel eval run config.yaml +nel eval run config.yaml --dry-run # generate scripts, don't run (inspect per-shard configs) +nel eval run config.yaml --resume # resume a partial / timed-out run + +# Quick mode: a single built-in benchmark against an already-served model +nel eval run --bench gpqa --model-url http://localhost:8000/v1 --model-id my-model --api-key dummy + +# Discover names +nel list -s builtin # also: -s skills / -s lm-eval + +# Validate before a long run: serve + run a few samples (catches serving/config errors cheap) +nel validate config.yaml + +# Track / control a submitted run +nel eval jobs # list tracked runs +nel eval status # progress of the current/last run +nel eval logs # tail logs +nel eval stop # cancel + +# Sharded run: after all shards finish, merge then export +nel eval merge / +nel export [ ...] --dest mlflow -o tracking_uri= -o experiment_name= # takes 1+ bundles +``` + +### Single-node vs sharded + +Sharding is **orthogonal to the benchmark config** — the `services:` / `benchmarks:` / `solver:` blocks +shown in the backend subsections below are identical either way. The only difference is one line in the +`cluster:` block plus a post-run merge: + +```text + single-node (no shards) sharded (cluster.shards: N) +---------------- --------------------------------- ---------------------------------------- +config diff cluster.shards absent (or = 1) cluster.shards: N (one added line) +benchmarks/solver identical identical +how it runs 1 worker serves the model, N independent single-node TP workers, each + runs ALL (problems × repeats) serves its OWN model copy, runs 1/N of them +launch nel eval run config.yaml nel eval run config.yaml (same command) +results one result bundle per benchmark N shard results -> nel eval merge / +export nel export ... nel export ... +when small / quick runs the long pole (e.g. aime25 repeats=64); ~N× faster +``` + +Key points: + +- **`repeats` is *not* divided — the *problem set* is.** Each shard runs a disjoint slice of all + `problems × repeats` tasks, so the merged result is mathematically identical to a single-node run, + just ~N× faster wall-clock. +- **This is the throughput mechanism, not data-parallel serving.** Sharding launches N separate + single-node jobs (each its own model instance); it does **not** rely on vLLM data-parallel (DP is + fragile on some stock vLLM builds — DP can fail in engine-core init). +- **The only extra step is `nel eval merge /`** — it stitches the N shard outputs into one + bundle per benchmark before you `nel export` (see "Export results to MLFlow" below). +- **Works cleanly for native (built-in / `skills://`).** Legacy `container://` is single-node only. A + `gym://` run *can* shard but it isn't a free swap — its one shared resource server must be at a + routable host (not loopback) reachable from every shard (see the `gym://` section); the validated gym + path is single-node `cluster: {type: local}`. + +### The `solver:` block — `simple` vs `tool_calling` + +`solver:` sets **how the model answers each problem**; `service:` names which `services:` entry to run +against (e.g. `service: model`). + +- **`simple`** — one model call per problem, no tools. For pure reasoning (knowledge / math / multichoice). +- **`tool_calling`** — a multi-turn ReAct loop: the model calls tools that a `sandbox:` block runs, up to + `max_turns` (turn-exhaustion = a failed episode). Needs `sandbox_tools: true` + a `sandbox:`, and a model + served with tool calling enabled (vLLM: `--enable-auto-tool-choice` + `--tool-call-parser`). For + tool-augmented tasks (run code, compute, search). + +**Backend and solver are independent.** A native built-in / `skills://` benchmark can run *either* solver +— e.g. `gpqa` with `simple` (no-tools) or with `tool_calling` + a `local` sandbox (the "with-tools" +variant). `tool_calling` is **not** exclusive to `gym://`. + + +### built-in and skills:// (native) + +The common path — every matrix row that is `yes` under **built-in** or **skills** with **no `†`**. The +engine serves the model itself and scores per-sample; just list the names in `benchmarks:` with a +`simple` solver (no sandbox, no server). Built-in takes a bare name; nemo-skills takes the `skills://` +prefix. + +Built-in benchmarks need nothing extra. For **`skills://`**, first install **NeMo-Skills** into your eval +image — `pip install nemo-skills` from [NVIDIA-NeMo/Skills](https://github.com/NVIDIA-NeMo/Skills); see the +repo README. It's baked into `cluster.eval_image` (and the dataset is prepped there). Likewise `lm-eval://` +needs `lm-eval` in the image. + +```yaml +benchmarks: + - name: gpqa # built-in (multichoice) + repeats: 8 + max_concurrent: 64 + solver: {type: simple, service: model} + - name: mmlu_pro # built-in (multichoice) + repeats: 1 + max_concurrent: 64 + solver: {type: simple, service: model} + - name: skills://aime25 # nemo-skills (math) + repeats: 64 + max_concurrent: 256 + solver: {type: simple, service: model} + # WITH-TOOLS variant of the SAME native benchmark — backend unchanged, just swap the solver and add a + # sandbox (see "The solver: block" above). Use instead of the simple gpqa entry to give it tools: + # - name: gpqa + # repeats: 8 + # max_concurrent: 64 + # solver: {type: tool_calling, service: model, sandbox_tools: true, max_turns: 100} + # sandbox: {type: local, concurrency: 64} + +cluster: + shards: 8 # N single-node workers, each runs 1/N of (problems × repeats); `nel eval merge` after + # ... + type/account/walltime/eval_image/node_pools (see r030_example_eval.yaml) +``` + +`nel eval run config.yaml`. `repeats` matches your model card's sampling (e.g. gpqa avg-of-8, aime25 avg-of-64); +sampling (`temperature`, `top_p`, `max_tokens`) lives under `services.model.generation`. Full runnable +config: [`recipes/examples/r030_example_eval.yaml`](../recipes/examples/r030_example_eval.yaml) +(simple / no-tools). For a tool-calling benchmark see the `gym://` example below. + +### gym:// (server + reward) + +`gym://` runs a NeMo-Gym **resource server** that scores each response — the faithful path for the `†` +benchmarks `skills://` mis-scores and for agentic envs. We run it as a **two-job split**: the model on a +GPU node (its own serve job), and the grader + evaluator on a CPU node. Why split: serving needs the vLLM +container, while `nel` and the Gym server live in their own venvs (see the last caveat in §7). + +#### 1. The pieces (two jobs) + +```text +program what it does runs on +----------------------------- ---------------------------------------------------- ------------------------------ +the grader (gym env start) checks each answer, returns a score CPU node (you start it there) +the conductor (nel eval run) sends questions -> model, answers -> grader, tallies CPU node — in-process + (cluster: {type: local}) +the model answers the questions GPU node — its own vllm serve + job (services.model.type: api) +``` + +Both the grader and the conductor run on the CPU node that `gym_eval.sbatch` allocates (`--partition=cpu`, +§6) — `cluster: {type: local}` means NEL runs the conductor **in-process there, not on the login node**. The +conductor reaches the **model** over the network (the GPU node's hostname) and the **grader** at `127.0.0.1` +(same CPU node). + +#### 2. Start the gym grader + get its port + +First install NeMo-Gym (the `gym`/`ng` CLI) — clone [NVIDIA-NeMo/Gym](https://github.com/NVIDIA-NeMo/Gym) +and `uv venv --python 3.12 && uv sync`; see the repo README. + +The grader is **not** started by NEL — you launch it on the **CPU node** (same node as the evaluator, so +it's reachable at `127.0.0.1`). Use **`--resources-server `** to bring up *only* the grader: a +benchmark's bundled config also defines an **agent** that needs its own model server, which you don't want +here (NEL is the conductor and serves its own model). It registers with a head process on `:11000` and +gets a **dynamic** port: + +```bash +export RAY_TMPDIR=/tmp # Lustre socket-path fix +gym env start --resources-server & # e.g. --resources-server ifbench +curl -s http://127.0.0.1:11000/server_instances | python3 -m json.tool # -> [{"name": .., "url": ".../"}] +``` + +The port is **assigned dynamically** — read it from `/server_instances` (the `url` field ends in +`:`) and drop it into the conductor config's `gym://127.0.0.1:` URI (§4). For scripting, parse it: + +```bash +PORT=$(curl -s http://127.0.0.1:11000/server_instances \ + | python3 -c "import json,sys; d=json.load(sys.stdin); print(d[0]['url'].rsplit(':',1)[1] if d else '')") +``` + +Running more than one server lists them all — match the row by `name`. + + +#### 3. Get the data + +The questions live in a **local `tasks.jsonl`** that the conductor reads (`data=` points at it). Two ways: + +**Way 1 - no download (quick):** use the bundled [`example.jsonl`](https://github.com/NVIDIA-NeMo/Gym/blob/main/resources_servers/ifbench/data/example.jsonl) already in the Gym repo (each server has one under +`resources_servers//data/`; small, prompt field pre-built): + +```text +data=/resources_servers//data/example.jsonl +``` + +**Way 2 - full set (downloads from HuggingFace):** run the benchmark's `prepare.py` — one per benchmark dir under [`Gym/benchmarks/`](https://github.com/NVIDIA-NeMo/Gym/tree/main/benchmarks) (e.g. [`benchmarks/ifbench/prepare.py`](https://github.com/NVIDIA-NeMo/Gym/blob/main/benchmarks/ifbench/prepare.py)): + +```bash +cd # your NeMo-Gym clone +export HF_HOME=/huggingface # keep the HF download off your home dir +PYTHONPATH=$PWD .venv/bin/python benchmarks//prepare.py # downloads from HF -> benchmarks//data/ +# then point data= at the output, e.g.: +# data=/benchmarks/ifbench/data/ifbench_benchmark_eval.jsonl +``` + +(Native path: each row needs a `responses_create_params` prompt field - bundled `example.jsonl` already +has it; `prepare.py` output may need it added.) + + +#### 4. The conductor config (what `nel eval run` reads) + +This is the config the **conductor** (`nel eval run` — the third piece in §1) reads. It tells the conductor +which **model** to call (§5), which **grader** + **data** to score against (§2/§3), where it runs, and where +to write results. It is *not* the grader's config (that's §2's gym server) — it's the run itself. + +Its placeholders are filled by the other steps — the grader port (§2), the data path (§3), and the model host (§5): +model = external endpoint (`type: api`), benchmark = the local gym grader, evaluator in-process +(`cluster: {type: local}`): + +```yaml +services: + model: + type: api + url: http://:8000/v1/chat/completions # from §5 (serve_host.txt) + model: + api_key: dummy + generation: + temperature: 1.0 + top_p: 0.95 + max_tokens: 32768 # reasoning model: big enough to finish reasoning + emit the answer +benchmarks: + # simple = single-shot (ifbench / ifeval); for agentic envs use `tool_calling` + a `sandbox:` block. + - name: "gym://127.0.0.1:?protocol=native&data=/.../tasks.jsonl" # from §2, data from §3 + repeats: 1 + max_concurrent: 64 + solver: {type: simple, service: model} +cluster: + type: local # evaluator runs in-process on the CPU node +output: + dir: +``` + +Full runnable config: [`recipes/examples/r030_gym.yaml`](../recipes/examples/r030_gym.yaml). + + +#### 5. Serve the model on a GPU node + +`type: api` means you serve the model yourself — a plain `vllm serve` in its own GPU `sbatch` job (the +vLLM **container**). Publish the node's hostname so the eval job can build `url`: + +```bash +#!/bin/bash +# serve.sbatch +#SBATCH --job-name=serve-policy +#SBATCH --partition= +#SBATCH --gres=gpu:4 +#SBATCH --nodes=1 +#SBATCH --ntasks-per-node=1 +#SBATCH --time=04:00:00 +#SBATCH --exclusive +hostname > /serve_host.txt # publish the node for the eval job +srun --container-image \ + --container-mounts :/model:ro,:/cache \ + bash -lc 'vllm serve /model --served-model-name --host 0.0.0.0 --port 8000 \ + --tensor-parallel-size 4 --max-model-len 262144 --trust-remote-code \ + --reasoning-parser

--tool-call-parser

...' +``` + +- **`--host 0.0.0.0`** (not `127.0.0.1`) so the CPU eval node can reach it cross-node; the GPU node serves only the model. +- **`` is resolved at runtime** — you don't know the node until SLURM dispatches the job, so the serve job writes its `hostname` to a shared file and the eval job reads it (`POLICY_HOST=$(cat /serve_host.txt)`) to fill `url`. + + +#### 6. Run it — the two-job pattern + +From the login node you submit **two** jobs — `serve.sbatch` (§5) and `gym_eval.sbatch` (below): + +```bash +sbatch serve.sbatch # GPU: §5 — vllm serve (publishes serve_host.txt) +sbatch gym_eval.sbatch # CPU: the script below +``` + +`gym_eval.sbatch` bundles §2 (start grader + get port) + the host hand-off + the conductor (§4 config): + +```bash +#!/bin/bash +# gym_eval.sbatch +#SBATCH --job-name=gym-eval +#SBATCH --partition=cpu +#SBATCH --nodes=1 +#SBATCH --cpus-per-task=32 +#SBATCH --mem=128G +#SBATCH --time=02:00:00 +export RAY_TMPDIR=/tmp # Lustre socket-path fix +# 1. start the grader (§2) on this node, then wait for it to register + read its DYNAMIC port +cd # so --resources-server resolves +gym env start --resources-server & +PORT="" +for i in $(seq 1 60); do # server takes a few s to register + PORT=$(curl -s http://127.0.0.1:11000/server_instances \ + | python3 -c "import json,sys; d=json.load(sys.stdin); print(d[0]['url'].rsplit(':',1)[1] if d else '')") + [ -n "$PORT" ] && break; sleep 5 +done +[ -z "$PORT" ] && { echo "gym server did not come up"; exit 1; } +# 2. wait for the GPU serve (serve_host.txt + model live), then fill the §4 config placeholders +until [ -f /serve_host.txt ]; do sleep 5; done +POLICY_HOST=$(cat /serve_host.txt) +until curl -s --max-time 8 "http://$POLICY_HOST:8000/v1/models" | grep -q '"id"'; do sleep 10; done +sed -i "s##$POLICY_HOST#; s##$PORT#" config.yaml +# 3. run the conductor (§4) +nel eval run config.yaml +``` + +Submit `serve.sbatch` first (a plain `sbatch --dependency=after` only waits for the serve job to *start*, +not for vLLM to finish loading — that's why `gym_eval.sbatch` polls `…/v1/models` above before running). + +#### 7. Caveats + +- **Reasoning models need a big enough `generation.max_tokens`** — too small truncates mid-reasoning, the + grader sees empty output, and the score collapses. +- **`example.jsonl` is a small subset** — use `prepare.py` (§3) for the full benchmark. +- **Sharding:** a gym run has one shared grader, so it doesn't shard across nodes cleanly (loopback breaks) + — keep it single-node. +- **Why not "one GPU node, NEL serves the model" (`type: vllm` + `cluster: {type: local}`)?** NEL's local + path runs `python -m vllm` from the `nel` venv (it ignores the container `image:`), and that venv has no + vLLM — so it errors at model startup. Serving needs the vLLM container, which is why §5 is a separate job. + + +### legacy container:// + +Backward-compat path for v0.2.x's eval-factory `*_aa_v3` container tasks that have no native 0.3.x +benchmark. The `container` solver injects a v1-format `run_config.yaml`, so v1 configs port with +minimal change — but **`cluster.shards:` is unsupported** for legacy container runs (single-node only). + +```yaml +benchmarks: + - name: mmlu_pro_aa_v3 # v0.2.x eval-factory container task + solver: {type: container, service: model} + +cluster: + # shards: N # UNSUPPORTED for legacy container — single-node only (omit it) + # ... + type/account/walltime/eval_image/node_pools +``` + +Prefer the native built-in / `skills://` equivalent (`mmlu_pro`, `gpqa`) when one exists — you get +sharding + the one eval image. Use `container://` only when there is no native port. + + +## Export results to MLFlow + +SLURM runs do **not** auto-export (the `output.export` block only fires for local runs). After all +shards finish, merge then export from the login node: + +```bash +# RUN = the run dir: / (one timestamped subdir per run; holds shard_0/ ... shard_N/) +RUN=/ + +# 1. merge the N shards -> writes the merged per-benchmark bundles BACK INTO $RUN (alongside the shards) +nel eval merge "$RUN" + +# 2. export each merged benchmark bundle ($RUN/) to MLflow +nel export "$RUN/gpqa" "$RUN/mmlu_pro" "$RUN/skills___aime25" --dest mlflow \ + -o tracking_uri= \ + -o experiment_name= +``` + +Both commands use the **same** run dir (`/`): merge reads its `shard_*/` +subdirectories and writes the merged `/` bundles next to them, which export then consumes. Tags +from `output.export_config.mlflow.tags` (e.g. `model`, `checkpoint_path`, `num_nodes`, `precision`, +`variant`) ride along and feed your MLflow dashboards / downstream viewers. The `/` in a +`skills://` name becomes `___` in the on-disk bundle dir (`skills___aime25`). From 4b4f4ee6d97b4300a8dd629fb20542cca166e644 Mon Sep 17 00:00:00 2001 From: Hung-Yueh Chiang Date: Sat, 27 Jun 2026 23:00:38 -0700 Subject: [PATCH 02/15] =?UTF-8?q?docs(eval):=20gym=20caveats=20=E2=80=94?= =?UTF-8?q?=20enable=5Fthinking=20via=20proxy.extra=5Fbody=20+=20manual=20?= =?UTF-8?q?resume?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - §7: how to enable reasoning (`chat_template_kwargs: {enable_thinking: true}` under services.model.proxy.extra_body) — a model setting, applies to native runs too. - §7: clarify that cluster:{type: local} has no auto_resume (slurm-only); recover a killed gym run by re-submitting with `nel eval run --resume` (progress is checkpointed). Signed-off-by: Hung-Yueh Chiang Co-Authored-By: Claude Opus 4.8 (1M context) --- .../skills/evaluation/references/nel-v0.3.0-migration.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.agents/skills/evaluation/references/nel-v0.3.0-migration.md b/.agents/skills/evaluation/references/nel-v0.3.0-migration.md index 5e503b4c3f6..46cc895ce41 100644 --- a/.agents/skills/evaluation/references/nel-v0.3.0-migration.md +++ b/.agents/skills/evaluation/references/nel-v0.3.0-migration.md @@ -539,6 +539,13 @@ not for vLLM to finish loading — that's why `gym_eval.sbatch` polls `…/v1/mo - **Reasoning models need a big enough `generation.max_tokens`** — too small truncates mid-reasoning, the grader sees empty output, and the score collapses. +- **Reasoning models: enable thinking via `proxy.extra_body`** — pass `chat_template_kwargs: {enable_thinking: + true}` under `services.model.proxy.extra_body` (it's merged into every request) so the chat template emits + the reasoning block. This is a *model* setting (applies to native runs too), not gym-specific. +- **Resuming a killed run is manual.** `cluster: {type: local}` does not auto-restart after a SLURM + wall-clock kill (NEL's `auto_resume` only chains `slurm`-cluster jobs). Finished rollouts are + checkpointed, though — just re-submit the job with `nel eval run --resume` and it skips the done ones + and continues. - **`example.jsonl` is a small subset** — use `prepare.py` (§3) for the full benchmark. - **Sharding:** a gym run has one shared grader, so it doesn't shard across nodes cleanly (loopback breaks) — keep it single-node. From 465bce79a9c20d4ee120fb02b020b3c1393ada75 Mon Sep 17 00:00:00 2001 From: Hung-Yueh Chiang Date: Sat, 27 Jun 2026 23:03:04 -0700 Subject: [PATCH 03/15] docs(eval): native (cluster:slurm) auto-resumes; gym (local) is manual --resume MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Clarify in the native section that built-in/skills runs use cluster:{type: slurm} (auto_resume: true) so NEL chains a successor job on a wall-clock kill — no manual step — in contrast to gym://'s cluster:{type: local} (manual `nel eval run --resume`). Signed-off-by: Hung-Yueh Chiang Co-Authored-By: Claude Opus 4.8 (1M context) --- .agents/skills/evaluation/references/nel-v0.3.0-migration.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.agents/skills/evaluation/references/nel-v0.3.0-migration.md b/.agents/skills/evaluation/references/nel-v0.3.0-migration.md index 46cc895ce41..b5496997ae4 100644 --- a/.agents/skills/evaluation/references/nel-v0.3.0-migration.md +++ b/.agents/skills/evaluation/references/nel-v0.3.0-migration.md @@ -352,6 +352,11 @@ sampling (`temperature`, `top_p`, `max_tokens`) lives under `services.model.gene config: [`recipes/examples/r030_example_eval.yaml`](../recipes/examples/r030_example_eval.yaml) (simple / no-tools). For a tool-calling benchmark see the `gym://` example below. +Because the native path uses `cluster: {type: slurm}` (`auto_resume: true` by default), NEL **auto-resumes** +these runs on a wall-clock kill — it chains a successor job, no manual step. (This differs from `gym://`, +which runs under `cluster: {type: local}` and must be resumed by hand with `nel eval run --resume` — see the +gym `§7` caveats.) + ### gym:// (server + reward) `gym://` runs a NeMo-Gym **resource server** that scores each response — the faithful path for the `†` From bb53eaebd1864aaaeb69c35e643015bd00c59854 Mon Sep 17 00:00:00 2001 From: Hung-Yueh Chiang Date: Sat, 27 Jun 2026 23:23:23 -0700 Subject: [PATCH 04/15] =?UTF-8?q?docs(eval):=20gym=20=C2=A72=20=E2=80=94?= =?UTF-8?q?=20document=20bundled-config=20instances=20+=20keep/drop=20rule?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Explain that a NeMo-Gym bundled config (resources_servers//configs/.yaml) packs multiple server instances: the resources_servers grader, plus a sample responses_api_agents agent that references an undefined responses_api_models. Add a component table (linked to upstream ifbench.yaml) and a keep/drop rule: NEL native needs only the grader; the agent is the gym-driven Responses-API path. Note that --resources-server / --config .yaml start the whole file and fail on the undefined model, so point --config at a grader-only config. Align §6 sbatch to match. Signed-off-by: Hung-Yueh Chiang Co-Authored-By: Claude Opus 4.8 (1M context) --- .../references/nel-v0.3.0-migration.md | 36 +++++++++++++++---- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/.agents/skills/evaluation/references/nel-v0.3.0-migration.md b/.agents/skills/evaluation/references/nel-v0.3.0-migration.md index b5496997ae4..03a84a84f41 100644 --- a/.agents/skills/evaluation/references/nel-v0.3.0-migration.md +++ b/.agents/skills/evaluation/references/nel-v0.3.0-migration.md @@ -387,14 +387,36 @@ First install NeMo-Gym (the `gym`/`ng` CLI) — clone [NVIDIA-NeMo/Gym](https:// and `uv venv --python 3.12 && uv sync`; see the repo README. The grader is **not** started by NEL — you launch it on the **CPU node** (same node as the evaluator, so -it's reachable at `127.0.0.1`). Use **`--resources-server `** to bring up *only* the grader: a -benchmark's bundled config also defines an **agent** that needs its own model server, which you don't want -here (NEL is the conductor and serves its own model). It registers with a head process on `:11000` and -gets a **dynamic** port: +it's reachable at `127.0.0.1`). A benchmark's bundled config (`resources_servers//configs/.yaml`) +usually packs **more than one server instance** into one file — you only start one of them. For +[ifbench](https://github.com/NVIDIA-NeMo/Gym/blob/main/resources_servers/ifbench/configs/ifbench.yaml): + +```text +top-level key in .yaml kind role NEL native: start it? +------------------------------ -------------------- ------------------------------- ---------------------- +ifbench resources_servers the GRADER — scores via /verify KEEP (all NEL needs) +ifbench_simple_agent responses_api_agents a sample gym-driven agent loop DROP + └ model_server: policy_model responses_api_models the model that agent would run (placeholder name only; + (referenced, NOT — gym-driven path, not NEL not defined in the file) + defined in the file) +``` + +**Keep / drop — which and when:** +- **NEL `gym://…?protocol=native` (this guide)** — NEL *is* the conductor: it calls the model itself, then + posts each answer to the grader's `/verify`. Start **only the `resources_servers` grader** (`ifbench`); + drop the agent. +- **gym-driven (Responses API)** — gym runs the `*_simple_agent` itself and needs a real + `responses_api_models` wired in (the bundle ships `policy_model` as an unresolved placeholder). Different + path; not what NEL native uses. + +`gym env start --resources-server ` (and `--config .yaml`) load the **whole** bundled file, so +they also try to start `*_simple_agent` and **fail** on the undefined `policy_model`. To bring up only the +grader, point `--config` at a config that keeps **just** the `resources_servers:` block (drop +`responses_api_agents:`). It registers with a head process on `:11000` and gets a **dynamic** port: ```bash export RAY_TMPDIR=/tmp # Lustre socket-path fix -gym env start --resources-server & # e.g. --resources-server ifbench +gym env start --config resources_servers//configs/.yaml & # Launch gym grader in background by using "&" curl -s http://127.0.0.1:11000/server_instances | python3 -m json.tool # -> [{"name": .., "url": ".../"}] ``` @@ -519,8 +541,8 @@ sbatch gym_eval.sbatch # CPU: the script below #SBATCH --time=02:00:00 export RAY_TMPDIR=/tmp # Lustre socket-path fix # 1. start the grader (§2) on this node, then wait for it to register + read its DYNAMIC port -cd # so --resources-server resolves -gym env start --resources-server & +cd +gym env start --config & # keep only resources_servers:, drop the agent (§2) PORT="" for i in $(seq 1 60); do # server takes a few s to register PORT=$(curl -s http://127.0.0.1:11000/server_instances \ From 7abb8e9e065a5a490d93d9c0dfa735cd8409caa8 Mon Sep 17 00:00:00 2001 From: Hung-Yueh Chiang Date: Sat, 27 Jun 2026 23:25:08 -0700 Subject: [PATCH 05/15] =?UTF-8?q?docs(eval):=20gym=20=C2=A76=20=E2=80=94?= =?UTF-8?q?=20match=20=C2=A72's=20grader=20start=20line=20+=20comment?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Align the §6 sbatch grader command with §2 (same --config resources_servers//configs/.yaml line and the "&" = background / keep resources_servers, drop the agent comment) so both code blocks are consistent. Signed-off-by: Hung-Yueh Chiang Co-Authored-By: Claude Opus 4.8 (1M context) --- .agents/skills/evaluation/references/nel-v0.3.0-migration.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.agents/skills/evaluation/references/nel-v0.3.0-migration.md b/.agents/skills/evaluation/references/nel-v0.3.0-migration.md index 03a84a84f41..9fa44383347 100644 --- a/.agents/skills/evaluation/references/nel-v0.3.0-migration.md +++ b/.agents/skills/evaluation/references/nel-v0.3.0-migration.md @@ -542,7 +542,8 @@ sbatch gym_eval.sbatch # CPU: the script below export RAY_TMPDIR=/tmp # Lustre socket-path fix # 1. start the grader (§2) on this node, then wait for it to register + read its DYNAMIC port cd -gym env start --config & # keep only resources_servers:, drop the agent (§2) +# Launch gym grader in background by using "&". Keep only resources_servers and drop the agent (§2) as needed +gym env start --config resources_servers//configs/.yaml & PORT="" for i in $(seq 1 60); do # server takes a few s to register PORT=$(curl -s http://127.0.0.1:11000/server_instances \ From 5314022f6f1e06c541250852acd40820cf3d09c1 Mon Sep 17 00:00:00 2001 From: Hung-Yueh Chiang Date: Sat, 27 Jun 2026 23:54:48 -0700 Subject: [PATCH 06/15] =?UTF-8?q?docs(eval):=20gym=20=C2=A77=20resume=20?= =?UTF-8?q?=E2=80=94=20two-step=20procedure=20+=20verified-cache=20note?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rewrite the gym "Resume the jobs" section into a clean two-step procedure (relaunch the serve, then resume the eval with --export=ALL,RESUME=--resume) and thread ${RESUME} into the §6 conductor line so the command works. Add that completed rollouts survive the resume even though the new serve node changes the model URL: NEL keys the skip on its verified log, which is retained on a config-hash change (only the un-scored inference cache is dropped). Tighten the native auto-resume paragraph and fix its cross-reference, and clarify the single-node/local sharding note in the §4 config snippet. Signed-off-by: Hung-Yueh Chiang Co-Authored-By: Claude Opus 4.8 (1M context) --- .../references/nel-v0.3.0-migration.md | 40 +++++++++++++------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/.agents/skills/evaluation/references/nel-v0.3.0-migration.md b/.agents/skills/evaluation/references/nel-v0.3.0-migration.md index 9fa44383347..f25ba5c93c8 100644 --- a/.agents/skills/evaluation/references/nel-v0.3.0-migration.md +++ b/.agents/skills/evaluation/references/nel-v0.3.0-migration.md @@ -352,10 +352,10 @@ sampling (`temperature`, `top_p`, `max_tokens`) lives under `services.model.gene config: [`recipes/examples/r030_example_eval.yaml`](../recipes/examples/r030_example_eval.yaml) (simple / no-tools). For a tool-calling benchmark see the `gym://` example below. -Because the native path uses `cluster: {type: slurm}` (`auto_resume: true` by default), NEL **auto-resumes** -these runs on a wall-clock kill — it chains a successor job, no manual step. (This differs from `gym://`, -which runs under `cluster: {type: local}` and must be resumed by hand with `nel eval run --resume` — see the -gym `§7` caveats.) +The native path runs under `cluster: {type: slurm}` with `auto_resume: true` (the default), so NEL +**auto-resumes** a wall-clock kill itself — it chains a successor SLURM job, no manual step. (A `gym://` run +is different: it runs under `cluster: {type: local}`, which has no `auto_resume`, so you resume it by hand — +see the gym section's "Resume the jobs".) ### gym:// (server + reward) @@ -484,7 +484,8 @@ benchmarks: max_concurrent: 64 solver: {type: simple, service: model} cluster: - type: local # evaluator runs in-process on the CPU node + # gym's single grader lives on 127.0.0.1, so a gym run can't shard across nodes — keep it single-node. + type: local # evaluator runs in-process on the CPU node (no shards) output: dir: ``` @@ -556,24 +557,39 @@ until [ -f /serve_host.txt ]; do sleep 5; done POLICY_HOST=$(cat /serve_host.txt) until curl -s --max-time 8 "http://$POLICY_HOST:8000/v1/models" | grep -q '"id"'; do sleep 10; done sed -i "s##$POLICY_HOST#; s##$PORT#" config.yaml -# 3. run the conductor (§4) -nel eval run config.yaml +# 3. run the conductor (§4); ${RESUME} is empty on a fresh run, "--resume" when resuming (§7) +nel eval run config.yaml ${RESUME:-} ``` Submit `serve.sbatch` first (a plain `sbatch --dependency=after` only waits for the serve job to *start*, not for vLLM to finish loading — that's why `gym_eval.sbatch` polls `…/v1/models` above before running). -#### 7. Caveats +#### 7. Resume the jobs + +Resuming a wall-clock kill is **manual** here. `cluster: {type: local}` has no `auto_resume` (that only +chains `slurm`-cluster jobs — see the native section), so neither job restarts itself, and the GPU serve +job dies too. A resume is therefore **two steps** — relaunch the serve, then resume the eval: + +```bash +# 1. relaunch the GPU serve — it rewrites serve_host.txt with the new node's hostname +sbatch serve.sbatch + +# 2. resume the eval — it waits for the fresh serve_host.txt + /v1/models, then continues +sbatch --export=ALL,RESUME=--resume gym_eval.sbatch # RESUME -> nel eval run … --resume (§6 step 3) +``` + +Completed rollouts are checkpointed per `(problem, repeat)`, so the resume **skips everything already +scored** and re-runs only what was in flight — even though the new serve node changes the model URL. (NEL +keys the skip on its *verified* log, which survives the config-hash change; only the un-scored inference +cache is dropped and regenerated.) + +#### 8. Caveats - **Reasoning models need a big enough `generation.max_tokens`** — too small truncates mid-reasoning, the grader sees empty output, and the score collapses. - **Reasoning models: enable thinking via `proxy.extra_body`** — pass `chat_template_kwargs: {enable_thinking: true}` under `services.model.proxy.extra_body` (it's merged into every request) so the chat template emits the reasoning block. This is a *model* setting (applies to native runs too), not gym-specific. -- **Resuming a killed run is manual.** `cluster: {type: local}` does not auto-restart after a SLURM - wall-clock kill (NEL's `auto_resume` only chains `slurm`-cluster jobs). Finished rollouts are - checkpointed, though — just re-submit the job with `nel eval run --resume` and it skips the done ones - and continues. - **`example.jsonl` is a small subset** — use `prepare.py` (§3) for the full benchmark. - **Sharding:** a gym run has one shared grader, so it doesn't shard across nodes cleanly (loopback breaks) — keep it single-node. From 835db3f082ce0480ddfefd17cdb3af416ed07381 Mon Sep 17 00:00:00 2001 From: Hung-Yueh Chiang Date: Sun, 28 Jun 2026 01:10:04 -0700 Subject: [PATCH 07/15] =?UTF-8?q?docs(eval):=20export=20=E2=80=94=20requir?= =?UTF-8?q?e=20a=20model=20tag=20for=20run=20attribution=20+=20tags=20exam?= =?UTF-8?q?ple?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a reminder in "Export results to MLflow" that downstream MLflow dashboards key/group runs by the `model` (and `checkpoint_path`) tag: a run exported without them can't be attributed (shows as an orphan/"no checkpoint" row), and a drifting `model` value splits one logical row. Include a concrete `output.export_config.mlflow.tags` YAML example. Signed-off-by: Hung-Yueh Chiang Co-Authored-By: Claude Opus 4.8 (1M context) --- .../references/nel-v0.3.0-migration.md | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/.agents/skills/evaluation/references/nel-v0.3.0-migration.md b/.agents/skills/evaluation/references/nel-v0.3.0-migration.md index f25ba5c93c8..56a15fd7a7c 100644 --- a/.agents/skills/evaluation/references/nel-v0.3.0-migration.md +++ b/.agents/skills/evaluation/references/nel-v0.3.0-migration.md @@ -641,3 +641,30 @@ subdirectories and writes the merged `/` bundles next to them, which expo from `output.export_config.mlflow.tags` (e.g. `model`, `checkpoint_path`, `num_nodes`, `precision`, `variant`) ride along and feed your MLflow dashboards / downstream viewers. The `/` in a `skills://` name becomes `___` in the on-disk bundle dir (`skills___aime25`). + +> **Always set a stable `model` (and ideally `checkpoint_path`) tag.** Dashboards key/group runs by +> these tags to attribute a score to a model or checkpoint. A run exported **without** them can't be +> attributed — it shows up as an orphan/"no checkpoint" row instead of landing on the right model — and +> a `model` value that drifts between runs (or differs from a baseline's) splits what should be one row. +> The engine logs only a generic headline metric (`pass_at_1` / `mean_reward`); the benchmark identity +> rides in the `benchmark` tag and the model identity in `model`, so both tags are what make the export +> usable downstream. + +Example `output` block with the tags set (lives in the run config; `nel export` also accepts the same +tracking URI / experiment via `-o`): + +```yaml +output: + dir: + export: [mlflow] + export_config: + mlflow: + tracking_uri: + experiment_name: + tags: + model: # set this — dashboards group/attribute runs by it + checkpoint_path: # the per-checkpoint row label downstream + benchmark: # benchmark identity (the engine logs only a generic metric key) + precision: bf16 + variant: # e.g. base / with-tools / 96k-thinking +``` From 4bed54ac258809ba09d24b035410811421fe00ed Mon Sep 17 00:00:00 2001 From: Hung-Yueh Chiang Date: Sun, 28 Jun 2026 09:55:33 -0700 Subject: [PATCH 08/15] =?UTF-8?q?docs(eval):=20example=20configs=20?= =?UTF-8?q?=E2=80=94=20export=20tags=20+=20discoverable=20enable=5Fthinkin?= =?UTF-8?q?g?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit r030_gym.yaml (cluster: local auto-exports): add the output.export_config.mlflow block with tags (model/checkpoint_path/benchmark/source/precision/variant) so a gym run is attributed downstream instead of landing as an orphan "no checkpoint" row. r030_example_eval.yaml: add checkpoint_path to the tags (and note why a multi-benchmark run carries no static benchmark tag). Both: add a commented services.model.proxy.extra_body.chat_template_kwargs.enable_thinking hint so users discover the reasoning toggle (with the max_tokens-truncation caveat). Signed-off-by: Hung-Yueh Chiang Co-Authored-By: Claude Opus 4.8 (1M context) --- .../recipes/examples/r030_example_eval.yaml | 12 +++++++++- .../evaluation/recipes/examples/r030_gym.yaml | 22 +++++++++++++++++-- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/.agents/skills/evaluation/recipes/examples/r030_example_eval.yaml b/.agents/skills/evaluation/recipes/examples/r030_example_eval.yaml index 78b27fb60e8..9b1ee70571b 100644 --- a/.agents/skills/evaluation/recipes/examples/r030_example_eval.yaml +++ b/.agents/skills/evaluation/recipes/examples/r030_example_eval.yaml @@ -17,6 +17,13 @@ services: model: /path/to/checkpoint served_model_name: my-model # required; referenced by benchmarks protocol: chat_completions + # Reasoning models: uncomment to emit the thinking block (see the migration doc's gym caveats). + # extra_body is merged into every request; pair it with a big enough generation max_tokens or the + # reasoning gets truncated -> empty answer -> score collapses. (Applies to native type:vllm too.) + # proxy: + # extra_body: + # chat_template_kwargs: + # enable_thinking: true port: 8000 node_pool: gpu # Single-instance serving. Pick TP/PP from model size & GPU count (references/parallelism.md). @@ -106,6 +113,9 @@ output: tracking_uri: experiment_name: tags: - model: my-model + model: my-model # set this — dashboards attribute/group runs by it + checkpoint_path: # per-checkpoint row label downstream precision: pipeline: r0.3.0 + # (no static `benchmark` tag — this config runs several benchmarks; the engine + # carries each one's identity in its own export bundle) diff --git a/.agents/skills/evaluation/recipes/examples/r030_gym.yaml b/.agents/skills/evaluation/recipes/examples/r030_gym.yaml index c854e2665cc..3955464a312 100644 --- a/.agents/skills/evaluation/recipes/examples/r030_gym.yaml +++ b/.agents/skills/evaluation/recipes/examples/r030_gym.yaml @@ -29,6 +29,13 @@ services: generation: temperature: 0.0 max_tokens: 512 + # Reasoning models: uncomment to emit the thinking block (see the migration doc's gym caveats). + # extra_body is merged into every request; pair it with a big enough generation.max_tokens or the + # reasoning gets truncated -> empty answer -> score collapses. + # proxy: + # extra_body: + # chat_template_kwargs: + # enable_thinking: true benchmarks: # The benchmark lives in the running server + the data file, NOT after the prefix. @@ -52,5 +59,16 @@ cluster: output: dir: /abs/path/to/rundir # MUST be absolute - # export is run manually after the job (see r030_example_eval.yaml / migration doc "Export to MLFlow"): - # nel export / --dest mlflow -o tracking_uri= -o experiment_name= + export: + - mlflow # cluster:local auto-exports at run end (or `nel export ` manually) + export_config: + mlflow: + tracking_uri: + experiment_name: + tags: # ride into the bundle -> MLflow; what dashboards attribute runs by + model: # set this — without it the run lands as an orphan "no checkpoint" row + checkpoint_path: # per-checkpoint row label downstream + benchmark: # a gym:// name is a URI, so set the clean benchmark id explicitly here + source: gym + precision: + variant: # e.g. 96k-thinking From 88c9edbca13e256a3c948a52cb02fccfbe77c5c1 Mon Sep 17 00:00:00 2001 From: Hung-Yueh Chiang Date: Sun, 28 Jun 2026 10:17:47 -0700 Subject: [PATCH 09/15] =?UTF-8?q?docs(eval):=20matrix=20=E2=80=94=20clarif?= =?UTF-8?q?y=20Harbor/OpenClaw=20+=20sandbox=20for=20agentic=20built-ins?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The "built-in" terminal-bench-v1/-hard and nmp_harbor are built-in NAMES but run via the Harbor package (HarborSolver) inside a sandbox, not in-process; pinchbench is built-in via the OpenClaw solver + sandbox. Tighten those four notes so the execution requirement (harbor pkg / OpenClaw + sandbox) is explicit. Signed-off-by: Hung-Yueh Chiang Co-Authored-By: Claude Opus 4.8 (1M context) --- .../skills/evaluation/references/nel-v0.3.0-migration.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.agents/skills/evaluation/references/nel-v0.3.0-migration.md b/.agents/skills/evaluation/references/nel-v0.3.0-migration.md index 56a15fd7a7c..b7aa441bce4 100644 --- a/.agents/skills/evaluation/references/nel-v0.3.0-migration.md +++ b/.agents/skills/evaluation/references/nel-v0.3.0-migration.md @@ -163,9 +163,9 @@ LONG-CONTEXT longcodebench — yes yes aalcr — yes†(judge) yes gym recommended (skills may mis-score) AGENTIC / TOOL / RL (mostly gym-only; the few built-ins are noted) - terminal-bench-v1 yes — — built-in; agentic terminal tasks - terminal-bench-hard yes — — built-in (+ -aa-split variant) - nmp_harbor yes — — built-in (harbor packaging) + terminal-bench-v1 yes — — built-in name; Harbor-run (harbor pkg + sandbox) + terminal-bench-hard yes — — Harbor-run (harbor pkg + sandbox); +-aa-split variant + nmp_harbor yes — — harbor-packaging umbrella; HarborSolver + sandbox tau2 — — yes gdpval — — yes browsecomp — — yes @@ -180,7 +180,7 @@ AGENTIC / TOOL / RL (mostly gym-only; the few built-ins are noted) xlam_fc — — yes ns_tools — — yes swe_pivot — — yes - pinchbench yes — — built-in only (NOT gym) + pinchbench yes — — built-in name (NOT gym); OpenClaw solver + sandbox ARENA / JUDGE arena_hard — yes†(judge) yes the "LM Arena" family arena_hard_v2 — yes†(judge) yes From c641b4ef88fdab43120ba65039418de79652d743 Mon Sep 17 00:00:00 2001 From: Hung-Yueh Chiang Date: Sun, 28 Jun 2026 21:02:44 -0700 Subject: [PATCH 10/15] =?UTF-8?q?docs(eval):=20matrix=20=E2=80=94=20add=20?= =?UTF-8?q?Harbor-playbook=20backend=20(TB=202.x,=20SWE-bench),=20fix=20sw?= =?UTF-8?q?e-bench/vlm=20rows?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The matrix only had built-in/skills/gym axes, so benchmarks reachable only via Harbor playbooks were invisible. Add `harbor://` to the backend list, a dedicated "Harbor playbooks" subsection (terminal-bench 2.0/2.1, SWE-bench Verified/Pro/ Multilingual, each `- playbook: ` → harbor:// target), and a harbor line in the launch-mapping. Correct the swe-bench note (Verified/Pro/Multilingual run via Harbor, not just SWE-RL gym servers) and the vlm_eval_kit row (vlmevalkit:// backend, not "gym only"). Signed-off-by: Hung-Yueh Chiang Co-Authored-By: Claude Opus 4.8 (1M context) --- .../references/nel-v0.3.0-migration.md | 29 +++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/.agents/skills/evaluation/references/nel-v0.3.0-migration.md b/.agents/skills/evaluation/references/nel-v0.3.0-migration.md index b7aa441bce4..47225f246c8 100644 --- a/.agents/skills/evaluation/references/nel-v0.3.0-migration.md +++ b/.agents/skills/evaluation/references/nel-v0.3.0-migration.md @@ -78,6 +78,8 @@ skills:// skills:// `nel list -s skills` (after prep) / lm-eval:// lm-eval:// `lm_eval --tasks list` gym:// gym://?protocol= Gym repo benchmarks/ + resources_servers/ native&data= (needs a running server → github.com/NVIDIA-NeMo/Gym) +harbor:// `- playbook: ` or src/nemo_evaluator/playbooks/ (terminal_bench_2, + harbor:// swebench_verified, …); HarborSolver + sandbox (harbor pkg) ``` Example — a `benchmarks:` block, one entry per benchmark. The `name:` prefix selects the backend (it is **not** uniformly `{backend}://{benchmark}`): built-in has **no prefix** (just `{benchmark}`); `skills://` and `lm-eval://` are `{backend}://{benchmark}`; `gym://` points at a running server, `gym://{host:port}?...&data=` (the benchmark lives in the data file, not after the prefix): @@ -145,7 +147,7 @@ CODE / SWE scicode — yes†(code) yes gym recommended (skills may mis-score) ioi — yes†(code) yes cvdp — yes†(code) yes - swe-bench — yes — gym has only SWE-RL servers (swe_pivot/swerl_*) + swe-bench — yes — Verified/Pro/Multiling via Harbor playbooks (below); gym=SWE-RL (swe_pivot/swerl_*) bird_sql — — yes gym only spider2_lite — — yes gym only code_gen — — yes gym only @@ -201,10 +203,32 @@ MULTILINGUAL / MULTIMODAL / AUDIO covost2 — yes — skills only (audio) fleurs — yes — skills only (audio) labbench2_vlm — — yes gym only (multimodal) - vlm_eval_kit — — yes gym only (multimodal) + vlm_eval_kit — — — vlmevalkit:// backend (multimodal); not gym ``` +### Harbor playbooks (terminal-bench 2.x, SWE-bench) — a separate mechanism + +These are **not** built-in / `skills://` / `gym://`, so they don't appear in the matrix above. They run +through the **Harbor** package via a `- playbook: ` entry (HarborSolver + sandbox), pointing at a +`harbor://` dataset: + +```text +playbook (`- playbook:`) harbor:// target what it is +------------------------- ----------------------------------------- ---------------------------------- +terminal_bench_2 harbor://terminal-bench@2.0 Terminal-Bench 2.0 +terminal_bench_2_1 harbor://terminal-bench/terminal-bench-2-1 Terminal-Bench 2.1 (needs a harbor + registry override — not yet in + upstream registry.json) +swebench_verified harbor://swebench-verified@1.0 SWE-bench Verified +swebench_pro harbor://swebenchpro@1.0 SWE-bench Pro +swebench_multilingual harbor://swebench_multilingual@1.0 SWE-bench Multilingual +``` + +All require the harbor package (`pip install nemo-evaluator[harbor]`) + a sandbox, like `terminal-bench-v1`. +(terminal-bench **v1** and `-hard` are `@register` built-ins; **2.0 / 2.1** are playbook-only.) + + ## Launch NEL v0.3.0 The launch pattern follows the **backend**, not the category. Map each matrix category to the matching @@ -217,6 +241,7 @@ built-in and skills:// (native) KNOWLEDGE/QA · MATH · LONG-CONTEXT · MULTI (every row `yes` under built-in or skills with NO †) gym:// (server + reward) AGENTIC/TOOL/RL (gym-only) + every † row (CODE/SWE, INSTRUCTION/FORMAT, ARENA/JUDGE, LONG-CONTEXT aalcr) +harbor:// (playbook + sandbox) terminal-bench v1/2/2.1, SWE-bench Verified/Pro/Multilingual, pinchbench legacy container:// v0.2.x `*_aa_v3` eval-factory container tasks ``` From 6a49d5f4965f4ff5a73dc93c863c437d5b71db9d Mon Sep 17 00:00:00 2001 From: Hung-Yueh Chiang Date: Sun, 28 Jun 2026 21:08:26 -0700 Subject: [PATCH 11/15] docs(eval): document harbor:// as a first-class (external) backend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Harbor is the Laude Institute's framework (github.com/laude-institute/harbor), integrated via `nemo-evaluator[harbor]`, with its own registry of 60+ agentic tasks (terminal-bench, swebench, usaco, bird-bench, …) — not a NeMo repo. Add it to the backend enumerations so it isn't hidden: the benchmark-families row, the source list (as the external 4th source), the name-prefix example (`harbor://… / - playbook:`), and the matrix disclaimer (like lm-eval://, not enumerated as rows). Also clarify that ALL terminal-bench variants execute via Harbor (HarborEnvironment + sandbox) — built-in name (v1/-hard) vs playbook (2.0/2.1) is only how you reference them. Signed-off-by: Hung-Yueh Chiang Co-Authored-By: Claude Opus 4.8 (1M context) --- .../references/nel-v0.3.0-migration.md | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/.agents/skills/evaluation/references/nel-v0.3.0-migration.md b/.agents/skills/evaluation/references/nel-v0.3.0-migration.md index 47225f246c8..1bf69f2b303 100644 --- a/.agents/skills/evaluation/references/nel-v0.3.0-migration.md +++ b/.agents/skills/evaluation/references/nel-v0.3.0-migration.md @@ -6,7 +6,7 @@ | --- | --- | --- | | Benchmark source | each task runs in its **own published eval-factory container** (`nvcr.io/nvidia/eval-factory/*`); you never build an eval image | **one** `cluster.eval_image`; benchmarks resolve from Python `@register()` modules | | Where harnesses live | baked into per-task containers upstream | **you pip-install** external harnesses (`nemo-skills`, `lm-eval`) **into the one eval image** | -| Benchmark families | eval-factory `container/` adapters, `ns_*` nemo-skills | **built-in** (17, `nel list -s builtin`), **`skills://`** (nemo-skills), **`lm-eval://`**, + legacy **`container://`** (BC) | +| Benchmark families | eval-factory `container/` adapters, `ns_*` nemo-skills | **built-in** (17, `nel list -s builtin`), **`skills://`** (nemo-skills), **`lm-eval://`**, **`harbor://`** (agentic, Harbor pkg), + legacy **`container://`** (BC) | | CLI | `nel run` / `ls` / `status` / `info` | `nel eval run` / `list` / `export` | | Config schema | Hydra `deployment:` / `evaluation:` / `execution:` | `services:` / `benchmarks:` / `cluster:` / `output:` (one file) | | Serving | a `deployment:` block | integrated under `services.model` | @@ -90,17 +90,21 @@ benchmarks: # the benchmarks block - name: skills://aime25 # NeMo-Skills - name: lm-eval://hellaswag # lm-eval - name: gym://127.0.0.1:8000?protocol=native&data=tasks.jsonl # NeMo-Gym (running server) + - name: harbor://terminal-bench@2.0 # Harbor (agentic; sandbox + harbor pkg) — or `- playbook: terminal_bench_2` ``` ### Supported Benchmarks -The list of supported benchmarks are from three NVIDIA-NeMo repositories: +The list of supported benchmarks are from three NVIDIA-NeMo repositories, plus the **external Harbor +framework** the engine integrates: - `NVIDIA-NeMo/Evaluator`: built-in engine [`@register`](https://github.com/NVIDIA-NeMo/Evaluator/tree/main/src/nemo_evaluator/benchmarks) - `NVIDIA-NeMo/Skills`: **`skills://`** from [`nemo_skills/dataset/`](https://github.com/NVIDIA-NeMo/Skills/tree/main/nemo_skills/dataset), - `NVIDIA-NeMo/Gym`: **`gym://`** from Gym [`benchmarks/`](https://github.com/NVIDIA-NeMo/Gym/tree/main/benchmarks)+[`resources_servers/`](https://github.com/NVIDIA-NeMo/Gym/tree/main/resources_servers). +- **Harbor** ([`laude-institute/harbor`](https://github.com/laude-institute/harbor), *not* a NeMo repo): **`harbor://`** agentic tasks from Harbor's own registry (60+: terminal-bench, swebench, usaco, bird-bench, …), integrated via `pip install nemo-evaluator[harbor]` and run in a sandbox (see "Harbor playbooks" below). -The matrix below covers only these three backends. **`lm-eval://`** (e.g. `lm-eval://hellaswag`) is also -supported but not enumerated here — discover its tasks with `lm_eval --tasks list`. Only the **built-in** +The matrix below covers only these three backends. **`lm-eval://`** (e.g. `lm-eval://hellaswag`) and +**`harbor://`** (60+ agentic tasks — discover via Harbor's registry; see "Harbor playbooks" below) are +also supported but not enumerated here — discover lm-eval tasks with `lm_eval --tasks list`. Only the **built-in** column is verified against this engine (`nel list -s builtin`); the **`skills://`** and **`gym://`** columns are compiled from the upstream repos and have **not** all been run — treat them as a routing guide, not a guarantee (see the per-row notes). @@ -225,8 +229,14 @@ swebench_pro harbor://swebenchpro@1.0 SWE-bench swebench_multilingual harbor://swebench_multilingual@1.0 SWE-bench Multilingual ``` -All require the harbor package (`pip install nemo-evaluator[harbor]`) + a sandbox, like `terminal-bench-v1`. -(terminal-bench **v1** and `-hard` are `@register` built-ins; **2.0 / 2.1** are playbook-only.) +All require the harbor package (`pip install nemo-evaluator[harbor]`) + a sandbox. + +**Note:** *every* terminal-bench variant executes through Harbor (`HarborEnvironment` + sandbox) — only the +invocation differs. **v1 / -hard / -hard-aa-split** are `@register` built-in *names* (subclasses of +`HarborEnvironment` that auto-download + map to Harbor format), so you run `- name: terminal-bench-v1`. +**2.0 / 2.1** aren't registered as names, so you run them as `- playbook: terminal_bench_2` pointing at the +`harbor://` dataset. None of them is "non-Harbor"; the built-in-vs-playbook split is just how you reference +them, not a different engine path. ## Launch NEL v0.3.0 From 06473d879e3aa89fa5162f8dbb3d729bfc3e8219 Mon Sep 17 00:00:00 2001 From: Hung-Yueh Chiang Date: Sun, 28 Jun 2026 23:17:11 -0700 Subject: [PATCH 12/15] docs(eval): expand harbor:// migration guide, add TOC + harbor example config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - harbor://: full setup walkthrough — readiness chain, creds, SSM verify, dataset directory, SSH sidecar build, eval image (cache-sqsh + by-hand), config, preflight, failure catalog (all cross-refs verified vs the engine) - add a Table of Contents - add recipes/examples/r030_harbor.yaml (terminal-bench-hard via ECS-Fargate) Signed-off-by: Hung-Yueh Chiang Co-Authored-By: Claude Opus 4.8 (1M context) --- .../recipes/examples/r030_harbor.yaml | 96 +++++ .../references/nel-v0.3.0-migration.md | 354 +++++++++++++++++- 2 files changed, 444 insertions(+), 6 deletions(-) create mode 100644 .agents/skills/evaluation/recipes/examples/r030_harbor.yaml diff --git a/.agents/skills/evaluation/recipes/examples/r030_harbor.yaml b/.agents/skills/evaluation/recipes/examples/r030_harbor.yaml new file mode 100644 index 00000000000..cd334651020 --- /dev/null +++ b/.agents/skills/evaluation/recipes/examples/r030_harbor.yaml @@ -0,0 +1,96 @@ +# NEL 0.3.x (engine) — AGENTIC harbor benchmark (terminal-bench) via the harbor solver + a remote AWS +# ECS-Fargate sandbox. READ the "harbor://" section of references/nel-v0.3.0-migration.md FIRST: the +# readiness chain, AWS creds, building the SSH sidecar, adding openssh-client to the eval image, and the +# dataset/CodeBuild gotcha. Preflight (creds -> SSM -> ECR sidecar -> eval-image ssh) before firing: +# set -a; source ; set +a +# nel eval run recipes/examples/r030_harbor.yaml --dry-run +# nel eval run recipes/examples/r030_harbor.yaml + +services: + model: + type: vllm + model: /path/to/checkpoint + served_model_name: my-model + protocol: chat_completions + port: 8000 + node_pool: gpu + tensor_parallel_size: 4 + pipeline_parallel_size: 1 + data_parallel_size: 1 + num_nodes: 1 + image: + startup_timeout: 3600.0 + extra_args: + - "--gpu-memory-utilization=0.9" + - "--max-model-len=262144" + - "--trust-remote-code" + - "--enable-auto-tool-choice" + - "--tool-call-parser=" + - "--reasoning-parser=" + - "--enable-expert-parallel" + - "--max-num-seqs=512" + extra_env: + HF_TOKEN: ${HF_TOKEN} + HF_HOME: /cache/huggingface + container_mounts: ["/path/to/hf-cache:/cache/huggingface"] + proxy: {verbose: true, request_timeout: 1800.0} + generation: + temperature: 1.0 + top_p: 1.0 + max_tokens: 196608 # per model-call; episode bounded by per-task `timeout` + agent turns + +benchmarks: + - name: terminal-bench-hard # curated task set; scored by each task's unit tests (no judge) + repeats: 1 + max_concurrent: 5 + timeout: 1800.0 # per-task wall (bounds the agentic episode) + # num_examples: 3 # smoke on the first N (already-cached) task images + solver: {type: harbor, service: model, agent: terminus-2} + sandbox: + type: ecs_fargate + region: # your harbor/ECS deployment region + cpu: "4096" + memory: "16384" + ephemeral_storage_gib: 50 + concurrency: 10 # parallel Fargate tasks — the throughput knob here (NOT cluster.shards) + stateful: true + ssh_sidecar: + image: .dkr.ecr..amazonaws.com/:terminalbench-ssh-sidecar__ + ssh_ready_timeout_sec: 900.0 + sshd_port: 52222 + private_key_secret_arn: # both ARNs come from //ecs-sandbox/config + public_key_secret_arn: + +cluster: + type: slurm + account: + walltime: "04:00:00" + auto_resume: true # verified-task log persists -> finished tasks skip on resume + eval_image: # MUST include the harbor solver AND openssh-client + mount_home: false + container_mounts: + - /path/to/hf-cache:/cache/huggingface + - /path/to/harbor_datasets:/cache/harbor_datasets # pre-mapped, edited dataset dir + container_env: + HF_HOME: /cache/huggingface + HF_TOKEN: ${HF_TOKEN} + HARBOR_DATASETS_DIR: /cache/harbor_datasets # pin so harbor reuses it, not re-clone + AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} + AWS_DEFAULT_REGION: + node_pools: + gpu: {partition: , nodes: 1, ntasks_per_node: 1, gres: "gpu:4"} + sbatch_extra_flags: {exclusive: true} + +output: + dir: /abs/path/to/rundir + export: [mlflow] # run `nel export` manually from the login node after the job + export_config: + mlflow: + tracking_uri: + experiment_name: + tags: + model: my-model # set this + checkpoint_path so the run attributes correctly + checkpoint_path: + pipeline: r0.3.0 + variant: agentic-terminal-bench diff --git a/.agents/skills/evaluation/references/nel-v0.3.0-migration.md b/.agents/skills/evaluation/references/nel-v0.3.0-migration.md index 1bf69f2b303..53f505ff823 100644 --- a/.agents/skills/evaluation/references/nel-v0.3.0-migration.md +++ b/.agents/skills/evaluation/references/nel-v0.3.0-migration.md @@ -1,5 +1,41 @@ # NEL v0.3.0 Migration +## Contents + +- [NEL 0.2.x (launcher) vs 0.3.x (engine)](#nel-02x-launcher-vs-03x-engine) + - [Comparing the Config File Structure](#comparing-the-config-file-structure) + - [Backward supporting v0.2.x](#backward-supporting-v02x) +- [NEL v0.3.0 Backend Support Matrix](#nel-v030-backend-support-matrix) + - [Supported Backends](#supported-backends) + - [Supported Benchmarks](#supported-benchmarks) + - [Harbor playbooks — a separate agentic backend](#harbor-playbooks--a-separate-agentic-backend) +- [Launch NEL v0.3.0](#launch-nel-v030) + - [Basic Usage](#basic-usage) + - [Single-node vs sharded](#single-node-vs-sharded) + - [The `solver:` block — `simple` vs `tool_calling`](#the-solver-block--simple-vs-tool_calling) + - [built-in and skills:// (native)](#built-in-and-skills-native) + - [gym:// (server + reward)](#gym-server--reward) + - [1. The pieces (two jobs)](#1-the-pieces-two-jobs) + - [2. Start the gym grader + get its port](#2-start-the-gym-grader--get-its-port) + - [3. Get the data](#3-get-the-data) + - [4. The conductor config (what `nel eval run` reads)](#4-the-conductor-config-what-nel-eval-run-reads) + - [5. Serve the model on a GPU node](#5-serve-the-model-on-a-gpu-node) + - [6. Run it — the two-job pattern](#6-run-it--the-two-job-pattern) + - [7. Resume the jobs](#7-resume-the-jobs) + - [8. Caveats](#8-caveats) + - [harbor:// (agentic, remote sandbox)](#harbor-agentic-remote-sandbox) + - [1. The readiness chain](#1-the-readiness-chain) + - [2. The creds you need (link 1)](#2-the-creds-you-need-link-1) + - [3. Verify SSM (link 2)](#3-verify-ssm-link-2) + - [4. Dataset directory (related to link 3)](#4-dataset-directory-related-to-link-3) + - [5. Build the SSH sidecar (links 4–5)](#5-build-the-ssh-sidecar-links-45) + - [6. The eval image (link 6)](#6-the-eval-image-link-6) + - [7. The config](#7-the-config) + - [8. Preflight + run](#8-preflight--run) + - [9. Failure catalog (symptom → link → fix)](#9-failure-catalog-symptom--link--fix) + - [legacy container://](#legacy-container) +- [Export results to MLFlow](#export-results-to-mlflow) + ## NEL 0.2.x (launcher) vs 0.3.x (engine) | | **0.2.x launcher** (`nemo_evaluator_launcher`) | **0.3.x engine** (`nemo_evaluator`) | @@ -82,7 +118,7 @@ harbor:// `- playbook: ` or src/nemo_evaluator/playbooks/ (term harbor:// swebench_verified, …); HarborSolver + sandbox (harbor pkg) ``` -Example — a `benchmarks:` block, one entry per benchmark. The `name:` prefix selects the backend (it is **not** uniformly `{backend}://{benchmark}`): built-in has **no prefix** (just `{benchmark}`); `skills://` and `lm-eval://` are `{backend}://{benchmark}`; `gym://` points at a running server, `gym://{host:port}?...&data=` (the benchmark lives in the data file, not after the prefix): +Example — a `benchmarks:` block, one entry per benchmark. The `name:` prefix selects the backend (it is **not** uniformly `{backend}://{benchmark}`): built-in has **no prefix** (just `{benchmark}`); `skills://` and `lm-eval://` are `{backend}://{benchmark}`; `gym://` points at a running server, `gym://{host:port}?...&data=` (the benchmark lives in the data file, not after the prefix); `harbor://` agentic tasks are usually entered as a `- playbook: ` (the preset bundles the `harbor://` name + `solver: {type: harbor}` + a `sandbox:`): ```yaml benchmarks: # the benchmarks block @@ -90,7 +126,7 @@ benchmarks: # the benchmarks block - name: skills://aime25 # NeMo-Skills - name: lm-eval://hellaswag # lm-eval - name: gym://127.0.0.1:8000?protocol=native&data=tasks.jsonl # NeMo-Gym (running server) - - name: harbor://terminal-bench@2.0 # Harbor (agentic; sandbox + harbor pkg) — or `- playbook: terminal_bench_2` + - playbook: terminal_bench_2 # Harbor (agentic): bundles `name: harbor://terminal-bench@2.0` + solver:harbor + sandbox ``` ### Supported Benchmarks @@ -102,7 +138,8 @@ framework** the engine integrates: - `NVIDIA-NeMo/Gym`: **`gym://`** from Gym [`benchmarks/`](https://github.com/NVIDIA-NeMo/Gym/tree/main/benchmarks)+[`resources_servers/`](https://github.com/NVIDIA-NeMo/Gym/tree/main/resources_servers). - **Harbor** ([`laude-institute/harbor`](https://github.com/laude-institute/harbor), *not* a NeMo repo): **`harbor://`** agentic tasks from Harbor's own registry (60+: terminal-bench, swebench, usaco, bird-bench, …), integrated via `pip install nemo-evaluator[harbor]` and run in a sandbox (see "Harbor playbooks" below). -The matrix below covers only these three backends. **`lm-eval://`** (e.g. `lm-eval://hellaswag`) and +The matrix below covers only its three columns — the **built-in**, **`skills://`**, and **`gym://`** +backends. **`lm-eval://`** (e.g. `lm-eval://hellaswag`) and **`harbor://`** (60+ agentic tasks — discover via Harbor's registry; see "Harbor playbooks" below) are also supported but not enumerated here — discover lm-eval tasks with `lm_eval --tasks list`. Only the **built-in** column is verified against this engine (`nel list -s builtin`); the **`skills://`** and **`gym://`** @@ -211,7 +248,7 @@ MULTILINGUAL / MULTIMODAL / AUDIO ``` -### Harbor playbooks (terminal-bench 2.x, SWE-bench) — a separate mechanism +### Harbor playbooks — a separate agentic backend These are **not** built-in / `skills://` / `gym://`, so they don't appear in the matrix above. They run through the **Harbor** package via a `- playbook: ` entry (HarborSolver + sandbox), pointing at a @@ -355,6 +392,26 @@ image — `pip install nemo-skills` from [NVIDIA-NeMo/Skills](https://github.com repo README. It's baked into `cluster.eval_image` (and the dataset is prepped there). Likewise `lm-eval://` needs `lm-eval` in the image. +```text + login node: `nel eval run ` (cluster: type: slurm) --submits sbatch--> the job below + | + v + one SLURM job (cluster: type: slurm; scale with shards) << everything in-process: no server, no sandbox + +---------------------------------------------------------------+ + | NEL engine <--(1) prompt / answer--> vLLM serve | + | | (services.model) | + | +--(2) scores each answer in-process: | + | built-in scorer / skills:// bridge (math, | + | multichoice) --> per-sample reward | + +---------------------------------------------------------------+ + | + +--(3)--> (sharded? `nel eval merge`) --> `nel export` --> MLflow + + 1 the engine serves the model itself and calls it per sample (avg-of-N via `repeats`) + 2 it scores each answer in-process — built-in scorer or the skills:// bridge — no server, no sandbox + 3 scale throughput with cluster.shards (N single-node workers, merged after); then `nel export` +``` + ```yaml benchmarks: - name: gpqa # built-in (multichoice) @@ -397,7 +454,29 @@ see the gym section's "Resume the jobs".) `gym://` runs a NeMo-Gym **resource server** that scores each response — the faithful path for the `†` benchmarks `skills://` mis-scores and for agentic envs. We run it as a **two-job split**: the model on a GPU node (its own serve job), and the grader + evaluator on a CPU node. Why split: serving needs the vLLM -container, while `nel` and the Gym server live in their own venvs (see the last caveat in §7). +container, while `nel` and the Gym server live in their own venvs (see the last caveat in §8). + +```text + login node: `sbatch serve.sbatch` `sbatch gym_eval.sbatch` (submit both jobs) + | | + v v + GPU node (serve.sbatch) CPU node (gym_eval.sbatch, cluster: type: local) + +-----------------------------+ +-------------------------------------------------+ + | vLLM serve :8000 | <--(2)---- | NEL conductor (nel eval run, in-process) | + | (services.model, type:api) | question | sends Q to the model, gets answer A | + | | --(2)----> | | (3) POST A -> /verify (loopback) | + | | answer | v | + | serve_host.txt --(1)-------|----------> | gym grader (gym env start) 127.0.0.1: | + | (hostname hand-off) | | scores A -> reward, back to the conductor | + +-----------------------------+ +-------------------------------------------------+ + | + +--(4)--> nel export --> MLflow + + 1 GPU serve job writes its hostname to serve_host.txt; the CPU job reads it + waits for /v1/models + 2 conductor calls the model cross-node (GPU :8000) for each rollout + 3 conductor POSTs each answer to the co-located gym grader's /verify (loopback :) + 4 grader returns a reward; conductor tallies -> nel export -> MLflow +``` #### 1. The pieces (two jobs) @@ -431,7 +510,7 @@ top-level key in .yaml kind role ------------------------------ -------------------- ------------------------------- ---------------------- ifbench resources_servers the GRADER — scores via /verify KEEP (all NEL needs) ifbench_simple_agent responses_api_agents a sample gym-driven agent loop DROP - └ model_server: policy_model responses_api_models the model that agent would run (placeholder name only; + model_server: policy_model responses_api_models the model that agent would run (placeholder name only; (referenced, NOT — gym-driven path, not NEL not defined in the file) defined in the file) ``` @@ -633,6 +712,269 @@ cache is dropped and regenerated.) vLLM — so it errors at model startup. Serving needs the vLLM container, which is why §5 is a separate job. +### harbor:// (agentic, remote sandbox) + +Harbor benchmarks (terminal-bench, swebench, …) run a multi-turn terminal agent +([`terminus-2`](https://www.harborframework.com/docs/agents/terminus-2) — Harbor's model-agnostic +reference agent; an interactive tmux session driven against your served model, not a model itself) inside +per-task containers that execute **remotely on [AWS ECS Fargate](https://aws.amazon.com/fargate/)**, scored by each task's own unit tests +(**no LLM judge**). The SLURM side only serves the model + runs the eval driver; the task containers fan +out on Fargate (`sandbox.concurrency` is the throughput knob, **not** `cluster.shards`). This is the +heaviest backend to stand up. Config snippet + full runnable example in §6 +([`recipes/examples/r030_harbor.yaml`](../recipes/examples/r030_harbor.yaml)). + +```text + login node: `nel eval run ` (cluster: type: slurm) --submits sbatch--> the GPU job below + | + v + SLURM (1 GPU node) AWS ECS Fargate (region + secret ARNs from SSM) + +-------------------------------+ +---------------------------------------------+ + | vLLM serve :8000 | <--(2)---- | task container: terminus-2 --tmux--> shell | + | (services.model) | model | ^ (runs cmds, edits files) | + | | completions| | (1) ssh | + | NEL eval driver | --(1) ssh->| SSH sidecar (sshd :52222) | + | (harbor solver) | | | | + | trigger image build | --(0)----->| Fargate pulls the sidecar + task images | + | | | (3) task's own test.sh --> pass/fail | + | | <--(4)-----| reward | + +-------------------------------+ +---------------------------------------------+ + | + +--(5)--> nel export --> MLflow + + 0 driver triggers CodeBuild (AWS) -> builds each task image -> ECR; Fargate pulls from ECR + (up front; ONE build failure aborts the whole benchmark) + 1 the driver SSHes in via the sidecar to install + run terminus-2 inside the task container, + and to stream files / run test.sh (needs openssh-client in the eval image) + 2 terminus-2 (in the task container) calls your served model on the GPU node each step + 3 when the agent stops, the task's own unit tests (test.sh) score it — no LLM judge + 4 reward returns to the driver over the ssh tunnel + 5 the driver aggregates, then `nel export` -> MLflow +``` + +The GPU node only **serves the model + runs the driver**; the agent's work happens in the remote Fargate +containers (so `sandbox.concurrency`, not `cluster.shards`, is the throughput knob). + +#### 1. The readiness chain + +Each link must be green before the next; most failures pin to one specific link: + +```text +creds → SSM → ECR/CodeBuild → sidecar pull → sshd ready → ssh client → agent (terminus-2) → score + 1 2 3 4 5 6 7 8 +``` +1. **creds** — boto finds the AWS keys in env (*fails:* `NoCredentialsError`). +2. **SSM** ([AWS Systems Manager](https://aws.amazon.com/systems-manager/) Parameter Store) — the engine's + [`resolve_ecs_config_from_ssm`](https://github.com/NVIDIA-NeMo/Evaluator/blob/main/src/nemo_evaluator/sandbox/ecs_fargate.py) + reads the parameter `//ecs-sandbox/config` for cluster/subnets/roles/ECR repo + the **SSH key + secret ARNs** (written by the reference Terraform — see §3). Don't hardcode the account/repo/ARNs — let + SSM resolve them (*fails:* `SSM parameter not found`). +3. **ECR/CodeBuild** — [CodeBuild](https://aws.amazon.com/codebuild/) **builds** each task's Dockerfile and + pushes the image to [ECR](https://aws.amazon.com/ecr/) (the registry; content-hash tags, cached) **up + front**; Fargate later pulls it from ECR (link 4). One CodeBuild failure is **FATAL to the whole + benchmark** (0 rows); `skip_failed` does **not** cover it (§4). +4. **sidecar pull** — Fargate pulls your baked SSH sidecar (*fails:* `CannotPullContainerError …ssh-sidecar…: not found` → ECR GC'd it → rebuild, §5). +5. **sshd ready** — the sidecar's `sshd` must answer within `ssh_ready_timeout_sec` (*fails:* `SSH not ready … after 300s` → the default sidecar installs openssh at task start, too slow → use the **baked** one). +6. **ssh client** — the driver shells out to `ssh` to tunnel in, so the **eval image** must ship `openssh-client` (*fails:* `[Errno 2] No such file or directory: 'ssh'`). +7. **agent** — multi-turn loop vs your served model (*fails:* turn / `run_timeout` exhaustion → reward 0). +8. **score** — runs the task's `test.sh`; reward from the verifier, no judge (*fails:* `verify_timeout`). + +#### 2. The creds you need (link 1) + +**AWS creds** for the harbor ECS/ECR/CodeBuild/SSM account, no judge key is needed here (unlike `gym://` judge benchmarks). + +```text +AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY IAM for harbor ECS/ECR/CodeBuild + SSM +AWS_DEFAULT_REGION your harbor deployment's region (cluster.container_env) +HF_TOKEN model/dataset fetch +(NOT needed) INFERENCE_API_KEY / JUDGE_API_KEY scored by unit tests, no judge +``` +Source without echoing — keys live in your `.env`; never open it with Read/Write/Edit (the harness mirrors +opened files into the transcript → leaks the keys), shell `source` only: +```bash +set -a; source ; set +a +echo "AWS=${AWS_ACCESS_KEY_ID:+set} HF=${HF_TOKEN:+set}" # prints "set", never the value +``` + +#### 3. Verify SSM (link 2) + +Verify the SSM (AWS Systems Manager) parameter resolves — this checks the engine can read it via +`resolve_ecs_config_from_ssm` before you fire a GPU node. + +The `//ecs-sandbox/config` parameter is written **once** by the repo's reference Terraform +([`terraform/modules/ecs-sandbox-region`](https://github.com/NVIDIA-NeMo/Evaluator/tree/main/terraform/modules/ecs-sandbox-region) → `ssm.tf`), +applied by your **cloud/infra team** — not by `nel` or the eval user, who only read it. + +```python +# user-launched preflight check (NOT automated by nel): does the SSM config resolve + have the run's keys? +from nemo_evaluator.sandbox.ecs_fargate import resolve_ecs_config_from_ssm +cfg = resolve_ecs_config_from_ssm("", "harbor") # raises if the parameter is missing +assert all(cfg.get(k) for k in ("cluster", "ecr_repository", "s3_bucket", "codebuild_service_role")), cfg +print("OK:", cfg["cluster"], cfg["ecr_repository"]) +``` +*Fails:* `SSM parameter '/harbor/ecs-sandbox/config' not found` → wrong region/project, the harbor stack +isn't deployed, or your IAM can't read the parameter. + +#### 4. Dataset directory (related to link 3) + +**Recommended for every harbor benchmark**, not just terminal-bench. The `harbor://` factory maps each +benchmark's dataset under `HARBOR_DATASETS_DIR` (default `./harbor_datasets`, relative to CWD — fragile). +Point it at an absolute dir **you** control and mount it, so the engine **reuses** the mapped dataset across +runs instead of re-mapping each time — and so you can **patch a task** if its build breaks (below). Set it in +`cluster.container_env` + `cluster.container_mounts`: +```bash +export HARBOR_DATASETS_DIR= # also: container_env.HARBOR_DATASETS_DIR + container_mounts [

:] +nel eval run .yaml # maps the benchmark's dataset under $HARBOR_DATASETS_DIR (cached; e.g. `.tbv1_mapped`) +``` +**If a task's image build fails** — a *run-time* failure (link 3): CodeBuild only fires once you launch the +run, so you won't see it until §2–§3 and the sidecar/eval-image setup (§5–§6) are green and you've fired it. +A task that builds a dataset at image-build time can break on dependency drift (classic: `load_dataset(...)` +hits a `datasets`/`huggingface_hub` mismatch → CodeBuild fails → the **whole benchmark aborts**, 0 rows; +`skip_failed` doesn't cover the build phase). `nel` won't fix it — patch the task in your mapped tree and +re-run (harbor reuses the edited copy, no re-map). Path is benchmark-specific; terminal-bench shown: +```bash +vi "$HARBOR_DATASETS_DIR/terminal-bench@1.0//environment/Dockerfile" # e.g. + RUN pip install 'huggingface_hub==0.24.6' +nel eval run .yaml # re-run; confirm reuse in the log: +grep -hE "cached tasks|Cloning Terminal-Bench" /logs/*.log +# good: "N cached tasks" | bad: "Cloning Terminal-Bench v1 …" (pin bypassed → your edit ignored) +``` +`num_examples` caps the task list (first N images built) — smoke on already-cached tasks before committing +CodeBuild time to the full set. + +#### 5. Build the SSH sidecar (links 4–5) + +The default sidecar `apk add`s openssh at task start → step-5 timeouts under load. Build one that +**pre-installs** `openssh-server`+`netcat` and push it to your harbor ECR — **no local Docker** (CodeBuild +does the build). User-launched, **not** automated by `nel` (source your `.env` first — §2): +```python +# build + push the SSH sidecar to harbor's ECR via CodeBuild (AWS creds + AWS_DEFAULT_REGION already in env) +import os, pathlib +from nemo_evaluator.sandbox.ecs_fargate import EcsFargateConfig, ImageBuilder, resolve_ecs_config_from_ssm +region = os.environ["AWS_DEFAULT_REGION"] +d = pathlib.Path("tb_ssh_sidecar"); d.mkdir(exist_ok=True) +(d / "Dockerfile").write_text("FROM alpine:3.20\nRUN apk add --no-cache openssh-server netcat-openbsd\n") +ssm = resolve_ecs_config_from_ssm(region, "harbor") # ECR repo / S3 / CodeBuild role (from SSM, §3) +cfg = EcsFargateConfig(region=region, environment_dir=str(d), ssm_project="harbor", + ecr_repository=ssm["ecr_repository"], s3_bucket=ssm["s3_bucket"], + s3_prefix=ssm.get("s3_prefix", "ecs-sandbox"), + codebuild_service_role=ssm["codebuild_service_role"], + codebuild_project=ssm.get("codebuild_project")) +url = ImageBuilder.ensure_image_built(cfg=cfg, environment_name="terminalbench-ssh-sidecar", force_build=True) +print("pushed:", url) # paste this image ref into sandbox.ssh_sidecar.image (§7) +``` +The tag is **content-addressed** (sha of the Dockerfile bytes) — an unchanged Dockerfile reproduces the +exact tag your config references (no config edit after a rebuild). Rebuild whenever link 4 reports +`not found` (ECR GC). + +#### 6. The eval image (link 6) + +The 0.3.x driver runs **inside** `cluster.eval_image` and shells out to `ssh` to reach the sidecar, so the +image must carry the **harbor framework + `openssh-client`**. The engine ships the Dockerfiles +(`docker/Dockerfile.{base,harbor,skills,lm-eval,gym,full}`) and a builder, `nel cache-sqsh`. The **`harbor`** +target bundles the harbor framework **and `openssh-client`** for you: +```bash +# builds docker/Dockerfile.harbor, pushes to the registry, then enroot-imports it to a .sqsh on the cluster +nel cache-sqsh harbor # -> /nel-harbor.sqsh +``` +**`cache-sqsh` needs Docker** on the machine you launch it from (it `docker build`s + `docker push`es, then +`enroot import`s on the login node). **No Docker (enroot-only login node)?** If a prebuilt image is published +to a registry your node can reach, `enroot import docker:///nemo-evaluator:-harbor` directly; +otherwise **build by hand** — pure enroot, on a login node with apt+pip egress: +```bash +# 0. base layer: import stock Ubuntu 24.04 to squashfs (one time) +enroot import -o ubuntu-noble.sqsh docker://ubuntu:24.04 +# 1. open a writable container from the base +enroot create --name nel-build ubuntu-noble.sqsh +# 2. install nemo-evaluator[harbor] + openssh-client INSIDE it. ENROOT_MOUNT_HOME=n keeps the host $HOME +# out, else a stale ~/.local/bin/nel shadows the fresh install. +ENROOT_MOUNT_HOME=n enroot start --root --rw nel-build bash <<'INNER' +set -euo pipefail +export DEBIAN_FRONTEND=noninteractive PIP_BREAK_SYSTEM_PACKAGES=1 +# enroot bind-mounts the host /etc/localtime (busy) -> tzdata postinst mv fails + cascades to python3. +# detach it + pre-seed UTC so tzdata configures cleanly: +umount -l /etc/localtime 2>/dev/null || true +rm -f /etc/localtime; ln -sf /usr/share/zoneinfo/Etc/UTC /etc/localtime; echo Etc/UTC > /etc/timezone +apt-get update -o Acquire::Retries=5 +apt-get install -y --no-install-recommends python3 python3-pip git curl wget openssh-client +# --ignore-installed: the ubuntu base ships system copies of some deps (click, fsspec, httpx, +# huggingface-hub, ...) that make a plain pip install SKIP them -> ModuleNotFoundError at runtime. +pip install --no-cache-dir --ignore-installed "nemo-evaluator[harbor]" +nel eval run --help >/dev/null && command -v ssh && python3 -c "import harbor; print('harbor ok')" # verify +INNER +# 3. freeze the container back to a .sqsh -> point cluster.eval_image at this +enroot export -o nel-harbor.sqsh nel-build && enroot remove -f nel-build +``` +Three non-obvious gotchas, all baked into the snippet: **(1)** `ENROOT_MOUNT_HOME=n` (a stale host +`~/.local/bin/nel` otherwise shadows the install); **(2)** detach `/etc/localtime` (enroot bind-mounts the +busy host copy → `tzdata` postinst fails → cascades to `python3`); **(3)** `pip --ignore-installed` (the base +ships system copies that a plain install skips → `ModuleNotFoundError`). Installing from a **source checkout** +(`pip install ".[harbor]"`) instead of PyPI carries local engine patches into the image. + +Only if your eval image is a **non-harbor** variant (e.g. `base`/`skills`) that lacks `ssh`: add `openssh-client` +to the existing sqsh without a rebuild (Docker-free; keeps everything already baked in): +```bash +enroot create --name ev .sqsh +enroot start --root --rw ev bash -c 'apt-get update && apt-get install -y --no-install-recommends openssh-client' +enroot export -o .new ev && mv .new .sqsh +``` + +#### 7. The config + +`solver: {type: harbor}` + a `sandbox: {type: ecs_fargate}`; the `- playbook:` presets bundle the same +thing. The harbor-specific part is the benchmark entry: + +```yaml +benchmarks: + - name: terminal-bench-hard # scored by each task's unit tests (no judge) + repeats: 1 + timeout: 1800.0 # per-task wall (bounds the agentic episode) + # num_examples: 3 # smoke on the first N (already-cached) task images + solver: {type: harbor, service: model, agent: terminus-2} + sandbox: + type: ecs_fargate + region: + concurrency: 10 # parallel Fargate tasks — the throughput knob (NOT cluster.shards) + ssh_sidecar: + image: .dkr.ecr..amazonaws.com/:terminalbench-ssh-sidecar__ + # private/public key secret ARNs resolve from SSM (link 2) +``` +plus the usual `services.model` (your vLLM serve), a `cluster` whose `eval_image` includes the harbor +solver **and** `openssh-client` (§6) with the AWS creds + `HARBOR_DATASETS_DIR` in `container_env` (§4), +and `output`. Full runnable config: +[`recipes/examples/r030_harbor.yaml`](../recipes/examples/r030_harbor.yaml). + +#### 8. Preflight + run + +Preflight links 1→4 + the eval-image ssh **before** you burn a GPU node — confirm the two AWS keys are set; +SSM resolves (`resolve_ecs_config_from_ssm` returns `ecr_repository`/`s3_bucket`/`codebuild_service_role`); +the sidecar tag exists in ECR (`ImageBuilder.image_exists_in_ecr(repo, tag, region)`); and `ssh` is on PATH +inside the eval image (`enroot start sh -c 'command -v ssh'`). Then: +```bash +set -a; source ; set +a +nel eval run recipes/examples/r030_harbor.yaml --dry-run +nel eval run recipes/examples/r030_harbor.yaml +# monitor (secret-filtered) — watch the chain advance: +grep -hiE "CannotPull|SSH ready|No such file.*ssh|RUNNING|reward|exhausted|ERROR" /logs/*.log \ + | grep -ivE "SECRET|ACCESS_KEY" +``` + +#### 9. Failure catalog (symptom → link → fix) + +```text +symptom in eval log link fix +---------------------------------------------------- ---- ----------------------------------------- +no "Found credentials in environment variables" 1 source your .env +"SSM parameter … not found" 2 check region / project / IAM for harbor +"CodeBuild failed for " → 0 rows 3 patch task Dockerfile + pin HARBOR_DATASETS_DIR (§4) +"Cloning Terminal-Bench…" when reuse expected 3 pin HARBOR_DATASETS_DIR to the mapped dir (§4) +"CannotPullContainerError …ssh-sidecar…: not found" 4 rebuild the sidecar (§5) — ECR GC evicted it +"SSH not ready … after 300s" 5 use the baked sidecar (§5) +"[Errno 2] No such file or directory: 'ssh'" 6 add openssh-client to the eval image (§6) +"max_turns_exhausted" / agent timeout 7 raise turn / run_timeout budget +verifier timeout 8 raise verify_timeout +``` + + + ### legacy container:// Backward-compat path for v0.2.x's eval-factory `*_aa_v3` container tasks that have no native 0.3.x From 30022d1e12cae11b61746d7946a14a761505f05d Mon Sep 17 00:00:00 2001 From: Hung-Yueh Chiang Date: Mon, 29 Jun 2026 02:01:21 -0700 Subject: [PATCH 13/15] docs(eval): clarify terminal-bench invocation styles + add Terminal-Bench 2.0 example MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - harbor note: registered names (terminal-bench-v1/-hard) still need an explicit solver:harbor + sandbox; playbooks (terminal_bench_2) bundle them — same backend, different invocation. Point the note at r030_harbor.yaml for both forms. - r030_harbor.yaml: add a commented Terminal-Bench 2.0 (- playbook: terminal_bench_2) alternative beside the registered-name terminal-bench-hard entry. Signed-off-by: Hung-Yueh Chiang Co-Authored-By: Claude Opus 4.8 (1M context) --- .../recipes/examples/r030_harbor.yaml | 17 +++++++++++++++++ .../references/nel-v0.3.0-migration.md | 13 +++++++++---- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/.agents/skills/evaluation/recipes/examples/r030_harbor.yaml b/.agents/skills/evaluation/recipes/examples/r030_harbor.yaml index cd334651020..8a53f2d71cd 100644 --- a/.agents/skills/evaluation/recipes/examples/r030_harbor.yaml +++ b/.agents/skills/evaluation/recipes/examples/r030_harbor.yaml @@ -61,6 +61,23 @@ benchmarks: private_key_secret_arn: # both ARNs come from //ecs-sandbox/config public_key_secret_arn: + # --- ALTERNATIVE: Terminal-Bench 2.0 (89 tasks) via its playbook -------------------------------- + # Same Harbor backend, different invocation. The `terminal_bench_2` playbook bundles + # name: harbor://terminal-bench@2.0 + solver:{type: harbor, agent: terminus-2} + # + an ecs_fargate sandbox (cpu/memory/concurrency/stateful) — so you only overlay the + # deployment-specific bits below; they deep-merge onto the playbook (your values win). + # To use it instead, comment out the `- name: terminal-bench-hard` entry above and uncomment this: + # - playbook: terminal_bench_2 + # sandbox: + # region: # your harbor/ECS deployment region + # # ecr_repository: .dkr.ecr..amazonaws.com/ # optional; else resolved from SSM + # ssh_sidecar: + # image: .dkr.ecr..amazonaws.com/:terminalbench-ssh-sidecar__ + # ssh_ready_timeout_sec: 900.0 + # sshd_port: 52222 + # private_key_secret_arn: + # public_key_secret_arn: + cluster: type: slurm account: diff --git a/.agents/skills/evaluation/references/nel-v0.3.0-migration.md b/.agents/skills/evaluation/references/nel-v0.3.0-migration.md index 53f505ff823..ccfdef263dd 100644 --- a/.agents/skills/evaluation/references/nel-v0.3.0-migration.md +++ b/.agents/skills/evaluation/references/nel-v0.3.0-migration.md @@ -270,10 +270,15 @@ All require the harbor package (`pip install nemo-evaluator[harbor]`) + a sandbo **Note:** *every* terminal-bench variant executes through Harbor (`HarborEnvironment` + sandbox) — only the invocation differs. **v1 / -hard / -hard-aa-split** are `@register` built-in *names* (subclasses of -`HarborEnvironment` that auto-download + map to Harbor format), so you run `- name: terminal-bench-v1`. -**2.0 / 2.1** aren't registered as names, so you run them as `- playbook: terminal_bench_2` pointing at the -`harbor://` dataset. None of them is "non-Harbor"; the built-in-vs-playbook split is just how you reference -them, not a different engine path. +`HarborEnvironment` that auto-download + map to Harbor format), so you reference them as +`- name: terminal-bench-v1` — but you still add `solver: {type: harbor}` + a `sandbox:` yourself, since no +playbook bundles them. **2.0 / 2.1** aren't registered as names, so you run them as `- playbook: terminal_bench_2`, +which bundles the `harbor://` dataset + solver + sandbox for you. None of them is "non-Harbor"; the +built-in-vs-playbook split is just how you reference them, not a different engine path. + +Both forms are in [`r030_harbor.yaml`](../recipes/examples/r030_harbor.yaml) side by side: the active +`- name: terminal-bench-hard` (registered name + explicit `solver`/`sandbox`) and a commented +`- playbook: terminal_bench_2` alternative (bundled, with just a small deployment-specific `sandbox` override). ## Launch NEL v0.3.0 From 6775ce601972864afcebe945693620ae55cc463b Mon Sep 17 00:00:00 2001 From: Hung-Yueh Chiang Date: Mon, 29 Jun 2026 02:11:52 -0700 Subject: [PATCH 14/15] docs(eval): satisfy markdownlint (underscore emphasis, fence/blank spacing) markdownlint-cli2 --fix normalizations on the harbor migration guide: *italic* -> _italic_ (MD049), blank lines around code fences (MD031), collapse consecutive blanks (MD012). Content unchanged. Signed-off-by: Hung-Yueh Chiang Co-Authored-By: Claude Opus 4.8 (1M context) --- .../references/nel-v0.3.0-migration.md | 73 ++++++++++--------- 1 file changed, 38 insertions(+), 35 deletions(-) diff --git a/.agents/skills/evaluation/references/nel-v0.3.0-migration.md b/.agents/skills/evaluation/references/nel-v0.3.0-migration.md index ccfdef263dd..7e0bc955724 100644 --- a/.agents/skills/evaluation/references/nel-v0.3.0-migration.md +++ b/.agents/skills/evaluation/references/nel-v0.3.0-migration.md @@ -85,7 +85,6 @@ Key moves: - `export:` → **`output.export` / `output.export_config`** - `defaults:` (Hydra presets) → **gone** — one explicit file, no preset resolution step - ### Backward supporting v0.2.x 0.3.x keeps a backward-compat path for v0.2.x's eval-factory `container/` task variants (e.g. @@ -94,8 +93,7 @@ name**. Two options: 1. run them via the **legacy `container://` backend** (`solver: {type: container}`, which injects a v1-format `run_config.yaml` so v1 configs port with minimal changes — but **`cluster.shards` is unsupported** for legacy container runs) -2. **recommended** — switch to the native built-in (`mmlu_pro`, `gpqa`) or `skills://` equivalents (sharding + one eval image). Note the launcher's AIME entry was already `ns_aime2025` (nemo-skills, *not* a container variant), so it ports directly to `skills://aime25`. - +2. **recommended** — switch to the native built-in (`mmlu_pro`, `gpqa`) or `skills://` equivalents (sharding + one eval image). Note the launcher's AIME entry was already `ns_aime2025` (nemo-skills, _not_ a container variant), so it ports directly to `skills://aime25`. ## NEL v0.3.0 Backend Support Matrix @@ -136,7 +134,7 @@ framework** the engine integrates: - `NVIDIA-NeMo/Evaluator`: built-in engine [`@register`](https://github.com/NVIDIA-NeMo/Evaluator/tree/main/src/nemo_evaluator/benchmarks) - `NVIDIA-NeMo/Skills`: **`skills://`** from [`nemo_skills/dataset/`](https://github.com/NVIDIA-NeMo/Skills/tree/main/nemo_skills/dataset), - `NVIDIA-NeMo/Gym`: **`gym://`** from Gym [`benchmarks/`](https://github.com/NVIDIA-NeMo/Gym/tree/main/benchmarks)+[`resources_servers/`](https://github.com/NVIDIA-NeMo/Gym/tree/main/resources_servers). -- **Harbor** ([`laude-institute/harbor`](https://github.com/laude-institute/harbor), *not* a NeMo repo): **`harbor://`** agentic tasks from Harbor's own registry (60+: terminal-bench, swebench, usaco, bird-bench, …), integrated via `pip install nemo-evaluator[harbor]` and run in a sandbox (see "Harbor playbooks" below). +- **Harbor** ([`laude-institute/harbor`](https://github.com/laude-institute/harbor), _not_ a NeMo repo): **`harbor://`** agentic tasks from Harbor's own registry (60+: terminal-bench, swebench, usaco, bird-bench, …), integrated via `pip install nemo-evaluator[harbor]` and run in a sandbox (see "Harbor playbooks" below). The matrix below covers only its three columns — the **built-in**, **`skills://`**, and **`gym://`** backends. **`lm-eval://`** (e.g. `lm-eval://hellaswag`) and @@ -146,7 +144,6 @@ column is verified against this engine (`nel list -s builtin`); the **`skills:// columns are compiled from the upstream repos and have **not** all been run — treat them as a routing guide, not a guarantee (see the per-row notes). - > **`†`** = `skills://` **may mis-score** this **judge / batch / code** benchmark: its bridge ([`skills.py`](https://github.com/NVIDIA-NeMo/Evaluator/blob/main/src/nemo_evaluator/environments/skills.py)) only scores math/multichoice; other types fall back to exact-match → ~0. Use `gym://` (or the nemo-skills container / 0.2.x launcher) instead. ```text @@ -247,7 +244,6 @@ MULTILINGUAL / MULTIMODAL / AUDIO vlm_eval_kit — — — vlmevalkit:// backend (multimodal); not gym ``` - ### Harbor playbooks — a separate agentic backend These are **not** built-in / `skills://` / `gym://`, so they don't appear in the matrix above. They run @@ -268,8 +264,8 @@ swebench_multilingual harbor://swebench_multilingual@1.0 SWE-bench All require the harbor package (`pip install nemo-evaluator[harbor]`) + a sandbox. -**Note:** *every* terminal-bench variant executes through Harbor (`HarborEnvironment` + sandbox) — only the -invocation differs. **v1 / -hard / -hard-aa-split** are `@register` built-in *names* (subclasses of +**Note:** _every_ terminal-bench variant executes through Harbor (`HarborEnvironment` + sandbox) — only the +invocation differs. **v1 / -hard / -hard-aa-split** are `@register` built-in _names_ (subclasses of `HarborEnvironment` that auto-download + map to Harbor format), so you reference them as `- name: terminal-bench-v1` — but you still add `solver: {type: harbor}` + a `sandbox:` yourself, since no playbook bundles them. **2.0 / 2.1** aren't registered as names, so you run them as `- playbook: terminal_bench_2`, @@ -280,7 +276,6 @@ Both forms are in [`r030_harbor.yaml`](../recipes/examples/r030_harbor.yaml) sid `- name: terminal-bench-hard` (registered name + explicit `solver`/`sandbox`) and a commented `- playbook: terminal_bench_2` alternative (bundled, with just a small deployment-specific `sandbox` override). - ## Launch NEL v0.3.0 The launch pattern follows the **backend**, not the category. Map each matrix category to the matching @@ -356,7 +351,7 @@ when small / quick runs the long pole (e.g. aime25 Key points: -- **`repeats` is *not* divided — the *problem set* is.** Each shard runs a disjoint slice of all +- **`repeats` is _not_ divided — the _problem set_ is.** Each shard runs a disjoint slice of all `problems × repeats` tasks, so the merged result is mathematically identical to a single-node run, just ~N× faster wall-clock. - **This is the throughput mechanism, not data-parallel serving.** Sharding launches N separate @@ -365,7 +360,7 @@ Key points: - **The only extra step is `nel eval merge /`** — it stitches the N shard outputs into one bundle per benchmark before you `nel export` (see "Export results to MLFlow" below). - **Works cleanly for native (built-in / `skills://`).** Legacy `container://` is single-node only. A - `gym://` run *can* shard but it isn't a free swap — its one shared resource server must be at a + `gym://` run _can_ shard but it isn't a free swap — its one shared resource server must be at a routable host (not loopback) reachable from every shard (see the `gym://` section); the validated gym path is single-node `cluster: {type: local}`. @@ -380,11 +375,10 @@ against (e.g. `service: model`). served with tool calling enabled (vLLM: `--enable-auto-tool-choice` + `--tool-call-parser`). For tool-augmented tasks (run code, compute, search). -**Backend and solver are independent.** A native built-in / `skills://` benchmark can run *either* solver +**Backend and solver are independent.** A native built-in / `skills://` benchmark can run _either_ solver — e.g. `gpqa` with `simple` (no-tools) or with `tool_calling` + a `local` sandbox (the "with-tools" variant). `tool_calling` is **not** exclusive to `gym://`. - ### built-in and skills:// (native) The common path — every matrix row that is `yes` under **built-in** or **skills** with **no `†`**. The @@ -521,7 +515,7 @@ ifbench_simple_agent responses_api_agents a sample gym-driven agent ``` **Keep / drop — which and when:** -- **NEL `gym://…?protocol=native` (this guide)** — NEL *is* the conductor: it calls the model itself, then +- **NEL `gym://…?protocol=native` (this guide)** — NEL _is_ the conductor: it calls the model itself, then posts each answer to the grader's `/verify`. Start **only the `resources_servers` grader** (`ifbench`); drop the agent. - **gym-driven (Responses API)** — gym runs the `*_simple_agent` itself and needs a real @@ -549,7 +543,6 @@ PORT=$(curl -s http://127.0.0.1:11000/server_instances \ Running more than one server lists them all — match the row by `name`. - #### 3. Get the data The questions live in a **local `tasks.jsonl`** that the conductor reads (`data=` points at it). Two ways: @@ -574,12 +567,11 @@ PYTHONPATH=$PWD .venv/bin/python benchmarks//prepare.py # downloads from (Native path: each row needs a `responses_create_params` prompt field - bundled `example.jsonl` already has it; `prepare.py` output may need it added.) - #### 4. The conductor config (what `nel eval run` reads) This is the config the **conductor** (`nel eval run` — the third piece in §1) reads. It tells the conductor which **model** to call (§5), which **grader** + **data** to score against (§2/§3), where it runs, and where -to write results. It is *not* the grader's config (that's §2's gym server) — it's the run itself. +to write results. It is _not_ the grader's config (that's §2's gym server) — it's the run itself. Its placeholders are filled by the other steps — the grader port (§2), the data path (§3), and the model host (§5): model = external endpoint (`type: api`), benchmark = the local gym grader, evaluator in-process @@ -611,7 +603,6 @@ output: Full runnable config: [`recipes/examples/r030_gym.yaml`](../recipes/examples/r030_gym.yaml). - #### 5. Serve the model on a GPU node `type: api` means you serve the model yourself — a plain `vllm serve` in its own GPU `sbatch` job (the @@ -638,7 +629,6 @@ srun --container-image \ - **`--host 0.0.0.0`** (not `127.0.0.1`) so the CPU eval node can reach it cross-node; the GPU node serves only the model. - **`` is resolved at runtime** — you don't know the node until SLURM dispatches the job, so the serve job writes its `hostname` to a shared file and the eval job reads it (`POLICY_HOST=$(cat /serve_host.txt)`) to fill `url`. - #### 6. Run it — the two-job pattern From the login node you submit **two** jobs — `serve.sbatch` (§5) and `gym_eval.sbatch` (below): @@ -680,7 +670,7 @@ sed -i "s##$POLICY_HOST#; s##$PORT#" config.yaml nel eval run config.yaml ${RESUME:-} ``` -Submit `serve.sbatch` first (a plain `sbatch --dependency=after` only waits for the serve job to *start*, +Submit `serve.sbatch` first (a plain `sbatch --dependency=after` only waits for the serve job to _start_, not for vLLM to finish loading — that's why `gym_eval.sbatch` polls `…/v1/models` above before running). #### 7. Resume the jobs @@ -699,7 +689,7 @@ sbatch --export=ALL,RESUME=--resume gym_eval.sbatch # RESUME -> nel eva Completed rollouts are checkpointed per `(problem, repeat)`, so the resume **skips everything already scored** and re-runs only what was in flight — even though the new serve node changes the model URL. (NEL -keys the skip on its *verified* log, which survives the config-hash change; only the un-scored inference +keys the skip on its _verified_ log, which survives the config-hash change; only the un-scored inference cache is dropped and regenerated.) #### 8. Caveats @@ -708,7 +698,7 @@ cache is dropped and regenerated.) grader sees empty output, and the score collapses. - **Reasoning models: enable thinking via `proxy.extra_body`** — pass `chat_template_kwargs: {enable_thinking: true}` under `services.model.proxy.extra_body` (it's merged into every request) so the chat template emits - the reasoning block. This is a *model* setting (applies to native runs too), not gym-specific. + the reasoning block. This is a _model_ setting (applies to native runs too), not gym-specific. - **`example.jsonl` is a small subset** — use `prepare.py` (§3) for the full benchmark. - **Sharding:** a gym run has one shared grader, so it doesn't shard across nodes cleanly (loopback breaks) — keep it single-node. @@ -716,7 +706,6 @@ cache is dropped and regenerated.) path runs `python -m vllm` from the `nel` venv (it ignores the container `image:`), and that venv has no vLLM — so it errors at model startup. Serving needs the vLLM container, which is why §5 is a separate job. - ### harbor:// (agentic, remote sandbox) Harbor benchmarks (terminal-bench, swebench, …) run a multi-turn terminal agent @@ -767,21 +756,22 @@ Each link must be green before the next; most failures pin to one specific link: creds → SSM → ECR/CodeBuild → sidecar pull → sshd ready → ssh client → agent (terminus-2) → score 1 2 3 4 5 6 7 8 ``` -1. **creds** — boto finds the AWS keys in env (*fails:* `NoCredentialsError`). + +1. **creds** — boto finds the AWS keys in env (_fails:_ `NoCredentialsError`). 2. **SSM** ([AWS Systems Manager](https://aws.amazon.com/systems-manager/) Parameter Store) — the engine's [`resolve_ecs_config_from_ssm`](https://github.com/NVIDIA-NeMo/Evaluator/blob/main/src/nemo_evaluator/sandbox/ecs_fargate.py) reads the parameter `//ecs-sandbox/config` for cluster/subnets/roles/ECR repo + the **SSH key secret ARNs** (written by the reference Terraform — see §3). Don't hardcode the account/repo/ARNs — let - SSM resolve them (*fails:* `SSM parameter not found`). + SSM resolve them (_fails:_ `SSM parameter not found`). 3. **ECR/CodeBuild** — [CodeBuild](https://aws.amazon.com/codebuild/) **builds** each task's Dockerfile and pushes the image to [ECR](https://aws.amazon.com/ecr/) (the registry; content-hash tags, cached) **up front**; Fargate later pulls it from ECR (link 4). One CodeBuild failure is **FATAL to the whole benchmark** (0 rows); `skip_failed` does **not** cover it (§4). -4. **sidecar pull** — Fargate pulls your baked SSH sidecar (*fails:* `CannotPullContainerError …ssh-sidecar…: not found` → ECR GC'd it → rebuild, §5). -5. **sshd ready** — the sidecar's `sshd` must answer within `ssh_ready_timeout_sec` (*fails:* `SSH not ready … after 300s` → the default sidecar installs openssh at task start, too slow → use the **baked** one). -6. **ssh client** — the driver shells out to `ssh` to tunnel in, so the **eval image** must ship `openssh-client` (*fails:* `[Errno 2] No such file or directory: 'ssh'`). -7. **agent** — multi-turn loop vs your served model (*fails:* turn / `run_timeout` exhaustion → reward 0). -8. **score** — runs the task's `test.sh`; reward from the verifier, no judge (*fails:* `verify_timeout`). +4. **sidecar pull** — Fargate pulls your baked SSH sidecar (_fails:_ `CannotPullContainerError …ssh-sidecar…: not found` → ECR GC'd it → rebuild, §5). +5. **sshd ready** — the sidecar's `sshd` must answer within `ssh_ready_timeout_sec` (_fails:_ `SSH not ready … after 300s` → the default sidecar installs openssh at task start, too slow → use the **baked** one). +6. **ssh client** — the driver shells out to `ssh` to tunnel in, so the **eval image** must ship `openssh-client` (_fails:_ `[Errno 2] No such file or directory: 'ssh'`). +7. **agent** — multi-turn loop vs your served model (_fails:_ turn / `run_timeout` exhaustion → reward 0). +8. **score** — runs the task's `test.sh`; reward from the verifier, no judge (_fails:_ `verify_timeout`). #### 2. The creds you need (link 1) @@ -793,8 +783,10 @@ AWS_DEFAULT_REGION your harbor deployment's region HF_TOKEN model/dataset fetch (NOT needed) INFERENCE_API_KEY / JUDGE_API_KEY scored by unit tests, no judge ``` + Source without echoing — keys live in your `.env`; never open it with Read/Write/Edit (the harness mirrors opened files into the transcript → leaks the keys), shell `source` only: + ```bash set -a; source ; set +a echo "AWS=${AWS_ACCESS_KEY_ID:+set} HF=${HF_TOKEN:+set}" # prints "set", never the value @@ -816,7 +808,8 @@ cfg = resolve_ecs_config_from_ssm("", "harbor") # raises if the parame assert all(cfg.get(k) for k in ("cluster", "ecr_repository", "s3_bucket", "codebuild_service_role")), cfg print("OK:", cfg["cluster"], cfg["ecr_repository"]) ``` -*Fails:* `SSM parameter '/harbor/ecs-sandbox/config' not found` → wrong region/project, the harbor stack + +_Fails:_ `SSM parameter '/harbor/ecs-sandbox/config' not found` → wrong region/project, the harbor stack isn't deployed, or your IAM can't read the parameter. #### 4. Dataset directory (related to link 3) @@ -826,22 +819,26 @@ benchmark's dataset under `HARBOR_DATASETS_DIR` (default `./harbor_datasets`, re Point it at an absolute dir **you** control and mount it, so the engine **reuses** the mapped dataset across runs instead of re-mapping each time — and so you can **patch a task** if its build breaks (below). Set it in `cluster.container_env` + `cluster.container_mounts`: + ```bash export HARBOR_DATASETS_DIR= # also: container_env.HARBOR_DATASETS_DIR + container_mounts [:] nel eval run .yaml # maps the benchmark's dataset under $HARBOR_DATASETS_DIR (cached; e.g. `.tbv1_mapped`) ``` -**If a task's image build fails** — a *run-time* failure (link 3): CodeBuild only fires once you launch the + +**If a task's image build fails** — a _run-time_ failure (link 3): CodeBuild only fires once you launch the run, so you won't see it until §2–§3 and the sidecar/eval-image setup (§5–§6) are green and you've fired it. A task that builds a dataset at image-build time can break on dependency drift (classic: `load_dataset(...)` hits a `datasets`/`huggingface_hub` mismatch → CodeBuild fails → the **whole benchmark aborts**, 0 rows; `skip_failed` doesn't cover the build phase). `nel` won't fix it — patch the task in your mapped tree and re-run (harbor reuses the edited copy, no re-map). Path is benchmark-specific; terminal-bench shown: + ```bash vi "$HARBOR_DATASETS_DIR/terminal-bench@1.0//environment/Dockerfile" # e.g. + RUN pip install 'huggingface_hub==0.24.6' nel eval run .yaml # re-run; confirm reuse in the log: grep -hE "cached tasks|Cloning Terminal-Bench" /logs/*.log # good: "N cached tasks" | bad: "Cloning Terminal-Bench v1 …" (pin bypassed → your edit ignored) ``` + `num_examples` caps the task list (first N images built) — smoke on already-cached tasks before committing CodeBuild time to the full set. @@ -850,6 +847,7 @@ CodeBuild time to the full set. The default sidecar `apk add`s openssh at task start → step-5 timeouts under load. Build one that **pre-installs** `openssh-server`+`netcat` and push it to your harbor ECR — **no local Docker** (CodeBuild does the build). User-launched, **not** automated by `nel` (source your `.env` first — §2): + ```python # build + push the SSH sidecar to harbor's ECR via CodeBuild (AWS creds + AWS_DEFAULT_REGION already in env) import os, pathlib @@ -866,6 +864,7 @@ cfg = EcsFargateConfig(region=region, environment_dir=str(d), ssm_project="harbo url = ImageBuilder.ensure_image_built(cfg=cfg, environment_name="terminalbench-ssh-sidecar", force_build=True) print("pushed:", url) # paste this image ref into sandbox.ssh_sidecar.image (§7) ``` + The tag is **content-addressed** (sha of the Dockerfile bytes) — an unchanged Dockerfile reproduces the exact tag your config references (no config edit after a rebuild). Rebuild whenever link 4 reports `not found` (ECR GC). @@ -876,14 +875,17 @@ The 0.3.x driver runs **inside** `cluster.eval_image` and shells out to `ssh` to image must carry the **harbor framework + `openssh-client`**. The engine ships the Dockerfiles (`docker/Dockerfile.{base,harbor,skills,lm-eval,gym,full}`) and a builder, `nel cache-sqsh`. The **`harbor`** target bundles the harbor framework **and `openssh-client`** for you: + ```bash # builds docker/Dockerfile.harbor, pushes to the registry, then enroot-imports it to a .sqsh on the cluster nel cache-sqsh harbor # -> /nel-harbor.sqsh ``` + **`cache-sqsh` needs Docker** on the machine you launch it from (it `docker build`s + `docker push`es, then `enroot import`s on the login node). **No Docker (enroot-only login node)?** If a prebuilt image is published to a registry your node can reach, `enroot import docker:///nemo-evaluator:-harbor` directly; otherwise **build by hand** — pure enroot, on a login node with apt+pip egress: + ```bash # 0. base layer: import stock Ubuntu 24.04 to squashfs (one time) enroot import -o ubuntu-noble.sqsh docker://ubuntu:24.04 @@ -908,6 +910,7 @@ INNER # 3. freeze the container back to a .sqsh -> point cluster.eval_image at this enroot export -o nel-harbor.sqsh nel-build && enroot remove -f nel-build ``` + Three non-obvious gotchas, all baked into the snippet: **(1)** `ENROOT_MOUNT_HOME=n` (a stale host `~/.local/bin/nel` otherwise shadows the install); **(2)** detach `/etc/localtime` (enroot bind-mounts the busy host copy → `tzdata` postinst fails → cascades to `python3`); **(3)** `pip --ignore-installed` (the base @@ -916,6 +919,7 @@ ships system copies that a plain install skips → `ModuleNotFoundError`). Insta Only if your eval image is a **non-harbor** variant (e.g. `base`/`skills`) that lacks `ssh`: add `openssh-client` to the existing sqsh without a rebuild (Docker-free; keeps everything already baked in): + ```bash enroot create --name ev .sqsh enroot start --root --rw ev bash -c 'apt-get update && apt-get install -y --no-install-recommends openssh-client' @@ -942,6 +946,7 @@ benchmarks: image: .dkr.ecr..amazonaws.com/:terminalbench-ssh-sidecar__ # private/public key secret ARNs resolve from SSM (link 2) ``` + plus the usual `services.model` (your vLLM serve), a `cluster` whose `eval_image` includes the harbor solver **and** `openssh-client` (§6) with the AWS creds + `HARBOR_DATASETS_DIR` in `container_env` (§4), and `output`. Full runnable config: @@ -953,6 +958,7 @@ Preflight links 1→4 + the eval-image ssh **before** you burn a GPU node — co SSM resolves (`resolve_ecs_config_from_ssm` returns `ecr_repository`/`s3_bucket`/`codebuild_service_role`); the sidecar tag exists in ECR (`ImageBuilder.image_exists_in_ecr(repo, tag, region)`); and `ssh` is on PATH inside the eval image (`enroot start sh -c 'command -v ssh'`). Then: + ```bash set -a; source ; set +a nel eval run recipes/examples/r030_harbor.yaml --dry-run @@ -978,8 +984,6 @@ no "Found credentials in environment variables" 1 source your .env verifier timeout 8 raise verify_timeout ``` - - ### legacy container:// Backward-compat path for v0.2.x's eval-factory `*_aa_v3` container tasks that have no native 0.3.x @@ -999,7 +1003,6 @@ cluster: Prefer the native built-in / `skills://` equivalent (`mmlu_pro`, `gpqa`) when one exists — you get sharding + the one eval image. Use `container://` only when there is no native port. - ## Export results to MLFlow SLURM runs do **not** auto-export (the `output.export` block only fires for local runs). After all From d0f818dcb41bed18175d4b0660a067192ddd2b0a Mon Sep 17 00:00:00 2001 From: Hung-Yueh Chiang Date: Tue, 30 Jun 2026 14:47:26 -0700 Subject: [PATCH 15/15] docs: require checkpoint paths for NEL exports Signed-off-by: Hung-Yueh Chiang --- .../references/nel-v0.3.0-migration.md | 31 ++++++++++++++----- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/.agents/skills/evaluation/references/nel-v0.3.0-migration.md b/.agents/skills/evaluation/references/nel-v0.3.0-migration.md index 7e0bc955724..3191e59183f 100644 --- a/.agents/skills/evaluation/references/nel-v0.3.0-migration.md +++ b/.agents/skills/evaluation/references/nel-v0.3.0-migration.md @@ -1027,13 +1027,17 @@ from `output.export_config.mlflow.tags` (e.g. `model`, `checkpoint_path`, `num_n `variant`) ride along and feed your MLflow dashboards / downstream viewers. The `/` in a `skills://` name becomes `___` in the on-disk bundle dir (`skills___aime25`). -> **Always set a stable `model` (and ideally `checkpoint_path`) tag.** Dashboards key/group runs by -> these tags to attribute a score to a model or checkpoint. A run exported **without** them can't be -> attributed — it shows up as an orphan/"no checkpoint" row instead of landing on the right model — and -> a `model` value that drifts between runs (or differs from a baseline's) splits what should be one row. -> The engine logs only a generic headline metric (`pass_at_1` / `mean_reward`); the benchmark identity -> rides in the `benchmark` tag and the model identity in `model`, so both tags are what make the export -> usable downstream. +> **Required before every NEL 0.3 MLflow export: set both `model` and `checkpoint_path`.** Use the +> resolved checkpoint path actually served by `services..model`; do not substitute the model name, +> Hugging Face ID, served-model name, or output directory. When NEL 0.2 and NEL 0.3 runs evaluate the same +> checkpoint in the same experiment, copy the exact NEL 0.2 `config.deployment.checkpoint_path` value into +> the NEL 0.3 `checkpoint_path` tag. Every shard, merged bundle, retry, and benchmark for that checkpoint +> must use the same value. Downstream viewers group same-experiment results by checkpoint path; a missing or +> different value creates a second row even when the `model` tag displays the same canonical model. +> +> Treat a missing `checkpoint_path` as an export-preflight failure. The engine logs only generic headline +> metrics such as `pass_at_1` / `mean_reward`; `benchmark`, `model`, and `checkpoint_path` are the identity +> metadata that make the export usable downstream. Example `output` block with the tags set (lives in the run config; `nel export` also accepts the same tracking URI / experiment via `-o`): @@ -1048,8 +1052,19 @@ output: experiment_name: tags: model: # set this — dashboards group/attribute runs by it - checkpoint_path: # the per-checkpoint row label downstream + checkpoint_path: # REQUIRED: exact path served by services..model benchmark: # benchmark identity (the engine logs only a generic metric key) precision: bf16 variant: # e.g. base / with-tools / 96k-thinking ``` + +After export, verify the tag on every created run before refreshing a dashboard. The value must be present +and identical across all bundles for the checkpoint: + +```python +from mlflow import MlflowClient + +client = MlflowClient(tracking_uri="") +run = client.get_run("") +assert run.data.tags["checkpoint_path"] == ".model>" +```