diff --git a/packages/nemo_evaluator_sdk/examples/agentic_eval_with_fabric.ipynb b/packages/nemo_evaluator_sdk/examples/agentic_eval_with_fabric.ipynb index 7414ba387e..ae75c40f85 100644 --- a/packages/nemo_evaluator_sdk/examples/agentic_eval_with_fabric.ipynb +++ b/packages/nemo_evaluator_sdk/examples/agentic_eval_with_fabric.ipynb @@ -599,13 +599,16 @@ "result = evaluator.run_sync(\n", " tasks=suite.tasks,\n", " target=target,\n", - " config=AgentEvalRunConfig(output_dir=OUTPUT_DIR, write_dashboard=True, parallelism=3),\n", + " config=AgentEvalRunConfig(work_dir=OUTPUT_DIR, parallelism=3),\n", ")\n", "\n", + "# Storing the run is its own step; it defaults to the work_dir the config named.\n", + "location = result.persist()\n", + "\n", "print(\"run_id :\", result.run_id)\n", "print(\"tasks :\", result.summary.task_count)\n", "print(\"trials :\", result.summary.trial_count)\n", - "print(\"dashboard :\", result.dashboard_path)" + "print(\"dashboard :\", location.dashboard_path)" ] }, { @@ -685,7 +688,7 @@ "cell_type": "markdown", "id": "cell-34", "metadata": {}, - "source": "To see **what the agent actually did** — its output, the files it changed, its step-by-step trajectory\n— open the HTML dashboard at `result.dashboard_path`, or read the persisted bundle under\n`result.output_dir` (`trials.jsonl`, `scores.jsonl`, `summary.json`). The trial evidence (the final\nworkspace and the ATIF trace) is what the metrics above opened to score each run." + "source": "To see **what the agent actually did** — its output, the files it changed, its step-by-step trajectory\n— open the HTML dashboard at `location.dashboard_path`, or read the persisted bundle under\n`location.output_dir` (`trials.jsonl`, `scores.jsonl`, `summary.json`). The trial evidence (the final\nworkspace and the ATIF trace) is what the metrics above opened to score each run." }, { "cell_type": "markdown", diff --git a/packages/nemo_evaluator_sdk/examples/codex_docker/example.py b/packages/nemo_evaluator_sdk/examples/codex_docker/example.py index 589fbaf7d2..e4d945576c 100644 --- a/packages/nemo_evaluator_sdk/examples/codex_docker/example.py +++ b/packages/nemo_evaluator_sdk/examples/codex_docker/example.py @@ -32,7 +32,7 @@ from nemo_evaluator_sdk import MetricInput, MetricOutput, MetricOutputSpec, MetricResult from nemo_evaluator_sdk.agent_eval.evaluator import AgentEvaluator -from nemo_evaluator_sdk.agent_eval.results import AgentEvalResult +from nemo_evaluator_sdk.agent_eval.results import AgentEvalResult, BundleLocation from nemo_evaluator_sdk.agent_eval.runtimes.codex.runtime import CodexDockerCliAgentRuntime from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask from nemo_evaluator_sdk.agent_eval.trials import AgentTaskRunner @@ -108,8 +108,12 @@ async def evaluate( output_dir: str | Path | None = None, runtime: AgentTaskRunner | None = None, write_dashboard: bool = True, -) -> AgentEvalResult: - """Run one Docker Codex task and score its host-readable workspace evidence.""" +) -> tuple[AgentEvalResult, BundleLocation]: + """Run one Docker Codex task, score its workspace evidence, and store the run. + + Returns the result and where it was written: ``run`` itself no longer persists, so storing is an + explicit step here. + """ resolved_output_dir = Path(output_dir).expanduser() if output_dir is not None else _new_output_dir() target = runtime or _docker_runtime(resolved_output_dir) @@ -127,21 +131,21 @@ async def evaluate( metrics=[WorkspaceArtifactMetric()], ) - return await AgentEvaluator().run( + result = await AgentEvaluator().run( tasks=[task], target=target, config=AgentEvalRunConfig( - output_dir=resolved_output_dir, + work_dir=resolved_output_dir, parallelism=1, - write_dashboard=write_dashboard, benchmark={"name": "codex-docker-evidence-sanity"}, ), ) + return result, result.persist(write_dashboard=write_dashboard) async def main() -> None: logging.basicConfig(level=logging.INFO, format="%(levelname)s:%(name)s:%(message)s") - result = await evaluate() + result, location = await evaluate() trial = result.trials[0] if trial.output is None or trial.evidence is None: @@ -156,7 +160,7 @@ async def main() -> None: print(f"artifact contents: {artifact.read_text(encoding='utf-8').strip()}") print(f"workspace_artifact.output_matches: {scores['workspace_artifact.output_matches']}") print(f"workspace_artifact.artifact_matches: {scores['workspace_artifact.artifact_matches']}") - print(f"run bundle: {result.output_dir}") + print(f"run bundle: {location.output_dir}") if __name__ == "__main__": diff --git a/packages/nemo_evaluator_sdk/examples/fabric_container/run_e2e.py b/packages/nemo_evaluator_sdk/examples/fabric_container/run_e2e.py index 6005d244c2..5ef3d0980f 100644 --- a/packages/nemo_evaluator_sdk/examples/fabric_container/run_e2e.py +++ b/packages/nemo_evaluator_sdk/examples/fabric_container/run_e2e.py @@ -63,7 +63,7 @@ async def main() -> int: ) output_dir = Path(os.environ.get("FABRIC_OUTPUT_DIR", "/tmp/fabric-container-e2e")) - (trial,) = await runtime.run_tasks([task], AgentEvalRunConfig(output_dir=output_dir)) + (trial,) = await runtime.run_tasks([task], AgentEvalRunConfig(work_dir=output_dir)) print("=== TRIAL ===") print("status:", trial.status) diff --git a/packages/nemo_evaluator_sdk/examples/gym/run_gym_eval.py b/packages/nemo_evaluator_sdk/examples/gym/run_gym_eval.py index dbf3e1e268..a8d9cfe376 100644 --- a/packages/nemo_evaluator_sdk/examples/gym/run_gym_eval.py +++ b/packages/nemo_evaluator_sdk/examples/gym/run_gym_eval.py @@ -97,15 +97,18 @@ async def _main(args: argparse.Namespace) -> int: result = await AgentEvaluator().run( tasks=tasks, target=runner, - config=AgentEvalRunConfig(output_dir=output_dir, parallelism=1), + config=AgentEvalRunConfig(work_dir=output_dir, parallelism=1), ) + # Storing the run is its own step. Defaults to the run's work_dir, so the bundle contains the + # evidence the trials point at. + location = result.persist() print("=== RESULT ===") print(f"tasks: {result.summary.task_count} trials: {result.summary.trial_count}") print("aggregate scores:") for aggregate in result.summary.scores.scores: print(f" {aggregate.name}: mean={aggregate.mean}") - print(f"\nRun bundle (run.json, trials.jsonl, scores.jsonl, report.html): {output_dir}") + print(f"\nRun bundle (run.json, trials.jsonl, scores.jsonl, report.html): {location.output_dir}") return 0 diff --git a/packages/nemo_evaluator_sdk/examples/profbench/runner.py b/packages/nemo_evaluator_sdk/examples/profbench/runner.py index d8435f32f7..1ba0e3ab25 100644 --- a/packages/nemo_evaluator_sdk/examples/profbench/runner.py +++ b/packages/nemo_evaluator_sdk/examples/profbench/runner.py @@ -121,13 +121,14 @@ async def run_profbench_mode( trials=trials, target=target, config=AgentEvalRunConfig( - output_dir=output_dir, + work_dir=output_dir, run_id=f"{run_instance_id}-{mode.value}", params=params, benchmark=benchmark_meta, - write_dashboard=False, ), ) + # This example renders its own dashboards below, so persistence skips the built-in one. + result.persist(write_dashboard=False) sdk_dashboard_path, dashboard_path = write_example_dashboards(result, output_dir) overall = _profbench_overall(result) diff --git a/packages/nemo_evaluator_sdk/examples/run_agent_eval/pipeline.py b/packages/nemo_evaluator_sdk/examples/run_agent_eval/pipeline.py index 67111f285b..7fb37b3c4e 100644 --- a/packages/nemo_evaluator_sdk/examples/run_agent_eval/pipeline.py +++ b/packages/nemo_evaluator_sdk/examples/run_agent_eval/pipeline.py @@ -74,7 +74,7 @@ async def run_tasks( target=target, config=self._run_config(output_dir=output_dir, run_id=run_id, benchmark=benchmark), ) - self._maybe_write_gate(result) + self._persist_and_gate(result, output_dir) return result async def score_trials( @@ -93,7 +93,7 @@ async def score_trials( trials=list(trials), config=self._run_config(output_dir=output_dir, run_id=run_id, benchmark=benchmark), ) - self._maybe_write_gate(result) + self._persist_and_gate(result, output_dir) return result def _run_config( @@ -104,10 +104,9 @@ def _run_config( benchmark: dict[str, object] | None, ) -> AgentEvalRunConfig: return AgentEvalRunConfig( - output_dir=output_dir, + work_dir=output_dir, run_id=run_id, parallelism=self.config.parallelism, - write_dashboard=self.config.write_dashboard, benchmark=dict(benchmark or {}), ) @@ -122,8 +121,12 @@ def _with_extra_metrics(self, task: AgentEvalTask) -> AgentEvalTask: return task return task.model_copy(update={"metrics": metrics + appended}) - def _maybe_write_gate(self, result: AgentEvalResult) -> None: - if not (self.config.write_gate and result.output_dir is not None): + def _persist_and_gate(self, result: AgentEvalResult, output_dir: Path | None) -> None: + """Store the run (when a directory was given) and write the gate report beside it.""" + if output_dir is None: + return + location = result.persist(write_dashboard=self.config.write_dashboard) + if not self.config.write_gate: return baseline = ( load_baseline_summary(self.config.baseline_summary_path) @@ -131,7 +134,7 @@ def _maybe_write_gate(self, result: AgentEvalResult) -> None: else None ) report = evaluate_gate(result, thresholds=self.config.gate_thresholds, baseline_summary=baseline) - write_gate_report(report, result.output_dir) + write_gate_report(report, location.output_dir) __all__ = [ diff --git a/packages/nemo_evaluator_sdk/examples/run_agent_eval/platform_runtime.py b/packages/nemo_evaluator_sdk/examples/run_agent_eval/platform_runtime.py index 8c97094d75..93ca51b297 100644 --- a/packages/nemo_evaluator_sdk/examples/run_agent_eval/platform_runtime.py +++ b/packages/nemo_evaluator_sdk/examples/run_agent_eval/platform_runtime.py @@ -105,7 +105,7 @@ def task_image_tag(task_id: str) -> str: def resolve_run_layout(task: AgentEvalTask, config: AgentEvalRunConfig | None) -> AgenticRunLayout: """Resolve/create the on-disk layout for one task run.""" - output_dir = config.output_dir if config is not None else None + output_dir = config.work_dir if config is not None else None run_dir = resolve_run_dir(output_dir, lambda: Path.cwd() / "nat-jobs" / task.id) / task.id base = prepare_run_layout(run_dir, str(task.inputs.get("instruction") or task.intent)) state_dir = base.run_dir / "state" diff --git a/packages/nemo_evaluator_sdk/examples/run_agent_eval/run_agent_eval.py b/packages/nemo_evaluator_sdk/examples/run_agent_eval/run_agent_eval.py index 3d564eae6e..1998d70da4 100644 --- a/packages/nemo_evaluator_sdk/examples/run_agent_eval/run_agent_eval.py +++ b/packages/nemo_evaluator_sdk/examples/run_agent_eval/run_agent_eval.py @@ -159,9 +159,9 @@ def _print_result(result: AgentEvalResult) -> None: if score.mean is not None: print(f" {score.name}: mean={score.mean:.3f}") _print_measurements(result) - if result.output_dir is not None: - print(f"output_dir: {result.output_dir}") - print(f"gate: {result.output_dir / 'gate.json'}") + if result.work_dir is not None: + print(f"work_dir: {result.work_dir}") + print(f"gate: {result.work_dir / 'gate.json'}") def _print_measurements(result: AgentEvalResult) -> None: diff --git a/packages/nemo_evaluator_sdk/examples/run_agent_eval/workflow_runtime.py b/packages/nemo_evaluator_sdk/examples/run_agent_eval/workflow_runtime.py index 0906e4afdc..05b1c0faa1 100644 --- a/packages/nemo_evaluator_sdk/examples/run_agent_eval/workflow_runtime.py +++ b/packages/nemo_evaluator_sdk/examples/run_agent_eval/workflow_runtime.py @@ -179,7 +179,7 @@ def _format_command(self, instruction_path: Path, workspace_dir: Path, input_jso return [substitutions.get(token, token) for token in self.config.command] def _run_dir(self, index: int, task: AgentEvalTask, config: AgentEvalRunConfig) -> Path: - root = (config.output_dir or Path.cwd()) / "evidence" / RUNTIME_NAME + root = (config.work_dir or Path.cwd()) / "evidence" / RUNTIME_NAME return root / (_safe_name(task.id) or f"task-{index}") diff --git a/packages/nemo_evaluator_sdk/examples/skill_eval/run_skill_eval.py b/packages/nemo_evaluator_sdk/examples/skill_eval/run_skill_eval.py index 0693efe21e..5927eada8c 100644 --- a/packages/nemo_evaluator_sdk/examples/skill_eval/run_skill_eval.py +++ b/packages/nemo_evaluator_sdk/examples/skill_eval/run_skill_eval.py @@ -258,15 +258,17 @@ async def _main() -> int: baseline = await AgentEvaluator().run( tasks=tasks, target=baseline_runtime, - config=AgentEvalRunConfig(run_id="baseline", output_dir=output_dir / "baseline", write_dashboard=False), + config=AgentEvalRunConfig(run_id="baseline", work_dir=output_dir / "baseline"), ) + baseline.persist(write_dashboard=False) treated = await AgentEvaluator().run( tasks=tasks, target=baseline_runtime.with_skill( skill ), # We include the skill in the treated arm, so the two runs differ in *exactly* the skill. - config=AgentEvalRunConfig(run_id="treated", output_dir=output_dir / "treated", write_dashboard=False), + config=AgentEvalRunConfig(run_id="treated", work_dir=output_dir / "treated"), ) + treated.persist(write_dashboard=False) except SkillInjectionError as exc: print(f"skill eval failed to load the bundled skill: {exc}") return 1 diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/evaluator.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/evaluator.py index 2880072880..892ff42c18 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/evaluator.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/evaluator.py @@ -19,8 +19,6 @@ import httpx import nemo_evaluator_sdk.inference as inference -from nemo_evaluator_sdk.agent_eval.dashboard import write_dashboard -from nemo_evaluator_sdk.agent_eval.persistence import persist_run from nemo_evaluator_sdk.agent_eval.results import AgentEvalResult, AgentEvalSummary from nemo_evaluator_sdk.agent_eval.scores import ( AgentEvalDiagnostic, @@ -180,10 +178,9 @@ async def run( scores=scores, summary=AgentEvalSummary.from_scores(scores, tasks=task_list), benchmark=benchmark, + work_dir=runtime_config.work_dir, ) - if runtime_config.output_dir is not None: - result = _persist_with_optional_dashboard(result, runtime_config.output_dir, runtime_config.write_dashboard) return result def run_sync( @@ -324,8 +321,8 @@ async def generate_one(index: int, task: AgentEvalTask) -> AgentEvalTrial: "invocation_id": f"{config.run_id}:{task.id}:{target.name}", } evidence_dir = ( - _task_evidence_dir(Path(config.output_dir), index=index, task_id=task.id) - if config.output_dir is not None and isinstance(target, AgentBase) + _task_evidence_dir(Path(config.work_dir), index=index, task_id=task.id) + if config.work_dir is not None and isinstance(target, AgentBase) else None ) resolved_inference_fn = self.inference_fn @@ -693,18 +690,6 @@ def _benchmark_metadata(tasks: list[AgentEvalTask]) -> dict[str, Any]: return {"benchmark": benchmarks[0] if len(benchmarks) == 1 else benchmarks} -def _persist_with_optional_dashboard( - result: AgentEvalResult, - output_dir: Path, - write_html: bool, -) -> AgentEvalResult: - path = Path(output_dir) - dashboard_path = None - if write_html: - dashboard_path = write_dashboard(result.model_copy(update={"output_dir": path}), path / "report.html") - return persist_run(result.model_copy(update={"output_dir": path, "dashboard_path": dashboard_path}), path) - - def _new_run_id() -> str: timestamp = datetime.now(UTC).strftime("%Y%m%d%H%M%S") return f"agent-eval-{timestamp}-{uuid.uuid4().hex[:8]}" diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/persistence.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/persistence.py index b2b820e0e7..2c7383c2e9 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/persistence.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/persistence.py @@ -10,32 +10,52 @@ from pathlib import Path from typing import Any -from nemo_evaluator_sdk.agent_eval.results import AgentEvalResult +from nemo_evaluator_sdk.agent_eval.dashboard import write_dashboard +from nemo_evaluator_sdk.agent_eval.results import AgentEvalResult, BundleLocation from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTrial from pydantic import BaseModel +#: Filename of the rendered HTML dashboard inside a bundle. +DASHBOARD_FILENAME = "report.html" -def persist_run(result: AgentEvalResult, output_dir: str | Path) -> AgentEvalResult: - """Persist a completed run bundle to ``output_dir``.""" + +def persist_run( + result: AgentEvalResult, + output_dir: str | Path, + *, + write_html_dashboard: bool = True, +) -> BundleLocation: + """Write a completed run to a bundle at ``output_dir`` and report where it landed. + + Explicit rather than a side effect of :meth:`AgentEvaluator.run`: computing an evaluation and + storing one are different decisions, and folding them together is what forced the result object to + carry paths it could not know at construction time. (Same reasoning as ``publish_to_intake``.) + + Set ``write_html_dashboard=False`` to skip rendering ``report.html`` — the dashboard is written + here so the manifest can record it in a single pass. + """ path = Path(output_dir) path.mkdir(parents=True, exist_ok=True) + # Render first so the manifest below can name it; the dashboard reads only the run's own contents. + dashboard_path = write_dashboard(result, path / DASHBOARD_FILENAME) if write_html_dashboard else None + _write_json(path / "benchmark.json", result.benchmark) _write_jsonl(path / "tasks.jsonl", result.tasks) _write_trials(path / "trials.jsonl", result.trials, base=path) _write_jsonl(path / "scores.jsonl", result.scores) _write_json(path / "summary.json", result.summary) - updated = result.model_copy(update={"output_dir": path}) - _write_json(path / "run.json", _run_manifest(updated)) - return updated + location = BundleLocation(output_dir=path, dashboard_path=dashboard_path) + _write_json(path / "run.json", _run_manifest(result, location)) + return location -def _run_manifest(result: AgentEvalResult) -> dict[str, Any]: +def _run_manifest(result: AgentEvalResult, location: BundleLocation) -> dict[str, Any]: return { "run_id": result.run_id, - "output_dir": str(result.output_dir) if result.output_dir is not None else None, - "dashboard_path": str(result.dashboard_path) if result.dashboard_path is not None else None, + "output_dir": str(location.output_dir), + "dashboard_path": str(location.dashboard_path) if location.dashboard_path is not None else None, "artifacts": { "benchmark": "benchmark.json", "tasks": "tasks.jsonl", diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py index 7eee3f6335..18cc01e564 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py @@ -68,8 +68,30 @@ def from_scores( ) +class BundleLocation(BaseModel): + """Where a run was written, returned by :meth:`AgentEvalResult.persist`. + + Kept off :class:`AgentEvalResult` because it is not a property of the evaluation — it is the + outcome of choosing to store it. Holding one means the bundle exists, so there is no optional to + re-check; a run that was never persisted simply has no ``BundleLocation``. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + output_dir: Path = Field(description="Directory the run bundle was written to.") + dashboard_path: Path | None = Field( + default=None, + description="Path to the rendered HTML dashboard, or None when dashboard writing was disabled.", + ) + + class AgentEvalResult(BaseModel): - """Root result for a completed agent evaluation: tasks, trials, scores, summary, and bundle metadata.""" + """Root result for a completed agent evaluation: tasks, trials, scores, and summary. + + Describes the evaluation and nothing else — storing it is a separate decision, made by calling + :meth:`persist`. Because the result carries no paths, it never holds a location that was unknown + when it was constructed, and nothing has to mutate it after the fact. + """ model_config = ConfigDict(extra="forbid") @@ -82,8 +104,39 @@ class AgentEvalResult(BaseModel): default_factory=dict, description="Benchmark metadata recorded for the run.", ) - output_dir: Path | None = Field(default=None, description="Directory the run bundle was written to, if any.") - dashboard_path: Path | None = Field(default=None, description="Path to the rendered dashboard, if written.") + work_dir: Path | None = Field( + default=None, + description="Directory the run worked in, where its runtimes wrote trial evidence. Known " + "before the run starts (it comes from the run config), so unlike a bundle location it is " + "never attached after the fact. None for a purely in-memory run.", + ) + + def persist(self, output_dir: str | Path | None = None, *, write_dashboard: bool = True) -> BundleLocation: + """Write this run to a bundle and return where it landed. + + Deliberately a call rather than something ``AgentEvaluator.run`` does for you: computing an + evaluation and storing one are separate decisions (the same reasoning as ``publish_to_intake``). + + Defaults to :attr:`work_dir`, which is the directory the trials' evidence already lives under — + so the bundle is self-contained and survives being moved. Passing a different ``output_dir`` + leaves those evidence references pointing back at the original directory. That is supported (a + re-scored run may reference an earlier run's deliverables) but the resulting bundle only + resolves while the original directory is still there. + + Set ``write_dashboard=False`` to skip rendering ``report.html``. + """ + # Imported here rather than at module scope: persistence imports this module for the types it + # writes, so a top-level import would be circular. + from nemo_evaluator_sdk.agent_eval.persistence import persist_run + + target = output_dir if output_dir is not None else self.work_dir + if target is None: + raise ValueError( + "this run has no work_dir to persist into (it ran in memory); pass an explicit " + "output_dir, or set work_dir on the AgentEvalRunConfig so evidence and bundle share " + "a directory" + ) + return persist_run(self, target, write_html_dashboard=write_dashboard) def _aggregate_scores( diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/codex/runtime.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/codex/runtime.py index 717d10d653..4d85593b02 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/codex/runtime.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/codex/runtime.py @@ -261,7 +261,7 @@ def _validate_artifact_permissions(self, evidence_dir: Path) -> None: def _evidence_dir(self, index: int, task: AgentEvalTask, config: AgentEvalRunConfig) -> Path: root = self._work_root if root is None: - root = (config.output_dir or Path.cwd()) / "evidence" / "codex" + root = (config.work_dir or Path.cwd()) / "evidence" / "codex" safe_task_id = _safe_path_name(task.id) task_dir = f"{index:06d}-{safe_task_id}" if safe_task_id else f"task-{index:06d}" return Path(root) / task_dir diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/docker_sandbox.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/docker_sandbox.py index 22b4252b30..7bc7f13739 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/docker_sandbox.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/docker_sandbox.py @@ -276,7 +276,7 @@ def _failed_trial(self, task: AgentEvalTask, exc: Exception, evidence_dir: Path) ) def _evidence_dir(self, index: int, task: AgentEvalTask, config: AgentEvalRunConfig) -> Path: - root = config.output_dir if config.output_dir is not None else self._work_root + root = config.work_dir if config.work_dir is not None else self._work_root if root is None: root = Path(tempfile.gettempdir()) / "nemo-evaluator-agent-runtime" run_id = config.run_id or _new_runtime_run_id() diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/container_runtime.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/container_runtime.py index 18ba73c905..285d44ca7c 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/container_runtime.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/container_runtime.py @@ -469,7 +469,7 @@ def _failed_trial( def _evidence_dir(self, index: int, task: AgentEvalTask, config: AgentEvalRunConfig) -> Path: # Evidence lands under the run's output dir (like every other runtime); the container's own # working state lives at /out inside the sandbox and is downloaded here. - root = (config.output_dir or Path.cwd()) / "evidence" / "fabric_container" + root = (config.work_dir or Path.cwd()) / "evidence" / "fabric_container" return root / _common.task_subdir_name(index, task.id) diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/runtime.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/runtime.py index 34d56b7c7a..f3fb330faf 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/runtime.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/runtime.py @@ -511,7 +511,7 @@ def _relay_config(self, relay_dir: Path) -> RelayObservabilityConfig: def _evidence_dir(self, index: int, task: AgentEvalTask, config: AgentEvalRunConfig) -> Path: root = self._work_root if root is None: - root = (config.output_dir or Path.cwd()) / "evidence" / "fabric" + root = (config.work_dir or Path.cwd()) / "evidence" / "fabric" # The run id isolates this run's evidence from other runs sharing the same root (A/B baseline # vs. skilled); run_tasks always populates it, so the fallback only guards a direct call. run_id = config.run_id or _new_run_id() diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym_runtime.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym_runtime.py index ab0e2a314f..fec77df130 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym_runtime.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym_runtime.py @@ -347,8 +347,8 @@ async def run_tasks( # phases — parallelism bounds concurrent scoring (SDK-side, cheap), while Gym's `--concurrency` # bounds concurrent rollouts against the model endpoint during collection (tuned to that endpoint's # limits via GymRuntimeConfig.concurrency). - if config is not None and config.output_dir is not None: - work_dir = Path(config.output_dir) / "gym_run" + if config is not None and config.work_dir is not None: + work_dir = Path(config.work_dir) / "gym_run" else: work_dir = Path(tempfile.mkdtemp(prefix="gym_run_")) work_dir.mkdir(parents=True, exist_ok=True) diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/harbor_runtime.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/harbor_runtime.py index 374f2defa6..0c56660e98 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/harbor_runtime.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/harbor_runtime.py @@ -1299,7 +1299,7 @@ async def run_harbor_eval( return await AgentEvaluator().run( tasks=tasks, target=runner, - config=run_config or AgentEvalRunConfig(write_dashboard=False), + config=run_config or AgentEvalRunConfig(), ) diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/tasks.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/tasks.py index 83e5d64520..c33fcb0a78 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/tasks.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/tasks.py @@ -214,9 +214,11 @@ class AgentEvalRunConfig(BaseModel): model_config = ConfigDict(extra="forbid") - output_dir: Path | None = Field( + work_dir: Path | None = Field( default=None, - description="Directory where the run bundle is written; in-memory only when omitted.", + description="Directory the run works in: runtimes write trial evidence beneath it, and it is " + "the default target for AgentEvalResult.persist so the bundle contains that evidence. Purely " + "in-memory when omitted.", ) run_id: str | None = Field(default=None, description="Explicit run identifier; generated when omitted.") prompt_template: str | dict[str, Any] | None = Field( @@ -228,7 +230,6 @@ class AgentEvalRunConfig(BaseModel): description="Inference/run parameters used when producing trials online.", ) parallelism: int = Field(default=4, ge=1, description="Maximum number of tasks scored concurrently.") - write_dashboard: bool = Field(default=True, description="Whether to render an HTML dashboard for the run.") benchmark: dict[str, Any] = Field( default_factory=dict, description="Benchmark metadata recorded alongside the run.", diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_codex_docker_example.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_codex_docker_example.py index 26036d16e6..0b3868f0ed 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_codex_docker_example.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_codex_docker_example.py @@ -63,7 +63,7 @@ def test_default_output_dir_is_under_repo_temp(monkeypatch: pytest.MonkeyPatch) @pytest.mark.asyncio async def test_codex_docker_example_scores_workspace_artifact(tmp_path: Path) -> None: - result = await codex_docker.evaluate( + result, location = await codex_docker.evaluate( output_dir=tmp_path / "run", runtime=_FakeCodexRuntime(tmp_path / "workspace"), write_dashboard=False, @@ -77,6 +77,8 @@ async def test_codex_docker_example_scores_workspace_artifact(tmp_path: Path) -> "workspace_artifact.output_matches": True, } assert (tmp_path / "run" / "run.json").is_file() + assert location.output_dir == tmp_path / "run" + assert location.dashboard_path is None # write_dashboard=False @pytest.mark.asyncio diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_docker_sandbox_runtime.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_docker_sandbox_runtime.py index ad62495708..c33faae163 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_docker_sandbox_runtime.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_docker_sandbox_runtime.py @@ -272,7 +272,7 @@ async def test_completed_run_writes_artifacts_and_evidence(monkeypatch: pytest.M trials = await runtime.run_tasks( [_task()], - config=AgentEvalRunConfig(output_dir=tmp_path, run_id="run-1", parallelism=1), + config=AgentEvalRunConfig(work_dir=tmp_path, run_id="run-1", parallelism=1), ) evidence_dir = tmp_path / "agent-runtime" / "run-1" / "000000-task-1" @@ -304,7 +304,7 @@ async def test_runtime_creates_and_deletes_one_sandbox_per_task( await runtime.run_tasks( [_task(task_id="task-1"), _task(task_id="task-2")], - config=AgentEvalRunConfig(output_dir=tmp_path, run_id="run-1", parallelism=2), + config=AgentEvalRunConfig(work_dir=tmp_path, run_id="run-1", parallelism=2), ) assert len(client.created) == 2 @@ -322,7 +322,7 @@ async def test_direct_runtime_call_uses_one_generated_run_id( await runtime.run_tasks( [_task(task_id="task-1"), _task(task_id="task-2")], - config=AgentEvalRunConfig(output_dir=tmp_path, parallelism=2), + config=AgentEvalRunConfig(work_dir=tmp_path, parallelism=2), ) run_dirs = list((tmp_path / "agent-runtime").iterdir()) @@ -343,7 +343,7 @@ async def test_parallelism_limits_concurrent_task_runs( await runtime.run_tasks( [_task(task_id=f"task-{index}") for index in range(4)], - config=AgentEvalRunConfig(output_dir=tmp_path, run_id="run-1", parallelism=2), + config=AgentEvalRunConfig(work_dir=tmp_path, run_id="run-1", parallelism=2), ) assert runner.max_active == 2 @@ -360,7 +360,7 @@ async def test_runtime_exception_returns_failed_trial( trials = await runtime.run_tasks( [_task()], - config=AgentEvalRunConfig(output_dir=tmp_path, run_id="run-1", parallelism=1), + config=AgentEvalRunConfig(work_dir=tmp_path, run_id="run-1", parallelism=1), ) error_path = tmp_path / "agent-runtime" / "run-1" / "000000-task-1" / "error.json" diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_evaluator.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_evaluator.py index e4caa233b0..8ff1781c4f 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_evaluator.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_evaluator.py @@ -295,16 +295,38 @@ def test_run_rejects_trials_and_target_together() -> None: ) +@pytest.mark.asyncio +async def test_run_writes_nothing_until_persist_is_called(tmp_path: Path) -> None: + # The point of the change: computing an evaluation and storing one are separate decisions, so a + # run given a work_dir still leaves it empty until the caller asks for a bundle. + result = await AgentEvaluator().run( + tasks=[_task()], + trials=[_candidate_trial()], + config=AgentEvalRunConfig(work_dir=tmp_path, parallelism=1), + ) + + assert not (tmp_path / "run.json").exists() + assert not (tmp_path / "report.html").exists() + # work_dir comes from the config, so it is known at construction — never patched on afterwards. + assert result.work_dir == tmp_path + + result.persist() + assert (tmp_path / "run.json").is_file() + + @pytest.mark.asyncio async def test_scores_imported_trials_with_metric_and_persists_bundle(tmp_path: Path) -> None: result = await AgentEvaluator().run( tasks=[_task()], trials=[_candidate_trial()], - config=AgentEvalRunConfig(output_dir=tmp_path, parallelism=1), + config=AgentEvalRunConfig(work_dir=tmp_path, parallelism=1), ) + # run() no longer writes anything; persisting is the caller's call and defaults to the work_dir. + location = result.persist() assert _score(result.summary, "constant_metric.score").mean == 0.75 - assert result.dashboard_path == tmp_path / "report.html" + assert location.output_dir == tmp_path + assert location.dashboard_path == tmp_path / "report.html" assert (tmp_path / "run.json").exists() assert (tmp_path / "scores.jsonl").exists() assert "run_id" not in json.loads((tmp_path / "benchmark.json").read_text(encoding="utf-8")) @@ -653,7 +675,6 @@ async def test_generation_boundary_names_agent_eval_context() -> None: config=AgentEvalRunConfig( run_id="run-123", params=RunConfigOnline(parallelism=1), - write_dashboard=False, ), ) @@ -689,10 +710,9 @@ async def test_default_agent_invocation_receives_run_context_and_evidence_dir(tm target=agent, config=AgentEvalRunConfig( run_id="run-123", - output_dir=tmp_path, + work_dir=tmp_path, prompt_template=prompt_template, params=RunConfigOnline(parallelism=1), - write_dashboard=False, ), ) @@ -733,9 +753,8 @@ async def fake_invoke(agent: Agent, request: dict[str, Any], **kwargs: Any) -> A target=agent, config=AgentEvalRunConfig( run_id="run-123", - output_dir=tmp_path, + work_dir=tmp_path, params=RunConfigOnline(parallelism=1), - write_dashboard=False, ), ) @@ -773,9 +792,8 @@ def factory(context: AgentInferenceContext): target=agent, config=AgentEvalRunConfig( run_id="run-123", - output_dir=tmp_path, + work_dir=tmp_path, params=RunConfigOnline(parallelism=1), - write_dashboard=False, ), ) diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_container_runtime.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_container_runtime.py index e9a6f3694b..1ef7e2b764 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_container_runtime.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_container_runtime.py @@ -135,7 +135,7 @@ def _runtime(provider: _FakeProvider, **kwargs: object) -> FabricContainerRuntim async def _run(runtime: FabricContainerRuntime, tasks: list[AgentEvalTask], tmp_path: Path) -> Sequence[AgentEvalTrial]: - return await runtime.run_tasks(tasks, AgentEvalRunConfig(output_dir=tmp_path)) + return await runtime.run_tasks(tasks, AgentEvalRunConfig(work_dir=tmp_path)) def _task() -> AgentEvalTask: @@ -480,7 +480,7 @@ async def test_native_skill_preserves_preconfigured_skill_paths( skill = AgentSkill.from_directory(_skill_bundle(tmp_path / "src")) provider = _FakeProvider() runtime = FabricContainerRuntime(config, provider=provider, skills=[skill]) # type: ignore[arg-type] - await runtime.run_tasks([_task()], AgentEvalRunConfig(output_dir=tmp_path)) + await runtime.run_tasks([_task()], AgentEvalRunConfig(work_dir=tmp_path)) paths = _seeded_skill_paths(provider) assert paths[:2] == ["/pre/existing-a", "/pre/existing-b"] @@ -497,7 +497,7 @@ async def test_native_skill_on_runtime_discovered_adapter(tmp_path: Path, monkey skill = AgentSkill.from_directory(_skill_bundle(tmp_path / "src")) provider = _FakeProvider() runtime = FabricContainerRuntime(custom, provider=provider, skills=[skill]) # type: ignore[arg-type] - (trial,) = await runtime.run_tasks([_task()], AgentEvalRunConfig(output_dir=tmp_path)) + (trial,) = await runtime.run_tasks([_task()], AgentEvalRunConfig(work_dir=tmp_path)) assert "/in/skills/code-review" in _seeded_skill_paths(provider) assert trial.metadata["skill"]["mode"] == "native" @@ -520,7 +520,7 @@ async def download_dir(self, handle: SandboxHandle, source_dir: str, target_dir: skill = AgentSkill.from_directory(_skill_bundle(tmp_path / "src")) provider = _CodexWorkspaceProvider() runtime = FabricContainerRuntime(_CODEX_CONFIG, provider=provider, skills=[skill]) # type: ignore[arg-type] - (trial,) = await runtime.run_tasks([_task()], AgentEvalRunConfig(output_dir=tmp_path)) + (trial,) = await runtime.run_tasks([_task()], AgentEvalRunConfig(work_dir=tmp_path)) # Codex discovers agentskills from .agents/skills/ in its working dir, so the bundle is seeded there in # the workspace (not /in), for the harness to self-discover during the run. @@ -545,7 +545,7 @@ async def test_skill_on_unsupported_adapter_fails_fast(tmp_path: Path, monkeypat runtime = FabricContainerRuntime(unsupported, provider=_FakeProvider(), skills=[skill]) # type: ignore[arg-type] with pytest.raises(RuntimeError, match="no known skill-injection strategy"): - await runtime.run_tasks([_task()], AgentEvalRunConfig(output_dir=tmp_path)) + await runtime.run_tasks([_task()], AgentEvalRunConfig(work_dir=tmp_path)) async def test_no_skill_leaves_metadata_none_and_skips_planner(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: @@ -605,7 +605,7 @@ async def download_dir(self, handle: SandboxHandle, source_dir: str, target_dir: ] provider = _CodexWorkspaceProvider() runtime = FabricContainerRuntime(_CODEX_CONFIG, provider=provider, skills=skills) # type: ignore[arg-type] - (trial,) = await runtime.run_tasks([_task()], AgentEvalRunConfig(output_dir=tmp_path)) + (trial,) = await runtime.run_tasks([_task()], AgentEvalRunConfig(work_dir=tmp_path)) # Both bundles seeded under the codex discovery dir, no skills path, all scrubbed from evidence. assert provider.seeded["/out/workspace/.agents/skills/docx/SKILL.md"].startswith("---") @@ -663,7 +663,7 @@ async def test_same_skill_from_both_injection_and_task_files_fails_task( "files": {".agents/skills/code-review/SKILL.md": "# override"}, }, ) - (trial,) = await runtime.run_tasks([task], AgentEvalRunConfig(output_dir=tmp_path)) + (trial,) = await runtime.run_tasks([task], AgentEvalRunConfig(work_dir=tmp_path)) assert trial.status == AgentEvalTrialStatus.FAILED error = json.loads(Path(trial.evidence.require("error").ref).read_text()) # type: ignore[arg-type] @@ -692,7 +692,7 @@ async def test_task_seeded_skill_coexists_with_a_different_injected_skill( "files": {".agents/skills/style-guide/SKILL.md": "---\nname: style-guide\n---\n"}, }, ) - (trial,) = await runtime.run_tasks([task], AgentEvalRunConfig(output_dir=tmp_path)) + (trial,) = await runtime.run_tasks([task], AgentEvalRunConfig(work_dir=tmp_path)) assert trial.status == AgentEvalTrialStatus.COMPLETED assert "/out/workspace/.agents/skills/code-review/SKILL.md" in provider.seeded diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_integration.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_integration.py index 9c12c223af..6327a36fba 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_integration.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_integration.py @@ -209,7 +209,7 @@ def __init__(self, **kwargs: Any) -> None: result = AgentEvaluator().run_sync( tasks=[_task()], target=runtime, - config=AgentEvalRunConfig(output_dir=tmp_path / "out", parallelism=1, write_dashboard=False), + config=AgentEvalRunConfig(work_dir=tmp_path / "out", parallelism=1), ) trial = result.trials[0] @@ -283,7 +283,7 @@ def test_fabric_codex_live_eval_captures_atif_trajectory(tmp_path: Path) -> None result = AgentEvaluator().run_sync( tasks=[_task()], target=runtime, - config=AgentEvalRunConfig(output_dir=tmp_path / "out", parallelism=1, write_dashboard=False), + config=AgentEvalRunConfig(work_dir=tmp_path / "out", parallelism=1), ) trial = result.trials[0] diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_harbor_runtime.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_harbor_runtime.py index 7b53c1c6c8..bf19d2ac02 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_harbor_runtime.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_harbor_runtime.py @@ -88,7 +88,7 @@ async def test_harbor_runner_scores_through_agent_evaluator_and_adapts_legacy_pa # run_job is awaited exactly once, then the job dir is adapted and scored end-to-end. calls = [] runner = HarborAgentTaskRunner(job_dir=job_dir, run_job=lambda: _record(calls)) - result = await AgentEvaluator().run(tasks=tasks, target=runner, config=AgentEvalRunConfig(write_dashboard=False)) + result = await AgentEvaluator().run(tasks=tasks, target=runner, config=AgentEvalRunConfig()) assert calls == ["ran"] rewards_by_task = {score.task_id: score.outputs[0].value for score in result.scores if score.outputs} diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_persistence.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_persistence.py index b1a83d9bdd..186e653617 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_persistence.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_persistence.py @@ -9,6 +9,7 @@ import shutil from pathlib import Path +import pytest from nemo_evaluator_sdk.agent_eval.persistence import persist_run, read_trials from nemo_evaluator_sdk.agent_eval.results import AgentEvalResult, AgentEvalSummary from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus, AgentOutput @@ -122,3 +123,34 @@ def test_persist_and_read_keep_external_evidence_refs_absolute(tmp_path: Path) - (trial,) = read_trials(bundle) assert trial.evidence.require("workspace").ref == external_ref # retained as-is + + +def test_persist_defaults_to_the_work_dir_so_evidence_lands_inside_the_bundle(tmp_path: Path) -> None: + # Defaulting to work_dir is what keeps a bundle self-contained: the evidence the trials point at + # is already underneath it, so persist can rewrite the refs bundle-relative. + workspace = tmp_path / "evidence" / "000000-taskA" / "workspace" + workspace.mkdir(parents=True) + result = AgentEvalResult( + run_id="r", + tasks=[], + trials=[_trial_with_workspace(str(workspace))], + scores=[], + summary=AgentEvalSummary.from_scores([], tasks=[]), + work_dir=tmp_path, + ) + + location = result.persist(write_dashboard=False) + + assert location.output_dir == tmp_path + stored = json.loads((tmp_path / "trials.jsonl").read_text(encoding="utf-8"))["evidence"]["descriptors"] + assert stored["workspace"]["ref"] == "evidence/000000-taskA/workspace" # relative => self-contained + + +def test_persist_without_a_work_dir_or_an_explicit_target_is_an_error() -> None: + # An in-memory run has nowhere to go; failing loudly beats inventing a directory. + result = AgentEvalResult( + run_id="r", tasks=[], trials=[], scores=[], summary=AgentEvalSummary.from_scores([], tasks=[]) + ) + + with pytest.raises(ValueError, match="no work_dir"): + result.persist() diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/intake/publish.py b/plugins/nemo-evaluator/src/nemo_evaluator/intake/publish.py index 11a2acf05b..a459ac9d2b 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/intake/publish.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/intake/publish.py @@ -194,7 +194,7 @@ def _publish_failure_message( failures: list[tuple[str, BaseException]], ) -> str: """Build an actionable error: what failed, where the results are cached, how to recover.""" - location = f"cached locally at {result.output_dir}" if result.output_dir is not None else "in the local run bundle" + location = f"cached locally at {result.work_dir}" if result.work_dir is not None else "in the local run bundle" detail = "\n ".join(f"{trial_id}: {type(error).__name__}: {error}" for trial_id, error in failures) return ( f"publish_to_intake: {len(failures)} of {len(result.trials)} trial(s) failed to publish " diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_evaluate.py b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_evaluate.py index 008e1338a7..4d890dcc80 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_evaluate.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_evaluate.py @@ -42,7 +42,6 @@ from nemo_evaluator.shared.metric_bundles.bundles import unbundle_metric from nemo_evaluator.task_refs import resolve_agent_eval_tasks from nemo_evaluator_sdk.agent_eval.evaluator import AgentEvaluator -from nemo_evaluator_sdk.agent_eval.persistence import persist_run from nemo_evaluator_sdk.agent_eval.results import AgentEvalResult from nemo_evaluator_sdk.agent_eval.runtimes.codex.runtime import CodexCliAgentRuntime from nemo_evaluator_sdk.agent_eval.runtimes.fabric.runtime import FabricAgentRuntime @@ -295,7 +294,9 @@ def _resolve_target( def _write_result_files(result: AgentEvalResult, persistent_dir: Path) -> AgentEvalResultFiles: """Persist the run bundle (trials/scores/tasks/summary) under the job's storage.""" bundle_dir = persistent_dir / AGENT_BUNDLE_DIR - persist_run(result, bundle_dir) + # No HTML dashboard for job runs: the artifact is consumed programmatically, and the job + # config asked for no dashboard before persistence became an explicit call. + result.persist(bundle_dir, write_dashboard=False) return AgentEvalResultFiles(bundle_dir=bundle_dir, summary=bundle_dir / SUMMARY_FILE_NAME) def run( @@ -316,7 +317,6 @@ def run( parallelism=spec.max_concurrent_tasks, benchmark=spec.benchmark, fail_fast=spec.fail_fast, - write_dashboard=False, ) # `run` may be injected a sync `sdk` (submitted jobs, via get_task_sdk) and/or an # `async_sdk`; forward whichever identity is present, preferring async when both are — the diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/evaluator.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/evaluator.py index 453ac48ab0..88ada94166 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/evaluator.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/evaluator.py @@ -19,8 +19,6 @@ import httpx import nemo_platform.beta.evaluator.inference as inference -from nemo_platform.beta.evaluator.agent_eval.dashboard import write_dashboard -from nemo_platform.beta.evaluator.agent_eval.persistence import persist_run from nemo_platform.beta.evaluator.agent_eval.results import AgentEvalResult, AgentEvalSummary from nemo_platform.beta.evaluator.agent_eval.scores import ( AgentEvalDiagnostic, @@ -180,10 +178,9 @@ async def run( scores=scores, summary=AgentEvalSummary.from_scores(scores, tasks=task_list), benchmark=benchmark, + work_dir=runtime_config.work_dir, ) - if runtime_config.output_dir is not None: - result = _persist_with_optional_dashboard(result, runtime_config.output_dir, runtime_config.write_dashboard) return result def run_sync( @@ -324,8 +321,8 @@ async def generate_one(index: int, task: AgentEvalTask) -> AgentEvalTrial: "invocation_id": f"{config.run_id}:{task.id}:{target.name}", } evidence_dir = ( - _task_evidence_dir(Path(config.output_dir), index=index, task_id=task.id) - if config.output_dir is not None and isinstance(target, AgentBase) + _task_evidence_dir(Path(config.work_dir), index=index, task_id=task.id) + if config.work_dir is not None and isinstance(target, AgentBase) else None ) resolved_inference_fn = self.inference_fn @@ -693,18 +690,6 @@ def _benchmark_metadata(tasks: list[AgentEvalTask]) -> dict[str, Any]: return {"benchmark": benchmarks[0] if len(benchmarks) == 1 else benchmarks} -def _persist_with_optional_dashboard( - result: AgentEvalResult, - output_dir: Path, - write_html: bool, -) -> AgentEvalResult: - path = Path(output_dir) - dashboard_path = None - if write_html: - dashboard_path = write_dashboard(result.model_copy(update={"output_dir": path}), path / "report.html") - return persist_run(result.model_copy(update={"output_dir": path, "dashboard_path": dashboard_path}), path) - - def _new_run_id() -> str: timestamp = datetime.now(UTC).strftime("%Y%m%d%H%M%S") return f"agent-eval-{timestamp}-{uuid.uuid4().hex[:8]}" diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/persistence.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/persistence.py index b9c26a04bf..7a232afe40 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/persistence.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/persistence.py @@ -10,32 +10,52 @@ from pathlib import Path from typing import Any -from nemo_platform.beta.evaluator.agent_eval.results import AgentEvalResult +from nemo_platform.beta.evaluator.agent_eval.dashboard import write_dashboard +from nemo_platform.beta.evaluator.agent_eval.results import AgentEvalResult, BundleLocation from nemo_platform.beta.evaluator.agent_eval.trials import AgentEvalTrial from pydantic import BaseModel +#: Filename of the rendered HTML dashboard inside a bundle. +DASHBOARD_FILENAME = "report.html" -def persist_run(result: AgentEvalResult, output_dir: str | Path) -> AgentEvalResult: - """Persist a completed run bundle to ``output_dir``.""" + +def persist_run( + result: AgentEvalResult, + output_dir: str | Path, + *, + write_html_dashboard: bool = True, +) -> BundleLocation: + """Write a completed run to a bundle at ``output_dir`` and report where it landed. + + Explicit rather than a side effect of :meth:`AgentEvaluator.run`: computing an evaluation and + storing one are different decisions, and folding them together is what forced the result object to + carry paths it could not know at construction time. (Same reasoning as ``publish_to_intake``.) + + Set ``write_html_dashboard=False`` to skip rendering ``report.html`` — the dashboard is written + here so the manifest can record it in a single pass. + """ path = Path(output_dir) path.mkdir(parents=True, exist_ok=True) + # Render first so the manifest below can name it; the dashboard reads only the run's own contents. + dashboard_path = write_dashboard(result, path / DASHBOARD_FILENAME) if write_html_dashboard else None + _write_json(path / "benchmark.json", result.benchmark) _write_jsonl(path / "tasks.jsonl", result.tasks) _write_trials(path / "trials.jsonl", result.trials, base=path) _write_jsonl(path / "scores.jsonl", result.scores) _write_json(path / "summary.json", result.summary) - updated = result.model_copy(update={"output_dir": path}) - _write_json(path / "run.json", _run_manifest(updated)) - return updated + location = BundleLocation(output_dir=path, dashboard_path=dashboard_path) + _write_json(path / "run.json", _run_manifest(result, location)) + return location -def _run_manifest(result: AgentEvalResult) -> dict[str, Any]: +def _run_manifest(result: AgentEvalResult, location: BundleLocation) -> dict[str, Any]: return { "run_id": result.run_id, - "output_dir": str(result.output_dir) if result.output_dir is not None else None, - "dashboard_path": str(result.dashboard_path) if result.dashboard_path is not None else None, + "output_dir": str(location.output_dir), + "dashboard_path": str(location.dashboard_path) if location.dashboard_path is not None else None, "artifacts": { "benchmark": "benchmark.json", "tasks": "tasks.jsonl", diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py index 897e30aeea..d99a845c82 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py @@ -68,8 +68,30 @@ def from_scores( ) +class BundleLocation(BaseModel): + """Where a run was written, returned by :meth:`AgentEvalResult.persist`. + + Kept off :class:`AgentEvalResult` because it is not a property of the evaluation — it is the + outcome of choosing to store it. Holding one means the bundle exists, so there is no optional to + re-check; a run that was never persisted simply has no ``BundleLocation``. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + output_dir: Path = Field(description="Directory the run bundle was written to.") + dashboard_path: Path | None = Field( + default=None, + description="Path to the rendered HTML dashboard, or None when dashboard writing was disabled.", + ) + + class AgentEvalResult(BaseModel): - """Root result for a completed agent evaluation: tasks, trials, scores, summary, and bundle metadata.""" + """Root result for a completed agent evaluation: tasks, trials, scores, and summary. + + Describes the evaluation and nothing else — storing it is a separate decision, made by calling + :meth:`persist`. Because the result carries no paths, it never holds a location that was unknown + when it was constructed, and nothing has to mutate it after the fact. + """ model_config = ConfigDict(extra="forbid") @@ -82,8 +104,39 @@ class AgentEvalResult(BaseModel): default_factory=dict, description="Benchmark metadata recorded for the run.", ) - output_dir: Path | None = Field(default=None, description="Directory the run bundle was written to, if any.") - dashboard_path: Path | None = Field(default=None, description="Path to the rendered dashboard, if written.") + work_dir: Path | None = Field( + default=None, + description="Directory the run worked in, where its runtimes wrote trial evidence. Known " + "before the run starts (it comes from the run config), so unlike a bundle location it is " + "never attached after the fact. None for a purely in-memory run.", + ) + + def persist(self, output_dir: str | Path | None = None, *, write_dashboard: bool = True) -> BundleLocation: + """Write this run to a bundle and return where it landed. + + Deliberately a call rather than something ``AgentEvaluator.run`` does for you: computing an + evaluation and storing one are separate decisions (the same reasoning as ``publish_to_intake``). + + Defaults to :attr:`work_dir`, which is the directory the trials' evidence already lives under — + so the bundle is self-contained and survives being moved. Passing a different ``output_dir`` + leaves those evidence references pointing back at the original directory. That is supported (a + re-scored run may reference an earlier run's deliverables) but the resulting bundle only + resolves while the original directory is still there. + + Set ``write_dashboard=False`` to skip rendering ``report.html``. + """ + # Imported here rather than at module scope: persistence imports this module for the types it + # writes, so a top-level import would be circular. + from nemo_platform.beta.evaluator.agent_eval.persistence import persist_run + + target = output_dir if output_dir is not None else self.work_dir + if target is None: + raise ValueError( + "this run has no work_dir to persist into (it ran in memory); pass an explicit " + "output_dir, or set work_dir on the AgentEvalRunConfig so evidence and bundle share " + "a directory" + ) + return persist_run(self, target, write_html_dashboard=write_dashboard) def _aggregate_scores( diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/codex/runtime.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/codex/runtime.py index 32bacefc36..ed02136bdb 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/codex/runtime.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/codex/runtime.py @@ -261,7 +261,7 @@ def _validate_artifact_permissions(self, evidence_dir: Path) -> None: def _evidence_dir(self, index: int, task: AgentEvalTask, config: AgentEvalRunConfig) -> Path: root = self._work_root if root is None: - root = (config.output_dir or Path.cwd()) / "evidence" / "codex" + root = (config.work_dir or Path.cwd()) / "evidence" / "codex" safe_task_id = _safe_path_name(task.id) task_dir = f"{index:06d}-{safe_task_id}" if safe_task_id else f"task-{index:06d}" return Path(root) / task_dir diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/docker_sandbox.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/docker_sandbox.py index 9113164c00..3240726ed0 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/docker_sandbox.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/docker_sandbox.py @@ -276,7 +276,7 @@ def _failed_trial(self, task: AgentEvalTask, exc: Exception, evidence_dir: Path) ) def _evidence_dir(self, index: int, task: AgentEvalTask, config: AgentEvalRunConfig) -> Path: - root = config.output_dir if config.output_dir is not None else self._work_root + root = config.work_dir if config.work_dir is not None else self._work_root if root is None: root = Path(tempfile.gettempdir()) / "nemo-evaluator-agent-runtime" run_id = config.run_id or _new_runtime_run_id() diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/container_runtime.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/container_runtime.py index 59fda190e6..28987c75d3 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/container_runtime.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/container_runtime.py @@ -469,7 +469,7 @@ def _failed_trial( def _evidence_dir(self, index: int, task: AgentEvalTask, config: AgentEvalRunConfig) -> Path: # Evidence lands under the run's output dir (like every other runtime); the container's own # working state lives at /out inside the sandbox and is downloaded here. - root = (config.output_dir or Path.cwd()) / "evidence" / "fabric_container" + root = (config.work_dir or Path.cwd()) / "evidence" / "fabric_container" return root / _common.task_subdir_name(index, task.id) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/runtime.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/runtime.py index 77423a4547..2ca60d3fe6 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/runtime.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/runtime.py @@ -511,7 +511,7 @@ def _relay_config(self, relay_dir: Path) -> RelayObservabilityConfig: def _evidence_dir(self, index: int, task: AgentEvalTask, config: AgentEvalRunConfig) -> Path: root = self._work_root if root is None: - root = (config.output_dir or Path.cwd()) / "evidence" / "fabric" + root = (config.work_dir or Path.cwd()) / "evidence" / "fabric" # The run id isolates this run's evidence from other runs sharing the same root (A/B baseline # vs. skilled); run_tasks always populates it, so the fallback only guards a direct call. run_id = config.run_id or _new_run_id() diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym_runtime.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym_runtime.py index 94526bae68..60c8a4e9f7 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym_runtime.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym_runtime.py @@ -347,8 +347,8 @@ async def run_tasks( # phases — parallelism bounds concurrent scoring (SDK-side, cheap), while Gym's `--concurrency` # bounds concurrent rollouts against the model endpoint during collection (tuned to that endpoint's # limits via GymRuntimeConfig.concurrency). - if config is not None and config.output_dir is not None: - work_dir = Path(config.output_dir) / "gym_run" + if config is not None and config.work_dir is not None: + work_dir = Path(config.work_dir) / "gym_run" else: work_dir = Path(tempfile.mkdtemp(prefix="gym_run_")) work_dir.mkdir(parents=True, exist_ok=True) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/harbor_runtime.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/harbor_runtime.py index 20d56349d3..a2efcbdf8b 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/harbor_runtime.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/harbor_runtime.py @@ -1299,7 +1299,7 @@ async def run_harbor_eval( return await AgentEvaluator().run( tasks=tasks, target=runner, - config=run_config or AgentEvalRunConfig(write_dashboard=False), + config=run_config or AgentEvalRunConfig(), ) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/tasks.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/tasks.py index 6c0da4ac21..3c23c2d8e8 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/tasks.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/tasks.py @@ -214,9 +214,11 @@ class AgentEvalRunConfig(BaseModel): model_config = ConfigDict(extra="forbid") - output_dir: Path | None = Field( + work_dir: Path | None = Field( default=None, - description="Directory where the run bundle is written; in-memory only when omitted.", + description="Directory the run works in: runtimes write trial evidence beneath it, and it is " + "the default target for AgentEvalResult.persist so the bundle contains that evidence. Purely " + "in-memory when omitted.", ) run_id: str | None = Field(default=None, description="Explicit run identifier; generated when omitted.") prompt_template: str | dict[str, Any] | None = Field( @@ -228,7 +230,6 @@ class AgentEvalRunConfig(BaseModel): description="Inference/run parameters used when producing trials online.", ) parallelism: int = Field(default=4, ge=1, description="Maximum number of tasks scored concurrently.") - write_dashboard: bool = Field(default=True, description="Whether to render an HTML dashboard for the run.") benchmark: dict[str, Any] = Field( default_factory=dict, description="Benchmark metadata recorded alongside the run.",