From f875731e6975ddb10f79f4558a1b9ce5669af216 Mon Sep 17 00:00:00 2001 From: Alec Khoury Date: Wed, 29 Jul 2026 10:54:21 -0500 Subject: [PATCH] feat(eval-author): scaffold the nemo eval-author CLI surface Register discover, audit, propose, run, and doctor so the command tree exists and each child ticket has a landing spot. Every body exits non-zero naming the ticket that owns it, because a placeholder that exits 0 reads as a run that did nothing. Flags belong to those tickets, so no verb declares options yet. The CLI module imports nothing from eval_author. Those agents build their LLM client while the class body executes, so importing one here would make the whole CLI require AUTHOR_* credentials, including the doctor verb whose job is to report them missing. Experimentalist keeps its runner behind a lazily assigned module global for the same reason. Declare nemo-platform-plugin directly instead of leaning on the transitive path through Experimentalist, which is the coupling this plugin is shedding. Cover the entry-point wiring in tests. A typo in the key or the import path does not fail an import, it just makes nemo eval-author quietly missing from the CLI, which no unit test of the module itself would catch. Signed-off-by: Alec Khoury --- plugins/nemo-eval-author/pyproject.toml | 4 + .../src/nemo_eval_author_plugin/cli.py | 73 +++++++++++++++++++ plugins/nemo-eval-author/tests/test_cli.py | 64 ++++++++++++++++ uv.lock | 9 +-- 4 files changed, 144 insertions(+), 6 deletions(-) create mode 100644 plugins/nemo-eval-author/src/nemo_eval_author_plugin/cli.py create mode 100644 plugins/nemo-eval-author/tests/test_cli.py diff --git a/plugins/nemo-eval-author/pyproject.toml b/plugins/nemo-eval-author/pyproject.toml index 94e72320b0..870d2dcefc 100644 --- a/plugins/nemo-eval-author/pyproject.toml +++ b/plugins/nemo-eval-author/pyproject.toml @@ -10,9 +10,13 @@ dependencies = [ "nemo-experimentalist-plugin", "nemo-insights-plugin", "nemo-platform", + "nemo-platform-plugin", "tomlkit>=0.13.3", ] +[project.entry-points."nemo.cli"] +eval-author = "nemo_eval_author_plugin.cli:EvalAuthorCLI" + [build-system] requires = ["hatchling"] build-backend = "hatchling.build" diff --git a/plugins/nemo-eval-author/src/nemo_eval_author_plugin/cli.py b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/cli.py new file mode 100644 index 0000000000..d7c5667414 --- /dev/null +++ b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/cli.py @@ -0,0 +1,73 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Eval Author plugin CLI — ``nemo eval-author ...`` subcommands. + +Scaffolding. Every verb is registered under its final name so the command tree is +discoverable and the child tickets have a landing spot, and each body exits non-zero +until its own ticket lands. Flags belong to those tickets, so nothing here declares +options yet. + +This module imports nothing from ``eval_author``: those agents build their LLM client +while the class body executes, so importing one here would make the whole CLI require +``AUTHOR_*`` credentials — including the ``doctor`` verb whose job is to report that they +are missing. Experimentalist's CLI keeps its runner behind a lazily-assigned module +global for the same reason. +""" + +from typing import ClassVar, NoReturn + +import typer +from nemo_platform_plugin.cli import NemoCLI + + +def _not_implemented(command: str, ticket: str) -> NoReturn: + """Fail loudly, so a placeholder verb can never be mistaken for a successful run.""" + typer.echo(f"`nemo eval-author {command}` is not implemented yet ({ticket}).", err=True) + raise typer.Exit(code=1) + + +class EvalAuthorCLI(NemoCLI): + """``nemo eval-author ...`` subcommands.""" + + name: ClassVar[str] = "eval-author" + description: ClassVar[str] = "NeMo Eval Author commands." + + def get_cli(self) -> typer.Typer: + app = typer.Typer(help=self.description, no_args_is_help=True) + + @app.callback() + def _root() -> None: + """Force subcommand dispatch even when only one verb is registered.""" + + @app.command("discover") + def discover() -> None: + """Discover candidate evaluation cases from agent traces.""" + # TODO(ASE-677): declare flags and wire discovery. + _not_implemented("discover", "ASE-677") + + @app.command("audit") + def audit() -> None: + """Audit an existing eval suite for coverage gaps.""" + # TODO(ASE-676): declare flags and wire the audit. + _not_implemented("audit", "ASE-676") + + @app.command("propose") + def propose() -> None: + """Propose eval suite additions for review.""" + # TODO(ASE-675): declare flags and wire the proposal. + _not_implemented("propose", "ASE-675") + + @app.command("run") + def run() -> None: + """Run the Eval Author pipeline end to end.""" + # TODO(ASE-673): declare flags and wire the pipeline to run_eval_author. + _not_implemented("run", "ASE-673") + + @app.command("doctor") + def doctor() -> None: + """Diagnose Eval Author setup: credentials, platform, runtime.""" + # TODO(ASE-678): report the prerequisites the other verbs gate on. + _not_implemented("doctor", "ASE-678") + + return app diff --git a/plugins/nemo-eval-author/tests/test_cli.py b/plugins/nemo-eval-author/tests/test_cli.py new file mode 100644 index 0000000000..663d0f15f1 --- /dev/null +++ b/plugins/nemo-eval-author/tests/test_cli.py @@ -0,0 +1,64 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Scaffolding tests: the command tree exists, and every verb still refuses to run. + +The entry-point cases cover the ``pyproject.toml`` wiring that nothing else exercises. A +typo in the key or the import path does not fail an import; it just makes ``nemo +eval-author`` quietly missing from the CLI, which no unit test of this module would catch. +""" + +from importlib.metadata import EntryPoint, entry_points + +import pytest +import typer +from nemo_eval_author_plugin import cli +from typer.testing import CliRunner + +runner = CliRunner() + +# Each verb is a placeholder owned by the child ticket named beside it. +_PLACEHOLDER_VERBS = [ + ("discover", "ASE-677"), + ("audit", "ASE-676"), + ("propose", "ASE-675"), + ("run", "ASE-673"), + ("doctor", "ASE-678"), +] + + +@pytest.fixture +def app() -> typer.Typer: + return cli.EvalAuthorCLI().get_cli() + + +def _eval_author_entry_point() -> EntryPoint: + matches = [entry for entry in entry_points(group="nemo.cli") if entry.name == "eval-author"] + assert matches, "no nemo.cli entry point named 'eval-author'; reinstall the plugin with uv sync" + return matches[0] + + +def test_help_lists_every_verb(app: typer.Typer) -> None: + result = runner.invoke(app, ["--help"]) + + assert result.exit_code == 0, result.output + for command, _ in _PLACEHOLDER_VERBS: + assert command in result.output + + +@pytest.mark.parametrize(("command", "ticket"), _PLACEHOLDER_VERBS) +def test_verb_refuses_to_run_and_names_its_ticket(app: typer.Typer, command: str, ticket: str) -> None: + result = runner.invoke(app, [command]) + + assert result.exit_code == 1, result.output + assert ticket in result.output + + +def test_entry_point_key_matches_the_cli_name() -> None: + """Discovery rejects a plugin whose entry-point key differs from its ``name``.""" + assert _eval_author_entry_point().value == "nemo_eval_author_plugin.cli:EvalAuthorCLI" + assert cli.EvalAuthorCLI.name == "eval-author" + + +def test_entry_point_loads_the_cli_class() -> None: + assert _eval_author_entry_point().load() is cli.EvalAuthorCLI diff --git a/uv.lock b/uv.lock index 61520d9a0e..99d2652508 100644 --- a/uv.lock +++ b/uv.lock @@ -4065,6 +4065,7 @@ dependencies = [ { name = "nemo-experimentalist-plugin", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "nemo-insights-plugin", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "nemo-platform", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nemo-platform-plugin", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "nooa", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "pydantic", extra = ["email"], marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "tomlkit", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, @@ -4076,7 +4077,8 @@ requires-dist = [ { name = "nemo-experimentalist-plugin", editable = "plugins/nemo-experimentalist" }, { name = "nemo-insights-plugin", editable = "plugins/nemo-insights" }, { name = "nemo-platform", editable = "packages/nemo_platform" }, - { name = "nooa", git = "https://github.com/NVIDIA-NeMo/labs-OO-Agents.git?tag=v0.0.6" }, + { name = "nemo-platform-plugin", editable = "packages/nemo_platform_plugin" }, + { name = "nooa", git = "https://github.com/NVIDIA-NeMo/labs-OO-Agents.git?rev=bea4614a0e2a6cf88f76225466159af883da80a0" }, { name = "pydantic", specifier = ">=2" }, { name = "tomlkit", specifier = ">=0.13.3" }, ] @@ -8887,11 +8889,6 @@ version = "24.0.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/91/13/13e1069b351bdc3881266e11147ffccf687505dbb0ea74036237f5d454a5/pyarrow-24.0.0.tar.gz", hash = "sha256:85fe721a14dd823aca09127acbb06c3ca723efbd436c004f16bca601b04dcc83", size = 1180261, upload-time = "2026-04-21T10:51:25.837Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/62/c9/a47ab7ece0d86cbe6678418a0fbd1ac4bb493b9184a3891dfa0e7f287ae0/pyarrow-24.0.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:b0e131f880cda8d04e076cee175a46fc0e8bc8b65c99c6c09dff6669335fde74", size = 35068898, upload-time = "2026-04-21T10:46:36.599Z" }, - { url = "https://files.pythonhosted.org/packages/eb/8e/fb178720400ef69db251eb4a9c3ccf4af269bc1feb5055529b8fc87170d1/pyarrow-24.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:0b3537c00fb8d384f15ac1e79b6eb6db04a16514c8c1d22e59a9b95c8ba42868", size = 45697931, upload-time = "2026-04-21T10:46:48.403Z" }, - { url = "https://files.pythonhosted.org/packages/f3/27/99c42abe8e21b44f4917f62631f3aa31404882a2c41d8a4cd5c110e13d52/pyarrow-24.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:14e31a3c9e35f1ab6356c6378f6f72830e6d2d5f1791df3774a7b097d18a6a1e", size = 48837449, upload-time = "2026-04-21T10:46:55.329Z" }, - { url = "https://files.pythonhosted.org/packages/36/b6/333749e2666e9032891125bf9c691146e92901bece62030ac1430e2e7c88/pyarrow-24.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b7d9a514e73bc42711e6a35aaccf3587c520024fe0a25d830a1a8a27c15f4f57", size = 49395949, upload-time = "2026-04-21T10:47:01.869Z" }, - { url = "https://files.pythonhosted.org/packages/17/25/c5201706a2dd374e8ba6ee3fd7a8c89fb7ffc16eed5217a91fd2bd7f7626/pyarrow-24.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b196eb3f931862af3fa84c2a253514d859c08e0d8fe020e07be12e75a5a9780c", size = 51912986, upload-time = "2026-04-21T10:47:09.872Z" }, { url = "https://files.pythonhosted.org/packages/b4/a9/9686d9f07837f91f775e8932659192e02c74f9d8920524b480b85212cc68/pyarrow-24.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:6233c9ed9ab9d1db47de57d9753256d9dcffbf42db341576099f0fd9f6bf4810", size = 34981559, upload-time = "2026-04-21T10:47:22.17Z" }, { url = "https://files.pythonhosted.org/packages/7c/3b/926382efe8ce27ba729071d3566ade6dfb86bdf112f366000196b2f5780a/pyarrow-24.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:1617043b99bd33e5318ae18eb2919af09c71322ef1ca46566cdafc6e6712fb66", size = 45679394, upload-time = "2026-04-21T10:47:34.821Z" }, { url = "https://files.pythonhosted.org/packages/b3/7a/829f7d9dfd37c207206081d6dad474d81dde29952401f07f2ba507814818/pyarrow-24.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:6165461f55ef6314f026de6638d661188e3455d3ec49834556a0ebbdbace18bb", size = 48863122, upload-time = "2026-04-21T10:47:42.056Z" },