From e1953e219865b2875f1d8e506b7fab5e3fe9ceae Mon Sep 17 00:00:00 2001 From: Eva <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@buzz.block.builderlab.xyz> Date: Thu, 6 Aug 2026 21:54:51 -0400 Subject: [PATCH] fix(bench): mention the orchestrator by pubkey when posting the task MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The harness posts each trial's task via `buzz messages send`, relying on `@` name resolution. Task text is untrusted payload: when it contains @-tokens of its own — TB 2.1's large-scale-text-editing task embeds Vim macros like `:%normal! @a` — the CLI's mention resolver rejects `@a` as an unknown channel member and refuses to send, killing the trial with RuntimeLaunchError before the agent ever saw the task. Pass the orchestrator's pubkey as an explicit --mention instead. The CLI demotes unresolved @-tokens in the text to presentation-only when any explicit identity is supplied, so delivery still targets exactly the orchestrator and every @-token in the task statement becomes inert. Live repro on the relay confirmed both halves: a fenced `@a` without --mention still hard-fails (the resolver is not markdown-aware); the same content with an explicit --mention sends clean. ## Testing - benchmarks/harbor-buzz-orchestra: full pytest suite, 35 passed (34 baseline + new test), ruff clean — validated against origin/main 769ac70b with this patch applied - testbed: full pytest suite, 23 passed / 1 skipped, ruff clean Originating Buzz thread: buzz://message?channel=c3252dd2-0142-4e01-88c7-a2183c3960a5&id=74a65a0990fd2197882b66b5ea2707169d4a3dbd2020d1610c45150fb99f140b Co-authored-by: Eva <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@buzz.block.builderlab.xyz> Signed-off-by: Eva <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@buzz.block.builderlab.xyz> --- .../container_runtime.py | 28 ++++++++++++----- .../tests/test_container_runtime.py | 31 +++++++++++++++++++ 2 files changed, 52 insertions(+), 7 deletions(-) diff --git a/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/container_runtime.py b/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/container_runtime.py index 149a5295a7..ed883a820a 100644 --- a/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/container_runtime.py +++ b/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/container_runtime.py @@ -157,9 +157,17 @@ async def run( ) # The task arrives exactly as it would in production Buzz: a # user prompt @mentioning the orchestrator. The harness never - # speaks as any agent. + # speaks as any agent. The orchestrator is mentioned by pubkey, + # not by name resolution: task text is untrusted payload, and any + # @-token inside it (e.g. Vim's `:%normal! @a`) would otherwise + # fail member resolution and kill the trial before the agent + # ever saw the task. An explicit --mention demotes unresolved + # @-tokens in the text to presentation-only. await self._send( - trial.user, trial, f"@{orchestrator.agent_id} {instruction}" + trial.user, + trial, + f"@{orchestrator.agent_id} {instruction}", + mention=orchestrator.nostr_pubkey, ) final_message = await asyncio.wait_for( self._wait_for_done(environment, orchestrator, trial, agents + infra), @@ -519,18 +527,24 @@ async def _verify_m1_output( ) async def _send( - self, credential: AgentCredential, trial: TrialHandle, content: str + self, + credential: AgentCredential, + trial: TrialHandle, + content: str, + *, + mention: str | None = None, ) -> None: - await self._buzz_json( - credential, - trial, + args = [ "messages", "send", "--channel", trial.channel_id, "--content", content, - ) + ] + if mention is not None: + args += ["--mention", mention] + await self._buzz_json(credential, trial, *args) async def _buzz_json( self, credential: AgentCredential, trial: TrialHandle, *args: str diff --git a/benchmarks/harbor-buzz-orchestra/tests/test_container_runtime.py b/benchmarks/harbor-buzz-orchestra/tests/test_container_runtime.py index ebf0eb4b5d..5fc0e63e54 100644 --- a/benchmarks/harbor-buzz-orchestra/tests/test_container_runtime.py +++ b/benchmarks/harbor-buzz-orchestra/tests/test_container_runtime.py @@ -372,6 +372,37 @@ async def test_m1_output_probe_matches_grader_and_is_condition_scoped( assert bool(probed) == (condition == "M1-hello-world") +async def test_send_mentions_by_pubkey_so_task_text_stays_inert( + tmp_path, monkeypatch +): + """Task text is untrusted payload: `:%normal! @a` in a task statement must + not be fed to member-name resolution (it would fail and kill the trial). + An explicit --mention pins delivery to the orchestrator's pubkey.""" + rt = runtime(tmp_path) + orch = credential("orch-1", "orchestrator", "orch-model") + trial = trial_handle((orch,)) + calls = [] + + async def buzz_json(credential, trial, *args): + calls.append(args) + return {} + + monkeypatch.setattr(rt, "_buzz_json", buzz_json) + + await rt._send( + trial.user, + trial, + "@orch-1 run `:%normal! @a` on the file", + mention=orch.nostr_pubkey, + ) + assert calls[-1][-2:] == ("--mention", "pubkey-orch-1") + + # Without an explicit mention the send is unchanged (name resolution). + await rt._send(trial.user, trial, "plain content") + assert "--mention" not in calls[-1] + assert calls[-1][-2:] == ("--content", "plain content") + + async def test_wait_for_done_requires_orchestrator_authorship(tmp_path, monkeypatch): rt = runtime(tmp_path, poll_seconds=0) orch = credential("orch-1", "orchestrator", "orch-model")