diff --git a/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/container_runtime.py b/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/container_runtime.py index 149a5295a7..ed883a820a 100644 --- a/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/container_runtime.py +++ b/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/container_runtime.py @@ -157,9 +157,17 @@ async def run( ) # The task arrives exactly as it would in production Buzz: a # user prompt @mentioning the orchestrator. The harness never - # speaks as any agent. + # speaks as any agent. The orchestrator is mentioned by pubkey, + # not by name resolution: task text is untrusted payload, and any + # @-token inside it (e.g. Vim's `:%normal! @a`) would otherwise + # fail member resolution and kill the trial before the agent + # ever saw the task. An explicit --mention demotes unresolved + # @-tokens in the text to presentation-only. await self._send( - trial.user, trial, f"@{orchestrator.agent_id} {instruction}" + trial.user, + trial, + f"@{orchestrator.agent_id} {instruction}", + mention=orchestrator.nostr_pubkey, ) final_message = await asyncio.wait_for( self._wait_for_done(environment, orchestrator, trial, agents + infra), @@ -519,18 +527,24 @@ async def _verify_m1_output( ) async def _send( - self, credential: AgentCredential, trial: TrialHandle, content: str + self, + credential: AgentCredential, + trial: TrialHandle, + content: str, + *, + mention: str | None = None, ) -> None: - await self._buzz_json( - credential, - trial, + args = [ "messages", "send", "--channel", trial.channel_id, "--content", content, - ) + ] + if mention is not None: + args += ["--mention", mention] + await self._buzz_json(credential, trial, *args) async def _buzz_json( self, credential: AgentCredential, trial: TrialHandle, *args: str diff --git a/benchmarks/harbor-buzz-orchestra/tests/test_container_runtime.py b/benchmarks/harbor-buzz-orchestra/tests/test_container_runtime.py index ebf0eb4b5d..5fc0e63e54 100644 --- a/benchmarks/harbor-buzz-orchestra/tests/test_container_runtime.py +++ b/benchmarks/harbor-buzz-orchestra/tests/test_container_runtime.py @@ -372,6 +372,37 @@ async def test_m1_output_probe_matches_grader_and_is_condition_scoped( assert bool(probed) == (condition == "M1-hello-world") +async def test_send_mentions_by_pubkey_so_task_text_stays_inert( + tmp_path, monkeypatch +): + """Task text is untrusted payload: `:%normal! @a` in a task statement must + not be fed to member-name resolution (it would fail and kill the trial). + An explicit --mention pins delivery to the orchestrator's pubkey.""" + rt = runtime(tmp_path) + orch = credential("orch-1", "orchestrator", "orch-model") + trial = trial_handle((orch,)) + calls = [] + + async def buzz_json(credential, trial, *args): + calls.append(args) + return {} + + monkeypatch.setattr(rt, "_buzz_json", buzz_json) + + await rt._send( + trial.user, + trial, + "@orch-1 run `:%normal! @a` on the file", + mention=orch.nostr_pubkey, + ) + assert calls[-1][-2:] == ("--mention", "pubkey-orch-1") + + # Without an explicit mention the send is unchanged (name resolution). + await rt._send(trial.user, trial, "plain content") + assert "--mention" not in calls[-1] + assert calls[-1][-2:] == ("--content", "plain content") + + async def test_wait_for_done_requires_orchestrator_authorship(tmp_path, monkeypatch): rt = runtime(tmp_path, poll_seconds=0) orch = credential("orch-1", "orchestrator", "orch-model")