Skip to content

Commit b51c8ba

Browse files
committed
feat(tournament): per-contestant model selection via --models
The headline use of a tournament is settling which model actually wins on your codebase, not just which of N identical runs got lucky. --models a,b,c races one contestant per model id (same provider/proxy as the parent), and the picker + live status show each contestant's model.
1 parent 258a64e commit b51c8ba

5 files changed

Lines changed: 102 additions & 38 deletions

File tree

CLAUDE.md

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -277,7 +277,10 @@ the available types.
277277
## Tournaments (`/tournament`)
278278

279279
`/tournament [n] <task>` (default n=3, max 5) races N agents on the same
280-
build task and lets you merge the winner. Called mid-build: it snapshots
280+
build task and lets you merge the winner. `--models opus,sonnet,haiku
281+
<task>` instead runs one contestant per model id (same provider/proxy as
282+
the parent, via per-agent model id-cloning) — race models head-to-head on
283+
your actual codebase. Called mid-build: it snapshots
281284
the **working tree** — tracked + untracked, via a scratch index so your
282285
real index is untouched (`src/agent/wip-snapshot.ts`) — and branches
283286
every contestant from that, so in-progress work is preserved. Each

src/commands/builtins/tournament.test.ts

Lines changed: 34 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,14 +4,16 @@ import { tournament } from "./tournament.js";
44

55
function makeCtx(withRunner: boolean): {
66
ctx: CommandContext;
7-
calls: { task: string; count: number }[];
7+
calls: { task: string; count: number; models?: string[] }[];
88
emits: string[];
99
} {
10-
const calls: { task: string; count: number }[] = [];
10+
const calls: { task: string; count: number; models?: string[] }[] = [];
1111
const emits: string[] = [];
1212
const ctx = {
1313
emit: (t: string) => emits.push(t),
14-
runTournament: withRunner ? (task: string, count: number) => calls.push({ task, count }) : undefined,
14+
runTournament: withRunner
15+
? (task: string, opts: { count: number; models?: string[] }) => calls.push({ task, ...opts })
16+
: undefined,
1517
} as unknown as CommandContext;
1618
return { ctx, calls, emits };
1719
}
@@ -20,20 +22,45 @@ describe("/tournament", () => {
2022
it("defaults to 3 contestants when no count is given", () => {
2123
const { ctx, calls } = makeCtx(true);
2224
tournament.handler("add pagination to the list", ctx);
23-
expect(calls).toEqual([{ task: "add pagination to the list", count: 3 }]);
25+
expect(calls).toEqual([{ task: "add pagination to the list", count: 3, models: undefined }]);
2426
});
2527

2628
it("parses and clamps a leading count to 2..5", () => {
2729
const { ctx, calls } = makeCtx(true);
2830
tournament.handler("9 refactor the parser", ctx);
29-
expect(calls[0]).toEqual({ task: "refactor the parser", count: 5 });
31+
expect(calls[0]).toMatchObject({ task: "refactor the parser", count: 5 });
3032
});
3133

3234
it("treats a number-only arg as the task, not a count", () => {
3335
const { ctx, calls } = makeCtx(true);
3436
tournament.handler("42", ctx);
35-
// "42" alone has no task after it, so it stays the task with default count.
36-
expect(calls[0]).toEqual({ task: "42", count: 3 });
37+
expect(calls[0]).toMatchObject({ task: "42", count: 3 });
38+
});
39+
40+
it("parses --models into one contestant per model", () => {
41+
const { ctx, calls } = makeCtx(true);
42+
tournament.handler("--models opus,sonnet,haiku fix the parser", ctx);
43+
expect(calls[0]).toEqual({ task: "fix the parser", count: 3, models: ["opus", "sonnet", "haiku"] });
44+
});
45+
46+
it("supports --models=a,b syntax and ignores a leading digit as task text", () => {
47+
const { ctx, calls } = makeCtx(true);
48+
tournament.handler("--models=a,b 2fa support", ctx);
49+
expect(calls[0]).toEqual({ task: "2fa support", count: 2, models: ["a", "b"] });
50+
});
51+
52+
it("rejects a single-model list", () => {
53+
const { ctx, calls, emits } = makeCtx(true);
54+
tournament.handler("--models solo do a thing", ctx);
55+
expect(calls).toHaveLength(0);
56+
expect(emits[0]).toMatch(/at least 2/);
57+
});
58+
59+
it("caps the model list at 5", () => {
60+
const { ctx, calls } = makeCtx(true);
61+
tournament.handler("--models a,b,c,d,e,f,g build", ctx);
62+
expect(calls[0].models).toHaveLength(5);
63+
expect(calls[0].count).toBe(5);
3764
});
3865

3966
it("shows usage when given no task", () => {
Lines changed: 46 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,41 +1,72 @@
11
import type { Command } from "../types.js";
22

3+
const MAX_CONTESTANTS = 5;
4+
35
/**
46
* /tournament [n] <task> — race N agents (default 3, max 5) on the same
5-
* build task in isolated worktrees, then pick the winner to merge. The
6-
* heavy lifting (snapshot, worktrees, judge, merge UI) lives in the App
7-
* via ctx.runTournament; here we just parse and hand off.
7+
* build task in isolated worktrees, then pick the winner to merge.
8+
* `--models a,b,c` runs one contestant per model id instead of N copies
9+
* of the current model. The heavy lifting (snapshot, worktrees, judge,
10+
* merge UI) lives in the App via ctx.runTournament; here we just parse.
811
*/
912
export const tournament: Command = {
1013
name: "tournament",
1114
aliases: ["race"],
12-
description: "Race N agents on a build task in parallel, then merge the winner. /tournament [n] <task>",
15+
description: "Race agents on a build task, then merge the winner. /tournament [n|--models a,b,c] <task>",
1316
mutates: true,
1417
handler: (args, ctx) => {
1518
if (!ctx.runTournament) {
1619
ctx.emit("/tournament needs the pi-tui UI (the default). It's not available in the legacy renderer.");
1720
return { handled: true };
1821
}
19-
const trimmed = args.trim();
20-
if (!trimmed) {
22+
let rest = args.trim();
23+
if (!rest) {
2124
ctx.emit(
22-
"Usage: /tournament [n] <what to build or change> — e.g. /tournament 3 add pagination to the users list",
25+
"Usage: /tournament [n] <task> or /tournament --models a,b,c <task>\n" +
26+
"e.g. /tournament 3 add pagination · /tournament --models opus,sonnet,haiku fix the parser",
2327
);
2428
return { handled: true };
2529
}
26-
// Optional leading contestant count.
27-
let count = 3;
28-
let task = trimmed;
29-
const m = trimmed.match(/^(\d+)\s+(.*)$/s);
30-
if (m) {
31-
count = Math.min(5, Math.max(2, Number.parseInt(m[1], 10)));
32-
task = m[2].trim();
30+
31+
// Pull out an optional --models / --model flag from anywhere in the args.
32+
let models: string[] | undefined;
33+
const mm = rest.match(/--models?(?:=|\s+)(\S+)/);
34+
if (mm) {
35+
models = mm[1]
36+
.split(",")
37+
.map((s) => s.trim())
38+
.filter(Boolean);
39+
rest = (rest.slice(0, mm.index) + rest.slice((mm.index ?? 0) + mm[0].length)).replace(/\s+/g, " ").trim();
40+
if (models.length < 2) {
41+
ctx.emit(
42+
"--models needs at least 2 model ids to race (or drop the flag for N copies of the current model).",
43+
);
44+
return { handled: true };
45+
}
46+
if (models.length > MAX_CONTESTANTS) {
47+
ctx.emit(`capping at ${MAX_CONTESTANTS} contestants; ignoring the extra models.`);
48+
models = models.slice(0, MAX_CONTESTANTS);
49+
}
3350
}
51+
52+
// A leading count only applies when no explicit model list was given
53+
// (otherwise the contestant count is the number of models, and a
54+
// leading digit is part of the task — e.g. "2fa support").
55+
let count = models ? models.length : 3;
56+
let task = rest;
57+
if (!models) {
58+
const m = rest.match(/^(\d+)\s+(.*)$/s);
59+
if (m) {
60+
count = Math.min(MAX_CONTESTANTS, Math.max(2, Number.parseInt(m[1], 10)));
61+
task = m[2].trim();
62+
}
63+
}
64+
3465
if (!task) {
3566
ctx.emit("Give the contestants something to build: /tournament <task>.");
3667
return { handled: true };
3768
}
38-
ctx.runTournament(task, count);
69+
ctx.runTournament(task, { count, models });
3970
return { handled: true };
4071
},
4172
};

src/commands/types.ts

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -44,11 +44,13 @@ export interface CommandContext {
4444
*/
4545
openRewindPicker?: () => void;
4646
/**
47-
* Run a /tournament: race `count` agents on `task` in isolated
48-
* worktrees, judge them, and open the merge picker (pi-tui only).
49-
* Undefined on UIs without it.
47+
* Run a /tournament: race agents on `task` in isolated worktrees, judge
48+
* them, and open the merge picker (pi-tui only). `models`, when given,
49+
* runs one contestant per model id (same provider/proxy as the parent);
50+
* otherwise `count` copies of the current model race. Undefined on UIs
51+
* without it.
5052
*/
51-
runTournament?: (task: string, count: number) => void;
53+
runTournament?: (task: string, opts: { count: number; models?: string[] }) => void;
5254
}
5355

5456
export interface CommandResult {

src/ui-pi/app.ts

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -659,12 +659,13 @@ export class App extends Container {
659659
}
660660

661661
/**
662-
* Run a /tournament: snapshot the working tree, race `count` general
663-
* agents on `task` in isolated worktrees, then open the results picker so
664-
* the user can merge a winner. Heavy + long-running, so it's gated
665-
* against the agent being busy or another tournament already going.
662+
* Run a /tournament: snapshot the working tree, race general agents on
663+
* `task` in isolated worktrees, then open the results picker so the user
664+
* can merge a winner. With `opts.models`, one contestant runs per model
665+
* id; otherwise `opts.count` copies of the current model race. Heavy +
666+
* long-running, so it's gated against the agent or another tournament.
666667
*/
667-
private async startTournament(task: string, count: number): Promise<void> {
668+
private async startTournament(task: string, opts: { count: number; models?: string[] }): Promise<void> {
668669
if (!this.tui) return;
669670
if (this.busy || this.tournamentRunning) {
670671
this.statusBar.note("can't start a tournament while the agent (or another tournament) is running.");
@@ -675,12 +676,12 @@ export class App extends Container {
675676
const cwd = this.bundle.toolContext.cwd;
676677
try {
677678
const snap = await snapshotWorkingTree(cwd, this.tournamentAbort.signal);
678-
const branches: BranchSpec[] = Array.from({ length: count }, (_, i) => ({
679-
id: String.fromCharCode(65 + i), // A, B, C…
680-
}));
679+
const branches: BranchSpec[] = opts.models
680+
? opts.models.map((model, i) => ({ id: String.fromCharCode(65 + i), model }))
681+
: Array.from({ length: opts.count }, (_, i) => ({ id: String.fromCharCode(65 + i) }));
681682
const status = new Map<string, string>(branches.map((b) => [b.id, "queued"]));
682683
const renderStatus = () => {
683-
const parts = branches.map((b) => `${b.id}:${status.get(b.id)}`);
684+
const parts = branches.map((b) => `${b.id}${b.model ? `(${b.model})` : ""}:${status.get(b.id)}`);
684685
this.statusBar.note(`🏁 tournament — ${parts.join(" ")}`);
685686
this.tui?.requestRender();
686687
};
@@ -1057,8 +1058,8 @@ export class App extends Container {
10571058
},
10581059
switchSession: (sessionId) => this.switchSession(sessionId),
10591060
openRewindPicker: () => this.showRewindOverlay(),
1060-
runTournament: (task, count) => {
1061-
void this.startTournament(task, count);
1061+
runTournament: (task, opts) => {
1062+
void this.startTournament(task, opts);
10621063
},
10631064
});
10641065
if (!result.handled) {

0 commit comments

Comments
 (0)