Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .agentv/targets.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ targets:
- name: default
provider: pi-coding-agent
executable: ${{ PI_CLI_PATH }}
pi_provider: openrouter
subprovider: openrouter
model: z-ai/glm-4.7
api_key: ${{ OPENROUTER_API_KEY }}
system_prompt: "Answer directly based on the information provided."
Expand Down
2 changes: 1 addition & 1 deletion examples/features/.agentv/targets.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ targets:
- name: pi
provider: pi-coding-agent
executable: ${{ PI_CLI_PATH }} # Optional: defaults to `pi` if omitted
pi_provider: openrouter
subprovider: openrouter
model: openai/gpt-5.4
api_key: ${{ OPENROUTER_API_KEY }}
grader_target: gemini-llm
Expand Down
4 changes: 2 additions & 2 deletions packages/core/src/evaluation/providers/pi-agent-sdk.ts
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ export class PiAgentSdkProvider implements Provider {

const startTimeIso = new Date().toISOString();
const startMs = Date.now();
const providerName = this.config.provider ?? 'anthropic';
const providerName = this.config.subprovider ?? 'anthropic';
const modelId = this.config.model ?? 'claude-sonnet-4-20250514';
// Use type assertion since getModel has strict generic constraints for compile-time known values
// but we're working with runtime configuration strings
Expand Down Expand Up @@ -248,7 +248,7 @@ export class PiAgentSdkProvider implements Provider {
messages: agentMessages,
systemPrompt,
model: this.config.model,
provider: this.config.provider,
subprovider: this.config.subprovider,
},
output,
tokenUsage,
Expand Down
6 changes: 3 additions & 3 deletions packages/core/src/evaluation/providers/pi-coding-agent.ts
Original file line number Diff line number Diff line change
Expand Up @@ -165,8 +165,8 @@ export class PiCodingAgentProvider implements Provider {
const args: string[] = [];

// Provider and model configuration
if (this.config.provider) {
args.push('--provider', this.config.provider);
if (this.config.subprovider) {
args.push('--provider', this.config.subprovider);
}
if (this.config.model) {
args.push('--model', this.config.model);
Expand Down Expand Up @@ -252,7 +252,7 @@ export class PiCodingAgentProvider implements Provider {

// Map provider-specific API key to the correct env var
if (this.config.apiKey) {
const provider = this.config.provider?.toLowerCase() ?? 'google';
const provider = this.config.subprovider?.toLowerCase() ?? 'google';
switch (provider) {
case 'google':
case 'gemini':
Expand Down
35 changes: 22 additions & 13 deletions packages/core/src/evaluation/providers/targets.ts
Original file line number Diff line number Diff line change
Expand Up @@ -487,7 +487,7 @@ export interface CopilotSdkResolvedConfig {

export interface PiCodingAgentResolvedConfig {
readonly executable: string;
readonly provider?: string;
readonly subprovider?: string;
readonly model?: string;
readonly apiKey?: string;
readonly tools?: string;
Expand All @@ -502,7 +502,7 @@ export interface PiCodingAgentResolvedConfig {
}

export interface PiAgentSdkResolvedConfig {
readonly provider?: string;
readonly subprovider?: string;
readonly model?: string;
readonly apiKey?: string;
readonly timeoutMs?: number;
Expand Down Expand Up @@ -1416,7 +1416,9 @@ function resolvePiCodingAgentConfig(
evalFilePath?: string,
): PiCodingAgentResolvedConfig {
const executableSource = target.executable ?? target.command ?? target.binary;
const providerSource = target.pi_provider ?? target.piProvider ?? target.llm_provider;
// subprovider is canonical; pi_provider, piProvider, llm_provider are deprecated aliases
const subproviderSource =
target.subprovider ?? target.pi_provider ?? target.piProvider ?? target.llm_provider;
const modelSource = target.model ?? target.pi_model ?? target.piModel;
const apiKeySource = target.api_key ?? target.apiKey;
const toolsSource = target.tools ?? target.pi_tools ?? target.piTools;
Expand All @@ -1436,10 +1438,15 @@ function resolvePiCodingAgentConfig(
optionalEnv: true,
}) ?? 'pi';

const provider = resolveOptionalString(providerSource, env, `${target.name} pi provider`, {
allowLiteral: true,
optionalEnv: true,
});
const subprovider = resolveOptionalString(
subproviderSource,
env,
`${target.name} pi subprovider`,
{
allowLiteral: true,
optionalEnv: true,
},
);

const model = resolveOptionalString(modelSource, env, `${target.name} pi model`, {
allowLiteral: true,
Expand Down Expand Up @@ -1507,7 +1514,7 @@ function resolvePiCodingAgentConfig(

return {
executable,
provider,
subprovider,
model,
apiKey,
tools,
Expand All @@ -1526,16 +1533,18 @@ function resolvePiAgentSdkConfig(
target: z.infer<typeof BASE_TARGET_SCHEMA>,
env: EnvLookup,
): PiAgentSdkResolvedConfig {
const providerSource = target.pi_provider ?? target.piProvider ?? target.llm_provider;
// subprovider is canonical; pi_provider, piProvider, llm_provider are deprecated aliases
const subproviderSource =
target.subprovider ?? target.pi_provider ?? target.piProvider ?? target.llm_provider;
const modelSource = target.model ?? target.pi_model ?? target.piModel;
const apiKeySource = target.api_key ?? target.apiKey;
const timeoutSource = target.timeout_seconds ?? target.timeoutSeconds;
const systemPromptSource = target.system_prompt ?? target.systemPrompt;

const provider = resolveOptionalString(
providerSource,
const subprovider = resolveOptionalString(
subproviderSource,
env,
`${target.name} pi-agent-sdk provider`,
`${target.name} pi-agent-sdk subprovider`,
{
allowLiteral: true,
optionalEnv: true,
Expand All @@ -1560,7 +1569,7 @@ function resolvePiAgentSdkConfig(
: undefined;

return {
provider,
subprovider,
model,
apiKey,
timeoutMs,
Expand Down