From 3513ab270501c45c925d975e1d1978981b54b150 Mon Sep 17 00:00:00 2001 From: zyysurely Date: Mon, 8 Jun 2026 13:34:19 -0700 Subject: [PATCH 1/5] [fix] -e environment switch support + relative dataset path + not stale agent version --- .../azure.ai.agents/internal/cmd/eval.go | 10 +-- .../internal/cmd/eval_helpers.go | 64 ++++++++++------- .../internal/cmd/eval_helpers_test.go | 28 ++++++-- .../azure.ai.agents/internal/cmd/eval_init.go | 5 +- .../internal/cmd/eval_init_jobs.go | 22 +++++- .../internal/cmd/eval_init_prompts.go | 2 +- .../internal/cmd/eval_init_test.go | 20 +++++- .../azure.ai.agents/internal/cmd/eval_list.go | 8 ++- .../internal/cmd/eval_list_test.go | 7 +- .../azure.ai.agents/internal/cmd/eval_run.go | 23 ++++--- .../azure.ai.agents/internal/cmd/eval_show.go | 6 +- .../internal/cmd/eval_show_test.go | 8 +-- .../internal/cmd/eval_update.go | 4 +- .../azure.ai.agents/internal/cmd/optimize.go | 69 ++++++++++++------- .../internal/cmd/optimize_apply.go | 13 ++-- .../internal/cmd/optimize_cancel.go | 7 +- .../internal/cmd/optimize_cancel_test.go | 5 +- .../internal/cmd/optimize_deploy.go | 20 ++++-- .../internal/cmd/optimize_deploy_test.go | 8 +-- .../internal/cmd/optimize_helpers.go | 68 +++++++++++++----- .../internal/cmd/optimize_helpers_test.go | 59 +++++++++++++--- .../internal/cmd/optimize_list.go | 11 +-- .../internal/cmd/optimize_list_test.go | 5 +- .../internal/cmd/optimize_prompts.go | 29 ++++---- .../internal/cmd/optimize_prompts_test.go | 6 +- .../internal/cmd/optimize_status.go | 16 +++-- .../internal/cmd/optimize_status_test.go | 5 +- .../internal/cmd/optimize_test.go | 20 +++--- 28 files changed, 370 insertions(+), 178 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval.go index 98f7d6aa27c..e14141f32f4 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval.go @@ -72,6 +72,7 @@ type evalResolvedContext struct { // evalContextOptions configures the behavior of resolveEvalContext. type evalContextOptions struct { + envName string // explicit environment name (from -e flag) agent string // explicit agent name (from --agent flag) projectEndpoint string // explicit project endpoint (from --project-endpoint flag) requireAgent bool // fail if agent name cannot be resolved @@ -95,8 +96,8 @@ Subcommands: cmd.AddCommand(newEvalInitCommand(extCtx)) cmd.AddCommand(newEvalRunCommand(extCtx)) cmd.AddCommand(newEvalUpdateCommand(extCtx)) - cmd.AddCommand(newEvalListCommand()) - cmd.AddCommand(newEvalShowCommand()) + cmd.AddCommand(newEvalListCommand(extCtx)) + cmd.AddCommand(newEvalShowCommand(extCtx)) return cmd } @@ -125,9 +126,8 @@ func resolveEvalContext(ctx context.Context, options evalContextOptions) (*evalR // Read the current azd environment once — used for agent info, endpoint, and env name. var envName string - envResp, envErr := azdClient.Environment().GetCurrent(ctx, &azdext.EmptyRequest{}) - if envErr == nil && envResp.Environment != nil { - envName = envResp.Environment.Name + if env := getExistingEnvironment(ctx, options.envName, azdClient); env != nil { + envName = env.Name } getEnvValue := func(key string) string { diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_helpers.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_helpers.go index 07df09a55e4..79700175504 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_helpers.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_helpers.go @@ -90,20 +90,37 @@ func relativeDisplay(absPath, projectDir string) string { return absPath } -// reconcileConfigAgentName reconciles the agent name in a config with the -// environment-resolved name. Environment takes precedence. Returns true if -// the config was changed. Used by both eval run and optimize. -func reconcileConfigAgentName(agent *opt_eval.AgentRef, envName, configSource string) bool { - if envName == "" || agent.Name == "" || agent.Name == envName { - if envName != "" && agent.Name == "" { - agent.Name = envName - } - return false - } - fmt.Printf(" %s agent name in %s (%q) differs from environment (%q) — using environment value\n", - color.YellowString("warning:"), configSource, agent.Name, envName) - agent.Name = envName - return true +// reconcileConfigAgent reconciles the agent name and version in a config +// with the environment-resolved values. Environment takes precedence for both. +// Version is always resolved from the azd environment (via AGENT_{SVC}_VERSION); +// stale values in the config are cleared. Returns true if the config was changed. +func reconcileConfigAgent(agent *opt_eval.AgentRef, envName, envVersion, configSource string) bool { + changed := false + + // --- Name --- + if envName != "" && agent.Name != "" && agent.Name != envName { + fmt.Printf(" %s agent name in %s (%q) differs from environment (%q) — using environment value\n", + color.YellowString("warning:"), configSource, agent.Name, envName) + agent.Name = envName + changed = true + } else if envName != "" && agent.Name == "" { + agent.Name = envName + } + + // --- Version --- + // Environment version always wins. If not set, clear any stale config value + // so the backend defaults to "latest". + if envVersion != "" && agent.Version != envVersion { + agent.Version = envVersion + changed = true + } else if envVersion == "" && agent.Version != "" { + fmt.Printf(" %s ignoring stale agent.version %q in %s — using latest from environment\n", + color.YellowString("warning:"), agent.Version, configSource) + agent.Version = "" + changed = true + } + + return changed } // resolveAgentConfig resolves agent configuration from config metadata @@ -328,6 +345,7 @@ func promptModelSelection( azdClient *azdext.AzdClient, message string, defaultModel string, + envName string, ) (string, error) { choices := buildModelSelectionChoices(defaultModel) defaultIndex := int32(0) @@ -347,7 +365,7 @@ func promptModelSelection( selected := choices[int(*resp.Value)].Value if selected == selectOtherDeploymentValue { - return promptAllDeployments(ctx, azdClient) + return promptAllDeployments(ctx, azdClient, envName) } return selected, nil @@ -372,8 +390,8 @@ func buildModelSelectionChoices(defaultModel string) []*azdext.SelectChoice { // promptAllDeployments fetches all model deployments from the Foundry project // and prompts the user to select one. -func promptAllDeployments(ctx context.Context, azdClient *azdext.AzdClient) (string, error) { - deployments := listDeploymentsFromEnv(ctx, azdClient) +func promptAllDeployments(ctx context.Context, azdClient *azdext.AzdClient, envName string) (string, error) { + deployments := listDeploymentsFromEnv(ctx, azdClient, envName) if len(deployments) == 0 { return "", fmt.Errorf("no model deployments found in the Foundry project") } @@ -410,14 +428,14 @@ func promptAllDeployments(ctx context.Context, azdClient *azdext.AzdClient) (str } // getDeployedModelFromEnv reads the AZURE_AI_MODEL_DEPLOYMENT_NAME from -// the current azd environment. Returns empty string if not available. -func getDeployedModelFromEnv(ctx context.Context, azdClient *azdext.AzdClient) string { - envResp, err := azdClient.Environment().GetCurrent(ctx, &azdext.EmptyRequest{}) - if err != nil || envResp == nil || envResp.Environment == nil { +// the specified (or current) azd environment. Returns empty string if not available. +func getDeployedModelFromEnv(ctx context.Context, azdClient *azdext.AzdClient, envName string) string { + env := getExistingEnvironment(ctx, envName, azdClient) + if env == nil { return "" } v, err := azdClient.Environment().GetValue(ctx, &azdext.GetEnvRequest{ - EnvName: envResp.Environment.Name, + EnvName: env.Name, Key: "AZURE_AI_MODEL_DEPLOYMENT_NAME", }) if err != nil || v.Value == "" { @@ -456,7 +474,7 @@ func promptDatasetSelection( return "", &opt_eval.DatasetRef{Name: value}, nil } - resolved, err := resolveLocalDatasetFile(value, agentProject) + resolved, err := resolveLocalDatasetFile(resolveCwdRelative(value), agentProject) if err != nil { return "", nil, err } diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_helpers_test.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_helpers_test.go index 63d9f743f4c..3b902947572 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_helpers_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_helpers_test.go @@ -39,14 +39,14 @@ func TestRelativeDisplay(t *testing.T) { } } -// ---- reconcileConfigAgentName ---- +// ---- reconcileConfigAgent ---- -func TestReconcileConfigAgentName(t *testing.T) { +func TestReconcileConfigAgent(t *testing.T) { t.Parallel() t.Run("no change when names match", func(t *testing.T) { t.Parallel() agent := &opt_eval.AgentRef{Name: "my-agent"} - changed := reconcileConfigAgentName(agent, "my-agent", "config.yaml") + changed := reconcileConfigAgent(agent, "my-agent", "", "config.yaml") assert.False(t, changed) assert.Equal(t, "my-agent", agent.Name) }) @@ -54,7 +54,7 @@ func TestReconcileConfigAgentName(t *testing.T) { t.Run("sets name when agent name is empty", func(t *testing.T) { t.Parallel() agent := &opt_eval.AgentRef{} - changed := reconcileConfigAgentName(agent, "env-agent", "config.yaml") + changed := reconcileConfigAgent(agent, "env-agent", "", "config.yaml") assert.False(t, changed) assert.Equal(t, "env-agent", agent.Name) }) @@ -62,7 +62,7 @@ func TestReconcileConfigAgentName(t *testing.T) { t.Run("overrides when names differ", func(t *testing.T) { t.Parallel() agent := &opt_eval.AgentRef{Name: "config-agent"} - changed := reconcileConfigAgentName(agent, "env-agent", "config.yaml") + changed := reconcileConfigAgent(agent, "env-agent", "", "config.yaml") assert.True(t, changed) assert.Equal(t, "env-agent", agent.Name) }) @@ -70,10 +70,26 @@ func TestReconcileConfigAgentName(t *testing.T) { t.Run("no change when envName is empty", func(t *testing.T) { t.Parallel() agent := &opt_eval.AgentRef{Name: "my-agent"} - changed := reconcileConfigAgentName(agent, "", "config.yaml") + changed := reconcileConfigAgent(agent, "", "", "config.yaml") assert.False(t, changed) assert.Equal(t, "my-agent", agent.Name) }) + + t.Run("clears stale version when env has none", func(t *testing.T) { + t.Parallel() + agent := &opt_eval.AgentRef{Name: "a", Version: "old-v"} + changed := reconcileConfigAgent(agent, "a", "", "config.yaml") + assert.True(t, changed) + assert.Empty(t, agent.Version) + }) + + t.Run("env version overrides config version", func(t *testing.T) { + t.Parallel() + agent := &opt_eval.AgentRef{Name: "a", Version: "old-v"} + changed := reconcileConfigAgent(agent, "a", "new-v", "config.yaml") + assert.True(t, changed) + assert.Equal(t, "new-v", agent.Version) + }) } // ---- statusLabelAndColor ---- diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_init.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_init.go index 126e6fced5e..c8e33277e03 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_init.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_init.go @@ -30,6 +30,7 @@ const DataGenerationAPIVersion = "v1" // evalInitFlags holds CLI flags and interactive prompt state for eval init. type evalInitFlags struct { // CLI flags. + envName string // explicit environment name (from -e flag) name string // eval suite name agent string // target agent name projectEndpoint string // Foundry project endpoint @@ -73,6 +74,7 @@ the agent project root. Use --no-wait to write pending operation IDs and return. defer logCleanup() flags.evalModelSet = cmd.Flags().Changed("eval-model") flags.maxSamplesSet = cmd.Flags().Changed("max-samples") + flags.envName = extCtx.Environment return runEvalInit(ctx, flags, extCtx.NoPrompt) }, } @@ -110,6 +112,7 @@ func runEvalInit(ctx context.Context, flags *evalInitFlags, noPrompt bool) error } resolved, err := resolveEvalContext(ctx, evalContextOptions{ + envName: flags.envName, agent: flags.agent, projectEndpoint: flags.projectEndpoint, requireAgent: true, @@ -306,7 +309,7 @@ func submitEvalJobs( needDatasetGen = flags.dataset == "" needEvalGen = true if !needDatasetGen { - datasetPath, err := resolveLocalDatasetFile(flags.dataset, resolved.agentProject) + datasetPath, err := resolveLocalDatasetFile(resolveCwdRelative(flags.dataset), resolved.agentProject) if err != nil { return nil, err } diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_init_jobs.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_init_jobs.go index cba4147600d..97b033e48e2 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_init_jobs.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_init_jobs.go @@ -47,9 +47,11 @@ func resolvedInstruction(flags *evalInitFlags) string { // newEvalConfig builds an evalConfig from flags and resolved context, applying defaults as needed. func newEvalConfig(flags *evalInitFlags, resolved *evalResolvedContext) *evalConfig { agent := evalAgentRef{ - Name: resolved.agentName, - Kind: resolved.agentKind, - Version: resolved.version, + Name: resolved.agentName, + Kind: resolved.agentKind, + // Version is intentionally omitted — it is resolved at run time + // from the azd environment (AGENT_{SVC}_VERSION) so eval.yaml + // never contains a stale version that drifts after redeployment. } if flags.configFile != "" { agent.ConfigFile = flags.configFile @@ -110,6 +112,20 @@ func submitEvaluatorGeneration( return resolved.evalClient.CreateEvaluatorGenerationJob(ctx, request, ProjectEndpointAPIVersion) } +// resolveCwdRelative converts a relative path to an absolute path based on +// the current working directory. Already-absolute paths are returned as-is. +// This should be called on CLI flag values before passing them to +// resolveLocalDatasetFile, which resolves against the agent project directory. +func resolveCwdRelative(path string) string { + if filepath.IsAbs(path) { + return path + } + if abs, err := filepath.Abs(path); err == nil { + return abs + } + return path +} + // resolveLocalDatasetFile resolves the dataset flag value to an absolute path // for the local JSONL file. If the value is relative it is resolved against // the agent project directory. diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_init_prompts.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_init_prompts.go index 4c93ff0023b..484066494c7 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_init_prompts.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_init_prompts.go @@ -151,7 +151,7 @@ func promptEvalInitOptions(ctx context.Context, resolved *evalResolvedContext, f } } - selected, err := promptModelSelection(ctx, azdClient, "Select the model for evaluation and generation", deployedModel) + selected, err := promptModelSelection(ctx, azdClient, "Select the model for evaluation and generation", deployedModel, resolved.envName) if err != nil { return err } diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_init_test.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_init_test.go index 7f90a513829..9eb26c424b2 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_init_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_init_test.go @@ -139,7 +139,7 @@ func TestNewEvalConfig(t *testing.T) { assert.Equal(t, defaultEvalName, cfg.Name) assert.Equal(t, "booking-agent", cfg.Agent.Name) assert.Equal(t, agent_yaml.AgentKindHosted, cfg.Agent.Kind) - assert.Equal(t, "v2", cfg.Agent.Version) + assert.Empty(t, cfg.Agent.Version, "version should not be persisted in eval config") assert.Equal(t, "gpt-4.1", cfg.Options.EvalModel) assert.Equal(t, "Test the booking agent", cfg.Agent.Instruction.Value) assert.Equal(t, 50, cfg.MaxSamples) @@ -479,6 +479,24 @@ func TestResolveLocalDatasetFile_NotFound(t *testing.T) { assert.Contains(t, err.Error(), "not accessible") } +// --------------------------------------------------------------------------- +// resolveCwdRelative +// --------------------------------------------------------------------------- + +func TestResolveCwdRelative_AbsoluteUnchanged(t *testing.T) { + t.Parallel() + abs := filepath.Join(t.TempDir(), "data.jsonl") + result := resolveCwdRelative(abs) + assert.Equal(t, abs, result) +} + +func TestResolveCwdRelative_RelativeResolvedToCwd(t *testing.T) { + t.Parallel() + result := resolveCwdRelative("data.jsonl") + assert.True(t, filepath.IsAbs(result), "expected absolute path, got %q", result) + assert.Equal(t, "data.jsonl", filepath.Base(result)) +} + // --------------------------------------------------------------------------- // tryLoadExistingEvalConfig // --------------------------------------------------------------------------- diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_list.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_list.go index 5861f0a2e28..5be82f87aba 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_list.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_list.go @@ -22,10 +22,11 @@ import ( // evalListFlags holds CLI flags for the eval list command. type evalListFlags struct { - limit int // maximum number of evals to return + envName string // explicit environment name (from -e flag) + limit int // maximum number of evals to return } -func newEvalListCommand() *cobra.Command { +func newEvalListCommand(extCtx *azdext.ExtensionContext) *cobra.Command { flags := &evalListFlags{limit: 10} cmd := &cobra.Command{ Use: "list", @@ -35,6 +36,7 @@ func newEvalListCommand() *cobra.Command { ctx := azdext.WithAccessToken(cmd.Context()) logCleanup := setupDebugLogging(cmd.Flags()) defer logCleanup() + flags.envName = extCtx.Environment return runEvalList(ctx, flags) }, } @@ -49,7 +51,7 @@ type evalRunSummary struct { } func runEvalList(ctx context.Context, flags *evalListFlags) error { - resolved, err := resolveEvalContext(ctx, evalContextOptions{}) + resolved, err := resolveEvalContext(ctx, evalContextOptions{envName: flags.envName}) if err != nil { return err } diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_list_test.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_list_test.go index 53ce6dfcb63..7444d89f9fb 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_list_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_list_test.go @@ -6,6 +6,7 @@ package cmd import ( "testing" + "github.com/azure/azure-dev/cli/azd/pkg/azdext" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -16,7 +17,7 @@ import ( func TestNewEvalListCommand_Flags(t *testing.T) { t.Parallel() - cmd := newEvalListCommand() + cmd := newEvalListCommand(&azdext.ExtensionContext{}) f := cmd.Flags().Lookup("limit") require.NotNil(t, f) @@ -25,13 +26,13 @@ func TestNewEvalListCommand_Flags(t *testing.T) { func TestNewEvalListCommand_NoArgs(t *testing.T) { t.Parallel() - cmd := newEvalListCommand() + cmd := newEvalListCommand(&azdext.ExtensionContext{}) assert.NoError(t, cmd.Args(cmd, nil)) assert.Error(t, cmd.Args(cmd, []string{"extra"})) } func TestNewEvalListCommand_UseString(t *testing.T) { t.Parallel() - cmd := newEvalListCommand() + cmd := newEvalListCommand(&azdext.ExtensionContext{}) assert.Equal(t, "list", cmd.Use) } diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_run.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_run.go index 64015942165..9ee3152bd8c 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_run.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_run.go @@ -26,9 +26,10 @@ import ( // evalRunFlags holds CLI flags for the eval run command. type evalRunFlags struct { - config string // eval config path - name string // eval run name - noWait bool // start and return immediately + envName string // explicit environment name (from -e flag) + config string // eval config path + name string // eval run name + noWait bool // start and return immediately } func newEvalRunCommand(extCtx *azdext.ExtensionContext) *cobra.Command { @@ -41,6 +42,7 @@ func newEvalRunCommand(extCtx *azdext.ExtensionContext) *cobra.Command { ctx := azdext.WithAccessToken(cmd.Context()) logCleanup := setupDebugLogging(cmd.Flags()) defer logCleanup() + flags.envName = extCtx.Environment return runEvalRun(ctx, flags, extCtx.NoPrompt) }, } @@ -51,7 +53,7 @@ func newEvalRunCommand(extCtx *azdext.ExtensionContext) *cobra.Command { } func runEvalRun(ctx context.Context, flags *evalRunFlags, noPrompt bool) error { - resolved, err := resolveEvalContext(ctx, evalContextOptions{}) + resolved, err := resolveEvalContext(ctx, evalContextOptions{envName: flags.envName}) if err != nil { return err } @@ -65,17 +67,12 @@ func runEvalRun(ctx context.Context, flags *evalRunFlags, noPrompt bool) error { // Reconcile agent name/version between environment and eval.yaml. // Environment values take precedence; warn and update the config if they differ. - configChanged := reconcileConfigAgentName(&evalCfg.Agent, resolved.agentName, flags.config) + configChanged := reconcileConfigAgent(&evalCfg.Agent, resolved.agentName, resolved.version, flags.config) if resolved.agentName == "" { resolved.agentName = evalCfg.Agent.Name } if resolved.version == "" { resolved.version = evalCfg.Agent.Version - } else if evalCfg.Agent.Version != "" && evalCfg.Agent.Version != resolved.version { - fmt.Printf(" %s agent version in %s (%q) differs from environment (%q) — using environment value\n", - color.YellowString("warning:"), flags.config, evalCfg.Agent.Version, resolved.version) - evalCfg.Agent.Version = resolved.version - configChanged = true } if configChanged { if err := eval_api.WriteEvalConfig(configPath, evalCfg); err != nil { @@ -136,7 +133,11 @@ func runEvalRun(ctx context.Context, flags *evalRunFlags, noPrompt bool) error { // Set source from local dataset file or remote dataset reference. if evalCfg.DatasetFile != "" { - items, err := loadJSONLFile[map[string]any](evalCfg.DatasetFile) + // Resolve relative paths against the agent project directory so + // eval.yaml files with project-relative dataset_file entries work + // regardless of the caller's working directory. + datasetPath := eval_api.ResolveRelPath(evalCfg.DatasetFile, resolved.agentProject) + items, err := loadJSONLFile[map[string]any](datasetPath) if err != nil { return err } diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_show.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_show.go index a7b7a76cffc..5b35b3e4095 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_show.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_show.go @@ -22,12 +22,13 @@ import ( // evalShowFlags holds CLI flags for the eval show command. type evalShowFlags struct { + envName string // explicit environment name (from -e flag) evalRunID string // specific eval run to show limit int // maximum number of runs to display output string // export results to JSON file } -func newEvalShowCommand() *cobra.Command { +func newEvalShowCommand(extCtx *azdext.ExtensionContext) *cobra.Command { flags := &evalShowFlags{limit: 20} cmd := &cobra.Command{ Use: "show [eval-id]", @@ -45,6 +46,7 @@ If eval-id is omitted, the most recent eval from the current environment is used if len(args) > 0 { evalID = args[0] } + flags.envName = extCtx.Environment return runEvalShow(ctx, evalID, flags) }, } @@ -55,7 +57,7 @@ If eval-id is omitted, the most recent eval from the current environment is used } func runEvalShow(ctx context.Context, evalID string, flags *evalShowFlags) error { - resolved, err := resolveEvalContext(ctx, evalContextOptions{}) + resolved, err := resolveEvalContext(ctx, evalContextOptions{envName: flags.envName}) if err != nil { return err } diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_show_test.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_show_test.go index 27fe3187b00..7bb17f13dbe 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_show_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_show_test.go @@ -15,13 +15,13 @@ import ( func TestNewEvalShowCommand_UseString(t *testing.T) { t.Parallel() - cmd := newEvalShowCommand() + cmd := newEvalShowCommand(&azdext.ExtensionContext{}) assert.Equal(t, "show [eval-id]", cmd.Use) } func TestNewEvalShowCommand_Flags(t *testing.T) { t.Parallel() - cmd := newEvalShowCommand() + cmd := newEvalShowCommand(&azdext.ExtensionContext{}) tests := []struct { name string @@ -49,14 +49,14 @@ func TestNewEvalShowCommand_Flags(t *testing.T) { func TestNewEvalShowCommand_AcceptsOptionalPositionalArg(t *testing.T) { t.Parallel() - cmd := newEvalShowCommand() + cmd := newEvalShowCommand(&azdext.ExtensionContext{}) // MaximumNArgs(1) — should accept 0 args without error from arg validation. assert.NotNil(t, cmd.Args) } func TestNewEvalShowCommand_HasOutFileShorthand(t *testing.T) { t.Parallel() - cmd := newEvalShowCommand() + cmd := newEvalShowCommand(&azdext.ExtensionContext{}) f := cmd.Flags().Lookup("out-file") require.NotNil(t, f) assert.Equal(t, "O", f.Shorthand) diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_update.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_update.go index f717ff00de7..c9596d79906 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_update.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_update.go @@ -24,6 +24,7 @@ import ( // evalUpdateFlags holds CLI flags for the eval update command. type evalUpdateFlags struct { + envName string // explicit environment name (from -e flag) config string // eval config path datasetOnly bool // only update the dataset evaluatorOnly bool // only update evaluators @@ -46,6 +47,7 @@ local changes. Use --dataset-only or --evaluator-only to skip prompts.`, ctx := azdext.WithAccessToken(cmd.Context()) logCleanup := setupDebugLogging(cmd.Flags()) defer logCleanup() + flags.envName = extCtx.Environment return runEvalUpdate(ctx, flags, extCtx.NoPrompt) }, } @@ -56,7 +58,7 @@ local changes. Use --dataset-only or --evaluator-only to skip prompts.`, } func runEvalUpdate(ctx context.Context, flags *evalUpdateFlags, noPrompt bool) error { - resolved, err := resolveEvalContext(ctx, evalContextOptions{}) + resolved, err := resolveEvalContext(ctx, evalContextOptions{envName: flags.envName}) if err != nil { return err } diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize.go index 4ea1c6625ae..1af9986ae86 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize.go @@ -32,6 +32,7 @@ import ( // for an optimization operation. type optimizeAgentContext struct { agentName string // deployed agent name + agentVersion string // deployed agent version (empty = latest) agentProject string // agent project directory (empty if not in an azd project) } @@ -40,7 +41,7 @@ type optimizeAgentContext struct { // 1. Explicit --agent flag // 2. azd project context (resolveAgentService + environment variables) // 3. Error with guidance -func resolveOptimizeAgent(ctx context.Context, flagValue string, noPrompt bool) (*optimizeAgentContext, error) { +func resolveOptimizeAgent(ctx context.Context, flagValue, envName string, noPrompt bool) (*optimizeAgentContext, error) { if flagValue != "" { return &optimizeAgentContext{agentName: flagValue}, nil } @@ -56,16 +57,24 @@ func resolveOptimizeAgent(ctx context.Context, flagValue string, noPrompt bool) agentProject := filepath.Join(project.Path, svc.RelativePath) serviceKey := toServiceKey(svc.Name) - // Read agent name from azd environment - envResp, envErr := azdClient.Environment().GetCurrent(ctx, &azdext.EmptyRequest{}) - if envErr == nil && envResp.Environment != nil { + // Read agent name and version from azd environment + if env := getExistingEnvironment(ctx, envName, azdClient); env != nil { nameKey := fmt.Sprintf("AGENT_%s_NAME", serviceKey) if v, e := azdClient.Environment().GetValue(ctx, &azdext.GetEnvRequest{ - EnvName: envResp.Environment.Name, + EnvName: env.Name, Key: nameKey, }); e == nil && v.Value != "" { + version := "" + versionKey := fmt.Sprintf("AGENT_%s_VERSION", serviceKey) + if vv, ve := azdClient.Environment().GetValue(ctx, &azdext.GetEnvRequest{ + EnvName: env.Name, + Key: versionKey, + }); ve == nil { + version = vv.Value + } return &optimizeAgentContext{ agentName: v.Value, + agentVersion: version, agentProject: agentProject, }, nil } @@ -92,7 +101,7 @@ type optimizeFlags struct { // newOptimizeCommand creates the top-level "optimize" command and registers its subcommands. func newOptimizeCommand(extCtx *azdext.ExtensionContext) *cobra.Command { flags := &optimizeFlags{} - action := &OptimizeAction{flags: flags, noPrompt: extCtx.NoPrompt} + action := &OptimizeAction{flags: flags, envName: extCtx.Environment, noPrompt: extCtx.NoPrompt} cmd := &cobra.Command{ Use: "optimize [agent-name]", @@ -140,11 +149,11 @@ Use --config for a custom YAML spec, or just provide the agent name to use sensi cmd.Flags().IntVar(&flags.pollInterval, "poll-interval", 5, "Polling interval in seconds") flags.optimizeConnectionFlags.register(cmd) - cmd.AddCommand(newOptimizeStatusCommand()) - cmd.AddCommand(newOptimizeListCommand()) - cmd.AddCommand(newOptimizeCancelCommand()) + cmd.AddCommand(newOptimizeStatusCommand(extCtx)) + cmd.AddCommand(newOptimizeListCommand(extCtx)) + cmd.AddCommand(newOptimizeCancelCommand(extCtx)) cmd.AddCommand(newOptimizeApplyCommand(extCtx)) - cmd.AddCommand(newOptimizeDeployCommand()) + cmd.AddCommand(newOptimizeDeployCommand(extCtx)) return cmd } @@ -152,6 +161,7 @@ Use --config for a custom YAML spec, or just provide the agent name to use sensi // OptimizeAction implements the optimize (submit job) command. type OptimizeAction struct { flags *optimizeFlags + envName string noPrompt bool } @@ -164,7 +174,7 @@ func (a *OptimizeAction) Run(ctx context.Context, cmd *cobra.Command) error { " Consider pinning to a specific version before starting optimization.\n\n", color.YellowString("Warning:")) - endpoint, err := a.flags.resolve(ctx) + endpoint, err := a.flags.resolve(ctx, a.envName) if err != nil { return err } @@ -196,7 +206,7 @@ func (a *OptimizeAction) Run(ctx context.Context, cmd *cobra.Command) error { if err != nil { return err } - printOptimizeResults(ctx, out, finalStatus, hasProject) + printOptimizeResults(ctx, out, finalStatus, hasProject, a.envName) } return nil @@ -214,17 +224,17 @@ func (a *OptimizeAction) resolveConfig( return nil, "", "", fmt.Errorf("%w\n\nCheck that the file path is correct and contains valid YAML", err) } - // Even with explicit --config, try to reconcile agent name with the environment. - resolved, resolveErr := resolveOptimizeAgent(ctx, a.flags.agent, a.noPrompt) + // Even with explicit --config, try to reconcile agent name/version with the environment. + resolved, resolveErr := resolveOptimizeAgent(ctx, a.flags.agent, a.envName, a.noPrompt) if resolveErr == nil { agentProject = resolved.agentProject - reconcileConfigAgentName(&cfg.Agent, resolved.agentName, a.flags.configFile) + reconcileConfigAgent(&cfg.Agent, resolved.agentName, resolved.agentVersion, a.flags.configFile) } return cfg, a.flags.configFile, agentProject, nil } - resolved, err := resolveOptimizeAgent(ctx, a.flags.agent, a.noPrompt) + resolved, err := resolveOptimizeAgent(ctx, a.flags.agent, a.envName, a.noPrompt) if err != nil { return nil, "", "", err } @@ -262,7 +272,7 @@ func (a *OptimizeAction) resolveConfig( if cfg == nil { cfg = defaultOptimizeConfig(resolved.agentName) } else { - reconcileConfigAgentName(&cfg.Agent, resolved.agentName, configSource) + reconcileConfigAgent(&cfg.Agent, resolved.agentName, resolved.agentVersion, configSource) } return cfg, configSource, agentProject, nil @@ -281,7 +291,7 @@ func (a *OptimizeAction) applyOverrides( cfg.DatasetReference = &opt_eval.DatasetRef{Name: a.flags.dataset} cfg.DatasetFile = "" } else { - resolved, err := resolveLocalDatasetFile(a.flags.dataset, agentProject) + resolved, err := resolveLocalDatasetFile(resolveCwdRelative(a.flags.dataset), agentProject) if err != nil { return err } @@ -322,7 +332,7 @@ func (a *OptimizeAction) applyOverrides( // If the model is still unknown, try the azd environment (set during deploy). if cfg.Agent.Model == "" && azdClient != nil { - if m := getDeployedModelFromEnv(ctx, azdClient); m != "" { + if m := getDeployedModelFromEnv(ctx, azdClient, a.envName); m != "" { cfg.Agent.Model = m } } @@ -361,7 +371,7 @@ func (a *OptimizeAction) applyOverrides( // Resolve eval_model: prompt user if not set. if cfg.Options.EvalModel == "" { - if err := resolveOptimizeEvalModel(ctx, azdClient, cfg, a.noPrompt); err != nil { + if err := resolveOptimizeEvalModel(ctx, azdClient, cfg, a.noPrompt, a.envName); err != nil { return err } } @@ -375,14 +385,14 @@ func (a *OptimizeAction) applyOverrides( // Resolve optimization_config.model: prompt user if not set. if !hasModelConfig(cfg.Options.OptimizationConfig) && !a.noPrompt { - if err := resolveOptimizeTargetModels(ctx, azdClient, cfg); err != nil { + if err := resolveOptimizeTargetModels(ctx, azdClient, cfg, a.envName); err != nil { return err } } // Resolve optimization_model: prompt user if not set. if cfg.Options.OptimizationModel == "" && !a.noPrompt { - if err := resolveOptimizeOptimizationModel(ctx, azdClient, cfg); err != nil { + if err := resolveOptimizeOptimizationModel(ctx, azdClient, cfg, a.envName); err != nil { return err } } @@ -440,6 +450,13 @@ func (a *OptimizeAction) submitJob( client := optimize_api.NewOptimizeClient(endpoint, credential) + // Resolve relative dataset path against the agent project directory so + // configs with project-relative dataset_file entries work regardless of + // the caller's working directory. + if cfg.DatasetFile != "" && !filepath.IsAbs(cfg.DatasetFile) && agentProject != "" { + cfg.DatasetFile = filepath.Join(agentProject, cfg.DatasetFile) + } + optimizeReq, warnings, err := cfg.ToRequest() if err != nil { return nil, nil, fmt.Errorf("failed to build optimization request: %w", err) @@ -477,10 +494,10 @@ func (a *OptimizeAction) submitJob( fmt.Fprintf(out, " Job ID: %s\n", color.CyanString(resp.OperationID)) fmt.Fprintf(out, " Status: %s\n", resp.Status) - printOptimizePortalLink(ctx, out, cfg.Agent.Name, resp.OperationID) + printOptimizePortalLink(ctx, out, cfg.Agent.Name, resp.OperationID, a.envName) fmt.Fprintln(out) - saveLastOptimizeJobID(ctx, resp.OperationID) + saveLastOptimizeJobID(ctx, resp.OperationID, a.envName) return resp, client, nil } @@ -534,7 +551,7 @@ func pollOptimizeJob( } // printOptimizeResults prints the optimization results table and next-step commands. -func printOptimizeResults(ctx context.Context, out io.Writer, status *optimize_api.OptimizeJobStatus, hasProject bool) { +func printOptimizeResults(ctx context.Context, out io.Writer, status *optimize_api.OptimizeJobStatus, hasProject bool, envName string) { if status.Error != nil { fmt.Fprintf(out, "\n %s %s\n", color.RedString("Error:"), status.Error.Message) } @@ -548,7 +565,7 @@ func printOptimizeResults(ctx context.Context, out io.Writer, status *optimize_a _, _ = bold.Fprintln(out, "\nResults:") // Resolve eval portal prefix once for building hyperlinks in the table. - evalURLs := buildCandidateEvalURLs(ctx, status.Candidates) + evalURLs := buildCandidateEvalURLs(ctx, status.Candidates, envName) hasEvalLinks := len(evalURLs) > 0 header := fmt.Sprintf(" %-20s %7s %7s", "Candidate", "Score", "Pass") diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_apply.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_apply.go index 68611d4079c..165237d84ad 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_apply.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_apply.go @@ -41,7 +41,7 @@ type optimizeApplyFlags struct { func newOptimizeApplyCommand(extCtx *azdext.ExtensionContext) *cobra.Command { flags := &optimizeApplyFlags{} - action := &OptimizeApplyAction{flags: flags, noPrompt: extCtx.NoPrompt} + action := &OptimizeApplyAction{flags: flags, envName: extCtx.Environment, noPrompt: extCtx.NoPrompt} cmd := &cobra.Command{ Use: "apply", @@ -72,6 +72,7 @@ After applying, run 'azd deploy' to deploy the optimized agent version.`, // OptimizeApplyAction implements the optimize apply command. type OptimizeApplyAction struct { flags *optimizeApplyFlags + envName string noPrompt bool } @@ -105,7 +106,7 @@ func (a *OptimizeApplyAction) apply( out io.Writer, bold *color.Color, ) error { - projectEndpoint, err := resolveProjectEndpointForDeploy(ctx, &a.flags.optimizeConnectionFlags) + projectEndpoint, err := resolveProjectEndpointForDeploy(ctx, &a.flags.optimizeConnectionFlags, a.envName) if err != nil { return err } @@ -159,14 +160,14 @@ func (a *OptimizeApplyAction) apply( // Step 4: Store candidate ID in the azd environment for tracking. serviceKey := toServiceKey(svc.Name) - envResp, err := azdClient.Environment().GetCurrent(ctx, &azdext.EmptyRequest{}) - if err != nil { - return fmt.Errorf("failed to get current environment: %w", err) + env := getExistingEnvironment(ctx, a.envName, azdClient) + if env == nil { + return fmt.Errorf("failed to resolve environment") } candidateKey := fmt.Sprintf("AGENT_%s_OPTIMIZATION_CANDIDATE_ID", serviceKey) if _, err := azdClient.Environment().SetValue(ctx, &azdext.SetEnvRequest{ - EnvName: envResp.Environment.Name, + EnvName: env.Name, Key: candidateKey, Value: a.flags.candidate, }); err != nil { diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_cancel.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_cancel.go index 597f3c8ce72..dfb2999bc85 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_cancel.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_cancel.go @@ -11,16 +11,18 @@ import ( "azureaiagent/internal/pkg/agents/optimize_api" + "github.com/azure/azure-dev/cli/azd/pkg/azdext" "github.com/fatih/color" "github.com/spf13/cobra" ) // optimizeCancelFlags holds connection settings for the cancel command. type optimizeCancelFlags struct { + envName string optimizeConnectionFlags } -func newOptimizeCancelCommand() *cobra.Command { +func newOptimizeCancelCommand(extCtx *azdext.ExtensionContext) *cobra.Command { flags := &optimizeCancelFlags{} cmd := &cobra.Command{ @@ -33,6 +35,7 @@ Only jobs in a non-terminal state (pending, running) can be cancelled.`, azd ai agent optimize cancel opt_abc123`, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { + flags.envName = extCtx.Environment return runOptimizeCancel(cmd, flags, args[0]) }, } @@ -43,7 +46,7 @@ Only jobs in a non-terminal state (pending, running) can be cancelled.`, } func runOptimizeCancel(cmd *cobra.Command, flags *optimizeCancelFlags, operationID string) error { - endpoint, err := flags.resolve(cmd.Context()) + endpoint, err := flags.resolve(cmd.Context(), flags.envName) if err != nil { return err } diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_cancel_test.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_cancel_test.go index af815c2128f..2f8196bef46 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_cancel_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_cancel_test.go @@ -6,11 +6,12 @@ package cmd import ( "testing" + "github.com/azure/azure-dev/cli/azd/pkg/azdext" "github.com/stretchr/testify/assert" ) func TestOptimizeCancelCommand_RequiresPositionalArg(t *testing.T) { - cmd := newOptimizeCancelCommand() + cmd := newOptimizeCancelCommand(&azdext.ExtensionContext{}) err := cmd.Args(cmd, []string{}) assert.Error(t, err) @@ -23,7 +24,7 @@ func TestOptimizeCancelCommand_RequiresPositionalArg(t *testing.T) { } func TestOptimizeCancelCommand_HasConnectionFlags(t *testing.T) { - cmd := newOptimizeCancelCommand() + cmd := newOptimizeCancelCommand(&azdext.ExtensionContext{}) assert.NotNil(t, cmd.Flags().Lookup("endpoint")) assert.NotNil(t, cmd.Flags().Lookup("project-endpoint")) diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_deploy.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_deploy.go index ab3a07429dc..e6da95724c5 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_deploy.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_deploy.go @@ -33,9 +33,9 @@ type optimizeDeployFlags struct { optimizeConnectionFlags } -func newOptimizeDeployCommand() *cobra.Command { +func newOptimizeDeployCommand(extCtx *azdext.ExtensionContext) *cobra.Command { flags := &optimizeDeployFlags{} - action := &OptimizeDeployAction{flags: flags} + action := &OptimizeDeployAction{flags: flags, envName: extCtx.Environment} cmd := &cobra.Command{ Use: "deploy [agent-name]", @@ -72,7 +72,8 @@ Use 'optimize apply' instead if you want to localize the config into your azd pr // OptimizeDeployAction implements the optimize deploy command. type OptimizeDeployAction struct { - flags *optimizeDeployFlags + flags *optimizeDeployFlags + envName string } func (a *OptimizeDeployAction) Run(ctx context.Context, cmd *cobra.Command) error { @@ -90,14 +91,14 @@ func (a *OptimizeDeployAction) runDirect( bold *color.Color, ) error { // Resolve agent name from flag or agent.yaml in current directory. - resolved, err := resolveOptimizeAgent(ctx, a.flags.agent, false) + resolved, err := resolveOptimizeAgent(ctx, a.flags.agent, a.envName, false) if err != nil { return err } agentName := resolved.agentName // Resolve project endpoint (for Foundry agent API). - projectEndpoint, err := resolveProjectEndpointForDeploy(ctx, &a.flags.optimizeConnectionFlags) + projectEndpoint, err := resolveProjectEndpointForDeploy(ctx, &a.flags.optimizeConnectionFlags, a.envName) if err != nil { return err } @@ -241,11 +242,18 @@ func upsertAgentYamlEnvVar(agentYamlPath, key, value string) error { // resolveProjectEndpointForDeploy resolves the Foundry project endpoint using // the same resolution chain as other agent commands. -func resolveProjectEndpointForDeploy(ctx context.Context, connFlags *optimizeConnectionFlags) (string, error) { +func resolveProjectEndpointForDeploy(ctx context.Context, connFlags *optimizeConnectionFlags, envName string) (string, error) { if connFlags.projectEndpoint != "" { return strings.TrimRight(connFlags.projectEndpoint, "/"), nil } + // When an explicit envName is provided, try the named environment first. + if envName != "" { + if ep := endpointFromNamedEnv(ctx, envName); ep != "" { + return strings.TrimRight(ep, "/"), nil + } + } + projectEndpoint, err := resolveAgentEndpoint(ctx, "", "") if err != nil { if ep := projectEndpointFromEnv(); ep != "" { diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_deploy_test.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_deploy_test.go index 5b58f43eb66..c725fe3cfed 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_deploy_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_deploy_test.go @@ -14,7 +14,7 @@ import ( ) func TestOptimizeDeployCommand_HasRequiredFlags(t *testing.T) { - cmd := newOptimizeDeployCommand() + cmd := newOptimizeDeployCommand(&azdext.ExtensionContext{}) candidateFlag := cmd.Flags().Lookup("candidate") require.NotNil(t, candidateFlag, "--candidate flag should be registered") @@ -24,7 +24,7 @@ func TestOptimizeDeployCommand_HasRequiredFlags(t *testing.T) { } func TestOptimizeDeployCommand_CandidateIsRequired(t *testing.T) { - cmd := newOptimizeDeployCommand() + cmd := newOptimizeDeployCommand(&azdext.ExtensionContext{}) // Set only --agent, omit --candidate cmd.SetArgs([]string{"--agent", "my-agent"}) @@ -34,7 +34,7 @@ func TestOptimizeDeployCommand_CandidateIsRequired(t *testing.T) { } func TestOptimizeDeployCommand_AgentResolvedFromFlagOrYaml(t *testing.T) { - cmd := newOptimizeDeployCommand() + cmd := newOptimizeDeployCommand(&azdext.ExtensionContext{}) // --agent is no longer MarkFlagRequired; it falls back to agent.yaml agentFlag := cmd.Flags().Lookup("agent") @@ -47,7 +47,7 @@ func TestOptimizeDeployCommand_AgentResolvedFromFlagOrYaml(t *testing.T) { } func TestOptimizeDeployCommand_HasConnectionFlags(t *testing.T) { - cmd := newOptimizeDeployCommand() + cmd := newOptimizeDeployCommand(&azdext.ExtensionContext{}) assert.NotNil(t, cmd.Flags().Lookup("endpoint")) assert.NotNil(t, cmd.Flags().Lookup("project-endpoint")) diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_helpers.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_helpers.go index 44d48c74768..fb783a8cd1b 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_helpers.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_helpers.go @@ -49,7 +49,8 @@ func projectEndpointFromEnv() string { } // Priority: --endpoint flag → --project-endpoint → azd environment → FOUNDRY_PROJECT_ENDPOINT env var. -func (f *optimizeConnectionFlags) resolve(ctx context.Context) (string, error) { +// envName selects a specific azd environment; empty means "current default". +func (f *optimizeConnectionFlags) resolve(ctx context.Context, envName string) (string, error) { if f.endpoint != "" { return strings.TrimRight(f.endpoint, "/"), nil } @@ -59,7 +60,16 @@ func (f *optimizeConnectionFlags) resolve(ctx context.Context) (string, error) { return strings.TrimRight(f.projectEndpoint, "/"), nil } - // Try azd environment (works when running under azd) + // When an explicit envName is provided, read FOUNDRY_PROJECT_ENDPOINT + // directly from that environment instead of relying on the default + // cascade (which always reads the current/default environment). + if envName != "" { + if ep := endpointFromNamedEnv(ctx, envName); ep != "" { + return strings.TrimRight(ep, "/"), nil + } + } + + // Try azd environment / global config cascade (works when running under azd) projectEndpoint, err := resolveAgentEndpoint(ctx, "", "") if err != nil { // Fall back to FOUNDRY_PROJECT_ENDPOINT or AZURE_AI_PROJECT_ENDPOINT env var (works standalone) @@ -74,25 +84,48 @@ func (f *optimizeConnectionFlags) resolve(ctx context.Context) (string, error) { return projectEndpoint, nil } +// endpointFromNamedEnv reads FOUNDRY_PROJECT_ENDPOINT from the specified +// azd environment. Returns empty string on any failure. +func endpointFromNamedEnv(ctx context.Context, envName string) string { + azdClient, err := azdext.NewAzdClient() + if err != nil { + return "" + } + defer azdClient.Close() + + env := getExistingEnvironment(ctx, envName, azdClient) + if env == nil { + return "" + } + v, err := azdClient.Environment().GetValue(ctx, &azdext.GetEnvRequest{ + EnvName: env.Name, + Key: "FOUNDRY_PROJECT_ENDPOINT", + }) + if err != nil || v.Value == "" { + return "" + } + return v.Value +} + // optimizeLastJobIDKey is the azd environment key for the last optimization job ID. const optimizeLastJobIDKey = "OPTIMIZE_LAST_OPERATION_ID" // saveLastOptimizeJobID stores the operation ID in the azd environment. // Best-effort — silently ignores errors (e.g., when running outside azd). -func saveLastOptimizeJobID(ctx context.Context, operationID string) { +func saveLastOptimizeJobID(ctx context.Context, operationID, envName string) { azdClient, err := azdext.NewAzdClient() if err != nil { return } defer azdClient.Close() - envResp, err := azdClient.Environment().GetCurrent(ctx, &azdext.EmptyRequest{}) - if err != nil || envResp == nil { + env := getExistingEnvironment(ctx, envName, azdClient) + if env == nil { return } _, _ = azdClient.Environment().SetValue(ctx, &azdext.SetEnvRequest{ - EnvName: envResp.Environment.Name, + EnvName: env.Name, Key: optimizeLastJobIDKey, Value: operationID, }) @@ -100,20 +133,20 @@ func saveLastOptimizeJobID(ctx context.Context, operationID string) { // loadLastOptimizeJobID retrieves the last operation ID from the azd environment. // Returns empty string if not available. -func loadLastOptimizeJobID(ctx context.Context) string { +func loadLastOptimizeJobID(ctx context.Context, envName string) string { azdClient, err := azdext.NewAzdClient() if err != nil { return "" } defer azdClient.Close() - envResp, err := azdClient.Environment().GetCurrent(ctx, &azdext.EmptyRequest{}) - if err != nil || envResp == nil { + env := getExistingEnvironment(ctx, envName, azdClient) + if env == nil { return "" } resp, err := azdClient.Environment().GetValue(ctx, &azdext.GetEnvRequest{ - EnvName: envResp.Environment.Name, + EnvName: env.Name, Key: optimizeLastJobIDKey, }) if err != nil || resp == nil { @@ -124,19 +157,19 @@ func loadLastOptimizeJobID(ctx context.Context) string { // printOptimizePortalLink prints the Foundry portal URL for an optimization job. // Best-effort — silently skips if the portal prefix cannot be resolved. -func printOptimizePortalLink(ctx context.Context, out io.Writer, agentName, operationID string) { +func printOptimizePortalLink(ctx context.Context, out io.Writer, agentName, operationID, envName string) { azdClient, err := azdext.NewAzdClient() if err != nil { return } defer azdClient.Close() - envResp, err := azdClient.Environment().GetCurrent(ctx, &azdext.EmptyRequest{}) - if err != nil || envResp == nil { + env := getExistingEnvironment(ctx, envName, azdClient) + if env == nil { return } - printPortalLink(ctx, out, azdClient, envResp.Environment.Name, func(prefix *eval_api.PortalPrefix) string { + printPortalLink(ctx, out, azdClient, env.Name, func(prefix *eval_api.PortalPrefix) string { return prefix.OptimizationURL(agentName, operationID) }) } @@ -159,6 +192,7 @@ func isInAzdProject(ctx context.Context) bool { func buildCandidateEvalURLs( ctx context.Context, candidates []optimize_api.CandidateResult, + explicitEnvName string, ) (result map[string]string) { defer func() { if r := recover(); r != nil { @@ -173,11 +207,11 @@ func buildCandidateEvalURLs( } defer azdClient.Close() - envResp, err := azdClient.Environment().GetCurrent(ctx, &azdext.EmptyRequest{}) - if err != nil || envResp == nil || envResp.Environment == nil { + env := getExistingEnvironment(ctx, explicitEnvName, azdClient) + if env == nil { return nil } - envName := envResp.Environment.Name + envName := env.Name urls := make(map[string]string) for _, c := range candidates { diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_helpers_test.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_helpers_test.go index 40bcb1c8b74..1b0d9a5c0ec 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_helpers_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_helpers_test.go @@ -27,7 +27,7 @@ func TestOptimizeConnectionFlags_Resolve_AllEmpty(t *testing.T) { t.Setenv("AZURE_AI_PROJECT_ENDPOINT", "") t.Setenv("AZD_SERVER", "") f := &optimizeConnectionFlags{} - _, err := f.resolve(t.Context()) + _, err := f.resolve(t.Context(), "") assert.Error(t, err) assert.Contains(t, err.Error(), "endpoint") } @@ -36,7 +36,7 @@ func TestOptimizeConnectionFlags_Resolve_FlagEndpoint(t *testing.T) { f := &optimizeConnectionFlags{ endpoint: "https://from-flag.com", } - endpoint, err := f.resolve(context.Background()) + endpoint, err := f.resolve(context.Background(), "") assert.NoError(t, err) assert.Equal(t, "https://from-flag.com", endpoint) } @@ -45,7 +45,7 @@ func TestOptimizeConnectionFlags_Resolve_TrimsTrailingSlash(t *testing.T) { f := &optimizeConnectionFlags{ endpoint: "https://example.com/", } - endpoint, err := f.resolve(context.Background()) + endpoint, err := f.resolve(context.Background(), "") assert.NoError(t, err) assert.Equal(t, "https://example.com", endpoint) } @@ -54,7 +54,7 @@ func TestOptimizeConnectionFlags_Resolve_ProjectEndpointFlag(t *testing.T) { f := &optimizeConnectionFlags{ projectEndpoint: "https://my-project.services.ai.azure.com/", } - endpoint, err := f.resolve(context.Background()) + endpoint, err := f.resolve(context.Background(), "") assert.NoError(t, err) assert.Equal(t, "https://my-project.services.ai.azure.com", endpoint) } @@ -352,7 +352,7 @@ func TestOptimizeConnectionFlags_Resolve_FoundryEnvVar(t *testing.T) { t.Setenv("AZD_SERVER", "") t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "https://foundry.example.com/") f := &optimizeConnectionFlags{} - endpoint, err := f.resolve(t.Context()) + endpoint, err := f.resolve(t.Context(), "") assert.NoError(t, err) assert.Equal(t, "https://foundry.example.com", endpoint) } @@ -362,7 +362,7 @@ func TestOptimizeConnectionFlags_Resolve_AzureAIEnvVar(t *testing.T) { t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "") t.Setenv("AZURE_AI_PROJECT_ENDPOINT", "https://azure-ai.example.com/") f := &optimizeConnectionFlags{} - endpoint, err := f.resolve(t.Context()) + endpoint, err := f.resolve(t.Context(), "") assert.NoError(t, err) assert.Equal(t, "https://azure-ai.example.com", endpoint) } @@ -372,7 +372,7 @@ func TestOptimizeConnectionFlags_Resolve_FoundryTakesPriorityOverAzureAI(t *test t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "https://foundry.example.com") t.Setenv("AZURE_AI_PROJECT_ENDPOINT", "https://azure-ai.example.com") f := &optimizeConnectionFlags{} - endpoint, err := f.resolve(t.Context()) + endpoint, err := f.resolve(t.Context(), "") assert.NoError(t, err) assert.Equal(t, "https://foundry.example.com", endpoint) } @@ -380,7 +380,7 @@ func TestOptimizeConnectionFlags_Resolve_FoundryTakesPriorityOverAzureAI(t *test func TestResolveProjectEndpointForDeploy_FoundryEnvVar(t *testing.T) { t.Setenv("AZD_SERVER", "") t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "https://foundry-deploy.example.com/") - ep, err := resolveProjectEndpointForDeploy(t.Context(), &optimizeConnectionFlags{}) + ep, err := resolveProjectEndpointForDeploy(t.Context(), &optimizeConnectionFlags{}, "") assert.NoError(t, err) assert.Equal(t, "https://foundry-deploy.example.com", ep) } @@ -389,7 +389,7 @@ func TestResolveProjectEndpointForDeploy_AzureAIEnvVar(t *testing.T) { t.Setenv("AZD_SERVER", "") t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "") t.Setenv("AZURE_AI_PROJECT_ENDPOINT", "https://azure-ai-deploy.example.com/") - ep, err := resolveProjectEndpointForDeploy(t.Context(), &optimizeConnectionFlags{}) + ep, err := resolveProjectEndpointForDeploy(t.Context(), &optimizeConnectionFlags{}, "") assert.NoError(t, err) assert.Equal(t, "https://azure-ai-deploy.example.com", ep) } @@ -398,7 +398,46 @@ func TestResolveProjectEndpointForDeploy_FoundryTakesPriorityOverAzureAI(t *test t.Setenv("AZD_SERVER", "") t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "https://foundry-deploy.example.com") t.Setenv("AZURE_AI_PROJECT_ENDPOINT", "https://azure-ai-deploy.example.com") - ep, err := resolveProjectEndpointForDeploy(t.Context(), &optimizeConnectionFlags{}) + ep, err := resolveProjectEndpointForDeploy(t.Context(), &optimizeConnectionFlags{}, "") assert.NoError(t, err) assert.Equal(t, "https://foundry-deploy.example.com", ep) } + +// --------------------------------------------------------------------------- +// getExistingEnvironment +// --------------------------------------------------------------------------- + +func TestGetExistingEnvironment_EmptyName_UsesCurrent(t *testing.T) { + t.Parallel() + envServer := &testEnvironmentServiceServer{ + current: &azdext.Environment{Name: "dev"}, + } + azdClient := newOptimizeTestAzdClient(t, envServer) + env := getExistingEnvironment(t.Context(), "", azdClient) + require.NotNil(t, env) + assert.Equal(t, "dev", env.Name) +} + +func TestGetExistingEnvironment_ExplicitName_UsesGet(t *testing.T) { + t.Parallel() + envServer := &testEnvironmentServiceServer{ + current: &azdext.Environment{Name: "dev"}, + environments: map[string]*azdext.Environment{ + "staging": {Name: "staging"}, + }, + } + azdClient := newOptimizeTestAzdClient(t, envServer) + env := getExistingEnvironment(t.Context(), "staging", azdClient) + require.NotNil(t, env) + assert.Equal(t, "staging", env.Name) +} + +func TestGetExistingEnvironment_NotFound_ReturnsNil(t *testing.T) { + t.Parallel() + envServer := &testEnvironmentServiceServer{ + environments: map[string]*azdext.Environment{}, + } + azdClient := newOptimizeTestAzdClient(t, envServer) + env := getExistingEnvironment(t.Context(), "nonexistent", azdClient) + assert.Nil(t, env) +} diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_list.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_list.go index 2b79ffe1274..0bd75ab603c 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_list.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_list.go @@ -13,18 +13,20 @@ import ( "azureaiagent/internal/pkg/agents/optimize_api" + "github.com/azure/azure-dev/cli/azd/pkg/azdext" "github.com/fatih/color" "github.com/spf13/cobra" ) // optimizeListFlags holds CLI flags for the optimize list command. type optimizeListFlags struct { - limit int // maximum number of results - status string // filter by job status + envName string // explicit environment name (from -e flag) + limit int // maximum number of results + status string // filter by job status optimizeConnectionFlags } -func newOptimizeListCommand() *cobra.Command { +func newOptimizeListCommand(extCtx *azdext.ExtensionContext) *cobra.Command { flags := &optimizeListFlags{} cmd := &cobra.Command{ @@ -42,6 +44,7 @@ Use --status to filter by job status and --limit to control page size.`, # Show last 5 runs azd ai agent optimize list --limit 5`, RunE: func(cmd *cobra.Command, args []string) error { + flags.envName = extCtx.Environment return runOptimizeList(cmd, flags) }, } @@ -62,7 +65,7 @@ func runOptimizeList(cmd *cobra.Command, flags *optimizeListFlags) error { } } - endpoint, err := flags.resolve(cmd.Context()) + endpoint, err := flags.resolve(cmd.Context(), flags.envName) if err != nil { return err } diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_list_test.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_list_test.go index 4aa5390a9f9..ea60517d564 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_list_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_list_test.go @@ -6,12 +6,13 @@ package cmd import ( "testing" + "github.com/azure/azure-dev/cli/azd/pkg/azdext" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestOptimizeListCommand_AcceptsLimitAndStatusFlags(t *testing.T) { - cmd := newOptimizeListCommand() + cmd := newOptimizeListCommand(&azdext.ExtensionContext{}) limitFlag := cmd.Flags().Lookup("limit") require.NotNil(t, limitFlag, "--limit flag should be registered") @@ -29,7 +30,7 @@ func TestOptimizeListCommand_AcceptsLimitAndStatusFlags(t *testing.T) { } func TestOptimizeListCommand_HasConnectionFlags(t *testing.T) { - cmd := newOptimizeListCommand() + cmd := newOptimizeListCommand(&azdext.ExtensionContext{}) assert.NotNil(t, cmd.Flags().Lookup("endpoint")) assert.NotNil(t, cmd.Flags().Lookup("project-endpoint")) diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_prompts.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_prompts.go index d4a34bc65de..8f5d2e621db 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_prompts.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_prompts.go @@ -309,6 +309,7 @@ func resolveOptimizeEvalModel( azdClient *azdext.AzdClient, cfg *OptimizeConfig, noPrompt bool, + envName string, ) error { if noPrompt { return fmt.Errorf("options.eval_model is required: use --eval-model to specify the evaluation model") @@ -318,9 +319,9 @@ func resolveOptimizeEvalModel( return fmt.Errorf("eval_model is required but cannot prompt") } - deployedModel := getDeployedModelFromEnv(ctx, azdClient) + deployedModel := getDeployedModelFromEnv(ctx, azdClient, envName) - selected, err := promptModelSelection(ctx, azdClient, "Select the model for evaluation", deployedModel) + selected, err := promptModelSelection(ctx, azdClient, "Select the model for evaluation", deployedModel, envName) if err != nil { return err } @@ -373,6 +374,7 @@ func resolveOptimizeTargetModels( ctx context.Context, azdClient *azdext.AzdClient, cfg *OptimizeConfig, + envName string, ) error { if azdClient == nil { return nil @@ -391,7 +393,7 @@ func resolveOptimizeTargetModels( } // Fetch deployed models from the Foundry project. - choices := buildOptimizeModelChoices(ctx, azdClient, currentModel) + choices := buildOptimizeModelChoices(ctx, azdClient, currentModel, envName) message := "Select target models for optimization" if currentModel != "" { @@ -451,13 +453,13 @@ func isRecommendedOptimizationModel(modelName string) bool { // model. Recommended deployments (gpt-5 family, deepseek) are listed first, // followed by the remaining deployments. If the user picks a model not in // the recommended set, a warning is printed. -func resolveOptimizeOptimizationModel(ctx context.Context, azdClient *azdext.AzdClient, cfg *OptimizeConfig) error { +func resolveOptimizeOptimizationModel(ctx context.Context, azdClient *azdext.AzdClient, cfg *OptimizeConfig, envName string) error { if azdClient == nil { return nil } // Fetch deployments once and build a single list: recommended first, then others. - deployments := listDeploymentsFromEnv(ctx, azdClient) + deployments := listDeploymentsFromEnv(ctx, azdClient, envName) var recommended, others []*azdext.SelectChoice seen := make(map[string]bool) @@ -515,8 +517,8 @@ func resolveOptimizeOptimizationModel(ctx context.Context, azdClient *azdext.Azd // MultiSelectChoice items. The baseline model (currentModel) is excluded // from the list since it is already used as the baseline. // Falls back to an empty list if deployments cannot be fetched. -func buildOptimizeModelChoices(ctx context.Context, azdClient *azdext.AzdClient, currentModel string) []*azdext.MultiSelectChoice { - deployments := listDeploymentsFromEnv(ctx, azdClient) +func buildOptimizeModelChoices(ctx context.Context, azdClient *azdext.AzdClient, currentModel, envName string) []*azdext.MultiSelectChoice { + deployments := listDeploymentsFromEnv(ctx, azdClient, envName) var choices []*azdext.MultiSelectChoice seen := make(map[string]bool) @@ -544,16 +546,17 @@ func buildOptimizeModelChoices(ctx context.Context, azdClient *azdext.AzdClient, return choices } -// listDeploymentsFromEnv reads AZURE_AI_PROJECT_ID from the azd environment -// and returns the Foundry project's model deployments. Returns nil on failure. -func listDeploymentsFromEnv(ctx context.Context, azdClient *azdext.AzdClient) []FoundryDeploymentInfo { - envResp, err := azdClient.Environment().GetCurrent(ctx, &azdext.EmptyRequest{}) - if err != nil || envResp == nil || envResp.Environment == nil { +// listDeploymentsFromEnv reads AZURE_AI_PROJECT_ID from the specified (or +// current) azd environment and returns the Foundry project's model +// deployments. Returns nil on failure. +func listDeploymentsFromEnv(ctx context.Context, azdClient *azdext.AzdClient, envName string) []FoundryDeploymentInfo { + env := getExistingEnvironment(ctx, envName, azdClient) + if env == nil { return nil } v, err := azdClient.Environment().GetValue(ctx, &azdext.GetEnvRequest{ - EnvName: envResp.Environment.Name, + EnvName: env.Name, Key: "AZURE_AI_PROJECT_ID", }) if err != nil || v.Value == "" { diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_prompts_test.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_prompts_test.go index 0e09b45b2a7..2e58da7c0f4 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_prompts_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_prompts_test.go @@ -137,7 +137,7 @@ func TestResolveOptimizeEvalModel_NilClient(t *testing.T) { t.Parallel() cfg := &OptimizeConfig{Options: &opt_eval.Options{}} - err := resolveOptimizeEvalModel(t.Context(), nil, cfg, false) + err := resolveOptimizeEvalModel(t.Context(), nil, cfg, false, "") assert.Error(t, err) assert.Contains(t, err.Error(), "cannot prompt") } @@ -159,7 +159,7 @@ func TestResolveOptimizeOptimizationModel_NilClient(t *testing.T) { t.Parallel() cfg := &OptimizeConfig{Options: &opt_eval.Options{}} - err := resolveOptimizeOptimizationModel(t.Context(), nil, cfg) + err := resolveOptimizeOptimizationModel(t.Context(), nil, cfg, "") assert.NoError(t, err) // silently skips assert.Empty(t, cfg.Options.OptimizationModel) } @@ -170,7 +170,7 @@ func TestResolveOptimizeTargetModels_NilClient(t *testing.T) { t.Parallel() cfg := &OptimizeConfig{Options: &opt_eval.Options{}} - err := resolveOptimizeTargetModels(t.Context(), nil, cfg) + err := resolveOptimizeTargetModels(t.Context(), nil, cfg, "") assert.NoError(t, err) // silently skips assert.Nil(t, cfg.Options.OptimizationConfig) } diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_status.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_status.go index 163607da41b..02c64b43cf2 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_status.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_status.go @@ -19,12 +19,13 @@ import ( // optimizeStatusFlags holds CLI flags for the optimize status command. type optimizeStatusFlags struct { - watch bool // poll until job completes - pollInterval int // polling interval in seconds + envName string // explicit environment name (from -e flag) + watch bool // poll until job completes + pollInterval int // polling interval in seconds optimizeConnectionFlags } -func newOptimizeStatusCommand() *cobra.Command { +func newOptimizeStatusCommand(extCtx *azdext.ExtensionContext) *cobra.Command { flags := &optimizeStatusFlags{} cmd := &cobra.Command{ @@ -45,11 +46,12 @@ Use --watch to poll until the job completes.`, Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { ctx := azdext.WithAccessToken(cmd.Context()) + flags.envName = extCtx.Environment operationID := "" if len(args) > 0 { operationID = args[0] } else { - operationID = loadLastOptimizeJobID(ctx) + operationID = loadLastOptimizeJobID(ctx, flags.envName) if operationID == "" { return fmt.Errorf("operation ID is required: provide it as an argument, or run 'azd ai agent optimize' first") } @@ -67,7 +69,7 @@ Use --watch to poll until the job completes.`, } func runOptimizeStatus(cmd *cobra.Command, flags *optimizeStatusFlags, operationID string) error { - endpoint, err := flags.resolve(cmd.Context()) + endpoint, err := flags.resolve(cmd.Context(), flags.envName) if err != nil { return err } @@ -94,9 +96,9 @@ func runOptimizeStatus(cmd *cobra.Command, flags *optimizeStatusFlags, operation if err != nil { return err } - printOptimizeResults(cmd.Context(), out, finalStatus, hasProject) + printOptimizeResults(cmd.Context(), out, finalStatus, hasProject, flags.envName) } else if len(status.Candidates) > 0 { - printOptimizeResults(cmd.Context(), out, status, hasProject) + printOptimizeResults(cmd.Context(), out, status, hasProject, flags.envName) } if status.Error != nil { diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_status_test.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_status_test.go index 7996b6dc7ee..0afa341140a 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_status_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_status_test.go @@ -6,12 +6,13 @@ package cmd import ( "testing" + "github.com/azure/azure-dev/cli/azd/pkg/azdext" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestOptimizeStatusCommand_AcceptsOptionalPositionalArg(t *testing.T) { - cmd := newOptimizeStatusCommand() + cmd := newOptimizeStatusCommand(&azdext.ExtensionContext{}) // Zero args is now OK (uses last job ID) err := cmd.Args(cmd, []string{}) @@ -27,7 +28,7 @@ func TestOptimizeStatusCommand_AcceptsOptionalPositionalArg(t *testing.T) { } func TestOptimizeStatusCommand_HasWatchFlag(t *testing.T) { - cmd := newOptimizeStatusCommand() + cmd := newOptimizeStatusCommand(&azdext.ExtensionContext{}) f := cmd.Flags().Lookup("watch") require.NotNil(t, f, "--watch flag should be registered") diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_test.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_test.go index 3f16c1d6be0..c58b5b2aedc 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_test.go @@ -110,9 +110,9 @@ func TestDefaultOptimizeConfig(t *testing.T) { assert.Equal(t, "builtin.task_adherence", cfg.Evaluators[0].Name) } -// ---- LoadOptimizeConfig + reconcileConfigAgentName (--config path) ---- +// ---- LoadOptimizeConfig + reconcileConfigAgent (--config path) ---- -func TestLoadOptimizeConfig_ReconcileAgentName(t *testing.T) { +func TestLoadOptimizeConfig_ReconcileAgent(t *testing.T) { t.Parallel() writeConfigYAML := func(t *testing.T, dir, agentName string) string { @@ -132,7 +132,7 @@ func TestLoadOptimizeConfig_ReconcileAgentName(t *testing.T) { require.NoError(t, err) assert.Equal(t, "config-agent", cfg.Agent.Name) - changed := reconcileConfigAgentName(&cfg.Agent, "env-agent", cfgPath) + changed := reconcileConfigAgent(&cfg.Agent, "env-agent", "", cfgPath) assert.True(t, changed, "should report change when names differ") assert.Equal(t, "env-agent", cfg.Agent.Name, "environment name should take precedence") }) @@ -145,7 +145,7 @@ func TestLoadOptimizeConfig_ReconcileAgentName(t *testing.T) { cfg, err := LoadOptimizeConfig(cfgPath) require.NoError(t, err) - changed := reconcileConfigAgentName(&cfg.Agent, "same-agent", cfgPath) + changed := reconcileConfigAgent(&cfg.Agent, "same-agent", "", cfgPath) assert.False(t, changed) assert.Equal(t, "same-agent", cfg.Agent.Name) }) @@ -161,7 +161,7 @@ func TestLoadOptimizeConfig_ReconcileAgentName(t *testing.T) { require.NoError(t, err) assert.Empty(t, cfg.Agent.Name) - changed := reconcileConfigAgentName(&cfg.Agent, "env-agent", cfgPath) + changed := reconcileConfigAgent(&cfg.Agent, "env-agent", "", cfgPath) assert.False(t, changed, "filling empty name is not a 'change' (no conflict)") assert.Equal(t, "env-agent", cfg.Agent.Name) }) @@ -174,7 +174,7 @@ func TestLoadOptimizeConfig_ReconcileAgentName(t *testing.T) { cfg, err := LoadOptimizeConfig(cfgPath) require.NoError(t, err) - changed := reconcileConfigAgentName(&cfg.Agent, "", cfgPath) + changed := reconcileConfigAgent(&cfg.Agent, "", "", cfgPath) assert.False(t, changed) assert.Equal(t, "config-agent", cfg.Agent.Name, "original name preserved when env is empty") }) @@ -386,7 +386,7 @@ func TestPrintOptimizeResults_TableHasCandidateScorePass(t *testing.T) { } var buf strings.Builder - printOptimizeResults(t.Context(), &buf, status, false) + printOptimizeResults(t.Context(), &buf, status, false, "") out := buf.String() // Verify header columns. @@ -419,7 +419,7 @@ func TestPrintOptimizeResults_BestMarkedWithStar(t *testing.T) { } var buf strings.Builder - printOptimizeResults(t.Context(), &buf, status, false) + printOptimizeResults(t.Context(), &buf, status, false, "") assert.Contains(t, buf.String(), "candidate_1 ★") } @@ -429,7 +429,7 @@ func TestPrintOptimizeResults_NoCandidates(t *testing.T) { status := &optimize_api.OptimizeJobStatus{} var buf strings.Builder - printOptimizeResults(t.Context(), &buf, status, false) + printOptimizeResults(t.Context(), &buf, status, false, "") // Should print nothing for an empty candidates list. assert.Empty(t, buf.String()) @@ -446,7 +446,7 @@ func TestPrintOptimizeResults_ShowsCandidateIDs(t *testing.T) { } var buf strings.Builder - printOptimizeResults(t.Context(), &buf, status, true) + printOptimizeResults(t.Context(), &buf, status, true, "") out := buf.String() assert.Contains(t, out, "Candidate IDs") From cb9c234f2d7dbeaa71d2cc181da7c2fb0f4c0ee2 Mon Sep 17 00:00:00 2001 From: zyysurely Date: Mon, 8 Jun 2026 14:34:02 -0700 Subject: [PATCH 2/5] fix comments --- .../azure.ai.agents/internal/cmd/eval_helpers.go | 7 ++++--- .../internal/cmd/eval_helpers_test.go | 13 +++++++------ .../azure.ai.agents/internal/cmd/eval_run.go | 3 ++- .../azure.ai.agents/internal/cmd/optimize.go | 4 ++-- .../azure.ai.agents/internal/cmd/optimize_test.go | 9 +++++---- 5 files changed, 20 insertions(+), 16 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_helpers.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_helpers.go index 79700175504..83d340d7c49 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_helpers.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_helpers.go @@ -94,12 +94,13 @@ func relativeDisplay(absPath, projectDir string) string { // with the environment-resolved values. Environment takes precedence for both. // Version is always resolved from the azd environment (via AGENT_{SVC}_VERSION); // stale values in the config are cleared. Returns true if the config was changed. -func reconcileConfigAgent(agent *opt_eval.AgentRef, envName, envVersion, configSource string) bool { +// Warnings are written to w (callers pass os.Stderr; tests pass io.Discard). +func reconcileConfigAgent(w io.Writer, agent *opt_eval.AgentRef, envName, envVersion, configSource string) bool { changed := false // --- Name --- if envName != "" && agent.Name != "" && agent.Name != envName { - fmt.Printf(" %s agent name in %s (%q) differs from environment (%q) — using environment value\n", + fmt.Fprintf(w, " %s agent name in %s (%q) differs from environment (%q) \u2014 using environment value\n", color.YellowString("warning:"), configSource, agent.Name, envName) agent.Name = envName changed = true @@ -114,7 +115,7 @@ func reconcileConfigAgent(agent *opt_eval.AgentRef, envName, envVersion, configS agent.Version = envVersion changed = true } else if envVersion == "" && agent.Version != "" { - fmt.Printf(" %s ignoring stale agent.version %q in %s — using latest from environment\n", + fmt.Fprintf(w, " %s ignoring stale agent.version %q in %s \u2014 using latest from environment\n", color.YellowString("warning:"), agent.Version, configSource) agent.Version = "" changed = true diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_helpers_test.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_helpers_test.go index 3b902947572..1b757a4a6e3 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_helpers_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_helpers_test.go @@ -4,6 +4,7 @@ package cmd import ( + "io" "os" "path/filepath" "testing" @@ -46,7 +47,7 @@ func TestReconcileConfigAgent(t *testing.T) { t.Run("no change when names match", func(t *testing.T) { t.Parallel() agent := &opt_eval.AgentRef{Name: "my-agent"} - changed := reconcileConfigAgent(agent, "my-agent", "", "config.yaml") + changed := reconcileConfigAgent(io.Discard, agent, "my-agent", "", "config.yaml") assert.False(t, changed) assert.Equal(t, "my-agent", agent.Name) }) @@ -54,7 +55,7 @@ func TestReconcileConfigAgent(t *testing.T) { t.Run("sets name when agent name is empty", func(t *testing.T) { t.Parallel() agent := &opt_eval.AgentRef{} - changed := reconcileConfigAgent(agent, "env-agent", "", "config.yaml") + changed := reconcileConfigAgent(io.Discard, agent, "env-agent", "", "config.yaml") assert.False(t, changed) assert.Equal(t, "env-agent", agent.Name) }) @@ -62,7 +63,7 @@ func TestReconcileConfigAgent(t *testing.T) { t.Run("overrides when names differ", func(t *testing.T) { t.Parallel() agent := &opt_eval.AgentRef{Name: "config-agent"} - changed := reconcileConfigAgent(agent, "env-agent", "", "config.yaml") + changed := reconcileConfigAgent(io.Discard, agent, "env-agent", "", "config.yaml") assert.True(t, changed) assert.Equal(t, "env-agent", agent.Name) }) @@ -70,7 +71,7 @@ func TestReconcileConfigAgent(t *testing.T) { t.Run("no change when envName is empty", func(t *testing.T) { t.Parallel() agent := &opt_eval.AgentRef{Name: "my-agent"} - changed := reconcileConfigAgent(agent, "", "", "config.yaml") + changed := reconcileConfigAgent(io.Discard, agent, "", "", "config.yaml") assert.False(t, changed) assert.Equal(t, "my-agent", agent.Name) }) @@ -78,7 +79,7 @@ func TestReconcileConfigAgent(t *testing.T) { t.Run("clears stale version when env has none", func(t *testing.T) { t.Parallel() agent := &opt_eval.AgentRef{Name: "a", Version: "old-v"} - changed := reconcileConfigAgent(agent, "a", "", "config.yaml") + changed := reconcileConfigAgent(io.Discard, agent, "a", "", "config.yaml") assert.True(t, changed) assert.Empty(t, agent.Version) }) @@ -86,7 +87,7 @@ func TestReconcileConfigAgent(t *testing.T) { t.Run("env version overrides config version", func(t *testing.T) { t.Parallel() agent := &opt_eval.AgentRef{Name: "a", Version: "old-v"} - changed := reconcileConfigAgent(agent, "a", "new-v", "config.yaml") + changed := reconcileConfigAgent(io.Discard, agent, "a", "new-v", "config.yaml") assert.True(t, changed) assert.Equal(t, "new-v", agent.Version) }) diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_run.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_run.go index 9ee3152bd8c..8915e9a8adb 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_run.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_run.go @@ -11,6 +11,7 @@ import ( "context" "fmt" "net/url" + "os" "strings" "time" @@ -67,7 +68,7 @@ func runEvalRun(ctx context.Context, flags *evalRunFlags, noPrompt bool) error { // Reconcile agent name/version between environment and eval.yaml. // Environment values take precedence; warn and update the config if they differ. - configChanged := reconcileConfigAgent(&evalCfg.Agent, resolved.agentName, resolved.version, flags.config) + configChanged := reconcileConfigAgent(os.Stderr, &evalCfg.Agent, resolved.agentName, resolved.version, flags.config) if resolved.agentName == "" { resolved.agentName = evalCfg.Agent.Name } diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize.go index 1af9986ae86..16417aef47b 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize.go @@ -228,7 +228,7 @@ func (a *OptimizeAction) resolveConfig( resolved, resolveErr := resolveOptimizeAgent(ctx, a.flags.agent, a.envName, a.noPrompt) if resolveErr == nil { agentProject = resolved.agentProject - reconcileConfigAgent(&cfg.Agent, resolved.agentName, resolved.agentVersion, a.flags.configFile) + reconcileConfigAgent(os.Stderr, &cfg.Agent, resolved.agentName, resolved.agentVersion, a.flags.configFile) } return cfg, a.flags.configFile, agentProject, nil @@ -272,7 +272,7 @@ func (a *OptimizeAction) resolveConfig( if cfg == nil { cfg = defaultOptimizeConfig(resolved.agentName) } else { - reconcileConfigAgent(&cfg.Agent, resolved.agentName, resolved.agentVersion, configSource) + reconcileConfigAgent(os.Stderr, &cfg.Agent, resolved.agentName, resolved.agentVersion, configSource) } return cfg, configSource, agentProject, nil diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_test.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_test.go index c58b5b2aedc..380bc58df93 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_test.go @@ -5,6 +5,7 @@ package cmd import ( "fmt" + "io" "os" "path/filepath" "strings" @@ -132,7 +133,7 @@ func TestLoadOptimizeConfig_ReconcileAgent(t *testing.T) { require.NoError(t, err) assert.Equal(t, "config-agent", cfg.Agent.Name) - changed := reconcileConfigAgent(&cfg.Agent, "env-agent", "", cfgPath) + changed := reconcileConfigAgent(io.Discard, &cfg.Agent, "env-agent", "", cfgPath) assert.True(t, changed, "should report change when names differ") assert.Equal(t, "env-agent", cfg.Agent.Name, "environment name should take precedence") }) @@ -145,7 +146,7 @@ func TestLoadOptimizeConfig_ReconcileAgent(t *testing.T) { cfg, err := LoadOptimizeConfig(cfgPath) require.NoError(t, err) - changed := reconcileConfigAgent(&cfg.Agent, "same-agent", "", cfgPath) + changed := reconcileConfigAgent(io.Discard, &cfg.Agent, "same-agent", "", cfgPath) assert.False(t, changed) assert.Equal(t, "same-agent", cfg.Agent.Name) }) @@ -161,7 +162,7 @@ func TestLoadOptimizeConfig_ReconcileAgent(t *testing.T) { require.NoError(t, err) assert.Empty(t, cfg.Agent.Name) - changed := reconcileConfigAgent(&cfg.Agent, "env-agent", "", cfgPath) + changed := reconcileConfigAgent(io.Discard, &cfg.Agent, "env-agent", "", cfgPath) assert.False(t, changed, "filling empty name is not a 'change' (no conflict)") assert.Equal(t, "env-agent", cfg.Agent.Name) }) @@ -174,7 +175,7 @@ func TestLoadOptimizeConfig_ReconcileAgent(t *testing.T) { cfg, err := LoadOptimizeConfig(cfgPath) require.NoError(t, err) - changed := reconcileConfigAgent(&cfg.Agent, "", "", cfgPath) + changed := reconcileConfigAgent(io.Discard, &cfg.Agent, "", "", cfgPath) assert.False(t, changed) assert.Equal(t, "config-agent", cfg.Agent.Name, "original name preserved when env is empty") }) From 30c2144a35a57e3aa4cd3dd2b1c52809f6df7f81 Mon Sep 17 00:00:00 2001 From: zyysurely Date: Tue, 9 Jun 2026 10:19:26 -0700 Subject: [PATCH 3/5] fix -e --- .../extensions/azure.ai.agents/internal/cmd/optimize.go | 4 ++++ .../azure.ai.agents/internal/cmd/optimize_apply.go | 8 +++++++- .../azure.ai.agents/internal/cmd/optimize_deploy.go | 6 +++++- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize.go index 16417aef47b..befb47fb85f 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize.go @@ -134,6 +134,10 @@ Use --config for a custom YAML spec, or just provide the agent name to use sensi flags.agent = args[0] } + // Read extCtx fields here (after PersistentPreRunE has populated them + // from -e / AZD_ENVIRONMENT), not at command construction time. + action.envName = extCtx.Environment + action.noPrompt = extCtx.NoPrompt return action.Run(ctx, cmd) }, } diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_apply.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_apply.go index 165237d84ad..95009c9010b 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_apply.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_apply.go @@ -41,7 +41,7 @@ type optimizeApplyFlags struct { func newOptimizeApplyCommand(extCtx *azdext.ExtensionContext) *cobra.Command { flags := &optimizeApplyFlags{} - action := &OptimizeApplyAction{flags: flags, envName: extCtx.Environment, noPrompt: extCtx.NoPrompt} + action := &OptimizeApplyAction{flags: flags} cmd := &cobra.Command{ Use: "apply", @@ -57,6 +57,12 @@ After applying, run 'azd deploy' to deploy the optimized agent version.`, RunE: func(cmd *cobra.Command, _ []string) error { ctx := azdext.WithAccessToken(cmd.Context()) setupDebugLogging(cmd.Flags()) + + // Read extCtx fields here (after PersistentPreRunE has populated them + // from -e / AZD_ENVIRONMENT), not at command construction time. + action.envName = extCtx.Environment + action.noPrompt = extCtx.NoPrompt + return action.Run(ctx, cmd) }, } diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_deploy.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_deploy.go index e6da95724c5..3dbd59cd75c 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_deploy.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_deploy.go @@ -35,7 +35,7 @@ type optimizeDeployFlags struct { func newOptimizeDeployCommand(extCtx *azdext.ExtensionContext) *cobra.Command { flags := &optimizeDeployFlags{} - action := &OptimizeDeployAction{flags: flags, envName: extCtx.Environment} + action := &OptimizeDeployAction{flags: flags} cmd := &cobra.Command{ Use: "deploy [agent-name]", @@ -54,6 +54,10 @@ Use 'optimize apply' instead if you want to localize the config into your azd pr ctx := azdext.WithAccessToken(cmd.Context()) setupDebugLogging(cmd.Flags()) + // Read extCtx fields here (after PersistentPreRunE has populated them + // from -e / AZD_ENVIRONMENT), not at command construction time. + action.envName = extCtx.Environment + if len(args) > 0 && flags.agent == "" { flags.agent = args[0] } From 3187a7bce4325c86831fa4763fdace75205e395c Mon Sep 17 00:00:00 2001 From: zyysurely Date: Tue, 9 Jun 2026 12:19:29 -0700 Subject: [PATCH 4/5] Deprecating eval init, change to eval generate --- .../azure.ai.agents/internal/cmd/eval.go | 30 ++++++++--- .../internal/cmd/eval_api_version_test.go | 4 +- .../cmd/{eval_init.go => eval_generate.go} | 36 ++++++------- ...val_init_jobs.go => eval_generate_jobs.go} | 28 +++++----- ...it_prompts.go => eval_generate_prompts.go} | 10 ++-- ...val_init_test.go => eval_generate_test.go} | 52 +++++++++---------- .../internal/cmd/eval_helpers.go | 8 +-- .../azure.ai.agents/internal/cmd/eval_run.go | 4 +- .../azure.ai.agents/internal/cmd/eval_show.go | 2 +- .../azure.ai.agents/internal/cmd/eval_test.go | 2 +- .../internal/cmd/optimize_config.go | 2 +- .../internal/cmd/optimize_prompts.go | 2 +- .../internal/cmd/optimize_test.go | 2 +- .../internal/pkg/agents/opt_eval/yaml.go | 4 +- .../internal/project/service_target_agent.go | 2 +- 15 files changed, 103 insertions(+), 85 deletions(-) rename cli/azd/extensions/azure.ai.agents/internal/cmd/{eval_init.go => eval_generate.go} (92%) rename cli/azd/extensions/azure.ai.agents/internal/cmd/{eval_init_jobs.go => eval_generate_jobs.go} (93%) rename cli/azd/extensions/azure.ai.agents/internal/cmd/{eval_init_prompts.go => eval_generate_prompts.go} (95%) rename cli/azd/extensions/azure.ai.agents/internal/cmd/{eval_init_test.go => eval_generate_test.go} (94%) diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval.go index e14141f32f4..f363375ed97 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval.go @@ -86,14 +86,15 @@ func newEvalCommand(extCtx *azdext.ExtensionContext) *cobra.Command { Long: `Create and run quick evals for an agent. Subcommands: - init Generate an eval config and dataset from a hosted agent - run Execute an evaluation run from eval.yaml - update Update an existing eval configuration - list List evaluations for the current project - show Show details of an evaluation run`, + generate Generate an eval config and dataset from a hosted agent + run Execute an evaluation run from eval.yaml + update Update an existing eval configuration + list List evaluations for the current project + show Show details of an evaluation run`, } - cmd.AddCommand(newEvalInitCommand(extCtx)) + cmd.AddCommand(newEvalGenerateCommand(extCtx)) + cmd.AddCommand(newDeprecatedEvalInitCommand()) cmd.AddCommand(newEvalRunCommand(extCtx)) cmd.AddCommand(newEvalUpdateCommand(extCtx)) cmd.AddCommand(newEvalListCommand(extCtx)) @@ -102,6 +103,23 @@ Subcommands: return cmd } +// newDeprecatedEvalInitCommand returns a hidden "init" command that tells users +// to use "eval generate" instead. This preserves discoverability during the +// deprecation period without silently accepting the old name. +func newDeprecatedEvalInitCommand() *cobra.Command { + return &cobra.Command{ + Use: "init", + Short: "(deprecated) Use 'eval generate' instead.", + Hidden: true, + Deprecated: "use 'azd ai agent eval generate' instead", + RunE: func(cmd *cobra.Command, args []string) error { + return fmt.Errorf( + "'eval init' has been renamed to 'eval generate'.\n\n" + + "Please run: azd ai agent eval generate") + }, + } +} + // resolveEvalContext resolves the context for an eval operation by reading azd project state, // environment variables, and optionally prompting the user. It returns an evalResolvedContext // with API clients and metadata needed to run eval commands. diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_api_version_test.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_api_version_test.go index 91890f2ae55..ae4c59b7335 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_api_version_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_api_version_test.go @@ -73,7 +73,7 @@ func TestSubmitDatasetGeneration_APIVersion(t *testing.T) { agentKind: agent_yaml.AgentKindHosted, version: "v1", } - flags := &evalInitFlags{ + flags := &evalGenerateFlags{ evalModel: "gpt-4o", maxSamples: 10, } @@ -108,7 +108,7 @@ func TestSubmitEvaluatorGeneration_APIVersion(t *testing.T) { agentKind: agent_yaml.AgentKindHosted, version: "v1", } - flags := &evalInitFlags{ + flags := &evalGenerateFlags{ evalModel: "gpt-4o", } diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_init.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_generate.go similarity index 92% rename from cli/azd/extensions/azure.ai.agents/internal/cmd/eval_init.go rename to cli/azd/extensions/azure.ai.agents/internal/cmd/eval_generate.go index c8e33277e03..10c7c14b3b4 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_init.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_generate.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -// eval_init.go implements the "eval init" command, which generates a local +// eval_generate.go implements the "eval generate" command, which generates a local // eval suite (eval.yaml) for a deployed agent. It resolves context, submits // dataset and evaluator generation jobs, polls for completion (unless // --no-wait), downloads review artifacts, and writes the eval config. @@ -27,8 +27,8 @@ import ( // DataGenerationAPIVersion is the API version used for data generation jobs. const DataGenerationAPIVersion = "v1" -// evalInitFlags holds CLI flags and interactive prompt state for eval init. -type evalInitFlags struct { +// evalGenerateFlags holds CLI flags and interactive prompt state for eval generate. +type evalGenerateFlags struct { // CLI flags. envName string // explicit environment name (from -e flag) name string // eval suite name @@ -53,20 +53,20 @@ type evalInitFlags struct { regenerateEvaluator bool } -func newEvalInitCommand(extCtx *azdext.ExtensionContext) *cobra.Command { - flags := &evalInitFlags{maxSamples: defaultEvalSamples, output: defaultEvalConfigName} +func newEvalGenerateCommand(extCtx *azdext.ExtensionContext) *cobra.Command { + flags := &evalGenerateFlags{maxSamples: defaultEvalSamples, output: defaultEvalConfigName} cmd := &cobra.Command{ - Use: "init", + Use: "generate", Short: "Generate a local eval suite for a deployed agent.", Long: `Generate a local eval suite for a deployed agent. By default, this command submits dataset and evaluator generation jobs, waits for completion, downloads review artifacts, and writes eval.yaml at the agent project root. Use --no-wait to write pending operation IDs and return.`, - Example: ` azd ai agent eval init - azd ai agent eval init --gen-instruction "This agent handles restaurant reservations." --eval-model gpt-4o --max-samples 50 - azd ai agent eval init --gen-instruction-file ./instructions.md --eval-model gpt-4o - azd ai agent eval init --dataset ./tests/golden.jsonl --evaluator builtin.intent_resolution`, + Example: ` azd ai agent eval generate + azd ai agent eval generate --gen-instruction "This agent handles restaurant reservations." --eval-model gpt-4o --max-samples 50 + azd ai agent eval generate --gen-instruction-file ./instructions.md --eval-model gpt-4o + azd ai agent eval generate --dataset ./tests/golden.jsonl --evaluator builtin.intent_resolution`, Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { ctx := azdext.WithAccessToken(cmd.Context()) @@ -75,7 +75,7 @@ the agent project root. Use --no-wait to write pending operation IDs and return. flags.evalModelSet = cmd.Flags().Changed("eval-model") flags.maxSamplesSet = cmd.Flags().Changed("max-samples") flags.envName = extCtx.Environment - return runEvalInit(ctx, flags, extCtx.NoPrompt) + return runEvalGenerate(ctx, flags, extCtx.NoPrompt) }, } @@ -96,10 +96,10 @@ the agent project root. Use --no-wait to write pending operation IDs and return. return cmd } -// runEvalInit executes the eval init command logic. It resolves context, +// runEvalGenerate executes the eval generate command logic. It resolves context, // prompts for missing options, submits generation jobs, polls for completion // (unless --no-wait), writes the eval config, and prints next steps. -func runEvalInit(ctx context.Context, flags *evalInitFlags, noPrompt bool) error { +func runEvalGenerate(ctx context.Context, flags *evalGenerateFlags, noPrompt bool) error { if flags.instruction != "" && flags.instructionFile != "" { return fmt.Errorf("cannot use both --gen-instruction and --gen-instruction-file; provide one or the other") } @@ -184,7 +184,7 @@ func runEvalInit(ctx context.Context, flags *evalInitFlags, noPrompt bool) error } } - if err := promptEvalInitOptions(ctx, resolved, flags, noPrompt); err != nil { + if err := promptEvalGenerateOptions(ctx, resolved, flags, noPrompt); err != nil { return err } @@ -226,13 +226,13 @@ func runEvalInit(ctx context.Context, flags *evalInitFlags, noPrompt bool) error if state.DatasetGenOpID != "" || state.EvalGenOpID != "" { state.InitStatus = opt_eval.InitStatusPending } - return writePendingEvalInit(ctx, resolved, configPath, evalCfg, state) + return writePendingEvalGenerate(ctx, resolved, configPath, evalCfg, state) } pollRes, err := pollAndFinalizeJobs(ctx, resolved, evalCfg, state, extraEvals) if err != nil { if _, ok := errors.AsType[*initTimeoutError](err); ok { - return writeTimedOutEvalInit(ctx, resolved, configPath, evalCfg, state) + return writeTimedOutEvalGenerate(ctx, resolved, configPath, evalCfg, state) } return err } @@ -251,7 +251,7 @@ func handleExistingEvalConfig( ctx context.Context, resolved *evalResolvedContext, existingCfg *evalConfig, - flags *evalInitFlags, + flags *evalGenerateFlags, noPrompt bool, ) (keepExisting bool, err error) { if noPrompt { @@ -287,7 +287,7 @@ func handleExistingEvalConfig( func submitEvalJobs( ctx context.Context, resolved *evalResolvedContext, - flags *evalInitFlags, + flags *evalGenerateFlags, evalCfg *evalConfig, existingCfg *evalConfig, isRegenerate bool, diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_init_jobs.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_generate_jobs.go similarity index 93% rename from cli/azd/extensions/azure.ai.agents/internal/cmd/eval_init_jobs.go rename to cli/azd/extensions/azure.ai.agents/internal/cmd/eval_generate_jobs.go index 97b033e48e2..bf342abf6c1 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_init_jobs.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_generate_jobs.go @@ -2,7 +2,7 @@ // Licensed under the MIT License. // eval_init_jobs.go handles generation job submission and polling for the -// eval init command. It submits dataset and evaluator generation requests, +// eval generate command. It submits dataset and evaluator generation requests, // polls for completion in parallel, downloads artifacts on success, and // persists state for resume on timeout. @@ -24,7 +24,7 @@ import ( ) // resolveEvalName returns the eval suite name from flags, falling back to defaultEvalName. -func resolveEvalName(flags *evalInitFlags) string { +func resolveEvalName(flags *evalGenerateFlags) string { if flags.name != "" { return flags.name } @@ -33,7 +33,7 @@ func resolveEvalName(flags *evalInitFlags) string { // resolvedInstruction returns the instruction content from flags, reading // from file if instructionFile is set. -func resolvedInstruction(flags *evalInitFlags) string { +func resolvedInstruction(flags *evalGenerateFlags) string { if flags.instructionFile != "" { data, err := os.ReadFile(flags.instructionFile) //nolint:gosec // user-provided path validated earlier if err != nil { @@ -45,7 +45,7 @@ func resolvedInstruction(flags *evalInitFlags) string { } // newEvalConfig builds an evalConfig from flags and resolved context, applying defaults as needed. -func newEvalConfig(flags *evalInitFlags, resolved *evalResolvedContext) *evalConfig { +func newEvalConfig(flags *evalGenerateFlags, resolved *evalResolvedContext) *evalConfig { agent := evalAgentRef{ Name: resolved.agentName, Kind: resolved.agentKind, @@ -79,7 +79,7 @@ func newEvalConfig(flags *evalInitFlags, resolved *evalResolvedContext) *evalCon func submitDatasetGeneration( ctx context.Context, resolved *evalResolvedContext, - flags *evalInitFlags, + flags *evalGenerateFlags, ) (*eval_api.GenerationJob, error) { // Traces are only supported for evaluator generation, not dataset generation. prompt := resolvedInstruction(flags) @@ -96,7 +96,7 @@ func submitDatasetGeneration( func submitEvaluatorGeneration( ctx context.Context, resolved *evalResolvedContext, - flags *evalInitFlags, + flags *evalGenerateFlags, ) (*eval_api.GenerationJob, error) { var traces *eval_api.TraceOptions if flags.traceDays > 0 { @@ -170,8 +170,8 @@ func buildOpenAIEvalRequest(evalCfg *evalConfig) *eval_api.CreateOpenAIEvalReque return evalCfg.ToAgentTargetAdaptableEvalGroupRequest() } -// resumeEvalInit handles resuming an eval init when generation jobs are still pending. It polls for job completion, updates state and config on success, and persists state for later resume if polling times out. -func resumeEvalInit( +// resumeEvalGenerate handles resuming an eval generate when generation jobs are still pending. It polls for job completion, updates state and config on success, and persists state for later resume if polling times out. +func resumeEvalGenerate( ctx context.Context, resolved *evalResolvedContext, configPath string, @@ -180,7 +180,7 @@ func resumeEvalInit( ) error { if _, err := pollAndFinalizeJobs(ctx, resolved, evalCfg, state, nil); err != nil { if _, ok := errors.AsType[*initTimeoutError](err); ok { - return writeTimedOutEvalInit(ctx, resolved, configPath, evalCfg, state) + return writeTimedOutEvalGenerate(ctx, resolved, configPath, evalCfg, state) } return err } @@ -394,7 +394,7 @@ func (e *initTimeoutError) Error() string { return "generation jobs did not complete within the polling timeout" } -func writePendingEvalInit( +func writePendingEvalGenerate( ctx context.Context, resolved *evalResolvedContext, configPath string, @@ -407,7 +407,7 @@ func writePendingEvalInit( if err := eval_api.WriteEvalConfig(configPath, evalCfg); err != nil { return err } - fmt.Println(color.YellowString("Eval init submitted (async)")) + fmt.Println(color.YellowString("Eval generate submitted (async)")) if state.DatasetGenOpID != "" { fmt.Printf(" dataset generation: %s (%s)\n", state.DatasetGenOpID, state.DatasetGenStatus) } @@ -420,9 +420,9 @@ func writePendingEvalInit( return nil } -// writeTimedOutEvalInit persists state and YAML when generation jobs exceed +// writeTimedOutEvalGenerate persists state and YAML when generation jobs exceed // the polling timeout, allowing the user to resume later. -func writeTimedOutEvalInit( +func writeTimedOutEvalGenerate( ctx context.Context, resolved *evalResolvedContext, configPath string, @@ -448,7 +448,7 @@ func writeTimedOutEvalInit( fmt.Println("\n To resume polling, run:") fmt.Println(" azd ai agent eval run") fmt.Println("\n To start fresh and clear timed-out state, run:") - fmt.Println(" azd ai agent eval init --reset-defaults") + fmt.Println(" azd ai agent eval generate --reset-defaults") return nil } diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_init_prompts.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_generate_prompts.go similarity index 95% rename from cli/azd/extensions/azure.ai.agents/internal/cmd/eval_init_prompts.go rename to cli/azd/extensions/azure.ai.agents/internal/cmd/eval_generate_prompts.go index 484066494c7..5f924891639 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_init_prompts.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_generate_prompts.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -// eval_init_prompts.go implements interactive prompts for the eval init +// eval_generate_prompts.go implements interactive prompts for the eval generate // command, including eval suite name, instruction source, trace inclusion, // eval model selection, and regeneration choices for existing configs. @@ -18,10 +18,10 @@ import ( "github.com/azure/azure-dev/cli/azd/pkg/azdext" ) -// promptEvalInitOptions runs interactive prompts for eval init options that +// promptEvalGenerateOptions runs interactive prompts for eval generate options that // were not provided via flags: name, instruction, trace days, eval model, // and max samples. -func promptEvalInitOptions(ctx context.Context, resolved *evalResolvedContext, flags *evalInitFlags, noPrompt bool) error { +func promptEvalGenerateOptions(ctx context.Context, resolved *evalResolvedContext, flags *evalGenerateFlags, noPrompt bool) error { azdClient := resolved.azdClient if noPrompt { return nil @@ -187,7 +187,7 @@ func promptRegenerateChoices( ctx context.Context, resolved *evalResolvedContext, existingCfg *evalConfig, - flags *evalInitFlags, + flags *evalGenerateFlags, ) error { prompt := resolved.azdClient.Prompt() @@ -240,7 +240,7 @@ func promptConfigConfirmation( ctx context.Context, azdClient *azdext.AzdClient, resolved *evalResolvedContext, - flags *evalInitFlags, + flags *evalGenerateFlags, ) error { prompt := azdClient.Prompt() projectDir := resolved.agentProject diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_init_test.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_generate_test.go similarity index 94% rename from cli/azd/extensions/azure.ai.agents/internal/cmd/eval_init_test.go rename to cli/azd/extensions/azure.ai.agents/internal/cmd/eval_generate_test.go index 9eb26c424b2..46f13b087a1 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_init_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_generate_test.go @@ -21,12 +21,12 @@ import ( ) // --------------------------------------------------------------------------- -// newEvalInitCommand — command shape +// newEvalGenerateCommand — command shape // --------------------------------------------------------------------------- -func TestNewEvalInitCommand_Flags(t *testing.T) { +func TestNewEvalGenerateCommand_Flags(t *testing.T) { t.Parallel() - cmd := newEvalInitCommand(&azdext.ExtensionContext{}) + cmd := newEvalGenerateCommand(&azdext.ExtensionContext{}) expectedFlags := []struct { name string @@ -54,16 +54,16 @@ func TestNewEvalInitCommand_Flags(t *testing.T) { } } -func TestNewEvalInitCommand_NoArgs(t *testing.T) { +func TestNewEvalGenerateCommand_NoArgs(t *testing.T) { t.Parallel() - cmd := newEvalInitCommand(&azdext.ExtensionContext{}) + cmd := newEvalGenerateCommand(&azdext.ExtensionContext{}) assert.NoError(t, cmd.Args(cmd, nil)) assert.Error(t, cmd.Args(cmd, []string{"extra"})) } -func TestNewEvalInitCommand_NoShortOutFile(t *testing.T) { +func TestNewEvalGenerateCommand_NoShortOutFile(t *testing.T) { t.Parallel() - cmd := newEvalInitCommand(&azdext.ExtensionContext{}) + cmd := newEvalGenerateCommand(&azdext.ExtensionContext{}) f := cmd.Flags().ShorthandLookup("o") assert.Nil(t, f, "flag -o shorthand must not exist (conflicts with azd global --output)") } @@ -72,44 +72,44 @@ func TestNewEvalInitCommand_NoShortOutFile(t *testing.T) { // --agent-instruction / --agent-instruction-file mutual exclusion // --------------------------------------------------------------------------- -func TestRunEvalInit_MutualExclusion(t *testing.T) { +func TestRunEvalGenerate_MutualExclusion(t *testing.T) { t.Parallel() - flags := &evalInitFlags{ + flags := &evalGenerateFlags{ instruction: "inline text", instructionFile: "some-file.txt", } - err := runEvalInit(t.Context(), flags, true) + err := runEvalGenerate(t.Context(), flags, true) require.Error(t, err) assert.Contains(t, err.Error(), "cannot use both --gen-instruction and --gen-instruction-file") } -func TestRunEvalInit_InstructionFile(t *testing.T) { +func TestRunEvalGenerate_InstructionFile(t *testing.T) { t.Parallel() tmpDir := t.TempDir() instrFile := filepath.Join(tmpDir, "instruction.md") require.NoError(t, os.WriteFile(instrFile, []byte(" Test booking agent \n"), 0600)) - flags := &evalInitFlags{ + flags := &evalGenerateFlags{ instructionFile: instrFile, evalModel: "test-model", maxSamples: 10, } - // runEvalInit will fail later (no azd client), but file validation should pass. - _ = runEvalInit(t.Context(), flags, true) + // runEvalGenerate will fail later (no azd client), but file validation should pass. + _ = runEvalGenerate(t.Context(), flags, true) // File path remains on the flag — content is NOT inlined. assert.Equal(t, instrFile, flags.instructionFile) assert.Empty(t, flags.instruction) } -func TestRunEvalInit_InstructionFileMissing(t *testing.T) { +func TestRunEvalGenerate_InstructionFileMissing(t *testing.T) { t.Parallel() // Use filepath.Join with TempDir to get a proper absolute path that doesn't exist. missingFile := filepath.Join(t.TempDir(), "nonexistent", "instruction.txt") - flags := &evalInitFlags{ + flags := &evalGenerateFlags{ instructionFile: missingFile, projectEndpoint: "https://example.ai.azure.com/", } - err := runEvalInit(t.Context(), flags, true) + err := runEvalGenerate(t.Context(), flags, true) require.Error(t, err) assert.Contains(t, err.Error(), "not accessible") } @@ -123,7 +123,7 @@ func TestNewEvalConfig(t *testing.T) { t.Run("uses default name", func(t *testing.T) { t.Parallel() - flags := &evalInitFlags{ + flags := &evalGenerateFlags{ instruction: "Test the booking agent", evalModel: "gpt-4.1", maxSamples: 50, @@ -147,7 +147,7 @@ func TestNewEvalConfig(t *testing.T) { t.Run("uses custom name from flag", func(t *testing.T) { t.Parallel() - flags := &evalInitFlags{ + flags := &evalGenerateFlags{ name: "my-suite", maxSamples: 10, } @@ -158,7 +158,7 @@ func TestNewEvalConfig(t *testing.T) { t.Run("stores instruction_file when file provided", func(t *testing.T) { t.Parallel() - flags := &evalInitFlags{ + flags := &evalGenerateFlags{ instructionFile: "./prompts/system.md", evalModel: "gpt-4o", maxSamples: 20, @@ -616,13 +616,13 @@ func TestResolveEvalName(t *testing.T) { t.Parallel() t.Run("returns flag name when set", func(t *testing.T) { t.Parallel() - flags := &evalInitFlags{name: "my-eval"} + flags := &evalGenerateFlags{name: "my-eval"} assert.Equal(t, "my-eval", resolveEvalName(flags)) }) t.Run("returns default when flag is empty", func(t *testing.T) { t.Parallel() - flags := &evalInitFlags{} + flags := &evalGenerateFlags{} assert.Equal(t, defaultEvalName, resolveEvalName(flags)) }) } @@ -635,7 +635,7 @@ func TestResolvedInstruction(t *testing.T) { t.Parallel() t.Run("returns inline instruction", func(t *testing.T) { t.Parallel() - flags := &evalInitFlags{instruction: "Be helpful."} + flags := &evalGenerateFlags{instruction: "Be helpful."} assert.Equal(t, "Be helpful.", resolvedInstruction(flags)) }) @@ -645,13 +645,13 @@ func TestResolvedInstruction(t *testing.T) { filePath := filepath.Join(dir, "prompt.md") require.NoError(t, os.WriteFile(filePath, []byte("File instruction."), 0600)) - flags := &evalInitFlags{instructionFile: filePath} + flags := &evalGenerateFlags{instructionFile: filePath} assert.Equal(t, "File instruction.", resolvedInstruction(flags)) }) t.Run("falls back to inline when file missing", func(t *testing.T) { t.Parallel() - flags := &evalInitFlags{ + flags := &evalGenerateFlags{ instructionFile: "/nonexistent/path.md", instruction: "fallback", } @@ -660,7 +660,7 @@ func TestResolvedInstruction(t *testing.T) { t.Run("returns empty when nothing set", func(t *testing.T) { t.Parallel() - flags := &evalInitFlags{} + flags := &evalGenerateFlags{} assert.Empty(t, resolvedInstruction(flags)) }) } diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_helpers.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_helpers.go index 83d340d7c49..09ca5622fe1 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_helpers.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_helpers.go @@ -134,7 +134,7 @@ func reconcileConfigAgent(w io.Writer, agent *opt_eval.AgentRef, envName, envVer // for an instruction and then call writeBaselineIfNeeded. // // The returned AgentConfig contains resolved instruction file path, model, -// skill_dir, and tools_file. Eval init uses only instruction fields; +// skill_dir, and tools_file. Eval generate uses only instruction fields; // optimize also uses skill_dir and tools_file. func resolveAgentConfig( existingConfig *opt_eval.Config, @@ -195,7 +195,7 @@ type baselineParams struct { // writeBaselineConfig writes a baseline agent config to .agent_configs/baseline/. // It creates metadata.yaml with file pointers and writes instructions.md. // When skillDir is empty, it auto-detects a "skills" or "skill" directory. -// Used by both eval init and optimize. +// Used by both eval generate and optimize. func writeBaselineConfig(agentProject string, p baselineParams) error { baseDir := filepath.Join(agentProject, opt_eval.AgentConfigsDir, opt_eval.BaselineDir) if err := os.MkdirAll(baseDir, 0750); err != nil { @@ -330,7 +330,7 @@ func padColorizedStatus(status string) string { } // --------------------------------------------------------------------------- -// Shared prompt helpers (used by eval init and optimize) +// Shared prompt helpers (used by eval generate and optimize) // --------------------------------------------------------------------------- // selectOtherDeploymentValue is the sentinel value for the "Select another @@ -468,7 +468,7 @@ func promptDatasetSelection( if value == "" { return "", nil, fmt.Errorf( "a dataset is required: use --dataset , or provide dataset_file / dataset_reference " + - "in your config, or run 'azd ai agent eval init' to generate one") + "in your config, or run 'azd ai agent eval generate' to generate one") } if eval_api.IsDatasetName(value) { diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_run.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_run.go index 8915e9a8adb..bbadd25cb66 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_run.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_run.go @@ -86,7 +86,7 @@ func runEvalRun(ctx context.Context, flags *evalRunFlags, noPrompt bool) error { state := opt_eval.LoadEvalState(ctx, resolved.azdClient, resolved.envName) if state.InitStatus == opt_eval.InitStatusPending { - if err := resumeEvalInit(ctx, resolved, configPath, evalCfg, state); err != nil { + if err := resumeEvalGenerate(ctx, resolved, configPath, evalCfg, state); err != nil { return err } } @@ -147,7 +147,7 @@ func runEvalRun(ctx context.Context, flags *evalRunFlags, noPrompt bool) error { fileID := buildDatasetFileID(resolved.projectEndpoint, evalCfg.DatasetReference) dataSource.SetFileID(fileID) } else { - return fmt.Errorf("no dataset configured; run 'azd ai agent eval init' or specify dataset_file / dataset_reference in the eval config") + return fmt.Errorf("no dataset configured; run 'azd ai agent eval generate' or specify dataset_file / dataset_reference in the eval config") } runReq.DataSource = dataSource diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_show.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_show.go index 5b35b3e4095..ba3febe1cc2 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_show.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_show.go @@ -69,7 +69,7 @@ func runEvalShow(ctx context.Context, evalID string, flags *evalShowFlags) error evalID = state.EvalID } if evalID == "" { - return fmt.Errorf("no eval-id provided and none found in the current environment; run 'azd ai agent eval init' first or pass an eval-id") + return fmt.Errorf("no eval-id provided and none found in the current environment; run 'azd ai agent eval generate' first or pass an eval-id") } if flags.evalRunID != "" { diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_test.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_test.go index 991d5509046..5c82b1f5cff 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_test.go @@ -194,7 +194,7 @@ func TestDetectEvalAgentKind(t *testing.T) { } // --------------------------------------------------------------------------- -// EvalState — stored in azd environment (integration-tested via eval init/run) +// EvalState — stored in azd environment (integration-tested via eval generate/run) // --------------------------------------------------------------------------- // --------------------------------------------------------------------------- diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_config.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_config.go index 28a6ceb48e7..2e49b47d34b 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_config.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_config.go @@ -70,7 +70,7 @@ func (c *OptimizeConfig) Validate() error { if !hasFile && !hasRef { return fmt.Errorf( "a dataset is required: provide dataset_file or dataset_reference in your config, " + - "or run 'azd ai agent eval init' to generate one") + "or run 'azd ai agent eval generate' to generate one") } return nil diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_prompts.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_prompts.go index 8f5d2e621db..029dde949a1 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_prompts.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_prompts.go @@ -342,7 +342,7 @@ func resolveOptimizeDataset( if noPrompt { return fmt.Errorf( "a dataset is required: use --dataset , or provide dataset_file / dataset_reference " + - "in your config, or run 'azd ai agent eval init' to generate one") + "in your config, or run 'azd ai agent eval generate' to generate one") } if azdClient == nil { diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_test.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_test.go index 380bc58df93..32ce411defb 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_test.go @@ -260,7 +260,7 @@ func TestLoadOptimizeConfig_EvalYAML_WithDatasetFile(t *testing.T) { t.Parallel() dir := t.TempDir() - // Simulate an eval.yaml with dataset_file — the format generated by "azd ai agent eval init". + // Simulate an eval.yaml with dataset_file — the format generated by "azd ai agent eval generate". dataFile := filepath.Join(dir, "data.jsonl") require.NoError(t, os.WriteFile(dataFile, []byte(`{"input":"hello"}`+"\n"), 0600)) diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/opt_eval/yaml.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/opt_eval/yaml.go index c795a4014ac..220d6d518b7 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/opt_eval/yaml.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/opt_eval/yaml.go @@ -162,7 +162,7 @@ func (el EvaluatorList) SetLocalURI(name, uri string) { // under AgentConfigsDir as a self-contained directory with a fixed layout: // // .agent_configs/ -// ├── baseline/ # original agent config captured by eval init or optimize +// ├── baseline/ # original agent config captured by eval generate or optimize // │ ├── metadata.yaml # MetadataFile — model, file pointers // │ ├── instructions.md # InstructionFile — system prompt // │ ├── skills/ # SkillsDir — skill definitions (optional) @@ -174,7 +174,7 @@ func (el EvaluatorList) SetLocalURI(name, uri string) { // └── tools.json // // Both eval and optimize commands share these constants and layout conventions. -// Eval init writes the baseline directory; optimize apply writes candidate +// Eval generate writes the baseline directory; optimize apply writes candidate // directories and reads the baseline for diff display. const ( // AgentConfigsDir is the top-level folder that holds agent configuration diff --git a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go index ed621e2c42d..db136194db6 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go +++ b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go @@ -1840,7 +1840,7 @@ func (p *AgentServiceTargetProvider) deployArtifacts( last := artifacts[len(artifacts)-1] last.Metadata["note"] = "For information on invoking the agent, see " + output.WithLinkFormat( "https://aka.ms/azd-agents-invoke") + - "\n\nSet up an evaluation suite to measure quality and impact in one step with " + output.WithHighLightFormat("azd ai agent eval init") + "\n\nSet up an evaluation suite to measure quality and impact in one step with " + output.WithHighLightFormat("azd ai agent eval generate") } } From bacd3d85fc9a5894b3ea604fab88bb6b5d01b9aa Mon Sep 17 00:00:00 2001 From: zyysurely Date: Tue, 9 Jun 2026 12:28:44 -0700 Subject: [PATCH 5/5] comments fix --- cli/azd/extensions/azure.ai.agents/internal/cmd/eval.go | 2 +- .../azure.ai.agents/internal/cmd/eval_generate_jobs.go | 2 +- cli/azd/extensions/azure.ai.agents/internal/cmd/eval_test.go | 4 +++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval.go index f363375ed97..37ad3934781 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval.go @@ -2,7 +2,7 @@ // Licensed under the MIT License. // eval.go implements the top-level "eval" command group and shared context -// resolution logic used by all eval subcommands (init, run, update, list, show). +// resolution logic used by all eval subcommands (generate, run, update, list, show). // // The evalResolvedContext struct holds the resolved agent, project, and // endpoint information. It is built from azd project state, environment diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_generate_jobs.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_generate_jobs.go index bf342abf6c1..8fe8ca243f9 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_generate_jobs.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_generate_jobs.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -// eval_init_jobs.go handles generation job submission and polling for the +// eval_generate_jobs.go handles generation job submission and polling for the // eval generate command. It submits dataset and evaluator generation requests, // polls for completion in parallel, downloads artifacts on success, and // persists state for resume on timeout. diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_test.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_test.go index 5c82b1f5cff..40e970368ab 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_test.go @@ -48,10 +48,12 @@ func TestNewEvalCommand_HasExpectedSubcommands(t *testing.T) { names = append(names, sub.Name()) } - assert.Contains(t, names, "init") + assert.Contains(t, names, "generate") assert.Contains(t, names, "run") assert.Contains(t, names, "list") assert.Contains(t, names, "show") + // "init" remains registered as a hidden deprecated alias for "generate". + assert.Contains(t, names, "init") } func TestNewEvalCommand_UseString(t *testing.T) {