Skip to content

[fix] -e environment switch support + relative dataset path + not sta… #8566

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Zyysurely merged 5 commits into from
Jun 9, 2026
Merged

[fix] -e environment switch support + relative dataset path + not sta… #8566

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
3513ab2
[fix] -e environment switch support + relative dataset path + not sta…
yingyingzhaozyying Jun 8, 2026
cb9c234
fix comments
yingyingzhaozyying Jun 8, 2026
30c2144
fix -e
yingyingzhaozyying Jun 9, 2026
3187a7b
Deprecating eval init, change to eval generate
yingyingzhaozyying Jun 9, 2026
bacd3d8
comments fix
yingyingzhaozyying Jun 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 30 additions & 12 deletions cli/azd/extensions/azure.ai.agents/internal/cmd/eval.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
// Licensed under the MIT License.

// eval.go implements the top-level "eval" command group and shared context
// resolution logic used by all eval subcommands (init, run, update, list, show).
// resolution logic used by all eval subcommands (generate, run, update, list, show).
//
// The evalResolvedContext struct holds the resolved agent, project, and
// endpoint information. It is built from azd project state, environment
Expand Down Expand Up @@ -72,6 +72,7 @@ type evalResolvedContext struct {

// evalContextOptions configures the behavior of resolveEvalContext.
type evalContextOptions struct {
envName string // explicit environment name (from -e flag)
agent string // explicit agent name (from --agent flag)
projectEndpoint string // explicit project endpoint (from --project-endpoint flag)
requireAgent bool // fail if agent name cannot be resolved
Expand All @@ -85,22 +86,40 @@ func newEvalCommand(extCtx *azdext.ExtensionContext) *cobra.Command {
Long: `Create and run quick evals for an agent.

Subcommands:
init Generate an eval config and dataset from a hosted agent
run Execute an evaluation run from eval.yaml
update Update an existing eval configuration
list List evaluations for the current project
show Show details of an evaluation run`,
generate Generate an eval config and dataset from a hosted agent
run Execute an evaluation run from eval.yaml
update Update an existing eval configuration
list List evaluations for the current project
show Show details of an evaluation run`,
}

cmd.AddCommand(newEvalInitCommand(extCtx))
cmd.AddCommand(newEvalGenerateCommand(extCtx))
cmd.AddCommand(newDeprecatedEvalInitCommand())
cmd.AddCommand(newEvalRunCommand(extCtx))
cmd.AddCommand(newEvalUpdateCommand(extCtx))
cmd.AddCommand(newEvalListCommand())
cmd.AddCommand(newEvalShowCommand())
cmd.AddCommand(newEvalListCommand(extCtx))
cmd.AddCommand(newEvalShowCommand(extCtx))

return cmd
}

// newDeprecatedEvalInitCommand returns a hidden "init" command that tells users
// to use "eval generate" instead. This preserves discoverability during the
// deprecation period without silently accepting the old name.
func newDeprecatedEvalInitCommand() *cobra.Command {
return &cobra.Command{
Use: "init",
Short: "(deprecated) Use 'eval generate' instead.",
Hidden: true,
Deprecated: "use 'azd ai agent eval generate' instead",
RunE: func(cmd *cobra.Command, args []string) error {
return fmt.Errorf(
"'eval init' has been renamed to 'eval generate'.\n\n" +
"Please run: azd ai agent eval generate")
},
}
}

// resolveEvalContext resolves the context for an eval operation by reading azd project state,
// environment variables, and optionally prompting the user. It returns an evalResolvedContext
// with API clients and metadata needed to run eval commands.
Expand All @@ -125,9 +144,8 @@ func resolveEvalContext(ctx context.Context, options evalContextOptions) (*evalR

// Read the current azd environment once — used for agent info, endpoint, and env name.
var envName string
envResp, envErr := azdClient.Environment().GetCurrent(ctx, &azdext.EmptyRequest{})
if envErr == nil && envResp.Environment != nil {
envName = envResp.Environment.Name
if env := getExistingEnvironment(ctx, options.envName, azdClient); env != nil {
envName = env.Name
}

getEnvValue := func(key string) string {
Expand Down
4 changes: 2 additions & 2 deletions cli/azd/extensions/azure.ai.agents/internal/cmd/eval_api_version_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ func TestSubmitDatasetGeneration_APIVersion(t *testing.T) {
agentKind: agent_yaml.AgentKindHosted,
version: "v1",
}
flags := &evalInitFlags{
flags := &evalGenerateFlags{
evalModel: "gpt-4o",
maxSamples: 10,
}
Expand Down Expand Up @@ -108,7 +108,7 @@ func TestSubmitEvaluatorGeneration_APIVersion(t *testing.T) {
agentKind: agent_yaml.AgentKindHosted,
version: "v1",
}
flags := &evalInitFlags{
flags := &evalGenerateFlags{
evalModel: "gpt-4o",
}

Expand Down
41 changes: 22 additions & 19 deletions ...azure.ai.agents/internal/cmd/eval_init.go → ...e.ai.agents/internal/cmd/eval_generate.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

// eval_init.go implements the "eval init" command, which generates a local
// eval_generate.go implements the "eval generate" command, which generates a local
// eval suite (eval.yaml) for a deployed agent. It resolves context, submits
// dataset and evaluator generation jobs, polls for completion (unless
// --no-wait), downloads review artifacts, and writes the eval config.
Expand All @@ -27,9 +27,10 @@ import (
// DataGenerationAPIVersion is the API version used for data generation jobs.
const DataGenerationAPIVersion = "v1"

// evalInitFlags holds CLI flags and interactive prompt state for eval init.
type evalInitFlags struct {
// evalGenerateFlags holds CLI flags and interactive prompt state for eval generate.
type evalGenerateFlags struct {
// CLI flags.
envName string // explicit environment name (from -e flag)
name string // eval suite name
agent string // target agent name
projectEndpoint string // Foundry project endpoint
Expand All @@ -52,28 +53,29 @@ type evalInitFlags struct {
regenerateEvaluator bool
}

func newEvalInitCommand(extCtx *azdext.ExtensionContext) *cobra.Command {
flags := &evalInitFlags{maxSamples: defaultEvalSamples, output: defaultEvalConfigName}
func newEvalGenerateCommand(extCtx *azdext.ExtensionContext) *cobra.Command {
flags := &evalGenerateFlags{maxSamples: defaultEvalSamples, output: defaultEvalConfigName}
cmd := &cobra.Command{
Use: "init",
Use: "generate",
Short: "Generate a local eval suite for a deployed agent.",
Long: `Generate a local eval suite for a deployed agent.

By default, this command submits dataset and evaluator generation jobs, waits for
completion, downloads review artifacts, and writes eval.yaml at
the agent project root. Use --no-wait to write pending operation IDs and return.`,
Example: ` azd ai agent eval init
azd ai agent eval init --gen-instruction "This agent handles restaurant reservations." --eval-model gpt-4o --max-samples 50
azd ai agent eval init --gen-instruction-file ./instructions.md --eval-model gpt-4o
azd ai agent eval init --dataset ./tests/golden.jsonl --evaluator builtin.intent_resolution`,
Example: ` azd ai agent eval generate
azd ai agent eval generate --gen-instruction "This agent handles restaurant reservations." --eval-model gpt-4o --max-samples 50
azd ai agent eval generate --gen-instruction-file ./instructions.md --eval-model gpt-4o
azd ai agent eval generate --dataset ./tests/golden.jsonl --evaluator builtin.intent_resolution`,
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, args []string) error {
ctx := azdext.WithAccessToken(cmd.Context())
logCleanup := setupDebugLogging(cmd.Flags())
defer logCleanup()
flags.evalModelSet = cmd.Flags().Changed("eval-model")
flags.maxSamplesSet = cmd.Flags().Changed("max-samples")
return runEvalInit(ctx, flags, extCtx.NoPrompt)
flags.envName = extCtx.Environment
return runEvalGenerate(ctx, flags, extCtx.NoPrompt)
},
}

Expand All @@ -94,10 +96,10 @@ the agent project root. Use --no-wait to write pending operation IDs and return.
return cmd
}

// runEvalInit executes the eval init command logic. It resolves context,
// runEvalGenerate executes the eval generate command logic. It resolves context,
// prompts for missing options, submits generation jobs, polls for completion
// (unless --no-wait), writes the eval config, and prints next steps.
func runEvalInit(ctx context.Context, flags *evalInitFlags, noPrompt bool) error {
func runEvalGenerate(ctx context.Context, flags *evalGenerateFlags, noPrompt bool) error {
if flags.instruction != "" && flags.instructionFile != "" {
return fmt.Errorf("cannot use both --gen-instruction and --gen-instruction-file; provide one or the other")
}
Expand All @@ -110,6 +112,7 @@ func runEvalInit(ctx context.Context, flags *evalInitFlags, noPrompt bool) error
}

resolved, err := resolveEvalContext(ctx, evalContextOptions{
envName: flags.envName,
agent: flags.agent,
projectEndpoint: flags.projectEndpoint,
requireAgent: true,
Expand Down Expand Up @@ -181,7 +184,7 @@ func runEvalInit(ctx context.Context, flags *evalInitFlags, noPrompt bool) error
}
}

if err := promptEvalInitOptions(ctx, resolved, flags, noPrompt); err != nil {
if err := promptEvalGenerateOptions(ctx, resolved, flags, noPrompt); err != nil {
return err
}

Expand Down Expand Up @@ -223,13 +226,13 @@ func runEvalInit(ctx context.Context, flags *evalInitFlags, noPrompt bool) error
if state.DatasetGenOpID != "" || state.EvalGenOpID != "" {
state.InitStatus = opt_eval.InitStatusPending
}
return writePendingEvalInit(ctx, resolved, configPath, evalCfg, state)
return writePendingEvalGenerate(ctx, resolved, configPath, evalCfg, state)
}

pollRes, err := pollAndFinalizeJobs(ctx, resolved, evalCfg, state, extraEvals)
if err != nil {
if _, ok := errors.AsType[*initTimeoutError](err); ok {
return writeTimedOutEvalInit(ctx, resolved, configPath, evalCfg, state)
return writeTimedOutEvalGenerate(ctx, resolved, configPath, evalCfg, state)
}
return err
}
Expand All @@ -248,7 +251,7 @@ func handleExistingEvalConfig(
ctx context.Context,
resolved *evalResolvedContext,
existingCfg *evalConfig,
flags *evalInitFlags,
flags *evalGenerateFlags,
noPrompt bool,
) (keepExisting bool, err error) {
if noPrompt {
Expand Down Expand Up @@ -284,7 +287,7 @@ func handleExistingEvalConfig(
func submitEvalJobs(
ctx context.Context,
resolved *evalResolvedContext,
flags *evalInitFlags,
flags *evalGenerateFlags,
evalCfg *evalConfig,
existingCfg *evalConfig,
isRegenerate bool,
Expand All @@ -306,7 +309,7 @@ func submitEvalJobs(
needDatasetGen = flags.dataset == ""
needEvalGen = true
if !needDatasetGen {
datasetPath, err := resolveLocalDatasetFile(flags.dataset, resolved.agentProject)
datasetPath, err := resolveLocalDatasetFile(resolveCwdRelative(flags.dataset), resolved.agentProject)
if err != nil {
return nil, err
}
Expand Down
52 changes: 34 additions & 18 deletions ....ai.agents/internal/cmd/eval_init_jobs.go → ...agents/internal/cmd/eval_generate_jobs.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

// eval_init_jobs.go handles generation job submission and polling for the
// eval init command. It submits dataset and evaluator generation requests,
// eval_generate_jobs.go handles generation job submission and polling for the
// eval generate command. It submits dataset and evaluator generation requests,
// polls for completion in parallel, downloads artifacts on success, and
// persists state for resume on timeout.

Expand All @@ -24,7 +24,7 @@ import (
)

// resolveEvalName returns the eval suite name from flags, falling back to defaultEvalName.
func resolveEvalName(flags *evalInitFlags) string {
func resolveEvalName(flags *evalGenerateFlags) string {
if flags.name != "" {
return flags.name
}
Expand All @@ -33,7 +33,7 @@ func resolveEvalName(flags *evalInitFlags) string {

// resolvedInstruction returns the instruction content from flags, reading
// from file if instructionFile is set.
func resolvedInstruction(flags *evalInitFlags) string {
func resolvedInstruction(flags *evalGenerateFlags) string {
if flags.instructionFile != "" {
data, err := os.ReadFile(flags.instructionFile) //nolint:gosec // user-provided path validated earlier
if err != nil {
Expand All @@ -45,11 +45,13 @@ func resolvedInstruction(flags *evalInitFlags) string {
}

// newEvalConfig builds an evalConfig from flags and resolved context, applying defaults as needed.
func newEvalConfig(flags *evalInitFlags, resolved *evalResolvedContext) *evalConfig {
func newEvalConfig(flags *evalGenerateFlags, resolved *evalResolvedContext) *evalConfig {
agent := evalAgentRef{
Name: resolved.agentName,
Kind: resolved.agentKind,
Version: resolved.version,
Name: resolved.agentName,
Kind: resolved.agentKind,
// Version is intentionally omitted — it is resolved at run time
// from the azd environment (AGENT_{SVC}_VERSION) so eval.yaml
// never contains a stale version that drifts after redeployment.
}
if flags.configFile != "" {
agent.ConfigFile = flags.configFile
Expand Down Expand Up @@ -77,7 +79,7 @@ func newEvalConfig(flags *evalInitFlags, resolved *evalResolvedContext) *evalCon
func submitDatasetGeneration(
ctx context.Context,
resolved *evalResolvedContext,
flags *evalInitFlags,
flags *evalGenerateFlags,
) (*eval_api.GenerationJob, error) {
// Traces are only supported for evaluator generation, not dataset generation.
prompt := resolvedInstruction(flags)
Expand All @@ -94,7 +96,7 @@ func submitDatasetGeneration(
func submitEvaluatorGeneration(
ctx context.Context,
resolved *evalResolvedContext,
flags *evalInitFlags,
flags *evalGenerateFlags,
) (*eval_api.GenerationJob, error) {
var traces *eval_api.TraceOptions
if flags.traceDays > 0 {
Expand All @@ -110,6 +112,20 @@ func submitEvaluatorGeneration(
return resolved.evalClient.CreateEvaluatorGenerationJob(ctx, request, ProjectEndpointAPIVersion)
}

// resolveCwdRelative converts a relative path to an absolute path based on
// the current working directory. Already-absolute paths are returned as-is.
// This should be called on CLI flag values before passing them to
// resolveLocalDatasetFile, which resolves against the agent project directory.
func resolveCwdRelative(path string) string {
if filepath.IsAbs(path) {
return path
}
if abs, err := filepath.Abs(path); err == nil {
return abs
}
return path
}

// resolveLocalDatasetFile resolves the dataset flag value to an absolute path
// for the local JSONL file. If the value is relative it is resolved against
// the agent project directory.
Expand Down Expand Up @@ -154,8 +170,8 @@ func buildOpenAIEvalRequest(evalCfg *evalConfig) *eval_api.CreateOpenAIEvalReque
return evalCfg.ToAgentTargetAdaptableEvalGroupRequest()
}

// resumeEvalInit handles resuming an eval init when generation jobs are still pending. It polls for job completion, updates state and config on success, and persists state for later resume if polling times out.
func resumeEvalInit(
// resumeEvalGenerate handles resuming an eval generate when generation jobs are still pending. It polls for job completion, updates state and config on success, and persists state for later resume if polling times out.
func resumeEvalGenerate(
ctx context.Context,
resolved *evalResolvedContext,
configPath string,
Expand All @@ -164,7 +180,7 @@ func resumeEvalInit(
) error {
if _, err := pollAndFinalizeJobs(ctx, resolved, evalCfg, state, nil); err != nil {
if _, ok := errors.AsType[*initTimeoutError](err); ok {
return writeTimedOutEvalInit(ctx, resolved, configPath, evalCfg, state)
return writeTimedOutEvalGenerate(ctx, resolved, configPath, evalCfg, state)
}
return err
}
Expand Down Expand Up @@ -378,7 +394,7 @@ func (e *initTimeoutError) Error() string {
return "generation jobs did not complete within the polling timeout"
}

func writePendingEvalInit(
func writePendingEvalGenerate(
ctx context.Context,
resolved *evalResolvedContext,
configPath string,
Expand All @@ -391,7 +407,7 @@ func writePendingEvalInit(
if err := eval_api.WriteEvalConfig(configPath, evalCfg); err != nil {
return err
}
fmt.Println(color.YellowString("Eval init submitted (async)"))
fmt.Println(color.YellowString("Eval generate submitted (async)"))
if state.DatasetGenOpID != "" {
fmt.Printf(" dataset generation: %s (%s)\n", state.DatasetGenOpID, state.DatasetGenStatus)
}
Expand All @@ -404,9 +420,9 @@ func writePendingEvalInit(
return nil
}

// writeTimedOutEvalInit persists state and YAML when generation jobs exceed
// writeTimedOutEvalGenerate persists state and YAML when generation jobs exceed
// the polling timeout, allowing the user to resume later.
func writeTimedOutEvalInit(
func writeTimedOutEvalGenerate(
ctx context.Context,
resolved *evalResolvedContext,
configPath string,
Expand All @@ -432,7 +448,7 @@ func writeTimedOutEvalInit(
fmt.Println("\n To resume polling, run:")
fmt.Println(" azd ai agent eval run")
fmt.Println("\n To start fresh and clear timed-out state, run:")
fmt.Println(" azd ai agent eval init --reset-defaults")
fmt.Println(" azd ai agent eval generate --reset-defaults")
return nil
}

Expand Down
Loading
Loading