From 898515d7cb7ac06423cdfcc28342623b41d23178 Mon Sep 17 00:00:00 2001 From: Wallace Breza Date: Wed, 18 Mar 2026 15:29:28 -0700 Subject: [PATCH 1/2] feat: add telemetry instrumentation for Copilot agent flows Add OpenTelemetry spans and usage attributes to track Copilot agent session lifecycle, initialization prompts, message usage, and consent decisions. All instrumentation is in the core agent packages so it works for both direct CLI and gRPC extension framework consumers. Changes: - Define 16 new AttributeKey fields in internal/tracing/fields/fields.go covering session (id, isNew, messageCount), init prompts (isFirstRun, reasoningEffort, model, consentScope), per-message metrics (model, tokens, billingRate, premiumRequests, durationMs), and consent counts (approvedCount, deniedCount) - Add copilot.initialize span in CopilotAgent.Initialize() tracking reasoning level, model selection, and isFirstRun - Add copilot.message span in CopilotAgent.SendMessage() tracking per-message usage metrics and cumulative message count - Add copilot.session span in ensureSession() tracking session creation vs resumption with hashed session ID - Track consent approved/denied counts in permission handler and record as usage attributes on agent Stop() - Track workflow consent scope selection in PromptWorkflowConsent() - Rename InitMethod from 'agent' to 'copilot' in cmd/init.go - Add InitMethod='environment' for previously untracked init branch - Record aggregate copilot metrics as usage attributes in initAppWithAgent Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- cli/azd/cmd/init.go | 18 ++- .../agent/consent/workflow_consent.go | 9 ++ cli/azd/internal/agent/copilot_agent.go | 99 ++++++++++++++- cli/azd/internal/tracing/fields/fields.go | 120 ++++++++++++++++++ 4 files changed, 237 insertions(+), 9 deletions(-) diff --git a/cli/azd/cmd/init.go b/cli/azd/cmd/init.go index b007ae5f8b4..4205432acbb 100644 --- a/cli/azd/cmd/init.go +++ b/cli/azd/cmd/init.go @@ -371,6 +371,7 @@ func (i *initAction) Run(ctx context.Context) (*actions.ActionResult, error) { return nil, err } case initEnvironment: + tracing.SetUsageAttributes(fields.InitMethod.String("environment")) env, err := i.initializeEnv(ctx, azdCtx, templates.Metadata{}) if err != nil { return nil, err @@ -379,7 +380,7 @@ func (i *initAction) Run(ctx context.Context) (*actions.ActionResult, error) { header = fmt.Sprintf("Initialized environment %s.", env.Name()) followUp = "" case initWithAgent: - tracing.SetUsageAttributes(fields.InitMethod.String("agent")) + tracing.SetUsageAttributes(fields.InitMethod.String("copilot")) if err := i.initAppWithAgent(ctx, azdCtx); err != nil { return nil, err } @@ -540,10 +541,21 @@ When complete, provide a brief summary of what was accomplished.` _ = azdCtx.ClearCopilotSession() } + // Record aggregate copilot metrics as usage attributes + metrics := copilotAgent.GetMetrics() + tracing.SetUsageAttributes( + fields.CopilotMode.String(string(agent.AgentModeInteractive)), + fields.CopilotMessageModel.String(metrics.Usage.Model), + fields.CopilotMessageInputTokens.Float64(metrics.Usage.InputTokens), + fields.CopilotMessageOutputTokens.Float64(metrics.Usage.OutputTokens), + fields.CopilotMessagePremiumRequests.Float64(metrics.Usage.PremiumRequests), + fields.CopilotMessageDurationMs.Float64(metrics.Usage.DurationMS), + ) + // Show session metrics (usage + file changes) - if metrics := copilotAgent.GetMetrics().String(); metrics != "" { + if metricsStr := metrics.String(); metricsStr != "" { i.console.Message(ctx, "") - i.console.Message(ctx, metrics) + i.console.Message(ctx, metricsStr) } i.console.Message(ctx, "") diff --git a/cli/azd/internal/agent/consent/workflow_consent.go b/cli/azd/internal/agent/consent/workflow_consent.go index 74649d3073d..c1ef8e23879 100644 --- a/cli/azd/internal/agent/consent/workflow_consent.go +++ b/cli/azd/internal/agent/consent/workflow_consent.go @@ -10,6 +10,8 @@ import ( "strings" "time" + "github.com/azure/azure-dev/cli/azd/internal/tracing" + "github.com/azure/azure-dev/cli/azd/internal/tracing/fields" "github.com/azure/azure-dev/cli/azd/pkg/output" "github.com/azure/azure-dev/cli/azd/pkg/ux" "github.com/mark3labs/mcp-go/mcp" @@ -29,6 +31,13 @@ func (cm *consentManager) PromptWorkflowConsent(ctx context.Context, servers []s return err } + // Track the consent scope selection + if scope == "" { + tracing.SetUsageAttributes(fields.CopilotInitConsentScope.String("prompt")) + } else { + tracing.SetUsageAttributes(fields.CopilotInitConsentScope.String(string(scope))) + } + // Empty scope means the user chose "No, prompt me for each operation" — no rules to add if scope == "" { return nil diff --git a/cli/azd/internal/agent/copilot_agent.go b/cli/azd/internal/agent/copilot_agent.go index ac8a66b37bd..80433fb5bd6 100644 --- a/cli/azd/internal/agent/copilot_agent.go +++ b/cli/azd/internal/agent/copilot_agent.go @@ -21,6 +21,8 @@ import ( "github.com/azure/azure-dev/cli/azd/internal/agent/consent" agentcopilot "github.com/azure/azure-dev/cli/azd/internal/agent/copilot" + "github.com/azure/azure-dev/cli/azd/internal/tracing" + "github.com/azure/azure-dev/cli/azd/internal/tracing/fields" "github.com/azure/azure-dev/cli/azd/pkg/config" "github.com/azure/azure-dev/cli/azd/pkg/input" "github.com/azure/azure-dev/cli/azd/pkg/output" @@ -57,6 +59,9 @@ type CopilotAgent struct { mu sync.Mutex // guards cumulative metrics and file changes cumulativeUsage UsageMetrics // cumulative metrics across multiple SendMessage calls accumulatedFileChanges watch.FileChanges // accumulated file changes across all SendMessage calls + messageCount int // number of messages sent in current session + consentApprovedCount int // running count of tool calls approved + consentDeniedCount int // running count of tool calls denied // Cleanup — ordered slice for deterministic teardown cleanupTasks []cleanupTask @@ -71,6 +76,9 @@ type cleanupTask struct { // and Copilot client startup. If config already exists, returns current values without // prompting. Use WithForcePrompt() to always show prompts. func (a *CopilotAgent) Initialize(ctx context.Context, opts ...InitOption) (*InitResult, error) { + ctx, span := tracing.Start(ctx, "copilot.initialize") + defer span.End() + options := &initOptions{} for _, opt := range opts { opt(options) @@ -106,11 +114,19 @@ func (a *CopilotAgent) Initialize(ctx context.Context, opts ...InitOption) (*Ini // If already configured and not forcing, return current config if (hasModel || hasEffort) && !options.forcePrompt { - return &InitResult{ + result := &InitResult{ Model: existingModel, ReasoningEffort: existingEffort, IsFirstRun: false, - }, nil + } + + tracing.SetUsageAttributes( + fields.CopilotInitIsFirstRun.Bool(false), + fields.CopilotInitModel.String(result.Model), + fields.CopilotInitReasoningEffort.String(result.ReasoningEffort), + ) + + return result, nil } // Prompt for reasoning effort @@ -194,11 +210,19 @@ func (a *CopilotAgent) Initialize(ctx context.Context, opts ...InitOption) (*Ini return nil, fmt.Errorf("failed to save config: %w", err) } - return &InitResult{ + result := &InitResult{ Model: selectedModel, ReasoningEffort: selectedEffort, IsFirstRun: true, - }, nil + } + + tracing.SetUsageAttributes( + fields.CopilotInitIsFirstRun.Bool(true), + fields.CopilotInitModel.String(result.Model), + fields.CopilotInitReasoningEffort.String(result.ReasoningEffort), + ) + + return result, nil } // SelectSession shows a UX picker with previous sessions for the current directory. @@ -273,6 +297,9 @@ func (a *CopilotAgent) ListSessions(ctx context.Context, cwd string) ([]SessionM // SendMessage sends a prompt to the agent and returns the result. // Creates a new session or resumes one if WithSessionID is provided. func (a *CopilotAgent) SendMessage(ctx context.Context, prompt string, opts ...SendOption) (*AgentResult, error) { + ctx, span := tracing.Start(ctx, "copilot.message") + defer span.End() + options := &sendOptions{} for _, opt := range opts { opt(options) @@ -281,6 +308,10 @@ func (a *CopilotAgent) SendMessage(ctx context.Context, prompt string, opts ...S // Update active context so SDK callbacks use this call's context a.activeCtx.Store(&ctx) + // Track whether this is a new session vs resumed + // A session is "not new" if it already exists OR if we're resuming by ID + isNewSession := a.session == nil && options.sessionID == "" + // Ensure session exists if err := a.ensureSession(ctx, options.sessionID); err != nil { return nil, err @@ -296,13 +327,47 @@ func (a *CopilotAgent) SendMessage(ctx context.Context, prompt string, opts ...S } } + // Set telemetry attributes for this message + span.SetAttributes( + fields.StringHashed(fields.CopilotSessionId, a.sessionID), + fields.CopilotSessionIsNew.Bool(isNewSession), + fields.CopilotMode.String(string(mode)), + ) + log.Printf("[copilot] SendMessage: sending prompt (%d chars, headless=%v)...", len(prompt), a.headless) + var result *AgentResult + var err error + if a.headless { - return a.sendMessageHeadless(ctx, prompt, mode) + result, err = a.sendMessageHeadless(ctx, prompt, mode) + } else { + result, err = a.sendMessageInteractive(ctx, prompt, mode) + } + + if err != nil { + return nil, err } - return a.sendMessageInteractive(ctx, prompt, mode) + // Increment message count only after successful send + a.messageCount++ + + // Record per-message usage metrics on the span + span.SetAttributes( + fields.CopilotMessageModel.String(result.Usage.Model), + fields.CopilotMessageInputTokens.Float64(result.Usage.InputTokens), + fields.CopilotMessageOutputTokens.Float64(result.Usage.OutputTokens), + fields.CopilotMessageBillingRate.Float64(result.Usage.BillingRate), + fields.CopilotMessagePremiumRequests.Float64(result.Usage.PremiumRequests), + fields.CopilotMessageDurationMs.Float64(result.Usage.DurationMS), + ) + + // Update cumulative usage attributes + tracing.SetUsageAttributes( + fields.CopilotSessionMessageCount.Int(a.messageCount), + ) + + return result, nil } // sendMessageInteractive sends a message with full interactive display and file watcher. @@ -470,6 +535,12 @@ func (a *CopilotAgent) SendMessageWithRetry(ctx context.Context, prompt string, // Stop terminates the agent, cleans up the Copilot SDK client and runtime process. // Cleanup tasks run in reverse registration order. Safe to call multiple times. func (a *CopilotAgent) Stop() error { + // Record final consent counts as usage attributes + tracing.SetUsageAttributes( + fields.CopilotConsentApprovedCount.Int(a.consentApprovedCount), + fields.CopilotConsentDeniedCount.Int(a.consentDeniedCount), + ) + tasks := a.cleanupTasks a.cleanupTasks = nil @@ -543,6 +614,12 @@ func (a *CopilotAgent) ensureSession(ctx context.Context, resumeSessionID string return nil } + ctx, span := tracing.Start(ctx, "copilot.session") + defer span.End() + + isResume := resumeSessionID != "" + span.SetAttributes(fields.CopilotSessionIsNew.Bool(!isResume)) + // Detach from the caller's cancellation so the client and session // outlive individual requests (e.g., gRPC calls). sessionCtx := context.WithoutCancel(ctx) @@ -634,6 +711,8 @@ func (a *CopilotAgent) ensureSession(ctx context.Context, resumeSessionID string a.onSessionStarted(a.sessionID) } + span.SetAttributes(fields.StringHashed(fields.CopilotSessionId, a.sessionID)) + return nil } @@ -647,6 +726,7 @@ func (a *CopilotAgent) createPermissionHandler() copilot.PermissionHandlerFunc { // In headless mode, auto-approve all permission requests if a.headless { log.Printf("[copilot] PermissionRequest (headless auto-approve): kind=%s", req.Kind) + a.consentApprovedCount++ return copilot.PermissionRequestResult{Kind: "approved"}, nil } @@ -667,10 +747,12 @@ func (a *CopilotAgent) createPermissionHandler() copilot.PermissionHandlerFunc { decision, err := a.consentManager.CheckConsent(a.activeContext(), consentReq) if err != nil { log.Printf("[copilot] Consent check error for %s: %v, denying", toolID, err) + a.consentDeniedCount++ return copilot.PermissionRequestResult{Kind: "denied-by-rules"}, nil } if decision.Allowed { + a.consentApprovedCount++ return copilot.PermissionRequestResult{Kind: "approved"}, nil } @@ -693,18 +775,23 @@ func (a *CopilotAgent) createPermissionHandler() copilot.PermissionHandlerFunc { if promptErr != nil { if errors.Is(promptErr, consent.ErrToolExecutionSkipped) { // Skip — deny this tool but let the agent continue + a.consentDeniedCount++ return copilot.PermissionRequestResult{Kind: "denied-by-rules"}, nil } if errors.Is(promptErr, consent.ErrToolExecutionDenied) { // Deny — block and exit the interaction + a.consentDeniedCount++ return copilot.PermissionRequestResult{Kind: "denied-interactively-by-user"}, nil } log.Printf("[copilot] Consent grant error for %s: %v", toolID, promptErr) + a.consentDeniedCount++ return copilot.PermissionRequestResult{Kind: "denied-by-rules"}, nil } + a.consentApprovedCount++ return copilot.PermissionRequestResult{Kind: "approved"}, nil } + a.consentDeniedCount++ return copilot.PermissionRequestResult{Kind: "denied-by-rules"}, nil } } diff --git a/cli/azd/internal/tracing/fields/fields.go b/cli/azd/internal/tracing/fields/fields.go index d06a48e9659..494149a9aac 100644 --- a/cli/azd/internal/tracing/fields/fields.go +++ b/cli/azd/internal/tracing/fields/fields.go @@ -621,3 +621,123 @@ var ( Purpose: FeatureInsight, } ) + +// Copilot agent session related fields +var ( + // CopilotSessionId is the hashed session ID for correlation across messages. + CopilotSessionId = AttributeKey{ + Key: attribute.Key("copilot.session.id"), + Classification: EndUserPseudonymizedInformation, + Purpose: FeatureInsight, + } + // CopilotSessionIsNew indicates whether this was a new session (true) or resumed (false). + CopilotSessionIsNew = AttributeKey{ + Key: attribute.Key("copilot.session.isNew"), + Classification: SystemMetadata, + Purpose: FeatureInsight, + } + // CopilotSessionMessageCount is the number of messages sent in the session. + CopilotSessionMessageCount = AttributeKey{ + Key: attribute.Key("copilot.session.messageCount"), + Classification: SystemMetadata, + Purpose: FeatureInsight, + IsMeasurement: true, + } +) + +// Copilot agent initialization related fields +var ( + // CopilotInitIsFirstRun indicates whether this was the user's first agent initialization. + CopilotInitIsFirstRun = AttributeKey{ + Key: attribute.Key("copilot.init.isFirstRun"), + Classification: SystemMetadata, + Purpose: FeatureInsight, + } + // CopilotInitReasoningEffort is the reasoning level selected (low/medium/high). + CopilotInitReasoningEffort = AttributeKey{ + Key: attribute.Key("copilot.init.reasoningEffort"), + Classification: SystemMetadata, + Purpose: FeatureInsight, + } + // CopilotInitModel is the model ID selected (empty string = default). + CopilotInitModel = AttributeKey{ + Key: attribute.Key("copilot.init.model"), + Classification: SystemMetadata, + Purpose: FeatureInsight, + } + // CopilotInitConsentScope is the workflow consent scope chosen (session/project/global). + CopilotInitConsentScope = AttributeKey{ + Key: attribute.Key("copilot.init.consentScope"), + Classification: SystemMetadata, + Purpose: FeatureInsight, + } +) + +// Copilot agent mode and message related fields +var ( + // CopilotMode is the agent operating mode (interactive/autopilot/plan). + CopilotMode = AttributeKey{ + Key: attribute.Key("copilot.mode"), + Classification: SystemMetadata, + Purpose: FeatureInsight, + } + // CopilotMessageModel is the model used for a specific message. + CopilotMessageModel = AttributeKey{ + Key: attribute.Key("copilot.message.model"), + Classification: SystemMetadata, + Purpose: FeatureInsight, + } + // CopilotMessageInputTokens is the number of input tokens consumed per message. + CopilotMessageInputTokens = AttributeKey{ + Key: attribute.Key("copilot.message.inputTokens"), + Classification: SystemMetadata, + Purpose: PerformanceAndHealth, + IsMeasurement: true, + } + // CopilotMessageOutputTokens is the number of output tokens consumed per message. + CopilotMessageOutputTokens = AttributeKey{ + Key: attribute.Key("copilot.message.outputTokens"), + Classification: SystemMetadata, + Purpose: PerformanceAndHealth, + IsMeasurement: true, + } + // CopilotMessageBillingRate is the billing rate multiplier per message. + CopilotMessageBillingRate = AttributeKey{ + Key: attribute.Key("copilot.message.billingRate"), + Classification: SystemMetadata, + Purpose: BusinessInsight, + IsMeasurement: true, + } + // CopilotMessagePremiumRequests is the number of premium requests used per message. + CopilotMessagePremiumRequests = AttributeKey{ + Key: attribute.Key("copilot.message.premiumRequests"), + Classification: SystemMetadata, + Purpose: BusinessInsight, + IsMeasurement: true, + } + // CopilotMessageDurationMs is the API call duration in milliseconds per message. + CopilotMessageDurationMs = AttributeKey{ + Key: attribute.Key("copilot.message.durationMs"), + Classification: SystemMetadata, + Purpose: PerformanceAndHealth, + IsMeasurement: true, + } +) + +// Copilot consent related fields +var ( + // CopilotConsentApprovedCount is the running count of tool calls approved during the session. + CopilotConsentApprovedCount = AttributeKey{ + Key: attribute.Key("copilot.consent.approvedCount"), + Classification: SystemMetadata, + Purpose: FeatureInsight, + IsMeasurement: true, + } + // CopilotConsentDeniedCount is the running count of tool calls denied during the session. + CopilotConsentDeniedCount = AttributeKey{ + Key: attribute.Key("copilot.consent.deniedCount"), + Classification: SystemMetadata, + Purpose: FeatureInsight, + IsMeasurement: true, + } +) From 3b0061a48b9e2ad9e04c4d9be58049ddb708b643 Mon Sep 17 00:00:00 2001 From: Wallace Breza Date: Thu, 19 Mar 2026 12:42:02 -0700 Subject: [PATCH 2/2] refactor: address PR feedback on copilot telemetry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Register event names in tracing/events/events.go (CopilotInitializeEvent, CopilotSessionEvent) per convention - Remove per-message copilot.message span — too chatty; emit only session-level metrics - Consolidate all cumulative metrics into Stop() (mode, messageCount, tokens, billing, consent counts, session ID) so telemetry is captured from the core agent package regardless of caller - Remove duplicate aggregate metrics from cmd/init.go — now handled by agent.Stop() - Use defer + named returns in Initialize() and ensureSession() to capture errors via span.EndWithStatus(err) and reduce duplication - Log resumeSessionID in ensureSession defer even on early failure Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- cli/azd/cmd/init.go | 13 +-- cli/azd/internal/agent/copilot_agent.go | 100 +++++++++------------- cli/azd/internal/tracing/events/events.go | 9 ++ 3 files changed, 52 insertions(+), 70 deletions(-) diff --git a/cli/azd/cmd/init.go b/cli/azd/cmd/init.go index 4205432acbb..7bec25d86a8 100644 --- a/cli/azd/cmd/init.go +++ b/cli/azd/cmd/init.go @@ -541,19 +541,8 @@ When complete, provide a brief summary of what was accomplished.` _ = azdCtx.ClearCopilotSession() } - // Record aggregate copilot metrics as usage attributes - metrics := copilotAgent.GetMetrics() - tracing.SetUsageAttributes( - fields.CopilotMode.String(string(agent.AgentModeInteractive)), - fields.CopilotMessageModel.String(metrics.Usage.Model), - fields.CopilotMessageInputTokens.Float64(metrics.Usage.InputTokens), - fields.CopilotMessageOutputTokens.Float64(metrics.Usage.OutputTokens), - fields.CopilotMessagePremiumRequests.Float64(metrics.Usage.PremiumRequests), - fields.CopilotMessageDurationMs.Float64(metrics.Usage.DurationMS), - ) - // Show session metrics (usage + file changes) - if metricsStr := metrics.String(); metricsStr != "" { + if metricsStr := copilotAgent.GetMetrics().String(); metricsStr != "" { i.console.Message(ctx, "") i.console.Message(ctx, metricsStr) } diff --git a/cli/azd/internal/agent/copilot_agent.go b/cli/azd/internal/agent/copilot_agent.go index 80433fb5bd6..33bf4825a52 100644 --- a/cli/azd/internal/agent/copilot_agent.go +++ b/cli/azd/internal/agent/copilot_agent.go @@ -22,6 +22,7 @@ import ( "github.com/azure/azure-dev/cli/azd/internal/agent/consent" agentcopilot "github.com/azure/azure-dev/cli/azd/internal/agent/copilot" "github.com/azure/azure-dev/cli/azd/internal/tracing" + "github.com/azure/azure-dev/cli/azd/internal/tracing/events" "github.com/azure/azure-dev/cli/azd/internal/tracing/fields" "github.com/azure/azure-dev/cli/azd/pkg/config" "github.com/azure/azure-dev/cli/azd/pkg/input" @@ -75,9 +76,18 @@ type cleanupTask struct { // Initialize handles first-run configuration (model/reasoning prompts), plugin install, // and Copilot client startup. If config already exists, returns current values without // prompting. Use WithForcePrompt() to always show prompts. -func (a *CopilotAgent) Initialize(ctx context.Context, opts ...InitOption) (*InitResult, error) { - ctx, span := tracing.Start(ctx, "copilot.initialize") - defer span.End() +func (a *CopilotAgent) Initialize(ctx context.Context, opts ...InitOption) (result *InitResult, err error) { + ctx, span := tracing.Start(ctx, events.CopilotInitializeEvent) + defer func() { + if result != nil { + tracing.SetUsageAttributes( + fields.CopilotInitIsFirstRun.Bool(result.IsFirstRun), + fields.CopilotInitModel.String(result.Model), + fields.CopilotInitReasoningEffort.String(result.ReasoningEffort), + ) + } + span.EndWithStatus(err) + }() options := &initOptions{} for _, opt := range opts { @@ -114,19 +124,11 @@ func (a *CopilotAgent) Initialize(ctx context.Context, opts ...InitOption) (*Ini // If already configured and not forcing, return current config if (hasModel || hasEffort) && !options.forcePrompt { - result := &InitResult{ + return &InitResult{ Model: existingModel, ReasoningEffort: existingEffort, IsFirstRun: false, - } - - tracing.SetUsageAttributes( - fields.CopilotInitIsFirstRun.Bool(false), - fields.CopilotInitModel.String(result.Model), - fields.CopilotInitReasoningEffort.String(result.ReasoningEffort), - ) - - return result, nil + }, nil } // Prompt for reasoning effort @@ -210,19 +212,11 @@ func (a *CopilotAgent) Initialize(ctx context.Context, opts ...InitOption) (*Ini return nil, fmt.Errorf("failed to save config: %w", err) } - result := &InitResult{ + return &InitResult{ Model: selectedModel, ReasoningEffort: selectedEffort, IsFirstRun: true, - } - - tracing.SetUsageAttributes( - fields.CopilotInitIsFirstRun.Bool(true), - fields.CopilotInitModel.String(result.Model), - fields.CopilotInitReasoningEffort.String(result.ReasoningEffort), - ) - - return result, nil + }, nil } // SelectSession shows a UX picker with previous sessions for the current directory. @@ -297,9 +291,6 @@ func (a *CopilotAgent) ListSessions(ctx context.Context, cwd string) ([]SessionM // SendMessage sends a prompt to the agent and returns the result. // Creates a new session or resumes one if WithSessionID is provided. func (a *CopilotAgent) SendMessage(ctx context.Context, prompt string, opts ...SendOption) (*AgentResult, error) { - ctx, span := tracing.Start(ctx, "copilot.message") - defer span.End() - options := &sendOptions{} for _, opt := range opts { opt(options) @@ -308,10 +299,6 @@ func (a *CopilotAgent) SendMessage(ctx context.Context, prompt string, opts ...S // Update active context so SDK callbacks use this call's context a.activeCtx.Store(&ctx) - // Track whether this is a new session vs resumed - // A session is "not new" if it already exists OR if we're resuming by ID - isNewSession := a.session == nil && options.sessionID == "" - // Ensure session exists if err := a.ensureSession(ctx, options.sessionID); err != nil { return nil, err @@ -327,13 +314,6 @@ func (a *CopilotAgent) SendMessage(ctx context.Context, prompt string, opts ...S } } - // Set telemetry attributes for this message - span.SetAttributes( - fields.StringHashed(fields.CopilotSessionId, a.sessionID), - fields.CopilotSessionIsNew.Bool(isNewSession), - fields.CopilotMode.String(string(mode)), - ) - log.Printf("[copilot] SendMessage: sending prompt (%d chars, headless=%v)...", len(prompt), a.headless) var result *AgentResult @@ -352,21 +332,6 @@ func (a *CopilotAgent) SendMessage(ctx context.Context, prompt string, opts ...S // Increment message count only after successful send a.messageCount++ - // Record per-message usage metrics on the span - span.SetAttributes( - fields.CopilotMessageModel.String(result.Usage.Model), - fields.CopilotMessageInputTokens.Float64(result.Usage.InputTokens), - fields.CopilotMessageOutputTokens.Float64(result.Usage.OutputTokens), - fields.CopilotMessageBillingRate.Float64(result.Usage.BillingRate), - fields.CopilotMessagePremiumRequests.Float64(result.Usage.PremiumRequests), - fields.CopilotMessageDurationMs.Float64(result.Usage.DurationMS), - ) - - // Update cumulative usage attributes - tracing.SetUsageAttributes( - fields.CopilotSessionMessageCount.Int(a.messageCount), - ) - return result, nil } @@ -535,11 +500,22 @@ func (a *CopilotAgent) SendMessageWithRetry(ctx context.Context, prompt string, // Stop terminates the agent, cleans up the Copilot SDK client and runtime process. // Cleanup tasks run in reverse registration order. Safe to call multiple times. func (a *CopilotAgent) Stop() error { - // Record final consent counts as usage attributes + // Record all cumulative session metrics as usage attributes tracing.SetUsageAttributes( + fields.CopilotMode.String(string(a.mode)), + fields.CopilotSessionMessageCount.Int(a.messageCount), + fields.CopilotMessageModel.String(a.cumulativeUsage.Model), + fields.CopilotMessageInputTokens.Float64(a.cumulativeUsage.InputTokens), + fields.CopilotMessageOutputTokens.Float64(a.cumulativeUsage.OutputTokens), + fields.CopilotMessageBillingRate.Float64(a.cumulativeUsage.BillingRate), + fields.CopilotMessagePremiumRequests.Float64(a.cumulativeUsage.PremiumRequests), + fields.CopilotMessageDurationMs.Float64(a.cumulativeUsage.DurationMS), fields.CopilotConsentApprovedCount.Int(a.consentApprovedCount), fields.CopilotConsentDeniedCount.Int(a.consentDeniedCount), ) + if a.sessionID != "" { + tracing.SetUsageAttributes(fields.StringHashed(fields.CopilotSessionId, a.sessionID)) + } tasks := a.cleanupTasks a.cleanupTasks = nil @@ -609,13 +585,23 @@ func (a *CopilotAgent) EnsureStarted(ctx context.Context) error { // ensureSession creates or resumes a Copilot session if one doesn't exist. // Uses context.WithoutCancel to prevent the SDK session from being torn down // when a per-request context (e.g., gRPC) is cancelled between calls. -func (a *CopilotAgent) ensureSession(ctx context.Context, resumeSessionID string) error { +func (a *CopilotAgent) ensureSession(ctx context.Context, resumeSessionID string) (err error) { if a.session != nil { return nil } - ctx, span := tracing.Start(ctx, "copilot.session") - defer span.End() + ctx, span := tracing.Start(ctx, events.CopilotSessionEvent) + defer func() { + // Log session ID even on failure (e.g., resume failure still has the attempted ID) + sessionID := a.sessionID + if sessionID == "" && resumeSessionID != "" { + sessionID = resumeSessionID + } + if sessionID != "" { + span.SetAttributes(fields.StringHashed(fields.CopilotSessionId, sessionID)) + } + span.EndWithStatus(err) + }() isResume := resumeSessionID != "" span.SetAttributes(fields.CopilotSessionIsNew.Bool(!isResume)) @@ -711,8 +697,6 @@ func (a *CopilotAgent) ensureSession(ctx context.Context, resumeSessionID string a.onSessionStarted(a.sessionID) } - span.SetAttributes(fields.StringHashed(fields.CopilotSessionId, a.sessionID)) - return nil } diff --git a/cli/azd/internal/tracing/events/events.go b/cli/azd/internal/tracing/events/events.go index e18a496b8b9..d160d895d7b 100644 --- a/cli/azd/internal/tracing/events/events.go +++ b/cli/azd/internal/tracing/events/events.go @@ -29,3 +29,12 @@ const ( ExtensionRunEvent = "ext.run" ExtensionInstallEvent = "ext.install" ) + +// Copilot agent related events. +const ( + // CopilotInitializeEvent tracks the agent initialization flow (model/reasoning config). + CopilotInitializeEvent = "copilot.initialize" + + // CopilotSessionEvent tracks session creation or resumption. + CopilotSessionEvent = "copilot.session" +)