Agent target traces samples

sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_builtin_with_traces.py

@@ -7,12 +7,21 @@
 """
 DESCRIPTION:
     Given an AIProjectClient, this sample demonstrates how to run Azure AI Evaluations
-    against agent traces collected in Azure Application Insights. The sample fetches
-    trace IDs for a given agent and time range, creates an evaluation group configured
-    for trace analysis, and monitors the evaluation run until it completes.
+    against agent traces collected in Azure Application Insights.
+
+    Supports three modes:
+      - Default mode (no flags): Queries Application Insights client-side for trace IDs
+        using the AGENT_ID environment variable, then passes them to the eval service.
+      - Agent ID mode (--agent-id): Passes the agent ID directly to the eval service,
+        which resolves traces server-side from Application Insights.
+      - Trace ID mode (--trace-ids): Passes explicit trace IDs to the eval service.
 
 USAGE:
     python sample_evaluations_builtin_with_traces.py
+    python sample_evaluations_builtin_with_traces.py --agent-id "my-agent:1"
+    python sample_evaluations_builtin_with_traces.py --trace-ids abc123 def456
+    python sample_evaluations_builtin_with_traces.py --agent-id "my-agent:1" --lookback-hours 48 --max-traces 20
+    python sample_evaluations_builtin_with_traces.py --no-cleanup
 
     Before running the sample:
 
@@ -21,27 +30,27 @@
     Set these environment variables with your own values:
     1) FOUNDRY_PROJECT_ENDPOINT - Required. The Azure AI Project endpoint, as found in the overview page of your
        Microsoft Foundry project. It has the form: https://<account_name>.services.ai.azure.com/api/projects/<project_name>.
-    2) APPINSIGHTS_RESOURCE_ID - Required. The Azure Application Insights resource ID that stores agent traces.
+    2) APPINSIGHTS_RESOURCE_ID - Required (for default mode). The Azure Application Insights resource ID that stores
+       agent traces. Not needed when using --agent-id or --trace-ids.
        It has the form: /subscriptions/<subscription_id>/resourceGroups/<rg_name>/providers/Microsoft.Insights/components/<resource_name>.
     3) AGENT_ID - Required. The agent identifier emitted by the Azure tracing integration, used to filter traces.
     4) FOUNDRY_MODEL_NAME - Required. The Azure OpenAI deployment name to use with the built-in evaluators.
     5) TRACE_LOOKBACK_HOURS - Optional. Number of hours to look back when querying traces and in the evaluation run.
        Defaults to 1.
 """
 
+import argparse
 import os
 import time
 from datetime import datetime, timedelta, timezone
 from pprint import pprint
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Optional
 from dotenv import load_dotenv
 from azure.identity import DefaultAzureCredential
 from azure.monitor.query import LogsQueryClient, LogsQueryStatus
 from azure.ai.projects import AIProjectClient
 from azure.ai.projects.models import (
-    AzureAIDataSourceConfig,
     TestingCriterionAzureAIEvaluator,
-    TracesPreviewEvalRunDataSource,
 )
 
 load_dotenv()
@@ -53,7 +62,7 @@
 ]  # Sample : /subscriptions/<subscription_id>/resourceGroups/<rg_name>/providers/Microsoft.Insights/components/<resource_name>
 agent_id = os.environ["AGENT_ID"]
 model_deployment_name = os.environ["FOUNDRY_MODEL_NAME"]
-trace_query_hours = int(os.environ.get("TRACE_LOOKBACK_HOURS", "1"))
+default_lookback_hours = int(os.environ.get("TRACE_LOOKBACK_HOURS", "1"))
 
 
 def _build_evaluator_config(name: str, evaluator_name: str) -> TestingCriterionAzureAIEvaluator:
@@ -63,9 +72,9 @@ def _build_evaluator_config(name: str, evaluator_name: str) -> TestingCriterionA
         name=name,
         evaluator_name=evaluator_name,
         data_mapping={
-            "query": "{{query}}",
-            "response": "{{response}}",
-            "tool_definitions": "{{tool_definitions}}",
+            "query": "{{sample.query}}",
+            "response": "{{sample.response}}",
+            "tool_definitions": "{{sample.tool_definitions}}",
         },
         initialization_parameters={
             "deployment_name": model_deployment_name,
@@ -121,92 +130,133 @@ def get_trace_ids(
     return []
 
 
-def main() -> None:
-    end_time = datetime.now(tz=timezone.utc)
-    start_time = end_time - timedelta(hours=trace_query_hours)
+def main() -> None:  # pylint: disable=too-many-statements
+    parser = argparse.ArgumentParser(description="Run Azure AI trace evaluations against agent traces.")
+    mode = parser.add_mutually_exclusive_group()
+    mode.add_argument("--agent-id", default=None, help="Agent ID for server-side trace resolution")
+    mode.add_argument("--trace-ids", nargs="+", default=None, help="Explicit trace IDs to evaluate")
+    parser.add_argument("--lookback-hours", type=int, default=None, help="Lookback window in hours")
+    parser.add_argument("--max-traces", type=int, default=50, help="Max traces in agent-id mode (default: 50)")
+    parser.add_argument("--no-cleanup", action="store_true", help="Keep eval group after run")
+    args = parser.parse_args()
+
+    lookback_hours = args.lookback_hours or default_lookback_hours
+    trace_ids: Optional[List[str]] = None
+    agent_id_for_server: Optional[str] = None
+    metadata: Dict[str, str] = {}
+
+    if args.agent_id:
+        agent_id_for_server = args.agent_id
+        print("Mode: Server-side agent ID resolution")
+        print(f"Agent ID: {args.agent_id}")
+        print(f"Lookback: {lookback_hours}h, Max traces: {args.max_traces}")
+        metadata["agent_id"] = args.agent_id
+
+    elif args.trace_ids:
+        trace_ids = list(args.trace_ids)
+        print(f"Mode: Explicit trace IDs ({len(trace_ids)} provided)")
+
+    else:
+        end_time = datetime.now(tz=timezone.utc)
+        start_time = end_time - timedelta(hours=lookback_hours)
 
-    print("Querying Application Insights for trace identifiers...")
-    print(f"Agent ID: {agent_id}")
-    print(f"Time range: {start_time.isoformat()} to {end_time.isoformat()}")
+        print("Querying Application Insights for trace identifiers...")
+        print(f"Agent ID: {agent_id}")
+        print(f"Time range: {start_time.isoformat()} to {end_time.isoformat()}")
 
-    trace_ids = get_trace_ids(appinsights_resource_id, agent_id, start_time, end_time)
+        trace_ids = get_trace_ids(appinsights_resource_id, agent_id, start_time, end_time)
 
-    if not trace_ids:
-        print("No trace IDs found for the provided agent and time window.")
-        return
+        if not trace_ids:
+            print("No trace IDs found for the provided agent and time window.")
+            return
 
-    print(f"\nFound {len(trace_ids)} trace IDs:")
-    for trace_id in trace_ids:
-        print(f"  - {trace_id}")
+        print(f"\nFound {len(trace_ids)} trace IDs:")
+        for tid in trace_ids:
+            print(f"  - {tid}")
+
+        metadata["agent_id"] = agent_id
+        metadata["start_time"] = start_time.isoformat()
+        metadata["end_time"] = end_time.isoformat()
 
     with DefaultAzureCredential() as credential:
         with AIProjectClient(endpoint=endpoint, credential=credential) as project_client:
             client = project_client.get_openai_client()
-            data_source_config = AzureAIDataSourceConfig(
-                type="azure_ai_source",
-                scenario="traces",
-            )
 
-        testing_criteria = [
-            _build_evaluator_config(
-                name="intent_resolution",
-                evaluator_name="builtin.intent_resolution",
-            ),
-            _build_evaluator_config(
-                name="task_adherence",
-                evaluator_name="builtin.task_adherence",
-            ),
-        ]
-
-        print("\nCreating evaluation")
-        eval_object = client.evals.create(
-            name="agent_trace_eval_group",
-            data_source_config=data_source_config,
-            testing_criteria=testing_criteria,
-        )
-        print(f"Evaluation created (id: {eval_object.id}, name: {eval_object.name})")
-
-        print("\nGet Evaluation by Id")
-        eval_object_response = client.evals.retrieve(eval_object.id)
-        print("Evaluation Response:")
-        pprint(eval_object_response)
-
-        print("\nCreating Eval Run with trace IDs")
-        run_name = f"agent_trace_eval_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
-        data_source = TracesPreviewEvalRunDataSource(
-            type="azure_ai_traces_preview",
-            trace_ids=trace_ids,
-            lookback_hours=trace_query_hours,
-        )
-        eval_run_object = client.evals.runs.create(
-            eval_id=eval_object.id,
-            name=run_name,
-            metadata={
-                "agent_id": agent_id,
-                "start_time": start_time.isoformat(),
-                "end_time": end_time.isoformat(),
-            },
-            data_source=data_source,
-        )
-        print("Eval Run created")
-        pprint(eval_run_object)
-
-        print("\nMonitoring Eval Run status...")
-        while True:
-            run = client.evals.runs.retrieve(run_id=eval_run_object.id, eval_id=eval_object.id)
-            print(f"Status: {run.status}")
-
-            if run.status in {"completed", "failed", "canceled"}:
-                print("\nEval Run finished!")
-                print("Final Eval Run Response:")
-                pprint(run)
-                break
-
-            time.sleep(5)
-            print("Waiting for eval run to complete...")
-
-        client.evals.delete(eval_id=eval_object.id)
-        print("Evaluation deleted")
+            data_source_config = {
+                "type": "azure_ai_source",
+                "scenario": "traces",
+            }
+
+            testing_criteria = [
+                _build_evaluator_config(
+                    name="intent_resolution",
+                    evaluator_name="builtin.intent_resolution",
+                ),
+                _build_evaluator_config(
+                    name="task_adherence",
+                    evaluator_name="builtin.task_adherence",
+                ),
+            ]
+
+            print("\nCreating evaluation")
+            eval_object = client.evals.create(
+                name="agent_trace_eval_group",
+                data_source_config=data_source_config,  # type: ignore
+                testing_criteria=testing_criteria,  # type: ignore
+            )
+            print(f"Evaluation created (id: {eval_object.id}, name: {eval_object.name})")
+
+            print("\nGet Evaluation by Id")
+            eval_object_response = client.evals.retrieve(eval_object.id)
+            print("Evaluation Response:")
+            pprint(eval_object_response)
+
+            # Build data source based on mode
+            if agent_id_for_server:
+                data_source: Dict[str, Any] = {
+                    "type": "azure_ai_traces",
+                    "agent_id": agent_id_for_server,
+                    "lookback_hours": lookback_hours,
+                    "max_traces": args.max_traces,
+                }
+            else:
+                assert trace_ids is not None
+                data_source = {
+                    "type": "azure_ai_traces",
+                    "trace_ids": trace_ids,
+                    "lookback_hours": lookback_hours,
+                }
+
+            print("\nCreating Eval Run")
+            run_name = f"agent_trace_eval_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
+            eval_run_object = client.evals.runs.create(
+                eval_id=eval_object.id,
+                name=run_name,
+                metadata=metadata if metadata else None,
+                data_source=data_source,  # type: ignore
+            )
+            print("Eval Run created")
+            pprint(eval_run_object)
+
+            print("\nMonitoring Eval Run status...")
+            while True:
+                run = client.evals.runs.retrieve(run_id=eval_run_object.id, eval_id=eval_object.id)
+                print(f"Status: {run.status}")
+
+                if run.status in {"completed", "failed", "canceled"}:
+                    print("\nEval Run finished!")
+                    print("Final Eval Run Response:")
+                    pprint(run)
+                    break
+
+                time.sleep(5)
+                print("Waiting for eval run to complete...")
+
+            if not args.no_cleanup:
+                client.evals.delete(eval_id=eval_object.id)
+                print("Evaluation deleted")
+            else:
+                print(f"Skipping cleanup (--no-cleanup). Eval ID: {eval_object.id}")
 
 
 if __name__ == "__main__":

sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_score_model_grader_with_audio.py

@@ -26,13 +26,15 @@
     3) AZURE_AI_MODEL_DEPLOYMENT_NAME_FOR_AUDIO - Required. The name of the model deployment for audio to use for evaluation, recommend to use "gpt-4o-audio-preview"
 """
 
-import os
 import base64
-
-from azure.identity import DefaultAzureCredential
-from azure.ai.projects import AIProjectClient
+import os
 import time
 from pprint import pprint
+
+from dotenv import load_dotenv
+from azure.ai.projects import AIProjectClient
+from azure.identity import DefaultAzureCredential
+from openai.types.eval_create_params import DataSourceConfigCustom
 from openai.types.evals.create_eval_completions_run_data_source_param import (
     CreateEvalCompletionsRunDataSourceParam,
     SourceFileContent,
@@ -41,8 +43,6 @@
     InputMessagesTemplateTemplateEvalItem,
 )
 from openai.types.responses import EasyInputMessageParam, ResponseInputAudioParam
-from openai.types.eval_create_params import DataSourceConfigCustom
-from dotenv import load_dotenv
 
 load_dotenv()
 file_path = os.path.abspath(__file__)
@@ -54,7 +54,13 @@
 
 
 def audio_to_base64(audio_path: str) -> str:
-    """Read an audio file and return its base64-encoded content."""
+    """Read an audio file and return its base64-encoded content.
+
+    :param audio_path: Absolute path to the input audio file.
+    :type audio_path: str
+    :return: Base64-encoded audio data.
+    :rtype: str
+    """
     with open(audio_path, "rb") as f:
         return base64.b64encode(f.read()).decode("utf-8")
 
@@ -183,7 +189,7 @@ def audio_to_base64(audio_path: str) -> str:
 
     while True:
         run = client.evals.runs.retrieve(run_id=eval_run_response.id, eval_id=eval_object.id)
-        if run.status == "completed" or run.status == "failed":
+        if run.status in {"completed", "failed"}:
             output_items = list(client.evals.runs.output_items.list(run_id=run.id, eval_id=eval_object.id))
             pprint(output_items)
             print(f"Eval Run Report URL: {run.report_url}")

sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_score_model_grader_with_audio_model_target.py

@@ -26,22 +26,22 @@
     3) AZURE_AI_MODEL_DEPLOYMENT_NAME_FOR_AUDIO - Required. The name of the model deployment for audio to use for evaluation, recommend to use "gpt-4o-audio-preview"
 """
 
-import os
 import base64
-
-from azure.identity import DefaultAzureCredential
-from azure.ai.projects import AIProjectClient
+import os
 import time
 from pprint import pprint
+
+from dotenv import load_dotenv
+from azure.ai.projects import AIProjectClient
+from azure.identity import DefaultAzureCredential
+from openai.types.eval_create_params import DataSourceConfigCustom
 from openai.types.evals.create_eval_completions_run_data_source_param import (
     SourceFileContent,
     SourceFileContentContent,
     InputMessagesTemplate,
     InputMessagesTemplateTemplateEvalItem,
 )
 from openai.types.responses import EasyInputMessageParam, ResponseInputAudioParam
-from openai.types.eval_create_params import DataSourceConfigCustom
-from dotenv import load_dotenv
 
 load_dotenv()
 file_path = os.path.abspath(__file__)
@@ -53,7 +53,13 @@
 
 
 def audio_to_base64(audio_path: str) -> str:
-    """Read an audio file and return its base64-encoded content."""
+    """Read an audio file and return its base64-encoded content.
+
+    :param audio_path: Absolute path to the input audio file.
+    :type audio_path: str
+    :return: Base64-encoded audio data.
+    :rtype: str
+    """
     with open(audio_path, "rb") as f:
         return base64.b64encode(f.read()).decode("utf-8")
 
@@ -188,7 +194,7 @@ def audio_to_base64(audio_path: str) -> str:
 
     while True:
         run = client.evals.runs.retrieve(run_id=eval_run_response.id, eval_id=eval_object.id)
-        if run.status == "completed" or run.status == "failed":
+        if run.status in {"completed", "failed"}:
             output_items = list(client.evals.runs.output_items.list(run_id=run.id, eval_id=eval_object.id))
             pprint(output_items)
             print(f"Eval Run Report URL: {run.report_url}")