diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_builtin_with_traces.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_builtin_with_traces.py index dd2271d4d6b7..a5e88ab53301 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_builtin_with_traces.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_builtin_with_traces.py @@ -7,12 +7,21 @@ """ DESCRIPTION: Given an AIProjectClient, this sample demonstrates how to run Azure AI Evaluations - against agent traces collected in Azure Application Insights. The sample fetches - trace IDs for a given agent and time range, creates an evaluation group configured - for trace analysis, and monitors the evaluation run until it completes. + against agent traces collected in Azure Application Insights. + + Supports three modes: + - Default mode (no flags): Queries Application Insights client-side for trace IDs + using the AGENT_ID environment variable, then passes them to the eval service. + - Agent ID mode (--agent-id): Passes the agent ID directly to the eval service, + which resolves traces server-side from Application Insights. + - Trace ID mode (--trace-ids): Passes explicit trace IDs to the eval service. USAGE: python sample_evaluations_builtin_with_traces.py + python sample_evaluations_builtin_with_traces.py --agent-id "my-agent:1" + python sample_evaluations_builtin_with_traces.py --trace-ids abc123 def456 + python sample_evaluations_builtin_with_traces.py --agent-id "my-agent:1" --lookback-hours 48 --max-traces 20 + python sample_evaluations_builtin_with_traces.py --no-cleanup Before running the sample: @@ -21,7 +30,8 @@ Set these environment variables with your own values: 1) FOUNDRY_PROJECT_ENDPOINT - Required. The Azure AI Project endpoint, as found in the overview page of your Microsoft Foundry project. It has the form: https://.services.ai.azure.com/api/projects/. - 2) APPINSIGHTS_RESOURCE_ID - Required. The Azure Application Insights resource ID that stores agent traces. + 2) APPINSIGHTS_RESOURCE_ID - Required (for default mode). The Azure Application Insights resource ID that stores + agent traces. Not needed when using --agent-id or --trace-ids. It has the form: /subscriptions//resourceGroups//providers/Microsoft.Insights/components/. 3) AGENT_ID - Required. The agent identifier emitted by the Azure tracing integration, used to filter traces. 4) FOUNDRY_MODEL_NAME - Required. The Azure OpenAI deployment name to use with the built-in evaluators. @@ -29,19 +39,18 @@ Defaults to 1. """ +import argparse import os import time from datetime import datetime, timedelta, timezone from pprint import pprint -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional from dotenv import load_dotenv from azure.identity import DefaultAzureCredential from azure.monitor.query import LogsQueryClient, LogsQueryStatus from azure.ai.projects import AIProjectClient from azure.ai.projects.models import ( - AzureAIDataSourceConfig, TestingCriterionAzureAIEvaluator, - TracesPreviewEvalRunDataSource, ) load_dotenv() @@ -53,7 +62,7 @@ ] # Sample : /subscriptions//resourceGroups//providers/Microsoft.Insights/components/ agent_id = os.environ["AGENT_ID"] model_deployment_name = os.environ["FOUNDRY_MODEL_NAME"] -trace_query_hours = int(os.environ.get("TRACE_LOOKBACK_HOURS", "1")) +default_lookback_hours = int(os.environ.get("TRACE_LOOKBACK_HOURS", "1")) def _build_evaluator_config(name: str, evaluator_name: str) -> TestingCriterionAzureAIEvaluator: @@ -63,9 +72,9 @@ def _build_evaluator_config(name: str, evaluator_name: str) -> TestingCriterionA name=name, evaluator_name=evaluator_name, data_mapping={ - "query": "{{query}}", - "response": "{{response}}", - "tool_definitions": "{{tool_definitions}}", + "query": "{{sample.query}}", + "response": "{{sample.response}}", + "tool_definitions": "{{sample.tool_definitions}}", }, initialization_parameters={ "deployment_name": model_deployment_name, @@ -121,92 +130,133 @@ def get_trace_ids( return [] -def main() -> None: - end_time = datetime.now(tz=timezone.utc) - start_time = end_time - timedelta(hours=trace_query_hours) +def main() -> None: # pylint: disable=too-many-statements + parser = argparse.ArgumentParser(description="Run Azure AI trace evaluations against agent traces.") + mode = parser.add_mutually_exclusive_group() + mode.add_argument("--agent-id", default=None, help="Agent ID for server-side trace resolution") + mode.add_argument("--trace-ids", nargs="+", default=None, help="Explicit trace IDs to evaluate") + parser.add_argument("--lookback-hours", type=int, default=None, help="Lookback window in hours") + parser.add_argument("--max-traces", type=int, default=50, help="Max traces in agent-id mode (default: 50)") + parser.add_argument("--no-cleanup", action="store_true", help="Keep eval group after run") + args = parser.parse_args() + + lookback_hours = args.lookback_hours or default_lookback_hours + trace_ids: Optional[List[str]] = None + agent_id_for_server: Optional[str] = None + metadata: Dict[str, str] = {} + + if args.agent_id: + agent_id_for_server = args.agent_id + print("Mode: Server-side agent ID resolution") + print(f"Agent ID: {args.agent_id}") + print(f"Lookback: {lookback_hours}h, Max traces: {args.max_traces}") + metadata["agent_id"] = args.agent_id + + elif args.trace_ids: + trace_ids = list(args.trace_ids) + print(f"Mode: Explicit trace IDs ({len(trace_ids)} provided)") + + else: + end_time = datetime.now(tz=timezone.utc) + start_time = end_time - timedelta(hours=lookback_hours) - print("Querying Application Insights for trace identifiers...") - print(f"Agent ID: {agent_id}") - print(f"Time range: {start_time.isoformat()} to {end_time.isoformat()}") + print("Querying Application Insights for trace identifiers...") + print(f"Agent ID: {agent_id}") + print(f"Time range: {start_time.isoformat()} to {end_time.isoformat()}") - trace_ids = get_trace_ids(appinsights_resource_id, agent_id, start_time, end_time) + trace_ids = get_trace_ids(appinsights_resource_id, agent_id, start_time, end_time) - if not trace_ids: - print("No trace IDs found for the provided agent and time window.") - return + if not trace_ids: + print("No trace IDs found for the provided agent and time window.") + return - print(f"\nFound {len(trace_ids)} trace IDs:") - for trace_id in trace_ids: - print(f" - {trace_id}") + print(f"\nFound {len(trace_ids)} trace IDs:") + for tid in trace_ids: + print(f" - {tid}") + + metadata["agent_id"] = agent_id + metadata["start_time"] = start_time.isoformat() + metadata["end_time"] = end_time.isoformat() with DefaultAzureCredential() as credential: with AIProjectClient(endpoint=endpoint, credential=credential) as project_client: client = project_client.get_openai_client() - data_source_config = AzureAIDataSourceConfig( - type="azure_ai_source", - scenario="traces", - ) - testing_criteria = [ - _build_evaluator_config( - name="intent_resolution", - evaluator_name="builtin.intent_resolution", - ), - _build_evaluator_config( - name="task_adherence", - evaluator_name="builtin.task_adherence", - ), - ] - - print("\nCreating evaluation") - eval_object = client.evals.create( - name="agent_trace_eval_group", - data_source_config=data_source_config, - testing_criteria=testing_criteria, - ) - print(f"Evaluation created (id: {eval_object.id}, name: {eval_object.name})") - - print("\nGet Evaluation by Id") - eval_object_response = client.evals.retrieve(eval_object.id) - print("Evaluation Response:") - pprint(eval_object_response) - - print("\nCreating Eval Run with trace IDs") - run_name = f"agent_trace_eval_{datetime.now().strftime('%Y%m%d_%H%M%S')}" - data_source = TracesPreviewEvalRunDataSource( - type="azure_ai_traces_preview", - trace_ids=trace_ids, - lookback_hours=trace_query_hours, - ) - eval_run_object = client.evals.runs.create( - eval_id=eval_object.id, - name=run_name, - metadata={ - "agent_id": agent_id, - "start_time": start_time.isoformat(), - "end_time": end_time.isoformat(), - }, - data_source=data_source, - ) - print("Eval Run created") - pprint(eval_run_object) - - print("\nMonitoring Eval Run status...") - while True: - run = client.evals.runs.retrieve(run_id=eval_run_object.id, eval_id=eval_object.id) - print(f"Status: {run.status}") - - if run.status in {"completed", "failed", "canceled"}: - print("\nEval Run finished!") - print("Final Eval Run Response:") - pprint(run) - break - - time.sleep(5) - print("Waiting for eval run to complete...") - - client.evals.delete(eval_id=eval_object.id) - print("Evaluation deleted") + data_source_config = { + "type": "azure_ai_source", + "scenario": "traces", + } + + testing_criteria = [ + _build_evaluator_config( + name="intent_resolution", + evaluator_name="builtin.intent_resolution", + ), + _build_evaluator_config( + name="task_adherence", + evaluator_name="builtin.task_adherence", + ), + ] + + print("\nCreating evaluation") + eval_object = client.evals.create( + name="agent_trace_eval_group", + data_source_config=data_source_config, # type: ignore + testing_criteria=testing_criteria, # type: ignore + ) + print(f"Evaluation created (id: {eval_object.id}, name: {eval_object.name})") + + print("\nGet Evaluation by Id") + eval_object_response = client.evals.retrieve(eval_object.id) + print("Evaluation Response:") + pprint(eval_object_response) + + # Build data source based on mode + if agent_id_for_server: + data_source: Dict[str, Any] = { + "type": "azure_ai_traces", + "agent_id": agent_id_for_server, + "lookback_hours": lookback_hours, + "max_traces": args.max_traces, + } + else: + assert trace_ids is not None + data_source = { + "type": "azure_ai_traces", + "trace_ids": trace_ids, + "lookback_hours": lookback_hours, + } + + print("\nCreating Eval Run") + run_name = f"agent_trace_eval_{datetime.now().strftime('%Y%m%d_%H%M%S')}" + eval_run_object = client.evals.runs.create( + eval_id=eval_object.id, + name=run_name, + metadata=metadata if metadata else None, + data_source=data_source, # type: ignore + ) + print("Eval Run created") + pprint(eval_run_object) + + print("\nMonitoring Eval Run status...") + while True: + run = client.evals.runs.retrieve(run_id=eval_run_object.id, eval_id=eval_object.id) + print(f"Status: {run.status}") + + if run.status in {"completed", "failed", "canceled"}: + print("\nEval Run finished!") + print("Final Eval Run Response:") + pprint(run) + break + + time.sleep(5) + print("Waiting for eval run to complete...") + + if not args.no_cleanup: + client.evals.delete(eval_id=eval_object.id) + print("Evaluation deleted") + else: + print(f"Skipping cleanup (--no-cleanup). Eval ID: {eval_object.id}") if __name__ == "__main__": diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_score_model_grader_with_audio.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_score_model_grader_with_audio.py index 7583f1e8636e..165bd0f43929 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_score_model_grader_with_audio.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_score_model_grader_with_audio.py @@ -26,13 +26,15 @@ 3) AZURE_AI_MODEL_DEPLOYMENT_NAME_FOR_AUDIO - Required. The name of the model deployment for audio to use for evaluation, recommend to use "gpt-4o-audio-preview" """ -import os import base64 - -from azure.identity import DefaultAzureCredential -from azure.ai.projects import AIProjectClient +import os import time from pprint import pprint + +from dotenv import load_dotenv +from azure.ai.projects import AIProjectClient +from azure.identity import DefaultAzureCredential +from openai.types.eval_create_params import DataSourceConfigCustom from openai.types.evals.create_eval_completions_run_data_source_param import ( CreateEvalCompletionsRunDataSourceParam, SourceFileContent, @@ -41,8 +43,6 @@ InputMessagesTemplateTemplateEvalItem, ) from openai.types.responses import EasyInputMessageParam, ResponseInputAudioParam -from openai.types.eval_create_params import DataSourceConfigCustom -from dotenv import load_dotenv load_dotenv() file_path = os.path.abspath(__file__) @@ -54,7 +54,13 @@ def audio_to_base64(audio_path: str) -> str: - """Read an audio file and return its base64-encoded content.""" + """Read an audio file and return its base64-encoded content. + + :param audio_path: Absolute path to the input audio file. + :type audio_path: str + :return: Base64-encoded audio data. + :rtype: str + """ with open(audio_path, "rb") as f: return base64.b64encode(f.read()).decode("utf-8") @@ -183,7 +189,7 @@ def audio_to_base64(audio_path: str) -> str: while True: run = client.evals.runs.retrieve(run_id=eval_run_response.id, eval_id=eval_object.id) - if run.status == "completed" or run.status == "failed": + if run.status in {"completed", "failed"}: output_items = list(client.evals.runs.output_items.list(run_id=run.id, eval_id=eval_object.id)) pprint(output_items) print(f"Eval Run Report URL: {run.report_url}") diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_score_model_grader_with_audio_model_target.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_score_model_grader_with_audio_model_target.py index 5c3c4ae51f78..75eb7b97f566 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_score_model_grader_with_audio_model_target.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_score_model_grader_with_audio_model_target.py @@ -26,13 +26,15 @@ 3) AZURE_AI_MODEL_DEPLOYMENT_NAME_FOR_AUDIO - Required. The name of the model deployment for audio to use for evaluation, recommend to use "gpt-4o-audio-preview" """ -import os import base64 - -from azure.identity import DefaultAzureCredential -from azure.ai.projects import AIProjectClient +import os import time from pprint import pprint + +from dotenv import load_dotenv +from azure.ai.projects import AIProjectClient +from azure.identity import DefaultAzureCredential +from openai.types.eval_create_params import DataSourceConfigCustom from openai.types.evals.create_eval_completions_run_data_source_param import ( SourceFileContent, SourceFileContentContent, @@ -40,8 +42,6 @@ InputMessagesTemplateTemplateEvalItem, ) from openai.types.responses import EasyInputMessageParam, ResponseInputAudioParam -from openai.types.eval_create_params import DataSourceConfigCustom -from dotenv import load_dotenv load_dotenv() file_path = os.path.abspath(__file__) @@ -53,7 +53,13 @@ def audio_to_base64(audio_path: str) -> str: - """Read an audio file and return its base64-encoded content.""" + """Read an audio file and return its base64-encoded content. + + :param audio_path: Absolute path to the input audio file. + :type audio_path: str + :return: Base64-encoded audio data. + :rtype: str + """ with open(audio_path, "rb") as f: return base64.b64encode(f.read()).decode("utf-8") @@ -188,7 +194,7 @@ def audio_to_base64(audio_path: str) -> str: while True: run = client.evals.runs.retrieve(run_id=eval_run_response.id, eval_id=eval_object.id) - if run.status == "completed" or run.status == "failed": + if run.status in {"completed", "failed"}: output_items = list(client.evals.runs.output_items.list(run_id=run.id, eval_id=eval_object.id)) pprint(output_items) print(f"Eval Run Report URL: {run.report_url}") diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_score_model_grader_with_image_model_target.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_score_model_grader_with_image_model_target.py index 88920549113d..01a22ccfe588 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_score_model_grader_with_image_model_target.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_score_model_grader_with_image_model_target.py @@ -23,15 +23,17 @@ 2) AZURE_AI_MODEL_DEPLOYMENT_NAME - Required. The name of the model deployment to use for evaluation. """ -import os import base64 -from PIL import Image +import os +import time from io import BytesIO +from pprint import pprint -from azure.identity import DefaultAzureCredential +from PIL import Image +from dotenv import load_dotenv from azure.ai.projects import AIProjectClient -import time -from pprint import pprint +from azure.identity import DefaultAzureCredential +from openai.types.eval_create_params import DataSourceConfigCustom from openai.types.evals.create_eval_completions_run_data_source_param import ( SourceFileContent, SourceFileContentContent, @@ -40,8 +42,6 @@ InputMessagesTemplateTemplateEvalItemContentInputImage, ) from openai.types.responses import EasyInputMessageParam -from openai.types.eval_create_params import DataSourceConfigCustom -from dotenv import load_dotenv load_dotenv() file_path = os.path.abspath(__file__) @@ -51,8 +51,8 @@ model_deployment_name = os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"] -def image_to_data_uri(image_path: str) -> str: - with Image.open(image_path) as img: +def image_to_data_uri(image_file_path: str) -> str: + with Image.open(image_file_path) as img: buffered = BytesIO() img.save(buffered, format=img.format or "PNG") img_str = base64.b64encode(buffered.getvalue()).decode() @@ -186,7 +186,7 @@ def image_to_data_uri(image_path: str) -> str: while True: run = client.evals.runs.retrieve(run_id=eval_run_response.id, eval_id=eval_object.id) - if run.status == "completed" or run.status == "failed": + if run.status in {"completed", "failed"}: output_items = list(client.evals.runs.output_items.list(run_id=run.id, eval_id=eval_object.id)) pprint(output_items) print(f"Eval Run Report URL: {run.report_url}")