From aec49159120df9631f63c89b4aff5fa3928cc123 Mon Sep 17 00:00:00 2001 From: Shruti Iyer <9905402+shrutiyer@users.noreply.github.com> Date: Fri, 10 Apr 2026 16:30:40 -0400 Subject: [PATCH 01/18] Enhance sample script for Azure AI evaluations Updated the sample script to support additional command-line arguments for trace evaluations, including agent ID and trace IDs. Modified the lookback hours default value and improved the overall structure for better clarity. --- .../sample_evaluations_builtin_with_traces.py | 225 +++++++++++------- 1 file changed, 138 insertions(+), 87 deletions(-) diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_builtin_with_traces.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_builtin_with_traces.py index 33aff799eb4f..738278e5259e 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_builtin_with_traces.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_builtin_with_traces.py @@ -7,12 +7,21 @@ """ DESCRIPTION: Given an AIProjectClient, this sample demonstrates how to run Azure AI Evaluations - against agent traces collected in Azure Application Insights. The sample fetches - trace IDs for a given agent and time range, creates an evaluation group configured - for trace analysis, and monitors the evaluation run until it completes. + against agent traces collected in Azure Application Insights. + + Supports three modes: + - Default mode (no flags): Queries Application Insights client-side for trace IDs + using the AGENT_ID environment variable, then passes them to the eval service. + - Agent ID mode (--agent-id): Passes the agent ID directly to the eval service, + which resolves traces server-side from Application Insights. + - Trace ID mode (--trace-ids): Passes explicit trace IDs to the eval service. USAGE: python sample_evaluations_builtin_with_traces.py + python sample_evaluations_builtin_with_traces.py --agent-id "my-agent:1" + python sample_evaluations_builtin_with_traces.py --trace-ids abc123 def456 + python sample_evaluations_builtin_with_traces.py --agent-id "my-agent:1" --lookback-hours 48 --max-traces 20 + python sample_evaluations_builtin_with_traces.py --no-cleanup Before running the sample: @@ -21,18 +30,21 @@ Set these environment variables with your own values: 1) AZURE_AI_PROJECT_ENDPOINT - Required. The Azure AI Project endpoint, as found in the overview page of your Microsoft Foundry project. It has the form: https://.services.ai.azure.com/api/projects/. - 2) APPINSIGHTS_RESOURCE_ID - Required. The Azure Application Insights resource ID that stores agent traces. + 2) APPINSIGHTS_RESOURCE_ID - Required (for default mode). The Azure Application Insights resource ID that stores + agent traces. Not needed when using --agent-id or --trace-ids. It has the form: /subscriptions//resourceGroups//providers/Microsoft.Insights/components/. - 3) AGENT_ID - Required. The agent identifier emitted by the Azure tracing integration, used to filter traces. + 3) AGENT_ID - Required (for default mode). The agent identifier emitted by the Azure tracing integration, + used to filter traces. Not needed when using --agent-id or --trace-ids. 4) AZURE_AI_MODEL_DEPLOYMENT_NAME - Required. The Azure OpenAI deployment name to use with the built-in evaluators. 5) TRACE_LOOKBACK_HOURS - Optional. Number of hours to look back when querying traces and in the evaluation run. Defaults to 1. """ +import argparse import os import time from datetime import datetime, timedelta, timezone -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional from dotenv import load_dotenv from azure.identity import DefaultAzureCredential @@ -45,12 +57,8 @@ endpoint = os.environ["AZURE_AI_PROJECT_ENDPOINT"] -appinsights_resource_id = os.environ[ - "APPINSIGHTS_RESOURCE_ID" -] # Sample : /subscriptions//resourceGroups//providers/Microsoft.Insights/components/ -agent_id = os.environ["AGENT_ID"] model_deployment_name = os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"] -trace_query_hours = int(os.environ.get("TRACE_LOOKBACK_HOURS", "1")) +default_lookback_hours = int(os.environ.get("TRACE_LOOKBACK_HOURS", "168")) def _build_evaluator_config(name: str, evaluator_name: str) -> Dict[str, Any]: @@ -119,91 +127,134 @@ def get_trace_ids( def main() -> None: - end_time = datetime.now(tz=timezone.utc) - start_time = end_time - timedelta(hours=trace_query_hours) - - print("Querying Application Insights for trace identifiers...") - print(f"Agent ID: {agent_id}") - print(f"Time range: {start_time.isoformat()} to {end_time.isoformat()}") - - trace_ids = get_trace_ids(appinsights_resource_id, agent_id, start_time, end_time) - - if not trace_ids: - print("No trace IDs found for the provided agent and time window.") - return - - print(f"\nFound {len(trace_ids)} trace IDs:") - for trace_id in trace_ids: - print(f" - {trace_id}") + parser = argparse.ArgumentParser(description="Run Azure AI trace evaluations against agent traces.") + mode = parser.add_mutually_exclusive_group() + mode.add_argument("--agent-id", default=None, help="Agent ID for server-side trace resolution") + mode.add_argument("--trace-ids", nargs="+", default=None, help="Explicit trace IDs to evaluate") + parser.add_argument("--lookback-hours", type=int, default=None, help="Lookback window in hours") + parser.add_argument("--max-traces", type=int, default=50, help="Max traces in agent-id mode (default: 50)") + parser.add_argument("--no-cleanup", action="store_true", help="Keep eval group after run") + args = parser.parse_args() + + lookback_hours = args.lookback_hours or default_lookback_hours + trace_ids: Optional[List[str]] = None + agent_id_for_server: Optional[str] = None + metadata: Dict[str, str] = {} + + if args.agent_id: + agent_id_for_server = args.agent_id + print(f"Mode: Server-side agent ID resolution") + print(f"Agent ID: {args.agent_id}") + print(f"Lookback: {lookback_hours}h, Max traces: {args.max_traces}") + metadata["agent_id"] = args.agent_id + + elif args.trace_ids: + trace_ids = args.trace_ids + print(f"Mode: Explicit trace IDs ({len(trace_ids)} provided)") + + else: + appinsights_resource_id = os.environ["APPINSIGHTS_RESOURCE_ID"] + agent_id = os.environ["AGENT_ID"] + end_time = datetime.now(tz=timezone.utc) + start_time = end_time - timedelta(hours=lookback_hours) + + print("Querying Application Insights for trace identifiers...") + print(f"Agent ID: {agent_id}") + print(f"Time range: {start_time.isoformat()} to {end_time.isoformat()}") + + trace_ids = get_trace_ids(appinsights_resource_id, agent_id, start_time, end_time) + + if not trace_ids: + print("No trace IDs found for the provided agent and time window.") + return + + print(f"\nFound {len(trace_ids)} trace IDs:") + for tid in trace_ids: + print(f" - {tid}") + + metadata["agent_id"] = agent_id + metadata["start_time"] = start_time.isoformat() + metadata["end_time"] = end_time.isoformat() with DefaultAzureCredential() as credential: with AIProjectClient(endpoint=endpoint, credential=credential) as project_client: client = project_client.get_openai_client() + data_source_config = { "type": "azure_ai_source", "scenario": "traces", } - testing_criteria = [ - _build_evaluator_config( - name="intent_resolution", - evaluator_name="builtin.intent_resolution", - ), - _build_evaluator_config( - name="task_adherence", - evaluator_name="builtin.task_adherence", - ), - ] - - print("\nCreating evaluation") - eval_object = client.evals.create( - name="agent_trace_eval_group", - data_source_config=data_source_config, # type: ignore - testing_criteria=testing_criteria, # type: ignore - ) - print(f"Evaluation created (id: {eval_object.id}, name: {eval_object.name})") - - print("\nGet Evaluation by Id") - eval_object_response = client.evals.retrieve(eval_object.id) - print("Evaluation Response:") - pprint(eval_object_response) - - print("\nCreating Eval Run with trace IDs") - run_name = f"agent_trace_eval_{datetime.now().strftime('%Y%m%d_%H%M%S')}" - data_source = { - "type": "azure_ai_traces", - "trace_ids": trace_ids, - "lookback_hours": trace_query_hours, - } - eval_run_object = client.evals.runs.create( - eval_id=eval_object.id, - name=run_name, - metadata={ - "agent_id": agent_id, - "start_time": start_time.isoformat(), - "end_time": end_time.isoformat(), - }, - data_source=data_source, # type: ignore - ) - print("Eval Run created") - pprint(eval_run_object) - - print("\nMonitoring Eval Run status...") - while True: - run = client.evals.runs.retrieve(run_id=eval_run_object.id, eval_id=eval_object.id) - print(f"Status: {run.status}") - - if run.status in {"completed", "failed", "canceled"}: - print("\nEval Run finished!") - print("Final Eval Run Response:") - pprint(run) - break - - time.sleep(5) - print("Waiting for eval run to complete...") - - client.evals.delete(eval_id=eval_object.id) - print("Evaluation deleted") + testing_criteria = [ + _build_evaluator_config( + name="intent_resolution", + evaluator_name="builtin.intent_resolution", + ), + _build_evaluator_config( + name="task_adherence", + evaluator_name="builtin.task_adherence", + ), + ] + + print("\nCreating evaluation") + eval_object = client.evals.create( + name="agent_trace_eval_group", + data_source_config=data_source_config, # type: ignore + testing_criteria=testing_criteria, # type: ignore + ) + print(f"Evaluation created (id: {eval_object.id}, name: {eval_object.name})") + + print("\nGet Evaluation by Id") + eval_object_response = client.evals.retrieve(eval_object.id) + print("Evaluation Response:") + pprint(eval_object_response) + + # Build data source based on mode + if agent_id_for_server: + data_source: Dict[str, Any] = { + "type": "azure_ai_traces", + "agent_id": agent_id_for_server, + "lookback_hours": lookback_hours, + "max_traces": args.max_traces, + } + else: + assert trace_ids is not None + data_source = { + "type": "azure_ai_traces", + "trace_ids": trace_ids, + "lookback_hours": lookback_hours, + } + + print("\nCreating Eval Run") + run_name = f"agent_trace_eval_{datetime.now().strftime('%Y%m%d_%H%M%S')}" + eval_run_object = client.evals.runs.create( + eval_id=eval_object.id, + name=run_name, + metadata=metadata if metadata else None, + data_source=data_source, # type: ignore + ) + print("Eval Run created") + pprint(eval_run_object) + + print("\nMonitoring Eval Run status...") + while True: + run = client.evals.runs.retrieve(run_id=eval_run_object.id, eval_id=eval_object.id) + print(f"Status: {run.status}") + + if run.status in {"completed", "failed", "canceled"}: + print("\nEval Run finished!") + print("Final Eval Run Response:") + pprint(run) + break + + time.sleep(5) + print("Waiting for eval run to complete...") + + if not args.no_cleanup: + client.evals.delete(eval_id=eval_object.id) + print("Evaluation deleted") + else: + print(f"Skipping cleanup (--no-cleanup). Eval ID: {eval_object.id}") if __name__ == "__main__": From 16a9d282ff8ea17e3933d501d1d97e2cab2a89c9 Mon Sep 17 00:00:00 2001 From: Shruti Iyer <9905402+shrutiyer@users.noreply.github.com> Date: Fri, 10 Apr 2026 16:45:28 -0400 Subject: [PATCH 02/18] Add sample for Azure AI Evaluations with hosted agent This sample demonstrates how to run Azure AI Evaluations against a hosted agent using the azure_ai_target_completions data source, evaluating agents live with built-in quality and safety evaluators. --- .../sample_evaluations_agent_as_target.py | 218 ++++++++++++++++++ 1 file changed, 218 insertions(+) create mode 100644 sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_agent_as_target.py diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_agent_as_target.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_agent_as_target.py new file mode 100644 index 000000000000..12003be10358 --- /dev/null +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_agent_as_target.py @@ -0,0 +1,218 @@ +# pylint: disable=line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ + +""" +DESCRIPTION: + Given an AIProjectClient, this sample demonstrates how to run Azure AI Evaluations against + a hosted agent using the azure_ai_target_completions data source. The evaluation service + invokes the agent live for each test query, collects responses, and runs built-in quality + and safety evaluators against them. + + This is different from trace evaluations (sample_evaluations_builtin_with_traces.py) which + evaluate historical agent traces. This sample evaluates agents live. + +USAGE: + python sample_evaluations_agent_as_target.py + + Before running the sample: + + pip install "azure-ai-projects>=2.0.0" python-dotenv + + Set these environment variables with your own values: + 1) AZURE_AI_PROJECT_ENDPOINT - Required. The Azure AI Project endpoint, as found in the overview page of your + Microsoft Foundry project. It has the form: https://.services.ai.azure.com/api/projects/. + 2) AZURE_AI_MODEL_DEPLOYMENT_NAME - Required. The Azure OpenAI deployment name to use as the judge model for + built-in evaluators. + 3) AGENT_NAME - Required. The hosted agent name to evaluate. + 4) AGENT_VERSION - Optional. The agent version to evaluate. Defaults to "1". +""" + +import os +import time +from datetime import datetime +from typing import Any, Dict, List + +from dotenv import load_dotenv +from azure.identity import DefaultAzureCredential +from azure.ai.projects import AIProjectClient + +from pprint import pprint + +load_dotenv() + + +endpoint = os.environ["AZURE_AI_PROJECT_ENDPOINT"] +model_deployment_name = os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"] +agent_name = os.environ["AGENT_NAME"] +agent_version = os.environ.get("AGENT_VERSION", "1") + +# Sample test queries. Replace with queries appropriate for your agent. +INPUT_QUERIES = [ + { + "item": { + "query": "Hello, what can you help me with?", + } + }, + { + "item": { + "query": "Can you give me a brief summary of your capabilities?", + } + }, +] + + +def _quality_evaluator(name: str, evaluator_name: str, response_field: str = "{{sample.output_text}}") -> Dict[str, Any]: + """Create a quality evaluator configuration block.""" + return { + "type": "azure_ai_evaluator", + "name": name, + "evaluator_name": evaluator_name, + "evaluator_version": "", + "initialization_parameters": { + "deployment_name": model_deployment_name, + }, + "data_mapping": { + "query": "{{item.query}}", + "response": response_field, + "context": "{{item.context}}", + "ground_truth": "{{item.ground_truth}}", + "tool_calls": "{{sample.tool_calls}}", + "tool_definitions": "{{sample.tool_definitions}}", + }, + } + + +def _safety_evaluator(name: str, evaluator_name: str) -> Dict[str, Any]: + """Create a safety evaluator configuration block.""" + return { + "type": "azure_ai_evaluator", + "name": name, + "evaluator_name": evaluator_name, + "evaluator_version": "", + "initialization_parameters": { + "threshold": 4, + }, + "data_mapping": { + "query": "{{item.query}}", + "response": "{{sample.output_text}}", + "context": "{{item.context}}", + "ground_truth": "{{item.ground_truth}}", + "tool_calls": "{{sample.tool_calls}}", + "tool_definitions": "{{sample.tool_definitions}}", + }, + } + + +def main() -> None: + testing_criteria: List[Dict[str, Any]] = [ + # Quality evaluators + _quality_evaluator("IntentResolution", "builtin.intent_resolution"), + _quality_evaluator("Relevance", "builtin.relevance"), + _quality_evaluator("Fluency", "builtin.fluency"), + _quality_evaluator("Coherence", "builtin.coherence"), + _quality_evaluator("Groundedness", "builtin.groundedness", "{{sample.output_items}}"), + _quality_evaluator("TaskCompletion", "builtin.task_completion", "{{sample.output_items}}"), + _quality_evaluator("ToolCallSuccess", "builtin.tool_call_success", "{{sample.output_items}}"), + # Safety evaluators + _safety_evaluator("Violence", "builtin.violence"), + _safety_evaluator("SelfHarm", "builtin.self_harm"), + _safety_evaluator("Sexual", "builtin.sexual"), + _safety_evaluator("HateAndUnfairness", "builtin.hate_unfairness"), + ] + + data_source_config = { + "type": "custom", + "item_schema": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "context": {"type": "string"}, + "ground_truth": {"type": "string"}, + }, + "required": ["query"], + }, + "include_sample_schema": True, + } + + input_messages = { + "type": "template", + "template": [ + { + "type": "message", + "role": "user", + "content": { + "type": "input_text", + "text": "{{item.query}}", + }, + } + ], + } + + print(f"Agent: {agent_name} v{agent_version}") + print(f"Evaluators: {len(testing_criteria)}") + print(f"Queries: {len(INPUT_QUERIES)}") + + with DefaultAzureCredential() as credential: + with AIProjectClient(endpoint=endpoint, credential=credential) as project_client: + client = project_client.get_openai_client() + + print("\nCreating evaluation") + eval_name = f"Hosted Agent Eval - {agent_name} - {datetime.now().strftime('%Y%m%d_%H%M%S')}" + eval_object = client.evals.create( + name=eval_name, + data_source_config=data_source_config, # type: ignore + testing_criteria=testing_criteria, # type: ignore + ) + print(f"Evaluation created (id: {eval_object.id}, name: {eval_object.name})") + + print("\nGet Evaluation by Id") + eval_object_response = client.evals.retrieve(eval_object.id) + print("Evaluation Response:") + pprint(eval_object_response) + + print("\nCreating Eval Run") + data_source = { + "type": "azure_ai_target_completions", + "source": { + "type": "file_content", + "content": INPUT_QUERIES, + }, + "input_messages": input_messages, + "target": { + "type": "azure_ai_agent", + "name": agent_name, + "version": agent_version, + }, + } + run_name = f"Run - {agent_name} - {datetime.now().strftime('%Y%m%d_%H%M%S')}" + eval_run_object = client.evals.runs.create( + eval_id=eval_object.id, + name=run_name, + data_source=data_source, # type: ignore + ) + print("Eval Run created") + pprint(eval_run_object) + + print("\nMonitoring Eval Run status...") + while True: + run = client.evals.runs.retrieve(run_id=eval_run_object.id, eval_id=eval_object.id) + print(f"Status: {run.status}") + + if run.status in {"completed", "failed", "canceled"}: + print("\nEval Run finished!") + print("Final Eval Run Response:") + pprint(run) + break + + time.sleep(5) + print("Waiting for eval run to complete...") + + client.evals.delete(eval_id=eval_object.id) + print("Evaluation deleted") + + +if __name__ == "__main__": + main() From c8c8021272646919a662a3dab0a1cf89e5145f55 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 14 Apr 2026 15:19:46 +0000 Subject: [PATCH 03/18] Revert TRACE_LOOKBACK_HOURS default to 1 hour to match docstring Agent-Logs-Url: https://github.com/Azure/azure-sdk-for-python/sessions/f15ae794-b773-4e8c-860b-5aea55873600 Co-authored-by: shrutiyer <9905402+shrutiyer@users.noreply.github.com> --- .../evaluations/sample_evaluations_builtin_with_traces.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_builtin_with_traces.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_builtin_with_traces.py index 738278e5259e..1675f01cce1f 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_builtin_with_traces.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_builtin_with_traces.py @@ -58,7 +58,7 @@ endpoint = os.environ["AZURE_AI_PROJECT_ENDPOINT"] model_deployment_name = os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"] -default_lookback_hours = int(os.environ.get("TRACE_LOOKBACK_HOURS", "168")) +default_lookback_hours = int(os.environ.get("TRACE_LOOKBACK_HOURS", "1")) def _build_evaluator_config(name: str, evaluator_name: str) -> Dict[str, Any]: From dd5590a7afe8a153070756b8f848919a1bec3834 Mon Sep 17 00:00:00 2001 From: Shruti Iyer Date: Wed, 15 Apr 2026 16:33:53 -0400 Subject: [PATCH 04/18] Add agent eval samples: trace evals and agent-as-target evals Add two new evaluation samples: - sample_evaluations_builtin_with_traces.py: Trace-based evaluation with three modes (client-side App Insights query, server-side agent ID, and explicit trace IDs) - sample_evaluations_agent_as_target.py: Live agent evaluation using azure_ai_target_completions data source Both samples use the azure_ai_evaluator config pattern with builtin intent_resolution and task_adherence evaluators. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../sample_evaluations_agent_as_target.py | 188 +++++++++++++ .../sample_evaluations_builtin_with_traces.py | 261 ++++++++++++++++++ 2 files changed, 449 insertions(+) create mode 100644 sdk/evaluation/azure-ai-evaluation/samples/sample_evaluations_agent_as_target.py create mode 100644 sdk/evaluation/azure-ai-evaluation/samples/sample_evaluations_builtin_with_traces.py diff --git a/sdk/evaluation/azure-ai-evaluation/samples/sample_evaluations_agent_as_target.py b/sdk/evaluation/azure-ai-evaluation/samples/sample_evaluations_agent_as_target.py new file mode 100644 index 000000000000..1d825c041c75 --- /dev/null +++ b/sdk/evaluation/azure-ai-evaluation/samples/sample_evaluations_agent_as_target.py @@ -0,0 +1,188 @@ +# pylint: disable=line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ + +""" +DESCRIPTION: + Given an AIProjectClient, this sample demonstrates how to run Azure AI Evaluations against + a hosted agent using the azure_ai_target_completions data source. The evaluation service + invokes the agent live for each test query, collects responses, and runs built-in quality + and safety evaluators against them. + + This is different from trace evaluations (sample_evaluations_builtin_with_traces.py) which + evaluate historical agent traces. This sample evaluates agents live. + +USAGE: + python sample_evaluations_agent_as_target.py + + Before running the sample: + + pip install "azure-ai-projects>=2.0.0" python-dotenv + + Set these environment variables with your own values: + 1) AZURE_AI_PROJECT_ENDPOINT - Required. The Azure AI Project endpoint, as found in the overview page of your + Microsoft Foundry project. It has the form: https://.services.ai.azure.com/api/projects/. + 2) AZURE_AI_MODEL_DEPLOYMENT_NAME - Required. The Azure OpenAI deployment name to use as the judge model for + built-in evaluators. + 3) AGENT_NAME - Required. The hosted agent name to evaluate. + 4) AGENT_VERSION - Optional. The agent version to evaluate. Defaults to "1". +""" + +import os +import time +from datetime import datetime +from typing import Any, Dict + +from dotenv import load_dotenv +from azure.identity import DefaultAzureCredential +from azure.ai.projects import AIProjectClient + +from pprint import pprint + +load_dotenv() + + +endpoint = os.environ["AZURE_AI_PROJECT_ENDPOINT"] +model_deployment_name = os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"] +agent_name = os.environ["AGENT_NAME"] +agent_version = os.environ.get("AGENT_VERSION", "1") + +# Sample test queries. Replace with queries appropriate for your agent. +INPUT_QUERIES = [ + { + "item": { + "query": "Hello, what can you help me with?", + } + }, + { + "item": { + "query": "Can you give me a brief summary of your capabilities?", + } + }, +] + + +def _build_evaluator_config(name: str, evaluator_name: str) -> Dict[str, Any]: + """Create a standard Azure AI evaluator configuration block.""" + return { + "type": "azure_ai_evaluator", + "name": name, + "evaluator_name": evaluator_name, + "data_mapping": { + "query": "{{query}}", + "response": "{{response}}", + "tool_definitions": "{{tool_definitions}}", + }, + "initialization_parameters": { + "deployment_name": model_deployment_name, + }, + } + + +def main() -> None: + testing_criteria = [ + _build_evaluator_config( + name="intent_resolution", + evaluator_name="builtin.intent_resolution", + ), + _build_evaluator_config( + name="task_adherence", + evaluator_name="builtin.task_adherence", + ), + ] + + data_source_config = { + "type": "custom", + "item_schema": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "context": {"type": "string"}, + "ground_truth": {"type": "string"}, + }, + "required": ["query"], + }, + "include_sample_schema": True, + } + + input_messages = { + "type": "template", + "template": [ + { + "type": "message", + "role": "user", + "content": { + "type": "input_text", + "text": "{{item.query}}", + }, + } + ], + } + + print(f"Agent: {agent_name} v{agent_version}") + print(f"Evaluators: {len(testing_criteria)}") + print(f"Queries: {len(INPUT_QUERIES)}") + + with DefaultAzureCredential() as credential: + with AIProjectClient(endpoint=endpoint, credential=credential) as project_client: + client = project_client.get_openai_client() + + print("\nCreating evaluation") + eval_name = f"Hosted Agent Eval - {agent_name} - {datetime.now().strftime('%Y%m%d_%H%M%S')}" + eval_object = client.evals.create( + name=eval_name, + data_source_config=data_source_config, # type: ignore + testing_criteria=testing_criteria, # type: ignore + ) + print(f"Evaluation created (id: {eval_object.id}, name: {eval_object.name})") + + print("\nGet Evaluation by Id") + eval_object_response = client.evals.retrieve(eval_object.id) + print("Evaluation Response:") + pprint(eval_object_response) + + print("\nCreating Eval Run") + data_source = { + "type": "azure_ai_target_completions", + "source": { + "type": "file_content", + "content": INPUT_QUERIES, + }, + "input_messages": input_messages, + "target": { + "type": "azure_ai_agent", + "name": agent_name, + "version": agent_version, + }, + } + run_name = f"Run - {agent_name} - {datetime.now().strftime('%Y%m%d_%H%M%S')}" + eval_run_object = client.evals.runs.create( + eval_id=eval_object.id, + name=run_name, + data_source=data_source, # type: ignore + ) + print("Eval Run created") + pprint(eval_run_object) + + print("\nMonitoring Eval Run status...") + while True: + run = client.evals.runs.retrieve(run_id=eval_run_object.id, eval_id=eval_object.id) + print(f"Status: {run.status}") + + if run.status in {"completed", "failed", "canceled"}: + print("\nEval Run finished!") + print("Final Eval Run Response:") + pprint(run) + break + + time.sleep(5) + print("Waiting for eval run to complete...") + + client.evals.delete(eval_id=eval_object.id) + print("Evaluation deleted") + + +if __name__ == "__main__": + main() diff --git a/sdk/evaluation/azure-ai-evaluation/samples/sample_evaluations_builtin_with_traces.py b/sdk/evaluation/azure-ai-evaluation/samples/sample_evaluations_builtin_with_traces.py new file mode 100644 index 000000000000..dcd4f535a370 --- /dev/null +++ b/sdk/evaluation/azure-ai-evaluation/samples/sample_evaluations_builtin_with_traces.py @@ -0,0 +1,261 @@ +# pylint: disable=line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ + +""" +DESCRIPTION: + Given an AIProjectClient, this sample demonstrates how to run Azure AI Evaluations + against agent traces collected in Azure Application Insights. + + Supports three modes: + - Default mode (no flags): Queries Application Insights client-side for trace IDs + using the AGENT_ID environment variable, then passes them to the eval service. + - Agent ID mode (--agent-id): Passes the agent ID directly to the eval service, + which resolves traces server-side from Application Insights. + - Trace ID mode (--trace-ids): Passes explicit trace IDs to the eval service. + +USAGE: + python sample_evaluations_builtin_with_traces.py + python sample_evaluations_builtin_with_traces.py --agent-id "my-agent:1" + python sample_evaluations_builtin_with_traces.py --trace-ids abc123 def456 + python sample_evaluations_builtin_with_traces.py --agent-id "my-agent:1" --lookback-hours 48 --max-traces 20 + python sample_evaluations_builtin_with_traces.py --no-cleanup + + Before running the sample: + + pip install "azure-ai-projects>=2.0.0" python-dotenv azure-monitor-query + + Set these environment variables with your own values: + 1) AZURE_AI_PROJECT_ENDPOINT - Required. The Azure AI Project endpoint, as found in the overview page of your + Microsoft Foundry project. It has the form: https://.services.ai.azure.com/api/projects/. + 2) APPINSIGHTS_RESOURCE_ID - Required (for default mode). The Azure Application Insights resource ID that stores + agent traces. Not needed when using --agent-id or --trace-ids. + It has the form: /subscriptions//resourceGroups//providers/Microsoft.Insights/components/. + 3) AGENT_ID - Required (for default mode). The agent identifier emitted by the Azure tracing integration, + used to filter traces. Not needed when using --agent-id or --trace-ids. + 4) AZURE_AI_MODEL_DEPLOYMENT_NAME - Required. The Azure OpenAI deployment name to use with the built-in evaluators. + 5) TRACE_LOOKBACK_HOURS - Optional. Number of hours to look back when querying traces and in the evaluation run. + Defaults to 1 (1 hour). +""" + +import argparse +import os +import time +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, List, Optional + +from dotenv import load_dotenv +from azure.identity import DefaultAzureCredential +from azure.monitor.query import LogsQueryClient, LogsQueryStatus +from azure.ai.projects import AIProjectClient + +from pprint import pprint + +load_dotenv() + + +endpoint = os.environ["AZURE_AI_PROJECT_ENDPOINT"] +model_deployment_name = os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"] +default_lookback_hours = int(os.environ.get("TRACE_LOOKBACK_HOURS", "1")) + + +def _build_evaluator_config(name: str, evaluator_name: str) -> Dict[str, Any]: + """Create a standard Azure AI evaluator configuration block for trace evaluations.""" + return { + "type": "azure_ai_evaluator", + "name": name, + "evaluator_name": evaluator_name, + "data_mapping": { + "query": "{{query}}", + "response": "{{response}}", + "tool_definitions": "{{tool_definitions}}", + }, + "initialization_parameters": { + "deployment_name": model_deployment_name, + }, + } + + +def get_trace_ids( + appinsight_resource_id: str, tracked_agent_id: str, start_time: datetime, end_time: datetime +) -> List[str]: + """ + Query Application Insights for trace IDs (operation_Id) based on agent ID and time range. + + Args: + appinsight_resource_id: The resource ID of the Application Insights instance. + tracked_agent_id: The agent ID to filter by. + start_time: Start time for the query. + end_time: End time for the query. + + Returns: + List of distinct operation IDs (trace IDs). + """ + query = f""" +dependencies +| where timestamp between (datetime({start_time.isoformat()}) .. datetime({end_time.isoformat()})) +| extend agent_id = tostring(customDimensions["gen_ai.agent.id"]) +| where agent_id == "{tracked_agent_id}" +| distinct operation_Id +""" + + try: + with DefaultAzureCredential() as credential: + client = LogsQueryClient(credential) + response = client.query_resource( + appinsight_resource_id, + query=query, + timespan=None, # Time range is specified in the query itself. + ) + except Exception as exc: # pylint: disable=broad-except + print(f"Error executing query: {exc}") + return [] + + if response.status == LogsQueryStatus.SUCCESS: + trace_ids: List[str] = [] + for table in response.tables: + for row in table.rows: + trace_ids.append(row[0]) + return trace_ids + + print(f"Query failed with status: {response.status}") + if response.partial_error: + print(f"Partial error: {response.partial_error}") + return [] + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run Azure AI trace evaluations against agent traces.") + mode = parser.add_mutually_exclusive_group() + mode.add_argument("--agent-id", default=None, help="Agent ID for server-side trace resolution") + mode.add_argument("--trace-ids", nargs="+", default=None, help="Explicit trace IDs to evaluate") + parser.add_argument("--lookback-hours", type=int, default=None, help="Lookback window in hours") + parser.add_argument("--max-traces", type=int, default=50, help="Max traces in agent-id mode (default: 50)") + parser.add_argument("--no-cleanup", action="store_true", help="Keep eval group after run") + args = parser.parse_args() + + lookback_hours = args.lookback_hours or default_lookback_hours + trace_ids: Optional[List[str]] = None + agent_id_for_server: Optional[str] = None + metadata: Dict[str, str] = {} + + if args.agent_id: + agent_id_for_server = args.agent_id + print(f"Mode: Server-side agent ID resolution") + print(f"Agent ID: {args.agent_id}") + print(f"Lookback: {lookback_hours}h, Max traces: {args.max_traces}") + metadata["agent_id"] = args.agent_id + + elif args.trace_ids: + trace_ids = args.trace_ids + print(f"Mode: Explicit trace IDs ({len(trace_ids)} provided)") + + else: + appinsights_resource_id = os.environ["APPINSIGHTS_RESOURCE_ID"] + agent_id = os.environ["AGENT_ID"] + end_time = datetime.now(tz=timezone.utc) + start_time = end_time - timedelta(hours=lookback_hours) + + print("Querying Application Insights for trace identifiers...") + print(f"Agent ID: {agent_id}") + print(f"Time range: {start_time.isoformat()} to {end_time.isoformat()}") + + trace_ids = get_trace_ids(appinsights_resource_id, agent_id, start_time, end_time) + + if not trace_ids: + print("No trace IDs found for the provided agent and time window.") + return + + print(f"\nFound {len(trace_ids)} trace IDs:") + for tid in trace_ids: + print(f" - {tid}") + + metadata["agent_id"] = agent_id + metadata["start_time"] = start_time.isoformat() + metadata["end_time"] = end_time.isoformat() + + with DefaultAzureCredential() as credential: + with AIProjectClient(endpoint=endpoint, credential=credential) as project_client: + client = project_client.get_openai_client() + + data_source_config = { + "type": "azure_ai_source", + "scenario": "traces", + } + + testing_criteria = [ + _build_evaluator_config( + name="intent_resolution", + evaluator_name="builtin.intent_resolution", + ), + _build_evaluator_config( + name="task_adherence", + evaluator_name="builtin.task_adherence", + ), + ] + + print("\nCreating evaluation") + eval_object = client.evals.create( + name="agent_trace_eval_group", + data_source_config=data_source_config, # type: ignore + testing_criteria=testing_criteria, # type: ignore + ) + print(f"Evaluation created (id: {eval_object.id}, name: {eval_object.name})") + + print("\nGet Evaluation by Id") + eval_object_response = client.evals.retrieve(eval_object.id) + print("Evaluation Response:") + pprint(eval_object_response) + + # Build data source based on mode + if agent_id_for_server: + data_source: Dict[str, Any] = { + "type": "azure_ai_traces", + "agent_id": agent_id_for_server, + "lookback_hours": lookback_hours, + "max_traces": args.max_traces, + } + else: + assert trace_ids is not None + data_source = { + "type": "azure_ai_traces", + "trace_ids": trace_ids, + "lookback_hours": lookback_hours, + } + + print("\nCreating Eval Run") + run_name = f"agent_trace_eval_{datetime.now().strftime('%Y%m%d_%H%M%S')}" + eval_run_object = client.evals.runs.create( + eval_id=eval_object.id, + name=run_name, + metadata=metadata if metadata else None, + data_source=data_source, # type: ignore + ) + print("Eval Run created") + pprint(eval_run_object) + + print("\nMonitoring Eval Run status...") + while True: + run = client.evals.runs.retrieve(run_id=eval_run_object.id, eval_id=eval_object.id) + print(f"Status: {run.status}") + + if run.status in {"completed", "failed", "canceled"}: + print("\nEval Run finished!") + print("Final Eval Run Response:") + pprint(run) + break + + time.sleep(5) + print("Waiting for eval run to complete...") + + if not args.no_cleanup: + client.evals.delete(eval_id=eval_object.id) + print("Evaluation deleted") + else: + print(f"Skipping cleanup (--no-cleanup). Eval ID: {eval_object.id}") + + +if __name__ == "__main__": + main() From f5315b9064937d16019e23bb380bc4766a7211da Mon Sep 17 00:00:00 2001 From: Shruti Iyer <9905402+shrutiyer@users.noreply.github.com> Date: Wed, 15 Apr 2026 16:46:06 -0400 Subject: [PATCH 05/18] Refactor evaluator configuration functions --- .../sample_evaluations_agent_as_target.py | 62 +++++-------------- 1 file changed, 16 insertions(+), 46 deletions(-) diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_agent_as_target.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_agent_as_target.py index 12003be10358..1d825c041c75 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_agent_as_target.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_agent_as_target.py @@ -33,7 +33,7 @@ import os import time from datetime import datetime -from typing import Any, Dict, List +from typing import Any, Dict from dotenv import load_dotenv from azure.identity import DefaultAzureCredential @@ -64,63 +64,33 @@ ] -def _quality_evaluator(name: str, evaluator_name: str, response_field: str = "{{sample.output_text}}") -> Dict[str, Any]: - """Create a quality evaluator configuration block.""" +def _build_evaluator_config(name: str, evaluator_name: str) -> Dict[str, Any]: + """Create a standard Azure AI evaluator configuration block.""" return { "type": "azure_ai_evaluator", "name": name, "evaluator_name": evaluator_name, - "evaluator_version": "", - "initialization_parameters": { - "deployment_name": model_deployment_name, - }, "data_mapping": { - "query": "{{item.query}}", - "response": response_field, - "context": "{{item.context}}", - "ground_truth": "{{item.ground_truth}}", - "tool_calls": "{{sample.tool_calls}}", - "tool_definitions": "{{sample.tool_definitions}}", + "query": "{{query}}", + "response": "{{response}}", + "tool_definitions": "{{tool_definitions}}", }, - } - - -def _safety_evaluator(name: str, evaluator_name: str) -> Dict[str, Any]: - """Create a safety evaluator configuration block.""" - return { - "type": "azure_ai_evaluator", - "name": name, - "evaluator_name": evaluator_name, - "evaluator_version": "", "initialization_parameters": { - "threshold": 4, - }, - "data_mapping": { - "query": "{{item.query}}", - "response": "{{sample.output_text}}", - "context": "{{item.context}}", - "ground_truth": "{{item.ground_truth}}", - "tool_calls": "{{sample.tool_calls}}", - "tool_definitions": "{{sample.tool_definitions}}", + "deployment_name": model_deployment_name, }, } def main() -> None: - testing_criteria: List[Dict[str, Any]] = [ - # Quality evaluators - _quality_evaluator("IntentResolution", "builtin.intent_resolution"), - _quality_evaluator("Relevance", "builtin.relevance"), - _quality_evaluator("Fluency", "builtin.fluency"), - _quality_evaluator("Coherence", "builtin.coherence"), - _quality_evaluator("Groundedness", "builtin.groundedness", "{{sample.output_items}}"), - _quality_evaluator("TaskCompletion", "builtin.task_completion", "{{sample.output_items}}"), - _quality_evaluator("ToolCallSuccess", "builtin.tool_call_success", "{{sample.output_items}}"), - # Safety evaluators - _safety_evaluator("Violence", "builtin.violence"), - _safety_evaluator("SelfHarm", "builtin.self_harm"), - _safety_evaluator("Sexual", "builtin.sexual"), - _safety_evaluator("HateAndUnfairness", "builtin.hate_unfairness"), + testing_criteria = [ + _build_evaluator_config( + name="intent_resolution", + evaluator_name="builtin.intent_resolution", + ), + _build_evaluator_config( + name="task_adherence", + evaluator_name="builtin.task_adherence", + ), ] data_source_config = { From 1c5638fe23a771efbb9e59023374c204c215e040 Mon Sep 17 00:00:00 2001 From: Shruti Iyer <9905402+shrutiyer@users.noreply.github.com> Date: Wed, 15 Apr 2026 16:46:58 -0400 Subject: [PATCH 06/18] Delete sdk/evaluation/azure-ai-evaluation/samples/sample_evaluations_agent_as_target.py --- .../sample_evaluations_agent_as_target.py | 188 ------------------ 1 file changed, 188 deletions(-) delete mode 100644 sdk/evaluation/azure-ai-evaluation/samples/sample_evaluations_agent_as_target.py diff --git a/sdk/evaluation/azure-ai-evaluation/samples/sample_evaluations_agent_as_target.py b/sdk/evaluation/azure-ai-evaluation/samples/sample_evaluations_agent_as_target.py deleted file mode 100644 index 1d825c041c75..000000000000 --- a/sdk/evaluation/azure-ai-evaluation/samples/sample_evaluations_agent_as_target.py +++ /dev/null @@ -1,188 +0,0 @@ -# pylint: disable=line-too-long,useless-suppression -# ------------------------------------ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. -# ------------------------------------ - -""" -DESCRIPTION: - Given an AIProjectClient, this sample demonstrates how to run Azure AI Evaluations against - a hosted agent using the azure_ai_target_completions data source. The evaluation service - invokes the agent live for each test query, collects responses, and runs built-in quality - and safety evaluators against them. - - This is different from trace evaluations (sample_evaluations_builtin_with_traces.py) which - evaluate historical agent traces. This sample evaluates agents live. - -USAGE: - python sample_evaluations_agent_as_target.py - - Before running the sample: - - pip install "azure-ai-projects>=2.0.0" python-dotenv - - Set these environment variables with your own values: - 1) AZURE_AI_PROJECT_ENDPOINT - Required. The Azure AI Project endpoint, as found in the overview page of your - Microsoft Foundry project. It has the form: https://.services.ai.azure.com/api/projects/. - 2) AZURE_AI_MODEL_DEPLOYMENT_NAME - Required. The Azure OpenAI deployment name to use as the judge model for - built-in evaluators. - 3) AGENT_NAME - Required. The hosted agent name to evaluate. - 4) AGENT_VERSION - Optional. The agent version to evaluate. Defaults to "1". -""" - -import os -import time -from datetime import datetime -from typing import Any, Dict - -from dotenv import load_dotenv -from azure.identity import DefaultAzureCredential -from azure.ai.projects import AIProjectClient - -from pprint import pprint - -load_dotenv() - - -endpoint = os.environ["AZURE_AI_PROJECT_ENDPOINT"] -model_deployment_name = os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"] -agent_name = os.environ["AGENT_NAME"] -agent_version = os.environ.get("AGENT_VERSION", "1") - -# Sample test queries. Replace with queries appropriate for your agent. -INPUT_QUERIES = [ - { - "item": { - "query": "Hello, what can you help me with?", - } - }, - { - "item": { - "query": "Can you give me a brief summary of your capabilities?", - } - }, -] - - -def _build_evaluator_config(name: str, evaluator_name: str) -> Dict[str, Any]: - """Create a standard Azure AI evaluator configuration block.""" - return { - "type": "azure_ai_evaluator", - "name": name, - "evaluator_name": evaluator_name, - "data_mapping": { - "query": "{{query}}", - "response": "{{response}}", - "tool_definitions": "{{tool_definitions}}", - }, - "initialization_parameters": { - "deployment_name": model_deployment_name, - }, - } - - -def main() -> None: - testing_criteria = [ - _build_evaluator_config( - name="intent_resolution", - evaluator_name="builtin.intent_resolution", - ), - _build_evaluator_config( - name="task_adherence", - evaluator_name="builtin.task_adherence", - ), - ] - - data_source_config = { - "type": "custom", - "item_schema": { - "type": "object", - "properties": { - "query": {"type": "string"}, - "context": {"type": "string"}, - "ground_truth": {"type": "string"}, - }, - "required": ["query"], - }, - "include_sample_schema": True, - } - - input_messages = { - "type": "template", - "template": [ - { - "type": "message", - "role": "user", - "content": { - "type": "input_text", - "text": "{{item.query}}", - }, - } - ], - } - - print(f"Agent: {agent_name} v{agent_version}") - print(f"Evaluators: {len(testing_criteria)}") - print(f"Queries: {len(INPUT_QUERIES)}") - - with DefaultAzureCredential() as credential: - with AIProjectClient(endpoint=endpoint, credential=credential) as project_client: - client = project_client.get_openai_client() - - print("\nCreating evaluation") - eval_name = f"Hosted Agent Eval - {agent_name} - {datetime.now().strftime('%Y%m%d_%H%M%S')}" - eval_object = client.evals.create( - name=eval_name, - data_source_config=data_source_config, # type: ignore - testing_criteria=testing_criteria, # type: ignore - ) - print(f"Evaluation created (id: {eval_object.id}, name: {eval_object.name})") - - print("\nGet Evaluation by Id") - eval_object_response = client.evals.retrieve(eval_object.id) - print("Evaluation Response:") - pprint(eval_object_response) - - print("\nCreating Eval Run") - data_source = { - "type": "azure_ai_target_completions", - "source": { - "type": "file_content", - "content": INPUT_QUERIES, - }, - "input_messages": input_messages, - "target": { - "type": "azure_ai_agent", - "name": agent_name, - "version": agent_version, - }, - } - run_name = f"Run - {agent_name} - {datetime.now().strftime('%Y%m%d_%H%M%S')}" - eval_run_object = client.evals.runs.create( - eval_id=eval_object.id, - name=run_name, - data_source=data_source, # type: ignore - ) - print("Eval Run created") - pprint(eval_run_object) - - print("\nMonitoring Eval Run status...") - while True: - run = client.evals.runs.retrieve(run_id=eval_run_object.id, eval_id=eval_object.id) - print(f"Status: {run.status}") - - if run.status in {"completed", "failed", "canceled"}: - print("\nEval Run finished!") - print("Final Eval Run Response:") - pprint(run) - break - - time.sleep(5) - print("Waiting for eval run to complete...") - - client.evals.delete(eval_id=eval_object.id) - print("Evaluation deleted") - - -if __name__ == "__main__": - main() From ff6987d43d2eb1fc190ef2c24135c19ef429e8d9 Mon Sep 17 00:00:00 2001 From: Shruti Iyer <9905402+shrutiyer@users.noreply.github.com> Date: Wed, 15 Apr 2026 16:47:32 -0400 Subject: [PATCH 07/18] Delete sdk/evaluation/azure-ai-evaluation/samples/sample_evaluations_builtin_with_traces.py --- .../sample_evaluations_builtin_with_traces.py | 261 ------------------ 1 file changed, 261 deletions(-) delete mode 100644 sdk/evaluation/azure-ai-evaluation/samples/sample_evaluations_builtin_with_traces.py diff --git a/sdk/evaluation/azure-ai-evaluation/samples/sample_evaluations_builtin_with_traces.py b/sdk/evaluation/azure-ai-evaluation/samples/sample_evaluations_builtin_with_traces.py deleted file mode 100644 index dcd4f535a370..000000000000 --- a/sdk/evaluation/azure-ai-evaluation/samples/sample_evaluations_builtin_with_traces.py +++ /dev/null @@ -1,261 +0,0 @@ -# pylint: disable=line-too-long,useless-suppression -# ------------------------------------ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. -# ------------------------------------ - -""" -DESCRIPTION: - Given an AIProjectClient, this sample demonstrates how to run Azure AI Evaluations - against agent traces collected in Azure Application Insights. - - Supports three modes: - - Default mode (no flags): Queries Application Insights client-side for trace IDs - using the AGENT_ID environment variable, then passes them to the eval service. - - Agent ID mode (--agent-id): Passes the agent ID directly to the eval service, - which resolves traces server-side from Application Insights. - - Trace ID mode (--trace-ids): Passes explicit trace IDs to the eval service. - -USAGE: - python sample_evaluations_builtin_with_traces.py - python sample_evaluations_builtin_with_traces.py --agent-id "my-agent:1" - python sample_evaluations_builtin_with_traces.py --trace-ids abc123 def456 - python sample_evaluations_builtin_with_traces.py --agent-id "my-agent:1" --lookback-hours 48 --max-traces 20 - python sample_evaluations_builtin_with_traces.py --no-cleanup - - Before running the sample: - - pip install "azure-ai-projects>=2.0.0" python-dotenv azure-monitor-query - - Set these environment variables with your own values: - 1) AZURE_AI_PROJECT_ENDPOINT - Required. The Azure AI Project endpoint, as found in the overview page of your - Microsoft Foundry project. It has the form: https://.services.ai.azure.com/api/projects/. - 2) APPINSIGHTS_RESOURCE_ID - Required (for default mode). The Azure Application Insights resource ID that stores - agent traces. Not needed when using --agent-id or --trace-ids. - It has the form: /subscriptions//resourceGroups//providers/Microsoft.Insights/components/. - 3) AGENT_ID - Required (for default mode). The agent identifier emitted by the Azure tracing integration, - used to filter traces. Not needed when using --agent-id or --trace-ids. - 4) AZURE_AI_MODEL_DEPLOYMENT_NAME - Required. The Azure OpenAI deployment name to use with the built-in evaluators. - 5) TRACE_LOOKBACK_HOURS - Optional. Number of hours to look back when querying traces and in the evaluation run. - Defaults to 1 (1 hour). -""" - -import argparse -import os -import time -from datetime import datetime, timedelta, timezone -from typing import Any, Dict, List, Optional - -from dotenv import load_dotenv -from azure.identity import DefaultAzureCredential -from azure.monitor.query import LogsQueryClient, LogsQueryStatus -from azure.ai.projects import AIProjectClient - -from pprint import pprint - -load_dotenv() - - -endpoint = os.environ["AZURE_AI_PROJECT_ENDPOINT"] -model_deployment_name = os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"] -default_lookback_hours = int(os.environ.get("TRACE_LOOKBACK_HOURS", "1")) - - -def _build_evaluator_config(name: str, evaluator_name: str) -> Dict[str, Any]: - """Create a standard Azure AI evaluator configuration block for trace evaluations.""" - return { - "type": "azure_ai_evaluator", - "name": name, - "evaluator_name": evaluator_name, - "data_mapping": { - "query": "{{query}}", - "response": "{{response}}", - "tool_definitions": "{{tool_definitions}}", - }, - "initialization_parameters": { - "deployment_name": model_deployment_name, - }, - } - - -def get_trace_ids( - appinsight_resource_id: str, tracked_agent_id: str, start_time: datetime, end_time: datetime -) -> List[str]: - """ - Query Application Insights for trace IDs (operation_Id) based on agent ID and time range. - - Args: - appinsight_resource_id: The resource ID of the Application Insights instance. - tracked_agent_id: The agent ID to filter by. - start_time: Start time for the query. - end_time: End time for the query. - - Returns: - List of distinct operation IDs (trace IDs). - """ - query = f""" -dependencies -| where timestamp between (datetime({start_time.isoformat()}) .. datetime({end_time.isoformat()})) -| extend agent_id = tostring(customDimensions["gen_ai.agent.id"]) -| where agent_id == "{tracked_agent_id}" -| distinct operation_Id -""" - - try: - with DefaultAzureCredential() as credential: - client = LogsQueryClient(credential) - response = client.query_resource( - appinsight_resource_id, - query=query, - timespan=None, # Time range is specified in the query itself. - ) - except Exception as exc: # pylint: disable=broad-except - print(f"Error executing query: {exc}") - return [] - - if response.status == LogsQueryStatus.SUCCESS: - trace_ids: List[str] = [] - for table in response.tables: - for row in table.rows: - trace_ids.append(row[0]) - return trace_ids - - print(f"Query failed with status: {response.status}") - if response.partial_error: - print(f"Partial error: {response.partial_error}") - return [] - - -def main() -> None: - parser = argparse.ArgumentParser(description="Run Azure AI trace evaluations against agent traces.") - mode = parser.add_mutually_exclusive_group() - mode.add_argument("--agent-id", default=None, help="Agent ID for server-side trace resolution") - mode.add_argument("--trace-ids", nargs="+", default=None, help="Explicit trace IDs to evaluate") - parser.add_argument("--lookback-hours", type=int, default=None, help="Lookback window in hours") - parser.add_argument("--max-traces", type=int, default=50, help="Max traces in agent-id mode (default: 50)") - parser.add_argument("--no-cleanup", action="store_true", help="Keep eval group after run") - args = parser.parse_args() - - lookback_hours = args.lookback_hours or default_lookback_hours - trace_ids: Optional[List[str]] = None - agent_id_for_server: Optional[str] = None - metadata: Dict[str, str] = {} - - if args.agent_id: - agent_id_for_server = args.agent_id - print(f"Mode: Server-side agent ID resolution") - print(f"Agent ID: {args.agent_id}") - print(f"Lookback: {lookback_hours}h, Max traces: {args.max_traces}") - metadata["agent_id"] = args.agent_id - - elif args.trace_ids: - trace_ids = args.trace_ids - print(f"Mode: Explicit trace IDs ({len(trace_ids)} provided)") - - else: - appinsights_resource_id = os.environ["APPINSIGHTS_RESOURCE_ID"] - agent_id = os.environ["AGENT_ID"] - end_time = datetime.now(tz=timezone.utc) - start_time = end_time - timedelta(hours=lookback_hours) - - print("Querying Application Insights for trace identifiers...") - print(f"Agent ID: {agent_id}") - print(f"Time range: {start_time.isoformat()} to {end_time.isoformat()}") - - trace_ids = get_trace_ids(appinsights_resource_id, agent_id, start_time, end_time) - - if not trace_ids: - print("No trace IDs found for the provided agent and time window.") - return - - print(f"\nFound {len(trace_ids)} trace IDs:") - for tid in trace_ids: - print(f" - {tid}") - - metadata["agent_id"] = agent_id - metadata["start_time"] = start_time.isoformat() - metadata["end_time"] = end_time.isoformat() - - with DefaultAzureCredential() as credential: - with AIProjectClient(endpoint=endpoint, credential=credential) as project_client: - client = project_client.get_openai_client() - - data_source_config = { - "type": "azure_ai_source", - "scenario": "traces", - } - - testing_criteria = [ - _build_evaluator_config( - name="intent_resolution", - evaluator_name="builtin.intent_resolution", - ), - _build_evaluator_config( - name="task_adherence", - evaluator_name="builtin.task_adherence", - ), - ] - - print("\nCreating evaluation") - eval_object = client.evals.create( - name="agent_trace_eval_group", - data_source_config=data_source_config, # type: ignore - testing_criteria=testing_criteria, # type: ignore - ) - print(f"Evaluation created (id: {eval_object.id}, name: {eval_object.name})") - - print("\nGet Evaluation by Id") - eval_object_response = client.evals.retrieve(eval_object.id) - print("Evaluation Response:") - pprint(eval_object_response) - - # Build data source based on mode - if agent_id_for_server: - data_source: Dict[str, Any] = { - "type": "azure_ai_traces", - "agent_id": agent_id_for_server, - "lookback_hours": lookback_hours, - "max_traces": args.max_traces, - } - else: - assert trace_ids is not None - data_source = { - "type": "azure_ai_traces", - "trace_ids": trace_ids, - "lookback_hours": lookback_hours, - } - - print("\nCreating Eval Run") - run_name = f"agent_trace_eval_{datetime.now().strftime('%Y%m%d_%H%M%S')}" - eval_run_object = client.evals.runs.create( - eval_id=eval_object.id, - name=run_name, - metadata=metadata if metadata else None, - data_source=data_source, # type: ignore - ) - print("Eval Run created") - pprint(eval_run_object) - - print("\nMonitoring Eval Run status...") - while True: - run = client.evals.runs.retrieve(run_id=eval_run_object.id, eval_id=eval_object.id) - print(f"Status: {run.status}") - - if run.status in {"completed", "failed", "canceled"}: - print("\nEval Run finished!") - print("Final Eval Run Response:") - pprint(run) - break - - time.sleep(5) - print("Waiting for eval run to complete...") - - if not args.no_cleanup: - client.evals.delete(eval_id=eval_object.id) - print("Evaluation deleted") - else: - print(f"Skipping cleanup (--no-cleanup). Eval ID: {eval_object.id}") - - -if __name__ == "__main__": - main() From 4d94d66c72786dc082fc28b96d61291f73072fe2 Mon Sep 17 00:00:00 2001 From: Shruti Iyer <9905402+shrutiyer@users.noreply.github.com> Date: Thu, 16 Apr 2026 15:06:14 -0400 Subject: [PATCH 08/18] Add sample_evaluations_agent_as_target to skip list --- .../azure-ai-projects/tests/samples/test_samples_evaluations.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/ai/azure-ai-projects/tests/samples/test_samples_evaluations.py b/sdk/ai/azure-ai-projects/tests/samples/test_samples_evaluations.py index 8fbb1bc90251..dd02f0c036c5 100644 --- a/sdk/ai/azure-ai-projects/tests/samples/test_samples_evaluations.py +++ b/sdk/ai/azure-ai-projects/tests/samples/test_samples_evaluations.py @@ -162,6 +162,7 @@ class TestSamplesEvaluations(AzureRecordedTestCase): samples_to_skip=[ "sample_evaluations_ai_assisted.py", # Similarity evaluator returns FAILED_EXECUTION ('query' is missing) "sample_evaluations_builtin_with_inline_data_oai.py", # 401 AuthenticationError (invalid subscription key or API endpoint) + "‎sample_evaluations_agent_as_target.py‎", # Missing required agent setup "sample_evaluations_builtin_with_traces.py", # Missing required env var APPINSIGHTS_RESOURCE_ID (KeyError) "sample_evaluations_score_model_grader_with_image.py", # Eval fails: image inputs not supported for configured grader model "sample_evaluations_score_model_grader_with_image_model_target.py", # Eval fails: image inputs not supported for configured grader model From 348e0a28bbeb63c5a05a1f7a69ef4cefcd0c48cb Mon Sep 17 00:00:00 2001 From: Shruti Iyer Date: Thu, 16 Apr 2026 16:20:07 -0400 Subject: [PATCH 09/18] Fix pyright error in trace evaluation samples Use list() to narrow args.trace_ids from Optional[List[str]] to List[str] inside the elif guard, resolving pyright reportArgumentType error. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../evaluations/sample_evaluations_builtin_with_traces.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_builtin_with_traces.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_builtin_with_traces.py index 1675f01cce1f..aae30052b2eb 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_builtin_with_traces.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_builtin_with_traces.py @@ -149,7 +149,7 @@ def main() -> None: metadata["agent_id"] = args.agent_id elif args.trace_ids: - trace_ids = args.trace_ids + trace_ids = list(args.trace_ids) print(f"Mode: Explicit trace IDs ({len(trace_ids)} provided)") else: From f9ed2748050461177031f36b174221ab22db1013 Mon Sep 17 00:00:00 2001 From: Shruti Iyer Date: Thu, 16 Apr 2026 16:23:47 -0400 Subject: [PATCH 10/18] Skip sample_evaluations_agent_as_target in test discovery The sample requires a deployed Azure AI Agent and has no recording, so it must be excluded from the parametrized test_evaluation_samples. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../tests/samples/test_samples_evaluations.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sdk/ai/azure-ai-projects/tests/samples/test_samples_evaluations.py b/sdk/ai/azure-ai-projects/tests/samples/test_samples_evaluations.py index dd02f0c036c5..0edef34b7e8d 100644 --- a/sdk/ai/azure-ai-projects/tests/samples/test_samples_evaluations.py +++ b/sdk/ai/azure-ai-projects/tests/samples/test_samples_evaluations.py @@ -142,6 +142,8 @@ class TestSamplesEvaluations(AzureRecordedTestCase): get_bearer_token_provider() which is incompatible with mock credentials. External service dependencies (require additional Azure services): + - sample_evaluations_agent_as_target.py: Requires a deployed Azure AI Agent for + agent-as-target evaluation. - sample_evaluations_builtin_with_traces.py: Requires Azure Application Insights and uses azure-monitor-query to fetch traces. - sample_scheduled_evaluations.py: Requires Azure RBAC assignment via @@ -162,7 +164,7 @@ class TestSamplesEvaluations(AzureRecordedTestCase): samples_to_skip=[ "sample_evaluations_ai_assisted.py", # Similarity evaluator returns FAILED_EXECUTION ('query' is missing) "sample_evaluations_builtin_with_inline_data_oai.py", # 401 AuthenticationError (invalid subscription key or API endpoint) - "‎sample_evaluations_agent_as_target.py‎", # Missing required agent setup + "sample_evaluations_agent_as_target.py", # Requires Azure AI Agent deployment (agent-as-target evaluation) "sample_evaluations_builtin_with_traces.py", # Missing required env var APPINSIGHTS_RESOURCE_ID (KeyError) "sample_evaluations_score_model_grader_with_image.py", # Eval fails: image inputs not supported for configured grader model "sample_evaluations_score_model_grader_with_image_model_target.py", # Eval fails: image inputs not supported for configured grader model From 1d6bd56157f6c61c6335dce27dea6163399b8b0a Mon Sep 17 00:00:00 2001 From: Shruti Iyer Date: Mon, 27 Apr 2026 20:20:07 -0400 Subject: [PATCH 11/18] Refactor the sample --- .../sample_evaluations_agent_as_target.py | 42 +++++++++++-------- .../sample_evaluations_builtin_with_traces.py | 6 +-- 2 files changed, 27 insertions(+), 21 deletions(-) diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_agent_as_target.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_agent_as_target.py index 1d825c041c75..448936f99018 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_agent_as_target.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_agent_as_target.py @@ -22,9 +22,9 @@ pip install "azure-ai-projects>=2.0.0" python-dotenv Set these environment variables with your own values: - 1) AZURE_AI_PROJECT_ENDPOINT - Required. The Azure AI Project endpoint, as found in the overview page of your + 1) FOUNDRY_PROJECT_ENDPOINT - Required. The Azure AI Project endpoint, as found in the overview page of your Microsoft Foundry project. It has the form: https://.services.ai.azure.com/api/projects/. - 2) AZURE_AI_MODEL_DEPLOYMENT_NAME - Required. The Azure OpenAI deployment name to use as the judge model for + 2) FOUNDRY_MODEL_NAME - Required. The Azure OpenAI deployment name to use as the judge model for built-in evaluators. 3) AGENT_NAME - Required. The hosted agent name to evaluate. 4) AGENT_VERSION - Optional. The agent version to evaluate. Defaults to "1". @@ -33,19 +33,19 @@ import os import time from datetime import datetime +from pprint import pprint from typing import Any, Dict from dotenv import load_dotenv -from azure.identity import DefaultAzureCredential from azure.ai.projects import AIProjectClient - -from pprint import pprint +from azure.ai.projects.models import TestingCriterionAzureAIEvaluator +from azure.identity import DefaultAzureCredential load_dotenv() -endpoint = os.environ["AZURE_AI_PROJECT_ENDPOINT"] -model_deployment_name = os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"] +endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] +model_deployment_name = os.environ["FOUNDRY_MODEL_NAME"] agent_name = os.environ["AGENT_NAME"] agent_version = os.environ.get("AGENT_VERSION", "1") @@ -64,21 +64,25 @@ ] -def _build_evaluator_config(name: str, evaluator_name: str) -> Dict[str, Any]: +def _build_evaluator_config( + name: str, + evaluator_name: str, + response_mapping: str, +) -> TestingCriterionAzureAIEvaluator: """Create a standard Azure AI evaluator configuration block.""" - return { - "type": "azure_ai_evaluator", - "name": name, - "evaluator_name": evaluator_name, - "data_mapping": { - "query": "{{query}}", - "response": "{{response}}", - "tool_definitions": "{{tool_definitions}}", + return TestingCriterionAzureAIEvaluator( + type="azure_ai_evaluator", + name=name, + evaluator_name=evaluator_name, + data_mapping={ + "query": "{{item.query}}", + "response": response_mapping, + "tool_definitions": "{{sample.tool_definitions}}", }, - "initialization_parameters": { + initialization_parameters={ "deployment_name": model_deployment_name, }, - } + ) def main() -> None: @@ -86,10 +90,12 @@ def main() -> None: _build_evaluator_config( name="intent_resolution", evaluator_name="builtin.intent_resolution", + response_mapping="{{sample.output_text}}", ), _build_evaluator_config( name="task_adherence", evaluator_name="builtin.task_adherence", + response_mapping="{{sample.output_items}}", ), ] diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_builtin_with_traces.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_builtin_with_traces.py index 47ee8d318e9c..2047bc52e0d8 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_builtin_with_traces.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_builtin_with_traces.py @@ -74,9 +74,9 @@ def _build_evaluator_config(name: str, evaluator_name: str) -> TestingCriterionA name=name, evaluator_name=evaluator_name, data_mapping={ - "query": "{{query}}", - "response": "{{response}}", - "tool_definitions": "{{tool_definitions}}", + "query": "{{sample.query}}", + "response": "{{sample.response}}", + "tool_definitions": "{{sample.tool_definitions}}", }, initialization_parameters={ "deployment_name": model_deployment_name, From 433be8a25868c4cf79e6f0160b57acab39340284 Mon Sep 17 00:00:00 2001 From: Shruti Iyer Date: Thu, 30 Apr 2026 15:29:55 -0400 Subject: [PATCH 12/18] pylint fixes --- .../evaluations/sample_evaluations_agent_as_target.py | 1 - .../evaluations/sample_evaluations_builtin_with_traces.py | 6 +----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_agent_as_target.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_agent_as_target.py index 448936f99018..3c1baff1d0b2 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_agent_as_target.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_agent_as_target.py @@ -34,7 +34,6 @@ import time from datetime import datetime from pprint import pprint -from typing import Any, Dict from dotenv import load_dotenv from azure.ai.projects import AIProjectClient diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_builtin_with_traces.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_builtin_with_traces.py index 2047bc52e0d8..30de5e7c463a 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_builtin_with_traces.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_builtin_with_traces.py @@ -50,9 +50,7 @@ from azure.monitor.query import LogsQueryClient, LogsQueryStatus from azure.ai.projects import AIProjectClient from azure.ai.projects.models import ( - AzureAIDataSourceConfig, TestingCriterionAzureAIEvaluator, - TracesPreviewEvalRunDataSource, ) load_dotenv() @@ -149,7 +147,7 @@ def main() -> None: if args.agent_id: agent_id_for_server = args.agent_id - print(f"Mode: Server-side agent ID resolution") + print("Mode: Server-side agent ID resolution") print(f"Agent ID: {args.agent_id}") print(f"Lookback: {lookback_hours}h, Max traces: {args.max_traces}") metadata["agent_id"] = args.agent_id @@ -159,8 +157,6 @@ def main() -> None: print(f"Mode: Explicit trace IDs ({len(trace_ids)} provided)") else: - appinsights_resource_id = os.environ["APPINSIGHTS_RESOURCE_ID"] - agent_id = os.environ["AGENT_ID"] end_time = datetime.now(tz=timezone.utc) start_time = end_time - timedelta(hours=lookback_hours) From a9f9f6467b28562a04ee53bf5b3e85a6a82d7185 Mon Sep 17 00:00:00 2001 From: Shruti Iyer Date: Thu, 30 Apr 2026 16:27:52 -0400 Subject: [PATCH 13/18] more pylint fixes --- .../sample_evaluations_agent_as_target.py | 12 +++++++++++- .../sample_evaluations_builtin_with_traces.py | 2 +- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_agent_as_target.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_agent_as_target.py index 3c1baff1d0b2..78b2e363eb21 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_agent_as_target.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_agent_as_target.py @@ -68,7 +68,17 @@ def _build_evaluator_config( evaluator_name: str, response_mapping: str, ) -> TestingCriterionAzureAIEvaluator: - """Create a standard Azure AI evaluator configuration block.""" + """Create a standard Azure AI evaluator configuration block. + + :param name: Display name for this testing criterion. + :type name: str + :param evaluator_name: Built-in evaluator identifier. + :type evaluator_name: str + :param response_mapping: Response mapping expression used by the evaluator. + :type response_mapping: str + :return: Evaluator configuration used in the evaluation group. + :rtype: ~azure.ai.projects.models.TestingCriterionAzureAIEvaluator + """ return TestingCriterionAzureAIEvaluator( type="azure_ai_evaluator", name=name, diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_builtin_with_traces.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_builtin_with_traces.py index 30de5e7c463a..a5e88ab53301 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_builtin_with_traces.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_builtin_with_traces.py @@ -130,7 +130,7 @@ def get_trace_ids( return [] -def main() -> None: +def main() -> None: # pylint: disable=too-many-statements parser = argparse.ArgumentParser(description="Run Azure AI trace evaluations against agent traces.") mode = parser.add_mutually_exclusive_group() mode.add_argument("--agent-id", default=None, help="Agent ID for server-side trace resolution") From 20f5e30d03ad3d91dcf687c016bc9b15fcc4fea0 Mon Sep 17 00:00:00 2001 From: Shruti Iyer Date: Thu, 30 Apr 2026 17:55:17 -0400 Subject: [PATCH 14/18] rename --- .../samples/evaluations/README.md | 4 +- ...y => sample_agent_as_target_evaluation.py} | 2 +- .../sample_evaluations_agent_as_target.py | 203 ------------------ ...y => sample_model_as_target_evaluation.py} | 2 +- .../tests/samples/test_samples_evaluations.py | 7 +- 5 files changed, 6 insertions(+), 212 deletions(-) rename sdk/ai/azure-ai-projects/samples/evaluations/{sample_agent_evaluation.py => sample_agent_as_target_evaluation.py} (99%) delete mode 100644 sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_agent_as_target.py rename sdk/ai/azure-ai-projects/samples/evaluations/{sample_model_evaluation.py => sample_model_as_target_evaluation.py} (99%) diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/README.md b/sdk/ai/azure-ai-projects/samples/evaluations/README.md index d601e9e22906..513b0d23052d 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/README.md +++ b/sdk/ai/azure-ai-projects/samples/evaluations/README.md @@ -42,10 +42,10 @@ python sample_evaluations_builtin_with_inline_data.py | Sample | Description | |--------|-------------| -| [sample_agent_evaluation.py](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/ai/azure-ai-projects/samples/evaluations/sample_agent_evaluation.py) | Create a response from an agent and evaluate | +| [sample_agent_as_target_evaluation.py](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/ai/azure-ai-projects/samples/evaluations/sample_agent_as_target_evaluation.py) | Create a response from an agent and evaluate | | [sample_agent_response_evaluation.py](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/ai/azure-ai-projects/samples/evaluations/sample_agent_response_evaluation.py) | Evaluate given agent responses | | [sample_agent_response_evaluation_with_function_tool.py](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/ai/azure-ai-projects/samples/evaluations/sample_agent_response_evaluation_with_function_tool.py) | Evaluate agent responses with function tools | -| [sample_model_evaluation.py](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/ai/azure-ai-projects/samples/evaluations/sample_model_evaluation.py) | Create response from model and evaluate | +| [sample_model_as_target_evaluation.py](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/ai/azure-ai-projects/samples/evaluations/sample_model_as_target_evaluation.py) | Create response from model and evaluate | | [sample_synthetic_data_agent_evaluation.py](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/ai/azure-ai-projects/samples/evaluations/sample_synthetic_data_agent_evaluation.py) | Generate synthetic test data, evaluate a Foundry agent | | [sample_synthetic_data_model_evaluation.py](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/ai/azure-ai-projects/samples/evaluations/sample_synthetic_data_model_evaluation.py) | Generate synthetic test data, evaluate a model | diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_agent_evaluation.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_agent_as_target_evaluation.py similarity index 99% rename from sdk/ai/azure-ai-projects/samples/evaluations/sample_agent_evaluation.py rename to sdk/ai/azure-ai-projects/samples/evaluations/sample_agent_as_target_evaluation.py index 4882737ce719..0d2aec0afb7c 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_agent_evaluation.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_agent_as_target_evaluation.py @@ -12,7 +12,7 @@ for more information. USAGE: - python sample_agent_evaluation.py + python sample_agent_as_target_evaluation.py Before running the sample: diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_agent_as_target.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_agent_as_target.py deleted file mode 100644 index 78b2e363eb21..000000000000 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_agent_as_target.py +++ /dev/null @@ -1,203 +0,0 @@ -# pylint: disable=line-too-long,useless-suppression -# ------------------------------------ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. -# ------------------------------------ - -""" -DESCRIPTION: - Given an AIProjectClient, this sample demonstrates how to run Azure AI Evaluations against - a hosted agent using the azure_ai_target_completions data source. The evaluation service - invokes the agent live for each test query, collects responses, and runs built-in quality - and safety evaluators against them. - - This is different from trace evaluations (sample_evaluations_builtin_with_traces.py) which - evaluate historical agent traces. This sample evaluates agents live. - -USAGE: - python sample_evaluations_agent_as_target.py - - Before running the sample: - - pip install "azure-ai-projects>=2.0.0" python-dotenv - - Set these environment variables with your own values: - 1) FOUNDRY_PROJECT_ENDPOINT - Required. The Azure AI Project endpoint, as found in the overview page of your - Microsoft Foundry project. It has the form: https://.services.ai.azure.com/api/projects/. - 2) FOUNDRY_MODEL_NAME - Required. The Azure OpenAI deployment name to use as the judge model for - built-in evaluators. - 3) AGENT_NAME - Required. The hosted agent name to evaluate. - 4) AGENT_VERSION - Optional. The agent version to evaluate. Defaults to "1". -""" - -import os -import time -from datetime import datetime -from pprint import pprint - -from dotenv import load_dotenv -from azure.ai.projects import AIProjectClient -from azure.ai.projects.models import TestingCriterionAzureAIEvaluator -from azure.identity import DefaultAzureCredential - -load_dotenv() - - -endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] -model_deployment_name = os.environ["FOUNDRY_MODEL_NAME"] -agent_name = os.environ["AGENT_NAME"] -agent_version = os.environ.get("AGENT_VERSION", "1") - -# Sample test queries. Replace with queries appropriate for your agent. -INPUT_QUERIES = [ - { - "item": { - "query": "Hello, what can you help me with?", - } - }, - { - "item": { - "query": "Can you give me a brief summary of your capabilities?", - } - }, -] - - -def _build_evaluator_config( - name: str, - evaluator_name: str, - response_mapping: str, -) -> TestingCriterionAzureAIEvaluator: - """Create a standard Azure AI evaluator configuration block. - - :param name: Display name for this testing criterion. - :type name: str - :param evaluator_name: Built-in evaluator identifier. - :type evaluator_name: str - :param response_mapping: Response mapping expression used by the evaluator. - :type response_mapping: str - :return: Evaluator configuration used in the evaluation group. - :rtype: ~azure.ai.projects.models.TestingCriterionAzureAIEvaluator - """ - return TestingCriterionAzureAIEvaluator( - type="azure_ai_evaluator", - name=name, - evaluator_name=evaluator_name, - data_mapping={ - "query": "{{item.query}}", - "response": response_mapping, - "tool_definitions": "{{sample.tool_definitions}}", - }, - initialization_parameters={ - "deployment_name": model_deployment_name, - }, - ) - - -def main() -> None: - testing_criteria = [ - _build_evaluator_config( - name="intent_resolution", - evaluator_name="builtin.intent_resolution", - response_mapping="{{sample.output_text}}", - ), - _build_evaluator_config( - name="task_adherence", - evaluator_name="builtin.task_adherence", - response_mapping="{{sample.output_items}}", - ), - ] - - data_source_config = { - "type": "custom", - "item_schema": { - "type": "object", - "properties": { - "query": {"type": "string"}, - "context": {"type": "string"}, - "ground_truth": {"type": "string"}, - }, - "required": ["query"], - }, - "include_sample_schema": True, - } - - input_messages = { - "type": "template", - "template": [ - { - "type": "message", - "role": "user", - "content": { - "type": "input_text", - "text": "{{item.query}}", - }, - } - ], - } - - print(f"Agent: {agent_name} v{agent_version}") - print(f"Evaluators: {len(testing_criteria)}") - print(f"Queries: {len(INPUT_QUERIES)}") - - with DefaultAzureCredential() as credential: - with AIProjectClient(endpoint=endpoint, credential=credential) as project_client: - client = project_client.get_openai_client() - - print("\nCreating evaluation") - eval_name = f"Hosted Agent Eval - {agent_name} - {datetime.now().strftime('%Y%m%d_%H%M%S')}" - eval_object = client.evals.create( - name=eval_name, - data_source_config=data_source_config, # type: ignore - testing_criteria=testing_criteria, # type: ignore - ) - print(f"Evaluation created (id: {eval_object.id}, name: {eval_object.name})") - - print("\nGet Evaluation by Id") - eval_object_response = client.evals.retrieve(eval_object.id) - print("Evaluation Response:") - pprint(eval_object_response) - - print("\nCreating Eval Run") - data_source = { - "type": "azure_ai_target_completions", - "source": { - "type": "file_content", - "content": INPUT_QUERIES, - }, - "input_messages": input_messages, - "target": { - "type": "azure_ai_agent", - "name": agent_name, - "version": agent_version, - }, - } - run_name = f"Run - {agent_name} - {datetime.now().strftime('%Y%m%d_%H%M%S')}" - eval_run_object = client.evals.runs.create( - eval_id=eval_object.id, - name=run_name, - data_source=data_source, # type: ignore - ) - print("Eval Run created") - pprint(eval_run_object) - - print("\nMonitoring Eval Run status...") - while True: - run = client.evals.runs.retrieve(run_id=eval_run_object.id, eval_id=eval_object.id) - print(f"Status: {run.status}") - - if run.status in {"completed", "failed", "canceled"}: - print("\nEval Run finished!") - print("Final Eval Run Response:") - pprint(run) - break - - time.sleep(5) - print("Waiting for eval run to complete...") - - client.evals.delete(eval_id=eval_object.id) - print("Evaluation deleted") - - -if __name__ == "__main__": - main() diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_model_evaluation.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_model_as_target_evaluation.py similarity index 99% rename from sdk/ai/azure-ai-projects/samples/evaluations/sample_model_evaluation.py rename to sdk/ai/azure-ai-projects/samples/evaluations/sample_model_as_target_evaluation.py index 82b18c6f045d..3857ab909d40 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_model_evaluation.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_model_as_target_evaluation.py @@ -13,7 +13,7 @@ for more information. USAGE: - python sample_model_evaluation.py + python sample_model_as_target_evaluation.py Before running the sample: diff --git a/sdk/ai/azure-ai-projects/tests/samples/test_samples_evaluations.py b/sdk/ai/azure-ai-projects/tests/samples/test_samples_evaluations.py index 31bae3292fe5..bfc0096c0ef0 100644 --- a/sdk/ai/azure-ai-projects/tests/samples/test_samples_evaluations.py +++ b/sdk/ai/azure-ai-projects/tests/samples/test_samples_evaluations.py @@ -76,8 +76,8 @@ class TestSamplesEvaluations(AzureRecordedTestCase): Included samples (28): Main evaluation samples (13): - - sample_agent_evaluation.py - - sample_model_evaluation.py + - sample_agent_as_target_evaluation.py + - sample_model_as_target_evaluation.py - sample_agent_response_evaluation.py - sample_agent_response_evaluation_with_function_tool.py - sample_evaluations_builtin_with_inline_data.py @@ -120,8 +120,6 @@ class TestSamplesEvaluations(AzureRecordedTestCase): get_bearer_token_provider() which is incompatible with mock credentials. External service dependencies (require additional Azure services): - - sample_evaluations_agent_as_target.py: Requires a deployed Azure AI Agent for - agent-as-target evaluation. - sample_evaluations_builtin_with_traces.py: Requires Azure Application Insights and uses azure-monitor-query to fetch traces. - sample_scheduled_evaluations.py: Requires Azure RBAC assignment via @@ -142,7 +140,6 @@ class TestSamplesEvaluations(AzureRecordedTestCase): samples_to_skip=[ "sample_evaluations_ai_assisted.py", # Similarity evaluator returns FAILED_EXECUTION ('query' is missing) "sample_evaluations_builtin_with_inline_data_oai.py", # 401 AuthenticationError (invalid subscription key or API endpoint) - "sample_evaluations_agent_as_target.py", # Requires Azure AI Agent deployment (agent-as-target evaluation) "sample_evaluations_builtin_with_traces.py", # Missing required env var APPINSIGHTS_RESOURCE_ID (KeyError) "sample_evaluations_score_model_grader_with_image.py", # Eval fails: image inputs not supported for configured grader model "sample_evaluations_score_model_grader_with_image_model_target.py", # Eval fails: image inputs not supported for configured grader model From 7f6d0284e1b9ce5a1e5aae9f590b296321dc6f45 Mon Sep 17 00:00:00 2001 From: Shruti Iyer Date: Thu, 30 Apr 2026 17:55:32 -0400 Subject: [PATCH 15/18] rename --- sdk/ai/azure-ai-projects/evaluation-typespec-issues.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sdk/ai/azure-ai-projects/evaluation-typespec-issues.md b/sdk/ai/azure-ai-projects/evaluation-typespec-issues.md index da81a132900e..eff5534462c7 100644 --- a/sdk/ai/azure-ai-projects/evaluation-typespec-issues.md +++ b/sdk/ai/azure-ai-projects/evaluation-typespec-issues.md @@ -47,7 +47,7 @@ The TypeSpec model TargetCompletionEvalRunDataSource has a property: where OpenAI.CreateEvalCompletionsRunDataSourceInputMessagesItemReference has a type discriminator: "item_reference". -Now looking at the Python sample `sample_agent_evaluation.py` it defines input_messages as the following: +Now looking at the Python sample `sample_agent_as_target_evaluation.py` it defines input_messages as the following: ``` "input_messages": { @@ -64,11 +64,11 @@ So I think our TypeSpec is missing a Union with other OpenAI TypeSpec models. ## Model `ResponseRetrievalItemGenerationParams`, property `max_runs_hourly` -Is defined in TypeSpec (using type discriminator "response_retrieval"), having `max_runs_hourly` as required properties. Yet the sample code `sample_agent_evaluation.py` does not set it. Should it be made optional in TypeSpec? Removed? +Is defined in TypeSpec (using type discriminator "response_retrieval"), having `max_runs_hourly` as required properties. Yet the sample code `sample_agent_as_target_evaluation.py` does not set it. Should it be made optional in TypeSpec? Removed? ## Model AzureAIResponsesEvalRunDataSource, properties `max_runs_hourly` and `event_configuration_id` -Is defined in TypeSpec (using type discriminator "azure_ai_responses"), having required properties max_runs_hourly and event_configuration_id. Yet the same sample code `sample_agent_evaluation.py` does not set them. Should they be made optional in TypeSpec? +Is defined in TypeSpec (using type discriminator "azure_ai_responses"), having required properties max_runs_hourly and event_configuration_id. Yet the same sample code `sample_agent_as_target_evaluation.py` does not set them. Should they be made optional in TypeSpec? ## Score Model Grader @@ -175,7 +175,7 @@ model ModelSamplingParams { } ``` -Note that all properties are REQUIRED on ModelSamplingParam. Yet when I look at the sample `samples\evaluations\sample_model_evaluation.py` it has: +Note that all properties are REQUIRED on ModelSamplingParam. Yet when I look at the sample `samples\evaluations\sample_model_as_target_evaluation.py` it has: ``` "sampling_params": { # Note: model sampling parameters are optional and can differ per model From 187f13dc12ee66aedcfc064d5f52cdd0a9d1f81f Mon Sep 17 00:00:00 2001 From: Shruti Iyer Date: Fri, 1 May 2026 10:20:18 -0400 Subject: [PATCH 16/18] one leftover rename --- .../azure-ai-projects/tests/samples/test_samples_evaluations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/ai/azure-ai-projects/tests/samples/test_samples_evaluations.py b/sdk/ai/azure-ai-projects/tests/samples/test_samples_evaluations.py index bfc0096c0ef0..8191b15b9f02 100644 --- a/sdk/ai/azure-ai-projects/tests/samples/test_samples_evaluations.py +++ b/sdk/ai/azure-ai-projects/tests/samples/test_samples_evaluations.py @@ -131,7 +131,7 @@ class TestSamplesEvaluations(AzureRecordedTestCase): """ # To run this test with a specific sample, use: - # pytest tests/samples/test_samples_evaluations.py::TestSamplesEvaluations::test_evaluation_samples[sample_agent_evaluation] + # pytest tests/samples/test_samples_evaluations.py::TestSamplesEvaluations::test_evaluation_samples[sample_agent_as_target_evaluation] @evaluationsPreparer() @pytest.mark.parametrize( "sample_path", From cafc55a37fc3c1cbf8e8edfc43156349476b01fe Mon Sep 17 00:00:00 2001 From: Shruti Iyer Date: Fri, 1 May 2026 10:38:21 -0400 Subject: [PATCH 17/18] undo naming --- sdk/ai/azure-ai-projects/evaluation-typespec-issues.md | 8 ++++---- sdk/ai/azure-ai-projects/samples/evaluations/README.md | 4 ++-- ...as_target_evaluation.py => sample_agent_evaluation.py} | 2 +- ...as_target_evaluation.py => sample_model_evaluation.py} | 2 +- .../tests/samples/test_samples_evaluations.py | 6 +++--- 5 files changed, 11 insertions(+), 11 deletions(-) rename sdk/ai/azure-ai-projects/samples/evaluations/{sample_agent_as_target_evaluation.py => sample_agent_evaluation.py} (99%) rename sdk/ai/azure-ai-projects/samples/evaluations/{sample_model_as_target_evaluation.py => sample_model_evaluation.py} (99%) diff --git a/sdk/ai/azure-ai-projects/evaluation-typespec-issues.md b/sdk/ai/azure-ai-projects/evaluation-typespec-issues.md index eff5534462c7..da81a132900e 100644 --- a/sdk/ai/azure-ai-projects/evaluation-typespec-issues.md +++ b/sdk/ai/azure-ai-projects/evaluation-typespec-issues.md @@ -47,7 +47,7 @@ The TypeSpec model TargetCompletionEvalRunDataSource has a property: where OpenAI.CreateEvalCompletionsRunDataSourceInputMessagesItemReference has a type discriminator: "item_reference". -Now looking at the Python sample `sample_agent_as_target_evaluation.py` it defines input_messages as the following: +Now looking at the Python sample `sample_agent_evaluation.py` it defines input_messages as the following: ``` "input_messages": { @@ -64,11 +64,11 @@ So I think our TypeSpec is missing a Union with other OpenAI TypeSpec models. ## Model `ResponseRetrievalItemGenerationParams`, property `max_runs_hourly` -Is defined in TypeSpec (using type discriminator "response_retrieval"), having `max_runs_hourly` as required properties. Yet the sample code `sample_agent_as_target_evaluation.py` does not set it. Should it be made optional in TypeSpec? Removed? +Is defined in TypeSpec (using type discriminator "response_retrieval"), having `max_runs_hourly` as required properties. Yet the sample code `sample_agent_evaluation.py` does not set it. Should it be made optional in TypeSpec? Removed? ## Model AzureAIResponsesEvalRunDataSource, properties `max_runs_hourly` and `event_configuration_id` -Is defined in TypeSpec (using type discriminator "azure_ai_responses"), having required properties max_runs_hourly and event_configuration_id. Yet the same sample code `sample_agent_as_target_evaluation.py` does not set them. Should they be made optional in TypeSpec? +Is defined in TypeSpec (using type discriminator "azure_ai_responses"), having required properties max_runs_hourly and event_configuration_id. Yet the same sample code `sample_agent_evaluation.py` does not set them. Should they be made optional in TypeSpec? ## Score Model Grader @@ -175,7 +175,7 @@ model ModelSamplingParams { } ``` -Note that all properties are REQUIRED on ModelSamplingParam. Yet when I look at the sample `samples\evaluations\sample_model_as_target_evaluation.py` it has: +Note that all properties are REQUIRED on ModelSamplingParam. Yet when I look at the sample `samples\evaluations\sample_model_evaluation.py` it has: ``` "sampling_params": { # Note: model sampling parameters are optional and can differ per model diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/README.md b/sdk/ai/azure-ai-projects/samples/evaluations/README.md index 513b0d23052d..d601e9e22906 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/README.md +++ b/sdk/ai/azure-ai-projects/samples/evaluations/README.md @@ -42,10 +42,10 @@ python sample_evaluations_builtin_with_inline_data.py | Sample | Description | |--------|-------------| -| [sample_agent_as_target_evaluation.py](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/ai/azure-ai-projects/samples/evaluations/sample_agent_as_target_evaluation.py) | Create a response from an agent and evaluate | +| [sample_agent_evaluation.py](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/ai/azure-ai-projects/samples/evaluations/sample_agent_evaluation.py) | Create a response from an agent and evaluate | | [sample_agent_response_evaluation.py](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/ai/azure-ai-projects/samples/evaluations/sample_agent_response_evaluation.py) | Evaluate given agent responses | | [sample_agent_response_evaluation_with_function_tool.py](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/ai/azure-ai-projects/samples/evaluations/sample_agent_response_evaluation_with_function_tool.py) | Evaluate agent responses with function tools | -| [sample_model_as_target_evaluation.py](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/ai/azure-ai-projects/samples/evaluations/sample_model_as_target_evaluation.py) | Create response from model and evaluate | +| [sample_model_evaluation.py](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/ai/azure-ai-projects/samples/evaluations/sample_model_evaluation.py) | Create response from model and evaluate | | [sample_synthetic_data_agent_evaluation.py](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/ai/azure-ai-projects/samples/evaluations/sample_synthetic_data_agent_evaluation.py) | Generate synthetic test data, evaluate a Foundry agent | | [sample_synthetic_data_model_evaluation.py](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/ai/azure-ai-projects/samples/evaluations/sample_synthetic_data_model_evaluation.py) | Generate synthetic test data, evaluate a model | diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_agent_as_target_evaluation.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_agent_evaluation.py similarity index 99% rename from sdk/ai/azure-ai-projects/samples/evaluations/sample_agent_as_target_evaluation.py rename to sdk/ai/azure-ai-projects/samples/evaluations/sample_agent_evaluation.py index 0d2aec0afb7c..4882737ce719 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_agent_as_target_evaluation.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_agent_evaluation.py @@ -12,7 +12,7 @@ for more information. USAGE: - python sample_agent_as_target_evaluation.py + python sample_agent_evaluation.py Before running the sample: diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_model_as_target_evaluation.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_model_evaluation.py similarity index 99% rename from sdk/ai/azure-ai-projects/samples/evaluations/sample_model_as_target_evaluation.py rename to sdk/ai/azure-ai-projects/samples/evaluations/sample_model_evaluation.py index 3857ab909d40..82b18c6f045d 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_model_as_target_evaluation.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_model_evaluation.py @@ -13,7 +13,7 @@ for more information. USAGE: - python sample_model_as_target_evaluation.py + python sample_model_evaluation.py Before running the sample: diff --git a/sdk/ai/azure-ai-projects/tests/samples/test_samples_evaluations.py b/sdk/ai/azure-ai-projects/tests/samples/test_samples_evaluations.py index 8191b15b9f02..2618d0320a05 100644 --- a/sdk/ai/azure-ai-projects/tests/samples/test_samples_evaluations.py +++ b/sdk/ai/azure-ai-projects/tests/samples/test_samples_evaluations.py @@ -76,8 +76,8 @@ class TestSamplesEvaluations(AzureRecordedTestCase): Included samples (28): Main evaluation samples (13): - - sample_agent_as_target_evaluation.py - - sample_model_as_target_evaluation.py + - sample_agent_evaluation.py + - sample_model_evaluation.py - sample_agent_response_evaluation.py - sample_agent_response_evaluation_with_function_tool.py - sample_evaluations_builtin_with_inline_data.py @@ -131,7 +131,7 @@ class TestSamplesEvaluations(AzureRecordedTestCase): """ # To run this test with a specific sample, use: - # pytest tests/samples/test_samples_evaluations.py::TestSamplesEvaluations::test_evaluation_samples[sample_agent_as_target_evaluation] + # pytest tests/samples/test_samples_evaluations.py::TestSamplesEvaluations::test_evaluation_samples[sample_agent_evaluation] @evaluationsPreparer() @pytest.mark.parametrize( "sample_path", From ecd2dcd7369ef9600d9759a434cc822b3fd25893 Mon Sep 17 00:00:00 2001 From: Shruti Iyer Date: Fri, 1 May 2026 11:35:46 -0400 Subject: [PATCH 18/18] fix other samples pylint --- ...aluations_score_model_grader_with_audio.py | 22 ++++++++++++------- ...re_model_grader_with_audio_model_target.py | 22 ++++++++++++------- ...re_model_grader_with_image_model_target.py | 20 ++++++++--------- 3 files changed, 38 insertions(+), 26 deletions(-) diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_score_model_grader_with_audio.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_score_model_grader_with_audio.py index 7583f1e8636e..165bd0f43929 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_score_model_grader_with_audio.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_score_model_grader_with_audio.py @@ -26,13 +26,15 @@ 3) AZURE_AI_MODEL_DEPLOYMENT_NAME_FOR_AUDIO - Required. The name of the model deployment for audio to use for evaluation, recommend to use "gpt-4o-audio-preview" """ -import os import base64 - -from azure.identity import DefaultAzureCredential -from azure.ai.projects import AIProjectClient +import os import time from pprint import pprint + +from dotenv import load_dotenv +from azure.ai.projects import AIProjectClient +from azure.identity import DefaultAzureCredential +from openai.types.eval_create_params import DataSourceConfigCustom from openai.types.evals.create_eval_completions_run_data_source_param import ( CreateEvalCompletionsRunDataSourceParam, SourceFileContent, @@ -41,8 +43,6 @@ InputMessagesTemplateTemplateEvalItem, ) from openai.types.responses import EasyInputMessageParam, ResponseInputAudioParam -from openai.types.eval_create_params import DataSourceConfigCustom -from dotenv import load_dotenv load_dotenv() file_path = os.path.abspath(__file__) @@ -54,7 +54,13 @@ def audio_to_base64(audio_path: str) -> str: - """Read an audio file and return its base64-encoded content.""" + """Read an audio file and return its base64-encoded content. + + :param audio_path: Absolute path to the input audio file. + :type audio_path: str + :return: Base64-encoded audio data. + :rtype: str + """ with open(audio_path, "rb") as f: return base64.b64encode(f.read()).decode("utf-8") @@ -183,7 +189,7 @@ def audio_to_base64(audio_path: str) -> str: while True: run = client.evals.runs.retrieve(run_id=eval_run_response.id, eval_id=eval_object.id) - if run.status == "completed" or run.status == "failed": + if run.status in {"completed", "failed"}: output_items = list(client.evals.runs.output_items.list(run_id=run.id, eval_id=eval_object.id)) pprint(output_items) print(f"Eval Run Report URL: {run.report_url}") diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_score_model_grader_with_audio_model_target.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_score_model_grader_with_audio_model_target.py index 5c3c4ae51f78..75eb7b97f566 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_score_model_grader_with_audio_model_target.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_score_model_grader_with_audio_model_target.py @@ -26,13 +26,15 @@ 3) AZURE_AI_MODEL_DEPLOYMENT_NAME_FOR_AUDIO - Required. The name of the model deployment for audio to use for evaluation, recommend to use "gpt-4o-audio-preview" """ -import os import base64 - -from azure.identity import DefaultAzureCredential -from azure.ai.projects import AIProjectClient +import os import time from pprint import pprint + +from dotenv import load_dotenv +from azure.ai.projects import AIProjectClient +from azure.identity import DefaultAzureCredential +from openai.types.eval_create_params import DataSourceConfigCustom from openai.types.evals.create_eval_completions_run_data_source_param import ( SourceFileContent, SourceFileContentContent, @@ -40,8 +42,6 @@ InputMessagesTemplateTemplateEvalItem, ) from openai.types.responses import EasyInputMessageParam, ResponseInputAudioParam -from openai.types.eval_create_params import DataSourceConfigCustom -from dotenv import load_dotenv load_dotenv() file_path = os.path.abspath(__file__) @@ -53,7 +53,13 @@ def audio_to_base64(audio_path: str) -> str: - """Read an audio file and return its base64-encoded content.""" + """Read an audio file and return its base64-encoded content. + + :param audio_path: Absolute path to the input audio file. + :type audio_path: str + :return: Base64-encoded audio data. + :rtype: str + """ with open(audio_path, "rb") as f: return base64.b64encode(f.read()).decode("utf-8") @@ -188,7 +194,7 @@ def audio_to_base64(audio_path: str) -> str: while True: run = client.evals.runs.retrieve(run_id=eval_run_response.id, eval_id=eval_object.id) - if run.status == "completed" or run.status == "failed": + if run.status in {"completed", "failed"}: output_items = list(client.evals.runs.output_items.list(run_id=run.id, eval_id=eval_object.id)) pprint(output_items) print(f"Eval Run Report URL: {run.report_url}") diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_score_model_grader_with_image_model_target.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_score_model_grader_with_image_model_target.py index 88920549113d..01a22ccfe588 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_score_model_grader_with_image_model_target.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_score_model_grader_with_image_model_target.py @@ -23,15 +23,17 @@ 2) AZURE_AI_MODEL_DEPLOYMENT_NAME - Required. The name of the model deployment to use for evaluation. """ -import os import base64 -from PIL import Image +import os +import time from io import BytesIO +from pprint import pprint -from azure.identity import DefaultAzureCredential +from PIL import Image +from dotenv import load_dotenv from azure.ai.projects import AIProjectClient -import time -from pprint import pprint +from azure.identity import DefaultAzureCredential +from openai.types.eval_create_params import DataSourceConfigCustom from openai.types.evals.create_eval_completions_run_data_source_param import ( SourceFileContent, SourceFileContentContent, @@ -40,8 +42,6 @@ InputMessagesTemplateTemplateEvalItemContentInputImage, ) from openai.types.responses import EasyInputMessageParam -from openai.types.eval_create_params import DataSourceConfigCustom -from dotenv import load_dotenv load_dotenv() file_path = os.path.abspath(__file__) @@ -51,8 +51,8 @@ model_deployment_name = os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"] -def image_to_data_uri(image_path: str) -> str: - with Image.open(image_path) as img: +def image_to_data_uri(image_file_path: str) -> str: + with Image.open(image_file_path) as img: buffered = BytesIO() img.save(buffered, format=img.format or "PNG") img_str = base64.b64encode(buffered.getvalue()).decode() @@ -186,7 +186,7 @@ def image_to_data_uri(image_path: str) -> str: while True: run = client.evals.runs.retrieve(run_id=eval_run_response.id, eval_id=eval_object.id) - if run.status == "completed" or run.status == "failed": + if run.status in {"completed", "failed"}: output_items = list(client.evals.runs.output_items.list(run_id=run.id, eval_id=eval_object.id)) pprint(output_items) print(f"Eval Run Report URL: {run.report_url}")