diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/README.md b/sdk/ai/azure-ai-projects/samples/evaluations/README.md index d601e9e22906..9a3dc1dc6f45 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/README.md +++ b/sdk/ai/azure-ai-projects/samples/evaluations/README.md @@ -80,6 +80,16 @@ These samples require additional setup or Azure services: | [sample_eval_catalog_code_based_evaluators.py](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/ai/azure-ai-projects/samples/evaluations/sample_eval_catalog_code_based_evaluators.py) | Custom code-based (inline) evaluators | | [sample_eval_catalog_prompt_based_evaluators.py](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/ai/azure-ai-projects/samples/evaluations/sample_eval_catalog_prompt_based_evaluators.py) | Custom prompt-based evaluators | +### Rubric Evaluators + +| Sample | Description | +|--------|-------------| +| [sample_rubric_evaluator_manual.py](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_manual.py) | Hand-author a rubric evaluator (dimensions, weights, pass threshold) with `create_version`, then use it in an OpenAI eval run | +| [sample_rubric_evaluator_generation_basic.py](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_basic.py) | Generate an evaluator from a single prompt source, inspect the produced dimensions, and use the auto-saved evaluator in an eval run | +| [sample_rubric_evaluator_generation_all_sources.py](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_all_sources.py) | Exercise every source type — prompt + agent + dataset in a combined job, plus a separate traces + agent-companion job | +| [sample_rubric_evaluator_generation_iterate.py](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_iterate.py) | Human-in-the-loop iteration: generate v1, edit dimensions locally, save as v2 with `create_version` | +| [sample_rubric_evaluator_generation_lifecycle.py](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_lifecycle.py) | Full generation-job lifecycle: idempotent `create_generation_job`, polling, list, and delete | + ### Agentic Evaluators Located in the [agentic_evaluators](https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/ai/azure-ai-projects/samples/evaluations/agentic_evaluators) subfolder: diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_all_sources.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_all_sources.py new file mode 100644 index 000000000000..5e29496f0eae --- /dev/null +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_all_sources.py @@ -0,0 +1,250 @@ +# pylint: disable=line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ + +""" +DESCRIPTION: + End-to-end scenario showing every rubric evaluator generation source type + in a single sample file. There are four source types: + + 1. `Prompt` - an inline natural-language description of the application. + 2. `Agent` - references an agent registered in the Foundry project. + 3. `Dataset` - references an uploaded dataset (name + version). + 4. `traces` - Application Insights conversation traces for an agent, + within a time window. + + The first three source types can be combined into a single generation job + to produce a richer rubric. The `traces` source type requires a companion + source (the service rejects `traces`-only source arrays), so this sample + submits it as a separate, second job paired with an `Agent` companion. + + Optional environment variables let you skip variants where you don't have + a suitable agent, dataset, or recent traces in the target Foundry project. + +USAGE: + python sample_rubric_evaluator_generation_all_sources.py + + Before running the sample: + + pip install "azure-ai-projects>=2.2.0" azure-identity python-dotenv + + Set these environment variables with your own values: + 1) FOUNDRY_PROJECT_ENDPOINT - Required. The Azure AI Project endpoint, as found + in the overview page of your Microsoft Foundry project. + 2) FOUNDRY_MODEL_NAME - Required. The name of the model the generation job + will use (e.g. `gpt-4o`, `gpt-4.1`). The generation runs inline server + side, so no deployment in your project is required. + 3) FOUNDRY_AGENT_NAME - Optional. Name of an agent registered in the project. + Enables the `Agent` source and the `traces`-source job. + 4) FOUNDRY_REFERENCE_DATASET_NAME - Optional. Name of an uploaded dataset. + Enables the `Dataset` source. + 5) FOUNDRY_REFERENCE_DATASET_VERSION - Optional. Version of the uploaded dataset. + Enables the `Dataset` source. + 6) FOUNDRY_TRACES_WINDOW_DAYS - Optional. Look-back window in days for the + `traces` source. Defaults to 7. + 7) POLL_INTERVAL_SECONDS - Optional. Number of seconds to sleep between + generation job status polls. Defaults to 10. +""" + +import os +import time +import uuid +from datetime import datetime, timezone +from typing import Any, Dict, List, cast + +from dotenv import load_dotenv + +from azure.identity import DefaultAzureCredential +from azure.ai.projects import AIProjectClient +from azure.ai.projects.models import JobStatus + +load_dotenv() + +endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] +model_name = os.environ["FOUNDRY_MODEL_NAME"] +agent_name = os.environ.get("FOUNDRY_AGENT_NAME") +dataset_name = os.environ.get("FOUNDRY_REFERENCE_DATASET_NAME") +dataset_version = os.environ.get("FOUNDRY_REFERENCE_DATASET_VERSION") +traces_window_days = int(os.environ.get("FOUNDRY_TRACES_WINDOW_DAYS", "7")) +poll_interval_seconds = int(os.environ.get("POLL_INTERVAL_SECONDS", "10")) + +# Unique per-run suffix so repeated runs do not collide on evaluator name. +ts = datetime.now(tz=timezone.utc).strftime("%Y%m%d%H%M%S") +short = uuid.uuid4().hex[:6] +multi_name = f"multi-source-{ts}-{short}" +traces_name = f"traces-source-{ts}-{short}" + +TERMINAL_STATUSES = {JobStatus.SUCCEEDED, JobStatus.FAILED, JobStatus.CANCELLED} + +multi_evaluator_version = "" +traces_evaluator_version = "" + +with ( + DefaultAzureCredential() as credential, + # `allow_preview` and `api_version` are required for the evaluator + # generation endpoints in this preview. + AIProjectClient( + endpoint=endpoint, + credential=credential, + allow_preview=True, + api_version="2025-11-15-preview", + ) as project_client, +): + + # ------------------------------------------------------------------ + # 1. Combined Prompt + Agent + Dataset generation job. + # ------------------------------------------------------------------ + print("Build sources for the combined generation job:") + multi_sources: List[Dict[str, Any]] = [ + { + "type": "Prompt", + "description": "Inline application overview.", + "prompt": ( + "You are evaluating a customer-support assistant that helps users " + "manage their accounts, troubleshoot issues, and place orders. The " + "assistant uses tools for account lookup, password reset, and order " + "creation. It must confirm intent before performing destructive " + "actions and maintain a patient, professional tone." + ), + } + ] + if agent_name: + print(f" Including Agent source: `{agent_name}`.") + multi_sources.append( + { + "type": "Agent", + "description": "Agent metadata enriches the rubric with tool and instruction signals.", + "agent_name": agent_name, + } + ) + else: + print(" Skipping Agent source (FOUNDRY_AGENT_NAME not set).") + + if dataset_name and dataset_version: + print(f" Including Dataset source: `{dataset_name}` v`{dataset_version}`.") + multi_sources.append( + { + "type": "Dataset", + "description": "Reference examples ground dimensions in real data.", + "name": dataset_name, + "version": dataset_version, + } + ) + else: + print(" Skipping Dataset source (FOUNDRY_REFERENCE_DATASET_NAME / _VERSION not set).") + + print(f"Create combined generation job for evaluator `{multi_name}`.") + multi_job = project_client.beta.evaluators.create_generation_job( + job={ + "model": model_name, + "name": "Multi-source generation", + "evaluator_name": multi_name, + "evaluator_display_name": "Customer Support Quality (multi-source)", + "evaluator_description": "Generated from prompt, agent, and dataset signals.", + "sources": multi_sources, + }, + operation_id=f"rubric-multi-{short}", + ) + print(f"Created generation job `{multi_job.id}` (status: `{cast(JobStatus, multi_job.status).value}`).") + + print(f"Poll job `{multi_job.id}` until it reaches a terminal state.", end="", flush=True) + while multi_job.status not in TERMINAL_STATUSES: + time.sleep(poll_interval_seconds) + multi_job = project_client.beta.evaluators.get_generation_job(multi_job.id) + print(".", end="", flush=True) + print() + print(f"Final job status: `{cast(JobStatus, multi_job.status).value}`.") + + if multi_job.status != JobStatus.SUCCEEDED: + message = multi_job.error.message if multi_job.error is not None else "" + print( + f"Combined job `{multi_job.id}` ended with status " + f"`{cast(JobStatus, multi_job.status).value}`: {message}" + ) + else: + evaluator = multi_job.result + multi_evaluator_version = evaluator.version or "" + print(f"Generated evaluator: name=`{evaluator.name}` version=`{evaluator.version}`.") + print(f"Categories: {[c.value for c in evaluator.categories]}") + print(f"Dimensions ({len(evaluator.definition.dimensions)}):") + for dim in evaluator.definition.dimensions: + marker = " [ALWAYS-ON]" if dim.always_applicable else "" + print(f" - {dim.id} (weight={dim.weight}){marker}") + + # ------------------------------------------------------------------ + # 2. Separate `traces` + Agent companion generation job. + # ------------------------------------------------------------------ + # The traces source requires a companion source because the service + # rejects sources arrays consisting only of traces. The Agent source + # is the typical companion. + if not agent_name: + print("Skip traces job: requires FOUNDRY_AGENT_NAME for both the traces source and companion.") + else: + print(f"Create traces-source generation job for evaluator `{traces_name}`.") + print(f" agent=`{agent_name}` look-back window: {traces_window_days} days") + now = int(time.time()) + start_time = now - traces_window_days * 24 * 3600 + end_time = now + 600 # small padding for clock skew + + traces_job = project_client.beta.evaluators.create_generation_job( + job={ + "model": model_name, + "name": "Traces-source generation", + "evaluator_name": traces_name, + "evaluator_display_name": "Customer Support Quality (from traces)", + "evaluator_description": "Generated from real Application Insights conversation traces.", + "sources": [ + { + "type": "traces", + "description": "Application Insights conversation traces for the agent.", + "agent_name": agent_name, + "start_time": start_time, + "end_time": end_time, + }, + { + "type": "Agent", + "description": "Companion source (service rejects traces-only).", + "agent_name": agent_name, + }, + ], + }, + operation_id=f"rubric-traces-{short}", + ) + print(f"Created generation job `{traces_job.id}` (status: `{cast(JobStatus, traces_job.status).value}`).") + + print(f"Poll job `{traces_job.id}` until it reaches a terminal state.", end="", flush=True) + while traces_job.status not in TERMINAL_STATUSES: + time.sleep(poll_interval_seconds) + traces_job = project_client.beta.evaluators.get_generation_job(traces_job.id) + print(".", end="", flush=True) + print() + print(f"Final job status: `{cast(JobStatus, traces_job.status).value}`.") + + if traces_job.status != JobStatus.SUCCEEDED: + message = traces_job.error.message if traces_job.error is not None else "" + print( + f"Traces job `{traces_job.id}` ended with status " + f"`{cast(JobStatus, traces_job.status).value}`: {message}" + ) + else: + evaluator = traces_job.result + traces_evaluator_version = evaluator.version or "" + print(f"Generated evaluator: name=`{evaluator.name}` version=`{evaluator.version}`.") + print(f"Categories: {[c.value for c in evaluator.categories]}") + print(f"Dimensions ({len(evaluator.definition.dimensions)}):") + for dim in evaluator.definition.dimensions: + marker = " [ALWAYS-ON]" if dim.always_applicable else "" + print(f" - {dim.id} (weight={dim.weight}){marker}") + + # ------------------------------------------------------------------ + # 3. Clean up. + # ------------------------------------------------------------------ + # `delete_version` cascades to delete the generation job record as well. + if multi_evaluator_version: + print(f"Delete evaluator `{multi_name}` version `{multi_evaluator_version}`.") + project_client.beta.evaluators.delete_version(name=multi_name, version=multi_evaluator_version) + if traces_evaluator_version: + print(f"Delete evaluator `{traces_name}` version `{traces_evaluator_version}`.") + project_client.beta.evaluators.delete_version(name=traces_name, version=traces_evaluator_version) diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_basic.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_basic.py new file mode 100644 index 000000000000..83d2df31056e --- /dev/null +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_basic.py @@ -0,0 +1,273 @@ +# pylint: disable=line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ + +""" +DESCRIPTION: + End-to-end scenario showing rubric evaluator generation from a single + `Prompt` source, followed by an OpenAI evaluation run that uses the + generated evaluator. The sample: + + 1. Creates an `EvaluatorGenerationJob` whose only source is an inline + natural-language description of the application's purpose, capabilities, + and tools. The service synthesizes a rubric tailored to that application. + 2. Polls the generation job to completion and resolves the generated + `EvaluatorVersion`. + 3. Creates an OpenAI evaluation (`client.evals.create`) referencing the + generated evaluator as a testing criterion. + 4. Runs the evaluation against inline JSONL sample data. + 5. Cleans up the evaluation and the evaluator version. Deleting the + evaluator version cascades to delete the generation job record. + + Other source types - `Agent`, `Dataset`, and `traces` - can be used in + place of (or alongside) the prompt source. See + `sample_rubric_evaluator_generation_all_sources.py` for examples of each. + +USAGE: + python sample_rubric_evaluator_generation_basic.py + + Before running the sample: + + pip install "azure-ai-projects>=2.2.0" azure-identity python-dotenv + + Set these environment variables with your own values: + 1) FOUNDRY_PROJECT_ENDPOINT - Required. The Azure AI Project endpoint, as found + in the overview page of your Microsoft Foundry project. + 2) FOUNDRY_MODEL_NAME - Required. The name of the model used by both the + generation job and the eval run's LLM judge (e.g. `gpt-4o`, `gpt-4.1`). + The generation runs inline server side (no deployment required), but the + eval run's grader does require a model deployment in your project. + 3) POLL_INTERVAL_SECONDS - Optional. Number of seconds to sleep between status + polls for both the generation job and the evaluation run. Defaults to 10. +""" + +import os +import time +import uuid +from datetime import datetime, timezone +from typing import cast + +from dotenv import load_dotenv +from openai.types.eval_create_params import DataSourceConfigCustom +from openai.types.evals.create_eval_jsonl_run_data_source_param import ( + CreateEvalJSONLRunDataSourceParam, + SourceFileContent, + SourceFileContentContent, +) + +from azure.identity import DefaultAzureCredential +from azure.ai.projects import AIProjectClient +from azure.ai.projects.models import JobStatus, TestingCriterionAzureAIEvaluator + +load_dotenv() + +endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] +model_name = os.environ["FOUNDRY_MODEL_NAME"] +poll_interval_seconds = int(os.environ.get("POLL_INTERVAL_SECONDS", "10")) + +# Unique per-run name so repeated runs do not collide. +ts = datetime.now(tz=timezone.utc).strftime("%Y%m%d%H%M%S") +short = uuid.uuid4().hex[:6] +evaluator_name = f"reservation-quality-generated-{ts}-{short}" + +TERMINAL_STATUSES = {JobStatus.SUCCEEDED, JobStatus.FAILED, JobStatus.CANCELLED} +TERMINAL_RUN_STATUSES = {"completed", "failed", "canceled"} + +with ( + DefaultAzureCredential() as credential, + # `allow_preview` and `api_version` are required for the evaluator + # generation endpoints in this preview. + AIProjectClient( + endpoint=endpoint, + credential=credential, + allow_preview=True, + api_version="2025-11-15-preview", + ) as project_client, + project_client.get_openai_client() as openai_client, +): + + # ------------------------------------------------------------------ + # 1. Generate an evaluator from a single `Prompt` source. + # ------------------------------------------------------------------ + # The body is sent as a plain dict to match the wire shape expected by + # the 2025-11-15-preview API. + print(f"Create generation job for evaluator `{evaluator_name}`.") + job = project_client.beta.evaluators.create_generation_job( + job={ + "model": model_name, + "name": "Reservation Quality (Generated)", + "evaluator_name": evaluator_name, + "evaluator_display_name": "Reservation Quality (Generated)", + "evaluator_description": ( + "Quality evaluator generated from a prompt describing a " + "restaurant reservation assistant." + ), + "sources": [ + { + "type": "Prompt", + "description": "Application overview - purpose, capabilities, and tools.", + "prompt": ( + "You are evaluating a restaurant reservation assistant. The assistant helps " + "users create, modify, and cancel reservations at participating restaurants. " + "It can:\n" + " - Search for restaurants by name, cuisine, or neighborhood.\n" + " - Check table availability for a requested date, time, and party size.\n" + " - Create, update, and cancel reservations on behalf of the user.\n" + " - Send SMS or email confirmations through a notifications tool.\n" + "It must always confirm the user's intent before committing changes, " + "ask follow-up questions when details are missing, and maintain a polite " + "restaurant-host tone." + ), + } + ], + }, + # `operation_id` makes the call idempotent - re-submitting the same id + # returns the existing job instead of creating a duplicate. + operation_id=f"rubric-eval-basic-{short}", + ) + print(f"Created generation job `{job.id}` (status: `{cast(JobStatus, job.status).value}`).") + + print(f"Poll job `{job.id}` until it reaches a terminal state.", end="", flush=True) + while job.status not in TERMINAL_STATUSES: + time.sleep(poll_interval_seconds) + job = project_client.beta.evaluators.get_generation_job(job.id) + print(".", end="", flush=True) + print() + print(f"Final job status: `{cast(JobStatus, job.status).value}`.") + + if job.status != JobStatus.SUCCEEDED: + message = job.error.message if job.error is not None else "" + raise RuntimeError( + f"Generation job `{job.id}` ended with status `{cast(JobStatus, job.status).value}`: {message}" + ) + + if job.usage is not None: + print(f"Token usage: {job.usage}") + + # On success, the evaluator is automatically saved as version 1. + evaluator = job.result + print(f"Generated evaluator: name=`{evaluator.name}` version=`{evaluator.version}`.") + print(f"Categories: {[c.value for c in evaluator.categories]}") + print(f"Pass threshold: {evaluator.definition.pass_threshold}") + print(f"Dimensions ({len(evaluator.definition.dimensions)}):") + for dim in evaluator.definition.dimensions: + # Quality evaluators always include a non-editable `general_quality` + # residual dimension with always_applicable=True. + marker = " [ALWAYS-ON]" if dim.always_applicable else "" + print(f" - {dim.id} (weight={dim.weight}){marker}: {dim.description[:120]}") + + # ------------------------------------------------------------------ + # 2. Create an OpenAI evaluation that uses the generated evaluator. + # ------------------------------------------------------------------ + data_source_config = DataSourceConfigCustom( + type="custom", + item_schema={ + "type": "object", + "properties": { + "query": {"type": "string"}, + "response": {"type": "string"}, + }, + "required": ["query", "response"], + }, + include_sample_schema=True, + ) + + testing_criteria = [ + TestingCriterionAzureAIEvaluator( + type="azure_ai_evaluator", + name=evaluator.name, + evaluator_name=evaluator.name, + initialization_parameters={"deployment_name": model_name}, + data_mapping={ + "query": "{{item.query}}", + "response": "{{item.response}}", + }, + ) + ] + + print("Create the evaluation.") + eval_object = openai_client.evals.create( + name=f"{evaluator.name}-eval", + data_source_config=data_source_config, + testing_criteria=testing_criteria, + ) + print(f"Evaluation created (id: {eval_object.id}).") + + # ------------------------------------------------------------------ + # 3. Run the evaluation against inline JSONL sample data. + # ------------------------------------------------------------------ + print(f"Create an evaluation run for eval `{eval_object.id}`.") + eval_run = openai_client.evals.runs.create( + eval_id=eval_object.id, + name=f"{evaluator.name}-run", + metadata={"sample": "rubric_evaluator_generation_basic"}, + data_source=CreateEvalJSONLRunDataSourceParam( + type="jsonl", + source=SourceFileContent( + type="file_content", + content=[ + SourceFileContentContent( + item={ + "query": "Book a table for 4 tomorrow at 7 PM.", + "response": ( + "Booked - table for 4 tomorrow at 7:00 PM. A confirmation " + "SMS is on its way." + ), + } + ), + SourceFileContentContent( + item={ + "query": "Cancel my reservation for Friday night.", + "response": "Sure.", + } + ), + ], + ), + ), + ) + print(f"Evaluation run created (id: {eval_run.id}).") + + print(f"Poll run `{eval_run.id}` until it reaches a terminal state.", end="", flush=True) + while eval_run.status not in TERMINAL_RUN_STATUSES: + time.sleep(poll_interval_seconds) + eval_run = openai_client.evals.runs.retrieve(run_id=eval_run.id, eval_id=eval_object.id) + print(".", end="", flush=True) + print() + print(f"Final eval run status: `{eval_run.status}`.") + + if eval_run.status == "completed": + print(f"Result counts: {eval_run.result_counts}") + if eval_run.report_url: + print(f"Eval run report URL: {eval_run.report_url}") + output_items = list(openai_client.evals.runs.output_items.list(run_id=eval_run.id, eval_id=eval_object.id)) + print(f"Output items (total: {len(output_items)}):") + for idx, item in enumerate(output_items, start=1): + results = getattr(item, "results", None) or [] + parts = [] + for r in results: + # Result entries are returned either as typed objects (Azure AI + # evaluators) or as plain dicts (some OpenAI-native evaluators). + if isinstance(r, dict): + name = r.get("name", "?") + score = r.get("score", "n/a") + passed = r.get("passed", "n/a") + else: + name = getattr(r, "name", "?") + score = getattr(r, "score", "n/a") + passed = getattr(r, "passed", "n/a") + parts.append(f"{name}={score} ({passed})") + print(f" item {idx}: status={item.status} | {', '.join(parts)}") + else: + print("Evaluation run did not complete successfully.") + + # ------------------------------------------------------------------ + # 4. Clean up. + # ------------------------------------------------------------------ + print(f"Delete evaluation `{eval_object.id}`.") + openai_client.evals.delete(eval_id=eval_object.id) + + print(f"Delete evaluator `{evaluator.name}` version `{evaluator.version}`.") + # `delete_version` cascades to delete the generation job record as well. + project_client.beta.evaluators.delete_version(name=evaluator.name, version=evaluator.version) diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_iterate.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_iterate.py new file mode 100644 index 000000000000..3c7c05afba2f --- /dev/null +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_iterate.py @@ -0,0 +1,218 @@ +# pylint: disable=line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ + +""" +DESCRIPTION: + End-to-end scenario showing the human-in-the-loop iteration workflow for + rubric evaluators. This is the typical pattern when the first generated + rubric is a good starting point but a domain expert wants to tune the + weighting or add custom dimensions. The sample: + + 1. Generates v1 of an evaluator from a single `Prompt` source. + 2. Inspects the dimensions the service produced. + 3. Edits the dimensions locally - boosts the highest-weight editable + dimension to 10, drops the lowest-weight editable dimension, and + adds a new custom dimension. The non-editable `general_quality` + ALWAYS-ON dimension is preserved verbatim. + 4. Saves the edited definition as v2 with `create_version`. + 5. Calls `list_versions` to enumerate v1 and v2. + 6. Cleans up by deleting both versions. + +USAGE: + python sample_rubric_evaluator_generation_iterate.py + + Before running the sample: + + pip install "azure-ai-projects>=2.2.0" azure-identity python-dotenv + + Set these environment variables with your own values: + 1) FOUNDRY_PROJECT_ENDPOINT - Required. The Azure AI Project endpoint, as found + in the overview page of your Microsoft Foundry project. + 2) FOUNDRY_MODEL_NAME - Required. The name of the model the generation job + will use (e.g. `gpt-4o`, `gpt-4.1`). The generation runs inline server + side, so no deployment in your project is required. + 3) POLL_INTERVAL_SECONDS - Optional. Number of seconds to sleep between status + polls for the generation job. Defaults to 10. +""" + +import os +import time +import uuid +from datetime import datetime, timezone +from typing import cast + +from dotenv import load_dotenv + +from azure.identity import DefaultAzureCredential +from azure.ai.projects import AIProjectClient +from azure.ai.projects.models import EvaluatorDefinitionType, JobStatus + +load_dotenv() + +endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] +model_name = os.environ["FOUNDRY_MODEL_NAME"] +poll_interval_seconds = int(os.environ.get("POLL_INTERVAL_SECONDS", "10")) + +# Unique per-run name so repeated runs do not collide. +ts = datetime.now(tz=timezone.utc).strftime("%Y%m%d%H%M%S") +short = uuid.uuid4().hex[:6] +evaluator_name = f"reservation-quality-iterate-{ts}-{short}" + +TERMINAL_STATUSES = {JobStatus.SUCCEEDED, JobStatus.FAILED, JobStatus.CANCELLED} + +with ( + DefaultAzureCredential() as credential, + # `allow_preview` and `api_version` are required for the evaluator + # generation endpoints in this preview. + AIProjectClient( + endpoint=endpoint, + credential=credential, + allow_preview=True, + api_version="2025-11-15-preview", + ) as project_client, +): + + # ------------------------------------------------------------------ + # 1. Generate v1 of the evaluator from a single `Prompt` source. + # ------------------------------------------------------------------ + print(f"Create generation job for evaluator `{evaluator_name}` (v1).") + job = project_client.beta.evaluators.create_generation_job( + job={ + "model": model_name, + "name": "Reservation Quality (iterate)", + "evaluator_name": evaluator_name, + "evaluator_display_name": "Reservation Quality (iterate)", + "evaluator_description": "Starting point for human-in-the-loop iteration.", + "sources": [ + { + "type": "Prompt", + "description": "Inline application overview.", + "prompt": ( + "You are evaluating a restaurant reservation assistant that creates, " + "modifies, and cancels reservations. It uses tools for restaurant " + "lookup, availability checking, and notifications. It must confirm " + "user intent before committing changes." + ), + } + ], + }, + operation_id=f"rubric-iterate-{short}", + ) + print(f"Created generation job `{job.id}` (status: `{cast(JobStatus, job.status).value}`).") + + print(f"Poll job `{job.id}` until it reaches a terminal state.", end="", flush=True) + while job.status not in TERMINAL_STATUSES: + time.sleep(poll_interval_seconds) + job = project_client.beta.evaluators.get_generation_job(job.id) + print(".", end="", flush=True) + print() + print(f"Final job status: `{cast(JobStatus, job.status).value}`.") + + if job.status != JobStatus.SUCCEEDED: + message = job.error.message if job.error is not None else "" + raise RuntimeError( + f"Generation job `{job.id}` ended with status `{cast(JobStatus, job.status).value}`: {message}" + ) + + v1 = job.result + print(f"v1 created: version=`{v1.version}`.") + print(f"v1 dimensions ({len(v1.definition.dimensions)}):") + for dim in v1.definition.dimensions: + marker = " [ALWAYS-ON]" if dim.always_applicable else "" + print(f" - {dim.id} (weight={dim.weight}){marker}") + + # ------------------------------------------------------------------ + # 2. Edit dimensions locally. + # ------------------------------------------------------------------ + # Domain-expert edits: + # * Always preserve the ALWAYS-ON `general_quality` residual dimension + # exactly as-is (id, weight, description, always_applicable). + # * Boost the most important editable dimension to weight 10. + # * Drop the lowest-weight editable dimension as redundant. + # * Add a new custom dimension specific to this assistant. + print("Apply human edits:") + editable = [d for d in v1.definition.dimensions if not d.always_applicable] + always_on = [d for d in v1.definition.dimensions if d.always_applicable] + + edited_dimensions = [] + if editable: + top = max(editable, key=lambda d: d.weight) + lowest = min(editable, key=lambda d: d.weight) + print(f" Boost `{top.id}` weight {top.weight} -> 10.") + print(f" Drop `{lowest.id}` (weight={lowest.weight}).") + for dim in editable: + if dim.id == lowest.id: + continue + edited_dimensions.append( + { + "id": dim.id, + "description": dim.description, + "weight": 10 if dim.id == top.id else dim.weight, + } + ) + + new_dimension = { + "id": "wait_time_expectations_set", + "description": ( + "Sets clear expectations about wait time, table readiness, or confirmation " + "delivery so the user knows what happens next." + ), + "weight": 4, + } + edited_dimensions.append(new_dimension) + print(f" Add new dimension `{new_dimension['id']}` (weight={new_dimension['weight']}).") + + # Preserve every ALWAYS-ON dimension verbatim. These are non-editable. + for dim in always_on: + print(f" Preserve ALWAYS-ON dimension `{dim.id}` (weight={dim.weight}) verbatim.") + edited_dimensions.append( + { + "id": dim.id, + "description": dim.description, + "weight": dim.weight, + "always_applicable": True, + } + ) + + # ------------------------------------------------------------------ + # 3. Save the edited definition as v2. + # ------------------------------------------------------------------ + print(f"Save edited definition as v2 of `{evaluator_name}`.") + v2 = project_client.beta.evaluators.create_version( + name=evaluator_name, + evaluator_version={ + "name": evaluator_name, + "categories": [c.value for c in v1.categories], + "display_name": v1.display_name, + "description": (v1.description or "") + " (edited)", + "definition": { + "type": EvaluatorDefinitionType.RUBRIC, + "dimensions": edited_dimensions, + "pass_threshold": v1.definition.pass_threshold or 0.6, + }, + }, + ) + print(f"v2 created: version=`{v2.version}`.") + print(f"v2 dimensions ({len(v2.definition.dimensions)}):") + for dim in v2.definition.dimensions: + marker = " [ALWAYS-ON]" if dim.always_applicable else "" + print(f" - {dim.id} (weight={dim.weight}){marker}") + + # ------------------------------------------------------------------ + # 4. List all versions of the evaluator. + # ------------------------------------------------------------------ + print(f"List all versions for evaluator `{evaluator_name}`:") + for ver in project_client.beta.evaluators.list_versions(name=evaluator_name): + print(f" - version=`{ver.version}` dimensions={len(ver.definition.dimensions)}") + + # ------------------------------------------------------------------ + # 5. Clean up. + # ------------------------------------------------------------------ + # Delete the highest version first to avoid any version-ordering issues. + for version in (v2.version, v1.version): + if version: + print(f"Delete evaluator `{evaluator_name}` version `{version}`.") + project_client.beta.evaluators.delete_version(name=evaluator_name, version=version) diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_lifecycle.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_lifecycle.py new file mode 100644 index 000000000000..da92503d9928 --- /dev/null +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_lifecycle.py @@ -0,0 +1,173 @@ +# pylint: disable=line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ + +""" +DESCRIPTION: + End-to-end scenario showing the full lifecycle of rubric evaluator + generation jobs. The sample exercises: + + * `create_generation_job` with `operation_id` for idempotent re-submits. + * `get_generation_job` to poll a single job to completion. + * `list_generation_jobs` to enumerate recent jobs in the project. + * `delete_generation_job` to remove a finished job record. + * `delete_version` to remove the persisted evaluator that the job produced. + + `cancel_generation_job` is shown in a comment - cancelling requires catching + a job mid-flight (jobs usually finish in under two minutes), so it is not + exercised inline. + + Note: `delete_version` cascades to delete the generation job record as well, + so `delete_generation_job` may return 404 - that is expected and tolerated + below. + +USAGE: + python sample_rubric_evaluator_generation_lifecycle.py + + Before running the sample: + + pip install "azure-ai-projects>=2.2.0" azure-identity python-dotenv + + Set these environment variables with your own values: + 1) FOUNDRY_PROJECT_ENDPOINT - Required. The Azure AI Project endpoint, as found + in the overview page of your Microsoft Foundry project. + 2) FOUNDRY_MODEL_NAME - Required. The name of the model the generation job + will use (e.g. `gpt-4o`, `gpt-4.1`). The generation runs inline server + side, so no deployment in your project is required. + 3) POLL_INTERVAL_SECONDS - Optional. Number of seconds to sleep between status + polls for the generation job. Defaults to 10. +""" + +import os +import time +import uuid +from datetime import datetime, timezone +from typing import cast + +from dotenv import load_dotenv + +from azure.core.exceptions import ResourceNotFoundError +from azure.identity import DefaultAzureCredential +from azure.ai.projects import AIProjectClient +from azure.ai.projects.models import JobStatus, PageOrder + +load_dotenv() + +endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] +model_name = os.environ["FOUNDRY_MODEL_NAME"] +poll_interval_seconds = int(os.environ.get("POLL_INTERVAL_SECONDS", "10")) + +# Unique per-run name so repeated runs do not collide. +ts = datetime.now(tz=timezone.utc).strftime("%Y%m%d%H%M%S") +short = uuid.uuid4().hex[:6] +evaluator_name = f"lifecycle-demo-{ts}-{short}" +operation_id = f"rubric-lifecycle-{short}" + +TERMINAL_STATUSES = {JobStatus.SUCCEEDED, JobStatus.FAILED, JobStatus.CANCELLED} + +# Shared job body used both for the initial create and the idempotency replay. +job_body = { + "model": model_name, + "name": "Lifecycle demo", + "evaluator_name": evaluator_name, + "evaluator_display_name": "Lifecycle demo", + "evaluator_description": "Minimal job used to demonstrate the LRO + list/delete lifecycle.", + "sources": [ + { + "type": "Prompt", + "description": "Inline application overview.", + "prompt": ( + "You are evaluating a simple Q&A assistant that answers factual " + "questions clearly and concisely." + ), + } + ], +} + +with ( + DefaultAzureCredential() as credential, + # `allow_preview` and `api_version` are required for the evaluator + # generation endpoints in this preview. + AIProjectClient( + endpoint=endpoint, + credential=credential, + allow_preview=True, + api_version="2025-11-15-preview", + ) as project_client, +): + + # ------------------------------------------------------------------ + # 1. Create the generation job. + # ------------------------------------------------------------------ + # `operation_id` makes this call idempotent - re-running with the same id + # returns the existing job instead of creating a duplicate. Useful for + # retry-safe automation. + print(f"Create generation job with operation_id `{operation_id}`.") + job = project_client.beta.evaluators.create_generation_job(job=job_body, operation_id=operation_id) + print(f"Created generation job `{job.id}` (status: `{cast(JobStatus, job.status).value}`).") + + # Re-issuing the same operation_id returns the SAME job rather than + # starting a new one. + replay = project_client.beta.evaluators.create_generation_job(job=job_body, operation_id=operation_id) + assert replay.id == job.id, "operation_id should make create_generation_job idempotent" + print(f"Idempotent replay returned the same id `{replay.id}`.") + + # ------------------------------------------------------------------ + # 2. Poll the job to completion. + # ------------------------------------------------------------------ + print(f"Poll job `{job.id}` until it reaches a terminal state.", end="", flush=True) + while job.status not in TERMINAL_STATUSES: + time.sleep(poll_interval_seconds) + job = project_client.beta.evaluators.get_generation_job(job.id) + print(".", end="", flush=True) + print() + print(f"Final job status: `{cast(JobStatus, job.status).value}`.") + + if job.status != JobStatus.SUCCEEDED: + message = job.error.message if job.error is not None else "" + raise RuntimeError( + f"Generation job `{job.id}` ended with status `{cast(JobStatus, job.status).value}`: {message}" + ) + + evaluator = job.result + print(f"Generated evaluator: name=`{evaluator.name}` version=`{evaluator.version}`.") + + # ------------------------------------------------------------------ + # 3. List recent generation jobs in this project. + # ------------------------------------------------------------------ + # `PageOrder.DESC` returns the most recently created jobs first. + print("List the 5 most recent generation jobs in this project:") + recent = list(project_client.beta.evaluators.list_generation_jobs(limit=5, order=PageOrder.DESC)) + if not recent: + print(" (no jobs returned)") + for entry in recent: + print( + f" - id=`{entry.id}` status=`{cast(JobStatus, entry.status).value}` " + f"evaluator_name=`{entry.inputs.evaluator_name}`" + ) + + # ------------------------------------------------------------------ + # 4. Cancel (commented for reference). + # ------------------------------------------------------------------ + # To cancel a job, call `cancel_generation_job` while it is still running. + # The job above already completed, so the call is shown here only for + # reference. + # + # cancelled = project_client.beta.evaluators.cancel_generation_job(some_running_job_id) + # print(f"Cancelled: id=`{cancelled.id}` status=`{cast(JobStatus, cancelled.status).value}`.") + + # ------------------------------------------------------------------ + # 5. Clean up. + # ------------------------------------------------------------------ + print(f"Delete evaluator `{evaluator.name}` version `{evaluator.version}`.") + project_client.beta.evaluators.delete_version(name=evaluator.name, version=evaluator.version) + + # `delete_version` above cascades to remove the generation job record as + # well; tolerate a 404 here. + print(f"Delete generation job `{job.id}`.") + try: + project_client.beta.evaluators.delete_generation_job(job.id) + except ResourceNotFoundError: + print(f" Job `{job.id}` was already removed by the delete_version cascade.") diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_manual.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_manual.py new file mode 100644 index 000000000000..a6844a39b9fe --- /dev/null +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_manual.py @@ -0,0 +1,266 @@ +# pylint: disable=line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ + +""" +DESCRIPTION: + End-to-end scenario showing how to manually author a rubric-based + evaluator and use it as a testing criterion of an OpenAI evaluation run. + The sample: + + 1. Creates a rubric evaluator with `project_client.beta.evaluators.create_version`, + supplying scoring dimensions (each with an id, description, and integer + weight from 1-10) and an optional pass threshold. + 2. Creates an OpenAI evaluation (`client.evals.create`) referencing the + custom evaluator as a testing criterion. + 3. Runs the evaluation against inline JSONL sample data. + 4. Polls the evaluation run to completion and prints per-item results. + 5. Cleans up the evaluation and the evaluator version. + + A rubric evaluator is a collection of independent scoring dimensions. At + evaluation time, an LLM judge scores each applicable dimension on a 1-5 + scale and the runtime emits a normalized aggregate score. Dimensions can + opt in to `always_applicable` to skip the applicability assessment. + + See `sample_rubric_evaluator_generation_basic.py` for the generation-based + workflow that produces the same rubric structure automatically from a + description of the application. + +USAGE: + python sample_rubric_evaluator_manual.py + + Before running the sample: + + pip install "azure-ai-projects>=2.2.0" azure-identity python-dotenv + + Set these environment variables with your own values: + 1) FOUNDRY_PROJECT_ENDPOINT - Required. The Azure AI Project endpoint, as found + in the overview page of your Microsoft Foundry project. + 2) FOUNDRY_MODEL_NAME - Required. The name of the LLM model deployment that + the rubric evaluator's judge will use at evaluation time (e.g. `gpt-4o`, `gpt-4.1`). + 3) POLL_INTERVAL_SECONDS - Optional. Number of seconds to sleep between status + polls for the evaluation run. Defaults to 10. +""" + +import os +import time +import uuid +from datetime import datetime, timezone + +from dotenv import load_dotenv +from openai.types.eval_create_params import DataSourceConfigCustom +from openai.types.evals.create_eval_jsonl_run_data_source_param import ( + CreateEvalJSONLRunDataSourceParam, + SourceFileContent, + SourceFileContentContent, +) + +from azure.identity import DefaultAzureCredential +from azure.ai.projects import AIProjectClient +from azure.ai.projects.models import ( + EvaluatorCategory, + EvaluatorDefinitionType, + TestingCriterionAzureAIEvaluator, +) + +load_dotenv() + +endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] +model_name = os.environ["FOUNDRY_MODEL_NAME"] +poll_interval_seconds = int(os.environ.get("POLL_INTERVAL_SECONDS", "10")) + +# Unique per-run name so repeated runs do not collide. +ts = datetime.now(tz=timezone.utc).strftime("%Y%m%d%H%M%S") +short = uuid.uuid4().hex[:6] +evaluator_name = f"reservation-quality-manual-{ts}-{short}" + +TERMINAL_RUN_STATUSES = {"completed", "failed", "canceled"} + +with ( + DefaultAzureCredential() as credential, + AIProjectClient(endpoint=endpoint, credential=credential) as project_client, + project_client.get_openai_client() as openai_client, +): + + # ------------------------------------------------------------------ + # 1. Author the rubric evaluator. + # ------------------------------------------------------------------ + # Each dimension is scored independently on a 1-5 scale at evaluation + # time. `weight` (1-10) controls how strongly each dimension contributes + # to the normalized aggregate score. + print(f"Create rubric evaluator `{evaluator_name}`.") + evaluator = project_client.beta.evaluators.create_version( + name=evaluator_name, + evaluator_version={ + "name": evaluator_name, + "categories": [EvaluatorCategory.QUALITY], + "display_name": "Reservation Quality (Manual)", + "description": ( + "Hand-authored rubric evaluating a reservation assistant on intent " + "resolution, completeness, and tone." + ), + "definition": { + "type": EvaluatorDefinitionType.RUBRIC, + "dimensions": [ + { + "id": "correct_intent_resolution", + "description": ( + "Correctly identifies the user's reservation intent (new booking, " + "modification, or cancellation) and pursues the right workflow." + ), + "weight": 9, + }, + { + "id": "completeness", + "description": ( + "Captures or confirms every reservation detail the user needs " + "(party size, date, time, contact info) before completing the task." + ), + "weight": 6, + }, + { + "id": "professional_tone", + "description": "Maintains a polite, professional, restaurant-host tone throughout.", + "weight": 3, + }, + ], + # `pass_threshold` sets the normalized 0.0-1.0 pass/fail threshold + # (default 0.5). The "any dimension scored 1 -> fail" rule applies + # regardless of this threshold. + "pass_threshold": 0.6, + }, + }, + ) + print(f"Created evaluator `{evaluator.name}` version `{evaluator.version}`.") + print(f"Categories: {[c.value for c in evaluator.categories]}") + print(f"Dimensions ({len(evaluator.definition.dimensions)}):") + for dim in evaluator.definition.dimensions: + marker = " [ALWAYS-ON]" if dim.always_applicable else "" + print(f" - {dim.id} (weight={dim.weight}){marker}") + + # ------------------------------------------------------------------ + # 2. Create an OpenAI evaluation that uses the rubric as a criterion. + # ------------------------------------------------------------------ + # The eval object describes the shape of the dataset items and the + # criteria to score against. The run below supplies inline sample data. + data_source_config = DataSourceConfigCustom( + type="custom", + item_schema={ + "type": "object", + "properties": { + "query": {"type": "string"}, + "response": {"type": "string"}, + }, + "required": ["query", "response"], + }, + include_sample_schema=True, + ) + + testing_criteria = [ + TestingCriterionAzureAIEvaluator( + type="azure_ai_evaluator", + name=evaluator_name, + evaluator_name=evaluator_name, + # The LLM judge for the rubric uses the deployment supplied here. + initialization_parameters={"deployment_name": model_name}, + data_mapping={ + "query": "{{item.query}}", + "response": "{{item.response}}", + }, + ) + ] + + print("Create the evaluation.") + eval_object = openai_client.evals.create( + name=f"{evaluator_name}-eval", + data_source_config=data_source_config, + testing_criteria=testing_criteria, + ) + print(f"Evaluation created (id: {eval_object.id}).") + + # ------------------------------------------------------------------ + # 3. Run the evaluation against inline JSONL sample data. + # ------------------------------------------------------------------ + print(f"Create an evaluation run for eval `{eval_object.id}`.") + eval_run = openai_client.evals.runs.create( + eval_id=eval_object.id, + name=f"{evaluator_name}-run", + metadata={"sample": "evaluator_rubric_manual"}, + data_source=CreateEvalJSONLRunDataSourceParam( + type="jsonl", + source=SourceFileContent( + type="file_content", + content=[ + SourceFileContentContent( + item={ + "query": "Can I book a table for 4 tomorrow at 7 PM?", + "response": ( + "Absolutely - I have you down for a table for 4 tomorrow at 7:00 PM. " + "Could you share a contact number in case anything changes?" + ), + } + ), + SourceFileContentContent( + item={ + "query": "I need to cancel my reservation for Friday.", + "response": "ok", + } + ), + SourceFileContentContent( + item={ + "query": "Can you move my Saturday 8 PM reservation to 8:30?", + "response": ( + "Of course. I've updated your Saturday reservation from 8:00 PM to 8:30 PM. " + "Anything else I can help with?" + ), + } + ), + ], + ), + ), + ) + print(f"Evaluation run created (id: {eval_run.id}).") + + print(f"Poll run `{eval_run.id}` until it reaches a terminal state.", end="", flush=True) + while eval_run.status not in TERMINAL_RUN_STATUSES: + time.sleep(poll_interval_seconds) + eval_run = openai_client.evals.runs.retrieve(run_id=eval_run.id, eval_id=eval_object.id) + print(".", end="", flush=True) + print() + print(f"Final eval run status: `{eval_run.status}`.") + + if eval_run.status == "completed": + print(f"Result counts: {eval_run.result_counts}") + if eval_run.report_url: + print(f"Eval run report URL: {eval_run.report_url}") + output_items = list(openai_client.evals.runs.output_items.list(run_id=eval_run.id, eval_id=eval_object.id)) + print(f"Output items (total: {len(output_items)}):") + for idx, item in enumerate(output_items, start=1): + results = getattr(item, "results", None) or [] + parts = [] + for r in results: + # Result entries are returned either as typed objects (Azure AI + # evaluators) or as plain dicts (some OpenAI-native evaluators). + if isinstance(r, dict): + name = r.get("name", "?") + score = r.get("score", "n/a") + passed = r.get("passed", "n/a") + else: + name = getattr(r, "name", "?") + score = getattr(r, "score", "n/a") + passed = getattr(r, "passed", "n/a") + parts.append(f"{name}={score} ({passed})") + print(f" item {idx}: status={item.status} | {', '.join(parts)}") + else: + print("Evaluation run did not complete successfully.") + + # ------------------------------------------------------------------ + # 4. Clean up. + # ------------------------------------------------------------------ + print(f"Delete evaluation `{eval_object.id}`.") + openai_client.evals.delete(eval_id=eval_object.id) + + print(f"Delete evaluator `{evaluator_name}` version `{evaluator.version}`.") + project_client.beta.evaluators.delete_version(name=evaluator_name, version=evaluator.version)