From 4d90c9ca229873c93df9efa7e1f8cb9099fc9ec7 Mon Sep 17 00:00:00 2001 From: April Kwong Date: Tue, 26 May 2026 14:33:18 -0700 Subject: [PATCH 1/2] [azure-ai-projects] Fix mypy errors in rubric evaluator samples Address 65 mypy errors across 5 samples on feature/azure-ai-projects/2.2.0: - Add `assert ... is not None` after extracting `job.result` (`Optional[EvaluatorVersion]`). - Narrow `EvaluatorDefinition` to `RubricBasedEvaluatorDefinition` via `isinstance` before accessing `dimensions` / `pass_threshold` (which only exist on the rubric subclass). - Guard `EvaluatorCategory` enum `.value` access with `isinstance` so list comprehensions remain mypy-clean over `list[Union[str, EvaluatorCategory]]`. - Guard `entry.inputs.evaluator_name` for Optional inputs in the lifecycle listing loop. No runtime semantics changed; the asserts hold by construction after a successful job (the service guarantees the invariants, but they aren't expressible in the static type system). Files: - sample_rubric_evaluator_manual.py - sample_rubric_evaluator_generation_lifecycle.py - sample_rubric_evaluator_generation_basic.py - sample_rubric_evaluator_generation_iterate.py - sample_rubric_evaluator_generation_all_sources.py Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ...rubric_evaluator_generation_all_sources.py | 24 ++++++++++---- ...ample_rubric_evaluator_generation_basic.py | 18 ++++++++--- ...ple_rubric_evaluator_generation_iterate.py | 32 +++++++++++++------ ...e_rubric_evaluator_generation_lifecycle.py | 4 ++- .../sample_rubric_evaluator_manual.py | 9 ++++-- 5 files changed, 61 insertions(+), 26 deletions(-) diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_all_sources.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_all_sources.py index 5e29496f0eae..7a15dd15af2d 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_all_sources.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_all_sources.py @@ -58,7 +58,7 @@ from azure.identity import DefaultAzureCredential from azure.ai.projects import AIProjectClient -from azure.ai.projects.models import JobStatus +from azure.ai.projects.models import EvaluatorCategory, JobStatus, RubricBasedEvaluatorDefinition load_dotenv() @@ -165,11 +165,16 @@ ) else: evaluator = multi_job.result + assert evaluator is not None, "succeeded job must have a result" + definition = evaluator.definition + assert isinstance(definition, RubricBasedEvaluatorDefinition) multi_evaluator_version = evaluator.version or "" print(f"Generated evaluator: name=`{evaluator.name}` version=`{evaluator.version}`.") - print(f"Categories: {[c.value for c in evaluator.categories]}") - print(f"Dimensions ({len(evaluator.definition.dimensions)}):") - for dim in evaluator.definition.dimensions: + print( + f"Categories: {[c.value if isinstance(c, EvaluatorCategory) else c for c in evaluator.categories]}" + ) + print(f"Dimensions ({len(definition.dimensions)}):") + for dim in definition.dimensions: marker = " [ALWAYS-ON]" if dim.always_applicable else "" print(f" - {dim.id} (weight={dim.weight}){marker}") @@ -230,11 +235,16 @@ ) else: evaluator = traces_job.result + assert evaluator is not None, "succeeded job must have a result" + definition = evaluator.definition + assert isinstance(definition, RubricBasedEvaluatorDefinition) traces_evaluator_version = evaluator.version or "" print(f"Generated evaluator: name=`{evaluator.name}` version=`{evaluator.version}`.") - print(f"Categories: {[c.value for c in evaluator.categories]}") - print(f"Dimensions ({len(evaluator.definition.dimensions)}):") - for dim in evaluator.definition.dimensions: + print( + f"Categories: {[c.value if isinstance(c, EvaluatorCategory) else c for c in evaluator.categories]}" + ) + print(f"Dimensions ({len(definition.dimensions)}):") + for dim in definition.dimensions: marker = " [ALWAYS-ON]" if dim.always_applicable else "" print(f" - {dim.id} (weight={dim.weight}){marker}") diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_basic.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_basic.py index 220d001400c0..ee00eddcaa77 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_basic.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_basic.py @@ -59,7 +59,12 @@ from azure.identity import DefaultAzureCredential from azure.ai.projects import AIProjectClient -from azure.ai.projects.models import JobStatus, TestingCriterionAzureAIEvaluator +from azure.ai.projects.models import ( + EvaluatorCategory, + JobStatus, + RubricBasedEvaluatorDefinition, + TestingCriterionAzureAIEvaluator, +) load_dotenv() @@ -147,11 +152,14 @@ # On success, the evaluator is automatically saved as version 1. evaluator = job.result + assert evaluator is not None, "succeeded job must have a result" + definition = evaluator.definition + assert isinstance(definition, RubricBasedEvaluatorDefinition) print(f"Generated evaluator: name=`{evaluator.name}` version=`{evaluator.version}`.") - print(f"Categories: {[c.value for c in evaluator.categories]}") - print(f"Pass threshold: {evaluator.definition.pass_threshold}") - print(f"Dimensions ({len(evaluator.definition.dimensions)}):") - for dim in evaluator.definition.dimensions: + print(f"Categories: {[c.value if isinstance(c, EvaluatorCategory) else c for c in evaluator.categories]}") + print(f"Pass threshold: {definition.pass_threshold}") + print(f"Dimensions ({len(definition.dimensions)}):") + for dim in definition.dimensions: # Quality evaluators always include a non-editable `general_quality` # residual dimension with always_applicable=True. marker = " [ALWAYS-ON]" if dim.always_applicable else "" diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_iterate.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_iterate.py index 3c7c05afba2f..bcb7f07d0cae 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_iterate.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_iterate.py @@ -48,7 +48,12 @@ from azure.identity import DefaultAzureCredential from azure.ai.projects import AIProjectClient -from azure.ai.projects.models import EvaluatorDefinitionType, JobStatus +from azure.ai.projects.models import ( + EvaluatorCategory, + EvaluatorDefinitionType, + JobStatus, + RubricBasedEvaluatorDefinition, +) load_dotenv() @@ -118,9 +123,12 @@ ) v1 = job.result + assert v1 is not None, "succeeded job must have a result" + v1_definition = v1.definition + assert isinstance(v1_definition, RubricBasedEvaluatorDefinition) print(f"v1 created: version=`{v1.version}`.") - print(f"v1 dimensions ({len(v1.definition.dimensions)}):") - for dim in v1.definition.dimensions: + print(f"v1 dimensions ({len(v1_definition.dimensions)}):") + for dim in v1_definition.dimensions: marker = " [ALWAYS-ON]" if dim.always_applicable else "" print(f" - {dim.id} (weight={dim.weight}){marker}") @@ -134,8 +142,8 @@ # * Drop the lowest-weight editable dimension as redundant. # * Add a new custom dimension specific to this assistant. print("Apply human edits:") - editable = [d for d in v1.definition.dimensions if not d.always_applicable] - always_on = [d for d in v1.definition.dimensions if d.always_applicable] + editable = [d for d in v1_definition.dimensions if not d.always_applicable] + always_on = [d for d in v1_definition.dimensions if d.always_applicable] edited_dimensions = [] if editable: @@ -185,19 +193,21 @@ name=evaluator_name, evaluator_version={ "name": evaluator_name, - "categories": [c.value for c in v1.categories], + "categories": [c.value if isinstance(c, EvaluatorCategory) else c for c in v1.categories], "display_name": v1.display_name, "description": (v1.description or "") + " (edited)", "definition": { "type": EvaluatorDefinitionType.RUBRIC, "dimensions": edited_dimensions, - "pass_threshold": v1.definition.pass_threshold or 0.6, + "pass_threshold": v1_definition.pass_threshold or 0.6, }, }, ) + v2_definition = v2.definition + assert isinstance(v2_definition, RubricBasedEvaluatorDefinition) print(f"v2 created: version=`{v2.version}`.") - print(f"v2 dimensions ({len(v2.definition.dimensions)}):") - for dim in v2.definition.dimensions: + print(f"v2 dimensions ({len(v2_definition.dimensions)}):") + for dim in v2_definition.dimensions: marker = " [ALWAYS-ON]" if dim.always_applicable else "" print(f" - {dim.id} (weight={dim.weight}){marker}") @@ -206,7 +216,9 @@ # ------------------------------------------------------------------ print(f"List all versions for evaluator `{evaluator_name}`:") for ver in project_client.beta.evaluators.list_versions(name=evaluator_name): - print(f" - version=`{ver.version}` dimensions={len(ver.definition.dimensions)}") + ver_definition = ver.definition + assert isinstance(ver_definition, RubricBasedEvaluatorDefinition) + print(f" - version=`{ver.version}` dimensions={len(ver_definition.dimensions)}") # ------------------------------------------------------------------ # 5. Clean up. diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_lifecycle.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_lifecycle.py index 392eb3735aae..20b3b096cd29 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_lifecycle.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_lifecycle.py @@ -131,6 +131,7 @@ ) evaluator = job.result + assert evaluator is not None, "succeeded job must have a result" print(f"Generated evaluator: name=`{evaluator.name}` version=`{evaluator.version}`.") # ------------------------------------------------------------------ @@ -142,9 +143,10 @@ if not recent: print(" (no jobs returned)") for entry in recent: + entry_evaluator_name = entry.inputs.evaluator_name if entry.inputs is not None else "" print( f" - id=`{entry.id}` status=`{cast(JobStatus, entry.status).value}` " - f"evaluator_name=`{entry.inputs.evaluator_name}`" + f"evaluator_name=`{entry_evaluator_name}`" ) # ------------------------------------------------------------------ diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_manual.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_manual.py index a6844a39b9fe..24d6b21b9b9d 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_manual.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_manual.py @@ -62,6 +62,7 @@ from azure.ai.projects.models import ( EvaluatorCategory, EvaluatorDefinitionType, + RubricBasedEvaluatorDefinition, TestingCriterionAzureAIEvaluator, ) @@ -134,9 +135,11 @@ }, ) print(f"Created evaluator `{evaluator.name}` version `{evaluator.version}`.") - print(f"Categories: {[c.value for c in evaluator.categories]}") - print(f"Dimensions ({len(evaluator.definition.dimensions)}):") - for dim in evaluator.definition.dimensions: + definition = evaluator.definition + assert isinstance(definition, RubricBasedEvaluatorDefinition) + print(f"Categories: {[c.value if isinstance(c, EvaluatorCategory) else c for c in evaluator.categories]}") + print(f"Dimensions ({len(definition.dimensions)}):") + for dim in definition.dimensions: marker = " [ALWAYS-ON]" if dim.always_applicable else "" print(f" - {dim.id} (weight={dim.weight}){marker}") From d57109b884a48b93da62b765e2aea2e4c5f59235 Mon Sep 17 00:00:00 2001 From: April Kwong Date: Tue, 26 May 2026 15:38:41 -0700 Subject: [PATCH 2/2] [azure-ai-projects] Trim noise in rubric evaluator samples Reduce per-step prints, ASCII section separators, polling progress dots, per-dimension enumeration loops, and verbose `Created X`/`Doing Y` announcements across the 5 rubric evaluator samples. Keep all service-constraint comments (`allow_preview`, `operation_id` idempotency, `traces` companion, `delete_version` cascade) and all type-narrowing `assert isinstance` calls. Total: 1217 -> 992 lines (-18.5%). mypy still clean. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ...rubric_evaluator_generation_all_sources.py | 82 +++------ ...ample_rubric_evaluator_generation_basic.py | 158 +++++------------- ...ple_rubric_evaluator_generation_iterate.py | 88 ++++------ ...e_rubric_evaluator_generation_lifecycle.py | 98 ++++------- .../sample_rubric_evaluator_manual.py | 146 ++++++---------- 5 files changed, 176 insertions(+), 396 deletions(-) diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_all_sources.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_all_sources.py index 7a15dd15af2d..4170bbf35078 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_all_sources.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_all_sources.py @@ -34,8 +34,7 @@ 1) FOUNDRY_PROJECT_ENDPOINT - Required. The Azure AI Project endpoint, as found in the overview page of your Microsoft Foundry project. 2) FOUNDRY_MODEL_NAME - Required. The name of the model the generation job - will use (e.g. `gpt-4o`, `gpt-4.1`). The generation runs inline server - side, so no deployment in your project is required. + will use (e.g. `gpt-4o`, `gpt-4.1`). 3) FOUNDRY_AGENT_NAME - Optional. Name of an agent registered in the project. Enables the `Agent` source and the `traces`-source job. 4) FOUNDRY_REFERENCE_DATASET_NAME - Optional. Name of an uploaded dataset. @@ -44,8 +43,8 @@ Enables the `Dataset` source. 6) FOUNDRY_TRACES_WINDOW_DAYS - Optional. Look-back window in days for the `traces` source. Defaults to 7. - 7) POLL_INTERVAL_SECONDS - Optional. Number of seconds to sleep between - generation job status polls. Defaults to 10. + 7) POLL_INTERVAL_SECONDS - Optional. Seconds to sleep between status polls. + Defaults to 10. """ import os @@ -58,7 +57,7 @@ from azure.identity import DefaultAzureCredential from azure.ai.projects import AIProjectClient -from azure.ai.projects.models import EvaluatorCategory, JobStatus, RubricBasedEvaluatorDefinition +from azure.ai.projects.models import JobStatus, RubricBasedEvaluatorDefinition load_dotenv() @@ -92,11 +91,7 @@ api_version="2025-11-15-preview", ) as project_client, ): - - # ------------------------------------------------------------------ # 1. Combined Prompt + Agent + Dataset generation job. - # ------------------------------------------------------------------ - print("Build sources for the combined generation job:") multi_sources: List[Dict[str, Any]] = [ { "type": "Prompt", @@ -111,7 +106,6 @@ } ] if agent_name: - print(f" Including Agent source: `{agent_name}`.") multi_sources.append( { "type": "Agent", @@ -120,10 +114,9 @@ } ) else: - print(" Skipping Agent source (FOUNDRY_AGENT_NAME not set).") + print("Skipping Agent source (FOUNDRY_AGENT_NAME not set).") if dataset_name and dataset_version: - print(f" Including Dataset source: `{dataset_name}` v`{dataset_version}`.") multi_sources.append( { "type": "Dataset", @@ -133,9 +126,8 @@ } ) else: - print(" Skipping Dataset source (FOUNDRY_REFERENCE_DATASET_NAME / _VERSION not set).") + print("Skipping Dataset source (FOUNDRY_REFERENCE_DATASET_NAME / _VERSION not set).") - print(f"Create combined generation job for evaluator `{multi_name}`.") multi_job = project_client.beta.evaluators.create_generation_job( job={ "model": model_name, @@ -147,48 +139,33 @@ }, operation_id=f"rubric-multi-{short}", ) - print(f"Created generation job `{multi_job.id}` (status: `{cast(JobStatus, multi_job.status).value}`).") - print(f"Poll job `{multi_job.id}` until it reaches a terminal state.", end="", flush=True) + print(f"Waiting for multi-source job `{multi_job.id}` to complete...") while multi_job.status not in TERMINAL_STATUSES: time.sleep(poll_interval_seconds) multi_job = project_client.beta.evaluators.get_generation_job(multi_job.id) - print(".", end="", flush=True) - print() - print(f"Final job status: `{cast(JobStatus, multi_job.status).value}`.") if multi_job.status != JobStatus.SUCCEEDED: message = multi_job.error.message if multi_job.error is not None else "" - print( - f"Combined job `{multi_job.id}` ended with status " - f"`{cast(JobStatus, multi_job.status).value}`: {message}" - ) + print(f"Multi-source job ended with status `{cast(JobStatus, multi_job.status).value}`: {message}") else: + # `isinstance` narrows the discriminated `definition` to the rubric subtype. evaluator = multi_job.result - assert evaluator is not None, "succeeded job must have a result" + assert evaluator is not None definition = evaluator.definition assert isinstance(definition, RubricBasedEvaluatorDefinition) multi_evaluator_version = evaluator.version or "" - print(f"Generated evaluator: name=`{evaluator.name}` version=`{evaluator.version}`.") print( - f"Categories: {[c.value if isinstance(c, EvaluatorCategory) else c for c in evaluator.categories]}" + f"Multi-source evaluator `{evaluator.name}` v{evaluator.version}: " + f"{len(definition.dimensions)} dimensions." ) - print(f"Dimensions ({len(definition.dimensions)}):") - for dim in definition.dimensions: - marker = " [ALWAYS-ON]" if dim.always_applicable else "" - print(f" - {dim.id} (weight={dim.weight}){marker}") - # ------------------------------------------------------------------ # 2. Separate `traces` + Agent companion generation job. - # ------------------------------------------------------------------ - # The traces source requires a companion source because the service - # rejects sources arrays consisting only of traces. The Agent source - # is the typical companion. + # The traces source requires a companion source because the service rejects + # sources arrays consisting only of traces. The Agent source is the typical companion. if not agent_name: - print("Skip traces job: requires FOUNDRY_AGENT_NAME for both the traces source and companion.") + print("Skipping traces job (requires FOUNDRY_AGENT_NAME for both the traces source and companion).") else: - print(f"Create traces-source generation job for evaluator `{traces_name}`.") - print(f" agent=`{agent_name}` look-back window: {traces_window_days} days") now = int(time.time()) start_time = now - traces_window_days * 24 * 3600 end_time = now + 600 # small padding for clock skew @@ -217,44 +194,29 @@ }, operation_id=f"rubric-traces-{short}", ) - print(f"Created generation job `{traces_job.id}` (status: `{cast(JobStatus, traces_job.status).value}`).") - print(f"Poll job `{traces_job.id}` until it reaches a terminal state.", end="", flush=True) + print(f"Waiting for traces job `{traces_job.id}` to complete...") while traces_job.status not in TERMINAL_STATUSES: time.sleep(poll_interval_seconds) traces_job = project_client.beta.evaluators.get_generation_job(traces_job.id) - print(".", end="", flush=True) - print() - print(f"Final job status: `{cast(JobStatus, traces_job.status).value}`.") if traces_job.status != JobStatus.SUCCEEDED: message = traces_job.error.message if traces_job.error is not None else "" - print( - f"Traces job `{traces_job.id}` ended with status " - f"`{cast(JobStatus, traces_job.status).value}`: {message}" - ) + print(f"Traces job ended with status `{cast(JobStatus, traces_job.status).value}`: {message}") else: evaluator = traces_job.result - assert evaluator is not None, "succeeded job must have a result" + assert evaluator is not None definition = evaluator.definition assert isinstance(definition, RubricBasedEvaluatorDefinition) traces_evaluator_version = evaluator.version or "" - print(f"Generated evaluator: name=`{evaluator.name}` version=`{evaluator.version}`.") print( - f"Categories: {[c.value if isinstance(c, EvaluatorCategory) else c for c in evaluator.categories]}" + f"Traces evaluator `{evaluator.name}` v{evaluator.version}: " + f"{len(definition.dimensions)} dimensions." ) - print(f"Dimensions ({len(definition.dimensions)}):") - for dim in definition.dimensions: - marker = " [ALWAYS-ON]" if dim.always_applicable else "" - print(f" - {dim.id} (weight={dim.weight}){marker}") - # ------------------------------------------------------------------ - # 3. Clean up. - # ------------------------------------------------------------------ - # `delete_version` cascades to delete the generation job record as well. + # 3. Clean up. `delete_version` cascades to delete the generation job record. + print("Cleaning up.") if multi_evaluator_version: - print(f"Delete evaluator `{multi_name}` version `{multi_evaluator_version}`.") project_client.beta.evaluators.delete_version(name=multi_name, version=multi_evaluator_version) if traces_evaluator_version: - print(f"Delete evaluator `{traces_name}` version `{traces_evaluator_version}`.") project_client.beta.evaluators.delete_version(name=traces_name, version=traces_evaluator_version) diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_basic.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_basic.py index ee00eddcaa77..d9514e0b6095 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_basic.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_basic.py @@ -15,8 +15,8 @@ and tools. The service synthesizes a rubric tailored to that application. 2. Polls the generation job to completion and resolves the generated `EvaluatorVersion`. - 3. Creates an OpenAI evaluation (`client.evals.create`) referencing the - generated evaluator as a testing criterion. + 3. Creates an OpenAI evaluation referencing the generated evaluator as a + testing criterion. 4. Runs the evaluation against inline JSONL sample data. 5. Cleans up the evaluation and the evaluator version. Deleting the evaluator version cascades to delete the generation job record. @@ -39,8 +39,8 @@ generation job and the eval run's LLM judge (e.g. `gpt-4o`, `gpt-4.1`). The generation runs inline server side (no deployment required), but the eval run's grader does require a model deployment in your project. - 3) POLL_INTERVAL_SECONDS - Optional. Number of seconds to sleep between status - polls for both the generation job and the evaluation run. Defaults to 10. + 3) POLL_INTERVAL_SECONDS - Optional. Seconds to sleep between status polls + for both the generation job and the evaluation run. Defaults to 10. """ import os @@ -60,7 +60,6 @@ from azure.identity import DefaultAzureCredential from azure.ai.projects import AIProjectClient from azure.ai.projects.models import ( - EvaluatorCategory, JobStatus, RubricBasedEvaluatorDefinition, TestingCriterionAzureAIEvaluator, @@ -92,22 +91,14 @@ ) as project_client, project_client.get_openai_client() as openai_client, ): - - # ------------------------------------------------------------------ # 1. Generate an evaluator from a single `Prompt` source. - # ------------------------------------------------------------------ - # The body is sent as a plain dict to match the wire shape expected by - # the 2025-11-15-preview API. - print(f"Create generation job for evaluator `{evaluator_name}`.") job = project_client.beta.evaluators.create_generation_job( job={ "model": model_name, "name": "Reservation Quality (Generated)", "evaluator_name": evaluator_name, "evaluator_display_name": "Reservation Quality (Generated)", - "evaluator_description": ( - "Quality evaluator generated from a prompt describing a " "restaurant reservation assistant." - ), + "evaluator_description": "Quality evaluator generated from a prompt describing a restaurant reservation assistant.", "sources": [ { "type": "Prompt", @@ -127,85 +118,63 @@ } ], }, - # `operation_id` makes the call idempotent - re-submitting the same id - # returns the existing job instead of creating a duplicate. + # `operation_id` makes the call idempotent - re-submitting the same id returns the existing job. operation_id=f"rubric-eval-basic-{short}", ) - print(f"Created generation job `{job.id}` (status: `{cast(JobStatus, job.status).value}`).") + print(f"Created generation job `{job.id}`.") - print(f"Poll job `{job.id}` until it reaches a terminal state.", end="", flush=True) + print(f"Waiting for job `{job.id}` to complete...") while job.status not in TERMINAL_STATUSES: time.sleep(poll_interval_seconds) job = project_client.beta.evaluators.get_generation_job(job.id) - print(".", end="", flush=True) - print() - print(f"Final job status: `{cast(JobStatus, job.status).value}`.") + print(f"Job finished with status `{cast(JobStatus, job.status).value}`.") if job.status != JobStatus.SUCCEEDED: message = job.error.message if job.error is not None else "" - raise RuntimeError( - f"Generation job `{job.id}` ended with status `{cast(JobStatus, job.status).value}`: {message}" - ) - - if job.usage is not None: - print(f"Token usage: {job.usage}") + raise RuntimeError(f"Generation job ended with status `{cast(JobStatus, job.status).value}`: {message}") # On success, the evaluator is automatically saved as version 1. + # `isinstance` narrows the discriminated `definition` to the rubric subtype. evaluator = job.result - assert evaluator is not None, "succeeded job must have a result" + assert evaluator is not None definition = evaluator.definition assert isinstance(definition, RubricBasedEvaluatorDefinition) - print(f"Generated evaluator: name=`{evaluator.name}` version=`{evaluator.version}`.") - print(f"Categories: {[c.value if isinstance(c, EvaluatorCategory) else c for c in evaluator.categories]}") - print(f"Pass threshold: {definition.pass_threshold}") - print(f"Dimensions ({len(definition.dimensions)}):") - for dim in definition.dimensions: - # Quality evaluators always include a non-editable `general_quality` - # residual dimension with always_applicable=True. - marker = " [ALWAYS-ON]" if dim.always_applicable else "" - print(f" - {dim.id} (weight={dim.weight}){marker}: {dim.description[:120]}") - - # ------------------------------------------------------------------ - # 2. Create an OpenAI evaluation that uses the generated evaluator. - # ------------------------------------------------------------------ - data_source_config = DataSourceConfigCustom( - type="custom", - item_schema={ - "type": "object", - "properties": { - "query": {"type": "string"}, - "response": {"type": "string"}, - }, - "required": ["query", "response"], - }, - include_sample_schema=True, + print( + f"Generated evaluator `{evaluator.name}` v{evaluator.version}: " + f"{len(definition.dimensions)} dimensions, pass_threshold={definition.pass_threshold}." ) + print(f" Dimensions: {', '.join(d.id for d in definition.dimensions)}") - testing_criteria = [ - TestingCriterionAzureAIEvaluator( - type="azure_ai_evaluator", - name=evaluator.name, - evaluator_name=evaluator.name, - initialization_parameters={"deployment_name": model_name}, - data_mapping={ - "query": "{{item.query}}", - "response": "{{item.response}}", - }, - ) - ] - - print("Create the evaluation.") + # 2. Create an OpenAI evaluation that uses the generated evaluator. eval_object = openai_client.evals.create( name=f"{evaluator.name}-eval", - data_source_config=data_source_config, - testing_criteria=testing_criteria, + data_source_config=DataSourceConfigCustom( + type="custom", + item_schema={ + "type": "object", + "properties": { + "query": {"type": "string"}, + "response": {"type": "string"}, + }, + "required": ["query", "response"], + }, + include_sample_schema=True, + ), + testing_criteria=[ + TestingCriterionAzureAIEvaluator( + type="azure_ai_evaluator", + name=evaluator.name, + evaluator_name=evaluator.name, + initialization_parameters={"deployment_name": model_name}, + data_mapping={ + "query": "{{item.query}}", + "response": "{{item.response}}", + }, + ) + ], ) - print(f"Evaluation created (id: {eval_object.id}).") - # ------------------------------------------------------------------ # 3. Run the evaluation against inline JSONL sample data. - # ------------------------------------------------------------------ - print(f"Create an evaluation run for eval `{eval_object.id}`.") eval_run = openai_client.evals.runs.create( eval_id=eval_object.id, name=f"{evaluator.name}-run", @@ -218,9 +187,7 @@ SourceFileContentContent( item={ "query": "Book a table for 4 tomorrow at 7 PM.", - "response": ( - "Booked - table for 4 tomorrow at 7:00 PM. A confirmation " "SMS is on its way." - ), + "response": "Booked - table for 4 tomorrow at 7:00 PM. A confirmation SMS is on its way.", } ), SourceFileContentContent( @@ -233,47 +200,14 @@ ), ), ) - print(f"Evaluation run created (id: {eval_run.id}).") - print(f"Poll run `{eval_run.id}` until it reaches a terminal state.", end="", flush=True) + print(f"Waiting for eval run `{eval_run.id}` to complete...") while eval_run.status not in TERMINAL_RUN_STATUSES: time.sleep(poll_interval_seconds) eval_run = openai_client.evals.runs.retrieve(run_id=eval_run.id, eval_id=eval_object.id) - print(".", end="", flush=True) - print() - print(f"Final eval run status: `{eval_run.status}`.") - - if eval_run.status == "completed": - print(f"Result counts: {eval_run.result_counts}") - if eval_run.report_url: - print(f"Eval run report URL: {eval_run.report_url}") - output_items = list(openai_client.evals.runs.output_items.list(run_id=eval_run.id, eval_id=eval_object.id)) - print(f"Output items (total: {len(output_items)}):") - for idx, item in enumerate(output_items, start=1): - results = getattr(item, "results", None) or [] - parts = [] - for r in results: - # Result entries are returned either as typed objects (Azure AI - # evaluators) or as plain dicts (some OpenAI-native evaluators). - if isinstance(r, dict): - name = r.get("name", "?") - score = r.get("score", "n/a") - passed = r.get("passed", "n/a") - else: - name = getattr(r, "name", "?") - score = getattr(r, "score", "n/a") - passed = getattr(r, "passed", "n/a") - parts.append(f"{name}={score} ({passed})") - print(f" item {idx}: status={item.status} | {', '.join(parts)}") - else: - print("Evaluation run did not complete successfully.") + print(f"Eval run finished with status `{eval_run.status}`. Result counts: {eval_run.result_counts}.") - # ------------------------------------------------------------------ - # 4. Clean up. - # ------------------------------------------------------------------ - print(f"Delete evaluation `{eval_object.id}`.") + # 4. Clean up. `delete_version` cascades to delete the generation job record. + print("Cleaning up.") openai_client.evals.delete(eval_id=eval_object.id) - - print(f"Delete evaluator `{evaluator.name}` version `{evaluator.version}`.") - # `delete_version` cascades to delete the generation job record as well. project_client.beta.evaluators.delete_version(name=evaluator.name, version=evaluator.version) diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_iterate.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_iterate.py index bcb7f07d0cae..e8f7df81a664 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_iterate.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_iterate.py @@ -12,14 +12,13 @@ weighting or add custom dimensions. The sample: 1. Generates v1 of an evaluator from a single `Prompt` source. - 2. Inspects the dimensions the service produced. - 3. Edits the dimensions locally - boosts the highest-weight editable + 2. Edits the dimensions locally - boosts the highest-weight editable dimension to 10, drops the lowest-weight editable dimension, and adds a new custom dimension. The non-editable `general_quality` ALWAYS-ON dimension is preserved verbatim. - 4. Saves the edited definition as v2 with `create_version`. - 5. Calls `list_versions` to enumerate v1 and v2. - 6. Cleans up by deleting both versions. + 3. Saves the edited definition as v2 with `create_version`. + 4. Calls `list_versions` to enumerate v1 and v2. + 5. Cleans up by deleting both versions. USAGE: python sample_rubric_evaluator_generation_iterate.py @@ -32,10 +31,9 @@ 1) FOUNDRY_PROJECT_ENDPOINT - Required. The Azure AI Project endpoint, as found in the overview page of your Microsoft Foundry project. 2) FOUNDRY_MODEL_NAME - Required. The name of the model the generation job - will use (e.g. `gpt-4o`, `gpt-4.1`). The generation runs inline server - side, so no deployment in your project is required. - 3) POLL_INTERVAL_SECONDS - Optional. Number of seconds to sleep between status - polls for the generation job. Defaults to 10. + will use (e.g. `gpt-4o`, `gpt-4.1`). + 3) POLL_INTERVAL_SECONDS - Optional. Seconds to sleep between status polls. + Defaults to 10. """ import os @@ -79,11 +77,7 @@ api_version="2025-11-15-preview", ) as project_client, ): - - # ------------------------------------------------------------------ # 1. Generate v1 of the evaluator from a single `Prompt` source. - # ------------------------------------------------------------------ - print(f"Create generation job for evaluator `{evaluator_name}` (v1).") job = project_client.beta.evaluators.create_generation_job( job={ "model": model_name, @@ -106,42 +100,30 @@ }, operation_id=f"rubric-iterate-{short}", ) - print(f"Created generation job `{job.id}` (status: `{cast(JobStatus, job.status).value}`).") - print(f"Poll job `{job.id}` until it reaches a terminal state.", end="", flush=True) + print(f"Waiting for job `{job.id}` to complete...") while job.status not in TERMINAL_STATUSES: time.sleep(poll_interval_seconds) job = project_client.beta.evaluators.get_generation_job(job.id) - print(".", end="", flush=True) - print() - print(f"Final job status: `{cast(JobStatus, job.status).value}`.") if job.status != JobStatus.SUCCEEDED: message = job.error.message if job.error is not None else "" - raise RuntimeError( - f"Generation job `{job.id}` ended with status `{cast(JobStatus, job.status).value}`: {message}" - ) + raise RuntimeError(f"Generation job ended with status `{cast(JobStatus, job.status).value}`: {message}") + # `isinstance` narrows the discriminated `definition` to the rubric subtype. v1 = job.result - assert v1 is not None, "succeeded job must have a result" + assert v1 is not None v1_definition = v1.definition assert isinstance(v1_definition, RubricBasedEvaluatorDefinition) - print(f"v1 created: version=`{v1.version}`.") - print(f"v1 dimensions ({len(v1_definition.dimensions)}):") - for dim in v1_definition.dimensions: - marker = " [ALWAYS-ON]" if dim.always_applicable else "" - print(f" - {dim.id} (weight={dim.weight}){marker}") + print(f"v1 created with {len(v1_definition.dimensions)} dimensions: {', '.join(d.id for d in v1_definition.dimensions)}") - # ------------------------------------------------------------------ # 2. Edit dimensions locally. - # ------------------------------------------------------------------ # Domain-expert edits: # * Always preserve the ALWAYS-ON `general_quality` residual dimension # exactly as-is (id, weight, description, always_applicable). # * Boost the most important editable dimension to weight 10. # * Drop the lowest-weight editable dimension as redundant. # * Add a new custom dimension specific to this assistant. - print("Apply human edits:") editable = [d for d in v1_definition.dimensions if not d.always_applicable] always_on = [d for d in v1_definition.dimensions if d.always_applicable] @@ -149,8 +131,6 @@ if editable: top = max(editable, key=lambda d: d.weight) lowest = min(editable, key=lambda d: d.weight) - print(f" Boost `{top.id}` weight {top.weight} -> 10.") - print(f" Drop `{lowest.id}` (weight={lowest.weight}).") for dim in editable: if dim.id == lowest.id: continue @@ -162,20 +142,19 @@ } ) - new_dimension = { - "id": "wait_time_expectations_set", - "description": ( - "Sets clear expectations about wait time, table readiness, or confirmation " - "delivery so the user knows what happens next." - ), - "weight": 4, - } - edited_dimensions.append(new_dimension) - print(f" Add new dimension `{new_dimension['id']}` (weight={new_dimension['weight']}).") + edited_dimensions.append( + { + "id": "wait_time_expectations_set", + "description": ( + "Sets clear expectations about wait time, table readiness, or confirmation " + "delivery so the user knows what happens next." + ), + "weight": 4, + } + ) # Preserve every ALWAYS-ON dimension verbatim. These are non-editable. for dim in always_on: - print(f" Preserve ALWAYS-ON dimension `{dim.id}` (weight={dim.weight}) verbatim.") edited_dimensions.append( { "id": dim.id, @@ -185,14 +164,12 @@ } ) - # ------------------------------------------------------------------ # 3. Save the edited definition as v2. - # ------------------------------------------------------------------ - print(f"Save edited definition as v2 of `{evaluator_name}`.") v2 = project_client.beta.evaluators.create_version( name=evaluator_name, evaluator_version={ "name": evaluator_name, + # Narrow each category to its enum value (the categories list is Union[str, EvaluatorCategory]). "categories": [c.value if isinstance(c, EvaluatorCategory) else c for c in v1.categories], "display_name": v1.display_name, "description": (v1.description or "") + " (edited)", @@ -205,26 +182,17 @@ ) v2_definition = v2.definition assert isinstance(v2_definition, RubricBasedEvaluatorDefinition) - print(f"v2 created: version=`{v2.version}`.") - print(f"v2 dimensions ({len(v2_definition.dimensions)}):") - for dim in v2_definition.dimensions: - marker = " [ALWAYS-ON]" if dim.always_applicable else "" - print(f" - {dim.id} (weight={dim.weight}){marker}") + print(f"v2 created with {len(v2_definition.dimensions)} dimensions: {', '.join(d.id for d in v2_definition.dimensions)}") - # ------------------------------------------------------------------ # 4. List all versions of the evaluator. - # ------------------------------------------------------------------ - print(f"List all versions for evaluator `{evaluator_name}`:") + print(f"All versions of `{evaluator_name}`:") for ver in project_client.beta.evaluators.list_versions(name=evaluator_name): ver_definition = ver.definition assert isinstance(ver_definition, RubricBasedEvaluatorDefinition) - print(f" - version=`{ver.version}` dimensions={len(ver_definition.dimensions)}") + print(f" - v{ver.version}: {len(ver_definition.dimensions)} dimensions") - # ------------------------------------------------------------------ - # 5. Clean up. - # ------------------------------------------------------------------ - # Delete the highest version first to avoid any version-ordering issues. + # 5. Clean up. Delete the highest version first to avoid any ordering issues. + print("Cleaning up.") for version in (v2.version, v1.version): if version: - print(f"Delete evaluator `{evaluator_name}` version `{version}`.") project_client.beta.evaluators.delete_version(name=evaluator_name, version=version) diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_lifecycle.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_lifecycle.py index 20b3b096cd29..8f949ffe3c41 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_lifecycle.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_lifecycle.py @@ -6,8 +6,8 @@ """ DESCRIPTION: - End-to-end scenario showing the full lifecycle of rubric evaluator - generation jobs. The sample exercises: + End-to-end scenario showing the lifecycle of rubric evaluator generation + jobs. The sample exercises: * `create_generation_job` with `operation_id` for idempotent re-submits. * `get_generation_job` to poll a single job to completion. @@ -15,9 +15,8 @@ * `delete_generation_job` to remove a finished job record. * `delete_version` to remove the persisted evaluator that the job produced. - `cancel_generation_job` is shown in a comment - cancelling requires catching - a job mid-flight (jobs usually finish in under two minutes), so it is not - exercised inline. + `cancel_generation_job` is not exercised here - cancelling requires catching + a job mid-flight and jobs usually finish in under two minutes. Note: `delete_version` cascades to delete the generation job record as well, so `delete_generation_job` may return 404 - that is expected and tolerated @@ -34,10 +33,9 @@ 1) FOUNDRY_PROJECT_ENDPOINT - Required. The Azure AI Project endpoint, as found in the overview page of your Microsoft Foundry project. 2) FOUNDRY_MODEL_NAME - Required. The name of the model the generation job - will use (e.g. `gpt-4o`, `gpt-4.1`). The generation runs inline server - side, so no deployment in your project is required. - 3) POLL_INTERVAL_SECONDS - Optional. Number of seconds to sleep between status - polls for the generation job. Defaults to 10. + will use (e.g. `gpt-4o`, `gpt-4.1`). + 3) POLL_INTERVAL_SECONDS - Optional. Seconds to sleep between status polls. + Defaults to 10. """ import os @@ -78,9 +76,7 @@ { "type": "Prompt", "description": "Inline application overview.", - "prompt": ( - "You are evaluating a simple Q&A assistant that answers factual " "questions clearly and concisely." - ), + "prompt": "You are evaluating a simple Q&A assistant that answers factual questions clearly and concisely.", } ], } @@ -96,79 +92,43 @@ api_version="2025-11-15-preview", ) as project_client, ): - - # ------------------------------------------------------------------ - # 1. Create the generation job. - # ------------------------------------------------------------------ - # `operation_id` makes this call idempotent - re-running with the same id - # returns the existing job instead of creating a duplicate. Useful for - # retry-safe automation. - print(f"Create generation job with operation_id `{operation_id}`.") + # 1. Create the generation job. `operation_id` makes the call idempotent - + # re-submitting with the same id returns the existing job. job = project_client.beta.evaluators.create_generation_job(job=job_body, operation_id=operation_id) - print(f"Created generation job `{job.id}` (status: `{cast(JobStatus, job.status).value}`).") + print(f"Created generation job `{job.id}`.") - # Re-issuing the same operation_id returns the SAME job rather than - # starting a new one. replay = project_client.beta.evaluators.create_generation_job(job=job_body, operation_id=operation_id) - assert replay.id == job.id, "operation_id should make create_generation_job idempotent" - print(f"Idempotent replay returned the same id `{replay.id}`.") + assert replay.id == job.id # idempotent replay returns the same job - # ------------------------------------------------------------------ # 2. Poll the job to completion. - # ------------------------------------------------------------------ - print(f"Poll job `{job.id}` until it reaches a terminal state.", end="", flush=True) + print(f"Waiting for job `{job.id}` to complete...") while job.status not in TERMINAL_STATUSES: time.sleep(poll_interval_seconds) job = project_client.beta.evaluators.get_generation_job(job.id) - print(".", end="", flush=True) - print() - print(f"Final job status: `{cast(JobStatus, job.status).value}`.") + print(f"Job finished with status `{cast(JobStatus, job.status).value}`.") if job.status != JobStatus.SUCCEEDED: message = job.error.message if job.error is not None else "" - raise RuntimeError( - f"Generation job `{job.id}` ended with status `{cast(JobStatus, job.status).value}`: {message}" - ) + raise RuntimeError(f"Generation job ended with status `{cast(JobStatus, job.status).value}`: {message}") evaluator = job.result - assert evaluator is not None, "succeeded job must have a result" - print(f"Generated evaluator: name=`{evaluator.name}` version=`{evaluator.version}`.") - - # ------------------------------------------------------------------ - # 3. List recent generation jobs in this project. - # ------------------------------------------------------------------ - # `PageOrder.DESC` returns the most recently created jobs first. - print("List the 5 most recent generation jobs in this project:") - recent = list(project_client.beta.evaluators.list_generation_jobs(limit=5, order=PageOrder.DESC)) - if not recent: - print(" (no jobs returned)") - for entry in recent: - entry_evaluator_name = entry.inputs.evaluator_name if entry.inputs is not None else "" - print( - f" - id=`{entry.id}` status=`{cast(JobStatus, entry.status).value}` " - f"evaluator_name=`{entry_evaluator_name}`" - ) - - # ------------------------------------------------------------------ - # 4. Cancel (commented for reference). - # ------------------------------------------------------------------ - # To cancel a job, call `cancel_generation_job` while it is still running. - # The job above already completed, so the call is shown here only for - # reference. - # + assert evaluator is not None + print(f"Generated evaluator `{evaluator.name}` version `{evaluator.version}`.") + + # 3. List the 5 most recent generation jobs in this project. + print("Recent generation jobs:") + for entry in project_client.beta.evaluators.list_generation_jobs(limit=5, order=PageOrder.DESC): + entry_name = entry.inputs.evaluator_name if entry.inputs is not None else "" + print(f" - id=`{entry.id}` status=`{cast(JobStatus, entry.status).value}` evaluator_name=`{entry_name}`") + + # 4. Cancel a running job (not exercised here; the job above already completed). # cancelled = project_client.beta.evaluators.cancel_generation_job(some_running_job_id) - # print(f"Cancelled: id=`{cancelled.id}` status=`{cast(JobStatus, cancelled.status).value}`.") - # ------------------------------------------------------------------ - # 5. Clean up. - # ------------------------------------------------------------------ - print(f"Delete evaluator `{evaluator.name}` version `{evaluator.version}`.") + # 5. Clean up. `delete_version` cascades to the generation job record, so + # the explicit delete below may return 404. + print("Cleaning up.") project_client.beta.evaluators.delete_version(name=evaluator.name, version=evaluator.version) - - # `delete_version` above cascades to remove the generation job record as - # well; tolerate a 404 here. - print(f"Delete generation job `{job.id}`.") try: project_client.beta.evaluators.delete_generation_job(job.id) except ResourceNotFoundError: - print(f" Job `{job.id}` was already removed by the delete_version cascade.") + pass # already removed by the delete_version cascade diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_manual.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_manual.py index 24d6b21b9b9d..9b8c6ae6be96 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_manual.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_manual.py @@ -13,11 +13,10 @@ 1. Creates a rubric evaluator with `project_client.beta.evaluators.create_version`, supplying scoring dimensions (each with an id, description, and integer weight from 1-10) and an optional pass threshold. - 2. Creates an OpenAI evaluation (`client.evals.create`) referencing the - custom evaluator as a testing criterion. - 3. Runs the evaluation against inline JSONL sample data. - 4. Polls the evaluation run to completion and prints per-item results. - 5. Cleans up the evaluation and the evaluator version. + 2. Creates an OpenAI evaluation referencing the rubric as a testing criterion. + 3. Runs the evaluation against inline JSONL sample data and prints per-item + scores. + 4. Cleans up the evaluation and the evaluator version. A rubric evaluator is a collection of independent scoring dimensions. At evaluation time, an LLM judge scores each applicable dimension on a 1-5 @@ -40,8 +39,8 @@ in the overview page of your Microsoft Foundry project. 2) FOUNDRY_MODEL_NAME - Required. The name of the LLM model deployment that the rubric evaluator's judge will use at evaluation time (e.g. `gpt-4o`, `gpt-4.1`). - 3) POLL_INTERVAL_SECONDS - Optional. Number of seconds to sleep between status - polls for the evaluation run. Defaults to 10. + 3) POLL_INTERVAL_SECONDS - Optional. Seconds to sleep between status polls + for the evaluation run. Defaults to 10. """ import os @@ -84,14 +83,10 @@ AIProjectClient(endpoint=endpoint, credential=credential) as project_client, project_client.get_openai_client() as openai_client, ): - - # ------------------------------------------------------------------ # 1. Author the rubric evaluator. - # ------------------------------------------------------------------ - # Each dimension is scored independently on a 1-5 scale at evaluation - # time. `weight` (1-10) controls how strongly each dimension contributes - # to the normalized aggregate score. - print(f"Create rubric evaluator `{evaluator_name}`.") + # Each dimension is scored independently on a 1-5 scale by an LLM judge at + # evaluation time. `weight` (1-10) controls how strongly each dimension + # contributes to the normalized aggregate score. evaluator = project_client.beta.evaluators.create_version( name=evaluator_name, evaluator_version={ @@ -127,66 +122,47 @@ "weight": 3, }, ], - # `pass_threshold` sets the normalized 0.0-1.0 pass/fail threshold - # (default 0.5). The "any dimension scored 1 -> fail" rule applies - # regardless of this threshold. + # `pass_threshold` sets the normalized 0.0-1.0 pass/fail threshold (default 0.5). "pass_threshold": 0.6, }, }, ) - print(f"Created evaluator `{evaluator.name}` version `{evaluator.version}`.") + # `isinstance` narrows the discriminated `definition` to the rubric subtype. definition = evaluator.definition assert isinstance(definition, RubricBasedEvaluatorDefinition) - print(f"Categories: {[c.value if isinstance(c, EvaluatorCategory) else c for c in evaluator.categories]}") - print(f"Dimensions ({len(definition.dimensions)}):") - for dim in definition.dimensions: - marker = " [ALWAYS-ON]" if dim.always_applicable else "" - print(f" - {dim.id} (weight={dim.weight}){marker}") - - # ------------------------------------------------------------------ - # 2. Create an OpenAI evaluation that uses the rubric as a criterion. - # ------------------------------------------------------------------ - # The eval object describes the shape of the dataset items and the - # criteria to score against. The run below supplies inline sample data. - data_source_config = DataSourceConfigCustom( - type="custom", - item_schema={ - "type": "object", - "properties": { - "query": {"type": "string"}, - "response": {"type": "string"}, - }, - "required": ["query", "response"], - }, - include_sample_schema=True, - ) + print(f"Created evaluator `{evaluator.name}` version `{evaluator.version}` with {len(definition.dimensions)} dimensions.") - testing_criteria = [ - TestingCriterionAzureAIEvaluator( - type="azure_ai_evaluator", - name=evaluator_name, - evaluator_name=evaluator_name, - # The LLM judge for the rubric uses the deployment supplied here. - initialization_parameters={"deployment_name": model_name}, - data_mapping={ - "query": "{{item.query}}", - "response": "{{item.response}}", - }, - ) - ] - - print("Create the evaluation.") + # 2. Create an OpenAI evaluation that uses the rubric as a testing criterion. eval_object = openai_client.evals.create( name=f"{evaluator_name}-eval", - data_source_config=data_source_config, - testing_criteria=testing_criteria, + data_source_config=DataSourceConfigCustom( + type="custom", + item_schema={ + "type": "object", + "properties": { + "query": {"type": "string"}, + "response": {"type": "string"}, + }, + "required": ["query", "response"], + }, + include_sample_schema=True, + ), + testing_criteria=[ + TestingCriterionAzureAIEvaluator( + type="azure_ai_evaluator", + name=evaluator_name, + evaluator_name=evaluator_name, + # The LLM judge for the rubric uses the deployment supplied here. + initialization_parameters={"deployment_name": model_name}, + data_mapping={ + "query": "{{item.query}}", + "response": "{{item.response}}", + }, + ) + ], ) - print(f"Evaluation created (id: {eval_object.id}).") - # ------------------------------------------------------------------ # 3. Run the evaluation against inline JSONL sample data. - # ------------------------------------------------------------------ - print(f"Create an evaluation run for eval `{eval_object.id}`.") eval_run = openai_client.evals.runs.create( eval_id=eval_object.id, name=f"{evaluator_name}-run", @@ -224,46 +200,26 @@ ), ), ) - print(f"Evaluation run created (id: {eval_run.id}).") - print(f"Poll run `{eval_run.id}` until it reaches a terminal state.", end="", flush=True) + print(f"Waiting for eval run `{eval_run.id}` to complete...") while eval_run.status not in TERMINAL_RUN_STATUSES: time.sleep(poll_interval_seconds) eval_run = openai_client.evals.runs.retrieve(run_id=eval_run.id, eval_id=eval_object.id) - print(".", end="", flush=True) - print() - print(f"Final eval run status: `{eval_run.status}`.") + print(f"Eval run finished with status `{eval_run.status}`. Result counts: {eval_run.result_counts}.") if eval_run.status == "completed": - print(f"Result counts: {eval_run.result_counts}") - if eval_run.report_url: - print(f"Eval run report URL: {eval_run.report_url}") - output_items = list(openai_client.evals.runs.output_items.list(run_id=eval_run.id, eval_id=eval_object.id)) - print(f"Output items (total: {len(output_items)}):") - for idx, item in enumerate(output_items, start=1): - results = getattr(item, "results", None) or [] - parts = [] - for r in results: - # Result entries are returned either as typed objects (Azure AI - # evaluators) or as plain dicts (some OpenAI-native evaluators). - if isinstance(r, dict): - name = r.get("name", "?") - score = r.get("score", "n/a") - passed = r.get("passed", "n/a") - else: - name = getattr(r, "name", "?") - score = getattr(r, "score", "n/a") - passed = getattr(r, "passed", "n/a") - parts.append(f"{name}={score} ({passed})") - print(f" item {idx}: status={item.status} | {', '.join(parts)}") - else: - print("Evaluation run did not complete successfully.") + for idx, item in enumerate( + openai_client.evals.runs.output_items.list(run_id=eval_run.id, eval_id=eval_object.id), start=1 + ): + # Result entries may be typed objects (Azure AI evaluators) or plain dicts (some OpenAI evaluators). + scores = [] + for r in getattr(item, "results", None) or []: + name = r.get("name", "?") if isinstance(r, dict) else getattr(r, "name", "?") + score = r.get("score", "n/a") if isinstance(r, dict) else getattr(r, "score", "n/a") + scores.append(f"{name}={score}") + print(f" item {idx} ({item.status}): {', '.join(scores)}") - # ------------------------------------------------------------------ # 4. Clean up. - # ------------------------------------------------------------------ - print(f"Delete evaluation `{eval_object.id}`.") + print("Cleaning up.") openai_client.evals.delete(eval_id=eval_object.id) - - print(f"Delete evaluator `{evaluator_name}` version `{evaluator.version}`.") project_client.beta.evaluators.delete_version(name=evaluator_name, version=evaluator.version)