From 77cb634fcbc47257d7ba2d3fb43edfa3b1c46481 Mon Sep 17 00:00:00 2001 From: Sean Gayler Date: Mon, 18 May 2026 13:03:49 -0700 Subject: [PATCH 01/11] Add sample for emitting human evaluation events Add `samples/evaluations/sample_human_evaluations.py`, an educational sample showing how Foundry customers can emit human evaluation telemetry as OpenTelemetry custom events (`gen_ai.evaluation.result`) that land in Application Insights `customEvents`, per the genai_human_evaluations spec. The sample defines a small `emit_human_evaluation_event` helper that hides the spec's bookkeeping (per-kind defaults for `binary` and `likert_5`, JSON-encoded `internal_properties`, conditional `response_id` / `enduser` / `tags` fields) so readers focus on what they're emitting. The project's ARM resource id is auto-derived from any connection's id (which is a full ARM path) to avoid an extra env var. Includes 25 pytest unit tests in `tests/samples/test_sample_human_evaluations.py` covering range + integer validation, pass/fail label derivation at the threshold boundary, JSON-encoding of `internal_properties`, and the conditional fields. No network, no Azure dependencies in the tests. Adds an index row in `samples/evaluations/README.md` under 'Additional Scenarios'. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../samples/evaluations/README.md | 1 + .../evaluations/sample_human_evaluations.py | 271 ++++++++++++++++ .../samples/test_sample_human_evaluations.py | 289 ++++++++++++++++++ 3 files changed, 561 insertions(+) create mode 100644 sdk/ai/azure-ai-projects/samples/evaluations/sample_human_evaluations.py create mode 100644 sdk/ai/azure-ai-projects/tests/samples/test_sample_human_evaluations.py diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/README.md b/sdk/ai/azure-ai-projects/samples/evaluations/README.md index d601e9e22906..3eed9e5ea6aa 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/README.md +++ b/sdk/ai/azure-ai-projects/samples/evaluations/README.md @@ -69,6 +69,7 @@ These samples require additional setup or Azure services: | [sample_evaluations_score_model_grader_with_audio.py](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_score_model_grader_with_audio.py) | Evaluate with audio data | Audio file, audio-capable model deployment | | [sample_evaluations_score_model_grader_with_audio_model_target.py](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_score_model_grader_with_audio_model_target.py) | Evaluate audio data using a model as the target | Audio file, audio-capable model deployment | | [sample_evaluations_builtin_with_inline_data_oai.py](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations_builtin_with_inline_data_oai.py) | Use OpenAI client directly | OpenAI SDK | +| [sample_human_evaluations.py](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/ai/azure-ai-projects/samples/evaluations/sample_human_evaluations.py) | Emit human evaluation events (binary / Likert-5) as OpenTelemetry custom events to Application Insights | Connected Application Insights on Foundry Project, `azure-monitor-opentelemetry` | ### Evaluator Types diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_human_evaluations.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_human_evaluations.py new file mode 100644 index 000000000000..27a9ecbe2159 --- /dev/null +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_human_evaluations.py @@ -0,0 +1,271 @@ +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ + +""" +DESCRIPTION: + This sample demonstrates how to emit human evaluation events from your + application as OpenTelemetry custom events that land in the `customEvents` + Application Insights table connected to your Microsoft Foundry project. + + Human evaluations capture signals that automated evaluators struggle with -- + tone, user satisfaction, factual nuance -- typically as a thumbs up/down + (binary) or a 5-point rating (likert_5) provided by an end user of your + application. + + The sample defines a small `emit_human_evaluation_event(...)` helper that + assembles and emits OTel-compliant events that carry additional metadata + for compatibility with Microsoft Azure services. The event is emitted via + Python `logging`, which `azure.monitor.opentelemetry.configure_azure_monitor` + routes through OpenTelemetry to Application Insights. + +USAGE: + python sample_human_evaluations.py + + Before running the sample: + + pip install "azure-ai-projects>=2.0.0" python-dotenv azure-monitor-opentelemetry + + Set these environment variables with your own values: + 1) FOUNDRY_PROJECT_ENDPOINT - Required. The Azure AI Project endpoint, as + found in the overview page of your Microsoft Foundry project. It has + the form: https://.services.ai.azure.com/api/projects/. +""" + +import json +import logging +import os +import uuid +from typing import Literal, Mapping, Optional + +from dotenv import load_dotenv +from azure.identity import DefaultAzureCredential +from azure.monitor.opentelemetry import configure_azure_monitor +from azure.ai.projects import AIProjectClient + +load_dotenv() + +# `configure_azure_monitor` installs an OpenTelemetry LoggingHandler on the +# root logger, so any standard Python `logging` call below this point flows +# through OTel to Application Insights as a log record. The +# `microsoft.custom_event.name` attribute is what routes the record to the +# `customEvents` table. +logger = logging.getLogger("human_evaluations") +logger.setLevel(logging.INFO) + + +_KIND_DEFAULTS = { + "binary": { + "min_value": 0.0, + "max_value": 1.0, + "threshold": 1.0, + "desirable_direction": "increase", + "type": "boolean", + }, + "likert_5": { + "min_value": 1.0, + "max_value": 5.0, + "threshold": 3.0, + "desirable_direction": "increase", + "type": "ordinal", + }, +} + + +# When identifying the end user, populate either or both of: +# enduser_id → the signed-in user's identity (e.g., AAD object id, +# email). This is PII and lands in App Insights as +# `user_AuthenticatedId`. +# enduser_pseudo_id → a random, non-identifying id you generated (e.g., a +# browser cookie or device id). Use when the user is +# anonymous or you don't want to log PII. Lands in +# App Insights as `user_Id`. +# Signed-in users often have both: `enduser_id` says who they are, and +# `enduser_pseudo_id` lets you correlate them with their earlier anonymous +# activity from the same browser/device. +def emit_human_evaluation_event( + *, + evaluation_name: str, + score_value: float, + kind: Literal["binary", "likert_5"], + explanation: Optional[str] = None, + response_id: Optional[str] = None, + project_resource_id: Optional[str] = None, + enduser_id: Optional[str] = None, + enduser_pseudo_id: Optional[str] = None, + tags: Optional[Mapping[str, str]] = None, + evaluation_id: Optional[str] = None, +) -> None: + """Emit a single `gen_ai.evaluation.result` human evaluation event. + + The helper takes care of all the bookkeeping (deriving min/max/ + threshold/type from `kind`, deriving the pass/fail label from the score, + JSON-encoding `internal_properties`, generating a correlation id, etc.). + + Args: + evaluation_name: The metric being evaluated. Free-form, but pick a + consistent snake_case name per metric. + Examples: "task_completion", "relevance", "helpfulness". + score_value: The numeric score the user gave. Must be a whole number + (no fractional part), expressed as a float. + For kind="binary": 0.0 (thumbs down) or 1.0 (thumbs up). + For kind="likert_5": 1.0, 2.0, 3.0, 4.0, or 5.0. + kind: The evaluation shape. "binary" for thumbs up/down; "likert_5" + for a 1-5 star rating. + explanation: Optional free-form text the user provided to justify + their score. + Example: "The agent answered the question accurately." + response_id: Optional id of the agent response being evaluated. + Typically the id of an OpenAI Responses API response. Used to + correlate the evaluation back to the response it judged. + Example: "resp_64904952b20872620069f8d600779c81908f58b0a3be090ef0". + project_resource_id: Optional Azure resource id of the Foundry + project the evaluation belongs to. Required by Microsoft systems + when you want the evaluation linked to a specific project. + Example: "/subscriptions//resourceGroups//providers/Microsoft.CognitiveServices/accounts//projects/". + enduser_id: Optional signed-in user identity (e.g., Entra ID object + id, email). This is PII and lands in App Insights as + `user_AuthenticatedId`. + Examples: "alice@contoso.com", "241964ad-a8db-4318-9f2e-5a7dc1f05349". + enduser_pseudo_id: Optional random, non-identifying id you generated + (e.g., a browser cookie or device id). Use when the user is + anonymous or you don't want to log PII. Lands in App Insights + as `user_Id`. + Example: "sess_QdH5CAWJgqVT4rOr0qtumf". + tags: Optional extra metadata to attach to the event. Each key is + emitted as `microsoft.human_evaluation.tags.` so you can + slice on it later in App Insights. + Example: {"subscription_tier": "basic_plan", "feature": "chat"}. + evaluation_id: Optional stable id for this specific evaluation event, + useful if you want to update or correlate it later. Defaults to a + fresh uuid4. + Example: "69d937a7-32e2-412e-97c9-119e2d282723". + """ + if kind not in _KIND_DEFAULTS: + raise ValueError(f"Unsupported kind '{kind}'. Use 'binary' or 'likert_5'.") + defaults = _KIND_DEFAULTS[kind] + + if not defaults["min_value"] <= score_value <= defaults["max_value"]: + raise ValueError( + f"score_value {score_value} is outside the allowed range " + f"[{defaults['min_value']}, {defaults['max_value']}] for kind '{kind}'." + ) + + # Ensure scores are whole numbers + if score_value != int(score_value): + raise ValueError( + f"score_value {score_value} must be a whole number (no fractional part) " + f"for kind '{kind}'." + ) + + # Per spec, the score is at or above the threshold = "pass", below = "fail". + # (Binary: 1.0 -> pass, 0.0 -> fail. Likert-5: >=3.0 -> pass, <3.0 -> fail.) + score_label = "pass" if score_value >= defaults["threshold"] else "fail" + + # `internal_properties` carries Microsoft-specific attributes. It MUST be a + # JSON-encoded string (not a nested object) to match how downstream systems + # like Azure Monitor and Foundry consume it. + internal_properties = { + "gen_ai.evaluation.threshold": str(defaults["threshold"]), + "gen_ai.evaluation.min_value": str(defaults["min_value"]), + "gen_ai.evaluation.max_value": str(defaults["max_value"]), + "gen_ai.evaluation.desirable_direction": defaults["desirable_direction"], + "gen_ai.evaluation.type": defaults["type"], + "microsoft.human_evaluation.source": "end_user", + "microsoft.human_evaluation.kind": kind, + "microsoft.human_evaluation.id": evaluation_id or str(uuid.uuid4()), + } + if project_resource_id: + internal_properties["gen_ai.azure_ai_project.id"] = project_resource_id + if response_id: + internal_properties["gen_ai.response.id.type"] = "responses" + if tags: + for tag_name, tag_value in tags.items(): + internal_properties[f"microsoft.human_evaluation.tags.{tag_name}"] = tag_value + + # Top-level event attributes follow the OTel `gen_ai.evaluation.result` + # event shape (except internal_properties). `microsoft.custom_event.name` + # is what routes the record to the `customEvents` App Insights table. + attributes = { + "microsoft.custom_event.name": "gen_ai.evaluation.result", + "gen_ai.evaluation.name": evaluation_name, + "gen_ai.evaluation.score.value": score_value, + "gen_ai.evaluation.score.label": score_label, + "internal_properties": json.dumps(internal_properties), + } + if explanation is not None: + attributes["gen_ai.evaluation.explanation"] = explanation + if response_id is not None: + attributes["gen_ai.response.id"] = response_id + if enduser_id is not None: + attributes["enduser.id"] = enduser_id + if enduser_pseudo_id is not None: + attributes["enduser.pseudo.id"] = enduser_pseudo_id + + logger.info("gen_ai.evaluation.result", extra=attributes) + + +if __name__ == "__main__": + endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] + + print("Point A") + + with ( + DefaultAzureCredential() as credential, + AIProjectClient(endpoint=endpoint, credential=credential) as project_client, + ): + # Pull the Application Insights connection string attached to your Foundry + # project and wire OpenTelemetry up to it. All `logger.info(...)` calls + # below will be exported to Application Insights. + connection_string = project_client.telemetry.get_application_insights_connection_string() + + print("Point B") + + configure_azure_monitor(connection_string=connection_string) + + print("Point C") + + # Optional: Derive the Foundry Project's resource id from any connection. The + # endpoint URL alone only gives us the account + project names, but every + # Connection's `id` is a full ARM path of the form: + # /subscriptions//resourceGroups//providers/Microsoft.CognitiveServices/accounts//projects//connections/ + # We just strip the trailing /connections/ to get the project id. + any_connection = next(iter(project_client.connections.list()), None) + project_resource_id = ( + any_connection.id.rsplit("/connections/", 1)[0] if any_connection else None + ) + + # The two examples below differ in evaluation kind (binary vs likert_5) + # AND in how the end user is identified, just to show both styles. In your + # own app, those two choices are independent. You can also pass both + # `enduser_id` and `enduser_pseudo_id` together -- typical for a + # signed-in user whose browser you've been tracking with a cookie. + + # Example 1: A signed-in end user gives a thumbs up on the agent's task + # completion. + emit_human_evaluation_event( + evaluation_name="task_completion", + score_value=1.0, + kind="binary", + explanation="The agent provided accurate weather information as requested.", + response_id="resp_64904952b20872620069f8d600779c81908f58b0a3be090ef0", + project_resource_id=project_resource_id, + enduser_id="241964ad-a8db-4318-9f2e-5a7dc1f05349", + tags={"subscription_tier": "basic_plan"}, + ) + print("Emitted binary human evaluation event.") + + # Example 2: An anonymous end user rates the agent's response 4 out of 5 + # stars for relevance. + emit_human_evaluation_event( + evaluation_name="relevance", + score_value=4.0, + kind="likert_5", + explanation="The agent's response is relevant to the query.", + response_id="resp_1234567890abcdef", + project_resource_id=project_resource_id, + enduser_pseudo_id="sess_QdH5CAWJgqVT4rOr0qtumf", + tags={"subscription_tier": "free_plan"}, + ) + print("Emitted likert_5 human evaluation event.") diff --git a/sdk/ai/azure-ai-projects/tests/samples/test_sample_human_evaluations.py b/sdk/ai/azure-ai-projects/tests/samples/test_sample_human_evaluations.py new file mode 100644 index 000000000000..b0450dab7e14 --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/samples/test_sample_human_evaluations.py @@ -0,0 +1,289 @@ +# pylint: disable=line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ +import json +import logging +import sys +import uuid +from pathlib import Path + +import pytest + +SAMPLES_EVALUATIONS_DIR = Path(__file__).resolve().parents[1] / ".." / "samples" / "evaluations" +sys.path.insert(0, str(SAMPLES_EVALUATIONS_DIR.resolve())) + +from sample_human_evaluations import emit_human_evaluation_event # noqa: E402 + + +class _RecordCapture(logging.Handler): + """Capture every ``LogRecord`` the helper emits so tests can introspect + the ``extra=`` kwargs that ended up as attributes on the record.""" + + def __init__(self) -> None: + super().__init__(level=logging.DEBUG) + self.records: list[logging.LogRecord] = [] + + def emit(self, record: logging.LogRecord) -> None: + self.records.append(record) + + +@pytest.fixture(autouse=True) +def capture(): + """Attach a fresh capture handler to the ``human_evaluations`` logger for + each test and detach on teardown.""" + logger = logging.getLogger("human_evaluations") + handler = _RecordCapture() + logger.addHandler(handler) + try: + yield handler + finally: + logger.removeHandler(handler) + + +def _only_record(capture: _RecordCapture) -> logging.LogRecord: + assert len(capture.records) == 1, f"expected exactly 1 emitted record, got {len(capture.records)}" + return capture.records[0] + + +# --------------------------------------------------------------------------- +# Validation: range / integer / unknown kind +# --------------------------------------------------------------------------- + + +def test_binary_score_0_emits_with_fail_label(capture): + emit_human_evaluation_event(evaluation_name="thumbs", score_value=0.0, kind="binary") + record = _only_record(capture) + assert record.__dict__["gen_ai.evaluation.score.label"] == "fail" + assert record.__dict__["gen_ai.evaluation.score.value"] == 0.0 + + +def test_binary_score_1_emits_with_pass_label(capture): + emit_human_evaluation_event(evaluation_name="thumbs", score_value=1.0, kind="binary") + record = _only_record(capture) + assert record.__dict__["gen_ai.evaluation.score.label"] == "pass" + assert record.__dict__["gen_ai.evaluation.score.value"] == 1.0 + + +def test_binary_score_fractional_raises(): + with pytest.raises(ValueError): + emit_human_evaluation_event(evaluation_name="thumbs", score_value=0.5, kind="binary") + + +def test_binary_score_out_of_range_raises(): + with pytest.raises(ValueError): + emit_human_evaluation_event(evaluation_name="thumbs", score_value=2.0, kind="binary") + + +def test_likert_5_score_1_emits_with_fail_label(capture): + emit_human_evaluation_event(evaluation_name="relevance", score_value=1.0, kind="likert_5") + assert _only_record(capture).__dict__["gen_ai.evaluation.score.label"] == "fail" + + +def test_likert_5_score_at_threshold_emits_with_pass_label(capture): + """Guards the `>=` vs `>` mistake: score == threshold must be pass.""" + emit_human_evaluation_event(evaluation_name="relevance", score_value=3.0, kind="likert_5") + assert _only_record(capture).__dict__["gen_ai.evaluation.score.label"] == "pass" + + +def test_likert_5_score_5_emits_with_pass_label(capture): + emit_human_evaluation_event(evaluation_name="relevance", score_value=5.0, kind="likert_5") + assert _only_record(capture).__dict__["gen_ai.evaluation.score.label"] == "pass" + + +def test_likert_5_non_integer_score_raises(): + with pytest.raises(ValueError): + emit_human_evaluation_event(evaluation_name="relevance", score_value=2.5, kind="likert_5") + + +def test_likert_5_score_out_of_range_raises(): + with pytest.raises(ValueError): + emit_human_evaluation_event(evaluation_name="relevance", score_value=6.0, kind="likert_5") + + +def test_unknown_kind_raises(): + with pytest.raises(ValueError): + emit_human_evaluation_event( + evaluation_name="relevance", + score_value=1.0, + kind="unknown", # type: ignore[arg-type] + ) + + +# --------------------------------------------------------------------------- +# Shape: top-level attributes + internal_properties JSON encoding +# --------------------------------------------------------------------------- + + +def test_top_level_attributes_have_canonical_keys_and_routing(capture): + emit_human_evaluation_event(evaluation_name="task_completion", score_value=1.0, kind="binary") + record = _only_record(capture) + attrs = record.__dict__ + assert attrs["microsoft.custom_event.name"] == "gen_ai.evaluation.result" + assert attrs["gen_ai.evaluation.name"] == "task_completion" + assert attrs["gen_ai.evaluation.score.value"] == 1.0 + assert attrs["gen_ai.evaluation.score.label"] == "pass" + # internal_properties must be present as a top-level attribute too. + assert "internal_properties" in attrs + + +def test_internal_properties_is_json_encoded_string_with_binary_defaults(capture): + """internal_properties MUST be a JSON-encoded string, not a nested dict, + per the genai_human_evaluations spec. + """ + emit_human_evaluation_event(evaluation_name="task_completion", score_value=1.0, kind="binary") + raw = _only_record(capture).__dict__["internal_properties"] + assert isinstance(raw, str) + decoded = json.loads(raw) + assert decoded["gen_ai.evaluation.threshold"] == "1.0" + assert decoded["gen_ai.evaluation.min_value"] == "0.0" + assert decoded["gen_ai.evaluation.max_value"] == "1.0" + assert decoded["gen_ai.evaluation.desirable_direction"] == "increase" + assert decoded["gen_ai.evaluation.type"] == "boolean" + assert decoded["microsoft.human_evaluation.source"] == "end_user" + assert decoded["microsoft.human_evaluation.kind"] == "binary" + assert "microsoft.human_evaluation.id" in decoded + + +def test_internal_properties_likert_5_defaults(capture): + emit_human_evaluation_event(evaluation_name="relevance", score_value=4.0, kind="likert_5") + decoded = json.loads(_only_record(capture).__dict__["internal_properties"]) + assert decoded["gen_ai.evaluation.threshold"] == "3.0" + assert decoded["gen_ai.evaluation.min_value"] == "1.0" + assert decoded["gen_ai.evaluation.max_value"] == "5.0" + assert decoded["gen_ai.evaluation.type"] == "ordinal" + assert decoded["microsoft.human_evaluation.kind"] == "likert_5" + + +# --------------------------------------------------------------------------- +# Conditional fields +# --------------------------------------------------------------------------- + + +def test_response_id_set_adds_top_level_id_and_internal_type(capture): + emit_human_evaluation_event( + evaluation_name="task_completion", + score_value=1.0, + kind="binary", + response_id="resp_abc123", + ) + record = _only_record(capture) + assert record.__dict__["gen_ai.response.id"] == "resp_abc123" + decoded = json.loads(record.__dict__["internal_properties"]) + assert decoded["gen_ai.response.id.type"] == "responses" + + +def test_response_id_omitted_omits_both_keys(capture): + emit_human_evaluation_event(evaluation_name="task_completion", score_value=1.0, kind="binary") + record = _only_record(capture) + assert "gen_ai.response.id" not in record.__dict__ + decoded = json.loads(record.__dict__["internal_properties"]) + assert "gen_ai.response.id.type" not in decoded + + +def test_project_resource_id_set_added_to_internal_properties(capture): + arm_id = ( + "/subscriptions/sub-id/resourceGroups/rg/providers/Microsoft.CognitiveServices" + "/accounts/acct/projects/proj" + ) + emit_human_evaluation_event( + evaluation_name="task_completion", + score_value=1.0, + kind="binary", + project_resource_id=arm_id, + ) + decoded = json.loads(_only_record(capture).__dict__["internal_properties"]) + assert decoded["gen_ai.azure_ai_project.id"] == arm_id + + +def test_project_resource_id_omitted_omits_key(capture): + emit_human_evaluation_event(evaluation_name="task_completion", score_value=1.0, kind="binary") + decoded = json.loads(_only_record(capture).__dict__["internal_properties"]) + assert "gen_ai.azure_ai_project.id" not in decoded + + +def test_enduser_id_only_sets_only_authenticated_attribute(capture): + emit_human_evaluation_event( + evaluation_name="task_completion", + score_value=1.0, + kind="binary", + enduser_id="user-oid-123", + ) + attrs = _only_record(capture).__dict__ + assert attrs["enduser.id"] == "user-oid-123" + assert "enduser.pseudo.id" not in attrs + + +def test_enduser_pseudo_id_only_sets_only_pseudo_attribute(capture): + emit_human_evaluation_event( + evaluation_name="task_completion", + score_value=1.0, + kind="binary", + enduser_pseudo_id="sess_abc", + ) + attrs = _only_record(capture).__dict__ + assert attrs["enduser.pseudo.id"] == "sess_abc" + assert "enduser.id" not in attrs + + +def test_both_enduser_ids_set_both_attributes(capture): + emit_human_evaluation_event( + evaluation_name="task_completion", + score_value=1.0, + kind="binary", + enduser_id="user-oid-123", + enduser_pseudo_id="sess_abc", + ) + attrs = _only_record(capture).__dict__ + assert attrs["enduser.id"] == "user-oid-123" + assert attrs["enduser.pseudo.id"] == "sess_abc" + + +def test_tags_fan_out_into_internal_properties(capture): + emit_human_evaluation_event( + evaluation_name="task_completion", + score_value=1.0, + kind="binary", + tags={"subscription_tier": "basic_plan", "department": "marketing"}, + ) + decoded = json.loads(_only_record(capture).__dict__["internal_properties"]) + assert decoded["microsoft.human_evaluation.tags.subscription_tier"] == "basic_plan" + assert decoded["microsoft.human_evaluation.tags.department"] == "marketing" + + +def test_evaluation_id_omitted_generates_uuid(capture): + emit_human_evaluation_event(evaluation_name="task_completion", score_value=1.0, kind="binary") + decoded = json.loads(_only_record(capture).__dict__["internal_properties"]) + generated = decoded["microsoft.human_evaluation.id"] + assert isinstance(generated, str) and generated + # Should parse as a UUID (raises if not). + uuid.UUID(generated) + + +def test_evaluation_id_provided_flows_through_verbatim(capture): + emit_human_evaluation_event( + evaluation_name="task_completion", + score_value=1.0, + kind="binary", + evaluation_id="custom-eval-id-42", + ) + decoded = json.loads(_only_record(capture).__dict__["internal_properties"]) + assert decoded["microsoft.human_evaluation.id"] == "custom-eval-id-42" + + +def test_explanation_flows_through_as_top_level_attribute(capture): + emit_human_evaluation_event( + evaluation_name="task_completion", + score_value=1.0, + kind="binary", + explanation="The agent answered correctly.", + ) + record = _only_record(capture) + assert record.__dict__["gen_ai.evaluation.explanation"] == "The agent answered correctly." + + +def test_explanation_omitted_omits_attribute(capture): + emit_human_evaluation_event(evaluation_name="task_completion", score_value=1.0, kind="binary") + record = _only_record(capture) + assert "gen_ai.evaluation.explanation" not in record.__dict__ From 16a02c377fb3ebc781c16e2c90a8cbf65377d20d Mon Sep 17 00:00:00 2001 From: Sean Gayler Date: Mon, 18 May 2026 15:32:18 -0700 Subject: [PATCH 02/11] remove debug print statements --- .../samples/evaluations/sample_human_evaluations.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_human_evaluations.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_human_evaluations.py index 27a9ecbe2159..43d39b87e8ea 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_human_evaluations.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_human_evaluations.py @@ -209,8 +209,6 @@ def emit_human_evaluation_event( if __name__ == "__main__": endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] - print("Point A") - with ( DefaultAzureCredential() as credential, AIProjectClient(endpoint=endpoint, credential=credential) as project_client, @@ -220,12 +218,8 @@ def emit_human_evaluation_event( # below will be exported to Application Insights. connection_string = project_client.telemetry.get_application_insights_connection_string() - print("Point B") - configure_azure_monitor(connection_string=connection_string) - print("Point C") - # Optional: Derive the Foundry Project's resource id from any connection. The # endpoint URL alone only gives us the account + project names, but every # Connection's `id` is a full ARM path of the form: From 868f9deba084c1e8ab220d7dbafb03dd6e3ca8e0 Mon Sep 17 00:00:00 2001 From: Sean Gayler Date: Mon, 18 May 2026 16:20:32 -0700 Subject: [PATCH 03/11] add preview disclaimer --- .../samples/evaluations/sample_human_evaluations.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_human_evaluations.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_human_evaluations.py index 43d39b87e8ea..058346afe33f 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_human_evaluations.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_human_evaluations.py @@ -20,6 +20,8 @@ Python `logging`, which `azure.monitor.opentelemetry.configure_azure_monitor` routes through OpenTelemetry to Application Insights. + NOTE: Human evaluations is in preview, and carries the risk of breaking changes. + USAGE: python sample_human_evaluations.py From 9052346c0d5804e35d72cd904e0d11feab1cbe67 Mon Sep 17 00:00:00 2001 From: Sean Gayler Date: Tue, 19 May 2026 12:29:13 -0700 Subject: [PATCH 04/11] Fix CI failures: move test out of tests/samples, fix mindependency import, skip sample in evaluations run * Move tests/samples/test_sample_human_evaluations.py to tests/evaluations/test_human_evaluations.py. tests/samples/ is for recorded sample-execution tests; this is a pure unit test of the emit_human_evaluation_event helper, so it belongs alongside other evaluation tests. * Narrow tests/conftest.py auto-skip rule so test_human_evaluations.py still runs in the PR pipeline (it has no Microsoft Foundry dependency, unlike the other evaluation tests). * Defer non-stdlib imports in samples/evaluations/sample_human_evaluations.py (os, dotenv, azure.identity, azure.monitor.opentelemetry, azure.ai.projects, load_dotenv()) into the __main__ block. The emit_human_evaluation_event helper only needs stdlib + typing, so this fixes the mindependency check failure caused by azure-monitor-opentelemetry's exporter requiring a newer opentelemetry-sdk than the min-pinned version provides. * Add sample_human_evaluations.py to samples_to_skip in tests/samples/test_samples_evaluations.py. The sample requires a real Application Insights connection string and emits OTel events, so it is not suitable for recorded playback (consistent with how sample_evaluations_builtin_with_traces.py and sample_scheduled_evaluations.py are handled). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- sdk/ai/azure-ai-projects/cspell.json | 7 ++- .../evaluations/sample_human_evaluations.py | 49 +++++++++++++------ sdk/ai/azure-ai-projects/tests/conftest.py | 7 ++- .../test_human_evaluations.py} | 0 .../tests/samples/test_samples_evaluations.py | 4 ++ 5 files changed, 48 insertions(+), 19 deletions(-) rename sdk/ai/azure-ai-projects/tests/{samples/test_sample_human_evaluations.py => evaluations/test_human_evaluations.py} (100%) diff --git a/sdk/ai/azure-ai-projects/cspell.json b/sdk/ai/azure-ai-projects/cspell.json index 7decf206d14a..6177d7708c01 100644 --- a/sdk/ai/azure-ai-projects/cspell.json +++ b/sdk/ai/azure-ai-projects/cspell.json @@ -33,7 +33,8 @@ "Udbk", "UPIA", "Vnext", - "xhigh" + "xhigh", + "likert" ], "ignorePaths": [ "*.csv", @@ -41,6 +42,8 @@ "*.jsonl" ], "words": [ - "Pxqzykebv" + "Pxqzykebv", + "qtumf", + "sess" ] } diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_human_evaluations.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_human_evaluations.py index 058346afe33f..bc7f38e9d456 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_human_evaluations.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_human_evaluations.py @@ -37,27 +37,35 @@ import json import logging -import os import uuid -from typing import Literal, Mapping, Optional - -from dotenv import load_dotenv -from azure.identity import DefaultAzureCredential -from azure.monitor.opentelemetry import configure_azure_monitor -from azure.ai.projects import AIProjectClient - -load_dotenv() - -# `configure_azure_monitor` installs an OpenTelemetry LoggingHandler on the -# root logger, so any standard Python `logging` call below this point flows -# through OTel to Application Insights as a log record. The -# `microsoft.custom_event.name` attribute is what routes the record to the -# `customEvents` table. +from typing import Literal, Mapping, Optional, TypedDict + +# NOTE: Azure SDK / Application Insights imports (azure-identity, +# azure-monitor-opentelemetry, azure-ai-projects, python-dotenv) are +# intentionally deferred into the `if __name__ == "__main__":` block at the +# bottom of this file. They are only needed when running the sample directly; +# the `emit_human_evaluation_event` helper itself depends only on the +# standard library and `typing`, which keeps the helper importable in test +# environments that do not (or cannot) install the full OTel stack. + +# `configure_azure_monitor` (called in __main__) installs an OpenTelemetry +# LoggingHandler on the root logger, so any standard Python `logging` call +# below this point flows through OTel to Application Insights as a log record. +# The `microsoft.custom_event.name` attribute is what routes the record to +# the `customEvents` table. logger = logging.getLogger("human_evaluations") logger.setLevel(logging.INFO) -_KIND_DEFAULTS = { +class _KindDefaults(TypedDict): + min_value: float + max_value: float + threshold: float + desirable_direction: str + type: str + + +_KIND_DEFAULTS: dict[str, _KindDefaults] = { "binary": { "min_value": 0.0, "max_value": 1.0, @@ -209,6 +217,15 @@ def emit_human_evaluation_event( if __name__ == "__main__": + import os + + from dotenv import load_dotenv + from azure.identity import DefaultAzureCredential + from azure.monitor.opentelemetry import configure_azure_monitor + from azure.ai.projects import AIProjectClient + + load_dotenv() + endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] with ( diff --git a/sdk/ai/azure-ai-projects/tests/conftest.py b/sdk/ai/azure-ai-projects/tests/conftest.py index e75608bf08ed..fbf03122f2da 100644 --- a/sdk/ai/azure-ai-projects/tests/conftest.py +++ b/sdk/ai/azure-ai-projects/tests/conftest.py @@ -33,7 +33,12 @@ def pytest_collection_modifyitems(items): if os.environ.get("AZURE_TEST_RUN_LIVE") == "true": return for item in items: - if "tests\\evaluation" in item.fspath.strpath or "tests/evaluation" in item.fspath.strpath: + path = item.fspath.strpath + if "tests\\evaluation" in path or "tests/evaluation" in path: + # test_human_evaluations.py is a pure unit test with no Microsoft Foundry + # dependency, so it must keep running in the PR pipeline. + if "test_human_evaluations" in os.path.basename(path): + continue item.add_marker( pytest.mark.skip( reason="Skip running Evaluations tests in PR pipeline until we can sort out the failures related to Microsoft Foundry project settings" diff --git a/sdk/ai/azure-ai-projects/tests/samples/test_sample_human_evaluations.py b/sdk/ai/azure-ai-projects/tests/evaluations/test_human_evaluations.py similarity index 100% rename from sdk/ai/azure-ai-projects/tests/samples/test_sample_human_evaluations.py rename to sdk/ai/azure-ai-projects/tests/evaluations/test_human_evaluations.py diff --git a/sdk/ai/azure-ai-projects/tests/samples/test_samples_evaluations.py b/sdk/ai/azure-ai-projects/tests/samples/test_samples_evaluations.py index 2618d0320a05..de730ce08b0f 100644 --- a/sdk/ai/azure-ai-projects/tests/samples/test_samples_evaluations.py +++ b/sdk/ai/azure-ai-projects/tests/samples/test_samples_evaluations.py @@ -124,6 +124,9 @@ class TestSamplesEvaluations(AzureRecordedTestCase): uses azure-monitor-query to fetch traces. - sample_scheduled_evaluations.py: Requires Azure RBAC assignment via azure-mgmt-authorization and azure-mgmt-resource, AND uploads Dataset. + - sample_human_evaluations.py: Requires Azure Application Insights (fetches + the connection string from the Foundry project and emits OTel events via + `azure-monitor-opentelemetry`); not meaningful to record/replay. Complex prerequisites (require manual portal setup): - sample_continuous_evaluation_rule.py: Requires manual RBAC assignment in Azure @@ -152,6 +155,7 @@ class TestSamplesEvaluations(AzureRecordedTestCase): "sample_synthetic_data_agent_evaluation.py", # Synthetic data gen is long-running preview feature "sample_synthetic_data_model_evaluation.py", # Synthetic data gen is long-running preview feature "sample_eval_catalog_prompt_based_evaluators.py", # For some reason fails with 500 (Internal server error) + "sample_human_evaluations.py", # Requires real Foundry App Insights connection string + emits OTel events; not suitable for recorded playback ], ), ) From 4608e3d0e77d5e0ecc28341bdbc2991e399f1ec4 Mon Sep 17 00:00:00 2001 From: Sean Gayler Date: Tue, 19 May 2026 15:08:32 -0700 Subject: [PATCH 05/11] move tags up to customDimensions top-level --- .../samples/evaluations/sample_human_evaluations.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_human_evaluations.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_human_evaluations.py index bc7f38e9d456..28b290c993b8 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_human_evaluations.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_human_evaluations.py @@ -184,15 +184,11 @@ def emit_human_evaluation_event( "gen_ai.evaluation.type": defaults["type"], "microsoft.human_evaluation.source": "end_user", "microsoft.human_evaluation.kind": kind, - "microsoft.human_evaluation.id": evaluation_id or str(uuid.uuid4()), } if project_resource_id: internal_properties["gen_ai.azure_ai_project.id"] = project_resource_id if response_id: internal_properties["gen_ai.response.id.type"] = "responses" - if tags: - for tag_name, tag_value in tags.items(): - internal_properties[f"microsoft.human_evaluation.tags.{tag_name}"] = tag_value # Top-level event attributes follow the OTel `gen_ai.evaluation.result` # event shape (except internal_properties). `microsoft.custom_event.name` @@ -213,6 +209,13 @@ def emit_human_evaluation_event( if enduser_pseudo_id is not None: attributes["enduser.pseudo.id"] = enduser_pseudo_id + # Some attributes are customer-defined and can be put in top-level with "microsoft" prefix + if tags: + for tag_name, tag_value in tags.items(): + attributes[f"microsoft.human_evaluation.tags.{tag_name}"] = tag_value + if evaluation_id: + attributes["microsoft.human_evaluation.id"] = evaluation_id + logger.info("gen_ai.evaluation.result", extra=attributes) From a1f9f55aeb5b01d33d019990160ca0f0cb3e079f Mon Sep 17 00:00:00 2001 From: Sean Gayler Date: Tue, 19 May 2026 15:15:39 -0700 Subject: [PATCH 06/11] updates --- .../evaluations/sample_human_evaluations.py | 2 ++ .../evaluations/test_human_evaluations.py | 35 +++++++++++-------- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_human_evaluations.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_human_evaluations.py index 28b290c993b8..dd2f9ac35010 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_human_evaluations.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_human_evaluations.py @@ -269,6 +269,7 @@ def emit_human_evaluation_event( project_resource_id=project_resource_id, enduser_id="241964ad-a8db-4318-9f2e-5a7dc1f05349", tags={"subscription_tier": "basic_plan"}, + evaluation_id="986ee25a-2db1-423c-8e3c-a2774a4d2da2", ) print("Emitted binary human evaluation event.") @@ -283,5 +284,6 @@ def emit_human_evaluation_event( project_resource_id=project_resource_id, enduser_pseudo_id="sess_QdH5CAWJgqVT4rOr0qtumf", tags={"subscription_tier": "free_plan"}, + evaluation_id="87e1fa43-46fc-4ddc-aaa6-c9af716fb47b" ) print("Emitted likert_5 human evaluation event.") diff --git a/sdk/ai/azure-ai-projects/tests/evaluations/test_human_evaluations.py b/sdk/ai/azure-ai-projects/tests/evaluations/test_human_evaluations.py index b0450dab7e14..422716ce0a24 100644 --- a/sdk/ai/azure-ai-projects/tests/evaluations/test_human_evaluations.py +++ b/sdk/ai/azure-ai-projects/tests/evaluations/test_human_evaluations.py @@ -6,7 +6,6 @@ import json import logging import sys -import uuid from pathlib import Path import pytest @@ -143,7 +142,6 @@ def test_internal_properties_is_json_encoded_string_with_binary_defaults(capture assert decoded["gen_ai.evaluation.type"] == "boolean" assert decoded["microsoft.human_evaluation.source"] == "end_user" assert decoded["microsoft.human_evaluation.kind"] == "binary" - assert "microsoft.human_evaluation.id" in decoded def test_internal_properties_likert_5_defaults(capture): @@ -240,36 +238,43 @@ def test_both_enduser_ids_set_both_attributes(capture): assert attrs["enduser.pseudo.id"] == "sess_abc" -def test_tags_fan_out_into_internal_properties(capture): +def test_tags_fan_out_as_top_level_attributes(capture): emit_human_evaluation_event( evaluation_name="task_completion", score_value=1.0, kind="binary", tags={"subscription_tier": "basic_plan", "department": "marketing"}, ) - decoded = json.loads(_only_record(capture).__dict__["internal_properties"]) - assert decoded["microsoft.human_evaluation.tags.subscription_tier"] == "basic_plan" - assert decoded["microsoft.human_evaluation.tags.department"] == "marketing" + attrs = _only_record(capture).__dict__ + assert attrs["microsoft.human_evaluation.tags.subscription_tier"] == "basic_plan" + assert attrs["microsoft.human_evaluation.tags.department"] == "marketing" + # And explicitly: tags must NOT also be inside internal_properties. + decoded = json.loads(attrs["internal_properties"]) + assert "microsoft.human_evaluation.tags.subscription_tier" not in decoded + assert "microsoft.human_evaluation.tags.department" not in decoded -def test_evaluation_id_omitted_generates_uuid(capture): +def test_evaluation_id_omitted_omits_attribute(capture): emit_human_evaluation_event(evaluation_name="task_completion", score_value=1.0, kind="binary") - decoded = json.loads(_only_record(capture).__dict__["internal_properties"]) - generated = decoded["microsoft.human_evaluation.id"] - assert isinstance(generated, str) and generated - # Should parse as a UUID (raises if not). - uuid.UUID(generated) + attrs = _only_record(capture).__dict__ + assert "microsoft.human_evaluation.id" not in attrs + # And explicitly: id must not be inside internal_properties either. + decoded = json.loads(attrs["internal_properties"]) + assert "microsoft.human_evaluation.id" not in decoded -def test_evaluation_id_provided_flows_through_verbatim(capture): +def test_evaluation_id_provided_flows_through_verbatim_as_top_level_attribute(capture): emit_human_evaluation_event( evaluation_name="task_completion", score_value=1.0, kind="binary", evaluation_id="custom-eval-id-42", ) - decoded = json.loads(_only_record(capture).__dict__["internal_properties"]) - assert decoded["microsoft.human_evaluation.id"] == "custom-eval-id-42" + attrs = _only_record(capture).__dict__ + assert attrs["microsoft.human_evaluation.id"] == "custom-eval-id-42" + # And explicitly: id must not also be inside internal_properties. + decoded = json.loads(attrs["internal_properties"]) + assert "microsoft.human_evaluation.id" not in decoded def test_explanation_flows_through_as_top_level_attribute(capture): From 802f2279e5b45ac3ebef6fcf3918b1e6733cc179 Mon Sep 17 00:00:00 2001 From: Sean Gayler Date: Tue, 26 May 2026 13:38:33 -0700 Subject: [PATCH 07/11] update for new spec --- .../evaluations/sample_human_evaluations.py | 394 ++++++++++-------- .../evaluations/test_human_evaluations.py | 255 +++++------- 2 files changed, 310 insertions(+), 339 deletions(-) diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_human_evaluations.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_human_evaluations.py index dd2f9ac35010..422b91a79e5a 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_human_evaluations.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_human_evaluations.py @@ -9,18 +9,22 @@ application as OpenTelemetry custom events that land in the `customEvents` Application Insights table connected to your Microsoft Foundry project. - Human evaluations capture signals that automated evaluators struggle with -- - tone, user satisfaction, factual nuance -- typically as a thumbs up/down - (binary) or a 5-point rating (likert_5) provided by an end user of your - application. + Human evaluations capture signals that automated evaluators struggle with, + such as tone, user satisfaction, and factual nuance. The helpers below emit + the `gen_ai.evaluation.result` event shape used by Microsoft Foundry: - The sample defines a small `emit_human_evaluation_event(...)` helper that - assembles and emits OTel-compliant events that carry additional metadata - for compatibility with Microsoft Azure services. The event is emitted via - Python `logging`, which `azure.monitor.opentelemetry.configure_azure_monitor` - routes through OpenTelemetry to Application Insights. + * `emit_boolean_evaluation(...)` emits binary scores such as thumbs up/down. + * `emit_5_point_ordinal_evaluation(...)` emits 1-5 ordered scores such as a + Likert or star rating. - NOTE: Human evaluations is in preview, and carries the risk of breaking changes. + Both public helpers call the shared `_emit_human_evaluation(...)` helper so + the OpenTelemetry and Microsoft-specific attribute mapping stays consistent. + This sample covers human evaluations submitted by end users of your + application and correlates evaluation events to OpenAI Responses API response + IDs when a response ID is provided. + + NOTE: Human evaluations are in preview and carry the risk of breaking + changes. USAGE: python sample_human_evaluations.py @@ -37,68 +41,70 @@ import json import logging -import uuid -from typing import Literal, Mapping, Optional, TypedDict +from typing import Literal, Mapping, Optional # NOTE: Azure SDK / Application Insights imports (azure-identity, # azure-monitor-opentelemetry, azure-ai-projects, python-dotenv) are # intentionally deferred into the `if __name__ == "__main__":` block at the # bottom of this file. They are only needed when running the sample directly; -# the `emit_human_evaluation_event` helper itself depends only on the -# standard library and `typing`, which keeps the helper importable in test -# environments that do not (or cannot) install the full OTel stack. +# the helper functions themselves depend only on the standard library and +# `typing`, which keeps them importable in test environments that do not install +# the full OTel stack. # `configure_azure_monitor` (called in __main__) installs an OpenTelemetry -# LoggingHandler on the root logger, so any standard Python `logging` call -# below this point flows through OTel to Application Insights as a log record. -# The `microsoft.custom_event.name` attribute is what routes the record to -# the `customEvents` table. +# LoggingHandler on the root logger, so any standard Python `logging` call below +# this point flows through OTel to Application Insights as a log record. The +# `microsoft.custom_event.name` attribute routes the record to the +# `customEvents` table. logger = logging.getLogger("human_evaluations") logger.setLevel(logging.INFO) +EvaluationType = Literal["boolean", "ordinal"] +DesirableDirection = Literal["increase", "decrease"] -class _KindDefaults(TypedDict): - min_value: float - max_value: float - threshold: float - desirable_direction: str - type: str - - -_KIND_DEFAULTS: dict[str, _KindDefaults] = { - "binary": { - "min_value": 0.0, - "max_value": 1.0, - "threshold": 1.0, - "desirable_direction": "increase", - "type": "boolean", - }, - "likert_5": { - "min_value": 1.0, - "max_value": 5.0, - "threshold": 3.0, - "desirable_direction": "increase", - "type": "ordinal", - }, -} - - -# When identifying the end user, populate either or both of: -# enduser_id → the signed-in user's identity (e.g., AAD object id, -# email). This is PII and lands in App Insights as -# `user_AuthenticatedId`. -# enduser_pseudo_id → a random, non-identifying id you generated (e.g., a -# browser cookie or device id). Use when the user is -# anonymous or you don't want to log PII. Lands in -# App Insights as `user_Id`. -# Signed-in users often have both: `enduser_id` says who they are, and -# `enduser_pseudo_id` lets you correlate them with their earlier anonymous -# activity from the same browser/device. -def emit_human_evaluation_event( +def _validate_score( *, - evaluation_name: str, score_value: float, - kind: Literal["binary", "likert_5"], + min_value: float, + max_value: float, + evaluation_type: EvaluationType, +) -> float: + score_value = float(score_value) + if not (min_value <= score_value <= max_value): + raise ValueError( + f"score_value {score_value} is outside the allowed range " + f"[{min_value}, {max_value}] for evaluation type '{evaluation_type}'." + ) + if score_value != int(score_value): + raise ValueError( + f"score_value {score_value} must be a double-encoded integer value " + f"for evaluation type '{evaluation_type}'." + ) + return score_value + + +def _get_score_label( + *, + score_value: float, + threshold: float, + desirable_direction: DesirableDirection, +) -> Literal["pass", "fail"]: + if desirable_direction == "increase": + return "pass" if score_value >= threshold else "fail" + if desirable_direction == "decrease": + return "pass" if score_value <= threshold else "fail" + raise ValueError(f"Unsupported desirable_direction: {desirable_direction!r}.") + + +def _emit_human_evaluation( + *, + evaluation_metric_name: str, + score_value: float, + evaluation_type: EvaluationType, + min_value: float, + max_value: float, + threshold: float, + desirable_direction: DesirableDirection, explanation: Optional[str] = None, response_id: Optional[str] = None, project_resource_id: Optional[str] = None, @@ -107,125 +113,160 @@ def emit_human_evaluation_event( tags: Optional[Mapping[str, str]] = None, evaluation_id: Optional[str] = None, ) -> None: - """Emit a single `gen_ai.evaluation.result` human evaluation event. + score_value = _validate_score( + score_value=score_value, + min_value=min_value, + max_value=max_value, + evaluation_type=evaluation_type, + ) + score_label = _get_score_label( + score_value=score_value, + threshold=threshold, + desirable_direction=desirable_direction, + ) - The helper takes care of all the bookkeeping (deriving min/max/ - threshold/type from `kind`, deriving the pass/fail label from the score, - JSON-encoding `internal_properties`, generating a correlation id, etc.). - - Args: - evaluation_name: The metric being evaluated. Free-form, but pick a - consistent snake_case name per metric. - Examples: "task_completion", "relevance", "helpfulness". - score_value: The numeric score the user gave. Must be a whole number - (no fractional part), expressed as a float. - For kind="binary": 0.0 (thumbs down) or 1.0 (thumbs up). - For kind="likert_5": 1.0, 2.0, 3.0, 4.0, or 5.0. - kind: The evaluation shape. "binary" for thumbs up/down; "likert_5" - for a 1-5 star rating. - explanation: Optional free-form text the user provided to justify - their score. - Example: "The agent answered the question accurately." - response_id: Optional id of the agent response being evaluated. - Typically the id of an OpenAI Responses API response. Used to - correlate the evaluation back to the response it judged. - Example: "resp_64904952b20872620069f8d600779c81908f58b0a3be090ef0". - project_resource_id: Optional Azure resource id of the Foundry - project the evaluation belongs to. Required by Microsoft systems - when you want the evaluation linked to a specific project. - Example: "/subscriptions//resourceGroups//providers/Microsoft.CognitiveServices/accounts//projects/". - enduser_id: Optional signed-in user identity (e.g., Entra ID object - id, email). This is PII and lands in App Insights as - `user_AuthenticatedId`. - Examples: "alice@contoso.com", "241964ad-a8db-4318-9f2e-5a7dc1f05349". - enduser_pseudo_id: Optional random, non-identifying id you generated - (e.g., a browser cookie or device id). Use when the user is - anonymous or you don't want to log PII. Lands in App Insights - as `user_Id`. - Example: "sess_QdH5CAWJgqVT4rOr0qtumf". - tags: Optional extra metadata to attach to the event. Each key is - emitted as `microsoft.human_evaluation.tags.` so you can - slice on it later in App Insights. - Example: {"subscription_tier": "basic_plan", "feature": "chat"}. - evaluation_id: Optional stable id for this specific evaluation event, - useful if you want to update or correlate it later. Defaults to a - fresh uuid4. - Example: "69d937a7-32e2-412e-97c9-119e2d282723". - """ - if kind not in _KIND_DEFAULTS: - raise ValueError(f"Unsupported kind '{kind}'. Use 'binary' or 'likert_5'.") - defaults = _KIND_DEFAULTS[kind] - - if not defaults["min_value"] <= score_value <= defaults["max_value"]: - raise ValueError( - f"score_value {score_value} is outside the allowed range " - f"[{defaults['min_value']}, {defaults['max_value']}] for kind '{kind}'." - ) - - # Ensure scores are whole numbers - if score_value != int(score_value): - raise ValueError( - f"score_value {score_value} must be a whole number (no fractional part) " - f"for kind '{kind}'." - ) - - # Per spec, the score is at or above the threshold = "pass", below = "fail". - # (Binary: 1.0 -> pass, 0.0 -> fail. Likert-5: >=3.0 -> pass, <3.0 -> fail.) - score_label = "pass" if score_value >= defaults["threshold"] else "fail" - - # `internal_properties` carries Microsoft-specific attributes. It MUST be a - # JSON-encoded string (not a nested object) to match how downstream systems - # like Azure Monitor and Foundry consume it. internal_properties = { - "gen_ai.evaluation.threshold": str(defaults["threshold"]), - "gen_ai.evaluation.min_value": str(defaults["min_value"]), - "gen_ai.evaluation.max_value": str(defaults["max_value"]), - "gen_ai.evaluation.desirable_direction": defaults["desirable_direction"], - "gen_ai.evaluation.type": defaults["type"], - "microsoft.human_evaluation.source": "end_user", - "microsoft.human_evaluation.kind": kind, + "gen_ai.evaluation.threshold": str(threshold), + "gen_ai.evaluation.min_value": str(min_value), + "gen_ai.evaluation.max_value": str(max_value), + "gen_ai.evaluation.desirable_direction": desirable_direction, + "gen_ai.evaluation.type": evaluation_type, } - if project_resource_id: + if project_resource_id is not None: internal_properties["gen_ai.azure_ai_project.id"] = project_resource_id - if response_id: - internal_properties["gen_ai.response.id.type"] = "responses" - # Top-level event attributes follow the OTel `gen_ai.evaluation.result` - # event shape (except internal_properties). `microsoft.custom_event.name` - # is what routes the record to the `customEvents` App Insights table. attributes = { "microsoft.custom_event.name": "gen_ai.evaluation.result", - "gen_ai.evaluation.name": evaluation_name, + "gen_ai.evaluation.name": evaluation_metric_name, "gen_ai.evaluation.score.value": score_value, "gen_ai.evaluation.score.label": score_label, + "microsoft.human_evaluation.source": "end_user", "internal_properties": json.dumps(internal_properties), } if explanation is not None: attributes["gen_ai.evaluation.explanation"] = explanation if response_id is not None: attributes["gen_ai.response.id"] = response_id + attributes["microsoft.gen_ai.response.id.type"] = "responses" if enduser_id is not None: attributes["enduser.id"] = enduser_id if enduser_pseudo_id is not None: attributes["enduser.pseudo.id"] = enduser_pseudo_id - - # Some attributes are customer-defined and can be put in top-level with "microsoft" prefix if tags: for tag_name, tag_value in tags.items(): - attributes[f"microsoft.human_evaluation.tags.{tag_name}"] = tag_value - if evaluation_id: - attributes["microsoft.human_evaluation.id"] = evaluation_id + attributes[f"microsoft.evaluation.tags.{tag_name}"] = tag_value + if evaluation_id is not None: + attributes["microsoft.evaluation.id"] = evaluation_id logger.info("gen_ai.evaluation.result", extra=attributes) +def emit_boolean_evaluation( + *, + evaluation_metric_name: str, + passed: bool, + explanation: Optional[str] = None, + response_id: Optional[str] = None, + project_resource_id: Optional[str] = None, + enduser_id: Optional[str] = None, + enduser_pseudo_id: Optional[str] = None, + tags: Optional[Mapping[str, str]] = None, + evaluation_id: Optional[str] = None, +) -> None: + """Emit a boolean human evaluation event. + + Boolean evaluations are typically represented as thumbs up/down, yes/no, or + pass/fail controls. `passed=True` emits a score of `1.0`; `passed=False` + emits a score of `0.0`. + + Args: + evaluation_metric_name: Name of the evaluated metric, such as + `"task_completion"` or `"helpfulness"`. + passed: Whether the human evaluation passed. + explanation: Optional free-form explanation from the end user. + response_id: Optional OpenAI Responses API response ID being evaluated. + project_resource_id: Optional ARM resource ID for the Foundry project. + enduser_id: Optional signed-in end-user ID. This may contain PII and maps + to `user_AuthenticatedId` in Application Insights. + enduser_pseudo_id: Optional pseudonymous end-user ID. This maps to + `user_Id` in Application Insights. + tags: Optional metadata emitted as `microsoft.evaluation.tags.`. + evaluation_id: Optional ID for the evaluation event itself. + """ + _emit_human_evaluation( + evaluation_metric_name=evaluation_metric_name, + score_value=1.0 if passed else 0.0, + evaluation_type="boolean", + min_value=0.0, + max_value=1.0, + threshold=1.0, + desirable_direction="increase", + explanation=explanation, + response_id=response_id, + project_resource_id=project_resource_id, + enduser_id=enduser_id, + enduser_pseudo_id=enduser_pseudo_id, + tags=tags, + evaluation_id=evaluation_id, + ) + + +def emit_5_point_ordinal_evaluation( + *, + evaluation_metric_name: str, + score_value: float, + explanation: Optional[str] = None, + response_id: Optional[str] = None, + project_resource_id: Optional[str] = None, + enduser_id: Optional[str] = None, + enduser_pseudo_id: Optional[str] = None, + tags: Optional[Mapping[str, str]] = None, + evaluation_id: Optional[str] = None, +) -> None: + """Emit a 5-point ordinal human evaluation event. + + Ordinal 5-point evaluations use integer scores from `1.0` through `5.0`; + scores at or above `3.0` are emitted with a `pass` label. + + Args: + evaluation_metric_name: Name of the evaluated metric, such as + `"relevance"` or `"helpfulness"`. + score_value: Integer score from `1.0` through `5.0`. + explanation: Optional free-form explanation from the end user. + response_id: Optional OpenAI Responses API response ID being evaluated. + project_resource_id: Optional ARM resource ID for the Foundry project. + enduser_id: Optional signed-in end-user ID. This may contain PII and maps + to `user_AuthenticatedId` in Application Insights. + enduser_pseudo_id: Optional pseudonymous end-user ID. This maps to + `user_Id` in Application Insights. + tags: Optional metadata emitted as `microsoft.evaluation.tags.`. + evaluation_id: Optional ID for the evaluation event itself. + """ + _emit_human_evaluation( + evaluation_metric_name=evaluation_metric_name, + score_value=score_value, + evaluation_type="ordinal", + min_value=1.0, + max_value=5.0, + threshold=3.0, + desirable_direction="increase", + explanation=explanation, + response_id=response_id, + project_resource_id=project_resource_id, + enduser_id=enduser_id, + enduser_pseudo_id=enduser_pseudo_id, + tags=tags, + evaluation_id=evaluation_id, + ) + + if __name__ == "__main__": import os - from dotenv import load_dotenv + from azure.ai.projects import AIProjectClient from azure.identity import DefaultAzureCredential from azure.monitor.opentelemetry import configure_azure_monitor - from azure.ai.projects import AIProjectClient + from dotenv import load_dotenv load_dotenv() @@ -242,48 +283,37 @@ def emit_human_evaluation_event( configure_azure_monitor(connection_string=connection_string) - # Optional: Derive the Foundry Project's resource id from any connection. The - # endpoint URL alone only gives us the account + project names, but every - # Connection's `id` is a full ARM path of the form: - # /subscriptions//resourceGroups//providers/Microsoft.CognitiveServices/accounts//projects//connections/ - # We just strip the trailing /connections/ to get the project id. + # Optional: derive the Foundry Project's resource id from any connection. + # The endpoint URL alone only gives us the account + project names, but + # every Connection's `id` is a full ARM path ending with /connections/. any_connection = next(iter(project_client.connections.list()), None) - project_resource_id = ( - any_connection.id.rsplit("/connections/", 1)[0] if any_connection else None - ) + project_resource_id = any_connection.id.rsplit("/connections/", 1)[0] if any_connection else None - # The two examples below differ in evaluation kind (binary vs likert_5) - # AND in how the end user is identified, just to show both styles. In your - # own app, those two choices are independent. You can also pass both - # `enduser_id` and `enduser_pseudo_id` together -- typical for a - # signed-in user whose browser you've been tracking with a cookie. - - # Example 1: A signed-in end user gives a thumbs up on the agent's task - # completion. - emit_human_evaluation_event( - evaluation_name="task_completion", - score_value=1.0, - kind="binary", + # Example 1: an anonymous end user gives a thumbs up on task completion. + emit_boolean_evaluation( + evaluation_metric_name="task_completion", + passed=True, explanation="The agent provided accurate weather information as requested.", response_id="resp_64904952b20872620069f8d600779c81908f58b0a3be090ef0", project_resource_id=project_resource_id, - enduser_id="241964ad-a8db-4318-9f2e-5a7dc1f05349", + enduser_pseudo_id="sess_123456", tags={"subscription_tier": "basic_plan"}, - evaluation_id="986ee25a-2db1-423c-8e3c-a2774a4d2da2", + evaluation_id="0b27be45-cd65-4671-ab08-c3eafd4c9613", ) - print("Emitted binary human evaluation event.") + print("Emitted boolean human evaluation event.") - # Example 2: An anonymous end user rates the agent's response 4 out of 5 - # stars for relevance. - emit_human_evaluation_event( - evaluation_name="relevance", + # Example 2: a signed-in end user rates relevance on a 5-point scale. + emit_5_point_ordinal_evaluation( + evaluation_metric_name="relevance", score_value=4.0, - kind="likert_5", - explanation="The agent's response is relevant to the query.", - response_id="resp_1234567890abcdef", + explanation=( + "The agent's response is relevant to the query, providing useful " + "information that addresses the user's intent." + ), + response_id="resp_64904952b20872620069f8d600779c81908f58b0a3be090ef0", project_resource_id=project_resource_id, - enduser_pseudo_id="sess_QdH5CAWJgqVT4rOr0qtumf", - tags={"subscription_tier": "free_plan"}, - evaluation_id="87e1fa43-46fc-4ddc-aaa6-c9af716fb47b" + enduser_id="oid:241964ad-a8db-4318-9f2e-5a7dc1f05349", + tags={"department": "marketing"}, + evaluation_id="69d937a7-32e2-412e-97c9-119e2d282723", ) - print("Emitted likert_5 human evaluation event.") + print("Emitted 5-point ordinal human evaluation event.") diff --git a/sdk/ai/azure-ai-projects/tests/evaluations/test_human_evaluations.py b/sdk/ai/azure-ai-projects/tests/evaluations/test_human_evaluations.py index 422716ce0a24..cb2f6e660d63 100644 --- a/sdk/ai/azure-ai-projects/tests/evaluations/test_human_evaluations.py +++ b/sdk/ai/azure-ai-projects/tests/evaluations/test_human_evaluations.py @@ -13,12 +13,11 @@ SAMPLES_EVALUATIONS_DIR = Path(__file__).resolve().parents[1] / ".." / "samples" / "evaluations" sys.path.insert(0, str(SAMPLES_EVALUATIONS_DIR.resolve())) -from sample_human_evaluations import emit_human_evaluation_event # noqa: E402 +from sample_human_evaluations import emit_5_point_ordinal_evaluation, emit_boolean_evaluation # noqa: E402 class _RecordCapture(logging.Handler): - """Capture every ``LogRecord`` the helper emits so tests can introspect - the ``extra=`` kwargs that ended up as attributes on the record.""" + """Capture emitted ``LogRecord`` instances so tests can inspect ``extra=`` attributes.""" def __init__(self) -> None: super().__init__(level=logging.DEBUG) @@ -30,8 +29,6 @@ def emit(self, record: logging.LogRecord) -> None: @pytest.fixture(autouse=True) def capture(): - """Attach a fresh capture handler to the ``human_evaluations`` logger for - each test and detach on teardown.""" logger = logging.getLogger("human_evaluations") handler = _RecordCapture() logger.addHandler(handler) @@ -41,143 +38,104 @@ def capture(): logger.removeHandler(handler) -def _only_record(capture: _RecordCapture) -> logging.LogRecord: +def _only_attrs(capture: _RecordCapture) -> dict: assert len(capture.records) == 1, f"expected exactly 1 emitted record, got {len(capture.records)}" - return capture.records[0] + return capture.records[0].__dict__ -# --------------------------------------------------------------------------- -# Validation: range / integer / unknown kind -# --------------------------------------------------------------------------- - - -def test_binary_score_0_emits_with_fail_label(capture): - emit_human_evaluation_event(evaluation_name="thumbs", score_value=0.0, kind="binary") - record = _only_record(capture) - assert record.__dict__["gen_ai.evaluation.score.label"] == "fail" - assert record.__dict__["gen_ai.evaluation.score.value"] == 0.0 - - -def test_binary_score_1_emits_with_pass_label(capture): - emit_human_evaluation_event(evaluation_name="thumbs", score_value=1.0, kind="binary") - record = _only_record(capture) - assert record.__dict__["gen_ai.evaluation.score.label"] == "pass" - assert record.__dict__["gen_ai.evaluation.score.value"] == 1.0 - - -def test_binary_score_fractional_raises(): - with pytest.raises(ValueError): - emit_human_evaluation_event(evaluation_name="thumbs", score_value=0.5, kind="binary") - - -def test_binary_score_out_of_range_raises(): - with pytest.raises(ValueError): - emit_human_evaluation_event(evaluation_name="thumbs", score_value=2.0, kind="binary") - - -def test_likert_5_score_1_emits_with_fail_label(capture): - emit_human_evaluation_event(evaluation_name="relevance", score_value=1.0, kind="likert_5") - assert _only_record(capture).__dict__["gen_ai.evaluation.score.label"] == "fail" +def _internal_properties(attrs: dict) -> dict: + raw = attrs["internal_properties"] + assert isinstance(raw, str) + return json.loads(raw) -def test_likert_5_score_at_threshold_emits_with_pass_label(capture): - """Guards the `>=` vs `>` mistake: score == threshold must be pass.""" - emit_human_evaluation_event(evaluation_name="relevance", score_value=3.0, kind="likert_5") - assert _only_record(capture).__dict__["gen_ai.evaluation.score.label"] == "pass" +def test_boolean_failed_emits_score_0_with_fail_label(capture): + emit_boolean_evaluation(evaluation_metric_name="task_completion", passed=False) + attrs = _only_attrs(capture) + assert attrs["gen_ai.evaluation.score.label"] == "fail" + assert attrs["gen_ai.evaluation.score.value"] == 0.0 -def test_likert_5_score_5_emits_with_pass_label(capture): - emit_human_evaluation_event(evaluation_name="relevance", score_value=5.0, kind="likert_5") - assert _only_record(capture).__dict__["gen_ai.evaluation.score.label"] == "pass" +def test_boolean_passed_emits_score_1_with_pass_label(capture): + emit_boolean_evaluation(evaluation_metric_name="task_completion", passed=True) + attrs = _only_attrs(capture) + assert attrs["gen_ai.evaluation.score.label"] == "pass" + assert attrs["gen_ai.evaluation.score.value"] == 1.0 -def test_likert_5_non_integer_score_raises(): - with pytest.raises(ValueError): - emit_human_evaluation_event(evaluation_name="relevance", score_value=2.5, kind="likert_5") +@pytest.mark.parametrize( + "score_value, expected_label", + [ + (1.0, "fail"), + (3.0, "pass"), + (5.0, "pass"), + ], +) +def test_ordinal_score_emits_expected_label(capture, score_value, expected_label): + emit_5_point_ordinal_evaluation(evaluation_metric_name="relevance", score_value=score_value) + attrs = _only_attrs(capture) + assert attrs["gen_ai.evaluation.score.label"] == expected_label + assert attrs["gen_ai.evaluation.score.value"] == score_value -def test_likert_5_score_out_of_range_raises(): +def test_ordinal_non_integer_score_raises(): with pytest.raises(ValueError): - emit_human_evaluation_event(evaluation_name="relevance", score_value=6.0, kind="likert_5") + emit_5_point_ordinal_evaluation(evaluation_metric_name="relevance", score_value=2.5) -def test_unknown_kind_raises(): +@pytest.mark.parametrize("score_value", [0.0, 6.0]) +def test_ordinal_score_out_of_range_raises(score_value): with pytest.raises(ValueError): - emit_human_evaluation_event( - evaluation_name="relevance", - score_value=1.0, - kind="unknown", # type: ignore[arg-type] - ) - - -# --------------------------------------------------------------------------- -# Shape: top-level attributes + internal_properties JSON encoding -# --------------------------------------------------------------------------- + emit_5_point_ordinal_evaluation(evaluation_metric_name="relevance", score_value=score_value) def test_top_level_attributes_have_canonical_keys_and_routing(capture): - emit_human_evaluation_event(evaluation_name="task_completion", score_value=1.0, kind="binary") - record = _only_record(capture) - attrs = record.__dict__ + emit_boolean_evaluation(evaluation_metric_name="task_completion", passed=True) + attrs = _only_attrs(capture) assert attrs["microsoft.custom_event.name"] == "gen_ai.evaluation.result" assert attrs["gen_ai.evaluation.name"] == "task_completion" assert attrs["gen_ai.evaluation.score.value"] == 1.0 assert attrs["gen_ai.evaluation.score.label"] == "pass" - # internal_properties must be present as a top-level attribute too. + assert attrs["microsoft.human_evaluation.source"] == "end_user" assert "internal_properties" in attrs -def test_internal_properties_is_json_encoded_string_with_binary_defaults(capture): - """internal_properties MUST be a JSON-encoded string, not a nested dict, - per the genai_human_evaluations spec. - """ - emit_human_evaluation_event(evaluation_name="task_completion", score_value=1.0, kind="binary") - raw = _only_record(capture).__dict__["internal_properties"] - assert isinstance(raw, str) - decoded = json.loads(raw) +def test_internal_properties_is_json_encoded_string_with_boolean_defaults(capture): + emit_boolean_evaluation(evaluation_metric_name="task_completion", passed=True) + decoded = _internal_properties(_only_attrs(capture)) assert decoded["gen_ai.evaluation.threshold"] == "1.0" assert decoded["gen_ai.evaluation.min_value"] == "0.0" assert decoded["gen_ai.evaluation.max_value"] == "1.0" assert decoded["gen_ai.evaluation.desirable_direction"] == "increase" assert decoded["gen_ai.evaluation.type"] == "boolean" - assert decoded["microsoft.human_evaluation.source"] == "end_user" - assert decoded["microsoft.human_evaluation.kind"] == "binary" -def test_internal_properties_likert_5_defaults(capture): - emit_human_evaluation_event(evaluation_name="relevance", score_value=4.0, kind="likert_5") - decoded = json.loads(_only_record(capture).__dict__["internal_properties"]) +def test_internal_properties_ordinal_defaults(capture): + emit_5_point_ordinal_evaluation(evaluation_metric_name="relevance", score_value=4.0) + decoded = _internal_properties(_only_attrs(capture)) assert decoded["gen_ai.evaluation.threshold"] == "3.0" assert decoded["gen_ai.evaluation.min_value"] == "1.0" assert decoded["gen_ai.evaluation.max_value"] == "5.0" + assert decoded["gen_ai.evaluation.desirable_direction"] == "increase" assert decoded["gen_ai.evaluation.type"] == "ordinal" - assert decoded["microsoft.human_evaluation.kind"] == "likert_5" - - -# --------------------------------------------------------------------------- -# Conditional fields -# --------------------------------------------------------------------------- -def test_response_id_set_adds_top_level_id_and_internal_type(capture): - emit_human_evaluation_event( - evaluation_name="task_completion", - score_value=1.0, - kind="binary", +def test_response_id_set_adds_top_level_id_and_response_id_type(capture): + emit_boolean_evaluation( + evaluation_metric_name="task_completion", + passed=True, response_id="resp_abc123", ) - record = _only_record(capture) - assert record.__dict__["gen_ai.response.id"] == "resp_abc123" - decoded = json.loads(record.__dict__["internal_properties"]) - assert decoded["gen_ai.response.id.type"] == "responses" + attrs = _only_attrs(capture) + assert attrs["gen_ai.response.id"] == "resp_abc123" + assert attrs["microsoft.gen_ai.response.id.type"] == "responses" -def test_response_id_omitted_omits_both_keys(capture): - emit_human_evaluation_event(evaluation_name="task_completion", score_value=1.0, kind="binary") - record = _only_record(capture) - assert "gen_ai.response.id" not in record.__dict__ - decoded = json.loads(record.__dict__["internal_properties"]) - assert "gen_ai.response.id.type" not in decoded +def test_response_id_omitted_omits_response_id_type(capture): + emit_boolean_evaluation(evaluation_metric_name="task_completion", passed=True) + attrs = _only_attrs(capture) + assert "gen_ai.response.id" not in attrs + assert "microsoft.gen_ai.response.id.type" not in attrs def test_project_resource_id_set_added_to_internal_properties(capture): @@ -185,110 +143,93 @@ def test_project_resource_id_set_added_to_internal_properties(capture): "/subscriptions/sub-id/resourceGroups/rg/providers/Microsoft.CognitiveServices" "/accounts/acct/projects/proj" ) - emit_human_evaluation_event( - evaluation_name="task_completion", - score_value=1.0, - kind="binary", + emit_boolean_evaluation( + evaluation_metric_name="task_completion", + passed=True, project_resource_id=arm_id, ) - decoded = json.loads(_only_record(capture).__dict__["internal_properties"]) + decoded = _internal_properties(_only_attrs(capture)) assert decoded["gen_ai.azure_ai_project.id"] == arm_id def test_project_resource_id_omitted_omits_key(capture): - emit_human_evaluation_event(evaluation_name="task_completion", score_value=1.0, kind="binary") - decoded = json.loads(_only_record(capture).__dict__["internal_properties"]) + emit_boolean_evaluation(evaluation_metric_name="task_completion", passed=True) + decoded = _internal_properties(_only_attrs(capture)) assert "gen_ai.azure_ai_project.id" not in decoded def test_enduser_id_only_sets_only_authenticated_attribute(capture): - emit_human_evaluation_event( - evaluation_name="task_completion", - score_value=1.0, - kind="binary", + emit_boolean_evaluation( + evaluation_metric_name="task_completion", + passed=True, enduser_id="user-oid-123", ) - attrs = _only_record(capture).__dict__ + attrs = _only_attrs(capture) assert attrs["enduser.id"] == "user-oid-123" assert "enduser.pseudo.id" not in attrs def test_enduser_pseudo_id_only_sets_only_pseudo_attribute(capture): - emit_human_evaluation_event( - evaluation_name="task_completion", - score_value=1.0, - kind="binary", + emit_boolean_evaluation( + evaluation_metric_name="task_completion", + passed=True, enduser_pseudo_id="sess_abc", ) - attrs = _only_record(capture).__dict__ + attrs = _only_attrs(capture) assert attrs["enduser.pseudo.id"] == "sess_abc" assert "enduser.id" not in attrs def test_both_enduser_ids_set_both_attributes(capture): - emit_human_evaluation_event( - evaluation_name="task_completion", - score_value=1.0, - kind="binary", + emit_boolean_evaluation( + evaluation_metric_name="task_completion", + passed=True, enduser_id="user-oid-123", enduser_pseudo_id="sess_abc", ) - attrs = _only_record(capture).__dict__ + attrs = _only_attrs(capture) assert attrs["enduser.id"] == "user-oid-123" assert attrs["enduser.pseudo.id"] == "sess_abc" def test_tags_fan_out_as_top_level_attributes(capture): - emit_human_evaluation_event( - evaluation_name="task_completion", - score_value=1.0, - kind="binary", + emit_boolean_evaluation( + evaluation_metric_name="task_completion", + passed=True, tags={"subscription_tier": "basic_plan", "department": "marketing"}, ) - attrs = _only_record(capture).__dict__ - assert attrs["microsoft.human_evaluation.tags.subscription_tier"] == "basic_plan" - assert attrs["microsoft.human_evaluation.tags.department"] == "marketing" - # And explicitly: tags must NOT also be inside internal_properties. - decoded = json.loads(attrs["internal_properties"]) - assert "microsoft.human_evaluation.tags.subscription_tier" not in decoded - assert "microsoft.human_evaluation.tags.department" not in decoded + attrs = _only_attrs(capture) + assert attrs["microsoft.evaluation.tags.subscription_tier"] == "basic_plan" + assert attrs["microsoft.evaluation.tags.department"] == "marketing" def test_evaluation_id_omitted_omits_attribute(capture): - emit_human_evaluation_event(evaluation_name="task_completion", score_value=1.0, kind="binary") - attrs = _only_record(capture).__dict__ - assert "microsoft.human_evaluation.id" not in attrs - # And explicitly: id must not be inside internal_properties either. - decoded = json.loads(attrs["internal_properties"]) - assert "microsoft.human_evaluation.id" not in decoded + emit_boolean_evaluation(evaluation_metric_name="task_completion", passed=True) + attrs = _only_attrs(capture) + assert "microsoft.evaluation.id" not in attrs def test_evaluation_id_provided_flows_through_verbatim_as_top_level_attribute(capture): - emit_human_evaluation_event( - evaluation_name="task_completion", - score_value=1.0, - kind="binary", + emit_boolean_evaluation( + evaluation_metric_name="task_completion", + passed=True, evaluation_id="custom-eval-id-42", ) - attrs = _only_record(capture).__dict__ - assert attrs["microsoft.human_evaluation.id"] == "custom-eval-id-42" - # And explicitly: id must not also be inside internal_properties. - decoded = json.loads(attrs["internal_properties"]) - assert "microsoft.human_evaluation.id" not in decoded + attrs = _only_attrs(capture) + assert attrs["microsoft.evaluation.id"] == "custom-eval-id-42" def test_explanation_flows_through_as_top_level_attribute(capture): - emit_human_evaluation_event( - evaluation_name="task_completion", - score_value=1.0, - kind="binary", + emit_boolean_evaluation( + evaluation_metric_name="task_completion", + passed=True, explanation="The agent answered correctly.", ) - record = _only_record(capture) - assert record.__dict__["gen_ai.evaluation.explanation"] == "The agent answered correctly." + attrs = _only_attrs(capture) + assert attrs["gen_ai.evaluation.explanation"] == "The agent answered correctly." def test_explanation_omitted_omits_attribute(capture): - emit_human_evaluation_event(evaluation_name="task_completion", score_value=1.0, kind="binary") - record = _only_record(capture) - assert "gen_ai.evaluation.explanation" not in record.__dict__ + emit_boolean_evaluation(evaluation_metric_name="task_completion", passed=True) + attrs = _only_attrs(capture) + assert "gen_ai.evaluation.explanation" not in attrs From ef1c9aa1bac321fdec3b93f1ba70511ade2c9d6d Mon Sep 17 00:00:00 2001 From: Sean Gayler Date: Tue, 26 May 2026 13:47:50 -0700 Subject: [PATCH 08/11] use more suitable tag --- .../samples/evaluations/sample_human_evaluations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_human_evaluations.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_human_evaluations.py index 422b91a79e5a..d869e6b50240 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_human_evaluations.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_human_evaluations.py @@ -297,7 +297,7 @@ def emit_5_point_ordinal_evaluation( response_id="resp_64904952b20872620069f8d600779c81908f58b0a3be090ef0", project_resource_id=project_resource_id, enduser_pseudo_id="sess_123456", - tags={"subscription_tier": "basic_plan"}, + tags={"subscription_tier": "free_plan"}, evaluation_id="0b27be45-cd65-4671-ab08-c3eafd4c9613", ) print("Emitted boolean human evaluation event.") From bda6123f843f76f8b1758ea97fe469deb3a37fd3 Mon Sep 17 00:00:00 2001 From: Sean Gayler Date: Wed, 27 May 2026 16:45:48 -0700 Subject: [PATCH 09/11] Update human evaluations sample spec Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../evaluations/sample_human_evaluations.py | 33 ++++++++++-------- .../evaluations/test_human_evaluations.py | 34 +++++++++++++------ 2 files changed, 41 insertions(+), 26 deletions(-) diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_human_evaluations.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_human_evaluations.py index d869e6b50240..ef34f32b438a 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_human_evaluations.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_human_evaluations.py @@ -62,6 +62,7 @@ EvaluationType = Literal["boolean", "ordinal"] DesirableDirection = Literal["increase", "decrease"] + def _validate_score( *, score_value: float, @@ -140,23 +141,23 @@ def _emit_human_evaluation( "gen_ai.evaluation.name": evaluation_metric_name, "gen_ai.evaluation.score.value": score_value, "gen_ai.evaluation.score.label": score_label, - "microsoft.human_evaluation.source": "end_user", + "microsoft.gen_ai.human_evaluation.source": "end_user", + "microsoft.gen_ai.evaluation.actor.type": "human", "internal_properties": json.dumps(internal_properties), } if explanation is not None: attributes["gen_ai.evaluation.explanation"] = explanation if response_id is not None: attributes["gen_ai.response.id"] = response_id - attributes["microsoft.gen_ai.response.id.type"] = "responses" if enduser_id is not None: attributes["enduser.id"] = enduser_id if enduser_pseudo_id is not None: attributes["enduser.pseudo.id"] = enduser_pseudo_id if tags: for tag_name, tag_value in tags.items(): - attributes[f"microsoft.evaluation.tags.{tag_name}"] = tag_value + attributes[f"microsoft.gen_ai.evaluation.tags.{tag_name}"] = tag_value if evaluation_id is not None: - attributes["microsoft.evaluation.id"] = evaluation_id + attributes["microsoft.gen_ai.human_evaluation.id"] = evaluation_id logger.info("gen_ai.evaluation.result", extra=attributes) @@ -186,11 +187,9 @@ def emit_boolean_evaluation( explanation: Optional free-form explanation from the end user. response_id: Optional OpenAI Responses API response ID being evaluated. project_resource_id: Optional ARM resource ID for the Foundry project. - enduser_id: Optional signed-in end-user ID. This may contain PII and maps - to `user_AuthenticatedId` in Application Insights. - enduser_pseudo_id: Optional pseudonymous end-user ID. This maps to - `user_Id` in Application Insights. - tags: Optional metadata emitted as `microsoft.evaluation.tags.`. + enduser_id: Optional signed-in end-user ID. This may contain PII. + enduser_pseudo_id: Optional pseudonymous end-user ID. + tags: Optional metadata associated with the evaluation. evaluation_id: Optional ID for the evaluation event itself. """ _emit_human_evaluation( @@ -215,6 +214,7 @@ def emit_5_point_ordinal_evaluation( *, evaluation_metric_name: str, score_value: float, + threshold: float = 3.0, explanation: Optional[str] = None, response_id: Optional[str] = None, project_resource_id: Optional[str] = None, @@ -232,23 +232,26 @@ def emit_5_point_ordinal_evaluation( evaluation_metric_name: Name of the evaluated metric, such as `"relevance"` or `"helpfulness"`. score_value: Integer score from `1.0` through `5.0`. + threshold: Score at or above this value is passing. explanation: Optional free-form explanation from the end user. response_id: Optional OpenAI Responses API response ID being evaluated. project_resource_id: Optional ARM resource ID for the Foundry project. - enduser_id: Optional signed-in end-user ID. This may contain PII and maps - to `user_AuthenticatedId` in Application Insights. - enduser_pseudo_id: Optional pseudonymous end-user ID. This maps to - `user_Id` in Application Insights. - tags: Optional metadata emitted as `microsoft.evaluation.tags.`. + enduser_id: Optional signed-in end-user ID. This may contain PII. + enduser_pseudo_id: Optional pseudonymous end-user ID. + tags: Optional metadata associated with the evaluation. evaluation_id: Optional ID for the evaluation event itself. """ + threshold = float(threshold) + if not 1.0 <= threshold <= 5.0: + raise ValueError(f"threshold {threshold} is outside the allowed range [1.0, 5.0].") + _emit_human_evaluation( evaluation_metric_name=evaluation_metric_name, score_value=score_value, evaluation_type="ordinal", min_value=1.0, max_value=5.0, - threshold=3.0, + threshold=threshold, desirable_direction="increase", explanation=explanation, response_id=response_id, diff --git a/sdk/ai/azure-ai-projects/tests/evaluations/test_human_evaluations.py b/sdk/ai/azure-ai-projects/tests/evaluations/test_human_evaluations.py index cb2f6e660d63..56d12cae3214 100644 --- a/sdk/ai/azure-ai-projects/tests/evaluations/test_human_evaluations.py +++ b/sdk/ai/azure-ai-projects/tests/evaluations/test_human_evaluations.py @@ -78,6 +78,14 @@ def test_ordinal_score_emits_expected_label(capture, score_value, expected_label assert attrs["gen_ai.evaluation.score.value"] == score_value +def test_ordinal_custom_threshold_controls_label_and_flows_to_internal_properties(capture): + emit_5_point_ordinal_evaluation(evaluation_metric_name="relevance", score_value=2.0, threshold=4.0) + attrs = _only_attrs(capture) + decoded = _internal_properties(attrs) + assert attrs["gen_ai.evaluation.score.label"] == "fail" + assert decoded["gen_ai.evaluation.threshold"] == "4.0" + + def test_ordinal_non_integer_score_raises(): with pytest.raises(ValueError): emit_5_point_ordinal_evaluation(evaluation_metric_name="relevance", score_value=2.5) @@ -89,6 +97,12 @@ def test_ordinal_score_out_of_range_raises(score_value): emit_5_point_ordinal_evaluation(evaluation_metric_name="relevance", score_value=score_value) +@pytest.mark.parametrize("threshold", [0.0, 6.0]) +def test_ordinal_threshold_out_of_range_raises(threshold): + with pytest.raises(ValueError): + emit_5_point_ordinal_evaluation(evaluation_metric_name="relevance", score_value=4.0, threshold=threshold) + + def test_top_level_attributes_have_canonical_keys_and_routing(capture): emit_boolean_evaluation(evaluation_metric_name="task_completion", passed=True) attrs = _only_attrs(capture) @@ -96,7 +110,8 @@ def test_top_level_attributes_have_canonical_keys_and_routing(capture): assert attrs["gen_ai.evaluation.name"] == "task_completion" assert attrs["gen_ai.evaluation.score.value"] == 1.0 assert attrs["gen_ai.evaluation.score.label"] == "pass" - assert attrs["microsoft.human_evaluation.source"] == "end_user" + assert attrs["microsoft.gen_ai.human_evaluation.source"] == "end_user" + assert attrs["microsoft.gen_ai.evaluation.actor.type"] == "human" assert "internal_properties" in attrs @@ -120,7 +135,7 @@ def test_internal_properties_ordinal_defaults(capture): assert decoded["gen_ai.evaluation.type"] == "ordinal" -def test_response_id_set_adds_top_level_id_and_response_id_type(capture): +def test_response_id_set_adds_top_level_id(capture): emit_boolean_evaluation( evaluation_metric_name="task_completion", passed=True, @@ -128,20 +143,17 @@ def test_response_id_set_adds_top_level_id_and_response_id_type(capture): ) attrs = _only_attrs(capture) assert attrs["gen_ai.response.id"] == "resp_abc123" - assert attrs["microsoft.gen_ai.response.id.type"] == "responses" -def test_response_id_omitted_omits_response_id_type(capture): +def test_response_id_omitted_omits_top_level_id(capture): emit_boolean_evaluation(evaluation_metric_name="task_completion", passed=True) attrs = _only_attrs(capture) assert "gen_ai.response.id" not in attrs - assert "microsoft.gen_ai.response.id.type" not in attrs def test_project_resource_id_set_added_to_internal_properties(capture): arm_id = ( - "/subscriptions/sub-id/resourceGroups/rg/providers/Microsoft.CognitiveServices" - "/accounts/acct/projects/proj" + "/subscriptions/sub-id/resourceGroups/rg/providers/Microsoft.CognitiveServices" "/accounts/acct/projects/proj" ) emit_boolean_evaluation( evaluation_metric_name="task_completion", @@ -199,14 +211,14 @@ def test_tags_fan_out_as_top_level_attributes(capture): tags={"subscription_tier": "basic_plan", "department": "marketing"}, ) attrs = _only_attrs(capture) - assert attrs["microsoft.evaluation.tags.subscription_tier"] == "basic_plan" - assert attrs["microsoft.evaluation.tags.department"] == "marketing" + assert attrs["microsoft.gen_ai.evaluation.tags.subscription_tier"] == "basic_plan" + assert attrs["microsoft.gen_ai.evaluation.tags.department"] == "marketing" def test_evaluation_id_omitted_omits_attribute(capture): emit_boolean_evaluation(evaluation_metric_name="task_completion", passed=True) attrs = _only_attrs(capture) - assert "microsoft.evaluation.id" not in attrs + assert "microsoft.gen_ai.human_evaluation.id" not in attrs def test_evaluation_id_provided_flows_through_verbatim_as_top_level_attribute(capture): @@ -216,7 +228,7 @@ def test_evaluation_id_provided_flows_through_verbatim_as_top_level_attribute(ca evaluation_id="custom-eval-id-42", ) attrs = _only_attrs(capture) - assert attrs["microsoft.evaluation.id"] == "custom-eval-id-42" + assert attrs["microsoft.gen_ai.human_evaluation.id"] == "custom-eval-id-42" def test_explanation_flows_through_as_top_level_attribute(capture): From ab7689118f357ba2876b844b675d6705f74c044a Mon Sep 17 00:00:00 2001 From: Sean Gayler Date: Wed, 27 May 2026 16:52:53 -0700 Subject: [PATCH 10/11] Add conversation ID to human evaluation sample Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../samples/evaluations/sample_human_evaluations.py | 11 +++++++++++ .../tests/evaluations/test_human_evaluations.py | 10 ++++++++++ 2 files changed, 21 insertions(+) diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_human_evaluations.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_human_evaluations.py index ef34f32b438a..14c46e1ec469 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_human_evaluations.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_human_evaluations.py @@ -108,6 +108,7 @@ def _emit_human_evaluation( desirable_direction: DesirableDirection, explanation: Optional[str] = None, response_id: Optional[str] = None, + conversation_id: Optional[str] = None, project_resource_id: Optional[str] = None, enduser_id: Optional[str] = None, enduser_pseudo_id: Optional[str] = None, @@ -149,6 +150,8 @@ def _emit_human_evaluation( attributes["gen_ai.evaluation.explanation"] = explanation if response_id is not None: attributes["gen_ai.response.id"] = response_id + if conversation_id is not None: + attributes["gen_ai.conversation.id"] = conversation_id if enduser_id is not None: attributes["enduser.id"] = enduser_id if enduser_pseudo_id is not None: @@ -168,6 +171,7 @@ def emit_boolean_evaluation( passed: bool, explanation: Optional[str] = None, response_id: Optional[str] = None, + conversation_id: Optional[str] = None, project_resource_id: Optional[str] = None, enduser_id: Optional[str] = None, enduser_pseudo_id: Optional[str] = None, @@ -186,6 +190,7 @@ def emit_boolean_evaluation( passed: Whether the human evaluation passed. explanation: Optional free-form explanation from the end user. response_id: Optional OpenAI Responses API response ID being evaluated. + conversation_id: Optional conversation ID associated with the evaluation. project_resource_id: Optional ARM resource ID for the Foundry project. enduser_id: Optional signed-in end-user ID. This may contain PII. enduser_pseudo_id: Optional pseudonymous end-user ID. @@ -202,6 +207,7 @@ def emit_boolean_evaluation( desirable_direction="increase", explanation=explanation, response_id=response_id, + conversation_id=conversation_id, project_resource_id=project_resource_id, enduser_id=enduser_id, enduser_pseudo_id=enduser_pseudo_id, @@ -217,6 +223,7 @@ def emit_5_point_ordinal_evaluation( threshold: float = 3.0, explanation: Optional[str] = None, response_id: Optional[str] = None, + conversation_id: Optional[str] = None, project_resource_id: Optional[str] = None, enduser_id: Optional[str] = None, enduser_pseudo_id: Optional[str] = None, @@ -235,6 +242,7 @@ def emit_5_point_ordinal_evaluation( threshold: Score at or above this value is passing. explanation: Optional free-form explanation from the end user. response_id: Optional OpenAI Responses API response ID being evaluated. + conversation_id: Optional conversation ID associated with the evaluation. project_resource_id: Optional ARM resource ID for the Foundry project. enduser_id: Optional signed-in end-user ID. This may contain PII. enduser_pseudo_id: Optional pseudonymous end-user ID. @@ -255,6 +263,7 @@ def emit_5_point_ordinal_evaluation( desirable_direction="increase", explanation=explanation, response_id=response_id, + conversation_id=conversation_id, project_resource_id=project_resource_id, enduser_id=enduser_id, enduser_pseudo_id=enduser_pseudo_id, @@ -298,6 +307,7 @@ def emit_5_point_ordinal_evaluation( passed=True, explanation="The agent provided accurate weather information as requested.", response_id="resp_64904952b20872620069f8d600779c81908f58b0a3be090ef0", + conversation_id="conv_5j66UpCpwteGg4YSxUnt7lPY", project_resource_id=project_resource_id, enduser_pseudo_id="sess_123456", tags={"subscription_tier": "free_plan"}, @@ -314,6 +324,7 @@ def emit_5_point_ordinal_evaluation( "information that addresses the user's intent." ), response_id="resp_64904952b20872620069f8d600779c81908f58b0a3be090ef0", + conversation_id="conv_5j66UpCpwteGg4YSxUnt7lPY", project_resource_id=project_resource_id, enduser_id="oid:241964ad-a8db-4318-9f2e-5a7dc1f05349", tags={"department": "marketing"}, diff --git a/sdk/ai/azure-ai-projects/tests/evaluations/test_human_evaluations.py b/sdk/ai/azure-ai-projects/tests/evaluations/test_human_evaluations.py index 56d12cae3214..1b6be3ad66eb 100644 --- a/sdk/ai/azure-ai-projects/tests/evaluations/test_human_evaluations.py +++ b/sdk/ai/azure-ai-projects/tests/evaluations/test_human_evaluations.py @@ -151,6 +151,16 @@ def test_response_id_omitted_omits_top_level_id(capture): assert "gen_ai.response.id" not in attrs +def test_conversation_id_set_adds_top_level_id(capture): + emit_boolean_evaluation( + evaluation_metric_name="task_completion", + passed=True, + conversation_id="conv_abc123", + ) + attrs = _only_attrs(capture) + assert attrs["gen_ai.conversation.id"] == "conv_abc123" + + def test_project_resource_id_set_added_to_internal_properties(capture): arm_id = ( "/subscriptions/sub-id/resourceGroups/rg/providers/Microsoft.CognitiveServices" "/accounts/acct/projects/proj" From 4a7cb04695f6337f5e014d4908eacbc42e2bbb01 Mon Sep 17 00:00:00 2001 From: Sean Gayler Date: Wed, 27 May 2026 17:12:23 -0700 Subject: [PATCH 11/11] Add trace context to human evaluation sample Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../evaluations/sample_human_evaluations.py | 99 +++++++++++++++++-- .../evaluations/test_human_evaluations.py | 87 ++++++++++++++-- 2 files changed, 169 insertions(+), 17 deletions(-) diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_human_evaluations.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_human_evaluations.py index 14c46e1ec469..3124c74eb9ee 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_human_evaluations.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_human_evaluations.py @@ -47,9 +47,9 @@ # azure-monitor-opentelemetry, azure-ai-projects, python-dotenv) are # intentionally deferred into the `if __name__ == "__main__":` block at the # bottom of this file. They are only needed when running the sample directly; -# the helper functions themselves depend only on the standard library and -# `typing`, which keeps them importable in test environments that do not install -# the full OTel stack. +# the helper functions remain importable in test environments that do not install +# the full OTel stack. The trace-context imports are only needed when a saved +# trace ID and span ID are provided. # `configure_azure_monitor` (called in __main__) installs an OpenTelemetry # LoggingHandler on the root logger, so any standard Python `logging` call below @@ -63,6 +63,63 @@ DesirableDirection = Literal["increase", "decrease"] +def _validate_hex_id(*, name: str, value: str, length: int) -> int: + if len(value) != length: + raise ValueError(f"{name} must be a {length}-character hexadecimal string.") + + try: + parsed_value = int(value, 16) + except ValueError as exc: + raise ValueError( + f"{name} must be a {length}-character hexadecimal string." + ) from exc + + if parsed_value == 0: + raise ValueError(f"{name} cannot be all zeros.") + + return parsed_value + + +def _log_evaluation_event( + *, + attributes: Mapping[str, object], + trace_id: Optional[str] = None, + span_id: Optional[str] = None, +) -> None: + if trace_id is None and span_id is None: + logger.info("gen_ai.evaluation.result", extra=attributes) + return + if trace_id is None or span_id is None: + raise ValueError("trace_id and span_id must be provided together.") + + trace_id_int = _validate_hex_id(name="trace_id", value=trace_id, length=32) + span_id_int = _validate_hex_id(name="span_id", value=span_id, length=16) + + from opentelemetry import context as otel_context + from opentelemetry import trace + from opentelemetry.trace import ( + NonRecordingSpan, + SpanContext, + TraceFlags, + TraceState, + ) + + span_context = SpanContext( + trace_id=trace_id_int, + span_id=span_id_int, + is_remote=True, + trace_flags=TraceFlags(TraceFlags.SAMPLED), + trace_state=TraceState(), + ) + token = otel_context.attach( + trace.set_span_in_context(NonRecordingSpan(span_context)) + ) + try: + logger.info("gen_ai.evaluation.result", extra=attributes) + finally: + otel_context.detach(token) + + def _validate_score( *, score_value: float, @@ -114,6 +171,8 @@ def _emit_human_evaluation( enduser_pseudo_id: Optional[str] = None, tags: Optional[Mapping[str, str]] = None, evaluation_id: Optional[str] = None, + trace_id: Optional[str] = None, + span_id: Optional[str] = None, ) -> None: score_value = _validate_score( score_value=score_value, @@ -162,7 +221,7 @@ def _emit_human_evaluation( if evaluation_id is not None: attributes["microsoft.gen_ai.human_evaluation.id"] = evaluation_id - logger.info("gen_ai.evaluation.result", extra=attributes) + _log_evaluation_event(attributes=attributes, trace_id=trace_id, span_id=span_id) def emit_boolean_evaluation( @@ -177,6 +236,8 @@ def emit_boolean_evaluation( enduser_pseudo_id: Optional[str] = None, tags: Optional[Mapping[str, str]] = None, evaluation_id: Optional[str] = None, + trace_id: Optional[str] = None, + span_id: Optional[str] = None, ) -> None: """Emit a boolean human evaluation event. @@ -196,6 +257,8 @@ def emit_boolean_evaluation( enduser_pseudo_id: Optional pseudonymous end-user ID. tags: Optional metadata associated with the evaluation. evaluation_id: Optional ID for the evaluation event itself. + trace_id: Optional trace ID captured when the evaluated response was created. + span_id: Optional span ID captured when the evaluated response was created. """ _emit_human_evaluation( evaluation_metric_name=evaluation_metric_name, @@ -213,6 +276,8 @@ def emit_boolean_evaluation( enduser_pseudo_id=enduser_pseudo_id, tags=tags, evaluation_id=evaluation_id, + trace_id=trace_id, + span_id=span_id, ) @@ -229,6 +294,8 @@ def emit_5_point_ordinal_evaluation( enduser_pseudo_id: Optional[str] = None, tags: Optional[Mapping[str, str]] = None, evaluation_id: Optional[str] = None, + trace_id: Optional[str] = None, + span_id: Optional[str] = None, ) -> None: """Emit a 5-point ordinal human evaluation event. @@ -248,10 +315,14 @@ def emit_5_point_ordinal_evaluation( enduser_pseudo_id: Optional pseudonymous end-user ID. tags: Optional metadata associated with the evaluation. evaluation_id: Optional ID for the evaluation event itself. + trace_id: Optional trace ID captured when the evaluated response was created. + span_id: Optional span ID captured when the evaluated response was created. """ threshold = float(threshold) if not 1.0 <= threshold <= 5.0: - raise ValueError(f"threshold {threshold} is outside the allowed range [1.0, 5.0].") + raise ValueError( + f"threshold {threshold} is outside the allowed range [1.0, 5.0]." + ) _emit_human_evaluation( evaluation_metric_name=evaluation_metric_name, @@ -269,6 +340,8 @@ def emit_5_point_ordinal_evaluation( enduser_pseudo_id=enduser_pseudo_id, tags=tags, evaluation_id=evaluation_id, + trace_id=trace_id, + span_id=span_id, ) @@ -291,7 +364,9 @@ def emit_5_point_ordinal_evaluation( # Pull the Application Insights connection string attached to your Foundry # project and wire OpenTelemetry up to it. All `logger.info(...)` calls # below will be exported to Application Insights. - connection_string = project_client.telemetry.get_application_insights_connection_string() + connection_string = ( + project_client.telemetry.get_application_insights_connection_string() + ) configure_azure_monitor(connection_string=connection_string) @@ -299,7 +374,13 @@ def emit_5_point_ordinal_evaluation( # The endpoint URL alone only gives us the account + project names, but # every Connection's `id` is a full ARM path ending with /connections/. any_connection = next(iter(project_client.connections.list()), None) - project_resource_id = any_connection.id.rsplit("/connections/", 1)[0] if any_connection else None + project_resource_id = ( + any_connection.id.rsplit("/connections/", 1)[0] if any_connection else None + ) + + # Sample trace and span IDs for demonstration purposes. + trace_id = "4bf92f3577b34da6a3ce929d0e0e4736" + span_id = "00f067aa0ba902b7" # Example 1: an anonymous end user gives a thumbs up on task completion. emit_boolean_evaluation( @@ -312,6 +393,8 @@ def emit_5_point_ordinal_evaluation( enduser_pseudo_id="sess_123456", tags={"subscription_tier": "free_plan"}, evaluation_id="0b27be45-cd65-4671-ab08-c3eafd4c9613", + trace_id=trace_id, + span_id=span_id, ) print("Emitted boolean human evaluation event.") @@ -329,5 +412,7 @@ def emit_5_point_ordinal_evaluation( enduser_id="oid:241964ad-a8db-4318-9f2e-5a7dc1f05349", tags={"department": "marketing"}, evaluation_id="69d937a7-32e2-412e-97c9-119e2d282723", + trace_id=trace_id, + span_id=span_id, ) print("Emitted 5-point ordinal human evaluation event.") diff --git a/sdk/ai/azure-ai-projects/tests/evaluations/test_human_evaluations.py b/sdk/ai/azure-ai-projects/tests/evaluations/test_human_evaluations.py index 1b6be3ad66eb..02a6626f42fd 100644 --- a/sdk/ai/azure-ai-projects/tests/evaluations/test_human_evaluations.py +++ b/sdk/ai/azure-ai-projects/tests/evaluations/test_human_evaluations.py @@ -10,10 +10,15 @@ import pytest -SAMPLES_EVALUATIONS_DIR = Path(__file__).resolve().parents[1] / ".." / "samples" / "evaluations" +SAMPLES_EVALUATIONS_DIR = ( + Path(__file__).resolve().parents[1] / ".." / "samples" / "evaluations" +) sys.path.insert(0, str(SAMPLES_EVALUATIONS_DIR.resolve())) -from sample_human_evaluations import emit_5_point_ordinal_evaluation, emit_boolean_evaluation # noqa: E402 +from sample_human_evaluations import ( + emit_5_point_ordinal_evaluation, + emit_boolean_evaluation, +) # noqa: E402 class _RecordCapture(logging.Handler): @@ -39,7 +44,9 @@ def capture(): def _only_attrs(capture: _RecordCapture) -> dict: - assert len(capture.records) == 1, f"expected exactly 1 emitted record, got {len(capture.records)}" + assert ( + len(capture.records) == 1 + ), f"expected exactly 1 emitted record, got {len(capture.records)}" return capture.records[0].__dict__ @@ -72,14 +79,20 @@ def test_boolean_passed_emits_score_1_with_pass_label(capture): ], ) def test_ordinal_score_emits_expected_label(capture, score_value, expected_label): - emit_5_point_ordinal_evaluation(evaluation_metric_name="relevance", score_value=score_value) + emit_5_point_ordinal_evaluation( + evaluation_metric_name="relevance", score_value=score_value + ) attrs = _only_attrs(capture) assert attrs["gen_ai.evaluation.score.label"] == expected_label assert attrs["gen_ai.evaluation.score.value"] == score_value -def test_ordinal_custom_threshold_controls_label_and_flows_to_internal_properties(capture): - emit_5_point_ordinal_evaluation(evaluation_metric_name="relevance", score_value=2.0, threshold=4.0) +def test_ordinal_custom_threshold_controls_label_and_flows_to_internal_properties( + capture, +): + emit_5_point_ordinal_evaluation( + evaluation_metric_name="relevance", score_value=2.0, threshold=4.0 + ) attrs = _only_attrs(capture) decoded = _internal_properties(attrs) assert attrs["gen_ai.evaluation.score.label"] == "fail" @@ -88,19 +101,25 @@ def test_ordinal_custom_threshold_controls_label_and_flows_to_internal_propertie def test_ordinal_non_integer_score_raises(): with pytest.raises(ValueError): - emit_5_point_ordinal_evaluation(evaluation_metric_name="relevance", score_value=2.5) + emit_5_point_ordinal_evaluation( + evaluation_metric_name="relevance", score_value=2.5 + ) @pytest.mark.parametrize("score_value", [0.0, 6.0]) def test_ordinal_score_out_of_range_raises(score_value): with pytest.raises(ValueError): - emit_5_point_ordinal_evaluation(evaluation_metric_name="relevance", score_value=score_value) + emit_5_point_ordinal_evaluation( + evaluation_metric_name="relevance", score_value=score_value + ) @pytest.mark.parametrize("threshold", [0.0, 6.0]) def test_ordinal_threshold_out_of_range_raises(threshold): with pytest.raises(ValueError): - emit_5_point_ordinal_evaluation(evaluation_metric_name="relevance", score_value=4.0, threshold=threshold) + emit_5_point_ordinal_evaluation( + evaluation_metric_name="relevance", score_value=4.0, threshold=threshold + ) def test_top_level_attributes_have_canonical_keys_and_routing(capture): @@ -161,9 +180,57 @@ def test_conversation_id_set_adds_top_level_id(capture): assert attrs["gen_ai.conversation.id"] == "conv_abc123" +def test_saved_trace_context_is_current_while_emitting(capture): + from opentelemetry import trace + + trace_id = "4bf92f3577b34da6a3ce929d0e0e4736" + span_id = "00f067aa0ba902b7" + captured_contexts = [] + + class _ContextCapture(logging.Handler): + def emit(self, record: logging.LogRecord) -> None: + captured_contexts.append(trace.get_current_span().get_span_context()) + + logger = logging.getLogger("human_evaluations") + handler = _ContextCapture() + logger.addHandler(handler) + try: + emit_boolean_evaluation( + evaluation_metric_name="task_completion", + passed=True, + trace_id=trace_id, + span_id=span_id, + ) + finally: + logger.removeHandler(handler) + + _only_attrs(capture) + assert len(captured_contexts) == 1 + assert captured_contexts[0].trace_id == int(trace_id, 16) + assert captured_contexts[0].span_id == int(span_id, 16) + + +@pytest.mark.parametrize( + "trace_id, span_id", + [ + ("4bf92f3577b34da6a3ce929d0e0e4736", None), + (None, "00f067aa0ba902b7"), + ], +) +def test_trace_id_and_span_id_must_be_provided_together(trace_id, span_id): + with pytest.raises(ValueError): + emit_boolean_evaluation( + evaluation_metric_name="task_completion", + passed=True, + trace_id=trace_id, + span_id=span_id, + ) + + def test_project_resource_id_set_added_to_internal_properties(capture): arm_id = ( - "/subscriptions/sub-id/resourceGroups/rg/providers/Microsoft.CognitiveServices" "/accounts/acct/projects/proj" + "/subscriptions/sub-id/resourceGroups/rg/providers/Microsoft.CognitiveServices" + "/accounts/acct/projects/proj" ) emit_boolean_evaluation( evaluation_metric_name="task_completion",