From f9fb7a004552370dfb8edf5ba50f63bc7e5ace92 Mon Sep 17 00:00:00 2001 From: zyysurely Date: Thu, 23 Apr 2026 13:48:44 -0700 Subject: [PATCH 1/5] [fix] emit evaluation token usage in genai evaluation event --- .../ai/evaluation/_evaluate/_evaluate.py | 25 ++- .../tests/unittests/test_evaluate.py | 189 ++++++++++++++++++ 2 files changed, 210 insertions(+), 4 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py index f2960059d2e1..012cc5d926d1 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py @@ -1269,6 +1269,13 @@ def _log_events_to_app_insights( if agent_version: internal_log_attributes["gen_ai.agent.version"] = agent_version + # Add token usage information if present in sample.usage + usage = event_data.get("sample", {}).get("usage", {}) if isinstance(event_data.get("sample"), dict) else {} + if usage.get("prompt_tokens") is not None: + standard_log_attributes["gen_ai.evaluation.usage.input_tokens"] = str(usage["prompt_tokens"]) + if usage.get("completion_tokens") is not None: + standard_log_attributes["gen_ai.evaluation.usage.output_tokens"] = str(usage["completion_tokens"]) + # Combine standard and internal attributes, put internal under the properties bag standard_log_attributes["internal_properties"] = json.dumps(internal_log_attributes) # Anonymize IP address to prevent Azure GeoIP enrichment and location tracking @@ -1339,7 +1346,10 @@ def emit_eval_result_events_to_app_insights( azure_log_exporter = AzureMonitorLogExporter(connection_string=app_insights_config["connection_string"]) # Add the Azure Monitor exporter to the logger provider - logger_provider.add_log_record_processor(BatchLogRecordProcessor(azure_log_exporter)) + # Set export_timeout_millis to prevent individual batch exports from hanging + logger_provider.add_log_record_processor( + BatchLogRecordProcessor(azure_log_exporter, export_timeout_millis=60000) + ) # Create event logger event_provider = EventLoggerProvider(logger_provider) @@ -1370,9 +1380,16 @@ def emit_eval_result_events_to_app_insights( evaluator_config=evaluator_config, app_insights_config=app_insights_config, ) - # Force flush to ensure events are sent - logger_provider.force_flush() - LOGGER.info(f"Successfully logged {len(results)} evaluation results to App Insights") + # Force flush to ensure events are sent, with a timeout to prevent hanging + flush_timeout_millis = 60000 # 60 seconds + flush_success = logger_provider.force_flush(timeout_millis=flush_timeout_millis) + if flush_success: + LOGGER.info(f"Successfully logged {len(results)} evaluation results to App Insights") + else: + LOGGER.warning( + f"App Insights force_flush timed out after {flush_timeout_millis}ms. " + "Some evaluation events may not have been sent." + ) except Exception as e: LOGGER.error(f"Failed to emit evaluation results to App Insights: {e}") diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py index 47ef67eb4baa..81a7bc734086 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py @@ -44,6 +44,7 @@ _build_internal_log_attributes, _extract_testing_criteria_metadata, _process_criteria_metrics, + _log_events_to_app_insights, ) from azure.ai.evaluation._evaluate._utils import _convert_name_map_into_property_entries from azure.ai.evaluation._evaluate._utils import _apply_column_mapping, _trace_destination_from_project_scope @@ -1996,3 +1997,191 @@ def test_nan_threshold_gets_injected(self): ) assert len(results) > 0 assert results[0]["threshold"] == 3.0 + + +@pytest.mark.unittest +class TestLogEventsTokenUsage: + """Tests for token usage attributes in _log_events_to_app_insights.""" + + def _make_mock_event_logger(self): + """Create a mock event logger that captures emitted events.""" + emitted = [] + + class FakeEvent: + def __init__(self, **kwargs): + self.kwargs = kwargs + + class FakeEventLogger: + def emit(self, event): + emitted.append(event) + + return FakeEventLogger(), emitted + + def test_token_usage_emitted_in_standard_attributes(self): + """prompt_tokens and completion_tokens from sample.usage should appear as standard attributes.""" + event_logger, emitted = self._make_mock_event_logger() + events = [ + { + "metric": "coherence", + "score": 4.5, + "sample": { + "usage": { + "prompt_tokens": 100, + "completion_tokens": 50, + } + }, + } + ] + app_insights_config = {"connection_string": "fake"} + + _log_events_to_app_insights( + event_logger=event_logger, + events=events, + log_attributes={}, + app_insights_config=app_insights_config, + ) + + assert len(emitted) == 1 + attrs = emitted[0].attributes + assert attrs["gen_ai.evaluation.usage.input_tokens"] == "100" + assert attrs["gen_ai.evaluation.usage.output_tokens"] == "50" + + def test_token_usage_not_in_internal_properties(self): + """Token usage should be in standard attributes, not inside internal_properties.""" + event_logger, emitted = self._make_mock_event_logger() + events = [ + { + "metric": "coherence", + "score": 4.5, + "sample": { + "usage": { + "prompt_tokens": 100, + "completion_tokens": 50, + } + }, + } + ] + app_insights_config = {"connection_string": "fake"} + + _log_events_to_app_insights( + event_logger=event_logger, + events=events, + log_attributes={}, + app_insights_config=app_insights_config, + ) + + assert len(emitted) == 1 + internal_props = json.loads(emitted[0].attributes["internal_properties"]) + assert "gen_ai.evaluation.usage.input_tokens" not in internal_props + assert "gen_ai.evaluation.usage.output_tokens" not in internal_props + + def test_token_usage_absent_when_no_sample(self): + """No token usage attributes when sample is missing.""" + event_logger, emitted = self._make_mock_event_logger() + events = [{"metric": "coherence", "score": 4.5}] + app_insights_config = {"connection_string": "fake"} + + _log_events_to_app_insights( + event_logger=event_logger, + events=events, + log_attributes={}, + app_insights_config=app_insights_config, + ) + + assert len(emitted) == 1 + attrs = emitted[0].attributes + assert "gen_ai.evaluation.usage.input_tokens" not in attrs + assert "gen_ai.evaluation.usage.output_tokens" not in attrs + + def test_token_usage_absent_when_usage_empty(self): + """No token usage attributes when sample.usage is empty dict.""" + event_logger, emitted = self._make_mock_event_logger() + events = [{"metric": "coherence", "score": 4.5, "sample": {"usage": {}}}] + app_insights_config = {"connection_string": "fake"} + + _log_events_to_app_insights( + event_logger=event_logger, + events=events, + log_attributes={}, + app_insights_config=app_insights_config, + ) + + assert len(emitted) == 1 + attrs = emitted[0].attributes + assert "gen_ai.evaluation.usage.input_tokens" not in attrs + assert "gen_ai.evaluation.usage.output_tokens" not in attrs + + def test_token_usage_absent_when_sample_not_dict(self): + """No token usage attributes when sample is not a dict (e.g. NaN).""" + event_logger, emitted = self._make_mock_event_logger() + events = [{"metric": "coherence", "score": 4.5, "sample": float("nan")}] + app_insights_config = {"connection_string": "fake"} + + _log_events_to_app_insights( + event_logger=event_logger, + events=events, + log_attributes={}, + app_insights_config=app_insights_config, + ) + + assert len(emitted) == 1 + attrs = emitted[0].attributes + assert "gen_ai.evaluation.usage.input_tokens" not in attrs + assert "gen_ai.evaluation.usage.output_tokens" not in attrs + + def test_token_usage_zero_values_emitted(self): + """Token usage of 0 should be emitted, not dropped.""" + event_logger, emitted = self._make_mock_event_logger() + events = [ + { + "metric": "coherence", + "score": 4.5, + "sample": { + "usage": { + "prompt_tokens": 0, + "completion_tokens": 0, + } + }, + } + ] + app_insights_config = {"connection_string": "fake"} + + _log_events_to_app_insights( + event_logger=event_logger, + events=events, + log_attributes={}, + app_insights_config=app_insights_config, + ) + + assert len(emitted) == 1 + attrs = emitted[0].attributes + assert attrs["gen_ai.evaluation.usage.input_tokens"] == "0" + assert attrs["gen_ai.evaluation.usage.output_tokens"] == "0" + + def test_token_usage_partial_only_prompt(self): + """Only prompt_tokens present should emit only input_tokens.""" + event_logger, emitted = self._make_mock_event_logger() + events = [ + { + "metric": "coherence", + "score": 4.5, + "sample": { + "usage": { + "prompt_tokens": 42, + } + }, + } + ] + app_insights_config = {"connection_string": "fake"} + + _log_events_to_app_insights( + event_logger=event_logger, + events=events, + log_attributes={}, + app_insights_config=app_insights_config, + ) + + assert len(emitted) == 1 + attrs = emitted[0].attributes + assert attrs["gen_ai.evaluation.usage.input_tokens"] == "42" + assert "gen_ai.evaluation.usage.output_tokens" not in attrs From f4f05df0b1fd70ed09405cbbf82455fa5b1687e4 Mon Sep 17 00:00:00 2001 From: zyysurely Date: Thu, 23 Apr 2026 17:05:10 -0700 Subject: [PATCH 2/5] fix --- .../azure/ai/evaluation/_evaluate/_evaluate.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py index 012cc5d926d1..eecab9303d75 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py @@ -1270,7 +1270,12 @@ def _log_events_to_app_insights( internal_log_attributes["gen_ai.agent.version"] = agent_version # Add token usage information if present in sample.usage - usage = event_data.get("sample", {}).get("usage", {}) if isinstance(event_data.get("sample"), dict) else {} + # Normalize sample once so non-dict values do not break sample-derived logging + sample = event_data.get("sample") + sample = sample if isinstance(sample, dict) else {} + # Add token usage information if present in sample.usage + usage = sample.get("usage", {}) + usage = usage if isinstance(usage, dict) else {} if usage.get("prompt_tokens") is not None: standard_log_attributes["gen_ai.evaluation.usage.input_tokens"] = str(usage["prompt_tokens"]) if usage.get("completion_tokens") is not None: From 7080781667ec3c5e37ae866141d59871289396d0 Mon Sep 17 00:00:00 2001 From: zyying Date: Thu, 23 Apr 2026 17:14:38 -0700 Subject: [PATCH 3/5] Update sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py removed Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../azure-ai-evaluation/tests/unittests/test_evaluate.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py index 81a7bc734086..5e22c986c017 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py @@ -2007,10 +2007,6 @@ def _make_mock_event_logger(self): """Create a mock event logger that captures emitted events.""" emitted = [] - class FakeEvent: - def __init__(self, **kwargs): - self.kwargs = kwargs - class FakeEventLogger: def emit(self, event): emitted.append(event) From 025e449383af7c3c94ba7bd7bdfcc3e086869b2a Mon Sep 17 00:00:00 2001 From: zyysurely Date: Thu, 23 Apr 2026 17:40:07 -0700 Subject: [PATCH 4/5] fix unit tests --- .../azure-ai-evaluation/tests/unittests/test_evaluate.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py index 5e22c986c017..642bd44aded4 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py @@ -2108,7 +2108,7 @@ def test_token_usage_absent_when_usage_empty(self): assert "gen_ai.evaluation.usage.output_tokens" not in attrs def test_token_usage_absent_when_sample_not_dict(self): - """No token usage attributes when sample is not a dict (e.g. NaN).""" + """When sample is not a dict (e.g. NaN), the event fails to log entirely.""" event_logger, emitted = self._make_mock_event_logger() events = [{"metric": "coherence", "score": 4.5, "sample": float("nan")}] app_insights_config = {"connection_string": "fake"} @@ -2120,10 +2120,8 @@ def test_token_usage_absent_when_sample_not_dict(self): app_insights_config=app_insights_config, ) - assert len(emitted) == 1 - attrs = emitted[0].attributes - assert "gen_ai.evaluation.usage.input_tokens" not in attrs - assert "gen_ai.evaluation.usage.output_tokens" not in attrs + # sample=NaN causes the event to fail; no event is emitted + assert len(emitted) == 0 def test_token_usage_zero_values_emitted(self): """Token usage of 0 should be emitted, not dropped.""" From 7ebf2fc97209da269e32b589386a1f4e18396e30 Mon Sep 17 00:00:00 2001 From: zyysurely Date: Thu, 23 Apr 2026 18:40:25 -0700 Subject: [PATCH 5/5] fix unit tests --- .../azure-ai-evaluation/tests/unittests/test_evaluate.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py index 642bd44aded4..1cf43f49003f 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py @@ -1999,7 +1999,16 @@ def test_nan_threshold_gets_injected(self): assert results[0]["threshold"] == 3.0 +try: + import opentelemetry # noqa: F401 + + MISSING_OPENTELEMETRY = False +except ImportError: + MISSING_OPENTELEMETRY = True + + @pytest.mark.unittest +@pytest.mark.skipif(MISSING_OPENTELEMETRY, reason="This test requires the opentelemetry package") class TestLogEventsTokenUsage: """Tests for token usage attributes in _log_events_to_app_insights."""