From cb3a6aa9160d65c5959951d06cf7f4dfb886a21e Mon Sep 17 00:00:00 2001 From: Miles Holland Date: Wed, 8 Jan 2025 15:44:17 -0500 Subject: [PATCH 01/15] add convo agg type, and have harm evals use max --- .../azure/ai/evaluation/_constants.py | 9 +++ .../_evaluators/_common/_base_eval.py | 15 ++++- .../_evaluators/_common/_base_rai_svc_eval.py | 8 ++- .../_content_safety/_hate_unfairness.py | 2 + .../_evaluators/_content_safety/_self_harm.py | 2 + .../_evaluators/_content_safety/_sexual.py | 2 + .../_evaluators/_content_safety/_violence.py | 2 + .../evaluate_test_data_conversation.jsonl | 2 + .../tests/unittests/test_evaluate.py | 55 ++++++++++++++++++- .../test_evaluators/test_inputs_evaluators.py | 42 ++++++++++++++ 10 files changed, 136 insertions(+), 3 deletions(-) create mode 100644 sdk/evaluation/azure-ai-evaluation/tests/unittests/data/evaluate_test_data_conversation.jsonl diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_constants.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_constants.py index 253a7efa2182..fd756d704d32 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_constants.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_constants.py @@ -3,6 +3,8 @@ # --------------------------------------------------------- from typing import Literal +from sympy import Min + class EvaluationMetrics: """Metrics for model evaluation.""" @@ -56,6 +58,13 @@ class EvaluationRunProperties: EVALUATION_RUN = "_azureml.evaluation_run" EVALUATION_SDK = "_azureml.evaluation_sdk_name" +class _ConversationNumericAggregationType: + """Defines how multiple conversation turns' worth of numeric results should be evaluated + to produce a single value to define the overall conversation result.""" + + MEAN = "mean" + MAX = "max" + MIN = "min" DEFAULT_EVALUATION_RESULTS_FILE_NAME = "evaluation_results.json" diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_eval.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_eval.py index 70f323470369..ad14c54ddd0d 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_eval.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_eval.py @@ -12,6 +12,7 @@ from azure.ai.evaluation._common.math import list_mean from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException from azure.ai.evaluation._common.utils import remove_optional_singletons +from azure.ai.evaluation._constants import _ConversationNumericAggregationType from azure.ai.evaluation._model_configurations import Conversation P = ParamSpec("P") @@ -70,6 +71,10 @@ class EvaluatorBase(ABC, Generic[T_EvalValue]): :type not_singleton_inputs: List[str] :param eval_last_turn: If True, only the last turn of the conversation will be evaluated. Default is False. :type eval_last_turn: bool + :param conversation_aggregation_type: The type of aggregation to perform on the per-turn results of a conversation + to produce a single result. + Default is ~azure.ai.evaluation._constants.ConversationNumericAggregationType.MEAN. + :type conversation_aggregation_type: ~azure.ai.evaluation._constants.ConversationNumericAggregationType """ # ~~~ METHODS THAT ALMOST ALWAYS NEED TO BE OVERRIDDEN BY CHILDREN~~~ @@ -81,11 +86,13 @@ def __init__( *, not_singleton_inputs: List[str] = ["conversation", "kwargs"], eval_last_turn: bool = False, + conversation_aggregation_type: str = _ConversationNumericAggregationType.MEAN, ): self._not_singleton_inputs = not_singleton_inputs self._eval_last_turn = eval_last_turn self._singleton_inputs = self._derive_singleton_inputs() self._async_evaluator = AsyncEvaluatorBase(self._real_call) + self._conversation_aggregation_type = conversation_aggregation_type # This needs to be overridden just to change the function header into something more informative, # and to be able to add a more specific docstring. The actual function contents should just be @@ -359,7 +366,13 @@ def _aggregate_results(self, per_turn_results: List[DoEvalResult[T_EvalValue]]) # Find and average all numeric values for metric, values in evaluation_per_turn.items(): if all(isinstance(value, (int, float)) for value in values): - aggregated[metric] = list_mean(cast(List[Union[int, float]], values)) + # Aggregate results in different ways depending on the aggregation type. + if self._conversation_aggregation_type == _ConversationNumericAggregationType.MEAN: + aggregated[metric] = list_mean(cast(List[Union[int, float]], values)) + elif self._conversation_aggregation_type == _ConversationNumericAggregationType.MAX: + aggregated[metric] = max(cast(List[Union[int, float]], values)) + elif self._conversation_aggregation_type == _ConversationNumericAggregationType.MIN: + aggregated[metric] = min(cast(List[Union[int, float]], values)) # Slap the per-turn results back in. aggregated["evaluation_per_turn"] = evaluation_per_turn return aggregated diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_rai_svc_eval.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_rai_svc_eval.py index 75bee3da43ad..a7b886361041 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_rai_svc_eval.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_rai_svc_eval.py @@ -15,6 +15,7 @@ from azure.ai.evaluation._common.utils import validate_azure_ai_project from azure.ai.evaluation._exceptions import EvaluationException from azure.ai.evaluation._common.utils import validate_conversation +from azure.ai.evaluation._constants import _ConversationNumericAggregationType from azure.core.credentials import TokenCredential from . import EvaluatorBase @@ -35,6 +36,10 @@ class RaiServiceEvaluatorBase(EvaluatorBase[T]): aggregated. Per-turn results are still be available in the output via the "evaluation_per_turn" key when this occurs. Default is False, resulting full conversation evaluation and aggregation. :type eval_last_turn: bool + :param conversation_aggregation_type: The type of aggregation to perform on the per-turn results of a conversation + to produce a single result. + Default is ~azure.ai.evaluation._constants.ConversationNumericAggregationType.MEAN. + :type conversation_aggregation_type: ~azure.ai.evaluation._constants.ConversationNumericAggregationType """ @override @@ -44,8 +49,9 @@ def __init__( azure_ai_project: dict, credential: TokenCredential, eval_last_turn: bool = False, + conversation_aggregation_type: str = _ConversationNumericAggregationType.MEAN, ): - super().__init__(eval_last_turn=eval_last_turn) + super().__init__(eval_last_turn=eval_last_turn, conversation_aggregation_type=conversation_aggregation_type) self._eval_metric = eval_metric self._azure_ai_project = validate_azure_ai_project(azure_ai_project) self._credential = credential diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_hate_unfairness.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_hate_unfairness.py index b0b15251844b..d60117b5bbf7 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_hate_unfairness.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_hate_unfairness.py @@ -9,6 +9,7 @@ from azure.ai.evaluation._common.constants import EvaluationMetrics from azure.ai.evaluation._evaluators._common import RaiServiceEvaluatorBase from azure.ai.evaluation._model_configurations import Conversation +from azure.ai.evaluation._constants import _ConversationNumericAggregationType @experimental @@ -71,6 +72,7 @@ def __init__( eval_metric=EvaluationMetrics.HATE_FAIRNESS, azure_ai_project=azure_ai_project, credential=credential, + conversation_aggregation_type=_ConversationNumericAggregationType.MAX, ) @overload diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_self_harm.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_self_harm.py index 1b408e2c1e27..fc25d7476794 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_self_harm.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_self_harm.py @@ -9,6 +9,7 @@ from azure.ai.evaluation._common.constants import EvaluationMetrics from azure.ai.evaluation._evaluators._common import RaiServiceEvaluatorBase from azure.ai.evaluation._model_configurations import Conversation +from azure.ai.evaluation._constants import _ConversationNumericAggregationType @experimental @@ -65,6 +66,7 @@ def __init__( eval_metric=EvaluationMetrics.SELF_HARM, azure_ai_project=azure_ai_project, credential=credential, + conversation_aggregation_type=_ConversationNumericAggregationType.MAX, ) @overload diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_sexual.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_sexual.py index 96a6ee0dbd73..f380bde8446a 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_sexual.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_sexual.py @@ -9,6 +9,7 @@ from azure.ai.evaluation._common.constants import EvaluationMetrics from azure.ai.evaluation._evaluators._common import RaiServiceEvaluatorBase from azure.ai.evaluation._model_configurations import Conversation +from azure.ai.evaluation._constants import _ConversationNumericAggregationType @experimental @@ -67,6 +68,7 @@ def __init__( eval_metric=EvaluationMetrics.SEXUAL, azure_ai_project=azure_ai_project, credential=credential, + conversation_aggregation_type=_ConversationNumericAggregationType.MAX, ) @overload diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_violence.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_violence.py index 0f3600a92cfc..99b6f4204241 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_violence.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_violence.py @@ -9,6 +9,7 @@ from azure.ai.evaluation._common.constants import EvaluationMetrics from azure.ai.evaluation._evaluators._common import RaiServiceEvaluatorBase from azure.ai.evaluation._model_configurations import Conversation +from azure.ai.evaluation._constants import _ConversationNumericAggregationType @experimental @@ -67,6 +68,7 @@ def __init__( eval_metric=EvaluationMetrics.VIOLENCE, azure_ai_project=azure_ai_project, credential=credential, + conversation_aggregation_type=_ConversationNumericAggregationType.MAX, ) @overload diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/data/evaluate_test_data_conversation.jsonl b/sdk/evaluation/azure-ai-evaluation/tests/unittests/data/evaluate_test_data_conversation.jsonl new file mode 100644 index 000000000000..037487fae1e4 --- /dev/null +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/data/evaluate_test_data_conversation.jsonl @@ -0,0 +1,2 @@ +{"conversation" : {"context" : "", "messages": [{"content": "What shape has 3 sides", "role" :"user", "context": null}, {"content": "A triangle", "role" :"assistant", "context": "The answer is a triangle."}, {"content": "Next, what shape has 4 sides", "role" :"user", "context": null}, {"content": "A square", "role" :"assistant", "context": "The answer is a square."}]}} +{"conversation" : {"context" : "User wants to know about state capitals", "messages": [{"content": "What is the capital of Hawaii`''\"{}{{]", "role" :"user", "context": "User wants to know the capital of Hawaii"}, {"content": "Honolulu", "role" :"assistant", "context": "The answer is a Honolulu."}, {"content": "Ok, what is the capital of Massachusetts", "role" :"user", "context": "User wants to know the capital of Massachusetts."}, {"content": "Boston", "role" :"assistant", "context": "The answer is Boston."}]}} diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py index 9095cd1ac960..cb81763734b7 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py @@ -16,8 +16,12 @@ GroundednessEvaluator, ProtectedMaterialEvaluator, evaluate, + ViolenceEvaluator, + SexualEvaluator, + SelfHarmEvaluator, + HateUnfairnessEvaluator ) -from azure.ai.evaluation._constants import DEFAULT_EVALUATION_RESULTS_FILE_NAME +from azure.ai.evaluation._constants import DEFAULT_EVALUATION_RESULTS_FILE_NAME, _ConversationNumericAggregationType from azure.ai.evaluation._evaluate._evaluate import ( _aggregate_metrics, _apply_target_to_data, @@ -48,6 +52,9 @@ def missing_columns_jsonl_file(): def evaluate_test_data_jsonl_file(): return _get_file("evaluate_test_data.jsonl") +@pytest.fixture +def evaluate_test_data_conversion_jsonl_file(): + return _get_file("evaluate_test_data_conversation.jsonl") @pytest.fixture def pf_client() -> PFClient: @@ -681,3 +688,49 @@ def test_optional_inputs_with_target(self, questions_file, questions_answers_bas ) # type: ignore assert double_override_results["rows"][0]["outputs.echo.echo_query"] == "new query" assert double_override_results["rows"][0]["outputs.echo.echo_response"] == "new response" + + def test_conversation_aggregation_types(self, evaluate_test_data_conversion_jsonl_file): + from test_evaluators.test_inputs_evaluators import CountingEval + + counting_eval = CountingEval() + evaluators = { "count": counting_eval} + # test default behavior - average + results = evaluate( + data=evaluate_test_data_conversion_jsonl_file, + evaluators=evaluators + ) + assert results['rows'][0]['outputs.count.response'] == 1.5 # average of 1 and 2 + assert results['rows'][1]['outputs.count.response'] == 3.5 # average of 3 and 4 + + # test maxing + counting_eval.reset() + counting_eval._conversation_aggregation_type = _ConversationNumericAggregationType.MAX + results = evaluate( + data=evaluate_test_data_conversion_jsonl_file, + evaluators=evaluators + ) + assert results['rows'][0]['outputs.count.response'] == 2 # max of 1 and 2 + assert results['rows'][1]['outputs.count.response'] == 4 # max of 3 and 4 + + # test minimizing + counting_eval.reset() + counting_eval._conversation_aggregation_type = _ConversationNumericAggregationType.MIN + results = evaluate( + data=evaluate_test_data_conversion_jsonl_file, + evaluators=evaluators + ) + assert results['rows'][0]['outputs.count.response'] == 1 # min of 1 and 2 + assert results['rows'][1]['outputs.count.response'] == 3 # min of 3 and 4 + + def test_default_conversation_aggregation_overrides(self): + fake_project = {"subscription_id": "123", "resource_group_name": "123", "project_name": "123"} + eval1 = ViolenceEvaluator(None, fake_project) + eval2 = SexualEvaluator(None, fake_project) + eval3 = SelfHarmEvaluator(None, fake_project) + eval4 = HateUnfairnessEvaluator(None, fake_project) + eval5 = F1ScoreEvaluator() # Test default + assert eval1._conversation_aggregation_type == _ConversationNumericAggregationType.MAX + assert eval2._conversation_aggregation_type == _ConversationNumericAggregationType.MAX + assert eval3._conversation_aggregation_type == _ConversationNumericAggregationType.MAX + assert eval4._conversation_aggregation_type == _ConversationNumericAggregationType.MAX + assert eval5._conversation_aggregation_type == _ConversationNumericAggregationType.MEAN diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluators/test_inputs_evaluators.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluators/test_inputs_evaluators.py index e09faa738c18..a9d17daab904 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluators/test_inputs_evaluators.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluators/test_inputs_evaluators.py @@ -5,6 +5,10 @@ # A collection of very simple evaluators designed to test column mappings. # (aka proper data file -> _call__ input mapping) +from typing import Dict, Union +from typing_extensions import overload, override + +from azure.ai.evaluation._evaluators._common import EvaluatorBase class NonOptionalEval: def __init__(self): @@ -44,3 +48,41 @@ def __init__(self): def __call__(self, *, query="default", response="default"): return {"echo_query": query, "echo_response": response} + + +class CountingEval(EvaluatorBase): + '''Returns an incrementing number, which can be reset as needed''' + def __init__(self, **kwargs): + self._count = 0 + super().__init__(**kwargs) + + + def reset(self): + self._count = 0 + + + @override + async def _do_eval(self, eval_input: Dict) -> Dict[str, int]: + self._count += 1 + return {"response" : self._count} + + @overload + def __call__( + self, + *, + query: str, + response: str, + ) -> Dict[str, Union[str, float]]: + """""" + + @overload + def __call__(self, *, conversation): + """""" + + @override + def __call__( # pylint: disable=docstring-missing-param + self, + *args, + **kwargs, + ): + return super().__call__(*args, **kwargs) From 3ee1a9fc9fc40fb329c92ba2455761894320c049 Mon Sep 17 00:00:00 2001 From: Miles Holland Date: Wed, 8 Jan 2025 15:47:11 -0500 Subject: [PATCH 02/15] analysis --- .../azure/ai/evaluation/_constants.py | 4 +- .../tests/unittests/test_evaluate.py | 37 ++++++++----------- .../test_evaluators/test_inputs_evaluators.py | 8 ++-- 3 files changed, 21 insertions(+), 28 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_constants.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_constants.py index fd756d704d32..5a8d212c53f9 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_constants.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_constants.py @@ -3,8 +3,6 @@ # --------------------------------------------------------- from typing import Literal -from sympy import Min - class EvaluationMetrics: """Metrics for model evaluation.""" @@ -58,6 +56,7 @@ class EvaluationRunProperties: EVALUATION_RUN = "_azureml.evaluation_run" EVALUATION_SDK = "_azureml.evaluation_sdk_name" + class _ConversationNumericAggregationType: """Defines how multiple conversation turns' worth of numeric results should be evaluated to produce a single value to define the overall conversation result.""" @@ -66,6 +65,7 @@ class _ConversationNumericAggregationType: MAX = "max" MIN = "min" + DEFAULT_EVALUATION_RESULTS_FILE_NAME = "evaluation_results.json" CONTENT_SAFETY_DEFECT_RATE_THRESHOLD_DEFAULT = 4 diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py index cb81763734b7..0a5ef3e711b5 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py @@ -19,7 +19,7 @@ ViolenceEvaluator, SexualEvaluator, SelfHarmEvaluator, - HateUnfairnessEvaluator + HateUnfairnessEvaluator, ) from azure.ai.evaluation._constants import DEFAULT_EVALUATION_RESULTS_FILE_NAME, _ConversationNumericAggregationType from azure.ai.evaluation._evaluate._evaluate import ( @@ -52,10 +52,12 @@ def missing_columns_jsonl_file(): def evaluate_test_data_jsonl_file(): return _get_file("evaluate_test_data.jsonl") + @pytest.fixture def evaluate_test_data_conversion_jsonl_file(): return _get_file("evaluate_test_data_conversation.jsonl") + @pytest.fixture def pf_client() -> PFClient: """The fixture, returning PRClient""" @@ -693,34 +695,25 @@ def test_conversation_aggregation_types(self, evaluate_test_data_conversion_json from test_evaluators.test_inputs_evaluators import CountingEval counting_eval = CountingEval() - evaluators = { "count": counting_eval} - # test default behavior - average - results = evaluate( - data=evaluate_test_data_conversion_jsonl_file, - evaluators=evaluators - ) - assert results['rows'][0]['outputs.count.response'] == 1.5 # average of 1 and 2 - assert results['rows'][1]['outputs.count.response'] == 3.5 # average of 3 and 4 + evaluators = {"count": counting_eval} + # test default behavior - average + results = evaluate(data=evaluate_test_data_conversion_jsonl_file, evaluators=evaluators) + assert results["rows"][0]["outputs.count.response"] == 1.5 # average of 1 and 2 + assert results["rows"][1]["outputs.count.response"] == 3.5 # average of 3 and 4 # test maxing counting_eval.reset() counting_eval._conversation_aggregation_type = _ConversationNumericAggregationType.MAX - results = evaluate( - data=evaluate_test_data_conversion_jsonl_file, - evaluators=evaluators - ) - assert results['rows'][0]['outputs.count.response'] == 2 # max of 1 and 2 - assert results['rows'][1]['outputs.count.response'] == 4 # max of 3 and 4 + results = evaluate(data=evaluate_test_data_conversion_jsonl_file, evaluators=evaluators) + assert results["rows"][0]["outputs.count.response"] == 2 # max of 1 and 2 + assert results["rows"][1]["outputs.count.response"] == 4 # max of 3 and 4 # test minimizing counting_eval.reset() counting_eval._conversation_aggregation_type = _ConversationNumericAggregationType.MIN - results = evaluate( - data=evaluate_test_data_conversion_jsonl_file, - evaluators=evaluators - ) - assert results['rows'][0]['outputs.count.response'] == 1 # min of 1 and 2 - assert results['rows'][1]['outputs.count.response'] == 3 # min of 3 and 4 + results = evaluate(data=evaluate_test_data_conversion_jsonl_file, evaluators=evaluators) + assert results["rows"][0]["outputs.count.response"] == 1 # min of 1 and 2 + assert results["rows"][1]["outputs.count.response"] == 3 # min of 3 and 4 def test_default_conversation_aggregation_overrides(self): fake_project = {"subscription_id": "123", "resource_group_name": "123", "project_name": "123"} @@ -728,7 +721,7 @@ def test_default_conversation_aggregation_overrides(self): eval2 = SexualEvaluator(None, fake_project) eval3 = SelfHarmEvaluator(None, fake_project) eval4 = HateUnfairnessEvaluator(None, fake_project) - eval5 = F1ScoreEvaluator() # Test default + eval5 = F1ScoreEvaluator() # Test default assert eval1._conversation_aggregation_type == _ConversationNumericAggregationType.MAX assert eval2._conversation_aggregation_type == _ConversationNumericAggregationType.MAX assert eval3._conversation_aggregation_type == _ConversationNumericAggregationType.MAX diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluators/test_inputs_evaluators.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluators/test_inputs_evaluators.py index a9d17daab904..004aec6741a2 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluators/test_inputs_evaluators.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluators/test_inputs_evaluators.py @@ -10,6 +10,7 @@ from azure.ai.evaluation._evaluators._common import EvaluatorBase + class NonOptionalEval: def __init__(self): pass @@ -51,20 +52,19 @@ def __call__(self, *, query="default", response="default"): class CountingEval(EvaluatorBase): - '''Returns an incrementing number, which can be reset as needed''' + """Returns an incrementing number, which can be reset as needed""" + def __init__(self, **kwargs): self._count = 0 super().__init__(**kwargs) - def reset(self): self._count = 0 - @override async def _do_eval(self, eval_input: Dict) -> Dict[str, int]: self._count += 1 - return {"response" : self._count} + return {"response": self._count} @overload def __call__( From a0caaf03d352e8f9f33ba8befd42acc775e1c3c6 Mon Sep 17 00:00:00 2001 From: Miles Holland Date: Wed, 8 Jan 2025 15:52:03 -0500 Subject: [PATCH 03/15] correct enum name in docs --- .../azure/ai/evaluation/_evaluators/_common/_base_eval.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_eval.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_eval.py index ad14c54ddd0d..f80aa491926e 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_eval.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_eval.py @@ -73,8 +73,8 @@ class EvaluatorBase(ABC, Generic[T_EvalValue]): :type eval_last_turn: bool :param conversation_aggregation_type: The type of aggregation to perform on the per-turn results of a conversation to produce a single result. - Default is ~azure.ai.evaluation._constants.ConversationNumericAggregationType.MEAN. - :type conversation_aggregation_type: ~azure.ai.evaluation._constants.ConversationNumericAggregationType + Default is ~azure.ai.evaluation._constants._ConversationNumericAggregationType.MEAN. + :type conversation_aggregation_type: ~azure.ai.evaluation._constants._ConversationNumericAggregationType """ # ~~~ METHODS THAT ALMOST ALWAYS NEED TO BE OVERRIDDEN BY CHILDREN~~~ From 171b2c251000ca9f063e3b9f1faea6c9ea98f350 Mon Sep 17 00:00:00 2001 From: Miles Holland Date: Mon, 13 Jan 2025 16:37:06 -0500 Subject: [PATCH 04/15] refactor checked enum into function field --- .../azure/ai/evaluation/__init__.py | 2 + .../azure/ai/evaluation/_constants.py | 4 +- .../_evaluators/_common/_base_eval.py | 54 ++++++++++++++----- .../_evaluators/_common/_base_rai_svc_eval.py | 8 +-- .../_common/_conversation_aggregators.py | 25 +++++++++ .../_content_safety/_hate_unfairness.py | 4 +- .../_evaluators/_content_safety/_self_harm.py | 4 +- .../_evaluators/_content_safety/_sexual.py | 4 +- .../_evaluators/_content_safety/_violence.py | 4 +- .../tests/unittests/test_evaluate.py | 43 ++++++++++----- 10 files changed, 113 insertions(+), 39 deletions(-) create mode 100644 sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_conversation_aggregators.py diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/__init__.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/__init__.py index c21a97a9531a..b9979814aeba 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/__init__.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/__init__.py @@ -42,6 +42,7 @@ Message, OpenAIModelConfiguration, ) +from ._constants import ConversationAggregationType __all__ = [ "evaluate", @@ -79,4 +80,5 @@ "SexualMultimodalEvaluator", "ViolenceMultimodalEvaluator", "ProtectedMaterialMultimodalEvaluator", + "ConversationAggregationType", ] diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_constants.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_constants.py index 5a8d212c53f9..d1cda626a060 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_constants.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_constants.py @@ -1,6 +1,7 @@ # --------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. # --------------------------------------------------------- +import enum from typing import Literal @@ -57,13 +58,14 @@ class EvaluationRunProperties: EVALUATION_SDK = "_azureml.evaluation_sdk_name" -class _ConversationNumericAggregationType: +class ConversationAggregationType(enum.Enum): """Defines how multiple conversation turns' worth of numeric results should be evaluated to produce a single value to define the overall conversation result.""" MEAN = "mean" MAX = "max" MIN = "min" + SUM = "sum" DEFAULT_EVALUATION_RESULTS_FILE_NAME = "evaluation_results.json" diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_eval.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_eval.py index f80aa491926e..6d6b4ef30399 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_eval.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_eval.py @@ -4,7 +4,7 @@ import inspect from abc import ABC, abstractmethod -from typing import Any, Callable, Dict, Generic, List, TypedDict, TypeVar, Union, cast, final +from typing import Any, Callable, Dict, Generic, List, TypedDict, TypeVar, Union, cast, final, Optional from promptflow._utils.async_utils import async_run_allowing_running_loop from typing_extensions import ParamSpec, TypeAlias, get_overloads @@ -12,9 +12,11 @@ from azure.ai.evaluation._common.math import list_mean from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException from azure.ai.evaluation._common.utils import remove_optional_singletons -from azure.ai.evaluation._constants import _ConversationNumericAggregationType +from azure.ai.evaluation._constants import ConversationAggregationType from azure.ai.evaluation._model_configurations import Conversation +from ._conversation_aggregators import GetAggregator + P = ParamSpec("P") T = TypeVar("T") T_EvalValue = TypeVar("T_EvalValue") @@ -73,8 +75,11 @@ class EvaluatorBase(ABC, Generic[T_EvalValue]): :type eval_last_turn: bool :param conversation_aggregation_type: The type of aggregation to perform on the per-turn results of a conversation to produce a single result. - Default is ~azure.ai.evaluation._constants._ConversationNumericAggregationType.MEAN. - :type conversation_aggregation_type: ~azure.ai.evaluation._constants._ConversationNumericAggregationType + Default is ~azure.ai.evaluation.ConversationAggregationType.MEAN. + :type conversation_aggregation_type: ~azure.ai.evaluation.ConversationAggregationType + :param conversation_aggregator_override: A function that will be used to aggregate per-turn results. If provided, + overrides the standard aggregator implied by conversation_aggregation_type. None by default. + :type conversation_aggregator_override: Optional[Callable[[List[float]], float]] """ # ~~~ METHODS THAT ALMOST ALWAYS NEED TO BE OVERRIDDEN BY CHILDREN~~~ @@ -86,13 +91,16 @@ def __init__( *, not_singleton_inputs: List[str] = ["conversation", "kwargs"], eval_last_turn: bool = False, - conversation_aggregation_type: str = _ConversationNumericAggregationType.MEAN, + conversation_aggregation_type: ConversationAggregationType = ConversationAggregationType.MEAN, + conversation_aggregator_override: Optional[Callable[[List[float]], float]] = None ): self._not_singleton_inputs = not_singleton_inputs self._eval_last_turn = eval_last_turn self._singleton_inputs = self._derive_singleton_inputs() self._async_evaluator = AsyncEvaluatorBase(self._real_call) - self._conversation_aggregation_type = conversation_aggregation_type + self._conversation_aggregation_function = GetAggregator(conversation_aggregation_type) + if conversation_aggregator_override!= None: + self._conversation_aggregation_function = conversation_aggregator_override # This needs to be overridden just to change the function header into something more informative, # and to be able to add a more specific docstring. The actual function contents should just be @@ -366,13 +374,7 @@ def _aggregate_results(self, per_turn_results: List[DoEvalResult[T_EvalValue]]) # Find and average all numeric values for metric, values in evaluation_per_turn.items(): if all(isinstance(value, (int, float)) for value in values): - # Aggregate results in different ways depending on the aggregation type. - if self._conversation_aggregation_type == _ConversationNumericAggregationType.MEAN: - aggregated[metric] = list_mean(cast(List[Union[int, float]], values)) - elif self._conversation_aggregation_type == _ConversationNumericAggregationType.MAX: - aggregated[metric] = max(cast(List[Union[int, float]], values)) - elif self._conversation_aggregation_type == _ConversationNumericAggregationType.MIN: - aggregated[metric] = min(cast(List[Union[int, float]], values)) + aggregated[metric] = self._conversation_aggregation_function(cast(List[Union[int, float]], values)) # Slap the per-turn results back in. aggregated["evaluation_per_turn"] = evaluation_per_turn return aggregated @@ -400,10 +402,36 @@ async def _real_call(self, **kwargs) -> Union[DoEvalResult[T_EvalValue], Aggrega # Otherwise, aggregate results. return self._aggregate_results(per_turn_results=per_turn_results) + # ~~~ METHODS THAT SHOULD NOT BE OVERRIDDEN BY CHILDREN~~~`` + @final def _to_async(self) -> "AsyncEvaluatorBase": return self._async_evaluator + @final + def set_conversation_aggregation_type(self, conversation_aggregation_type: ConversationAggregationType) -> None: + """Input a conversation aggregation type to re-assign the aggregator function used by this evaluator for + multi-turn conversations. This aggregator is used to combine numeric outputs from each evaluation of a + multi-turn conversation into a single top-level result. + + :param conversation_aggregation_type: The type of aggregation to perform on the per-turn results of a conversation + to produce a single result. + :type conversation_aggregation_type: ~azure.ai.evaluation.ConversationAggregationType + """ + self._conversation_aggregation_function = GetAggregator(conversation_aggregation_type) + + @final + def set_conversation_aggregator(self, aggregator: Callable[[List[float]], float]) -> None: + """Set the conversation aggregator function directly. This function will be applied to all numeric outputs + of an evaluator when it evaluates a conversation with multiple-turns thus ends up with multiple results per + evaluation that is needs to coalesce into a single result. Use when built-in aggregators do not + suit your needs, but use with caution. + + :param aggregator: The function to use to aggregate per-turn results. + :type aggregator: Callable[[List[float]], float] + """ + self._conversation_aggregation_function = aggregator + class AsyncEvaluatorBase: """The asynchronous evaluator hidden underneath all evaluators. This makes generous use passing functions diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_rai_svc_eval.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_rai_svc_eval.py index a7b886361041..08998dae70b4 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_rai_svc_eval.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_rai_svc_eval.py @@ -15,7 +15,7 @@ from azure.ai.evaluation._common.utils import validate_azure_ai_project from azure.ai.evaluation._exceptions import EvaluationException from azure.ai.evaluation._common.utils import validate_conversation -from azure.ai.evaluation._constants import _ConversationNumericAggregationType +from azure.ai.evaluation._constants import ConversationAggregationType from azure.core.credentials import TokenCredential from . import EvaluatorBase @@ -38,8 +38,8 @@ class RaiServiceEvaluatorBase(EvaluatorBase[T]): :type eval_last_turn: bool :param conversation_aggregation_type: The type of aggregation to perform on the per-turn results of a conversation to produce a single result. - Default is ~azure.ai.evaluation._constants.ConversationNumericAggregationType.MEAN. - :type conversation_aggregation_type: ~azure.ai.evaluation._constants.ConversationNumericAggregationType + Default is ~azure.ai.evaluation.ConversationAggregationType.MEAN. + :type conversation_aggregation_type: ~azure.ai.evaluation.ConversationAggregationType """ @override @@ -49,7 +49,7 @@ def __init__( azure_ai_project: dict, credential: TokenCredential, eval_last_turn: bool = False, - conversation_aggregation_type: str = _ConversationNumericAggregationType.MEAN, + conversation_aggregation_type: ConversationAggregationType = ConversationAggregationType.MEAN, ): super().__init__(eval_last_turn=eval_last_turn, conversation_aggregation_type=conversation_aggregation_type) self._eval_metric = eval_metric diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_conversation_aggregators.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_conversation_aggregators.py new file mode 100644 index 000000000000..bcc5afe84a06 --- /dev/null +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_conversation_aggregators.py @@ -0,0 +1,25 @@ +# --------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# --------------------------------------------------------- + +from typing import Callable, List +from azure.ai.evaluation._common.math import list_mean +from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException +from azure.ai.evaluation._constants import ConversationAggregationType + +def GetAggregator(aggregation_type: ConversationAggregationType) -> Callable[[List[float]], float]: + if (aggregation_type == ConversationAggregationType.SUM): + return sum + elif (aggregation_type == ConversationAggregationType.MEAN): + return list_mean + elif (aggregation_type == ConversationAggregationType.MAX): + return max + elif (aggregation_type == ConversationAggregationType.MIN): + return min + else: + raise EvaluationException( + message=f"Unaccounted for aggregation type: {aggregation_type}", + blame=ErrorBlame.UNKNOWN, + category=ErrorCategory.INVALID_VALUE, + target=ErrorTarget.EVALUATE + ) diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_hate_unfairness.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_hate_unfairness.py index d60117b5bbf7..3a90f13df563 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_hate_unfairness.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_hate_unfairness.py @@ -9,7 +9,7 @@ from azure.ai.evaluation._common.constants import EvaluationMetrics from azure.ai.evaluation._evaluators._common import RaiServiceEvaluatorBase from azure.ai.evaluation._model_configurations import Conversation -from azure.ai.evaluation._constants import _ConversationNumericAggregationType +from azure.ai.evaluation._constants import ConversationAggregationType @experimental @@ -72,7 +72,7 @@ def __init__( eval_metric=EvaluationMetrics.HATE_FAIRNESS, azure_ai_project=azure_ai_project, credential=credential, - conversation_aggregation_type=_ConversationNumericAggregationType.MAX, + conversation_aggregation_type=ConversationAggregationType.MAX, ) @overload diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_self_harm.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_self_harm.py index fc25d7476794..a04799f2c53d 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_self_harm.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_self_harm.py @@ -9,7 +9,7 @@ from azure.ai.evaluation._common.constants import EvaluationMetrics from azure.ai.evaluation._evaluators._common import RaiServiceEvaluatorBase from azure.ai.evaluation._model_configurations import Conversation -from azure.ai.evaluation._constants import _ConversationNumericAggregationType +from azure.ai.evaluation._constants import ConversationAggregationType @experimental @@ -66,7 +66,7 @@ def __init__( eval_metric=EvaluationMetrics.SELF_HARM, azure_ai_project=azure_ai_project, credential=credential, - conversation_aggregation_type=_ConversationNumericAggregationType.MAX, + conversation_aggregation_type=ConversationAggregationType.MAX, ) @overload diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_sexual.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_sexual.py index f380bde8446a..758050fa4ec9 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_sexual.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_sexual.py @@ -9,7 +9,7 @@ from azure.ai.evaluation._common.constants import EvaluationMetrics from azure.ai.evaluation._evaluators._common import RaiServiceEvaluatorBase from azure.ai.evaluation._model_configurations import Conversation -from azure.ai.evaluation._constants import _ConversationNumericAggregationType +from azure.ai.evaluation._constants import ConversationAggregationType @experimental @@ -68,7 +68,7 @@ def __init__( eval_metric=EvaluationMetrics.SEXUAL, azure_ai_project=azure_ai_project, credential=credential, - conversation_aggregation_type=_ConversationNumericAggregationType.MAX, + conversation_aggregation_type=ConversationAggregationType.MAX, ) @overload diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_violence.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_violence.py index 99b6f4204241..a8d1f33d320e 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_violence.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_violence.py @@ -9,7 +9,7 @@ from azure.ai.evaluation._common.constants import EvaluationMetrics from azure.ai.evaluation._evaluators._common import RaiServiceEvaluatorBase from azure.ai.evaluation._model_configurations import Conversation -from azure.ai.evaluation._constants import _ConversationNumericAggregationType +from azure.ai.evaluation._constants import ConversationAggregationType @experimental @@ -68,7 +68,7 @@ def __init__( eval_metric=EvaluationMetrics.VIOLENCE, azure_ai_project=azure_ai_project, credential=credential, - conversation_aggregation_type=_ConversationNumericAggregationType.MAX, + conversation_aggregation_type=ConversationAggregationType.MAX, ) @overload diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py index 0a5ef3e711b5..7d97d324cd61 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py @@ -10,6 +10,7 @@ from pandas.testing import assert_frame_equal from promptflow.client import PFClient +from azure.ai.evaluation._common.math import list_mean from azure.ai.evaluation import ( ContentSafetyEvaluator, F1ScoreEvaluator, @@ -21,7 +22,7 @@ SelfHarmEvaluator, HateUnfairnessEvaluator, ) -from azure.ai.evaluation._constants import DEFAULT_EVALUATION_RESULTS_FILE_NAME, _ConversationNumericAggregationType +from azure.ai.evaluation._constants import DEFAULT_EVALUATION_RESULTS_FILE_NAME, ConversationAggregationType from azure.ai.evaluation._evaluate._evaluate import ( _aggregate_metrics, _apply_target_to_data, @@ -696,24 +697,40 @@ def test_conversation_aggregation_types(self, evaluate_test_data_conversion_json counting_eval = CountingEval() evaluators = {"count": counting_eval} - # test default behavior - average + # test default behavior - mean results = evaluate(data=evaluate_test_data_conversion_jsonl_file, evaluators=evaluators) assert results["rows"][0]["outputs.count.response"] == 1.5 # average of 1 and 2 assert results["rows"][1]["outputs.count.response"] == 3.5 # average of 3 and 4 # test maxing counting_eval.reset() - counting_eval._conversation_aggregation_type = _ConversationNumericAggregationType.MAX + counting_eval.set_conversation_aggregation_type(ConversationAggregationType.MAX) results = evaluate(data=evaluate_test_data_conversion_jsonl_file, evaluators=evaluators) - assert results["rows"][0]["outputs.count.response"] == 2 # max of 1 and 2 - assert results["rows"][1]["outputs.count.response"] == 4 # max of 3 and 4 + assert results["rows"][0]["outputs.count.response"] == 2 + assert results["rows"][1]["outputs.count.response"] == 4 # test minimizing counting_eval.reset() - counting_eval._conversation_aggregation_type = _ConversationNumericAggregationType.MIN + counting_eval.set_conversation_aggregation_type(ConversationAggregationType.MIN) results = evaluate(data=evaluate_test_data_conversion_jsonl_file, evaluators=evaluators) - assert results["rows"][0]["outputs.count.response"] == 1 # min of 1 and 2 - assert results["rows"][1]["outputs.count.response"] == 3 # min of 3 and 4 + assert results["rows"][0]["outputs.count.response"] == 1 + assert results["rows"][1]["outputs.count.response"] == 3 + + # test sum + counting_eval.reset() + counting_eval.set_conversation_aggregation_type(ConversationAggregationType.SUM) + results = evaluate(data=evaluate_test_data_conversion_jsonl_file, evaluators=evaluators) + assert results["rows"][0]["outputs.count.response"] == 3 + assert results["rows"][1]["outputs.count.response"] == 7 + + # test custom aggregator + def custom_aggregator(values): + return sum(values) + 1 + counting_eval.reset() + counting_eval.set_conversation_aggregator(custom_aggregator) + results = evaluate(data=evaluate_test_data_conversion_jsonl_file, evaluators=evaluators) + assert results["rows"][0]["outputs.count.response"] == 4 + assert results["rows"][1]["outputs.count.response"] == 8 def test_default_conversation_aggregation_overrides(self): fake_project = {"subscription_id": "123", "resource_group_name": "123", "project_name": "123"} @@ -722,8 +739,8 @@ def test_default_conversation_aggregation_overrides(self): eval3 = SelfHarmEvaluator(None, fake_project) eval4 = HateUnfairnessEvaluator(None, fake_project) eval5 = F1ScoreEvaluator() # Test default - assert eval1._conversation_aggregation_type == _ConversationNumericAggregationType.MAX - assert eval2._conversation_aggregation_type == _ConversationNumericAggregationType.MAX - assert eval3._conversation_aggregation_type == _ConversationNumericAggregationType.MAX - assert eval4._conversation_aggregation_type == _ConversationNumericAggregationType.MAX - assert eval5._conversation_aggregation_type == _ConversationNumericAggregationType.MEAN + assert eval1._conversation_aggregation_function == max + assert eval2._conversation_aggregation_function == max + assert eval3._conversation_aggregation_function == max + assert eval4._conversation_aggregation_function == max + assert eval5._conversation_aggregation_function == list_mean From 9ab974cf042cd9f3c57707d93f2c0e2c43603a9a Mon Sep 17 00:00:00 2001 From: Miles Holland Date: Tue, 14 Jan 2025 10:44:20 -0500 Subject: [PATCH 05/15] cl and analysis --- .../azure-ai-evaluation/CHANGELOG.md | 3 +++ .../_evaluators/_common/_base_eval.py | 12 +++++----- .../_common/_conversation_aggregators.py | 22 +++++++++---------- .../tests/unittests/test_evaluate.py | 1 + 4 files changed, 21 insertions(+), 17 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md index de2c1cfe7ae0..a6aac0fcacb6 100644 --- a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md +++ b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md @@ -10,6 +10,9 @@ - Removed `[remote]` extra. This is no longer needed when tracking results in Azure AI Studio. - Fixed `AttributeError: 'NoneType' object has no attribute 'get'` while running simulator with 1000+ results - Fixed the non adversarial simulator to run in task-free mode +- Content safety evaluators (violence, self harm, sexual, hate/unfairness) return the maximum result as the + main score when aggregating multiple per-turn evaluations from a conversation into a single overall + evaluation score. Other conversation-capable evaluators. ### Other Changes - Changed minimum required python version to use this package from 3.8 to 3.9 diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_eval.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_eval.py index 6d6b4ef30399..a4e45694455b 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_eval.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_eval.py @@ -9,7 +9,6 @@ from promptflow._utils.async_utils import async_run_allowing_running_loop from typing_extensions import ParamSpec, TypeAlias, get_overloads -from azure.ai.evaluation._common.math import list_mean from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException from azure.ai.evaluation._common.utils import remove_optional_singletons from azure.ai.evaluation._constants import ConversationAggregationType @@ -92,15 +91,16 @@ def __init__( not_singleton_inputs: List[str] = ["conversation", "kwargs"], eval_last_turn: bool = False, conversation_aggregation_type: ConversationAggregationType = ConversationAggregationType.MEAN, - conversation_aggregator_override: Optional[Callable[[List[float]], float]] = None + conversation_aggregator_override: Optional[Callable[[List[float]], float]] = None, ): self._not_singleton_inputs = not_singleton_inputs self._eval_last_turn = eval_last_turn self._singleton_inputs = self._derive_singleton_inputs() self._async_evaluator = AsyncEvaluatorBase(self._real_call) self._conversation_aggregation_function = GetAggregator(conversation_aggregation_type) - if conversation_aggregator_override!= None: - self._conversation_aggregation_function = conversation_aggregator_override + if conversation_aggregator_override is not None: + # Type ignore since we already checked for None, but mypy doesn't know that. + self._conversation_aggregation_function = conversation_aggregator_override # type: ignore[assignment] # This needs to be overridden just to change the function header into something more informative, # and to be able to add a more specific docstring. The actual function contents should just be @@ -414,8 +414,8 @@ def set_conversation_aggregation_type(self, conversation_aggregation_type: Conve multi-turn conversations. This aggregator is used to combine numeric outputs from each evaluation of a multi-turn conversation into a single top-level result. - :param conversation_aggregation_type: The type of aggregation to perform on the per-turn results of a conversation - to produce a single result. + :param conversation_aggregation_type: The type of aggregation to perform on the per-turn + results of a conversation to produce a single result. :type conversation_aggregation_type: ~azure.ai.evaluation.ConversationAggregationType """ self._conversation_aggregation_function = GetAggregator(conversation_aggregation_type) diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_conversation_aggregators.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_conversation_aggregators.py index bcc5afe84a06..3b5839f50d5c 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_conversation_aggregators.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_conversation_aggregators.py @@ -7,19 +7,19 @@ from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException from azure.ai.evaluation._constants import ConversationAggregationType + def GetAggregator(aggregation_type: ConversationAggregationType) -> Callable[[List[float]], float]: - if (aggregation_type == ConversationAggregationType.SUM): + if aggregation_type == ConversationAggregationType.SUM: return sum - elif (aggregation_type == ConversationAggregationType.MEAN): + if aggregation_type == ConversationAggregationType.MEAN: return list_mean - elif (aggregation_type == ConversationAggregationType.MAX): + if aggregation_type == ConversationAggregationType.MAX: return max - elif (aggregation_type == ConversationAggregationType.MIN): + if aggregation_type == ConversationAggregationType.MIN: return min - else: - raise EvaluationException( - message=f"Unaccounted for aggregation type: {aggregation_type}", - blame=ErrorBlame.UNKNOWN, - category=ErrorCategory.INVALID_VALUE, - target=ErrorTarget.EVALUATE - ) + raise EvaluationException( + message=f"Unaccounted for aggregation type: {aggregation_type}", + blame=ErrorBlame.UNKNOWN, + category=ErrorCategory.INVALID_VALUE, + target=ErrorTarget.EVALUATE, + ) diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py index 7d97d324cd61..9f22d4bfba7a 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py @@ -726,6 +726,7 @@ def test_conversation_aggregation_types(self, evaluate_test_data_conversion_json # test custom aggregator def custom_aggregator(values): return sum(values) + 1 + counting_eval.reset() counting_eval.set_conversation_aggregator(custom_aggregator) results = evaluate(data=evaluate_test_data_conversion_jsonl_file, evaluators=evaluators) From 51592ce74cb0d074c9570716f91bfceb477fe15d Mon Sep 17 00:00:00 2001 From: Miles Holland Date: Tue, 14 Jan 2025 13:56:14 -0500 Subject: [PATCH 06/15] change enum name and update CL --- sdk/evaluation/azure-ai-evaluation/CHANGELOG.md | 6 +++++- .../azure/ai/evaluation/__init__.py | 4 ++-- .../azure/ai/evaluation/_constants.py | 2 +- .../ai/evaluation/_evaluators/_common/_base_eval.py | 12 ++++++------ .../_evaluators/_common/_base_rai_svc_eval.py | 8 ++++---- .../_evaluators/_common/_conversation_aggregators.py | 12 ++++++------ .../_evaluators/_content_safety/_hate_unfairness.py | 4 ++-- .../_evaluators/_content_safety/_self_harm.py | 4 ++-- .../_evaluators/_content_safety/_sexual.py | 4 ++-- .../_evaluators/_content_safety/_violence.py | 4 ++-- .../tests/unittests/test_evaluate.py | 8 ++++---- 11 files changed, 36 insertions(+), 32 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md index a6aac0fcacb6..3c9de1ebb7f4 100644 --- a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md +++ b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md @@ -12,7 +12,11 @@ - Fixed the non adversarial simulator to run in task-free mode - Content safety evaluators (violence, self harm, sexual, hate/unfairness) return the maximum result as the main score when aggregating multiple per-turn evaluations from a conversation into a single overall - evaluation score. Other conversation-capable evaluators. + evaluation score. Other conversation-capable evaluators still default to a mean. +- Evaluator aggregation for producing a main score multi-turn conversations is now configurable. Can be configured + quickly by providing an Enum value (`~azure.ai.evaluation.AggregationType`) via + `eval.set_conversation_aggregation_type` or the exact aggregation function can be assigned directly via + `eval.set_conversation_aggregator`. New options are MEAN, SUM, MAX, and MIN. ### Other Changes - Changed minimum required python version to use this package from 3.8 to 3.9 diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/__init__.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/__init__.py index b9979814aeba..71bf744c7b80 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/__init__.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/__init__.py @@ -42,7 +42,7 @@ Message, OpenAIModelConfiguration, ) -from ._constants import ConversationAggregationType +from ._constants import AggregationType __all__ = [ "evaluate", @@ -80,5 +80,5 @@ "SexualMultimodalEvaluator", "ViolenceMultimodalEvaluator", "ProtectedMaterialMultimodalEvaluator", - "ConversationAggregationType", + "AggregationType", ] diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_constants.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_constants.py index d1cda626a060..d047e5e2a3f0 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_constants.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_constants.py @@ -58,7 +58,7 @@ class EvaluationRunProperties: EVALUATION_SDK = "_azureml.evaluation_sdk_name" -class ConversationAggregationType(enum.Enum): +class AggregationType(enum.Enum): """Defines how multiple conversation turns' worth of numeric results should be evaluated to produce a single value to define the overall conversation result.""" diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_eval.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_eval.py index a4e45694455b..ff6fd83df233 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_eval.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_eval.py @@ -11,7 +11,7 @@ from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException from azure.ai.evaluation._common.utils import remove_optional_singletons -from azure.ai.evaluation._constants import ConversationAggregationType +from azure.ai.evaluation._constants import AggregationType from azure.ai.evaluation._model_configurations import Conversation from ._conversation_aggregators import GetAggregator @@ -74,8 +74,8 @@ class EvaluatorBase(ABC, Generic[T_EvalValue]): :type eval_last_turn: bool :param conversation_aggregation_type: The type of aggregation to perform on the per-turn results of a conversation to produce a single result. - Default is ~azure.ai.evaluation.ConversationAggregationType.MEAN. - :type conversation_aggregation_type: ~azure.ai.evaluation.ConversationAggregationType + Default is ~azure.ai.evaluation.AggregationType.MEAN. + :type conversation_aggregation_type: ~azure.ai.evaluation.AggregationType :param conversation_aggregator_override: A function that will be used to aggregate per-turn results. If provided, overrides the standard aggregator implied by conversation_aggregation_type. None by default. :type conversation_aggregator_override: Optional[Callable[[List[float]], float]] @@ -90,7 +90,7 @@ def __init__( *, not_singleton_inputs: List[str] = ["conversation", "kwargs"], eval_last_turn: bool = False, - conversation_aggregation_type: ConversationAggregationType = ConversationAggregationType.MEAN, + conversation_aggregation_type: AggregationType = AggregationType.MEAN, conversation_aggregator_override: Optional[Callable[[List[float]], float]] = None, ): self._not_singleton_inputs = not_singleton_inputs @@ -409,14 +409,14 @@ def _to_async(self) -> "AsyncEvaluatorBase": return self._async_evaluator @final - def set_conversation_aggregation_type(self, conversation_aggregation_type: ConversationAggregationType) -> None: + def set_conversation_aggregation_type(self, conversation_aggregation_type: AggregationType) -> None: """Input a conversation aggregation type to re-assign the aggregator function used by this evaluator for multi-turn conversations. This aggregator is used to combine numeric outputs from each evaluation of a multi-turn conversation into a single top-level result. :param conversation_aggregation_type: The type of aggregation to perform on the per-turn results of a conversation to produce a single result. - :type conversation_aggregation_type: ~azure.ai.evaluation.ConversationAggregationType + :type conversation_aggregation_type: ~azure.ai.evaluation.AggregationType """ self._conversation_aggregation_function = GetAggregator(conversation_aggregation_type) diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_rai_svc_eval.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_rai_svc_eval.py index 08998dae70b4..26608db9f4f1 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_rai_svc_eval.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_rai_svc_eval.py @@ -15,7 +15,7 @@ from azure.ai.evaluation._common.utils import validate_azure_ai_project from azure.ai.evaluation._exceptions import EvaluationException from azure.ai.evaluation._common.utils import validate_conversation -from azure.ai.evaluation._constants import ConversationAggregationType +from azure.ai.evaluation._constants import AggregationType from azure.core.credentials import TokenCredential from . import EvaluatorBase @@ -38,8 +38,8 @@ class RaiServiceEvaluatorBase(EvaluatorBase[T]): :type eval_last_turn: bool :param conversation_aggregation_type: The type of aggregation to perform on the per-turn results of a conversation to produce a single result. - Default is ~azure.ai.evaluation.ConversationAggregationType.MEAN. - :type conversation_aggregation_type: ~azure.ai.evaluation.ConversationAggregationType + Default is ~azure.ai.evaluation.AggregationType.MEAN. + :type conversation_aggregation_type: ~azure.ai.evaluation.AggregationType """ @override @@ -49,7 +49,7 @@ def __init__( azure_ai_project: dict, credential: TokenCredential, eval_last_turn: bool = False, - conversation_aggregation_type: ConversationAggregationType = ConversationAggregationType.MEAN, + conversation_aggregation_type: AggregationType = AggregationType.MEAN, ): super().__init__(eval_last_turn=eval_last_turn, conversation_aggregation_type=conversation_aggregation_type) self._eval_metric = eval_metric diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_conversation_aggregators.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_conversation_aggregators.py index 3b5839f50d5c..79bb4612cb12 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_conversation_aggregators.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_conversation_aggregators.py @@ -5,17 +5,17 @@ from typing import Callable, List from azure.ai.evaluation._common.math import list_mean from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException -from azure.ai.evaluation._constants import ConversationAggregationType +from azure.ai.evaluation._constants import AggregationType -def GetAggregator(aggregation_type: ConversationAggregationType) -> Callable[[List[float]], float]: - if aggregation_type == ConversationAggregationType.SUM: +def GetAggregator(aggregation_type: AggregationType) -> Callable[[List[float]], float]: + if aggregation_type == AggregationType.SUM: return sum - if aggregation_type == ConversationAggregationType.MEAN: + if aggregation_type == AggregationType.MEAN: return list_mean - if aggregation_type == ConversationAggregationType.MAX: + if aggregation_type == AggregationType.MAX: return max - if aggregation_type == ConversationAggregationType.MIN: + if aggregation_type == AggregationType.MIN: return min raise EvaluationException( message=f"Unaccounted for aggregation type: {aggregation_type}", diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_hate_unfairness.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_hate_unfairness.py index 3a90f13df563..154f2bac52ba 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_hate_unfairness.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_hate_unfairness.py @@ -9,7 +9,7 @@ from azure.ai.evaluation._common.constants import EvaluationMetrics from azure.ai.evaluation._evaluators._common import RaiServiceEvaluatorBase from azure.ai.evaluation._model_configurations import Conversation -from azure.ai.evaluation._constants import ConversationAggregationType +from azure.ai.evaluation._constants import AggregationType @experimental @@ -72,7 +72,7 @@ def __init__( eval_metric=EvaluationMetrics.HATE_FAIRNESS, azure_ai_project=azure_ai_project, credential=credential, - conversation_aggregation_type=ConversationAggregationType.MAX, + conversation_aggregation_type=AggregationType.MAX, ) @overload diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_self_harm.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_self_harm.py index a04799f2c53d..de2d2d20236b 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_self_harm.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_self_harm.py @@ -9,7 +9,7 @@ from azure.ai.evaluation._common.constants import EvaluationMetrics from azure.ai.evaluation._evaluators._common import RaiServiceEvaluatorBase from azure.ai.evaluation._model_configurations import Conversation -from azure.ai.evaluation._constants import ConversationAggregationType +from azure.ai.evaluation._constants import AggregationType @experimental @@ -66,7 +66,7 @@ def __init__( eval_metric=EvaluationMetrics.SELF_HARM, azure_ai_project=azure_ai_project, credential=credential, - conversation_aggregation_type=ConversationAggregationType.MAX, + conversation_aggregation_type=AggregationType.MAX, ) @overload diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_sexual.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_sexual.py index 758050fa4ec9..e990e73a3522 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_sexual.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_sexual.py @@ -9,7 +9,7 @@ from azure.ai.evaluation._common.constants import EvaluationMetrics from azure.ai.evaluation._evaluators._common import RaiServiceEvaluatorBase from azure.ai.evaluation._model_configurations import Conversation -from azure.ai.evaluation._constants import ConversationAggregationType +from azure.ai.evaluation._constants import AggregationType @experimental @@ -68,7 +68,7 @@ def __init__( eval_metric=EvaluationMetrics.SEXUAL, azure_ai_project=azure_ai_project, credential=credential, - conversation_aggregation_type=ConversationAggregationType.MAX, + conversation_aggregation_type=AggregationType.MAX, ) @overload diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_violence.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_violence.py index a8d1f33d320e..c0c71f49983c 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_violence.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_violence.py @@ -9,7 +9,7 @@ from azure.ai.evaluation._common.constants import EvaluationMetrics from azure.ai.evaluation._evaluators._common import RaiServiceEvaluatorBase from azure.ai.evaluation._model_configurations import Conversation -from azure.ai.evaluation._constants import ConversationAggregationType +from azure.ai.evaluation._constants import AggregationType @experimental @@ -68,7 +68,7 @@ def __init__( eval_metric=EvaluationMetrics.VIOLENCE, azure_ai_project=azure_ai_project, credential=credential, - conversation_aggregation_type=ConversationAggregationType.MAX, + conversation_aggregation_type=AggregationType.MAX, ) @overload diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py index fe1b63b70286..a7272af8591f 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py @@ -22,7 +22,7 @@ SelfHarmEvaluator, HateUnfairnessEvaluator, ) -from azure.ai.evaluation._constants import DEFAULT_EVALUATION_RESULTS_FILE_NAME, ConversationAggregationType +from azure.ai.evaluation._constants import DEFAULT_EVALUATION_RESULTS_FILE_NAME, AggregationType from azure.ai.evaluation._evaluate._evaluate import ( _aggregate_metrics, _apply_target_to_data, @@ -701,21 +701,21 @@ def test_conversation_aggregation_types(self, evaluate_test_data_conversion_json # test maxing counting_eval.reset() - counting_eval.set_conversation_aggregation_type(ConversationAggregationType.MAX) + counting_eval.set_conversation_aggregation_type(AggregationType.MAX) results = evaluate(data=evaluate_test_data_conversion_jsonl_file, evaluators=evaluators) assert results["rows"][0]["outputs.count.response"] == 2 assert results["rows"][1]["outputs.count.response"] == 4 # test minimizing counting_eval.reset() - counting_eval.set_conversation_aggregation_type(ConversationAggregationType.MIN) + counting_eval.set_conversation_aggregation_type(AggregationType.MIN) results = evaluate(data=evaluate_test_data_conversion_jsonl_file, evaluators=evaluators) assert results["rows"][0]["outputs.count.response"] == 1 assert results["rows"][1]["outputs.count.response"] == 3 # test sum counting_eval.reset() - counting_eval.set_conversation_aggregation_type(ConversationAggregationType.SUM) + counting_eval.set_conversation_aggregation_type(AggregationType.SUM) results = evaluate(data=evaluate_test_data_conversion_jsonl_file, evaluators=evaluators) assert results["rows"][0]["outputs.count.response"] == 3 assert results["rows"][1]["outputs.count.response"] == 7 From 22a02f2aa29c6082cf80a9cc4d0fd35562847352 Mon Sep 17 00:00:00 2001 From: Miles Holland Date: Wed, 15 Jan 2025 11:12:12 -0500 Subject: [PATCH 07/15] change function names to private, allow agg type retrieval --- .../azure/ai/evaluation/_constants.py | 9 +++++-- .../_evaluators/_common/_base_eval.py | 17 +++++++++--- .../_common/_conversation_aggregators.py | 24 +++++++++++++++++ .../tests/unittests/test_evaluate.py | 27 ++++++++++++++++--- 4 files changed, 68 insertions(+), 9 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_constants.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_constants.py index d047e5e2a3f0..4a4938f57e93 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_constants.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_constants.py @@ -59,13 +59,18 @@ class EvaluationRunProperties: class AggregationType(enum.Enum): - """Defines how multiple conversation turns' worth of numeric results should be evaluated - to produce a single value to define the overall conversation result.""" + """Defines how numeric evaluation results should be aggregated + to produce a single value. Used by individual evaluators to combine per-turn results for + a conversation-based input. In general, wherever this enum is used, it is also possible + to directly assign the underlying aggregation function for more complex use cases. + The 'custom' value is generally not an acceptable input, and should only be used as an output + to indicate that a custom aggregation function has been injected.""" MEAN = "mean" MAX = "max" MIN = "min" SUM = "sum" + CUSTOM = "custom" DEFAULT_EVALUATION_RESULTS_FILE_NAME = "evaluation_results.json" diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_eval.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_eval.py index ff6fd83df233..883ee6a07f28 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_eval.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_eval.py @@ -14,7 +14,7 @@ from azure.ai.evaluation._constants import AggregationType from azure.ai.evaluation._model_configurations import Conversation -from ._conversation_aggregators import GetAggregator +from ._conversation_aggregators import GetAggregator, GetAggregatorType P = ParamSpec("P") T = TypeVar("T") @@ -409,7 +409,7 @@ def _to_async(self) -> "AsyncEvaluatorBase": return self._async_evaluator @final - def set_conversation_aggregation_type(self, conversation_aggregation_type: AggregationType) -> None: + def _set_conversation_aggregation_type(self, conversation_aggregation_type: AggregationType) -> None: """Input a conversation aggregation type to re-assign the aggregator function used by this evaluator for multi-turn conversations. This aggregator is used to combine numeric outputs from each evaluation of a multi-turn conversation into a single top-level result. @@ -421,7 +421,7 @@ def set_conversation_aggregation_type(self, conversation_aggregation_type: Aggre self._conversation_aggregation_function = GetAggregator(conversation_aggregation_type) @final - def set_conversation_aggregator(self, aggregator: Callable[[List[float]], float]) -> None: + def _set_conversation_aggregator(self, aggregator: Callable[[List[float]], float]) -> None: """Set the conversation aggregator function directly. This function will be applied to all numeric outputs of an evaluator when it evaluates a conversation with multiple-turns thus ends up with multiple results per evaluation that is needs to coalesce into a single result. Use when built-in aggregators do not @@ -432,6 +432,17 @@ def set_conversation_aggregator(self, aggregator: Callable[[List[float]], float] """ self._conversation_aggregation_function = aggregator + def _get_conversation_aggregator_type(self) -> AggregationType: + """Get the current conversation aggregation type used by this evaluator. This refers to the + method used when a single input produces multiple evaluation results (ex: when a multi-turn conversation + is inputted into an evaluator that evaluates each turn individually). The individual inputs + are combined by the function implied here to produce a single overall result. + + :return: The conversation aggregation type. + :rtype: ~azure.ai.evaluation.AggregationType + """ + return GetAggregatorType(self._conversation_aggregation_function) + class AsyncEvaluatorBase: """The asynchronous evaluator hidden underneath all evaluators. This makes generous use passing functions diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_conversation_aggregators.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_conversation_aggregators.py index 79bb4612cb12..df76a33413cf 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_conversation_aggregators.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_conversation_aggregators.py @@ -17,9 +17,33 @@ def GetAggregator(aggregation_type: AggregationType) -> Callable[[List[float]], return max if aggregation_type == AggregationType.MIN: return min + if aggregation_type == AggregationType.CUSTOM: + msg = ( + "Cannot 'get' aggregator function associated with custom aggregation enum." + + " This enum value should only be outputted as an indicator of an injected" + + " aggregation function, not inputted directly" + ) + raise EvaluationException( + message=msg, + blame=ErrorBlame.UNKNOWN, + category=ErrorCategory.INVALID_VALUE, + target=ErrorTarget.EVALUATE, + ) raise EvaluationException( message=f"Unaccounted for aggregation type: {aggregation_type}", blame=ErrorBlame.UNKNOWN, category=ErrorCategory.INVALID_VALUE, target=ErrorTarget.EVALUATE, ) + + +def GetAggregatorType(aggregation_function: Callable) -> AggregationType: + if aggregation_function == sum: # pylint: disable=comparison-with-callable + return AggregationType.SUM + if aggregation_function == list_mean: # pylint: disable=comparison-with-callable + return AggregationType.MEAN + if aggregation_function == max: # pylint: disable=comparison-with-callable + return AggregationType.MAX + if aggregation_function == min: # pylint: disable=comparison-with-callable + return AggregationType.MIN + return AggregationType.CUSTOM diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py index a7272af8591f..1a7c369af11c 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py @@ -701,21 +701,21 @@ def test_conversation_aggregation_types(self, evaluate_test_data_conversion_json # test maxing counting_eval.reset() - counting_eval.set_conversation_aggregation_type(AggregationType.MAX) + counting_eval._set_conversation_aggregation_type(AggregationType.MAX) results = evaluate(data=evaluate_test_data_conversion_jsonl_file, evaluators=evaluators) assert results["rows"][0]["outputs.count.response"] == 2 assert results["rows"][1]["outputs.count.response"] == 4 # test minimizing counting_eval.reset() - counting_eval.set_conversation_aggregation_type(AggregationType.MIN) + counting_eval._set_conversation_aggregation_type(AggregationType.MIN) results = evaluate(data=evaluate_test_data_conversion_jsonl_file, evaluators=evaluators) assert results["rows"][0]["outputs.count.response"] == 1 assert results["rows"][1]["outputs.count.response"] == 3 # test sum counting_eval.reset() - counting_eval.set_conversation_aggregation_type(AggregationType.SUM) + counting_eval._set_conversation_aggregation_type(AggregationType.SUM) results = evaluate(data=evaluate_test_data_conversion_jsonl_file, evaluators=evaluators) assert results["rows"][0]["outputs.count.response"] == 3 assert results["rows"][1]["outputs.count.response"] == 7 @@ -725,7 +725,7 @@ def custom_aggregator(values): return sum(values) + 1 counting_eval.reset() - counting_eval.set_conversation_aggregator(custom_aggregator) + counting_eval._set_conversation_aggregator(custom_aggregator) results = evaluate(data=evaluate_test_data_conversion_jsonl_file, evaluators=evaluators) assert results["rows"][0]["outputs.count.response"] == 4 assert results["rows"][1]["outputs.count.response"] == 8 @@ -742,3 +742,22 @@ def test_default_conversation_aggregation_overrides(self): assert eval3._conversation_aggregation_function == max assert eval4._conversation_aggregation_function == max assert eval5._conversation_aggregation_function == list_mean + + def test_conversation_aggregation_type_returns(self): + fake_project = {"subscription_id": "123", "resource_group_name": "123", "project_name": "123"} + eval1 = ViolenceEvaluator(None, fake_project) + # Test builtins + assert eval1._get_conversation_aggregator_type() == AggregationType.MAX + eval1._set_conversation_aggregation_type(AggregationType.SUM) + assert eval1._get_conversation_aggregator_type() == AggregationType.SUM + eval1._set_conversation_aggregation_type(AggregationType.MAX) + assert eval1._get_conversation_aggregator_type() == AggregationType.MAX + eval1._set_conversation_aggregation_type(AggregationType.MIN) + assert eval1._get_conversation_aggregator_type() == AggregationType.MIN + + # test custom + def custom_aggregator(values): + return sum(values) + 1 + + eval1._set_conversation_aggregator(custom_aggregator) + assert eval1._get_conversation_aggregator_type() == AggregationType.CUSTOM From f1f4b82cdeb8a74f0ca3c80100e6190d897f4110 Mon Sep 17 00:00:00 2001 From: Miles Holland Date: Wed, 15 Jan 2025 14:45:47 -0500 Subject: [PATCH 08/15] PR comments --- sdk/evaluation/azure-ai-evaluation/CHANGELOG.md | 8 ++------ .../azure-ai-evaluation/azure/ai/evaluation/_constants.py | 2 ++ .../azure/ai/evaluation/_evaluators/_common/_base_eval.py | 5 +++++ 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md index 3c9de1ebb7f4..63cefd6d0126 100644 --- a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md +++ b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md @@ -11,12 +11,8 @@ - Fixed `AttributeError: 'NoneType' object has no attribute 'get'` while running simulator with 1000+ results - Fixed the non adversarial simulator to run in task-free mode - Content safety evaluators (violence, self harm, sexual, hate/unfairness) return the maximum result as the - main score when aggregating multiple per-turn evaluations from a conversation into a single overall - evaluation score. Other conversation-capable evaluators still default to a mean. -- Evaluator aggregation for producing a main score multi-turn conversations is now configurable. Can be configured - quickly by providing an Enum value (`~azure.ai.evaluation.AggregationType`) via - `eval.set_conversation_aggregation_type` or the exact aggregation function can be assigned directly via - `eval.set_conversation_aggregator`. New options are MEAN, SUM, MAX, and MIN. + main score when aggregating per-turn evaluations from a conversation into an overall + evaluation score. Other conversation-capable evaluators still default to a mean for aggregation. ### Other Changes - Changed minimum required python version to use this package from 3.8 to 3.9 diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_constants.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_constants.py index 4a4938f57e93..0a783838e586 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_constants.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_constants.py @@ -3,6 +3,7 @@ # --------------------------------------------------------- import enum from typing import Literal +from azure.ai.evaluation._common._experimental import experimental class EvaluationMetrics: @@ -58,6 +59,7 @@ class EvaluationRunProperties: EVALUATION_SDK = "_azureml.evaluation_sdk_name" +@experimental class AggregationType(enum.Enum): """Defines how numeric evaluation results should be aggregated to produce a single value. Used by individual evaluators to combine per-turn results for diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_eval.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_eval.py index 883ee6a07f28..f7ccf449de5f 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_eval.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_eval.py @@ -13,6 +13,7 @@ from azure.ai.evaluation._common.utils import remove_optional_singletons from azure.ai.evaluation._constants import AggregationType from azure.ai.evaluation._model_configurations import Conversation +from azure.ai.evaluation._common._experimental import experimental from ._conversation_aggregators import GetAggregator, GetAggregatorType @@ -408,6 +409,7 @@ async def _real_call(self, **kwargs) -> Union[DoEvalResult[T_EvalValue], Aggrega def _to_async(self) -> "AsyncEvaluatorBase": return self._async_evaluator + @experimental @final def _set_conversation_aggregation_type(self, conversation_aggregation_type: AggregationType) -> None: """Input a conversation aggregation type to re-assign the aggregator function used by this evaluator for @@ -420,6 +422,7 @@ def _set_conversation_aggregation_type(self, conversation_aggregation_type: Aggr """ self._conversation_aggregation_function = GetAggregator(conversation_aggregation_type) + @experimental @final def _set_conversation_aggregator(self, aggregator: Callable[[List[float]], float]) -> None: """Set the conversation aggregator function directly. This function will be applied to all numeric outputs @@ -432,6 +435,8 @@ def _set_conversation_aggregator(self, aggregator: Callable[[List[float]], float """ self._conversation_aggregation_function = aggregator + @experimental + @final def _get_conversation_aggregator_type(self) -> AggregationType: """Get the current conversation aggregation type used by this evaluator. This refers to the method used when a single input produces multiple evaluation results (ex: when a multi-turn conversation From ca6b11e3270b9c90934045e927f818affe1d31b5 Mon Sep 17 00:00:00 2001 From: Miles Holland Date: Wed, 15 Jan 2025 15:33:49 -0500 Subject: [PATCH 09/15] test serialization --- .../tests/unittests/test_evaluate.py | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py index 1a7c369af11c..04ddb3455707 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py @@ -1,3 +1,4 @@ +from typing import List, Dict, Union import json import math import os @@ -761,3 +762,32 @@ def custom_aggregator(values): eval1._set_conversation_aggregator(custom_aggregator) assert eval1._get_conversation_aggregator_type() == AggregationType.CUSTOM + + @pytest.mark.parametrize("use_async", ["true", "false"]) # Strings intended + def test_aggregation_serialization(self, evaluate_test_data_conversion_jsonl_file, use_async): + # This test exists to ensure that PF doesn't crash when trying to serialize a + # complex aggregation function. + from test_evaluators.test_inputs_evaluators import CountingEval + + counting_eval = CountingEval() + evaluators = {"count": counting_eval} + + def custom_aggregator(values: List[float]) -> float: + return sum(values) + 1 + + os.environ["AI_EVALS_BATCH_USE_ASYNC"] = use_async + _ = evaluate(data=evaluate_test_data_conversion_jsonl_file, evaluators=evaluators) + counting_eval._set_conversation_aggregation_type(AggregationType.MIN) + _ = evaluate(data=evaluate_test_data_conversion_jsonl_file, evaluators=evaluators) + counting_eval._set_conversation_aggregation_type(AggregationType.SUM) + _ = evaluate(data=evaluate_test_data_conversion_jsonl_file, evaluators=evaluators) + counting_eval._set_conversation_aggregation_type(AggregationType.MAX) + _ = evaluate(data=evaluate_test_data_conversion_jsonl_file, evaluators=evaluators) + if use_async == "true": + counting_eval._set_conversation_aggregator(custom_aggregator) + _ = evaluate(data=evaluate_test_data_conversion_jsonl_file, evaluators=evaluators) + else: + with pytest.raises(EvaluationException) as exc_info: + counting_eval._set_conversation_aggregator(custom_aggregator) + _ = evaluate(data=evaluate_test_data_conversion_jsonl_file, evaluators=evaluators) + assert "Can't pickle local object" in exc_info.value.args[0] From aeb82245f0e31e4d813e54f5369944a83a402d42 Mon Sep 17 00:00:00 2001 From: Miles Holland Date: Wed, 15 Jan 2025 16:14:22 -0500 Subject: [PATCH 10/15] CL --- sdk/evaluation/azure-ai-evaluation/CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md index 63cefd6d0126..01de45c07df2 100644 --- a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md +++ b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md @@ -17,6 +17,8 @@ ### Other Changes - Changed minimum required python version to use this package from 3.8 to 3.9 - Stop dependency on the local promptflow service. No promptflow service will automatically start when running evaluation. +- Evaluators internally allow for custom aggregation. However, this causes serialization failures if evaluated while the + environment variable `AI_EVALS_BATCH_USE_ASYNC` is set to false. ## 1.1.0 (2024-12-12) From e448817da77ef94ddb03262c12dbe4fb45bafc92 Mon Sep 17 00:00:00 2001 From: Miles Holland Date: Thu, 16 Jan 2025 11:20:58 -0500 Subject: [PATCH 11/15] CI adjustment --- .../azure-ai-evaluation/tests/unittests/test_evaluate.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py index 04ddb3455707..d94e1f6e9251 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py @@ -11,6 +11,7 @@ from pandas.testing import assert_frame_equal from promptflow.client import PFClient +from ci_tools.variables import in_ci from azure.ai.evaluation._common.math import list_mean from azure.ai.evaluation import ( ContentSafetyEvaluator, @@ -790,4 +791,8 @@ def custom_aggregator(values: List[float]) -> float: with pytest.raises(EvaluationException) as exc_info: counting_eval._set_conversation_aggregator(custom_aggregator) _ = evaluate(data=evaluate_test_data_conversion_jsonl_file, evaluators=evaluators) - assert "Can't pickle local object" in exc_info.value.args[0] + # CI produces a slightly different error message + if in_ci(): + assert "Can't get local object" in exc_info.value.args[0] + else: + assert "Can't pickle local object" in exc_info.value.args[0] From cb12bc5df1b27095677f5ca0ba8a78705df6879a Mon Sep 17 00:00:00 2001 From: Miles Holland Date: Thu, 16 Jan 2025 12:08:06 -0500 Subject: [PATCH 12/15] try again --- .../azure-ai-evaluation/tests/unittests/test_evaluate.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py index d94e1f6e9251..7263c50ee814 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py @@ -11,7 +11,6 @@ from pandas.testing import assert_frame_equal from promptflow.client import PFClient -from ci_tools.variables import in_ci from azure.ai.evaluation._common.math import list_mean from azure.ai.evaluation import ( ContentSafetyEvaluator, @@ -791,8 +790,4 @@ def custom_aggregator(values: List[float]) -> float: with pytest.raises(EvaluationException) as exc_info: counting_eval._set_conversation_aggregator(custom_aggregator) _ = evaluate(data=evaluate_test_data_conversion_jsonl_file, evaluators=evaluators) - # CI produces a slightly different error message - if in_ci(): - assert "Can't get local object" in exc_info.value.args[0] - else: - assert "Can't pickle local object" in exc_info.value.args[0] + assert "TestEvaluate.test_aggregation_serialization..custom_aggregator" in exc_info.value.args[0] From 2c846d2438782b878259671c2635d16a4db16c37 Mon Sep 17 00:00:00 2001 From: Miles Holland Date: Tue, 21 Jan 2025 08:39:12 -0500 Subject: [PATCH 13/15] perf --- .../tests/unittests/test_evaluate_performance.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate_performance.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate_performance.py index d71483220069..d2df21c72fbf 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate_performance.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate_performance.py @@ -71,7 +71,8 @@ def test_evaluate_parallelism(self, ten_queries_file): diff = end - start # Assume any system running this test can manage to multithread 10 runs into # 2 batches at most, so it should take between 1 and 1.5 seconds. - max_duration = 1.5 + # Increasee to 1.75 to account for CI lag. + max_duration = 1.75 assert diff < max_duration row_result_df = pd.DataFrame(result["rows"]) assert "outputs.slow.result" in row_result_df.columns From 013842a545c81f8c57fd27a09d77e5088a9d251a Mon Sep 17 00:00:00 2001 From: Miles Holland Date: Tue, 21 Jan 2025 10:03:06 -0500 Subject: [PATCH 14/15] skip perf --- .../azure-ai-evaluation/tests/unittests/test_evaluate.py | 1 - .../tests/unittests/test_evaluate_performance.py | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py index f320121bdddc..a8dd0f1226ae 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py @@ -830,4 +830,3 @@ def test_malformed_file_inputs(self, model_config, missing_header_csv_file, miss ) assert "Either 'conversation' or individual inputs must be provided." in str(exc_info.value) - diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate_performance.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate_performance.py index d2df21c72fbf..8245ec231586 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate_performance.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate_performance.py @@ -30,6 +30,7 @@ def ten_queries_file(): @pytest.mark.unittest class TestEvaluatePerformance: @pytest.mark.performance_test + @pytest.mark.skipif(in_ci(), reason="Causes a -9 pytest failure in CI") def test_bulk_evaluate(self, big_f1_data_file): """Test local-only evaluation against 100 inputs.""" f1_score_eval = F1ScoreEvaluator() @@ -71,7 +72,7 @@ def test_evaluate_parallelism(self, ten_queries_file): diff = end - start # Assume any system running this test can manage to multithread 10 runs into # 2 batches at most, so it should take between 1 and 1.5 seconds. - # Increasee to 1.75 to account for CI lag. + # Increase to 1.75 to account for CI lag. max_duration = 1.75 assert diff < max_duration row_result_df = pd.DataFrame(result["rows"]) From 77ca51eb13742054813d8040b11598a6bf6a7b6f Mon Sep 17 00:00:00 2001 From: Miles Holland Date: Tue, 21 Jan 2025 12:02:28 -0500 Subject: [PATCH 15/15] remove skip --- .../tests/unittests/test_evaluate_performance.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate_performance.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate_performance.py index 8245ec231586..515fd6508f1c 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate_performance.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate_performance.py @@ -30,7 +30,6 @@ def ten_queries_file(): @pytest.mark.unittest class TestEvaluatePerformance: @pytest.mark.performance_test - @pytest.mark.skipif(in_ci(), reason="Causes a -9 pytest failure in CI") def test_bulk_evaluate(self, big_f1_data_file): """Test local-only evaluation against 100 inputs.""" f1_score_eval = F1ScoreEvaluator()