From de76ccc63ab824c21e9072c3d3160140fb94c857 Mon Sep 17 00:00:00 2001 From: Sydney Lister Date: Mon, 21 Apr 2025 13:42:46 -0700 Subject: [PATCH] Add CodeVulnerabilityEvaluator and UngroundedAttributesEvaluator to _SafetyEvaluation --- .../_safety_evaluation/_safety_evaluation.py | 41 ++++++++++++++++++- .../tests/unittests/test_safety_evaluation.py | 36 ++++++++++++++++ 2 files changed, 75 insertions(+), 2 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_safety_evaluation/_safety_evaluation.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_safety_evaluation/_safety_evaluation.py index 805f7cf4ab33..d12cb355c3ec 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_safety_evaluation/_safety_evaluation.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_safety_evaluation/_safety_evaluation.py @@ -20,6 +20,8 @@ _fluency, _xpia, _coherence, + _code_vulnerability, + _ungrounded_attributes, ) from azure.ai.evaluation._evaluators._eci._eci import ECIEvaluator from azure.ai.evaluation._evaluate import _evaluate @@ -31,7 +33,7 @@ AdversarialScenario, AdversarialScenarioJailbreak, IndirectAttackSimulator, - DirectAttackSimulator , + DirectAttackSimulator, ) from azure.ai.evaluation.simulator._adversarial_scenario import _UnstableAdversarialScenario from azure.ai.evaluation.simulator._utils import JsonLineList @@ -71,6 +73,7 @@ class _SafetyEvaluator(Enum): """ CONTENT_SAFETY = "content_safety" + CODE_VULNERABILITY = "code_vulnerability" GROUNDEDNESS = "groundedness" PROTECTED_MATERIAL = "protected_material" RELEVANCE = "relevance" @@ -80,6 +83,7 @@ class _SafetyEvaluator(Enum): INDIRECT_ATTACK = "indirect_attack" DIRECT_ATTACK = "direct_attack" ECI = "eci" + UNGROUNDED_ATTRIBUTES = "ungrounded_attributes" @experimental @@ -372,6 +376,10 @@ def _get_scenario( ) if evaluator == _SafetyEvaluator.ECI: return _UnstableAdversarialScenario.ECI + if evaluator == _SafetyEvaluator.CODE_VULNERABILITY: + return AdversarialScenario.ADVERSARIAL_CODE_VULNERABILITY + if evaluator == _SafetyEvaluator.UNGROUNDED_ATTRIBUTES: + return AdversarialScenario.ADVERSARIAL_UNGROUNDED_ATTRIBUTES if evaluator in [ _SafetyEvaluator.GROUNDEDNESS, _SafetyEvaluator.RELEVANCE, @@ -453,6 +461,14 @@ def _get_evaluators( evaluators_dict["eci"] = ECIEvaluator( azure_ai_project=self.azure_ai_project, credential=self.credential ) + elif evaluator == _SafetyEvaluator.CODE_VULNERABILITY: + evaluators_dict["code_vulnerability"] = _code_vulnerability.CodeVulnerabilityEvaluator( + azure_ai_project=self.azure_ai_project, credential=self.credential + ) + elif evaluator == _SafetyEvaluator.UNGROUNDED_ATTRIBUTES: + evaluators_dict["ungrounded_attributes"] = _ungrounded_attributes.UngroundedAttributesEvaluator( + azure_ai_project=self.azure_ai_project, credential=self.credential + ) else: msg = ( f"Invalid evaluator: {evaluator}. Supported evaluators are: {_SafetyEvaluator.__members__.values()}" @@ -560,7 +576,28 @@ def _validate_inputs( category=ErrorCategory.INVALID_VALUE, blame=ErrorBlame.USER_ERROR, ) - + + if _SafetyEvaluator.CODE_VULNERABILITY in evaluators and num_turns > 1: + self.logger.error("Code vulnerability evaluation only supports single-turn conversations.") + msg = "Code vulnerability evaluation only supports single-turn conversations." + raise EvaluationException( + message=msg, + internal_message=msg, + target=ErrorTarget.UNKNOWN, + category=ErrorCategory.INVALID_VALUE, + blame=ErrorBlame.USER_ERROR, + ) + if _SafetyEvaluator.UNGROUNDED_ATTRIBUTES in evaluators and num_turns > 1: + self.logger.error("Ungrounded attributes evaluation only supports single-turn conversations.") + msg = "Ungrounded attributes evaluation only supports single-turn conversations." + raise EvaluationException( + message=msg, + internal_message=msg, + target=ErrorTarget.UNKNOWN, + category=ErrorCategory.INVALID_VALUE, + blame=ErrorBlame.USER_ERROR, + ) + if _SafetyEvaluator.CONTENT_SAFETY in evaluators and scenario and num_turns > 1 and scenario != AdversarialScenario.ADVERSARIAL_CONVERSATION: self.logger.error(f"Adversarial scenario {scenario} is not supported for content safety evaluation with more than 1 turn.") msg = f"Adversarial scenario {scenario} is not supported for content safety evaluation with more than 1 turn." diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_safety_evaluation.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_safety_evaluation.py index a87763dcfd42..49582853a712 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_safety_evaluation.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_safety_evaluation.py @@ -243,3 +243,39 @@ async def test_simulate_no_results(self, mock_call, mock_init, safety_eval, mock target=mock_target, adversarial_scenario=AdversarialScenario.ADVERSARIAL_QA ) assert "outputs generated by the simulator" in str(exc_info.value) + + def test_get_scenario_code_vulnerability(self, safety_eval): + scenario = safety_eval._get_scenario([_SafetyEvaluator.CODE_VULNERABILITY]) + assert scenario == AdversarialScenario.ADVERSARIAL_CODE_VULNERABILITY + + def test_get_scenario_ungrounded_attributes(self, safety_eval): + scenario = safety_eval._get_scenario([_SafetyEvaluator.UNGROUNDED_ATTRIBUTES]) + assert scenario == AdversarialScenario.ADVERSARIAL_UNGROUNDED_ATTRIBUTES + + def test_get_evaluators_code_vulnerability(self, safety_eval): + evaluators = safety_eval._get_evaluators([_SafetyEvaluator.CODE_VULNERABILITY]) + assert "code_vulnerability" in evaluators + assert evaluators["code_vulnerability"].__class__.__name__ == "CodeVulnerabilityEvaluator" + + def test_get_evaluators_ungrounded_attributes(self, safety_eval): + evaluators = safety_eval._get_evaluators([_SafetyEvaluator.UNGROUNDED_ATTRIBUTES]) + assert "ungrounded_attributes" in evaluators + assert evaluators["ungrounded_attributes"].__class__.__name__ == "UngroundedAttributesEvaluator" + + def test_validate_inputs_code_vulnerability_multi_turn(self, safety_eval, mock_target): + with pytest.raises(EvaluationException) as exc_info: + safety_eval._validate_inputs( + target=mock_target, + evaluators=[_SafetyEvaluator.CODE_VULNERABILITY], + num_turns=3, + ) + assert "Code vulnerability evaluation only supports single-turn conversations" in str(exc_info.value) + + def test_validate_inputs_ungrounded_attributes_multi_turn(self, safety_eval, mock_target): + with pytest.raises(EvaluationException) as exc_info: + safety_eval._validate_inputs( + target=mock_target, + evaluators=[_SafetyEvaluator.UNGROUNDED_ATTRIBUTES], + num_turns=3, + ) + assert "Ungrounded attributes evaluation only supports single-turn conversations" in str(exc_info.value)