Introduce Content Safety evaluators

Author: shyamnamboodiripad

src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/README.md

@@ -4,7 +4,7 @@
 
 * [`Microsoft.Extensions.AI.Evaluation`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation) - Defines core abstractions and types for supporting evaluation.
 * [`Microsoft.Extensions.AI.Evaluation.Quality`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Quality) - Contains evaluators that can be used to evaluate the quality of AI responses in your projects including Relevance, Truth, Completeness, Fluency, Coherence, Equivalence and Groundedness.
-* [`Microsoft.Extensions.AI.Evaluation.Safety`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Safety) - Contains evaluators that can be used to evaluate the content safety of AI responses in your projects including Hate and Fairness, Self-Harm, Violence etc.
+* [`Microsoft.Extensions.AI.Evaluation.Safety`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Safety) - Contains a set of evaluators that are built atop the Azure AI Content Safety service that can be used to evaluate the content safety of AI responses in your projects including Protected Material, Groundedness Pro, Ungrounded Attributes, Hate and Unfairness, Self Harm, Violence, Sexual, Code Vulnerability and Indirect Attack.
 * [`Microsoft.Extensions.AI.Evaluation.Reporting`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Reporting) - Contains support for caching LLM responses, storing the results of evaluations and generating reports from that data.
 * [`Microsoft.Extensions.AI.Evaluation.Reporting.Azure`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Reporting.Azure) - Supports the `Microsoft.Extensions.AI.Evaluation.Reporting` library with an implementation for caching LLM responses and storing the evaluation results in an Azure Storage container.
 * [`Microsoft.Extensions.AI.Evaluation.Console`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Console) - A command line dotnet tool for generating reports and managing evaluation data.

src/Libraries/Microsoft.Extensions.AI.Evaluation.Quality/EquivalenceEvaluatorContext.cs

@@ -9,7 +9,8 @@
 namespace Microsoft.Extensions.AI.Evaluation.Quality;
 
 /// <summary>
-/// Contextual information required to evaluate the 'Equivalence' of a response.
+/// Contextual information that the <see cref="EquivalenceEvaluator"/> uses to evaluate the 'Equivalence' of a
+/// response.
 /// </summary>
 /// <param name="groundTruth">
 /// The ground truth response against which the response that is being evaluated is compared.

src/Libraries/Microsoft.Extensions.AI.Evaluation.Quality/GroundednessEvaluatorContext.cs

@@ -9,7 +9,8 @@
 namespace Microsoft.Extensions.AI.Evaluation.Quality;
 
 /// <summary>
-/// Contextual information required to evaluate the 'Groundedness' of a response.
+/// Contextual information that the <see cref="GroundednessEvaluator"/> uses to evaluate the 'Groundedness' of a
+/// response.
 /// </summary>
 /// <param name="groundingContext">
 /// Contextual information against which the 'Groundedness' of a response is evaluated.

src/Libraries/Microsoft.Extensions.AI.Evaluation.Quality/README.md

@@ -4,7 +4,7 @@
 
 * [`Microsoft.Extensions.AI.Evaluation`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation) - Defines core abstractions and types for supporting evaluation.
 * [`Microsoft.Extensions.AI.Evaluation.Quality`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Quality) - Contains evaluators that can be used to evaluate the quality of AI responses in your projects including Relevance, Truth, Completeness, Fluency, Coherence, Equivalence and Groundedness.
-* [`Microsoft.Extensions.AI.Evaluation.Safety`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Safety) - Contains evaluators that can be used to evaluate the content safety of AI responses in your projects including Hate and Fairness, Self-Harm, Violence etc.
+* [`Microsoft.Extensions.AI.Evaluation.Safety`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Safety) - Contains a set of evaluators that are built atop the Azure AI Content Safety service that can be used to evaluate the content safety of AI responses in your projects including Protected Material, Groundedness Pro, Ungrounded Attributes, Hate and Unfairness, Self Harm, Violence, Sexual, Code Vulnerability and Indirect Attack.
 * [`Microsoft.Extensions.AI.Evaluation.Reporting`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Reporting) - Contains support for caching LLM responses, storing the results of evaluations and generating reports from that data.
 * [`Microsoft.Extensions.AI.Evaluation.Reporting.Azure`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Reporting.Azure) - Supports the `Microsoft.Extensions.AI.Evaluation.Reporting` library with an implementation for caching LLM responses and storing the evaluation results in an Azure Storage container.
 * [`Microsoft.Extensions.AI.Evaluation.Console`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Console) - A command line dotnet tool for generating reports and managing evaluation data.

src/Libraries/Microsoft.Extensions.AI.Evaluation.Quality/RelevanceTruthAndCompletenessEvaluator.cs

@@ -207,31 +207,33 @@ void UpdateResult()
             const string Rationales = "Rationales";
             const string Separator = "; ";
 
-            var commonMetadata = new Dictionary<string, string> { ["rtc_evaluation_duration"] = duration };
+            var commonMetadata = new Dictionary<string, string>();
 
             if (!string.IsNullOrWhiteSpace(evaluationResponse.ModelId))
             {
-                commonMetadata["rtc_evaluation_model_used"] = evaluationResponse.ModelId!;
+                commonMetadata["rtc-evaluation-model-used"] = evaluationResponse.ModelId!;
             }
 
             if (evaluationResponse.Usage is UsageDetails usage)
             {
                 if (usage.InputTokenCount is not null)
                 {
-                    commonMetadata["rtc_evaluation_input_tokens_used"] = $"{usage.InputTokenCount}";
+                    commonMetadata["rtc-evaluation-input-tokens-used"] = $"{usage.InputTokenCount}";
                 }
 
                 if (usage.OutputTokenCount is not null)
                 {
-                    commonMetadata["rtc_evaluation_output_tokens_used"] = $"{usage.OutputTokenCount}";
+                    commonMetadata["rtc-evaluation-output-tokens-used"] = $"{usage.OutputTokenCount}";
                 }
 
                 if (usage.TotalTokenCount is not null)
                 {
-                    commonMetadata["rtc_evaluation_total_tokens_used"] = $"{usage.TotalTokenCount}";
+                    commonMetadata["rtc-evaluation-total-tokens-used"] = $"{usage.TotalTokenCount}";
                 }
             }
 
+            commonMetadata["rtc-evaluation-duration"] = duration;
+
             NumericMetric relevance = result.Get<NumericMetric>(RelevanceMetricName);
             relevance.Value = rating.Relevance;
             relevance.Interpretation = relevance.InterpretScore();

src/Libraries/Microsoft.Extensions.AI.Evaluation.Quality/SingleNumericMetricEvaluator.cs

@@ -80,24 +80,24 @@ await chatConfiguration.ChatClient.GetResponseAsync(
 
             if (!string.IsNullOrWhiteSpace(evaluationResponse.ModelId))
             {
-                metric.AddOrUpdateMetadata(name: "evaluation_model_used", value: evaluationResponse.ModelId!);
+                metric.AddOrUpdateMetadata(name: "evaluation-model-used", value: evaluationResponse.ModelId!);
             }
 
             if (evaluationResponse.Usage is UsageDetails usage)
             {
                 if (usage.InputTokenCount is not null)
                 {
-                    metric.AddOrUpdateMetadata(name: "evaluation_input_tokens_used", value: $"{usage.InputTokenCount}");
+                    metric.AddOrUpdateMetadata(name: "evaluation-input-tokens-used", value: $"{usage.InputTokenCount}");
                 }
 
                 if (usage.OutputTokenCount is not null)
                 {
-                    metric.AddOrUpdateMetadata(name: "evaluation_output_tokens_used", value: $"{usage.OutputTokenCount}");
+                    metric.AddOrUpdateMetadata(name: "evaluation-output-tokens-used", value: $"{usage.OutputTokenCount}");
                 }
 
                 if (usage.TotalTokenCount is not null)
                 {
-                    metric.AddOrUpdateMetadata(name: "evaluation_total_tokens_used", value: $"{usage.TotalTokenCount}");
+                    metric.AddOrUpdateMetadata(name: "evaluation-total-tokens-used", value: $"{usage.TotalTokenCount}");
                 }
             }
 
@@ -126,7 +126,7 @@ await chatConfiguration.ChatClient.GetResponseAsync(
         {
             stopwatch.Stop();
             string duration = $"{stopwatch.Elapsed.TotalSeconds.ToString("F2", CultureInfo.InvariantCulture)} s";
-            metric.AddOrUpdateMetadata(name: "evaluation_duration", value: duration);
+            metric.AddOrUpdateMetadata(name: "evaluation-duration", value: duration);
         }
     }
 }

src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/README.md

@@ -4,7 +4,7 @@
 
 * [`Microsoft.Extensions.AI.Evaluation`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation) - Defines core abstractions and types for supporting evaluation.
 * [`Microsoft.Extensions.AI.Evaluation.Quality`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Quality) - Contains evaluators that can be used to evaluate the quality of AI responses in your projects including Relevance, Truth, Completeness, Fluency, Coherence, Equivalence and Groundedness.
-* [`Microsoft.Extensions.AI.Evaluation.Safety`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Safety) - Contains evaluators that can be used to evaluate the content safety of AI responses in your projects including Hate and Fairness, Self-Harm, Violence etc.
+* [`Microsoft.Extensions.AI.Evaluation.Safety`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Safety) - Contains a set of evaluators that are built atop the Azure AI Content Safety service that can be used to evaluate the content safety of AI responses in your projects including Protected Material, Groundedness Pro, Ungrounded Attributes, Hate and Unfairness, Self Harm, Violence, Sexual, Code Vulnerability and Indirect Attack.
 * [`Microsoft.Extensions.AI.Evaluation.Reporting`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Reporting) - Contains support for caching LLM responses, storing the results of evaluations and generating reports from that data.
 * [`Microsoft.Extensions.AI.Evaluation.Reporting.Azure`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Reporting.Azure) - Supports the `Microsoft.Extensions.AI.Evaluation.Reporting` library with an implementation for caching LLM responses and storing the evaluation results in an Azure Storage container.
 * [`Microsoft.Extensions.AI.Evaluation.Console`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Console) - A command line dotnet tool for generating reports and managing evaluation data.

src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/README.md

@@ -4,7 +4,7 @@
 
 * [`Microsoft.Extensions.AI.Evaluation`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation) - Defines core abstractions and types for supporting evaluation.
 * [`Microsoft.Extensions.AI.Evaluation.Quality`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Quality) - Contains evaluators that can be used to evaluate the quality of AI responses in your projects including Relevance, Truth, Completeness, Fluency, Coherence, Equivalence and Groundedness.
-* [`Microsoft.Extensions.AI.Evaluation.Safety`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Safety) - Contains evaluators that can be used to evaluate the content safety of AI responses in your projects including Hate and Fairness, Self-Harm, Violence etc.
+* [`Microsoft.Extensions.AI.Evaluation.Safety`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Safety) - Contains a set of evaluators that are built atop the Azure AI Content Safety service that can be used to evaluate the content safety of AI responses in your projects including Protected Material, Groundedness Pro, Ungrounded Attributes, Hate and Unfairness, Self Harm, Violence, Sexual, Code Vulnerability and Indirect Attack.
 * [`Microsoft.Extensions.AI.Evaluation.Reporting`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Reporting) - Contains support for caching LLM responses, storing the results of evaluations and generating reports from that data.
 * [`Microsoft.Extensions.AI.Evaluation.Reporting.Azure`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Reporting.Azure) - Supports the `Microsoft.Extensions.AI.Evaluation.Reporting` library with an implementation for caching LLM responses and storing the evaluation results in an Azure Storage container.
 * [`Microsoft.Extensions.AI.Evaluation.Console`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Console) - A command line dotnet tool for generating reports and managing evaluation data.

src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/MetricCard.tsx

@@ -42,7 +42,7 @@ const useCardStyles = makeStyles({
         padding: '.75rem', 
         border: `1px solid ${tokens.colorNeutralStroke2}`,
         borderRadius: '4px',
-        width: '8rem',
+        width: '12.5rem',
         cursor: 'pointer',
         transition: 'box-shadow 0.2s ease-in-out, outline 0.2s ease-in-out',
         position: 'relative',
@@ -241,4 +241,4 @@ export const MetricDisplay = ({metric}: {metric: MetricWithNoValue | NumericMetr
         classes.metricPill,
     );
     return (<div className={pillClass}><span className={fg}>{metricValue}</span></div>);
-};
+};

src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/CodeVulnerabilityEvaluator.cs

@@ -0,0 +1,88 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Microsoft.Extensions.AI.Evaluation.Safety;
+
+/// <summary>
+/// An <see cref="IEvaluator"/> that utilizes the Azure AI Content Safety service to evaluate code completion responses
+/// produced by an AI model for the presence of vulnerable code.
+/// </summary>
+/// <remarks>
+/// <para>
+/// <see cref="CodeVulnerabilityEvaluator"/> supports evaluation of code vulnerabilities in the following programming
+/// languages: Python, Java, C++, C#, Go, JavaScript and SQL. It can identify a variety of code vulnerabilities such as
+/// sql injection, stack trace exposure, hardcoded credentials etc.
+/// </para>
+/// <para>
+/// <see cref="CodeVulnerabilityEvaluator"/> returns a <see cref="BooleanMetric"/> with a value of
+/// <see langword="true"/> indicating the presence of an vulnerable code in the evaluated response, and a value of
+/// <see langword="false"/> indicating the absence of vulnerable code.
+/// </para>
+/// <para>
+/// Note that <see cref="CodeVulnerabilityEvaluator"/> does not support evaluation of multimodal content present in
+/// the evaluated responses. Images and other multimodal content present in the evaluated responses will be ignored.
+/// Also note that if a multi-turn conversation is supplied as input, <see cref="CodeVulnerabilityEvaluator"/> will
+/// only evaluate the code present in the last conversation turn. Any code present in the previous conversation turns
+/// will be ignored.
+/// </para>
+/// </remarks>
+/// <param name="contentSafetyServiceConfiguration">
+/// Specifies the Azure AI project that should be used and credentials that should be used when this
+/// <see cref="ContentSafetyEvaluator"/> communicates with the Azure AI Content Safety service to perform
+/// evaluations.
+/// </param>
+public sealed class CodeVulnerabilityEvaluator(ContentSafetyServiceConfiguration contentSafetyServiceConfiguration)
+    : ContentSafetyEvaluator(
+        contentSafetyServiceConfiguration,
+        contentSafetyServiceAnnotationTask: "code vulnerability",
+        evaluatorName: nameof(CodeVulnerabilityEvaluator))
+{
+    /// <summary>
+    /// Gets the <see cref="EvaluationMetric.Name"/> of the <see cref="BooleanMetric"/> returned by
+    /// <see cref="CodeVulnerabilityEvaluator"/>.
+    /// </summary>
+    public static string CodeVulnerabilityMetricName => "Code Vulnerability";
+
+    /// <inheritdoc/>
+    public override IReadOnlyCollection<string> EvaluationMetricNames => [CodeVulnerabilityMetricName];
+
+    /// <inheritdoc/>
+    public override async ValueTask<EvaluationResult> EvaluateAsync(
+        IEnumerable<ChatMessage> messages,
+        ChatResponse modelResponse,
+        ChatConfiguration? chatConfiguration = null,
+        IEnumerable<EvaluationContext>? additionalContext = null,
+        CancellationToken cancellationToken = default)
+    {
+        const string CodeVulnerabilityContentSafetyServiceMetricName = "code_vulnerability";
+
+        EvaluationResult result =
+            await EvaluateContentSafetyAsync(
+                messages,
+                modelResponse,
+                contentSafetyServicePayloadFormat: ContentSafetyServicePayloadFormat.ContextCompletion.ToString(),
+                contentSafetyServiceMetricName: CodeVulnerabilityContentSafetyServiceMetricName,
+                cancellationToken: cancellationToken).ConfigureAwait(false);
+
+        IEnumerable<EvaluationMetric> updatedMetrics =
+            result.Metrics.Values.Select(
+                metric =>
+                {
+                    if (metric.Name == CodeVulnerabilityContentSafetyServiceMetricName)
+                    {
+                        metric.Name = CodeVulnerabilityMetricName;
+                    }
+
+                    return metric;
+                });
+
+        result = new EvaluationResult(updatedMetrics);
+        result.Interpret(metric => metric is BooleanMetric booleanMetric ? booleanMetric.InterpretScore() : null);
+        return result;
+    }
+}