diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.json b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.json index 2d105bc2a32..c283afdfe26 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.json +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.json @@ -4851,4 +4851,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/src/Libraries/Microsoft.Extensions.DataIngestion/Microsoft.Extensions.DataIngestion.csproj b/src/Libraries/Microsoft.Extensions.DataIngestion/Microsoft.Extensions.DataIngestion.csproj index 06c7a0f1804..846ea3c2810 100644 --- a/src/Libraries/Microsoft.Extensions.DataIngestion/Microsoft.Extensions.DataIngestion.csproj +++ b/src/Libraries/Microsoft.Extensions.DataIngestion/Microsoft.Extensions.DataIngestion.csproj @@ -18,6 +18,7 @@ + diff --git a/src/Libraries/Microsoft.Extensions.DataIngestion/OcrDocumentReader.cs b/src/Libraries/Microsoft.Extensions.DataIngestion/OcrDocumentReader.cs new file mode 100644 index 00000000000..e892cff2810 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DataIngestion/OcrDocumentReader.cs @@ -0,0 +1,114 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#pragma warning disable MEDE0001 // Document extraction abstractions are experimental. + +using System; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DocumentExtraction; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.DataIngestion; + +/// +/// Reads documents by extracting structured OCR output using an . +/// +public sealed class OcrDocumentReader : IngestionDocumentReader +{ + private const string BoundingBoxMetadataKey = "bounding_box"; + private const string BoundingRegionMetadataKey = "bounding_region"; + + private readonly IDocumentExtractionClient _documentExtractionClient; + private readonly DocumentExtractionOptions _options; + + /// + /// Initializes a new instance of the class. + /// + /// The OCR client to use for document extraction. + /// Optional OCR options. + public OcrDocumentReader(IDocumentExtractionClient documentExtractionClient, DocumentExtractionOptions? options = null) + { + _documentExtractionClient = Throw.IfNull(documentExtractionClient); + _options = options?.Clone() ?? new DocumentExtractionOptions(); + } + + /// + public override async Task ReadAsync(Stream source, string identifier, string mediaType, CancellationToken cancellationToken = default) + { + _ = Throw.IfNull(source); + _ = Throw.IfNullOrEmpty(identifier); + _ = Throw.IfNullOrEmpty(mediaType); + + DocumentExtractionResult documentExtractionResult = await _documentExtractionClient + .ExtractAsync(source, mediaType, _options.Clone(), cancellationToken: cancellationToken) + .ConfigureAwait(false); + + return Map(documentExtractionResult, identifier); + } + + private static IngestionDocument Map(DocumentExtractionResult documentExtractionResult, string identifier) + { + IngestionDocument document = new(identifier); + + foreach (DocumentPage page in documentExtractionResult.Pages) + { + IngestionDocumentSection section = new(); + int pageNumber = page.PageNumber; + + if (!string.IsNullOrWhiteSpace(page.Text)) + { + section.Elements.Add(new IngestionDocumentParagraph(page.Text) + { + Text = page.Text, + PageNumber = pageNumber + }); + } + + foreach (DocumentImage image in page.Elements.OfType()) + { + section.Elements.Add(MapImage(image, pageNumber)); + } + + document.Sections.Add(section); + } + + return document; + } + + private static IngestionDocumentImage MapImage(DocumentImage image, int pageNumber) + { + DataContent? content = image.Content; + IngestionDocumentImage element = new(CreateImageMarkdown(image)) + { + Content = content?.Data, + MediaType = content?.MediaType, + AlternativeText = image.Caption, + PageNumber = image.BoundingRegion?.PageNumber ?? pageNumber + }; + + if (image.BoundingRegion is not null) + { + if (image.BoundingRegion.GetBounds() is { } bounds) + { + (float left, float top, float right, float bottom) = bounds; + element.Metadata[BoundingBoxMetadataKey] = new[] { left, top, right, bottom }; + } + + element.Metadata[BoundingRegionMetadataKey] = image.BoundingRegion.Polygon.SelectMany(static p => new[] { p.X, p.Y }).ToArray(); + } + + return element; + } + + private static string CreateImageMarkdown(DocumentImage image) + { + string altText = image.Caption ?? string.Empty; + string uri = image.Content?.Uri ?? string.Empty; + + return $"![{altText}]({uri})"; + } +} diff --git a/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DelegatingDocumentExtractionClient.cs b/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DelegatingDocumentExtractionClient.cs new file mode 100644 index 00000000000..27190aafeb4 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DelegatingDocumentExtractionClient.cs @@ -0,0 +1,82 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.DocumentExtraction; + +/// Provides an optional base class for an that passes through calls to another instance. +/// +/// This is recommended as a base type when building clients that can be chained in any order around an +/// underlying . The default implementation simply passes each call to the inner +/// client instance. +/// +[Experimental(DiagnosticIds.Experiments.DocumentExtraction, UrlFormat = DiagnosticIds.UrlFormat)] +public class DelegatingDocumentExtractionClient : IDocumentExtractionClient +{ + /// Initializes a new instance of the class. + /// The wrapped client instance. + /// is . + protected DelegatingDocumentExtractionClient(IDocumentExtractionClient innerClient) + { + InnerClient = Throw.IfNull(innerClient); + } + + /// Gets the inner . + protected IDocumentExtractionClient InnerClient { get; } + + /// + public void Dispose() + { + Dispose(disposing: true); + GC.SuppressFinalize(this); + } + + /// + public virtual Task ExtractAsync( + Stream document, + string mediaType, + DocumentExtractionOptions? options = null, + CancellationToken cancellationToken = default) + { + return InnerClient.ExtractAsync(document, mediaType, options, cancellationToken); + } + + /// + public virtual IAsyncEnumerable ExtractPagesAsync( + Stream document, + string mediaType, + DocumentExtractionOptions? options = null, + CancellationToken cancellationToken = default) + { + return InnerClient.ExtractPagesAsync(document, mediaType, options, cancellationToken); + } + + /// + public virtual object? GetService(Type serviceType, object? serviceKey = null) + { + _ = Throw.IfNull(serviceType); + + // If the key is non-null, we don't know what it means so pass through to the inner service. + return + serviceKey is null && serviceType.IsInstanceOfType(this) ? this : + InnerClient.GetService(serviceType, serviceKey); + } + + /// Provides a mechanism for releasing unmanaged resources. + /// if being called from ; otherwise, . + protected virtual void Dispose(bool disposing) + { + if (disposing) + { + InnerClient.Dispose(); + } + } +} diff --git a/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentBlock.cs b/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentBlock.cs new file mode 100644 index 00000000000..a5ec4601508 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentBlock.cs @@ -0,0 +1,27 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.DocumentExtraction; + +/// Represents a positioned layout block, such as a paragraph, heading, or figure. +[Experimental(DiagnosticIds.Experiments.DocumentExtraction, UrlFormat = DiagnosticIds.UrlFormat)] +public class DocumentBlock : DocumentElement +{ + /// Initializes a new instance of the class. + /// The text content of the block. + /// is . + public DocumentBlock(string text) + { + Text = Throw.IfNull(text); + } + + /// Gets the text content of the block. + public string Text { get; } + + /// Gets or sets the kind of block, for example , , or . + public DocumentBlockKind? Kind { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentBlockKind.cs b/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentBlockKind.cs new file mode 100644 index 00000000000..b2028cccfa8 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentBlockKind.cs @@ -0,0 +1,93 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.ComponentModel; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Text.Json; +using System.Text.Json.Serialization; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.DocumentExtraction; + +/// Describes the kind of an , such as a paragraph, title, or figure. +/// +/// This is a small open set modeled on : the well-known values cover the common +/// layout categories, and a provider may introduce its own value when an engine reports a kind that is +/// not represented here. +/// +[Experimental(DiagnosticIds.Experiments.DocumentExtraction, UrlFormat = DiagnosticIds.UrlFormat)] +[JsonConverter(typeof(Converter))] +[DebuggerDisplay("{Value,nq}")] +public readonly struct DocumentBlockKind : IEquatable +{ + /// Gets the kind representing a paragraph of body text. + public static DocumentBlockKind Paragraph { get; } = new("paragraph"); + + /// Gets the kind representing a title or heading. + public static DocumentBlockKind Title { get; } = new("title"); + + /// Gets the kind representing a figure or image region. + public static DocumentBlockKind Figure { get; } = new("figure"); + + /// Gets the value associated with this . + public string Value { get; } + + /// Initializes a new instance of the struct with the provided value. + /// The value to associate with this . + /// is . + /// is empty or composed entirely of whitespace. + [JsonConstructor] + public DocumentBlockKind(string value) + { + Value = Throw.IfNullOrWhitespace(value); + } + + /// Returns a value indicating whether two instances are equivalent, using a case-insensitive comparison. + /// The first instance to compare. + /// The second instance to compare. + /// if left and right have equivalent values; otherwise, . + public static bool operator ==(DocumentBlockKind left, DocumentBlockKind right) + { + return left.Equals(right); + } + + /// Returns a value indicating whether two instances are not equivalent, using a case-insensitive comparison. + /// The first instance to compare. + /// The second instance to compare. + /// if left and right have different values; otherwise, . + public static bool operator !=(DocumentBlockKind left, DocumentBlockKind right) + { + return !(left == right); + } + + /// + public override bool Equals([NotNullWhen(true)] object? obj) + => obj is DocumentBlockKind otherKind && Equals(otherKind); + + /// + public bool Equals(DocumentBlockKind other) + => string.Equals(Value, other.Value, StringComparison.OrdinalIgnoreCase); + + /// + public override int GetHashCode() + => StringComparer.OrdinalIgnoreCase.GetHashCode(Value); + + /// + public override string ToString() => Value; + + /// Provides a for serializing instances. + [EditorBrowsable(EditorBrowsableState.Never)] + public sealed class Converter : JsonConverter + { + /// + public override DocumentBlockKind Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) => + new(reader.GetString()!); + + /// + public override void Write(Utf8JsonWriter writer, DocumentBlockKind value, JsonSerializerOptions options) => + Throw.IfNull(writer).WriteStringValue(value.Value); + } +} diff --git a/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentBoundingBox.cs b/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentBoundingBox.cs new file mode 100644 index 00000000000..7fa2b53fd4a --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentBoundingBox.cs @@ -0,0 +1,15 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Shared.DiagnosticIds; + +namespace Microsoft.Extensions.DocumentExtraction; + +/// Represents the axis-aligned bounds of a bounding polygon, in the coordinate space defined by the OCR engine. +/// The minimum horizontal coordinate. +/// The minimum vertical coordinate. +/// The maximum horizontal coordinate. +/// The maximum vertical coordinate. +[Experimental(DiagnosticIds.Experiments.DocumentExtraction, UrlFormat = DiagnosticIds.UrlFormat)] +public readonly record struct DocumentBoundingBox(float Left, float Top, float Right, float Bottom); diff --git a/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentBoundingRegion.cs b/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentBoundingRegion.cs new file mode 100644 index 00000000000..fc9f5e3bbf3 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentBoundingRegion.cs @@ -0,0 +1,81 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.DocumentExtraction; + +/// Represents a positioned region on a page. +/// +/// The region is a polygon (a clockwise sequence of vertices) so it can +/// faithfully carry a possibly rotation-skewed quadrilateral, such as Azure Document Intelligence's +/// BoundingRegion.Polygon, without loss. Engines that emit only an axis-aligned rectangle +/// (such as Mistral OCR) can convert via . The same type is reused for +/// layout-block geometry and for field grounding, providing one region primitive across providers. +/// +[Experimental(DiagnosticIds.Experiments.DocumentExtraction, UrlFormat = DiagnosticIds.UrlFormat)] +public class DocumentBoundingRegion +{ + /// Initializes a new instance of the class. + /// The one-based page number the region is on. + /// The clockwise polygon vertices. + /// is . + public DocumentBoundingRegion(int pageNumber, IReadOnlyList polygon) + { + PageNumber = pageNumber; + Polygon = Throw.IfNull(polygon); + } + + /// Gets the one-based page number the region is on. + /// A region can reference a different page than its parent element. + public int PageNumber { get; } + + /// Gets the polygon vertices, in clockwise order. + /// An Azure Document Intelligence quadrilateral is four points. + public IReadOnlyList Polygon { get; } + + /// Builds a clockwise quadrilateral region from an axis-aligned rectangle. + /// The one-based page number the region is on. + /// The left coordinate. + /// The top coordinate. + /// The right coordinate. + /// The bottom coordinate. + /// A region whose polygon is the four corners of the rectangle. + public static DocumentBoundingRegion FromRectangle(int pageNumber, float left, float top, float right, float bottom) + => new(pageNumber, + [ + new DocumentPoint(left, top), + new DocumentPoint(right, top), + new DocumentPoint(right, bottom), + new DocumentPoint(left, bottom), + ]); + + /// Computes the axis-aligned bounds of the polygon. + /// The axis-aligned bounds, or when the polygon has no vertices. + /// + /// Returning for an empty polygon avoids conflating "no geometry" with a real + /// zero-size box located at the origin. + /// + public DocumentBoundingBox? GetBounds() + { + if (Polygon.Count == 0) + { + return null; + } + + float minX = float.MaxValue, minY = float.MaxValue, maxX = float.MinValue, maxY = float.MinValue; + foreach (DocumentPoint point in Polygon) + { + minX = Math.Min(minX, point.X); + maxX = Math.Max(maxX, point.X); + minY = Math.Min(minY, point.Y); + maxY = Math.Max(maxY, point.Y); + } + + return new DocumentBoundingBox(minX, minY, maxX, maxY); + } +} diff --git a/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentCoordinateOrigin.cs b/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentCoordinateOrigin.cs new file mode 100644 index 00000000000..95dad93c8be --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentCoordinateOrigin.cs @@ -0,0 +1,27 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Shared.DiagnosticIds; + +namespace Microsoft.Extensions.DocumentExtraction; + +/// +/// Describes the origin corner and vertical axis direction of an OCR coordinate space. +/// +/// +/// Engines disagree on where a page's coordinate origin sits and which way the y axis grows: rasterized +/// page images place the origin at the top-left with y increasing downward, whereas PDF-native +/// (point) coordinates place it at the bottom-left with y increasing upward. The origin is reported per +/// page on , alongside , so bounding +/// regions from different engines can be compared and normalized without guessing the convention. +/// +[Experimental(DiagnosticIds.Experiments.DocumentExtraction, UrlFormat = DiagnosticIds.UrlFormat)] +public enum DocumentCoordinateOrigin +{ + /// The origin sits at the top-left corner, with the y axis increasing downward. The convention for rasterized page images. + TopLeft, + + /// The origin sits at the bottom-left corner, with the y axis increasing upward. The convention for PDF-native point coordinates. + BottomLeft, +} diff --git a/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentCoordinateUnit.cs b/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentCoordinateUnit.cs new file mode 100644 index 00000000000..d8aa185691e --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentCoordinateUnit.cs @@ -0,0 +1,37 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Shared.DiagnosticIds; + +namespace Microsoft.Extensions.DocumentExtraction; + +/// +/// Describes the unit in which OCR geometry coordinates ( and +/// ) are expressed. +/// +/// +/// Coordinate conventions differ across OCR engines: some report pixels of the rendered page image, +/// some report a physical unit such as points or inches, and some normalize to the page. The unit is +/// reported per page on , paired with an +/// and the page dimensions (), so a +/// consumer can interpret or normalize regions with engine-agnostic code. It is per page because engines +/// can emit different units for different pages of one document (for example, a batch mixing image and +/// PDF inputs). Unlike the taxonomy kinds (, ), +/// the set of coordinate units is physically bounded, so it is modeled as a closed enumeration. +/// +[Experimental(DiagnosticIds.Experiments.DocumentExtraction, UrlFormat = DiagnosticIds.UrlFormat)] +public enum DocumentCoordinateUnit +{ + /// Coordinates expressed in pixels of the rendered page image. + Pixel, + + /// Coordinates expressed in points (1/72 inch), the native unit of PDF content. + Point, + + /// Coordinates expressed in inches. + Inch, + + /// Coordinates normalized to the range [0, 1] relative to the page width and height. + Normalized, +} diff --git a/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentElement.cs b/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentElement.cs new file mode 100644 index 00000000000..386eccc8287 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentElement.cs @@ -0,0 +1,48 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using System.Text.Json.Serialization; +using Microsoft.Extensions.AI; +using Microsoft.Shared.DiagnosticIds; + +namespace Microsoft.Extensions.DocumentExtraction; + +/// Represents a single positioned element within an , such as a block, table, or image. +/// +/// Elements appear in in reading order, so a consumer can walk a page as one +/// heterogeneous stream and project the kinds it cares about with +/// (for example +/// page.Elements.OfType<DocumentTable>()). The full page text is available directly on +/// . The base is shaped to be promotable to a future shared document-element type. +/// +[Experimental(DiagnosticIds.Experiments.DocumentExtraction, UrlFormat = DiagnosticIds.UrlFormat)] +[JsonPolymorphic(TypeDiscriminatorPropertyName = "$type")] +[JsonDerivedType(typeof(DocumentBlock), typeDiscriminator: "block")] +[JsonDerivedType(typeof(DocumentTable), typeDiscriminator: "table")] +[JsonDerivedType(typeof(DocumentImage), typeDiscriminator: "image")] +public abstract class DocumentElement +{ + /// Initializes a new instance of the class. + protected DocumentElement() + { + } + + /// Gets or sets the region of the page the element occupies, when the engine provides geometry. + public DocumentBoundingRegion? BoundingRegion { get; set; } + + /// Gets or sets the confidence for the element in the range [0, 1], when available. + public double? Confidence { get; set; } + + /// Gets or sets the provider-native object underlying this element. + /// + /// If an is created to represent an underlying object from another object model, + /// this property can store that original object. This can be useful for debugging or for enabling a + /// consumer to access the underlying object model if needed. + /// + [JsonIgnore] + public object? RawRepresentation { get; set; } + + /// Gets or sets any additional properties associated with the element. + public AdditionalPropertiesDictionary? AdditionalProperties { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentExtractionClientExtensions.cs b/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentExtractionClientExtensions.cs new file mode 100644 index 00000000000..88dc4b5106f --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentExtractionClientExtensions.cs @@ -0,0 +1,224 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Net.Http; +using System.Runtime.InteropServices; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.DocumentExtraction; + +/// Provides extension methods for . +[Experimental(DiagnosticIds.Experiments.DocumentExtraction, UrlFormat = DiagnosticIds.UrlFormat)] +public static class DocumentExtractionClientExtensions +{ + /// Asks the for an object of type . + /// The type of the object to be retrieved. + /// The client. + /// An optional key that can be used to help identify the target service. + /// The found object, otherwise . + /// is . + /// + /// The purpose of this method is to allow for the retrieval of strongly typed services that might be + /// provided by the , including itself or any services it might be wrapping. + /// + public static TService? GetService(this IDocumentExtractionClient client, object? serviceKey = null) + { + _ = Throw.IfNull(client); + + return (TService?)client.GetService(typeof(TService), serviceKey); + } + + /// Runs OCR over a single document provided as a . + /// The client. + /// The document content to parse. + /// The OCR options to configure the request. + /// The to monitor for cancellation requests. The default is . + /// The structured OCR result. + /// or is . + public static Task ExtractAsync( + this IDocumentExtractionClient client, + DataContent document, + DocumentExtractionOptions? options = null, + CancellationToken cancellationToken = default) + { + _ = Throw.IfNull(client); + _ = Throw.IfNull(document); + + var documentStream = MemoryMarshal.TryGetArray(document.Data, out var array) ? + new MemoryStream(array.Array!, array.Offset, array.Count) : + new MemoryStream(document.Data.ToArray()); + + return client.ExtractAsync(documentStream, document.MediaType, options, cancellationToken); + } + + /// Runs OCR over a single document referenced by a . + /// The client. + /// The document reference to parse. + /// The OCR options to configure the request. + /// The to monitor for cancellation requests. The default is . + /// The structured OCR result. + /// or is . + /// The references a remote URI, which this overload does not fetch. + /// + /// This overload handles self-contained data: URIs by delegating to the + /// overload. It intentionally does no file or network IO: for + /// file: and remote (http/https) URIs it throws, because whether to read/download + /// the bytes or hand the URL to the engine natively (for example Mistral document_url or Azure + /// Document Intelligence uriSource) is an open design question the abstraction does not decide. + /// To download a remote document explicitly, use + /// + /// with a caller-supplied ; or read the bytes yourself and pass a + /// or ; or use an engine that accepts a URL directly. + /// + public static Task ExtractAsync( + this IDocumentExtractionClient client, + UriContent document, + DocumentExtractionOptions? options = null, + CancellationToken cancellationToken = default) + { + _ = Throw.IfNull(client); + _ = Throw.IfNull(document); + + Uri uri = document.Uri; + if (uri.IsAbsoluteUri && string.Equals(uri.Scheme, "data", StringComparison.OrdinalIgnoreCase)) + { + // Reuse DataContent's data: URI parsing, then defer to the DataContent overload. + return client.ExtractAsync(new DataContent(uri), options, cancellationToken); + } + + throw new NotSupportedException( + "This overload handles only self-contained data: URIs. For file or remote URIs, read the bytes " + + "and pass a stream or DataContent, or use an engine that accepts a URL natively."); + } + + /// + /// Runs OCR over a single document referenced by a , explicitly downloading the + /// bytes with a caller-supplied when the reference is remote. + /// + /// The client. + /// The document reference to download and parse. + /// The used to download remote (http/https) documents. + /// The OCR options to configure the request. + /// The to monitor for cancellation requests. The default is . + /// The structured OCR result. + /// , , or is . + /// The uses a scheme other than data:, http, or https. + /// + /// This is the explicit, opt-in counterpart to + /// , + /// which never touches the network. Self-contained data: URIs are handled inline (no download); + /// http/https URIs are fetched with the supplied and passed + /// to the stream-based extraction path. The abstraction performs no ambient network IO: the caller owns + /// the and therefore its handlers, authentication, timeouts, and lifetime. + /// Engines that accept a URL natively (for example Azure Document Intelligence uriSource) should + /// expose that on the concrete client instead; this extension serves the bytes-only majority. + /// + public static async Task ExtractFromUriAsync( + this IDocumentExtractionClient client, + UriContent document, + HttpClient httpClient, + DocumentExtractionOptions? options = null, + CancellationToken cancellationToken = default) + { + _ = Throw.IfNull(client); + _ = Throw.IfNull(document); + _ = Throw.IfNull(httpClient); + + Uri uri = document.Uri; + + // Self-contained data: URIs carry their own bytes - no download needed. + if (uri.IsAbsoluteUri && string.Equals(uri.Scheme, "data", StringComparison.OrdinalIgnoreCase)) + { + return await client.ExtractAsync(document, options, cancellationToken).ConfigureAwait(false); + } + + if (!uri.IsAbsoluteUri || + (!string.Equals(uri.Scheme, "http", StringComparison.OrdinalIgnoreCase) && + !string.Equals(uri.Scheme, "https", StringComparison.OrdinalIgnoreCase))) + { + throw new NotSupportedException( + "ExtractFromUriAsync downloads only http/https URIs (and inlines data: URIs). For other " + + "schemes, read the bytes and pass a stream or DataContent."); + } + + using HttpResponseMessage response = await httpClient + .GetAsync(uri, HttpCompletionOption.ResponseHeadersRead, cancellationToken) + .ConfigureAwait(false); + _ = response.EnsureSuccessStatusCode(); + + string mediaType = response.Content.Headers.ContentType?.MediaType ?? document.MediaType; + +#if NET + using Stream contentStream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); +#else + using Stream contentStream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false); +#endif + return await client.ExtractAsync(contentStream, mediaType, options, cancellationToken).ConfigureAwait(false); + } + + /// Runs streaming OCR over a single document provided as a . + /// The client. + /// The document content to parse. + /// The OCR options to configure the request. + /// The to monitor for cancellation requests. The default is . + /// The structured OCR updates representing the streamed output. + /// or is . + public static IAsyncEnumerable ExtractPagesAsync( + this IDocumentExtractionClient client, + DataContent document, + DocumentExtractionOptions? options = null, + CancellationToken cancellationToken = default) + { + _ = Throw.IfNull(client); + _ = Throw.IfNull(document); + + var documentStream = MemoryMarshal.TryGetArray(document.Data, out var array) ? + new MemoryStream(array.Array!, array.Offset, array.Count) : + new MemoryStream(document.Data.ToArray()); + + return client.ExtractPagesAsync(documentStream, document.MediaType, options, cancellationToken); + } + + /// Runs streaming OCR over a single document referenced by a . + /// The client. + /// The document reference to parse. + /// The OCR options to configure the request. + /// The to monitor for cancellation requests. The default is . + /// The structured OCR updates representing the streamed output. + /// or is . + /// The references a remote URI, which this overload does not fetch. + /// + /// This is the streaming counterpart to + /// ; it handles only + /// self-contained data: URIs and does no file or network IO. For file: and remote URIs it + /// throws, for the same reasons documented on the unary overload. + /// + public static IAsyncEnumerable ExtractPagesAsync( + this IDocumentExtractionClient client, + UriContent document, + DocumentExtractionOptions? options = null, + CancellationToken cancellationToken = default) + { + _ = Throw.IfNull(client); + _ = Throw.IfNull(document); + + Uri uri = document.Uri; + if (uri.IsAbsoluteUri && string.Equals(uri.Scheme, "data", StringComparison.OrdinalIgnoreCase)) + { + // Reuse DataContent's data: URI parsing, then defer to the DataContent overload. + return client.ExtractPagesAsync(new DataContent(uri), options, cancellationToken); + } + + throw new NotSupportedException( + "This overload handles only self-contained data: URIs. For file or remote URIs, read the bytes " + + "and pass a stream or DataContent, or use an engine that accepts a URL natively."); + } +} diff --git a/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentExtractionClientMetadata.cs b/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentExtractionClientMetadata.cs new file mode 100644 index 00000000000..5f22cfc391a --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentExtractionClientMetadata.cs @@ -0,0 +1,38 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Diagnostics.CodeAnalysis; +using Microsoft.Shared.DiagnosticIds; + +namespace Microsoft.Extensions.DocumentExtraction; + +/// Provides metadata about an . +[Experimental(DiagnosticIds.Experiments.DocumentExtraction, UrlFormat = DiagnosticIds.UrlFormat)] +public class DocumentExtractionClientMetadata +{ + /// Initializes a new instance of the class. + /// The name of the OCR provider, if applicable. + /// The URL for accessing the OCR provider, if applicable. + /// The identifier of the model used by default, if applicable. + public DocumentExtractionClientMetadata(string? providerName = null, Uri? providerUri = null, string? defaultModelId = null) + { + DefaultModelId = defaultModelId; + ProviderName = providerName; + ProviderUri = providerUri; + } + + /// Gets the name of the OCR provider. + public string? ProviderName { get; } + + /// Gets the URL for accessing the OCR provider. + public Uri? ProviderUri { get; } + + /// Gets the identifier of the default model used by this OCR client. + /// + /// This value can be if the name is unknown or if there are multiple possible + /// models associated with this instance. An individual request can override this value via + /// . + /// + public string? DefaultModelId { get; } +} diff --git a/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentExtractionOptions.cs b/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentExtractionOptions.cs new file mode 100644 index 00000000000..86cd9b1b3c1 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentExtractionOptions.cs @@ -0,0 +1,32 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.AI; +using Microsoft.Shared.DiagnosticIds; + +namespace Microsoft.Extensions.DocumentExtraction; + +/// Represents the options to configure an OCR request. +/// +/// Normalized options common to engines, plus an bag for +/// provider-specific settings, mirroring ChatOptions. +/// +[Experimental(DiagnosticIds.Experiments.DocumentExtraction, UrlFormat = DiagnosticIds.UrlFormat)] +public class DocumentExtractionOptions +{ + /// Gets or sets the model or deployment identifier to use for this request. + public string? ModelId { get; set; } + + /// Gets or sets any additional provider-specific request settings. + public AdditionalPropertiesDictionary? AdditionalProperties { get; set; } + + /// Produces a clone of the current instance. + /// A shallow clone of the options instance. + public DocumentExtractionOptions Clone() => + new() + { + ModelId = ModelId, + AdditionalProperties = AdditionalProperties?.Clone(), + }; +} diff --git a/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentExtractionPageResult.cs b/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentExtractionPageResult.cs new file mode 100644 index 00000000000..ccf708168ab --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentExtractionPageResult.cs @@ -0,0 +1,64 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using System.Text.Json.Serialization; +using Microsoft.Extensions.AI; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.DocumentExtraction; + +/// Represents a single streaming update from an . +/// +/// +/// An request produces a sequence of +/// instances, one per page as that page finishes (carrying the completed +/// ). Progress rides along on each result via and +/// for long-running operations such as Azure Document Intelligence. Completion is +/// signaled by the end of the sequence. +/// +/// +/// The relationship between and is codified in +/// , which reassembles a stream of updates into a +/// single . The conversion can be slightly lossy: for example, only one +/// slot is available on the assembled . +/// +/// +[Experimental(DiagnosticIds.Experiments.DocumentExtraction, UrlFormat = DiagnosticIds.UrlFormat)] +public class DocumentExtractionPageResult +{ + /// Initializes a new instance of the class with the page completed in this update. + /// The page produced in this update. + /// is . + [JsonConstructor] + public DocumentExtractionPageResult(DocumentPage page) + { + Page = Throw.IfNull(page); + } + + /// Gets the page produced in this update. + public DocumentPage Page { get; } + + /// Gets or sets the number of pages processed so far, when known. + public int? PagesProcessed { get; set; } + + /// Gets or sets the total number of pages, when known. + public int? TotalPages { get; set; } + + /// Gets or sets usage details associated with the request, when reported. + /// Usage is typically carried on a terminal update once the full document has been processed. + public DocumentExtractionUsage? Usage { get; set; } + + /// Gets or sets the provider-native object underlying this update. + /// + /// If an is created to represent an underlying object from another object + /// model, this property can store that original object. This can be useful for debugging or for enabling + /// a consumer to access the underlying object model if needed. + /// + [JsonIgnore] + public object? RawRepresentation { get; set; } + + /// Gets or sets any additional properties associated with the update. + public AdditionalPropertiesDictionary? AdditionalProperties { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentExtractionPageResultExtensions.cs b/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentExtractionPageResultExtensions.cs new file mode 100644 index 00000000000..822f6fdc2f3 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentExtractionPageResultExtensions.cs @@ -0,0 +1,91 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.DocumentExtraction; + +/// Provides extension methods for working with instances. +[Experimental(DiagnosticIds.Experiments.DocumentExtraction, UrlFormat = DiagnosticIds.UrlFormat)] +public static class DocumentExtractionPageResultExtensions +{ + /// Combines instances into a single . + /// The updates to be combined. + /// The combined . + /// is . + public static DocumentExtractionResult ToDocumentExtractionResult(this IEnumerable updates) + { + _ = Throw.IfNull(updates); + + List pages = []; + DocumentExtractionResult result = new(pages); + + foreach (var update in updates) + { + ProcessUpdate(update, pages, result); + } + + return result; + } + + /// Combines instances into a single . + /// The updates to be combined. + /// The to monitor for cancellation requests. The default is . + /// The combined . + /// is . + public static Task ToDocumentExtractionResultAsync( + this IAsyncEnumerable updates, CancellationToken cancellationToken = default) + { + _ = Throw.IfNull(updates); + + return ToResultAsync(updates, cancellationToken); + + static async Task ToResultAsync( + IAsyncEnumerable updates, CancellationToken cancellationToken) + { + List pages = []; + DocumentExtractionResult result = new(pages); + + await foreach (var update in updates.WithCancellation(cancellationToken).ConfigureAwait(false)) + { + ProcessUpdate(update, pages, result); + } + + return result; + } + } + + /// Incorporates one into the assembled . + /// The update to process. + /// The accumulating list of pages backing . + /// The being assembled. + private static void ProcessUpdate(DocumentExtractionPageResult update, List pages, DocumentExtractionResult result) + { + pages.Add(update.Page); + + if (update.Usage is not null) + { + result.Usage = update.Usage; + } + + if (update.AdditionalProperties is not null) + { + if (result.AdditionalProperties is null) + { + result.AdditionalProperties = new(update.AdditionalProperties); + } + else + { + foreach (var entry in update.AdditionalProperties) + { + result.AdditionalProperties[entry.Key] = entry.Value; + } + } + } + } +} diff --git a/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentExtractionResult.cs b/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentExtractionResult.cs new file mode 100644 index 00000000000..feda650c1e8 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentExtractionResult.cs @@ -0,0 +1,49 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using Microsoft.Extensions.AI; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.DocumentExtraction; + +/// Represents the structured result of an OCR / document-parsing request. +/// +/// The result normalizes the content common to every engine (text, pages, tables, bounding +/// regions) while preserving everything provider-specific via +/// and , mirroring how +/// ChatResponse normalizes the common surface and preserves the raw. +/// +[Experimental(DiagnosticIds.Experiments.DocumentExtraction, UrlFormat = DiagnosticIds.UrlFormat)] +public class DocumentExtractionResult +{ + /// Initializes a new instance of the class. + /// The per-page structured content. + /// is . + public DocumentExtractionResult(IReadOnlyList pages) + { + Pages = Throw.IfNull(pages); + } + + /// Gets the per-page structured content (text, tables, blocks). + public IReadOnlyList Pages { get; } + + /// Gets the full-document text, formed by joining the per-page text. + public string Text => string.Join("\n\n", Pages.Select(p => p.Text)); + + /// Gets or sets usage details associated with the request. + public DocumentExtractionUsage? Usage { get; set; } + + /// Gets or sets the provider-native object underlying this result. + /// + /// The escape hatch for provider richness that does not map onto the normalized surface, mirroring + /// ChatResponse.RawRepresentation. Nothing is lost. + /// + public object? RawRepresentation { get; set; } + + /// Gets or sets any additional properties associated with the result. + public AdditionalPropertiesDictionary? AdditionalProperties { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentExtractionUsage.cs b/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentExtractionUsage.cs new file mode 100644 index 00000000000..83e1f5176a2 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentExtractionUsage.cs @@ -0,0 +1,29 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.AI; +using Microsoft.Shared.DiagnosticIds; + +namespace Microsoft.Extensions.DocumentExtraction; + +/// Represents usage details associated with an OCR request. +[Experimental(DiagnosticIds.Experiments.DocumentExtraction, UrlFormat = DiagnosticIds.UrlFormat)] +public class DocumentExtractionUsage +{ + /// Gets or sets the number of pages processed by the request, when known. + public int? PagesProcessed { get; set; } + + /// Gets or sets the number of input tokens consumed, when the engine reports token usage. + /// Typically reported only by vision-LLM OCR paths; classic OCR engines usually leave this . + public int? InputTokenCount { get; set; } + + /// Gets or sets the number of output tokens produced, when the engine reports token usage. + public int? OutputTokenCount { get; set; } + + /// Gets or sets the total number of tokens (input plus output), when the engine reports token usage. + public int? TotalTokenCount { get; set; } + + /// Gets or sets any additional provider-specific usage details. + public AdditionalPropertiesDictionary? AdditionalProperties { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentImage.cs b/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentImage.cs new file mode 100644 index 00000000000..29be12e5880 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentImage.cs @@ -0,0 +1,26 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.AI; +using Microsoft.Shared.DiagnosticIds; + +namespace Microsoft.Extensions.DocumentExtraction; + +/// Represents an image or figure extracted from a page during OCR. +/// +/// Populated when the engine supports it and images are present. Every +/// member is optional so each implementer fills what it can provide: document-native engines (for +/// example Mistral OCR inline images, or Azure Document Intelligence figures) populate +/// with the rendered image bytes, whereas a vision-LLM transcriber that cannot emit bytes may instead +/// populate only . This lets one shape serve both provider archetypes. +/// +[Experimental(DiagnosticIds.Experiments.DocumentExtraction, UrlFormat = DiagnosticIds.UrlFormat)] +public class DocumentImage : DocumentElement +{ + /// Gets or sets the rendered image bytes, when the engine returns them. + public DataContent? Content { get; set; } + + /// Gets or sets a caption or description of the image, when available. + public string? Caption { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentPage.cs b/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentPage.cs new file mode 100644 index 00000000000..7b0ee146089 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentPage.cs @@ -0,0 +1,76 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Text.Json.Serialization; +using Microsoft.Extensions.AI; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.DocumentExtraction; + +/// Represents one page of structured OCR output. +[Experimental(DiagnosticIds.Experiments.DocumentExtraction, UrlFormat = DiagnosticIds.UrlFormat)] +public class DocumentPage +{ + /// Initializes a new instance of the class. + /// The one-based page number. + /// The structured text for this page. + /// is . + public DocumentPage(int pageNumber, string text) + { + PageNumber = pageNumber; + Text = Throw.IfNull(text); + } + + /// Gets the one-based page number. + public int PageNumber { get; } + + /// Gets the structured text for this page, with headings, tables, and reading order preserved. + public string Text { get; } + + /// Gets or sets the elements extracted from this page, in reading order. + /// + /// A single heterogeneous stream of blocks, tables, and images in the order a human would read them. + /// Project a specific kind with , + /// for example Elements.OfType<DocumentTable>(). The full page text is available directly on + /// , so reading-order consumers do not need geometry math. + /// + public IReadOnlyList Elements { get; set; } = []; + + /// Gets or sets the page dimensions (width and height), expressed in , when the engine provides them. + /// + /// Together with and , the dimensions let a consumer + /// interpret or normalize the geometry ( / ) on this page with + /// engine-agnostic code. For example, dividing a coordinate by the corresponding dimension yields a page-relative + /// [0, 1] value regardless of the native unit. + /// + public DocumentPageDimensions? Dimensions { get; set; } + + /// Gets or sets the unit in which this page's geometry coordinates are expressed, when known. + /// + /// Reported per page: engines can emit different units for different pages of one document (for example, a batch + /// mixing image and PDF inputs). Applies to every on the page and to + /// . When , the geometry should be treated as an opaque, + /// provider-specific coordinate space. + /// + public DocumentCoordinateUnit? CoordinateUnit { get; set; } + + /// Gets or sets the origin corner and axis direction of this page's geometry coordinates, when known. + public DocumentCoordinateOrigin? CoordinateOrigin { get; set; } + + /// Gets or sets the provider-native object underlying this page. + /// + /// If an is created to represent an underlying object from another object model, this + /// property can store that original object. This can be useful for debugging or for enabling a consumer to + /// access the underlying object model if needed. Because the page node rides through + /// reduction, provider-native page data set here survives + /// into . + /// + [JsonIgnore] + public object? RawRepresentation { get; set; } + + /// Gets or sets any additional properties associated with the page. + public AdditionalPropertiesDictionary? AdditionalProperties { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentPageDimensions.cs b/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentPageDimensions.cs new file mode 100644 index 00000000000..1c601fbde88 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentPageDimensions.cs @@ -0,0 +1,19 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Shared.DiagnosticIds; + +namespace Microsoft.Extensions.DocumentExtraction; + +/// Represents the width and height of an , expressed in the page's . +/// The page width. +/// The page height. +/// +/// Together with the page's and , the +/// dimensions let a consumer interpret or normalize the geometry ( / ) +/// on the page with engine-agnostic code. For example, dividing a coordinate by the corresponding dimension yields a +/// page-relative [0, 1] value regardless of the native unit. +/// +[Experimental(DiagnosticIds.Experiments.DocumentExtraction, UrlFormat = DiagnosticIds.UrlFormat)] +public readonly record struct DocumentPageDimensions(float Width, float Height); diff --git a/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentPoint.cs b/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentPoint.cs new file mode 100644 index 00000000000..0e8b2584277 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentPoint.cs @@ -0,0 +1,13 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Shared.DiagnosticIds; + +namespace Microsoft.Extensions.DocumentExtraction; + +/// Represents a single vertex of a bounding polygon, in the coordinate space defined by the OCR engine. +/// The horizontal coordinate. +/// The vertical coordinate. +[Experimental(DiagnosticIds.Experiments.DocumentExtraction, UrlFormat = DiagnosticIds.UrlFormat)] +public readonly record struct DocumentPoint(float X, float Y); diff --git a/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentTable.cs b/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentTable.cs new file mode 100644 index 00000000000..6f8f2d0466a --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentTable.cs @@ -0,0 +1,50 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using Microsoft.Shared.DiagnosticIds; + +namespace Microsoft.Extensions.DocumentExtraction; + +/// Represents a table extracted from a document. +/// +/// Cells are the primary, structured representation (row and column indices with spans, the Azure +/// Document Intelligence shape) and are authoritative when non-. +/// is the fallback for engines that only emit markdown or HTML +/// (such as Mistral OCR). Consumers prefer when present and fall back to +/// otherwise. On the markdown-only path and +/// may be 0 because the structure was not enumerated. +/// +[Experimental(DiagnosticIds.Experiments.DocumentExtraction, UrlFormat = DiagnosticIds.UrlFormat)] +public class DocumentTable : DocumentElement +{ + /// Initializes a new instance of the class. + /// The number of rows in the table. + /// The number of columns in the table. + /// The structured cells, or when only markdown is available. + /// The markdown or HTML representation, or when cells are available. + public DocumentTable( + int rowCount, + int columnCount, + IReadOnlyList? cells = null, + string? markdownRepresentation = null) + { + RowCount = rowCount; + ColumnCount = columnCount; + Cells = cells; + MarkdownRepresentation = markdownRepresentation; + } + + /// Gets the number of rows in the table. + public int RowCount { get; } + + /// Gets the number of columns in the table. + public int ColumnCount { get; } + + /// Gets the structured cells, or when the engine only returned markdown. + public IReadOnlyList? Cells { get; } + + /// Gets the markdown or HTML table text, or when only cells were returned. + public string? MarkdownRepresentation { get; } +} diff --git a/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentTableCell.cs b/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentTableCell.cs new file mode 100644 index 00000000000..a3df51e245d --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentTableCell.cs @@ -0,0 +1,78 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Text.Json.Serialization; +using Microsoft.Extensions.AI; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.DocumentExtraction; + +/// Represents a single cell within an . +[Experimental(DiagnosticIds.Experiments.DocumentExtraction, UrlFormat = DiagnosticIds.UrlFormat)] +public class DocumentTableCell +{ + /// Initializes a new instance of the class. + /// The zero-based row index of the cell. + /// The zero-based column index of the cell. + /// The text content of the cell. + /// is . + public DocumentTableCell(int rowIndex, int columnIndex, string content) + { + RowIndex = rowIndex; + ColumnIndex = columnIndex; + Content = Throw.IfNull(content); + } + + /// Gets or sets the role of the cell, for example or . + public DocumentTableCellKind? Kind { get; set; } + + /// Gets the zero-based row index of the cell. + public int RowIndex { get; } + + /// Gets the zero-based column index of the cell. + public int ColumnIndex { get; } + + /// Gets or sets the number of rows the cell spans. The default is 1. + public int RowSpan { get; set; } = 1; + + /// Gets or sets the number of columns the cell spans. The default is 1. + public int ColumnSpan { get; set; } = 1; + + /// Gets the text content of the cell. + public string Content { get; } + + /// Gets or sets the nested content of the cell, in reading order, when the engine provides structured cell content. + /// + /// When , the cell is text-only and carries its text. When present, + /// the cell holds richer structured content (for example nested blocks or tables), and + /// remains a flat-text convenience. This mirrors the nested-content model used by engines such as Docling and + /// Google Document AI. + /// + public IReadOnlyList? Elements { get; set; } + + /// Gets or sets the region of the page the cell occupies, when the engine provides geometry. + /// + /// A cell can carry its own rectangle even when it is empty or padded, so this is not always derivable from + /// the geometry of its nested . These members mirror so a cell + /// can be promoted to a shared positioned-node type later without a reshape. + /// + public DocumentBoundingRegion? BoundingRegion { get; set; } + + /// Gets or sets the confidence for the cell in the range [0, 1], when available. + public double? Confidence { get; set; } + + /// Gets or sets the provider-native object underlying this cell. + /// + /// If an is created to represent an underlying object from another object model, + /// this property can store that original object. This can be useful for debugging or for enabling a + /// consumer to access the underlying object model if needed. + /// + [JsonIgnore] + public object? RawRepresentation { get; set; } + + /// Gets or sets any additional properties associated with the cell. + public AdditionalPropertiesDictionary? AdditionalProperties { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentTableCellKind.cs b/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentTableCellKind.cs new file mode 100644 index 00000000000..7254a557dcf --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/DocumentTableCellKind.cs @@ -0,0 +1,96 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.ComponentModel; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Text.Json; +using System.Text.Json.Serialization; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.DocumentExtraction; + +/// Describes the role of an , such as a column header or a content cell. +/// +/// This is a small open set modeled on : the well-known values cover the common +/// table cell roles, and a provider may introduce its own value when an engine reports a role that is +/// not represented here. +/// +[Experimental(DiagnosticIds.Experiments.DocumentExtraction, UrlFormat = DiagnosticIds.UrlFormat)] +[JsonConverter(typeof(Converter))] +[DebuggerDisplay("{Value,nq}")] +public readonly struct DocumentTableCellKind : IEquatable +{ + /// Gets the kind representing a column header cell. + public static DocumentTableCellKind ColumnHeader { get; } = new("columnHeader"); + + /// Gets the kind representing a regular content cell. + public static DocumentTableCellKind Content { get; } = new("content"); + + /// Gets the kind representing a row header cell (a header that labels the row it sits in). + public static DocumentTableCellKind RowHeader { get; } = new("rowHeader"); + + /// Gets the kind representing a cell that introduces a labeled section spanning subsequent rows. + public static DocumentTableCellKind RowSection { get; } = new("rowSection"); + + /// Gets the value associated with this . + public string Value { get; } + + /// Initializes a new instance of the struct with the provided value. + /// The value to associate with this . + /// is . + /// is empty or composed entirely of whitespace. + [JsonConstructor] + public DocumentTableCellKind(string value) + { + Value = Throw.IfNullOrWhitespace(value); + } + + /// Returns a value indicating whether two instances are equivalent, using a case-insensitive comparison. + /// The first instance to compare. + /// The second instance to compare. + /// if left and right have equivalent values; otherwise, . + public static bool operator ==(DocumentTableCellKind left, DocumentTableCellKind right) + { + return left.Equals(right); + } + + /// Returns a value indicating whether two instances are not equivalent, using a case-insensitive comparison. + /// The first instance to compare. + /// The second instance to compare. + /// if left and right have different values; otherwise, . + public static bool operator !=(DocumentTableCellKind left, DocumentTableCellKind right) + { + return !(left == right); + } + + /// + public override bool Equals([NotNullWhen(true)] object? obj) + => obj is DocumentTableCellKind otherKind && Equals(otherKind); + + /// + public bool Equals(DocumentTableCellKind other) + => string.Equals(Value, other.Value, StringComparison.OrdinalIgnoreCase); + + /// + public override int GetHashCode() + => StringComparer.OrdinalIgnoreCase.GetHashCode(Value); + + /// + public override string ToString() => Value; + + /// Provides a for serializing instances. + [EditorBrowsable(EditorBrowsableState.Never)] + public sealed class Converter : JsonConverter + { + /// + public override DocumentTableCellKind Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) => + new(reader.GetString()!); + + /// + public override void Write(Utf8JsonWriter writer, DocumentTableCellKind value, JsonSerializerOptions options) => + Throw.IfNull(writer).WriteStringValue(value.Value); + } +} diff --git a/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/IDocumentExtractionClient.cs b/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/IDocumentExtractionClient.cs new file mode 100644 index 00000000000..ce40de65493 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/IDocumentExtractionClient.cs @@ -0,0 +1,81 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Shared.DiagnosticIds; + +namespace Microsoft.Extensions.DocumentExtraction; + +/// Represents an optical character recognition (OCR) / document-parsing client. +/// +/// +/// An transcribes a document or image into structured output: text, +/// per-page content, tables, layout blocks with bounding regions, and confidence. It is the +/// capability sibling to , IEmbeddingGenerator, and +/// ISpeechToTextClient for the document-extraction problem. +/// +/// +/// The contract is independent of . Most OCR / document-AI engines are not +/// chat models: they emit structured output (tables, bounding regions, confidence, reading order) +/// that does not map onto a chat response. An implementation may wrap a vision-capable +/// as the lowest-fidelity, transcription-only path, but the interface does +/// not require one. +/// +/// +/// Unless otherwise specified, all members of are thread-safe for concurrent +/// use. Implementations might mutate the supplied to +/// and ; consumers should avoid sharing a single options instance across +/// concurrent invocations when that is a concern. The document stream passed to these methods is not +/// disposed by the implementation. +/// +/// +[Experimental(DiagnosticIds.Experiments.DocumentExtraction, UrlFormat = DiagnosticIds.UrlFormat)] +public interface IDocumentExtractionClient : IDisposable +{ + /// Runs OCR / document parsing over a document stream and returns structured output. + /// The document or image content to parse. + /// The media type of , for example application/pdf or image/png. + /// The OCR options to configure the request. + /// The to monitor for cancellation requests. The default is . + /// The structured OCR result. + Task ExtractAsync( + Stream document, + string mediaType, + DocumentExtractionOptions? options = null, + CancellationToken cancellationToken = default); + + /// Runs OCR / document parsing over a document stream and streams back structured output as it is produced. + /// The document or image content to parse. + /// The media type of , for example application/pdf or image/png. + /// The OCR options to configure the request. + /// The to monitor for cancellation requests. The default is . + /// The structured OCR updates representing the streamed output. + /// + /// Engines that produce pages incrementally (for example, while polling a long-running operation such as + /// Azure Document Intelligence) can yield each page as it completes, letting a consumer begin processing + /// early pages before later pages finish. Synchronous engines may yield a single terminal update. Use + /// to reassemble the stream into an + /// . + /// + IAsyncEnumerable ExtractPagesAsync( + Stream document, + string mediaType, + DocumentExtractionOptions? options = null, + CancellationToken cancellationToken = default); + + /// Asks the for an object of the specified type . + /// The type of object being requested. + /// An optional key that can be used to help identify the target service. + /// The found object, otherwise . + /// is . + /// + /// The purpose of this method is to allow for the retrieval of strongly typed services that might be + /// provided by the , including itself or any services it might be wrapping. + /// + object? GetService(Type serviceType, object? serviceKey = null); +} diff --git a/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/Microsoft.Extensions.DocumentExtraction.Abstractions.csproj b/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/Microsoft.Extensions.DocumentExtraction.Abstractions.csproj new file mode 100644 index 00000000000..de09935641c --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/Microsoft.Extensions.DocumentExtraction.Abstractions.csproj @@ -0,0 +1,51 @@ + + + + Microsoft.Extensions.DocumentExtraction + Abstractions representing document extraction components (OCR, layout, tables, and figures). + AI + document;extraction;ocr;layout;ai + true + + + + normal + false + 75 + 75 + + + + $(TargetFrameworks);netstandard2.0 + $(NoWarn);MEAI001;MEDE0001 + true + true + + + + true + true + true + true + true + true + true + true + true + true + + + + + + + + + + + + + + + + diff --git a/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/Microsoft.Extensions.DocumentExtraction.Abstractions.json b/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/Microsoft.Extensions.DocumentExtraction.Abstractions.json new file mode 100644 index 00000000000..dc5b01961b7 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/Microsoft.Extensions.DocumentExtraction.Abstractions.json @@ -0,0 +1,845 @@ +{ + "Name": "Microsoft.Extensions.DocumentExtraction.Abstractions, Version=10.8.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35", + "Types": [ + { + "Type": "class Microsoft.Extensions.DocumentExtraction.DelegatingDocumentExtractionClient : Microsoft.Extensions.DocumentExtraction.IDocumentExtractionClient, System.IDisposable", + "Stage": "Experimental", + "Methods": [ + { + "Member": "Microsoft.Extensions.DocumentExtraction.DelegatingDocumentExtractionClient.DelegatingDocumentExtractionClient(Microsoft.Extensions.DocumentExtraction.IDocumentExtractionClient innerClient);", + "Stage": "Experimental" + }, + { + "Member": "void Microsoft.Extensions.DocumentExtraction.DelegatingDocumentExtractionClient.Dispose();", + "Stage": "Experimental" + }, + { + "Member": "virtual void Microsoft.Extensions.DocumentExtraction.DelegatingDocumentExtractionClient.Dispose(bool disposing);", + "Stage": "Experimental" + }, + { + "Member": "virtual System.Threading.Tasks.Task Microsoft.Extensions.DocumentExtraction.DelegatingDocumentExtractionClient.ExtractAsync(System.IO.Stream document, string mediaType, Microsoft.Extensions.DocumentExtraction.DocumentExtractionOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", + "Stage": "Experimental" + }, + { + "Member": "virtual System.Collections.Generic.IAsyncEnumerable Microsoft.Extensions.DocumentExtraction.DelegatingDocumentExtractionClient.ExtractPagesAsync(System.IO.Stream document, string mediaType, Microsoft.Extensions.DocumentExtraction.DocumentExtractionOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", + "Stage": "Experimental" + }, + { + "Member": "virtual object? Microsoft.Extensions.DocumentExtraction.DelegatingDocumentExtractionClient.GetService(System.Type serviceType, object? serviceKey = null);", + "Stage": "Experimental" + } + ], + "Properties": [ + { + "Member": "Microsoft.Extensions.DocumentExtraction.IDocumentExtractionClient Microsoft.Extensions.DocumentExtraction.DelegatingDocumentExtractionClient.InnerClient { get; }", + "Stage": "Experimental" + } + ] + }, + { + "Type": "class Microsoft.Extensions.DocumentExtraction.DocumentBlock : Microsoft.Extensions.DocumentExtraction.DocumentElement", + "Stage": "Experimental", + "Methods": [ + { + "Member": "Microsoft.Extensions.DocumentExtraction.DocumentBlock.DocumentBlock(string text);", + "Stage": "Experimental" + } + ], + "Properties": [ + { + "Member": "Microsoft.Extensions.DocumentExtraction.DocumentBlockKind? Microsoft.Extensions.DocumentExtraction.DocumentBlock.Kind { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "string Microsoft.Extensions.DocumentExtraction.DocumentBlock.Text { get; }", + "Stage": "Experimental" + } + ] + }, + { + "Type": "readonly struct Microsoft.Extensions.DocumentExtraction.DocumentBlockKind : System.IEquatable", + "Stage": "Experimental", + "Methods": [ + { + "Member": "Microsoft.Extensions.DocumentExtraction.DocumentBlockKind.DocumentBlockKind(string value);", + "Stage": "Experimental" + }, + { + "Member": "Microsoft.Extensions.DocumentExtraction.DocumentBlockKind.DocumentBlockKind();", + "Stage": "Experimental" + }, + { + "Member": "override bool Microsoft.Extensions.DocumentExtraction.DocumentBlockKind.Equals(object? obj);", + "Stage": "Experimental" + }, + { + "Member": "bool Microsoft.Extensions.DocumentExtraction.DocumentBlockKind.Equals(Microsoft.Extensions.DocumentExtraction.DocumentBlockKind other);", + "Stage": "Experimental" + }, + { + "Member": "override int Microsoft.Extensions.DocumentExtraction.DocumentBlockKind.GetHashCode();", + "Stage": "Experimental" + }, + { + "Member": "static bool Microsoft.Extensions.DocumentExtraction.DocumentBlockKind.operator ==(Microsoft.Extensions.DocumentExtraction.DocumentBlockKind left, Microsoft.Extensions.DocumentExtraction.DocumentBlockKind right);", + "Stage": "Experimental" + }, + { + "Member": "static bool Microsoft.Extensions.DocumentExtraction.DocumentBlockKind.operator !=(Microsoft.Extensions.DocumentExtraction.DocumentBlockKind left, Microsoft.Extensions.DocumentExtraction.DocumentBlockKind right);", + "Stage": "Experimental" + }, + { + "Member": "override string Microsoft.Extensions.DocumentExtraction.DocumentBlockKind.ToString();", + "Stage": "Experimental" + } + ], + "Properties": [ + { + "Member": "static Microsoft.Extensions.DocumentExtraction.DocumentBlockKind Microsoft.Extensions.DocumentExtraction.DocumentBlockKind.Figure { get; }", + "Stage": "Experimental" + }, + { + "Member": "static Microsoft.Extensions.DocumentExtraction.DocumentBlockKind Microsoft.Extensions.DocumentExtraction.DocumentBlockKind.Paragraph { get; }", + "Stage": "Experimental" + }, + { + "Member": "static Microsoft.Extensions.DocumentExtraction.DocumentBlockKind Microsoft.Extensions.DocumentExtraction.DocumentBlockKind.Title { get; }", + "Stage": "Experimental" + }, + { + "Member": "string Microsoft.Extensions.DocumentExtraction.DocumentBlockKind.Value { get; }", + "Stage": "Experimental" + } + ] + }, + { + "Type": "sealed class Microsoft.Extensions.DocumentExtraction.DocumentBlockKind.Converter", + "Stage": "Experimental", + "Methods": [ + { + "Member": "Microsoft.Extensions.DocumentExtraction.DocumentBlockKind.Converter.Converter();", + "Stage": "Experimental" + }, + { + "Member": "override Microsoft.Extensions.DocumentExtraction.DocumentBlockKind Microsoft.Extensions.DocumentExtraction.DocumentBlockKind.Converter.Read(ref System.Text.Json.Utf8JsonReader reader, System.Type typeToConvert, System.Text.Json.JsonSerializerOptions options);", + "Stage": "Experimental" + }, + { + "Member": "override void Microsoft.Extensions.DocumentExtraction.DocumentBlockKind.Converter.Write(System.Text.Json.Utf8JsonWriter writer, Microsoft.Extensions.DocumentExtraction.DocumentBlockKind value, System.Text.Json.JsonSerializerOptions options);", + "Stage": "Experimental" + } + ] + }, + { + "Type": "readonly class Microsoft.Extensions.DocumentExtraction.DocumentBoundingBox(float Left, float Top, float Right, float Bottom)", + "Stage": "Experimental", + "Methods": [ + { + "Member": "Microsoft.Extensions.DocumentExtraction.DocumentBoundingBox.DocumentBoundingBox(float Left, float Top, float Right, float Bottom);", + "Stage": "Experimental" + }, + { + "Member": "Microsoft.Extensions.DocumentExtraction.DocumentBoundingBox.DocumentBoundingBox();", + "Stage": "Experimental" + }, + { + "Member": "void Microsoft.Extensions.DocumentExtraction.DocumentBoundingBox.Deconstruct(out float Left, out float Top, out float Right, out float Bottom);", + "Stage": "Experimental" + }, + { + "Member": "override bool Microsoft.Extensions.DocumentExtraction.DocumentBoundingBox.Equals(object obj);", + "Stage": "Experimental" + }, + { + "Member": "bool Microsoft.Extensions.DocumentExtraction.DocumentBoundingBox.Equals(Microsoft.Extensions.DocumentExtraction.DocumentBoundingBox other);", + "Stage": "Experimental" + }, + { + "Member": "override int Microsoft.Extensions.DocumentExtraction.DocumentBoundingBox.GetHashCode();", + "Stage": "Experimental" + }, + { + "Member": "static bool Microsoft.Extensions.DocumentExtraction.DocumentBoundingBox.operator ==(Microsoft.Extensions.DocumentExtraction.DocumentBoundingBox left, Microsoft.Extensions.DocumentExtraction.DocumentBoundingBox right);", + "Stage": "Experimental" + }, + { + "Member": "static bool Microsoft.Extensions.DocumentExtraction.DocumentBoundingBox.operator !=(Microsoft.Extensions.DocumentExtraction.DocumentBoundingBox left, Microsoft.Extensions.DocumentExtraction.DocumentBoundingBox right);", + "Stage": "Experimental" + }, + { + "Member": "override string Microsoft.Extensions.DocumentExtraction.DocumentBoundingBox.ToString();", + "Stage": "Experimental" + } + ], + "Properties": [ + { + "Member": "float Microsoft.Extensions.DocumentExtraction.DocumentBoundingBox.Bottom { get; init; }", + "Stage": "Experimental" + }, + { + "Member": "float Microsoft.Extensions.DocumentExtraction.DocumentBoundingBox.Left { get; init; }", + "Stage": "Experimental" + }, + { + "Member": "float Microsoft.Extensions.DocumentExtraction.DocumentBoundingBox.Right { get; init; }", + "Stage": "Experimental" + }, + { + "Member": "float Microsoft.Extensions.DocumentExtraction.DocumentBoundingBox.Top { get; init; }", + "Stage": "Experimental" + } + ] + }, + { + "Type": "class Microsoft.Extensions.DocumentExtraction.DocumentBoundingRegion", + "Stage": "Experimental", + "Methods": [ + { + "Member": "Microsoft.Extensions.DocumentExtraction.DocumentBoundingRegion.DocumentBoundingRegion(int pageNumber, System.Collections.Generic.IReadOnlyList polygon);", + "Stage": "Experimental" + }, + { + "Member": "static Microsoft.Extensions.DocumentExtraction.DocumentBoundingRegion Microsoft.Extensions.DocumentExtraction.DocumentBoundingRegion.FromRectangle(int pageNumber, float left, float top, float right, float bottom);", + "Stage": "Experimental" + }, + { + "Member": "Microsoft.Extensions.DocumentExtraction.DocumentBoundingBox? Microsoft.Extensions.DocumentExtraction.DocumentBoundingRegion.GetBounds();", + "Stage": "Experimental" + } + ], + "Properties": [ + { + "Member": "int Microsoft.Extensions.DocumentExtraction.DocumentBoundingRegion.PageNumber { get; }", + "Stage": "Experimental" + }, + { + "Member": "System.Collections.Generic.IReadOnlyList Microsoft.Extensions.DocumentExtraction.DocumentBoundingRegion.Polygon { get; }", + "Stage": "Experimental" + } + ] + }, + { + "Type": "enum Microsoft.Extensions.DocumentExtraction.DocumentCoordinateOrigin", + "Stage": "Experimental", + "Methods": [ + { + "Member": "Microsoft.Extensions.DocumentExtraction.DocumentCoordinateOrigin.DocumentCoordinateOrigin();", + "Stage": "Experimental" + } + ], + "Fields": [ + { + "Member": "const Microsoft.Extensions.DocumentExtraction.DocumentCoordinateOrigin Microsoft.Extensions.DocumentExtraction.DocumentCoordinateOrigin.BottomLeft", + "Stage": "Experimental", + "Value": "1" + }, + { + "Member": "const Microsoft.Extensions.DocumentExtraction.DocumentCoordinateOrigin Microsoft.Extensions.DocumentExtraction.DocumentCoordinateOrigin.TopLeft", + "Stage": "Experimental", + "Value": "0" + } + ] + }, + { + "Type": "enum Microsoft.Extensions.DocumentExtraction.DocumentCoordinateUnit", + "Stage": "Experimental", + "Methods": [ + { + "Member": "Microsoft.Extensions.DocumentExtraction.DocumentCoordinateUnit.DocumentCoordinateUnit();", + "Stage": "Experimental" + } + ], + "Fields": [ + { + "Member": "const Microsoft.Extensions.DocumentExtraction.DocumentCoordinateUnit Microsoft.Extensions.DocumentExtraction.DocumentCoordinateUnit.Inch", + "Stage": "Experimental", + "Value": "2" + }, + { + "Member": "const Microsoft.Extensions.DocumentExtraction.DocumentCoordinateUnit Microsoft.Extensions.DocumentExtraction.DocumentCoordinateUnit.Normalized", + "Stage": "Experimental", + "Value": "3" + }, + { + "Member": "const Microsoft.Extensions.DocumentExtraction.DocumentCoordinateUnit Microsoft.Extensions.DocumentExtraction.DocumentCoordinateUnit.Pixel", + "Stage": "Experimental", + "Value": "0" + }, + { + "Member": "const Microsoft.Extensions.DocumentExtraction.DocumentCoordinateUnit Microsoft.Extensions.DocumentExtraction.DocumentCoordinateUnit.Point", + "Stage": "Experimental", + "Value": "1" + } + ] + }, + { + "Type": "abstract class Microsoft.Extensions.DocumentExtraction.DocumentElement", + "Stage": "Experimental", + "Methods": [ + { + "Member": "Microsoft.Extensions.DocumentExtraction.DocumentElement.DocumentElement();", + "Stage": "Experimental" + } + ], + "Properties": [ + { + "Member": "Microsoft.Extensions.AI.AdditionalPropertiesDictionary? Microsoft.Extensions.DocumentExtraction.DocumentElement.AdditionalProperties { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "Microsoft.Extensions.DocumentExtraction.DocumentBoundingRegion? Microsoft.Extensions.DocumentExtraction.DocumentElement.BoundingRegion { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "double? Microsoft.Extensions.DocumentExtraction.DocumentElement.Confidence { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "object? Microsoft.Extensions.DocumentExtraction.DocumentElement.RawRepresentation { get; set; }", + "Stage": "Experimental" + } + ] + }, + { + "Type": "static class Microsoft.Extensions.DocumentExtraction.DocumentExtractionClientExtensions", + "Stage": "Experimental", + "Methods": [ + { + "Member": "static System.Threading.Tasks.Task Microsoft.Extensions.DocumentExtraction.DocumentExtractionClientExtensions.ExtractAsync(this Microsoft.Extensions.DocumentExtraction.IDocumentExtractionClient client, Microsoft.Extensions.AI.DataContent document, Microsoft.Extensions.DocumentExtraction.DocumentExtractionOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", + "Stage": "Experimental" + }, + { + "Member": "static System.Threading.Tasks.Task Microsoft.Extensions.DocumentExtraction.DocumentExtractionClientExtensions.ExtractAsync(this Microsoft.Extensions.DocumentExtraction.IDocumentExtractionClient client, Microsoft.Extensions.AI.UriContent document, Microsoft.Extensions.DocumentExtraction.DocumentExtractionOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", + "Stage": "Experimental" + }, + { + "Member": "static System.Threading.Tasks.Task Microsoft.Extensions.DocumentExtraction.DocumentExtractionClientExtensions.ExtractFromUriAsync(this Microsoft.Extensions.DocumentExtraction.IDocumentExtractionClient client, Microsoft.Extensions.AI.UriContent document, System.Net.Http.HttpClient httpClient, Microsoft.Extensions.DocumentExtraction.DocumentExtractionOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", + "Stage": "Experimental" + }, + { + "Member": "static System.Collections.Generic.IAsyncEnumerable Microsoft.Extensions.DocumentExtraction.DocumentExtractionClientExtensions.ExtractPagesAsync(this Microsoft.Extensions.DocumentExtraction.IDocumentExtractionClient client, Microsoft.Extensions.AI.DataContent document, Microsoft.Extensions.DocumentExtraction.DocumentExtractionOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", + "Stage": "Experimental" + }, + { + "Member": "static System.Collections.Generic.IAsyncEnumerable Microsoft.Extensions.DocumentExtraction.DocumentExtractionClientExtensions.ExtractPagesAsync(this Microsoft.Extensions.DocumentExtraction.IDocumentExtractionClient client, Microsoft.Extensions.AI.UriContent document, Microsoft.Extensions.DocumentExtraction.DocumentExtractionOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", + "Stage": "Experimental" + }, + { + "Member": "static TService? Microsoft.Extensions.DocumentExtraction.DocumentExtractionClientExtensions.GetService(this Microsoft.Extensions.DocumentExtraction.IDocumentExtractionClient client, object? serviceKey = null);", + "Stage": "Experimental" + } + ] + }, + { + "Type": "class Microsoft.Extensions.DocumentExtraction.DocumentExtractionClientMetadata", + "Stage": "Experimental", + "Methods": [ + { + "Member": "Microsoft.Extensions.DocumentExtraction.DocumentExtractionClientMetadata.DocumentExtractionClientMetadata(string? providerName = null, System.Uri? providerUri = null, string? defaultModelId = null);", + "Stage": "Experimental" + } + ], + "Properties": [ + { + "Member": "string? Microsoft.Extensions.DocumentExtraction.DocumentExtractionClientMetadata.DefaultModelId { get; }", + "Stage": "Experimental" + }, + { + "Member": "string? Microsoft.Extensions.DocumentExtraction.DocumentExtractionClientMetadata.ProviderName { get; }", + "Stage": "Experimental" + }, + { + "Member": "System.Uri? Microsoft.Extensions.DocumentExtraction.DocumentExtractionClientMetadata.ProviderUri { get; }", + "Stage": "Experimental" + } + ] + }, + { + "Type": "class Microsoft.Extensions.DocumentExtraction.DocumentExtractionOptions", + "Stage": "Experimental", + "Methods": [ + { + "Member": "Microsoft.Extensions.DocumentExtraction.DocumentExtractionOptions.DocumentExtractionOptions();", + "Stage": "Experimental" + }, + { + "Member": "Microsoft.Extensions.DocumentExtraction.DocumentExtractionOptions Microsoft.Extensions.DocumentExtraction.DocumentExtractionOptions.Clone();", + "Stage": "Experimental" + } + ], + "Properties": [ + { + "Member": "Microsoft.Extensions.AI.AdditionalPropertiesDictionary? Microsoft.Extensions.DocumentExtraction.DocumentExtractionOptions.AdditionalProperties { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "string? Microsoft.Extensions.DocumentExtraction.DocumentExtractionOptions.ModelId { get; set; }", + "Stage": "Experimental" + } + ] + }, + { + "Type": "class Microsoft.Extensions.DocumentExtraction.DocumentExtractionPageResult", + "Stage": "Experimental", + "Methods": [ + { + "Member": "Microsoft.Extensions.DocumentExtraction.DocumentExtractionPageResult.DocumentExtractionPageResult(Microsoft.Extensions.DocumentExtraction.DocumentPage page);", + "Stage": "Experimental" + } + ], + "Properties": [ + { + "Member": "Microsoft.Extensions.AI.AdditionalPropertiesDictionary? Microsoft.Extensions.DocumentExtraction.DocumentExtractionPageResult.AdditionalProperties { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "Microsoft.Extensions.DocumentExtraction.DocumentPage Microsoft.Extensions.DocumentExtraction.DocumentExtractionPageResult.Page { get; }", + "Stage": "Experimental" + }, + { + "Member": "int? Microsoft.Extensions.DocumentExtraction.DocumentExtractionPageResult.PagesProcessed { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "object? Microsoft.Extensions.DocumentExtraction.DocumentExtractionPageResult.RawRepresentation { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "int? Microsoft.Extensions.DocumentExtraction.DocumentExtractionPageResult.TotalPages { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "Microsoft.Extensions.DocumentExtraction.DocumentExtractionUsage? Microsoft.Extensions.DocumentExtraction.DocumentExtractionPageResult.Usage { get; set; }", + "Stage": "Experimental" + } + ] + }, + { + "Type": "static class Microsoft.Extensions.DocumentExtraction.DocumentExtractionPageResultExtensions", + "Stage": "Experimental", + "Methods": [ + { + "Member": "static Microsoft.Extensions.DocumentExtraction.DocumentExtractionResult Microsoft.Extensions.DocumentExtraction.DocumentExtractionPageResultExtensions.ToDocumentExtractionResult(this System.Collections.Generic.IEnumerable updates);", + "Stage": "Experimental" + }, + { + "Member": "static System.Threading.Tasks.Task Microsoft.Extensions.DocumentExtraction.DocumentExtractionPageResultExtensions.ToDocumentExtractionResultAsync(this System.Collections.Generic.IAsyncEnumerable updates, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", + "Stage": "Experimental" + } + ] + }, + { + "Type": "class Microsoft.Extensions.DocumentExtraction.DocumentExtractionResult", + "Stage": "Experimental", + "Methods": [ + { + "Member": "Microsoft.Extensions.DocumentExtraction.DocumentExtractionResult.DocumentExtractionResult(System.Collections.Generic.IReadOnlyList pages);", + "Stage": "Experimental" + } + ], + "Properties": [ + { + "Member": "Microsoft.Extensions.AI.AdditionalPropertiesDictionary? Microsoft.Extensions.DocumentExtraction.DocumentExtractionResult.AdditionalProperties { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "System.Collections.Generic.IReadOnlyList Microsoft.Extensions.DocumentExtraction.DocumentExtractionResult.Pages { get; }", + "Stage": "Experimental" + }, + { + "Member": "object? Microsoft.Extensions.DocumentExtraction.DocumentExtractionResult.RawRepresentation { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "string Microsoft.Extensions.DocumentExtraction.DocumentExtractionResult.Text { get; }", + "Stage": "Experimental" + }, + { + "Member": "Microsoft.Extensions.DocumentExtraction.DocumentExtractionUsage? Microsoft.Extensions.DocumentExtraction.DocumentExtractionResult.Usage { get; set; }", + "Stage": "Experimental" + } + ] + }, + { + "Type": "class Microsoft.Extensions.DocumentExtraction.DocumentExtractionUsage", + "Stage": "Experimental", + "Methods": [ + { + "Member": "Microsoft.Extensions.DocumentExtraction.DocumentExtractionUsage.DocumentExtractionUsage();", + "Stage": "Experimental" + } + ], + "Properties": [ + { + "Member": "Microsoft.Extensions.AI.AdditionalPropertiesDictionary? Microsoft.Extensions.DocumentExtraction.DocumentExtractionUsage.AdditionalProperties { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "int? Microsoft.Extensions.DocumentExtraction.DocumentExtractionUsage.InputTokenCount { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "int? Microsoft.Extensions.DocumentExtraction.DocumentExtractionUsage.OutputTokenCount { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "int? Microsoft.Extensions.DocumentExtraction.DocumentExtractionUsage.PagesProcessed { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "int? Microsoft.Extensions.DocumentExtraction.DocumentExtractionUsage.TotalTokenCount { get; set; }", + "Stage": "Experimental" + } + ] + }, + { + "Type": "class Microsoft.Extensions.DocumentExtraction.DocumentImage : Microsoft.Extensions.DocumentExtraction.DocumentElement", + "Stage": "Experimental", + "Methods": [ + { + "Member": "Microsoft.Extensions.DocumentExtraction.DocumentImage.DocumentImage();", + "Stage": "Experimental" + } + ], + "Properties": [ + { + "Member": "string? Microsoft.Extensions.DocumentExtraction.DocumentImage.Caption { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "Microsoft.Extensions.AI.DataContent? Microsoft.Extensions.DocumentExtraction.DocumentImage.Content { get; set; }", + "Stage": "Experimental" + } + ] + }, + { + "Type": "class Microsoft.Extensions.DocumentExtraction.DocumentPage", + "Stage": "Experimental", + "Methods": [ + { + "Member": "Microsoft.Extensions.DocumentExtraction.DocumentPage.DocumentPage(int pageNumber, string text);", + "Stage": "Experimental" + } + ], + "Properties": [ + { + "Member": "Microsoft.Extensions.AI.AdditionalPropertiesDictionary? Microsoft.Extensions.DocumentExtraction.DocumentPage.AdditionalProperties { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "Microsoft.Extensions.DocumentExtraction.DocumentCoordinateOrigin? Microsoft.Extensions.DocumentExtraction.DocumentPage.CoordinateOrigin { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "Microsoft.Extensions.DocumentExtraction.DocumentCoordinateUnit? Microsoft.Extensions.DocumentExtraction.DocumentPage.CoordinateUnit { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "Microsoft.Extensions.DocumentExtraction.DocumentPageDimensions? Microsoft.Extensions.DocumentExtraction.DocumentPage.Dimensions { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "System.Collections.Generic.IReadOnlyList Microsoft.Extensions.DocumentExtraction.DocumentPage.Elements { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "int Microsoft.Extensions.DocumentExtraction.DocumentPage.PageNumber { get; }", + "Stage": "Experimental" + }, + { + "Member": "object? Microsoft.Extensions.DocumentExtraction.DocumentPage.RawRepresentation { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "string Microsoft.Extensions.DocumentExtraction.DocumentPage.Text { get; }", + "Stage": "Experimental" + } + ] + }, + { + "Type": "readonly class Microsoft.Extensions.DocumentExtraction.DocumentPageDimensions(float Width, float Height)", + "Stage": "Experimental", + "Methods": [ + { + "Member": "Microsoft.Extensions.DocumentExtraction.DocumentPageDimensions.DocumentPageDimensions(float Width, float Height);", + "Stage": "Experimental" + }, + { + "Member": "Microsoft.Extensions.DocumentExtraction.DocumentPageDimensions.DocumentPageDimensions();", + "Stage": "Experimental" + }, + { + "Member": "void Microsoft.Extensions.DocumentExtraction.DocumentPageDimensions.Deconstruct(out float Width, out float Height);", + "Stage": "Experimental" + }, + { + "Member": "override bool Microsoft.Extensions.DocumentExtraction.DocumentPageDimensions.Equals(object obj);", + "Stage": "Experimental" + }, + { + "Member": "bool Microsoft.Extensions.DocumentExtraction.DocumentPageDimensions.Equals(Microsoft.Extensions.DocumentExtraction.DocumentPageDimensions other);", + "Stage": "Experimental" + }, + { + "Member": "override int Microsoft.Extensions.DocumentExtraction.DocumentPageDimensions.GetHashCode();", + "Stage": "Experimental" + }, + { + "Member": "static bool Microsoft.Extensions.DocumentExtraction.DocumentPageDimensions.operator ==(Microsoft.Extensions.DocumentExtraction.DocumentPageDimensions left, Microsoft.Extensions.DocumentExtraction.DocumentPageDimensions right);", + "Stage": "Experimental" + }, + { + "Member": "static bool Microsoft.Extensions.DocumentExtraction.DocumentPageDimensions.operator !=(Microsoft.Extensions.DocumentExtraction.DocumentPageDimensions left, Microsoft.Extensions.DocumentExtraction.DocumentPageDimensions right);", + "Stage": "Experimental" + }, + { + "Member": "override string Microsoft.Extensions.DocumentExtraction.DocumentPageDimensions.ToString();", + "Stage": "Experimental" + } + ], + "Properties": [ + { + "Member": "float Microsoft.Extensions.DocumentExtraction.DocumentPageDimensions.Height { get; init; }", + "Stage": "Experimental" + }, + { + "Member": "float Microsoft.Extensions.DocumentExtraction.DocumentPageDimensions.Width { get; init; }", + "Stage": "Experimental" + } + ] + }, + { + "Type": "readonly class Microsoft.Extensions.DocumentExtraction.DocumentPoint(float X, float Y)", + "Stage": "Experimental", + "Methods": [ + { + "Member": "Microsoft.Extensions.DocumentExtraction.DocumentPoint.DocumentPoint(float X, float Y);", + "Stage": "Experimental" + }, + { + "Member": "Microsoft.Extensions.DocumentExtraction.DocumentPoint.DocumentPoint();", + "Stage": "Experimental" + }, + { + "Member": "void Microsoft.Extensions.DocumentExtraction.DocumentPoint.Deconstruct(out float X, out float Y);", + "Stage": "Experimental" + }, + { + "Member": "override bool Microsoft.Extensions.DocumentExtraction.DocumentPoint.Equals(object obj);", + "Stage": "Experimental" + }, + { + "Member": "bool Microsoft.Extensions.DocumentExtraction.DocumentPoint.Equals(Microsoft.Extensions.DocumentExtraction.DocumentPoint other);", + "Stage": "Experimental" + }, + { + "Member": "override int Microsoft.Extensions.DocumentExtraction.DocumentPoint.GetHashCode();", + "Stage": "Experimental" + }, + { + "Member": "static bool Microsoft.Extensions.DocumentExtraction.DocumentPoint.operator ==(Microsoft.Extensions.DocumentExtraction.DocumentPoint left, Microsoft.Extensions.DocumentExtraction.DocumentPoint right);", + "Stage": "Experimental" + }, + { + "Member": "static bool Microsoft.Extensions.DocumentExtraction.DocumentPoint.operator !=(Microsoft.Extensions.DocumentExtraction.DocumentPoint left, Microsoft.Extensions.DocumentExtraction.DocumentPoint right);", + "Stage": "Experimental" + }, + { + "Member": "override string Microsoft.Extensions.DocumentExtraction.DocumentPoint.ToString();", + "Stage": "Experimental" + } + ], + "Properties": [ + { + "Member": "float Microsoft.Extensions.DocumentExtraction.DocumentPoint.X { get; init; }", + "Stage": "Experimental" + }, + { + "Member": "float Microsoft.Extensions.DocumentExtraction.DocumentPoint.Y { get; init; }", + "Stage": "Experimental" + } + ] + }, + { + "Type": "class Microsoft.Extensions.DocumentExtraction.DocumentTable : Microsoft.Extensions.DocumentExtraction.DocumentElement", + "Stage": "Experimental", + "Methods": [ + { + "Member": "Microsoft.Extensions.DocumentExtraction.DocumentTable.DocumentTable(int rowCount, int columnCount, System.Collections.Generic.IReadOnlyList? cells = null, string? markdownRepresentation = null);", + "Stage": "Experimental" + } + ], + "Properties": [ + { + "Member": "System.Collections.Generic.IReadOnlyList? Microsoft.Extensions.DocumentExtraction.DocumentTable.Cells { get; }", + "Stage": "Experimental" + }, + { + "Member": "int Microsoft.Extensions.DocumentExtraction.DocumentTable.ColumnCount { get; }", + "Stage": "Experimental" + }, + { + "Member": "string? Microsoft.Extensions.DocumentExtraction.DocumentTable.MarkdownRepresentation { get; }", + "Stage": "Experimental" + }, + { + "Member": "int Microsoft.Extensions.DocumentExtraction.DocumentTable.RowCount { get; }", + "Stage": "Experimental" + } + ] + }, + { + "Type": "class Microsoft.Extensions.DocumentExtraction.DocumentTableCell", + "Stage": "Experimental", + "Methods": [ + { + "Member": "Microsoft.Extensions.DocumentExtraction.DocumentTableCell.DocumentTableCell(int rowIndex, int columnIndex, string content);", + "Stage": "Experimental" + } + ], + "Properties": [ + { + "Member": "Microsoft.Extensions.AI.AdditionalPropertiesDictionary? Microsoft.Extensions.DocumentExtraction.DocumentTableCell.AdditionalProperties { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "Microsoft.Extensions.DocumentExtraction.DocumentBoundingRegion? Microsoft.Extensions.DocumentExtraction.DocumentTableCell.BoundingRegion { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "int Microsoft.Extensions.DocumentExtraction.DocumentTableCell.ColumnIndex { get; }", + "Stage": "Experimental" + }, + { + "Member": "int Microsoft.Extensions.DocumentExtraction.DocumentTableCell.ColumnSpan { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "double? Microsoft.Extensions.DocumentExtraction.DocumentTableCell.Confidence { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "string Microsoft.Extensions.DocumentExtraction.DocumentTableCell.Content { get; }", + "Stage": "Experimental" + }, + { + "Member": "System.Collections.Generic.IReadOnlyList? Microsoft.Extensions.DocumentExtraction.DocumentTableCell.Elements { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "Microsoft.Extensions.DocumentExtraction.DocumentTableCellKind? Microsoft.Extensions.DocumentExtraction.DocumentTableCell.Kind { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "object? Microsoft.Extensions.DocumentExtraction.DocumentTableCell.RawRepresentation { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "int Microsoft.Extensions.DocumentExtraction.DocumentTableCell.RowIndex { get; }", + "Stage": "Experimental" + }, + { + "Member": "int Microsoft.Extensions.DocumentExtraction.DocumentTableCell.RowSpan { get; set; }", + "Stage": "Experimental" + } + ] + }, + { + "Type": "readonly struct Microsoft.Extensions.DocumentExtraction.DocumentTableCellKind : System.IEquatable", + "Stage": "Experimental", + "Methods": [ + { + "Member": "Microsoft.Extensions.DocumentExtraction.DocumentTableCellKind.DocumentTableCellKind(string value);", + "Stage": "Experimental" + }, + { + "Member": "Microsoft.Extensions.DocumentExtraction.DocumentTableCellKind.DocumentTableCellKind();", + "Stage": "Experimental" + }, + { + "Member": "override bool Microsoft.Extensions.DocumentExtraction.DocumentTableCellKind.Equals(object? obj);", + "Stage": "Experimental" + }, + { + "Member": "bool Microsoft.Extensions.DocumentExtraction.DocumentTableCellKind.Equals(Microsoft.Extensions.DocumentExtraction.DocumentTableCellKind other);", + "Stage": "Experimental" + }, + { + "Member": "override int Microsoft.Extensions.DocumentExtraction.DocumentTableCellKind.GetHashCode();", + "Stage": "Experimental" + }, + { + "Member": "static bool Microsoft.Extensions.DocumentExtraction.DocumentTableCellKind.operator ==(Microsoft.Extensions.DocumentExtraction.DocumentTableCellKind left, Microsoft.Extensions.DocumentExtraction.DocumentTableCellKind right);", + "Stage": "Experimental" + }, + { + "Member": "static bool Microsoft.Extensions.DocumentExtraction.DocumentTableCellKind.operator !=(Microsoft.Extensions.DocumentExtraction.DocumentTableCellKind left, Microsoft.Extensions.DocumentExtraction.DocumentTableCellKind right);", + "Stage": "Experimental" + }, + { + "Member": "override string Microsoft.Extensions.DocumentExtraction.DocumentTableCellKind.ToString();", + "Stage": "Experimental" + } + ], + "Properties": [ + { + "Member": "static Microsoft.Extensions.DocumentExtraction.DocumentTableCellKind Microsoft.Extensions.DocumentExtraction.DocumentTableCellKind.ColumnHeader { get; }", + "Stage": "Experimental" + }, + { + "Member": "static Microsoft.Extensions.DocumentExtraction.DocumentTableCellKind Microsoft.Extensions.DocumentExtraction.DocumentTableCellKind.Content { get; }", + "Stage": "Experimental" + }, + { + "Member": "static Microsoft.Extensions.DocumentExtraction.DocumentTableCellKind Microsoft.Extensions.DocumentExtraction.DocumentTableCellKind.RowHeader { get; }", + "Stage": "Experimental" + }, + { + "Member": "static Microsoft.Extensions.DocumentExtraction.DocumentTableCellKind Microsoft.Extensions.DocumentExtraction.DocumentTableCellKind.RowSection { get; }", + "Stage": "Experimental" + }, + { + "Member": "string Microsoft.Extensions.DocumentExtraction.DocumentTableCellKind.Value { get; }", + "Stage": "Experimental" + } + ] + }, + { + "Type": "sealed class Microsoft.Extensions.DocumentExtraction.DocumentTableCellKind.Converter", + "Stage": "Experimental", + "Methods": [ + { + "Member": "Microsoft.Extensions.DocumentExtraction.DocumentTableCellKind.Converter.Converter();", + "Stage": "Experimental" + }, + { + "Member": "override Microsoft.Extensions.DocumentExtraction.DocumentTableCellKind Microsoft.Extensions.DocumentExtraction.DocumentTableCellKind.Converter.Read(ref System.Text.Json.Utf8JsonReader reader, System.Type typeToConvert, System.Text.Json.JsonSerializerOptions options);", + "Stage": "Experimental" + }, + { + "Member": "override void Microsoft.Extensions.DocumentExtraction.DocumentTableCellKind.Converter.Write(System.Text.Json.Utf8JsonWriter writer, Microsoft.Extensions.DocumentExtraction.DocumentTableCellKind value, System.Text.Json.JsonSerializerOptions options);", + "Stage": "Experimental" + } + ] + }, + { + "Type": "interface Microsoft.Extensions.DocumentExtraction.IDocumentExtractionClient : System.IDisposable", + "Stage": "Experimental", + "Methods": [ + { + "Member": "System.Threading.Tasks.Task Microsoft.Extensions.DocumentExtraction.IDocumentExtractionClient.ExtractAsync(System.IO.Stream document, string mediaType, Microsoft.Extensions.DocumentExtraction.DocumentExtractionOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", + "Stage": "Experimental" + }, + { + "Member": "System.Collections.Generic.IAsyncEnumerable Microsoft.Extensions.DocumentExtraction.IDocumentExtractionClient.ExtractPagesAsync(System.IO.Stream document, string mediaType, Microsoft.Extensions.DocumentExtraction.DocumentExtractionOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", + "Stage": "Experimental" + }, + { + "Member": "object? Microsoft.Extensions.DocumentExtraction.IDocumentExtractionClient.GetService(System.Type serviceType, object? serviceKey = null);", + "Stage": "Experimental" + } + ] + } + ] +} \ No newline at end of file diff --git a/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/README.md b/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/README.md new file mode 100644 index 00000000000..5de2d60d413 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions/README.md @@ -0,0 +1,39 @@ +# Microsoft.Extensions.DocumentExtraction.Abstractions + +.NET developers need to turn documents such as scanned images and PDFs into structured, AI-ready content, recovering text along with layout, tables, figures, and coordinates in a provider-neutral way. The `Microsoft.Extensions.DocumentExtraction` libraries provide a unified approach for representing document-extraction components, complementing `Microsoft.Extensions.DataIngestion` (extraction pulls content out of documents; ingestion feeds that content into a retrieval pipeline). + +## The packages + +The [Microsoft.Extensions.DocumentExtraction.Abstractions](https://www.nuget.org/packages/Microsoft.Extensions.DocumentExtraction.Abstractions) package provides the core exchange types, including [`IDocumentExtractionClient`](https://learn.microsoft.com/dotnet/api/microsoft.extensions.documentextraction.idocumentextractionclient), [`DocumentExtractionResult`](https://learn.microsoft.com/dotnet/api/microsoft.extensions.documentextraction.documentextractionresult), [`DocumentPage`](https://learn.microsoft.com/dotnet/api/microsoft.extensions.documentextraction.documentpage), and the reading-order [`DocumentElement`](https://learn.microsoft.com/dotnet/api/microsoft.extensions.documentextraction.documentelement) model (blocks, tables, and images with bounding regions and confidence). Any .NET library that provides a document-extraction engine can implement these abstractions to enable seamless integration with consuming code. + +The [Microsoft.Extensions.DocumentExtraction](https://www.nuget.org/packages/Microsoft.Extensions.DocumentExtraction) package has an implicit dependency on the `Microsoft.Extensions.DocumentExtraction.Abstractions` package. This package enables you to easily integrate components such as logging, telemetry, and options configuration into your applications using familiar dependency injection and builder patterns. + +## Which package to reference + +Libraries that provide implementations of the abstractions typically reference only `Microsoft.Extensions.DocumentExtraction.Abstractions`. + +To also have access to higher-level utilities for working with document-extraction clients, reference the `Microsoft.Extensions.DocumentExtraction` package instead (which itself references `Microsoft.Extensions.DocumentExtraction.Abstractions`). Most consuming applications and services should reference the `Microsoft.Extensions.DocumentExtraction` package along with a library that provides a concrete implementation of the abstractions. + +## Install the package + +From the command-line: + +```console +dotnet add package Microsoft.Extensions.DocumentExtraction.Abstractions --prerelease +``` + +Or directly in the C# project file: + +```xml + + + +``` + +## Documentation + +Refer to the [Microsoft.Extensions.DocumentExtraction libraries documentation](https://learn.microsoft.com/dotnet/api/microsoft.extensions.documentextraction) for more information and API usage examples. + +## Feedback & Contributing + +We welcome feedback and contributions in [our GitHub repo](https://github.com/dotnet/extensions). diff --git a/src/Libraries/Microsoft.Extensions.DocumentExtraction/ConfigureOptionsDocumentExtractionClient.cs b/src/Libraries/Microsoft.Extensions.DocumentExtraction/ConfigureOptionsDocumentExtractionClient.cs new file mode 100644 index 00000000000..63b69aedc98 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DocumentExtraction/ConfigureOptionsDocumentExtractionClient.cs @@ -0,0 +1,68 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.DocumentExtraction; + +/// Represents a delegating OCR client that configures an instance used by the remainder of the pipeline. +[Experimental(DiagnosticIds.Experiments.DocumentExtraction, UrlFormat = DiagnosticIds.UrlFormat)] +public sealed class ConfigureOptionsDocumentExtractionClient : DelegatingDocumentExtractionClient +{ + /// The callback delegate used to configure options. + private readonly Action _configureOptions; + + /// Initializes a new instance of the class with the specified callback. + /// The inner client. + /// + /// The delegate to invoke to configure the instance. It is passed a clone of the caller-supplied instance + /// (or a newly constructed instance if the caller-supplied instance is ). + /// + /// + /// The delegate is passed either a new instance of if + /// the caller didn't supply an instance, or a clone (via ) of the caller-supplied + /// instance if one was supplied. + /// + public ConfigureOptionsDocumentExtractionClient(IDocumentExtractionClient innerClient, Action configure) + : base(innerClient) + { + _configureOptions = Throw.IfNull(configure); + } + + /// + public override async Task ExtractAsync( + Stream document, + string mediaType, + DocumentExtractionOptions? options = null, + CancellationToken cancellationToken = default) + { + return await base.ExtractAsync(document, mediaType, Configure(options), cancellationToken); + } + + /// + public override IAsyncEnumerable ExtractPagesAsync( + Stream document, + string mediaType, + DocumentExtractionOptions? options = null, + CancellationToken cancellationToken = default) + { + return base.ExtractPagesAsync(document, mediaType, Configure(options), cancellationToken); + } + + /// Creates and configures the to pass along to the inner client. + private DocumentExtractionOptions Configure(DocumentExtractionOptions? options) + { + options = options?.Clone() ?? new(); + + _configureOptions(options); + + return options; + } +} diff --git a/src/Libraries/Microsoft.Extensions.DocumentExtraction/ConfigureOptionsDocumentExtractionClientBuilderExtensions.cs b/src/Libraries/Microsoft.Extensions.DocumentExtraction/ConfigureOptionsDocumentExtractionClientBuilderExtensions.cs new file mode 100644 index 00000000000..c56a6476fde --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DocumentExtraction/ConfigureOptionsDocumentExtractionClientBuilderExtensions.cs @@ -0,0 +1,37 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Diagnostics.CodeAnalysis; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.DocumentExtraction; + +/// Provides extensions for configuring instances. +[Experimental(DiagnosticIds.Experiments.DocumentExtraction, UrlFormat = DiagnosticIds.UrlFormat)] +public static class ConfigureOptionsDocumentExtractionClientBuilderExtensions +{ + /// + /// Adds a callback that configures an to be passed to the next client in the pipeline. + /// + /// The . + /// + /// The delegate to invoke to configure the instance. + /// It is passed a clone of the caller-supplied instance (or a newly constructed instance if the caller-supplied instance is ). + /// + /// + /// This method can be used to set default options. The delegate is passed either a new instance of + /// if the caller didn't supply an instance, or a clone (via ) + /// of the caller-supplied instance if one was supplied. + /// + /// The . + public static DocumentExtractionClientBuilder ConfigureOptions( + this DocumentExtractionClientBuilder builder, Action configure) + { + _ = Throw.IfNull(builder); + _ = Throw.IfNull(configure); + + return builder.Use(innerClient => new ConfigureOptionsDocumentExtractionClient(innerClient, configure)); + } +} diff --git a/src/Libraries/Microsoft.Extensions.DocumentExtraction/DocumentExtractionClientBuilder.cs b/src/Libraries/Microsoft.Extensions.DocumentExtraction/DocumentExtractionClientBuilder.cs new file mode 100644 index 00000000000..8e37ca298ea --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DocumentExtraction/DocumentExtractionClientBuilder.cs @@ -0,0 +1,82 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.DocumentExtraction; + +/// A builder for creating pipelines of . +[Experimental(DiagnosticIds.Experiments.DocumentExtraction, UrlFormat = DiagnosticIds.UrlFormat)] +public sealed class DocumentExtractionClientBuilder +{ + private readonly Func _innerClientFactory; + + /// The registered client factory instances. + private List>? _clientFactories; + + /// Initializes a new instance of the class. + /// The inner that represents the underlying backend. + public DocumentExtractionClientBuilder(IDocumentExtractionClient innerClient) + { + _ = Throw.IfNull(innerClient); + _innerClientFactory = _ => innerClient; + } + + /// Initializes a new instance of the class. + /// A callback that produces the inner that represents the underlying backend. + public DocumentExtractionClientBuilder(Func innerClientFactory) + { + _innerClientFactory = Throw.IfNull(innerClientFactory); + } + + /// Builds an that represents the entire pipeline. Calls to this instance will pass through each of the pipeline stages in turn. + /// + /// The that should provide services to the instances. + /// If null, an empty will be used. + /// + /// An instance of that represents the entire pipeline. + public IDocumentExtractionClient Build(IServiceProvider? services = null) + { + services ??= EmptyServiceProvider.Instance; + var documentExtractionClient = _innerClientFactory(services); + + // To match intuitive expectations, apply the factories in reverse order, so that the first factory added is the outermost. + if (_clientFactories is not null) + { + for (var i = _clientFactories.Count - 1; i >= 0; i--) + { + documentExtractionClient = _clientFactories[i](documentExtractionClient, services) ?? + throw new InvalidOperationException( + $"The {nameof(DocumentExtractionClientBuilder)} entry at index {i} returned null. " + + $"Ensure that the callbacks passed to {nameof(Use)} return non-null {nameof(IDocumentExtractionClient)} instances."); + } + } + + return documentExtractionClient; + } + + /// Adds a factory for an intermediate OCR client to the OCR client pipeline. + /// The client factory function. + /// The updated instance. + public DocumentExtractionClientBuilder Use(Func clientFactory) + { + _ = Throw.IfNull(clientFactory); + + return Use((innerClient, _) => clientFactory(innerClient)); + } + + /// Adds a factory for an intermediate OCR client to the OCR client pipeline. + /// The client factory function. + /// The updated instance. + public DocumentExtractionClientBuilder Use(Func clientFactory) + { + _ = Throw.IfNull(clientFactory); + + (_clientFactories ??= []).Add(clientFactory); + return this; + } +} diff --git a/src/Libraries/Microsoft.Extensions.DocumentExtraction/DocumentExtractionClientBuilderExtensions.cs b/src/Libraries/Microsoft.Extensions.DocumentExtraction/DocumentExtractionClientBuilderExtensions.cs new file mode 100644 index 00000000000..32255c9d4fb --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DocumentExtraction/DocumentExtractionClientBuilderExtensions.cs @@ -0,0 +1,27 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.DocumentExtraction; + +/// Provides extension methods for working with in the context of . +[Experimental(DiagnosticIds.Experiments.DocumentExtraction, UrlFormat = DiagnosticIds.UrlFormat)] +public static class DocumentExtractionClientBuilderExtensions +{ + /// Creates a new using as its inner client. + /// The client to use as the inner client. + /// The new instance. + /// + /// This method is equivalent to using the constructor directly, + /// specifying as the inner client. + /// + public static DocumentExtractionClientBuilder AsBuilder(this IDocumentExtractionClient innerClient) + { + _ = Throw.IfNull(innerClient); + + return new DocumentExtractionClientBuilder(innerClient); + } +} diff --git a/src/Libraries/Microsoft.Extensions.DocumentExtraction/DocumentExtractionClientBuilderServiceCollectionExtensions.cs b/src/Libraries/Microsoft.Extensions.DocumentExtraction/DocumentExtractionClientBuilderServiceCollectionExtensions.cs new file mode 100644 index 00000000000..a85f1f45d80 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DocumentExtraction/DocumentExtractionClientBuilderServiceCollectionExtensions.cs @@ -0,0 +1,85 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.DocumentExtraction; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.DependencyInjection; + +/// Provides extension methods for registering with a . +[Experimental(DiagnosticIds.Experiments.DocumentExtraction, UrlFormat = DiagnosticIds.UrlFormat)] +public static class DocumentExtractionClientBuilderServiceCollectionExtensions +{ + /// Registers a singleton in the . + /// The to which the client should be added. + /// The inner that represents the underlying backend. + /// The service lifetime for the client. Defaults to . + /// An that can be used to build a pipeline around the inner client. + /// is . + /// is . + public static DocumentExtractionClientBuilder AddDocumentExtractionClient( + this IServiceCollection serviceCollection, + IDocumentExtractionClient innerClient, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + => AddDocumentExtractionClient(serviceCollection, _ => innerClient, lifetime); + + /// Registers a singleton in the . + /// The to which the client should be added. + /// A callback that produces the inner that represents the underlying backend. + /// The service lifetime for the client. Defaults to . + /// An that can be used to build a pipeline around the inner client. + /// is . + /// is . + public static DocumentExtractionClientBuilder AddDocumentExtractionClient( + this IServiceCollection serviceCollection, + Func innerClientFactory, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + { + _ = Throw.IfNull(serviceCollection); + _ = Throw.IfNull(innerClientFactory); + + var builder = new DocumentExtractionClientBuilder(innerClientFactory); + serviceCollection.Add(new ServiceDescriptor(typeof(IDocumentExtractionClient), builder.Build, lifetime)); + return builder; + } + + /// Registers a keyed singleton in the . + /// The to which the client should be added. + /// The key with which to associate the client. + /// The inner that represents the underlying backend. + /// The service lifetime for the client. Defaults to . + /// An that can be used to build a pipeline around the inner client. + /// is . + /// is . + public static DocumentExtractionClientBuilder AddKeyedDocumentExtractionClient( + this IServiceCollection serviceCollection, + object? serviceKey, + IDocumentExtractionClient innerClient, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + => AddKeyedDocumentExtractionClient(serviceCollection, serviceKey, _ => innerClient, lifetime); + + /// Registers a keyed singleton in the . + /// The to which the client should be added. + /// The key with which to associate the client. + /// A callback that produces the inner that represents the underlying backend. + /// The service lifetime for the client. Defaults to . + /// An that can be used to build a pipeline around the inner client. + /// is . + /// is . + public static DocumentExtractionClientBuilder AddKeyedDocumentExtractionClient( + this IServiceCollection serviceCollection, + object? serviceKey, + Func innerClientFactory, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + { + _ = Throw.IfNull(serviceCollection); + _ = Throw.IfNull(innerClientFactory); + + var builder = new DocumentExtractionClientBuilder(innerClientFactory); + serviceCollection.Add(new ServiceDescriptor(typeof(IDocumentExtractionClient), serviceKey, factory: (services, serviceKey) => builder.Build(services), lifetime)); + return builder; + } +} diff --git a/src/Libraries/Microsoft.Extensions.DocumentExtraction/EmptyServiceProvider.cs b/src/Libraries/Microsoft.Extensions.DocumentExtraction/EmptyServiceProvider.cs new file mode 100644 index 00000000000..cd827104ae8 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DocumentExtraction/EmptyServiceProvider.cs @@ -0,0 +1,25 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using Microsoft.Extensions.DependencyInjection; + +namespace Microsoft.Extensions.DocumentExtraction; + +/// Provides an implementation of that contains no services. +internal sealed class EmptyServiceProvider : IKeyedServiceProvider +{ + /// Gets a singleton instance of . + public static EmptyServiceProvider Instance { get; } = new(); + + /// + public object? GetService(Type serviceType) => null; + + /// + public object? GetKeyedService(Type serviceType, object? serviceKey) => null; + + /// + public object GetRequiredKeyedService(Type serviceType, object? serviceKey) => + GetKeyedService(serviceType, serviceKey) ?? + throw new InvalidOperationException($"No service for type '{serviceType}' and key '{serviceKey}' has been registered."); +} diff --git a/src/Libraries/Microsoft.Extensions.DocumentExtraction/LoggingDocumentExtractionClient.cs b/src/Libraries/Microsoft.Extensions.DocumentExtraction/LoggingDocumentExtractionClient.cs new file mode 100644 index 00000000000..bc7d5c536d4 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DocumentExtraction/LoggingDocumentExtractionClient.cs @@ -0,0 +1,215 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Runtime.CompilerServices; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.Logging; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.DocumentExtraction; + +/// A delegating OCR client that logs OCR operations to an . +/// +/// +/// The provided implementation of is thread-safe for concurrent use so long as the +/// employed is also thread-safe for concurrent use. +/// +/// +/// When the employed enables , the contents of +/// options and results are logged. These may contain sensitive application data. +/// is disabled by default and should never be enabled in a production environment. +/// Options and results are not logged at other logging levels. +/// +/// +[Experimental(DiagnosticIds.Experiments.DocumentExtraction, UrlFormat = DiagnosticIds.UrlFormat)] +public partial class LoggingDocumentExtractionClient : DelegatingDocumentExtractionClient +{ + /// An instance used for all logging. + private readonly ILogger _logger; + + /// The to use for serialization of state written to the logger. + private JsonSerializerOptions _jsonSerializerOptions; + + /// Initializes a new instance of the class. + /// The underlying . + /// An instance that will be used for all logging. + public LoggingDocumentExtractionClient(IDocumentExtractionClient innerClient, ILogger logger) + : base(innerClient) + { + _logger = Throw.IfNull(logger); + _jsonSerializerOptions = AIJsonUtilities.DefaultOptions; + } + + /// Gets or sets JSON serialization options to use when serializing logging data. + public JsonSerializerOptions JsonSerializerOptions + { + get => _jsonSerializerOptions; + set => _jsonSerializerOptions = Throw.IfNull(value); + } + + /// + public override async Task ExtractAsync( + Stream document, + string mediaType, + DocumentExtractionOptions? options = null, + CancellationToken cancellationToken = default) + { + if (_logger.IsEnabled(LogLevel.Debug)) + { + if (_logger.IsEnabled(LogLevel.Trace)) + { + LogInvokedSensitive(nameof(ExtractAsync), mediaType, AsJson(options), AsJson(this.GetService())); + } + else + { + LogInvoked(nameof(ExtractAsync)); + } + } + + try + { + var result = await base.ExtractAsync(document, mediaType, options, cancellationToken); + + if (_logger.IsEnabled(LogLevel.Debug)) + { + if (_logger.IsEnabled(LogLevel.Trace)) + { + LogCompletedSensitive(nameof(ExtractAsync), AsJson(result)); + } + else + { + LogCompleted(nameof(ExtractAsync)); + } + } + + return result; + } + catch (OperationCanceledException) + { + LogInvocationCanceled(nameof(ExtractAsync)); + throw; + } + catch (Exception ex) + { + LogInvocationFailed(nameof(ExtractAsync), ex); + throw; + } + } + + /// + public override async IAsyncEnumerable ExtractPagesAsync( + Stream document, + string mediaType, + DocumentExtractionOptions? options = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + if (_logger.IsEnabled(LogLevel.Debug)) + { + if (_logger.IsEnabled(LogLevel.Trace)) + { + LogInvokedSensitive(nameof(ExtractPagesAsync), mediaType, AsJson(options), AsJson(this.GetService())); + } + else + { + LogInvoked(nameof(ExtractPagesAsync)); + } + } + + IAsyncEnumerator e; + try + { + e = base.ExtractPagesAsync(document, mediaType, options, cancellationToken).GetAsyncEnumerator(cancellationToken); + } + catch (OperationCanceledException) + { + LogInvocationCanceled(nameof(ExtractPagesAsync)); + throw; + } + catch (Exception ex) + { + LogInvocationFailed(nameof(ExtractPagesAsync), ex); + throw; + } + + try + { + DocumentExtractionPageResult? update = null; + while (true) + { + try + { + if (!await e.MoveNextAsync()) + { + break; + } + + update = e.Current; + } + catch (OperationCanceledException) + { + LogInvocationCanceled(nameof(ExtractPagesAsync)); + throw; + } + catch (Exception ex) + { + LogInvocationFailed(nameof(ExtractPagesAsync), ex); + throw; + } + + if (_logger.IsEnabled(LogLevel.Debug)) + { + if (_logger.IsEnabled(LogLevel.Trace)) + { + LogStreamingUpdateSensitive(AsJson(update)); + } + else + { + LogStreamingUpdate(); + } + } + + yield return update; + } + + LogCompleted(nameof(ExtractPagesAsync)); + } + finally + { + await e.DisposeAsync(); + } + } + + private string AsJson(T value) => TelemetryHelpers.AsJson(value, _jsonSerializerOptions); + + [LoggerMessage(LogLevel.Debug, "{MethodName} invoked.")] + private partial void LogInvoked(string methodName); + + [LoggerMessage(LogLevel.Trace, "{MethodName} invoked: MediaType: {MediaType}. Options: {DocumentExtractionOptions}. Metadata: {DocumentExtractionClientMetadata}.")] + private partial void LogInvokedSensitive(string methodName, string mediaType, string documentExtractionOptions, string documentExtractionClientMetadata); + + [LoggerMessage(LogLevel.Debug, "{MethodName} completed.")] + private partial void LogCompleted(string methodName); + + [LoggerMessage(LogLevel.Trace, "{MethodName} completed: {DocumentExtractionResult}.")] + private partial void LogCompletedSensitive(string methodName, string documentExtractionResult); + + [LoggerMessage(LogLevel.Debug, "ExtractPagesAsync received update.")] + private partial void LogStreamingUpdate(); + + [LoggerMessage(LogLevel.Trace, "ExtractPagesAsync received update: {DocumentExtractionPageResult}")] + private partial void LogStreamingUpdateSensitive(string documentExtractionPageResult); + + [LoggerMessage(LogLevel.Debug, "{MethodName} canceled.")] + private partial void LogInvocationCanceled(string methodName); + + [LoggerMessage(LogLevel.Error, "{MethodName} failed.")] + private partial void LogInvocationFailed(string methodName, Exception error); +} diff --git a/src/Libraries/Microsoft.Extensions.DocumentExtraction/LoggingDocumentExtractionClientBuilderExtensions.cs b/src/Libraries/Microsoft.Extensions.DocumentExtraction/LoggingDocumentExtractionClientBuilderExtensions.cs new file mode 100644 index 00000000000..88259ad1252 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DocumentExtraction/LoggingDocumentExtractionClientBuilderExtensions.cs @@ -0,0 +1,57 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.DocumentExtraction; + +/// Provides extensions for configuring instances. +[Experimental(DiagnosticIds.Experiments.DocumentExtraction, UrlFormat = DiagnosticIds.UrlFormat)] +public static class LoggingDocumentExtractionClientBuilderExtensions +{ + /// Adds logging to the OCR client pipeline. + /// The . + /// + /// An optional used to create a logger with which logging should be performed. + /// If not supplied, a required instance will be resolved from the service provider. + /// + /// An optional callback that can be used to configure the instance. + /// The . + /// + /// + /// When the employed enables , the contents of + /// options and results are logged. These may contain sensitive application data. + /// is disabled by default and should never be enabled in a production environment. + /// Options and results are not logged at other logging levels. + /// + /// + public static DocumentExtractionClientBuilder UseLogging( + this DocumentExtractionClientBuilder builder, + ILoggerFactory? loggerFactory = null, + Action? configure = null) + { + _ = Throw.IfNull(builder); + + return builder.Use((innerClient, services) => + { + loggerFactory ??= services.GetRequiredService(); + + // If the factory we resolve is for the null logger, the LoggingDocumentExtractionClient will end up + // being an expensive nop, so skip adding it and just return the inner client. + if (loggerFactory == NullLoggerFactory.Instance) + { + return innerClient; + } + + var documentExtractionClient = new LoggingDocumentExtractionClient(innerClient, loggerFactory.CreateLogger(typeof(LoggingDocumentExtractionClient))); + configure?.Invoke(documentExtractionClient); + return documentExtractionClient; + }); + } +} diff --git a/src/Libraries/Microsoft.Extensions.DocumentExtraction/Microsoft.Extensions.DocumentExtraction.csproj b/src/Libraries/Microsoft.Extensions.DocumentExtraction/Microsoft.Extensions.DocumentExtraction.csproj new file mode 100644 index 00000000000..55e7657c7ed --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DocumentExtraction/Microsoft.Extensions.DocumentExtraction.csproj @@ -0,0 +1,54 @@ + + + + Microsoft.Extensions.DocumentExtraction + Utilities for working with document extraction components. + AI + document;extraction;ocr;layout;ai + true + + + + normal + false + 75 + 75 + + + + $(TargetFrameworks);netstandard2.0 + $(NoWarn);MEAI001;MEDE0001 + + + $(NoWarn);CA2007 + + true + true + + + + true + true + true + true + true + false + + + + + + + + + + + + + + + + + diff --git a/src/Libraries/Microsoft.Extensions.DocumentExtraction/Microsoft.Extensions.DocumentExtraction.json b/src/Libraries/Microsoft.Extensions.DocumentExtraction/Microsoft.Extensions.DocumentExtraction.json new file mode 100644 index 00000000000..a18edd9df6e --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DocumentExtraction/Microsoft.Extensions.DocumentExtraction.json @@ -0,0 +1,167 @@ +{ + "Name": "Microsoft.Extensions.DocumentExtraction, Version=10.8.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35", + "Types": [ + { + "Type": "sealed class Microsoft.Extensions.DocumentExtraction.ConfigureOptionsDocumentExtractionClient : Microsoft.Extensions.DocumentExtraction.DelegatingDocumentExtractionClient", + "Stage": "Experimental", + "Methods": [ + { + "Member": "Microsoft.Extensions.DocumentExtraction.ConfigureOptionsDocumentExtractionClient.ConfigureOptionsDocumentExtractionClient(Microsoft.Extensions.DocumentExtraction.IDocumentExtractionClient innerClient, System.Action configure);", + "Stage": "Experimental" + }, + { + "Member": "override System.Threading.Tasks.Task Microsoft.Extensions.DocumentExtraction.ConfigureOptionsDocumentExtractionClient.ExtractAsync(System.IO.Stream document, string mediaType, Microsoft.Extensions.DocumentExtraction.DocumentExtractionOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", + "Stage": "Experimental" + }, + { + "Member": "override System.Collections.Generic.IAsyncEnumerable Microsoft.Extensions.DocumentExtraction.ConfigureOptionsDocumentExtractionClient.ExtractPagesAsync(System.IO.Stream document, string mediaType, Microsoft.Extensions.DocumentExtraction.DocumentExtractionOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", + "Stage": "Experimental" + } + ] + }, + { + "Type": "static class Microsoft.Extensions.DocumentExtraction.ConfigureOptionsDocumentExtractionClientBuilderExtensions", + "Stage": "Experimental", + "Methods": [ + { + "Member": "static Microsoft.Extensions.DocumentExtraction.DocumentExtractionClientBuilder Microsoft.Extensions.DocumentExtraction.ConfigureOptionsDocumentExtractionClientBuilderExtensions.ConfigureOptions(this Microsoft.Extensions.DocumentExtraction.DocumentExtractionClientBuilder builder, System.Action configure);", + "Stage": "Experimental" + } + ] + }, + { + "Type": "sealed class Microsoft.Extensions.DocumentExtraction.DocumentExtractionClientBuilder", + "Stage": "Experimental", + "Methods": [ + { + "Member": "Microsoft.Extensions.DocumentExtraction.DocumentExtractionClientBuilder.DocumentExtractionClientBuilder(Microsoft.Extensions.DocumentExtraction.IDocumentExtractionClient innerClient);", + "Stage": "Experimental" + }, + { + "Member": "Microsoft.Extensions.DocumentExtraction.DocumentExtractionClientBuilder.DocumentExtractionClientBuilder(System.Func innerClientFactory);", + "Stage": "Experimental" + }, + { + "Member": "Microsoft.Extensions.DocumentExtraction.IDocumentExtractionClient Microsoft.Extensions.DocumentExtraction.DocumentExtractionClientBuilder.Build(System.IServiceProvider? services = null);", + "Stage": "Experimental" + }, + { + "Member": "Microsoft.Extensions.DocumentExtraction.DocumentExtractionClientBuilder Microsoft.Extensions.DocumentExtraction.DocumentExtractionClientBuilder.Use(System.Func clientFactory);", + "Stage": "Experimental" + }, + { + "Member": "Microsoft.Extensions.DocumentExtraction.DocumentExtractionClientBuilder Microsoft.Extensions.DocumentExtraction.DocumentExtractionClientBuilder.Use(System.Func clientFactory);", + "Stage": "Experimental" + } + ] + }, + { + "Type": "static class Microsoft.Extensions.DocumentExtraction.DocumentExtractionClientBuilderExtensions", + "Stage": "Experimental", + "Methods": [ + { + "Member": "static Microsoft.Extensions.DocumentExtraction.DocumentExtractionClientBuilder Microsoft.Extensions.DocumentExtraction.DocumentExtractionClientBuilderExtensions.AsBuilder(this Microsoft.Extensions.DocumentExtraction.IDocumentExtractionClient innerClient);", + "Stage": "Experimental" + } + ] + }, + { + "Type": "static class Microsoft.Extensions.DependencyInjection.DocumentExtractionClientBuilderServiceCollectionExtensions", + "Stage": "Experimental", + "Methods": [ + { + "Member": "static Microsoft.Extensions.DocumentExtraction.DocumentExtractionClientBuilder Microsoft.Extensions.DependencyInjection.DocumentExtractionClientBuilderServiceCollectionExtensions.AddDocumentExtractionClient(this Microsoft.Extensions.DependencyInjection.IServiceCollection serviceCollection, Microsoft.Extensions.DocumentExtraction.IDocumentExtractionClient innerClient, Microsoft.Extensions.DependencyInjection.ServiceLifetime lifetime = Microsoft.Extensions.DependencyInjection.ServiceLifetime.Singleton);", + "Stage": "Experimental" + }, + { + "Member": "static Microsoft.Extensions.DocumentExtraction.DocumentExtractionClientBuilder Microsoft.Extensions.DependencyInjection.DocumentExtractionClientBuilderServiceCollectionExtensions.AddDocumentExtractionClient(this Microsoft.Extensions.DependencyInjection.IServiceCollection serviceCollection, System.Func innerClientFactory, Microsoft.Extensions.DependencyInjection.ServiceLifetime lifetime = Microsoft.Extensions.DependencyInjection.ServiceLifetime.Singleton);", + "Stage": "Experimental" + }, + { + "Member": "static Microsoft.Extensions.DocumentExtraction.DocumentExtractionClientBuilder Microsoft.Extensions.DependencyInjection.DocumentExtractionClientBuilderServiceCollectionExtensions.AddKeyedDocumentExtractionClient(this Microsoft.Extensions.DependencyInjection.IServiceCollection serviceCollection, object? serviceKey, Microsoft.Extensions.DocumentExtraction.IDocumentExtractionClient innerClient, Microsoft.Extensions.DependencyInjection.ServiceLifetime lifetime = Microsoft.Extensions.DependencyInjection.ServiceLifetime.Singleton);", + "Stage": "Experimental" + }, + { + "Member": "static Microsoft.Extensions.DocumentExtraction.DocumentExtractionClientBuilder Microsoft.Extensions.DependencyInjection.DocumentExtractionClientBuilderServiceCollectionExtensions.AddKeyedDocumentExtractionClient(this Microsoft.Extensions.DependencyInjection.IServiceCollection serviceCollection, object? serviceKey, System.Func innerClientFactory, Microsoft.Extensions.DependencyInjection.ServiceLifetime lifetime = Microsoft.Extensions.DependencyInjection.ServiceLifetime.Singleton);", + "Stage": "Experimental" + } + ] + }, + { + "Type": "class Microsoft.Extensions.DocumentExtraction.LoggingDocumentExtractionClient : Microsoft.Extensions.DocumentExtraction.DelegatingDocumentExtractionClient", + "Stage": "Experimental", + "Methods": [ + { + "Member": "Microsoft.Extensions.DocumentExtraction.LoggingDocumentExtractionClient.LoggingDocumentExtractionClient(Microsoft.Extensions.DocumentExtraction.IDocumentExtractionClient innerClient, Microsoft.Extensions.Logging.ILogger logger);", + "Stage": "Experimental" + }, + { + "Member": "override System.Threading.Tasks.Task Microsoft.Extensions.DocumentExtraction.LoggingDocumentExtractionClient.ExtractAsync(System.IO.Stream document, string mediaType, Microsoft.Extensions.DocumentExtraction.DocumentExtractionOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", + "Stage": "Experimental" + }, + { + "Member": "override System.Collections.Generic.IAsyncEnumerable Microsoft.Extensions.DocumentExtraction.LoggingDocumentExtractionClient.ExtractPagesAsync(System.IO.Stream document, string mediaType, Microsoft.Extensions.DocumentExtraction.DocumentExtractionOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", + "Stage": "Experimental" + } + ], + "Properties": [ + { + "Member": "System.Text.Json.JsonSerializerOptions Microsoft.Extensions.DocumentExtraction.LoggingDocumentExtractionClient.JsonSerializerOptions { get; set; }", + "Stage": "Experimental" + } + ] + }, + { + "Type": "static class Microsoft.Extensions.DocumentExtraction.LoggingDocumentExtractionClientBuilderExtensions", + "Stage": "Experimental", + "Methods": [ + { + "Member": "static Microsoft.Extensions.DocumentExtraction.DocumentExtractionClientBuilder Microsoft.Extensions.DocumentExtraction.LoggingDocumentExtractionClientBuilderExtensions.UseLogging(this Microsoft.Extensions.DocumentExtraction.DocumentExtractionClientBuilder builder, Microsoft.Extensions.Logging.ILoggerFactory? loggerFactory = null, System.Action? configure = null);", + "Stage": "Experimental" + } + ] + }, + { + "Type": "sealed class Microsoft.Extensions.DocumentExtraction.OpenTelemetryDocumentExtractionClient : Microsoft.Extensions.DocumentExtraction.DelegatingDocumentExtractionClient", + "Stage": "Experimental", + "Methods": [ + { + "Member": "Microsoft.Extensions.DocumentExtraction.OpenTelemetryDocumentExtractionClient.OpenTelemetryDocumentExtractionClient(Microsoft.Extensions.DocumentExtraction.IDocumentExtractionClient innerClient, Microsoft.Extensions.Logging.ILogger? logger = null, string? sourceName = null);", + "Stage": "Experimental" + }, + { + "Member": "override void Microsoft.Extensions.DocumentExtraction.OpenTelemetryDocumentExtractionClient.Dispose(bool disposing);", + "Stage": "Experimental" + }, + { + "Member": "override System.Threading.Tasks.Task Microsoft.Extensions.DocumentExtraction.OpenTelemetryDocumentExtractionClient.ExtractAsync(System.IO.Stream document, string mediaType, Microsoft.Extensions.DocumentExtraction.DocumentExtractionOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", + "Stage": "Experimental" + }, + { + "Member": "override System.Collections.Generic.IAsyncEnumerable Microsoft.Extensions.DocumentExtraction.OpenTelemetryDocumentExtractionClient.ExtractPagesAsync(System.IO.Stream document, string mediaType, Microsoft.Extensions.DocumentExtraction.DocumentExtractionOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", + "Stage": "Experimental" + }, + { + "Member": "override object? Microsoft.Extensions.DocumentExtraction.OpenTelemetryDocumentExtractionClient.GetService(System.Type serviceType, object? serviceKey = null);", + "Stage": "Experimental" + } + ], + "Properties": [ + { + "Member": "bool Microsoft.Extensions.DocumentExtraction.OpenTelemetryDocumentExtractionClient.EnableSensitiveData { get; set; }", + "Stage": "Experimental" + } + ] + }, + { + "Type": "static class Microsoft.Extensions.DocumentExtraction.OpenTelemetryDocumentExtractionClientBuilderExtensions", + "Stage": "Experimental", + "Methods": [ + { + "Member": "static Microsoft.Extensions.DocumentExtraction.DocumentExtractionClientBuilder Microsoft.Extensions.DocumentExtraction.OpenTelemetryDocumentExtractionClientBuilderExtensions.UseOpenTelemetry(this Microsoft.Extensions.DocumentExtraction.DocumentExtractionClientBuilder builder, Microsoft.Extensions.Logging.ILoggerFactory? loggerFactory = null, string? sourceName = null, System.Action? configure = null);", + "Stage": "Experimental" + } + ] + } + ] +} \ No newline at end of file diff --git a/src/Libraries/Microsoft.Extensions.DocumentExtraction/OpenTelemetryConsts.cs b/src/Libraries/Microsoft.Extensions.DocumentExtraction/OpenTelemetryConsts.cs new file mode 100644 index 00000000000..7a15ef264ce --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DocumentExtraction/OpenTelemetryConsts.cs @@ -0,0 +1,82 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.Extensions.DocumentExtraction; + +#pragma warning disable S4041 // Type names should not match namespaces + +/// Provides constants used by various telemetry services. +internal static class OpenTelemetryConsts +{ + public const string DefaultSourceName = "Experimental.Microsoft.Extensions.DocumentExtraction"; + + public const string SecondsUnit = "s"; + public const string TokensUnit = "{token}"; + + /// Environment variable name for controlling whether sensitive content should be captured in telemetry by default. + public const string GenAICaptureMessageContentEnvVar = "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT"; + + public const string TypeText = "text"; + + public static class Error + { + public const string Type = "error.type"; + } + + public static class GenAI + { + public const string GenerateContentName = "generate_content"; + + public static class Client + { + public static class OperationDuration + { + public const string Description = "Measures the duration of a GenAI operation"; + public const string Name = "gen_ai.client.operation.duration"; + public static readonly double[] ExplicitBucketBoundaries = [0.01, 0.02, 0.04, 0.08, 0.16, 0.32, 0.64, 1.28, 2.56, 5.12, 10.24, 20.48, 40.96, 81.92]; + } + + public static class TokenUsage + { + public const string Description = "Measures number of input and output tokens used"; + public const string Name = "gen_ai.client.token.usage"; + public static readonly int[] ExplicitBucketBoundaries = [1, 4, 16, 64, 256, 1_024, 4_096, 16_384, 65_536, 262_144, 1_048_576, 4_194_304, 16_777_216, 67_108_864]; + } + } + + public static class Operation + { + public const string Name = "gen_ai.operation.name"; + } + + public static class Output + { + public const string Type = "gen_ai.output.type"; + } + + public static class Provider + { + public const string Name = "gen_ai.provider.name"; + } + + public static class Request + { + public const string Model = "gen_ai.request.model"; + } + + public static class Usage + { + /// + /// Number of document pages processed by a document-extraction request. + /// This attribute is NOT part of the OpenTelemetry GenAI semantic conventions (as of v1.41). + /// + public const string PagesProcessed = "gen_ai.usage.pages_processed"; // Non-standard + } + } + + public static class Server + { + public const string Address = "server.address"; + public const string Port = "server.port"; + } +} diff --git a/src/Libraries/Microsoft.Extensions.DocumentExtraction/OpenTelemetryDocumentExtractionClient.cs b/src/Libraries/Microsoft.Extensions.DocumentExtraction/OpenTelemetryDocumentExtractionClient.cs new file mode 100644 index 00000000000..5189728ca64 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DocumentExtraction/OpenTelemetryDocumentExtractionClient.cs @@ -0,0 +1,294 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Diagnostics.Metrics; +using System.IO; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.DocumentExtraction; + +/// Represents a delegating OCR client that implements the OpenTelemetry Semantic Conventions for Generative AI systems. +/// +/// This class provides an implementation of the Semantic Conventions for Generative AI systems v1.41, defined at . +/// The specification is still experimental and subject to change; as such, the telemetry output by this client is also subject to change. +/// +[Experimental(DiagnosticIds.Experiments.DocumentExtraction, UrlFormat = DiagnosticIds.UrlFormat)] +public sealed class OpenTelemetryDocumentExtractionClient : DelegatingDocumentExtractionClient +{ + private readonly ActivitySource _activitySource; + private readonly Meter _meter; + + private readonly Histogram _operationDurationHistogram; + + private readonly string? _defaultModelId; + private readonly string? _providerName; + private readonly string? _serverAddress; + private readonly int _serverPort; + + private readonly ILogger? _logger; + + /// Initializes a new instance of the class. + /// The underlying . + /// The to use for emitting any logging data from the client. + /// An optional source name that will be used on the telemetry data. + public OpenTelemetryDocumentExtractionClient(IDocumentExtractionClient innerClient, ILogger? logger = null, string? sourceName = null) + : base(innerClient) + { + Debug.Assert(innerClient is not null, "Should have been validated by the base ctor"); + + _logger = logger; + + if (innerClient!.GetService() is DocumentExtractionClientMetadata metadata) + { + _defaultModelId = metadata.DefaultModelId; + _providerName = metadata.ProviderName; + _serverAddress = metadata.ProviderUri?.Host; + _serverPort = metadata.ProviderUri?.Port ?? 0; + } + + string name = string.IsNullOrEmpty(sourceName) ? OpenTelemetryConsts.DefaultSourceName : sourceName!; + _activitySource = new(name); + _meter = new(name); + + _operationDurationHistogram = OtelMetricHelpers.CreateGenAIOperationDurationHistogram(_meter); + } + + /// + protected override void Dispose(bool disposing) + { + if (disposing) + { + _activitySource.Dispose(); + _meter.Dispose(); + } + + base.Dispose(disposing); + } + + /// + /// Gets or sets a value indicating whether potentially sensitive information should be included in telemetry. + /// + /// + /// if potentially sensitive information should be included in telemetry; + /// if telemetry shouldn't include raw inputs and outputs. + /// The default value is , unless the OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT + /// environment variable is set to "true" (case-insensitive). + /// + /// + /// By default, telemetry includes metadata, such as page counts, but not raw inputs + /// and outputs, such as document content. The default value can be overridden by setting the + /// OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT environment variable to "true". + /// Explicitly setting this property will override the environment variable. + /// + public bool EnableSensitiveData { get; set; } = TelemetryHelpers.EnableSensitiveDataDefault; + + /// + public override object? GetService(Type serviceType, object? serviceKey = null) => + serviceType == typeof(ActivitySource) ? _activitySource : + base.GetService(serviceType, serviceKey); + + /// + public override async Task ExtractAsync( + Stream document, + string mediaType, + DocumentExtractionOptions? options = null, + CancellationToken cancellationToken = default) + { + _ = Throw.IfNull(document); + + using Activity? activity = CreateAndConfigureActivity(options); + Stopwatch? stopwatch = _operationDurationHistogram.Enabled ? Stopwatch.StartNew() : null; + string? requestModelId = options?.ModelId ?? _defaultModelId; + + DocumentExtractionResult? response = null; + Exception? error = null; + try + { + response = await base.ExtractAsync(document, mediaType, options, cancellationToken); + return response; + } + catch (Exception ex) + { + error = ex; + throw; + } + finally + { + TraceResponse(activity, requestModelId, response, error, stopwatch); + } + } + + /// + public override async IAsyncEnumerable ExtractPagesAsync( + Stream document, + string mediaType, + DocumentExtractionOptions? options = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + _ = Throw.IfNull(document); + + using Activity? activity = CreateAndConfigureActivity(options); + Stopwatch? stopwatch = _operationDurationHistogram.Enabled ? Stopwatch.StartNew() : null; + string? requestModelId = options?.ModelId ?? _defaultModelId; + + IAsyncEnumerable updates; + try + { + updates = base.ExtractPagesAsync(document, mediaType, options, cancellationToken); + } + catch (Exception ex) + { + TraceResponse(activity, requestModelId, response: null, ex, stopwatch); + throw; + } + + var responseEnumerator = updates.GetAsyncEnumerator(cancellationToken); + List trackedUpdates = []; + Exception? error = null; + try + { + while (true) + { + DocumentExtractionPageResult update; + try + { + if (!await responseEnumerator.MoveNextAsync()) + { + break; + } + + update = responseEnumerator.Current; + } + catch (Exception ex) + { + error = ex; + throw; + } + + trackedUpdates.Add(update); + yield return update; + if (activity is not null) + { + Activity.Current = activity; // workaround for https://github.com/dotnet/runtime/issues/47802 + } + } + } + finally + { + TraceResponse(activity, requestModelId, trackedUpdates.ToDocumentExtractionResult(), error, stopwatch); + + await responseEnumerator.DisposeAsync(); + } + } + + /// Creates an activity for an OCR request, or returns if not enabled. + private Activity? CreateAndConfigureActivity(DocumentExtractionOptions? options) + { + Activity? activity = null; + if (_activitySource.HasListeners()) + { + string? modelId = options?.ModelId ?? _defaultModelId; + + activity = _activitySource.StartActivity( + string.IsNullOrWhiteSpace(modelId) ? OpenTelemetryConsts.GenAI.GenerateContentName : $"{OpenTelemetryConsts.GenAI.GenerateContentName} {modelId}", + ActivityKind.Client); + + if (activity is { IsAllDataRequested: true }) + { + _ = activity + .AddTag(OpenTelemetryConsts.GenAI.Operation.Name, OpenTelemetryConsts.GenAI.GenerateContentName) + .AddTag(OpenTelemetryConsts.GenAI.Request.Model, modelId) + .AddTag(OpenTelemetryConsts.GenAI.Provider.Name, _providerName) + .AddTag(OpenTelemetryConsts.GenAI.Output.Type, OpenTelemetryConsts.TypeText); + + if (_serverAddress is not null) + { + _ = activity + .AddTag(OpenTelemetryConsts.Server.Address, _serverAddress) + .AddTag(OpenTelemetryConsts.Server.Port, _serverPort); + } + + if (EnableSensitiveData && options?.AdditionalProperties is { } props) + { + // Log all additional request options as raw values on the span. + // Since AdditionalProperties has undefined meaning, we treat it as potentially sensitive data. + foreach (KeyValuePair prop in props) + { + _ = activity.AddTag(prop.Key, prop.Value); + } + } + } + } + + return activity; + } + + /// Adds OCR response information to the activity. + private void TraceResponse( + Activity? activity, + string? requestModelId, + DocumentExtractionResult? response, + Exception? error, + Stopwatch? stopwatch) + { + if (_operationDurationHistogram.Enabled && stopwatch is not null) + { + TagList tags = default; + + AddMetricTags(ref tags, requestModelId); + if (error is not null) + { + tags.Add(OpenTelemetryConsts.Error.Type, error.GetType().FullName); + } + + _operationDurationHistogram.Record(stopwatch.Elapsed.TotalSeconds, tags); + } + + OpenTelemetryLog.RecordOperationError(activity, _logger, error); + + if (response is not null && activity is not null) + { + if (response.Usage?.PagesProcessed is int pages) + { + _ = activity.AddTag(OpenTelemetryConsts.GenAI.Usage.PagesProcessed, pages); + } + + // Log all additional response properties as raw values on the span. + // Since AdditionalProperties has undefined meaning, we treat it as potentially sensitive data. + if (EnableSensitiveData && response.AdditionalProperties is { } props) + { + foreach (KeyValuePair prop in props) + { + _ = activity.AddTag(prop.Key, prop.Value); + } + } + } + + void AddMetricTags(ref TagList tags, string? requestModelId) + { + tags.Add(OpenTelemetryConsts.GenAI.Operation.Name, OpenTelemetryConsts.GenAI.GenerateContentName); + + if (requestModelId is not null) + { + tags.Add(OpenTelemetryConsts.GenAI.Request.Model, requestModelId); + } + + tags.Add(OpenTelemetryConsts.GenAI.Provider.Name, _providerName); + + if (_serverAddress is string endpointAddress) + { + tags.Add(OpenTelemetryConsts.Server.Address, endpointAddress); + tags.Add(OpenTelemetryConsts.Server.Port, _serverPort); + } + } + } +} diff --git a/src/Libraries/Microsoft.Extensions.DocumentExtraction/OpenTelemetryDocumentExtractionClientBuilderExtensions.cs b/src/Libraries/Microsoft.Extensions.DocumentExtraction/OpenTelemetryDocumentExtractionClientBuilderExtensions.cs new file mode 100644 index 00000000000..c9d68582d2f --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DocumentExtraction/OpenTelemetryDocumentExtractionClientBuilderExtensions.cs @@ -0,0 +1,43 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.DocumentExtraction; + +/// Provides extensions for configuring instances. +[Experimental(DiagnosticIds.Experiments.DocumentExtraction, UrlFormat = DiagnosticIds.UrlFormat)] +public static class OpenTelemetryDocumentExtractionClientBuilderExtensions +{ + /// + /// Adds OpenTelemetry support to the OCR client pipeline, following the OpenTelemetry Semantic Conventions for Generative AI systems. + /// + /// + /// The draft specification this follows is available at . + /// The specification is still experimental and subject to change; as such, the telemetry output by this client is also subject to change. + /// + /// The . + /// An optional to use to create a logger for logging events. + /// An optional source name that will be used on the telemetry data. + /// An optional callback that can be used to configure the instance. + /// The . + public static DocumentExtractionClientBuilder UseOpenTelemetry( + this DocumentExtractionClientBuilder builder, + ILoggerFactory? loggerFactory = null, + string? sourceName = null, + Action? configure = null) => + Throw.IfNull(builder).Use((innerClient, services) => + { + loggerFactory ??= services.GetService(); + + var client = new OpenTelemetryDocumentExtractionClient(innerClient, loggerFactory?.CreateLogger(typeof(OpenTelemetryDocumentExtractionClient)), sourceName); + configure?.Invoke(client); + + return client; + }); +} diff --git a/src/Libraries/Microsoft.Extensions.DocumentExtraction/OpenTelemetryLog.cs b/src/Libraries/Microsoft.Extensions.DocumentExtraction/OpenTelemetryLog.cs new file mode 100644 index 00000000000..abc38cccf12 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DocumentExtraction/OpenTelemetryLog.cs @@ -0,0 +1,38 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Diagnostics; +using Microsoft.Extensions.Logging; + +namespace Microsoft.Extensions.DocumentExtraction; + +/// Shared log methods for OpenTelemetry instrumentation classes. +internal static partial class OpenTelemetryLog +{ + [LoggerMessage( + EventName = "gen_ai.client.operation.exception", + Level = LogLevel.Warning, + Message = "gen_ai.client.operation.exception")] + internal static partial void OperationException(ILogger logger, Exception error); + + /// Stamps the operation error tag/status on and logs the exception. + /// No-op when is . + internal static void RecordOperationError(Activity? activity, ILogger? logger, Exception? error) + { + if (error is null) + { + return; + } + + _ = activity? + .AddTag(OpenTelemetryConsts.Error.Type, error.GetType().FullName) + .SetStatus(ActivityStatusCode.Error, error.Message); + + if (logger is not null) + { + OperationException(logger, error); + } + } +} + diff --git a/src/Libraries/Microsoft.Extensions.DocumentExtraction/OtelMetricHelpers.cs b/src/Libraries/Microsoft.Extensions.DocumentExtraction/OtelMetricHelpers.cs new file mode 100644 index 00000000000..a70f5576630 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DocumentExtraction/OtelMetricHelpers.cs @@ -0,0 +1,26 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.Metrics; + +namespace Microsoft.Extensions.DocumentExtraction; + +/// Shared metric instrument factories for the OpenTelemetry* clients. +internal static class OtelMetricHelpers +{ + /// Creates the standard gen_ai.client.token.usage histogram on . + public static Histogram CreateGenAITokenUsageHistogram(Meter meter) => + meter.CreateHistogram( + OpenTelemetryConsts.GenAI.Client.TokenUsage.Name, + OpenTelemetryConsts.TokensUnit, + OpenTelemetryConsts.GenAI.Client.TokenUsage.Description, + advice: new() { HistogramBucketBoundaries = OpenTelemetryConsts.GenAI.Client.TokenUsage.ExplicitBucketBoundaries }); + + /// Creates the standard gen_ai.client.operation.duration histogram on . + public static Histogram CreateGenAIOperationDurationHistogram(Meter meter) => + meter.CreateHistogram( + OpenTelemetryConsts.GenAI.Client.OperationDuration.Name, + OpenTelemetryConsts.SecondsUnit, + OpenTelemetryConsts.GenAI.Client.OperationDuration.Description, + advice: new() { HistogramBucketBoundaries = OpenTelemetryConsts.GenAI.Client.OperationDuration.ExplicitBucketBoundaries }); +} diff --git a/src/Libraries/Microsoft.Extensions.DocumentExtraction/README.md b/src/Libraries/Microsoft.Extensions.DocumentExtraction/README.md new file mode 100644 index 00000000000..af528f218eb --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DocumentExtraction/README.md @@ -0,0 +1,35 @@ +# Microsoft.Extensions.DocumentExtraction + +.NET developers need to turn documents such as scanned images and PDFs into structured, AI-ready content, recovering text along with layout, tables, figures, and coordinates in a provider-neutral way. The `Microsoft.Extensions.DocumentExtraction` libraries provide a unified approach for representing document-extraction components, complementing `Microsoft.Extensions.DataIngestion` (extraction pulls content out of documents; ingestion feeds that content into a retrieval pipeline). + +## The packages + +The [Microsoft.Extensions.DocumentExtraction.Abstractions](https://www.nuget.org/packages/Microsoft.Extensions.DocumentExtraction.Abstractions) package provides the core exchange types, including [`IDocumentExtractionClient`](https://learn.microsoft.com/dotnet/api/microsoft.extensions.documentextraction.idocumentextractionclient), [`DocumentExtractionResult`](https://learn.microsoft.com/dotnet/api/microsoft.extensions.documentextraction.documentextractionresult), [`DocumentPage`](https://learn.microsoft.com/dotnet/api/microsoft.extensions.documentextraction.documentpage), and the reading-order [`DocumentElement`](https://learn.microsoft.com/dotnet/api/microsoft.extensions.documentextraction.documentelement) model (blocks, tables, and images with bounding regions and confidence). Any .NET library that provides a document-extraction engine can implement these abstractions to enable seamless integration with consuming code. + +The [Microsoft.Extensions.DocumentExtraction](https://www.nuget.org/packages/Microsoft.Extensions.DocumentExtraction) package has an implicit dependency on the `Microsoft.Extensions.DocumentExtraction.Abstractions` package. This package enables you to easily integrate components such as logging, telemetry, and options configuration into your applications using familiar dependency injection and builder patterns. For example, it provides [`DocumentExtractionClientBuilder`](https://learn.microsoft.com/dotnet/api/microsoft.extensions.documentextraction.documentextractionclientbuilder) and the `AddDocumentExtractionClient` service-collection extensions, along with logging and OpenTelemetry delegating clients that can be composed into a client pipeline. + +## Which package to reference + +Libraries that provide implementations of the abstractions typically reference only `Microsoft.Extensions.DocumentExtraction.Abstractions`. + +To also have access to higher-level utilities for working with document-extraction clients, reference the `Microsoft.Extensions.DocumentExtraction` package instead (which itself references `Microsoft.Extensions.DocumentExtraction.Abstractions`). Most consuming applications and services should reference the `Microsoft.Extensions.DocumentExtraction` package along with a library that provides a concrete implementation of the abstractions. + +## Install the package + +From the command-line: + +```console +dotnet add package Microsoft.Extensions.DocumentExtraction --prerelease +``` + +Or directly in the C# project file: + +```xml + + + +``` + +## Feedback & Contributing + +We welcome feedback and contributions in [our GitHub repo](https://github.com/dotnet/extensions). diff --git a/src/Libraries/Microsoft.Extensions.DocumentExtraction/TelemetryHelpers.cs b/src/Libraries/Microsoft.Extensions.DocumentExtraction/TelemetryHelpers.cs new file mode 100644 index 00000000000..3b94f4f394c --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DocumentExtraction/TelemetryHelpers.cs @@ -0,0 +1,39 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Text.Json; +using Microsoft.Extensions.AI; + +namespace Microsoft.Extensions.DocumentExtraction; + +/// Provides internal helpers for implementing telemetry. +internal static class TelemetryHelpers +{ + /// Gets a value indicating whether the OpenTelemetry clients should enable their EnableSensitiveData property's by default. + /// Defaults to false. May be overridden by setting the OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT environment variable to "true". + public static bool EnableSensitiveDataDefault { get; } = + Environment.GetEnvironmentVariable(OpenTelemetryConsts.GenAICaptureMessageContentEnvVar) is string envVar && + string.Equals(envVar, "true", StringComparison.OrdinalIgnoreCase); + + /// Serializes as JSON for logging purposes. + public static string AsJson(T value, JsonSerializerOptions? options) + { + if (options?.TryGetTypeInfo(typeof(T), out var typeInfo) is true || + AIJsonUtilities.DefaultOptions.TryGetTypeInfo(typeof(T), out typeInfo)) + { + try + { + return JsonSerializer.Serialize(value, typeInfo); + } + catch + { + // If we fail to serialize, just fall through to returning "{}". + } + } + + // If we're unable to get a type info for the value, or if we fail to serialize, + // return an empty JSON object. We do not want lack of type info to disrupt application behavior with exceptions. + return "{}"; + } +} diff --git a/src/Shared/DiagnosticIds/DiagnosticIds.cs b/src/Shared/DiagnosticIds/DiagnosticIds.cs index 4b4925850a4..9a4309a5da1 100644 --- a/src/Shared/DiagnosticIds/DiagnosticIds.cs +++ b/src/Shared/DiagnosticIds/DiagnosticIds.cs @@ -76,8 +76,13 @@ internal static class Experiments // constants to manage which experiment each API belongs to. internal const string VectorDataProviderServices = VectorDataExperiments; + // All Document Extraction experiments share a diagnostic ID but have different + // constants to manage which experiment each API belongs to. + internal const string DocumentExtraction = DocumentExtractionExperiments; + private const string AIExperiments = "MEAI001"; private const string VectorDataExperiments = "MEVD9001"; + private const string DocumentExtractionExperiments = "MEDE0001"; } internal static class LoggerMessage diff --git a/test/Libraries/Microsoft.Extensions.DataIngestion.Tests/Readers/OcrDocumentReaderTests.cs b/test/Libraries/Microsoft.Extensions.DataIngestion.Tests/Readers/OcrDocumentReaderTests.cs new file mode 100644 index 00000000000..2a435b4ea7c --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.DataIngestion.Tests/Readers/OcrDocumentReaderTests.cs @@ -0,0 +1,86 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#pragma warning disable MEDE0001 // Document extraction abstractions are experimental. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DocumentExtraction; +using Xunit; + +namespace Microsoft.Extensions.DataIngestion.Readers.Tests; + +public class OcrDocumentReaderTests +{ + [Fact] + public async Task MapsOcrImagesToIngestionDocumentImages() + { + byte[] imageBytes = [1, 2, 3, 4, 5]; + DocumentExtractionResult documentExtractionResult = new( + [ + new DocumentPage(2, "Page text") + { + Elements = + [ + new DocumentImage + { + Content = new DataContent(imageBytes, "image/png"), + Caption = "Architecture diagram", + BoundingRegion = DocumentBoundingRegion.FromRectangle(3, left: 1, top: 2, right: 10, bottom: 20) + } + ] + } + ]); + using TestDocumentExtractionClient documentExtractionClient = new(documentExtractionResult); + OcrDocumentReader reader = new(documentExtractionClient); + + using MemoryStream source = new([42]); + IngestionDocument document = await reader.ReadAsync(source, "doc-id", "application/pdf"); + + IngestionDocumentImage image = Assert.Single(document.EnumerateContent().OfType()); + Assert.Equal(imageBytes, image.Content?.ToArray()); + Assert.Equal("image/png", image.MediaType); + Assert.Equal("Architecture diagram", image.AlternativeText); + Assert.Equal(3, image.PageNumber); + Assert.Equal([1f, 2f, 10f, 20f], Assert.IsType(image.Metadata["bounding_box"])); + Assert.Equal([1f, 2f, 10f, 2f, 10f, 20f, 1f, 20f], Assert.IsType(image.Metadata["bounding_region"])); + Assert.Equal("application/pdf", documentExtractionClient.MediaType); + Assert.NotNull(documentExtractionClient.Options); + } + + private sealed class TestDocumentExtractionClient(DocumentExtractionResult result) : IDocumentExtractionClient + { + public string? MediaType { get; private set; } + + public DocumentExtractionOptions? Options { get; private set; } + + public Task ExtractAsync( + Stream document, + string mediaType, + DocumentExtractionOptions? options = null, + CancellationToken cancellationToken = default) + { + MediaType = mediaType; + Options = options; + return Task.FromResult(result); + } + + public IAsyncEnumerable ExtractPagesAsync( + Stream document, + string mediaType, + DocumentExtractionOptions? options = null, + CancellationToken cancellationToken = default) + => throw new NotSupportedException(); + + public object? GetService(Type serviceType, object? serviceKey = null) => null; + + public void Dispose() + { + } + } +} diff --git a/test/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions.Tests/DelegatingDocumentExtractionClientTests.cs b/test/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions.Tests/DelegatingDocumentExtractionClientTests.cs new file mode 100644 index 00000000000..dab3b38c16e --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions.Tests/DelegatingDocumentExtractionClientTests.cs @@ -0,0 +1,167 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Xunit; + +namespace Microsoft.Extensions.DocumentExtraction; + +public class DelegatingDocumentExtractionClientTests +{ + [Fact] + public void RequiresInnerDocumentExtractionClient() + { + Assert.Throws("innerClient", () => new NoOpDelegatingDocumentExtractionClient(null!)); + } + + [Fact] + public async Task ExtractAsyncDefaultsToInnerClientAsync() + { + // Arrange + using var expectedDocument = new MemoryStream(); + var expectedMediaType = "application/pdf"; + var expectedOptions = new DocumentExtractionOptions(); + var expectedCancellationToken = CancellationToken.None; + var expectedResult = new TaskCompletionSource(); + var expectedResponse = new DocumentExtractionResult([]); + using var inner = new TestDocumentExtractionClient + { + ExtractAsyncCallback = (document, mediaType, options, cancellationToken) => + { + Assert.Same(expectedDocument, document); + Assert.Same(expectedMediaType, mediaType); + Assert.Same(expectedOptions, options); + Assert.Equal(expectedCancellationToken, cancellationToken); + return expectedResult.Task; + } + }; + + using var delegating = new NoOpDelegatingDocumentExtractionClient(inner); + + // Act + var resultTask = delegating.ExtractAsync(expectedDocument, expectedMediaType, expectedOptions, expectedCancellationToken); + + // Assert + Assert.False(resultTask.IsCompleted); + expectedResult.SetResult(expectedResponse); + Assert.True(resultTask.IsCompleted); + Assert.Same(expectedResponse, await resultTask); + } + + [Fact] + public async Task ExtractPagesAsyncDefaultsToInnerClientAsync() + { + // Arrange + using var expectedDocument = new MemoryStream(); + var expectedMediaType = "application/pdf"; + var expectedOptions = new DocumentExtractionOptions(); + using var cts = new CancellationTokenSource(); + DocumentExtractionPageResult[] expectedUpdates = + [ + new(new DocumentPage(1, "page one")), + new(new DocumentPage(2, "page two")), + ]; + + using var inner = new TestDocumentExtractionClient + { + ExtractPagesAsyncCallback = (document, mediaType, options, cancellationToken) => + { + Assert.Same(expectedDocument, document); + Assert.Same(expectedMediaType, mediaType); + Assert.Same(expectedOptions, options); + Assert.Equal(cts.Token, cancellationToken); + return YieldAsync(expectedUpdates); + } + }; + + using var delegating = new NoOpDelegatingDocumentExtractionClient(inner); + + // Act + List received = []; + await foreach (var update in delegating.ExtractPagesAsync(expectedDocument, expectedMediaType, expectedOptions, cts.Token)) + { + received.Add(update); + } + + // Assert + Assert.Equal(expectedUpdates, received); + } + + private static async IAsyncEnumerable YieldAsync(IEnumerable updates) + { + foreach (var update in updates) + { + await Task.Yield(); + yield return update; + } + } + + [Fact] + public void GetServiceThrowsForNullType() + { + using var inner = new TestDocumentExtractionClient(); + using var delegating = new NoOpDelegatingDocumentExtractionClient(inner); + Assert.Throws("serviceType", () => delegating.GetService(null!)); + } + + [Fact] + public void GetServiceReturnsSelfIfCompatibleWithRequestAndKeyIsNull() + { + // Arrange + using var inner = new TestDocumentExtractionClient(); + using var delegating = new NoOpDelegatingDocumentExtractionClient(inner); + + // Act + var client = delegating.GetService(); + + // Assert + Assert.Same(delegating, client); + } + + [Fact] + public void GetServiceDelegatesToInnerIfKeyIsNotNull() + { + // Arrange + var expectedKey = new object(); + using var expectedResult = new TestDocumentExtractionClient(); + using var inner = new TestDocumentExtractionClient + { + GetServiceCallback = (_, _) => expectedResult + }; + using var delegating = new NoOpDelegatingDocumentExtractionClient(inner); + + // Act + var client = delegating.GetService(expectedKey); + + // Assert + Assert.Same(expectedResult, client); + } + + [Fact] + public void GetServiceDelegatesToInnerIfNotCompatibleWithRequest() + { + // Arrange + var expectedResult = TimeZoneInfo.Local; + var expectedKey = new object(); + using var inner = new TestDocumentExtractionClient + { + GetServiceCallback = (type, key) => type == expectedResult.GetType() && key == expectedKey + ? expectedResult + : throw new InvalidOperationException("Unexpected call") + }; + using var delegating = new NoOpDelegatingDocumentExtractionClient(inner); + + // Act + var tzi = delegating.GetService(expectedKey); + + // Assert + Assert.Same(expectedResult, tzi); + } + + private sealed class NoOpDelegatingDocumentExtractionClient(IDocumentExtractionClient innerClient) + : DelegatingDocumentExtractionClient(innerClient); +} diff --git a/test/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions.Tests/DocumentBoundingRegionTests.cs b/test/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions.Tests/DocumentBoundingRegionTests.cs new file mode 100644 index 00000000000..0af86c6095d --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions.Tests/DocumentBoundingRegionTests.cs @@ -0,0 +1,49 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using Xunit; + +namespace Microsoft.Extensions.DocumentExtraction; + +public class DocumentBoundingRegionTests +{ + [Fact] + public void Constructor_NullPolygon_Throws() + { + Assert.Throws("polygon", () => new DocumentBoundingRegion(1, null!)); + } + + [Fact] + public void FromRectangle_ProducesClockwiseQuadrilateral() + { + var region = DocumentBoundingRegion.FromRectangle(2, left: 10, top: 20, right: 110, bottom: 220); + + Assert.Equal(2, region.PageNumber); + Assert.Equal(new[] { new DocumentPoint(10, 20), new DocumentPoint(110, 20), new DocumentPoint(110, 220), new DocumentPoint(10, 220) }, region.Polygon); + } + + [Fact] + public void GetBounds_ReturnsAxisAlignedExtents() + { + var region = new DocumentBoundingRegion(1, [new DocumentPoint(30, 40), new DocumentPoint(100, 35), new DocumentPoint(110, 90), new DocumentPoint(25, 95)]); + + var bounds = region.GetBounds(); + + Assert.NotNull(bounds); + var (left, top, right, bottom) = bounds.Value; + + Assert.Equal(25, left); + Assert.Equal(35, top); + Assert.Equal(110, right); + Assert.Equal(95, bottom); + } + + [Fact] + public void GetBounds_EmptyPolygon_ReturnsNull() + { + var region = new DocumentBoundingRegion(1, []); + + Assert.Null(region.GetBounds()); + } +} diff --git a/test/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions.Tests/DocumentElementTests.cs b/test/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions.Tests/DocumentElementTests.cs new file mode 100644 index 00000000000..f9886715b23 --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions.Tests/DocumentElementTests.cs @@ -0,0 +1,110 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Linq; +using System.Text.Json; +using Microsoft.Extensions.AI; +using Xunit; + +namespace Microsoft.Extensions.DocumentExtraction; + +public class DocumentElementTests +{ + [Fact] + public void Elements_OfType_ProjectsEachKindInReadingOrder() + { + DocumentPage page = new(1, "page text") + { + Elements = + [ + new DocumentBlock("intro"), + new DocumentTable(1, 1), + new DocumentImage { Caption = "figure" }, + new DocumentBlock("outro"), + ], + }; + + Assert.Equal(4, page.Elements.Count); + Assert.Equal(["intro", "outro"], page.Elements.OfType().Select(b => b.Text)); + Assert.Single(page.Elements.OfType()); + Assert.Equal("figure", Assert.Single(page.Elements.OfType()).Caption); + } + + [Fact] + public void Elements_SerializePolymorphically_RoundTrip() + { + DocumentExtractionResult result = new( + [ + new DocumentPage(1, "page text") + { + CoordinateUnit = DocumentCoordinateUnit.Point, + CoordinateOrigin = DocumentCoordinateOrigin.BottomLeft, + Elements = + [ + new DocumentBlock("title") { Kind = DocumentBlockKind.Title, Confidence = 0.9 }, + new DocumentTable(1, 2, [new DocumentTableCell(0, 0, "a") { Kind = DocumentTableCellKind.RowHeader }, new DocumentTableCell(0, 1, "b")]), + new DocumentImage { Caption = "figure", Confidence = 0.5 }, + ], + }, + ]); + + string json = JsonSerializer.Serialize(result, AIJsonUtilities.DefaultOptions); + + Assert.Contains("$type", json); + Assert.Contains("block", json); + Assert.Contains("table", json); + Assert.Contains("image", json); + Assert.Contains("Point", json); + Assert.Contains("BottomLeft", json); + + DocumentExtractionResult roundTripped = JsonSerializer.Deserialize(json, AIJsonUtilities.DefaultOptions)!; + + DocumentPage page = Assert.Single(roundTripped.Pages); + Assert.Equal(DocumentCoordinateUnit.Point, page.CoordinateUnit); + Assert.Equal(DocumentCoordinateOrigin.BottomLeft, page.CoordinateOrigin); + Assert.Collection( + page.Elements, + e => Assert.Equal("title", Assert.IsType(e).Text), + e => Assert.Equal(2, Assert.IsType(e).ColumnCount), + e => Assert.Equal("figure", Assert.IsType(e).Caption)); + Assert.Equal(0.9, page.Elements.OfType().Single().Confidence); + } + + [Fact] + public void TableCell_NestedElements_RoundTrip() + { + DocumentTableCell cell = new(0, 0, "flat text") + { + Elements = [new DocumentBlock("nested paragraph")], + }; + + string json = JsonSerializer.Serialize(cell, AIJsonUtilities.DefaultOptions); + DocumentTableCell roundTripped = JsonSerializer.Deserialize(json, AIJsonUtilities.DefaultOptions)!; + + Assert.Equal("flat text", roundTripped.Content); + Assert.NotNull(roundTripped.Elements); + Assert.Equal("nested paragraph", Assert.IsType(Assert.Single(roundTripped.Elements!)).Text); + } + + [Fact] + public void TableCell_GeometryConfidenceAndProperties_RoundTrip() + { + DocumentTableCell cell = new(0, 0, "flat text") + { + BoundingRegion = DocumentBoundingRegion.FromRectangle(1, left: 10, top: 20, right: 110, bottom: 220), + Confidence = 0.75, + RawRepresentation = new { ignored = true }, + AdditionalProperties = new() { ["detectedLanguages"] = "en" }, + }; + + string json = JsonSerializer.Serialize(cell, AIJsonUtilities.DefaultOptions); + DocumentTableCell roundTripped = JsonSerializer.Deserialize(json, AIJsonUtilities.DefaultOptions)!; + + Assert.NotNull(roundTripped.BoundingRegion); + Assert.Equal(1, roundTripped.BoundingRegion!.PageNumber); + Assert.Equal(0.75, roundTripped.Confidence); + Assert.NotNull(roundTripped.AdditionalProperties); + Assert.True(roundTripped.AdditionalProperties!.ContainsKey("detectedLanguages")); + Assert.Null(roundTripped.RawRepresentation); + } +} diff --git a/test/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions.Tests/DocumentExtractionClientExtensionsTests.cs b/test/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions.Tests/DocumentExtractionClientExtensionsTests.cs new file mode 100644 index 00000000000..9b85326cce0 --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions.Tests/DocumentExtractionClientExtensionsTests.cs @@ -0,0 +1,70 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; +using Xunit; + +namespace Microsoft.Extensions.DocumentExtraction; + +public class DocumentExtractionClientExtensionsTests +{ + [Fact] + public void GetService_InvalidArgs_Throws() + { + Assert.Throws("client", () => + { + _ = DocumentExtractionClientExtensions.GetService(null!); + }); + } + + [Fact] + public async Task ExtractAsync_InvalidArgs_Throws() + { + IDocumentExtractionClient? client = null; + var content = new DataContent("data:application/pdf;base64,AQIDBA=="); + var ex1 = await Assert.ThrowsAsync(() => DocumentExtractionClientExtensions.ExtractAsync(client!, content)); + Assert.Equal("client", ex1.ParamName); + + using var testClient = new TestDocumentExtractionClient(); + DataContent? nullContent = null; + var ex2 = await Assert.ThrowsAsync(() => DocumentExtractionClientExtensions.ExtractAsync(testClient, nullContent!)); + Assert.Equal("document", ex2.ParamName); + } + + [Fact] + public async Task ExtractAsync_DataContent_PassesStreamAndMediaTypeAsync() + { + // Arrange + var expectedResponse = new DocumentExtractionResult([new DocumentPage(1, "hello")]); + string? observedMediaType = null; + byte[]? observedBytes = null; + + using var client = new TestDocumentExtractionClient + { + ExtractAsyncCallback = async (document, mediaType, options, cancellationToken) => + { + observedMediaType = mediaType; + using var ms = new MemoryStream(); + await document.CopyToAsync( + ms, +#if !NET + 80 * 1024, // same as the default buffer size +#endif + cancellationToken); + observedBytes = ms.ToArray(); + return expectedResponse; + } + }; + + // Act + var result = await DocumentExtractionClientExtensions.ExtractAsync(client, new DataContent("data:application/pdf;base64,AQIDBA==")); + + // Assert + Assert.Same(expectedResponse, result); + Assert.Equal("application/pdf", observedMediaType); + Assert.Equal(new byte[] { 1, 2, 3, 4 }, observedBytes); + } +} diff --git a/test/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions.Tests/DocumentExtractionClientMetadataTests.cs b/test/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions.Tests/DocumentExtractionClientMetadataTests.cs new file mode 100644 index 00000000000..c9ae7e77a1e --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions.Tests/DocumentExtractionClientMetadataTests.cs @@ -0,0 +1,29 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using Xunit; + +namespace Microsoft.Extensions.DocumentExtraction; + +public class DocumentExtractionClientMetadataTests +{ + [Fact] + public void Constructor_NullValues_AllowedAndRoundtrip() + { + DocumentExtractionClientMetadata metadata = new(null, null, null); + Assert.Null(metadata.ProviderName); + Assert.Null(metadata.ProviderUri); + Assert.Null(metadata.DefaultModelId); + } + + [Fact] + public void Constructor_Value_Roundtrips() + { + var uri = new Uri("https://example.com"); + DocumentExtractionClientMetadata metadata = new("providerName", uri, "theModel"); + Assert.Equal("providerName", metadata.ProviderName); + Assert.Same(uri, metadata.ProviderUri); + Assert.Equal("theModel", metadata.DefaultModelId); + } +} diff --git a/test/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions.Tests/DocumentExtractionOptionsTests.cs b/test/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions.Tests/DocumentExtractionOptionsTests.cs new file mode 100644 index 00000000000..b6403017132 --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions.Tests/DocumentExtractionOptionsTests.cs @@ -0,0 +1,40 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Xunit; + +namespace Microsoft.Extensions.DocumentExtraction; + +public class DocumentExtractionOptionsTests +{ + [Fact] + public void Clone_CopiesAllProperties() + { + var options = new DocumentExtractionOptions + { + ModelId = "mistral-ocr-4-0", + AdditionalProperties = new() { ["custom"] = "value" }, + }; + + var clone = options.Clone(); + + Assert.NotSame(options, clone); + Assert.Equal("mistral-ocr-4-0", clone.ModelId); + Assert.NotNull(clone.AdditionalProperties); + Assert.Equal("value", clone.AdditionalProperties!["custom"]); + } + + [Fact] + public void Clone_DeepCopiesAdditionalProperties() + { + var options = new DocumentExtractionOptions + { + AdditionalProperties = new() { ["key"] = "original" }, + }; + + var clone = options.Clone(); + clone.AdditionalProperties!["key"] = "changed"; + + Assert.Equal("original", options.AdditionalProperties!["key"]); + } +} diff --git a/test/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions.Tests/DocumentExtractionPageResultExtensionsTests.cs b/test/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions.Tests/DocumentExtractionPageResultExtensionsTests.cs new file mode 100644 index 00000000000..0a163f0f30a --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions.Tests/DocumentExtractionPageResultExtensionsTests.cs @@ -0,0 +1,132 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Xunit; + +namespace Microsoft.Extensions.DocumentExtraction; + +public class DocumentExtractionPageResultExtensionsTests +{ + [Fact] + public void ToDocumentExtractionResult_NullUpdates_Throws() + { + Assert.Throws("updates", () => ((IEnumerable)null!).ToDocumentExtractionResult()); + } + + [Fact] + public async Task ToDocumentExtractionResultAsync_NullUpdates_ThrowsAsync() + { + await Assert.ThrowsAsync("updates", () => ((IAsyncEnumerable)null!).ToDocumentExtractionResultAsync()); + } + + [Fact] + public void ToDocumentExtractionResult_AssemblesPagesAndUsage() + { + DocumentExtractionPageResult[] updates = + [ + new(new DocumentPage(1, "page one")) { PagesProcessed = 1, TotalPages = 2 }, + new(new DocumentPage(2, "page two")) { Usage = new() { PagesProcessed = 2 } }, + ]; + + DocumentExtractionResult result = updates.ToDocumentExtractionResult(); + + Assert.Equal(2, result.Pages.Count); + Assert.Equal("page one\n\npage two", result.Text); + Assert.NotNull(result.Usage); + Assert.Equal(2, result.Usage!.PagesProcessed); + } + + [Fact] + public async Task ToDocumentExtractionResultAsync_AssemblesPagesAndUsageAsync() + { + DocumentExtractionPageResult[] updates = + [ + new(new DocumentPage(1, "page one")), + new(new DocumentPage(2, "page two")), + ]; + + DocumentExtractionResult result = await YieldAsync(updates).ToDocumentExtractionResultAsync(); + + Assert.Equal(2, result.Pages.Count); + Assert.Equal("page one\n\npage two", result.Text); + } + + [Fact] + public void ToDocumentExtractionResult_MergesAdditionalProperties() + { + DocumentExtractionPageResult[] updates = + [ + new(new DocumentPage(1, "page one")) { AdditionalProperties = new() { ["a"] = "1" } }, + new(new DocumentPage(2, "page two")) { AdditionalProperties = new() { ["b"] = "2" } }, + ]; + + DocumentExtractionResult result = updates.ToDocumentExtractionResult(); + + Assert.NotNull(result.AdditionalProperties); + Assert.Equal("1", result.AdditionalProperties!["a"]); + Assert.Equal("2", result.AdditionalProperties!["b"]); + } + + [Fact] + public void ToDocumentExtractionResult_PreservesPerPageCoordinateMetadata() + { + DocumentExtractionPageResult[] updates = + [ + new(new DocumentPage(1, "page one") { CoordinateUnit = DocumentCoordinateUnit.Pixel, CoordinateOrigin = DocumentCoordinateOrigin.TopLeft }), + new(new DocumentPage(2, "page two")), + new(new DocumentPage(3, "page three") { CoordinateUnit = DocumentCoordinateUnit.Point, CoordinateOrigin = DocumentCoordinateOrigin.BottomLeft }), + ]; + + DocumentExtractionResult result = updates.ToDocumentExtractionResult(); + + Assert.Collection( + result.Pages, + p => + { + Assert.Equal(DocumentCoordinateUnit.Pixel, p.CoordinateUnit); + Assert.Equal(DocumentCoordinateOrigin.TopLeft, p.CoordinateOrigin); + }, + p => + { + Assert.Null(p.CoordinateUnit); + Assert.Null(p.CoordinateOrigin); + }, + p => + { + Assert.Equal(DocumentCoordinateUnit.Point, p.CoordinateUnit); + Assert.Equal(DocumentCoordinateOrigin.BottomLeft, p.CoordinateOrigin); + }); + } + + [Fact] + public void ToDocumentExtractionResult_PreservesPerPageRawRepresentation() + { + object rawPageOne = new { page = 1 }; + object rawPageTwo = new { page = 2 }; + + DocumentExtractionPageResult[] updates = + [ + new(new DocumentPage(1, "page one") { RawRepresentation = rawPageOne }), + new(new DocumentPage(2, "page two") { RawRepresentation = rawPageTwo }), + ]; + + DocumentExtractionResult result = updates.ToDocumentExtractionResult(); + + Assert.Collection( + result.Pages, + p => Assert.Same(rawPageOne, p.RawRepresentation), + p => Assert.Same(rawPageTwo, p.RawRepresentation)); + } + + private static async IAsyncEnumerable YieldAsync(IEnumerable updates) + { + foreach (var update in updates) + { + await Task.Yield(); + yield return update; + } + } +} diff --git a/test/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions.Tests/DocumentExtractionResultTests.cs b/test/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions.Tests/DocumentExtractionResultTests.cs new file mode 100644 index 00000000000..da2987a26f6 --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions.Tests/DocumentExtractionResultTests.cs @@ -0,0 +1,25 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using Xunit; + +namespace Microsoft.Extensions.DocumentExtraction; + +public class DocumentExtractionResultTests +{ + [Fact] + public void Constructor_NullPages_Throws() + { + Assert.Throws("pages", () => new DocumentExtractionResult(null!)); + } + + [Fact] + public void Text_JoinsPerPageText() + { + var result = new DocumentExtractionResult([new DocumentPage(1, "page one"), new DocumentPage(2, "page two")]); + + Assert.Equal("page one\n\npage two", result.Text); + Assert.Equal(2, result.Pages.Count); + } +} diff --git a/test/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions.Tests/Microsoft.Extensions.DocumentExtraction.Abstractions.Tests.csproj b/test/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions.Tests/Microsoft.Extensions.DocumentExtraction.Abstractions.Tests.csproj new file mode 100644 index 00000000000..65ad80091e8 --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions.Tests/Microsoft.Extensions.DocumentExtraction.Abstractions.Tests.csproj @@ -0,0 +1,28 @@ + + + Microsoft.Extensions.DocumentExtraction + Unit tests for Microsoft.Extensions.DocumentExtraction.Abstractions. + + + + $(NoWarn);MEDE0001;CA1063;CA1861;CA2201;VSTHRD003;S104 + true + + + + true + true + true + true + true + true + + + + + + + + + + diff --git a/test/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions.Tests/TestDocumentExtractionClient.cs b/test/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions.Tests/TestDocumentExtractionClient.cs new file mode 100644 index 00000000000..00d32a9a33a --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.DocumentExtraction.Abstractions.Tests/TestDocumentExtractionClient.cs @@ -0,0 +1,63 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.Extensions.DocumentExtraction; + +public sealed class TestDocumentExtractionClient : IDocumentExtractionClient +{ + public TestDocumentExtractionClient() + { + GetServiceCallback = DefaultGetServiceCallback; + } + + public IServiceProvider? Services { get; set; } + + public Func>? + ExtractAsyncCallback + { get; set; } + + public Func>? + ExtractPagesAsyncCallback + { get; set; } + + public Func GetServiceCallback { get; set; } + + private object? DefaultGetServiceCallback(Type serviceType, object? serviceKey) + => serviceType is not null && serviceKey is null && serviceType.IsInstanceOfType(this) ? this : null; + + public Task ExtractAsync( + Stream document, + string mediaType, + DocumentExtractionOptions? options = null, + CancellationToken cancellationToken = default) + => ExtractAsyncCallback!.Invoke(document, mediaType, options, cancellationToken); + + public IAsyncEnumerable ExtractPagesAsync( + Stream document, + string mediaType, + DocumentExtractionOptions? options = null, + CancellationToken cancellationToken = default) + => ExtractPagesAsyncCallback!.Invoke(document, mediaType, options, cancellationToken); + + public object? GetService(Type serviceType, object? serviceKey = null) + => GetServiceCallback!.Invoke(serviceType, serviceKey); + + public void Dispose() + { + // Dispose of resources if any. + } +} diff --git a/test/Libraries/Microsoft.Extensions.DocumentExtraction.Tests/ConfigureOptionsDocumentExtractionClientTests.cs b/test/Libraries/Microsoft.Extensions.DocumentExtraction.Tests/ConfigureOptionsDocumentExtractionClientTests.cs new file mode 100644 index 00000000000..dc2d4a67337 --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.DocumentExtraction.Tests/ConfigureOptionsDocumentExtractionClientTests.cs @@ -0,0 +1,71 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Xunit; + +namespace Microsoft.Extensions.DocumentExtraction; + +public class ConfigureOptionsDocumentExtractionClientTests +{ + [Fact] + public void ConfigureOptionsDocumentExtractionClient_InvalidArgs_Throws() + { + Assert.Throws("innerClient", () => new ConfigureOptionsDocumentExtractionClient(null!, _ => { })); + Assert.Throws("configure", () => new ConfigureOptionsDocumentExtractionClient(new TestDocumentExtractionClient(), null!)); + } + + [Fact] + public void ConfigureOptions_InvalidArgs_Throws() + { + using var innerClient = new TestDocumentExtractionClient(); + var builder = innerClient.AsBuilder(); + Assert.Throws("configure", () => builder.ConfigureOptions(null!)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ConfigureOptions_ReturnedInstancePassedToNextClient(bool nullProvidedOptions) + { + DocumentExtractionOptions? providedOptions = nullProvidedOptions ? null : new() { ModelId = "test" }; + DocumentExtractionOptions? returnedOptions = null; + DocumentExtractionResult expectedResult = new([new DocumentPage(1, "blue whale")]); + using CancellationTokenSource cts = new(); + + using IDocumentExtractionClient innerClient = new TestDocumentExtractionClient + { + ExtractAsyncCallback = (document, mediaType, options, cancellationToken) => + { + Assert.Same(returnedOptions, options); + Assert.Equal(cts.Token, cancellationToken); + return Task.FromResult(expectedResult); + }, + }; + + using var client = innerClient + .AsBuilder() + .ConfigureOptions(options => + { + Assert.NotSame(providedOptions, options); + if (nullProvidedOptions) + { + Assert.Null(options.ModelId); + } + else + { + Assert.Equal(providedOptions!.ModelId, options.ModelId); + } + + returnedOptions = options; + }) + .Build(); + + using var document = new MemoryStream(new byte[] { 1, 2, 3, 4 }); + var result = await client.ExtractAsync(document, "application/pdf", providedOptions, cts.Token); + Assert.Same(expectedResult, result); + } +} diff --git a/test/Libraries/Microsoft.Extensions.DocumentExtraction.Tests/DocumentExtractionClientBuilderTests.cs b/test/Libraries/Microsoft.Extensions.DocumentExtraction.Tests/DocumentExtractionClientBuilderTests.cs new file mode 100644 index 00000000000..31eae36e902 --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.DocumentExtraction.Tests/DocumentExtractionClientBuilderTests.cs @@ -0,0 +1,103 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using Microsoft.Extensions.DependencyInjection; +using Xunit; + +namespace Microsoft.Extensions.DocumentExtraction; + +public class DocumentExtractionClientBuilderTests +{ + [Fact] + public void PassingNullInnerClientThrows() + { + Assert.Throws("innerClient", () => new DocumentExtractionClientBuilder((IDocumentExtractionClient)null!)); + Assert.Throws("innerClientFactory", () => new DocumentExtractionClientBuilder((Func)null!)); + } + + [Fact] + public void BuildReturnsInnerClientWhenNoMiddleware() + { + using var inner = new TestDocumentExtractionClient(); + var builder = inner.AsBuilder(); + + var built = builder.Build(); + + Assert.Same(inner, built); + } + + [Fact] + public void UseAppliesFactoriesInReverseOrderSoFirstAddedIsOutermost() + { + // Arrange + using var inner = new TestDocumentExtractionClient(); + var order = new List(); + + var built = inner.AsBuilder() + .Use(c => + { + order.Add("outer-built"); + return new InspectorDocumentExtractionClient(c, "outer"); + }) + .Use(c => + { + order.Add("inner-built"); + return new InspectorDocumentExtractionClient(c, "inner"); + }) + .Build(); + + // The first factory added should be the outermost wrapper. + var outer = Assert.IsType(built); + Assert.Equal("outer", outer.Name); + var innerWrapper = Assert.IsType(outer.InnerClientPublic); + Assert.Equal("inner", innerWrapper.Name); + Assert.Same(inner, innerWrapper.InnerClientPublic); + + // Reverse-order application: inner factory runs before outer factory. + Assert.Equal(["inner-built", "outer-built"], order); + } + + [Fact] + public void BuildThrowsWhenFactoryReturnsNull() + { + using var inner = new TestDocumentExtractionClient(); + var builder = inner.AsBuilder().Use(_ => null!); + + Assert.Throws(() => builder.Build()); + } + + [Fact] + public void UseNullFactoryThrows() + { + using var inner = new TestDocumentExtractionClient(); + var builder = inner.AsBuilder(); + Assert.Throws("clientFactory", () => builder.Use((Func)null!)); + Assert.Throws("clientFactory", () => builder.Use((Func)null!)); + } + + [Fact] + public void ServicesAreFlowedThroughBuild() + { + using var inner = new TestDocumentExtractionClient(); + IServiceProvider? observed = null; + + var services = new ServiceCollection().BuildServiceProvider(); + _ = inner.AsBuilder() + .Use((c, sp) => + { + observed = sp; + return c; + }) + .Build(services); + + Assert.Same(services, observed); + } + + private sealed class InspectorDocumentExtractionClient(IDocumentExtractionClient inner, string name) : DelegatingDocumentExtractionClient(inner) + { + public string Name => name; + public IDocumentExtractionClient InnerClientPublic => InnerClient; + } +} diff --git a/test/Libraries/Microsoft.Extensions.DocumentExtraction.Tests/DocumentExtractionClientDependencyInjectionPatterns.cs b/test/Libraries/Microsoft.Extensions.DocumentExtraction.Tests/DocumentExtractionClientDependencyInjectionPatterns.cs new file mode 100644 index 00000000000..3d1dc80dfd6 --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.DocumentExtraction.Tests/DocumentExtractionClientDependencyInjectionPatterns.cs @@ -0,0 +1,107 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using Microsoft.Extensions.DependencyInjection; +using Xunit; + +namespace Microsoft.Extensions.DocumentExtraction; + +public class DocumentExtractionClientDependencyInjectionPatterns +{ + private IServiceCollection ServiceCollection { get; } = new ServiceCollection(); + + [Fact] + public void CanRegisterSingletonUsingFactory() + { + ServiceCollection.AddDocumentExtractionClient(services => new TestDocumentExtractionClient { Services = services }) + .Use((inner, services) => new SingletonMiddleware(inner, services)); + + var services = ServiceCollection.BuildServiceProvider(); + using var scope1 = services.CreateScope(); + using var scope2 = services.CreateScope(); + + var instance1 = scope1.ServiceProvider.GetRequiredService(); + var instance1Copy = scope1.ServiceProvider.GetRequiredService(); + var instance2 = scope2.ServiceProvider.GetRequiredService(); + + var instance = Assert.IsType(instance1); + Assert.Same(instance, instance1Copy); + Assert.Same(instance, instance2); + Assert.IsType(instance.InnerClientPublic); + } + + [Fact] + public void CanRegisterKeyedSingletonUsingSharedInstance() + { + using var singleton = new TestDocumentExtractionClient(); + ServiceCollection.AddKeyedDocumentExtractionClient("mykey", singleton) + .Use((inner, services) => new SingletonMiddleware(inner, services)); + + var services = ServiceCollection.BuildServiceProvider(); + using var scope1 = services.CreateScope(); + using var scope2 = services.CreateScope(); + + Assert.Null(services.GetService()); + + var instance1 = scope1.ServiceProvider.GetRequiredKeyedService("mykey"); + var instance1Copy = scope1.ServiceProvider.GetRequiredKeyedService("mykey"); + var instance2 = scope2.ServiceProvider.GetRequiredKeyedService("mykey"); + + var instance = Assert.IsType(instance1); + Assert.Same(instance, instance1Copy); + Assert.Same(instance, instance2); + Assert.IsType(instance.InnerClientPublic); + } + + [Theory] + [InlineData(null)] + [InlineData(ServiceLifetime.Singleton)] + [InlineData(ServiceLifetime.Scoped)] + [InlineData(ServiceLifetime.Transient)] + public void AddDocumentExtractionClient_RegistersExpectedLifetime(ServiceLifetime? lifetime) + { + ServiceCollection sc = new(); + ServiceLifetime expectedLifetime = lifetime ?? ServiceLifetime.Singleton; + _ = lifetime.HasValue + ? sc.AddDocumentExtractionClient(services => new TestDocumentExtractionClient(), lifetime.Value) + : sc.AddDocumentExtractionClient(services => new TestDocumentExtractionClient()); + + ServiceDescriptor sd = Assert.Single(sc); + Assert.Equal(typeof(IDocumentExtractionClient), sd.ServiceType); + Assert.False(sd.IsKeyedService); + Assert.Null(sd.ImplementationInstance); + Assert.NotNull(sd.ImplementationFactory); + Assert.IsType(sd.ImplementationFactory!(null!)); + Assert.Equal(expectedLifetime, sd.Lifetime); + } + + [Theory] + [InlineData(null)] + [InlineData(ServiceLifetime.Singleton)] + [InlineData(ServiceLifetime.Scoped)] + [InlineData(ServiceLifetime.Transient)] + public void AddKeyedDocumentExtractionClient_RegistersExpectedLifetime(ServiceLifetime? lifetime) + { + ServiceCollection sc = new(); + ServiceLifetime expectedLifetime = lifetime ?? ServiceLifetime.Singleton; + _ = lifetime.HasValue + ? sc.AddKeyedDocumentExtractionClient("key", services => new TestDocumentExtractionClient(), lifetime.Value) + : sc.AddKeyedDocumentExtractionClient("key", services => new TestDocumentExtractionClient()); + + ServiceDescriptor sd = Assert.Single(sc); + Assert.Equal(typeof(IDocumentExtractionClient), sd.ServiceType); + Assert.True(sd.IsKeyedService); + Assert.Equal("key", sd.ServiceKey); + Assert.Null(sd.KeyedImplementationInstance); + Assert.NotNull(sd.KeyedImplementationFactory); + Assert.IsType(sd.KeyedImplementationFactory!(null!, null!)); + Assert.Equal(expectedLifetime, sd.Lifetime); + } + + public class SingletonMiddleware(IDocumentExtractionClient inner, IServiceProvider services) : DelegatingDocumentExtractionClient(inner) + { + public IDocumentExtractionClient InnerClientPublic => InnerClient; + public IServiceProvider Services => services; + } +} diff --git a/test/Libraries/Microsoft.Extensions.DocumentExtraction.Tests/LoggingDocumentExtractionClientTests.cs b/test/Libraries/Microsoft.Extensions.DocumentExtraction.Tests/LoggingDocumentExtractionClientTests.cs new file mode 100644 index 00000000000..6bbe7789ff5 --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.DocumentExtraction.Tests/LoggingDocumentExtractionClientTests.cs @@ -0,0 +1,87 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Logging.Testing; +using Xunit; + +namespace Microsoft.Extensions.DocumentExtraction; + +public class LoggingDocumentExtractionClientTests +{ + [Fact] + public void LoggingDocumentExtractionClient_InvalidArgs_Throws() + { + Assert.Throws("innerClient", () => new LoggingDocumentExtractionClient(null!, NullLogger.Instance)); + Assert.Throws("logger", () => new LoggingDocumentExtractionClient(new TestDocumentExtractionClient(), null!)); + } + + [Fact] + public void UseLogging_AvoidsInjectingNopClient() + { + using var innerClient = new TestDocumentExtractionClient(); + + Assert.Null(innerClient.AsBuilder().UseLogging(NullLoggerFactory.Instance).Build().GetService(typeof(LoggingDocumentExtractionClient))); + Assert.Same(innerClient, innerClient.AsBuilder().UseLogging(NullLoggerFactory.Instance).Build().GetService(typeof(IDocumentExtractionClient))); + + using var factory = LoggerFactory.Create(b => b.AddFakeLogging()); + Assert.NotNull(innerClient.AsBuilder().UseLogging(factory).Build().GetService(typeof(LoggingDocumentExtractionClient))); + + ServiceCollection c = new(); + c.AddFakeLogging(); + var services = c.BuildServiceProvider(); + Assert.NotNull(innerClient.AsBuilder().UseLogging().Build(services).GetService(typeof(LoggingDocumentExtractionClient))); + Assert.NotNull(innerClient.AsBuilder().UseLogging(null).Build(services).GetService(typeof(LoggingDocumentExtractionClient))); + Assert.Null(innerClient.AsBuilder().UseLogging(NullLoggerFactory.Instance).Build(services).GetService(typeof(LoggingDocumentExtractionClient))); + } + + [Theory] + [InlineData(LogLevel.Trace)] + [InlineData(LogLevel.Debug)] + [InlineData(LogLevel.Information)] + public async Task ExtractAsync_LogsInvocationAndCompletion(LogLevel level) + { + var collector = new FakeLogCollector(); + + ServiceCollection c = new(); + c.AddLogging(b => b.AddProvider(new FakeLoggerProvider(collector)).SetMinimumLevel(level)); + var services = c.BuildServiceProvider(); + + using IDocumentExtractionClient innerClient = new TestDocumentExtractionClient + { + ExtractAsyncCallback = (document, mediaType, options, cancellationToken) => + Task.FromResult(new DocumentExtractionResult([new DocumentPage(1, "blue whale")])), + }; + + using IDocumentExtractionClient client = innerClient + .AsBuilder() + .UseLogging() + .Build(services); + + using var document = new MemoryStream(new byte[] { 1, 2, 3, 4 }); + await client.ExtractAsync(document, "application/pdf", new DocumentExtractionOptions { ModelId = "mistral-ocr-4-0" }); + + var logs = collector.GetSnapshot(); + if (level is LogLevel.Trace) + { + Assert.Collection(logs, + entry => Assert.True(entry.Message.Contains($"{nameof(IDocumentExtractionClient.ExtractAsync)} invoked:") && entry.Message.Contains("mistral-ocr-4-0")), + entry => Assert.True(entry.Message.Contains($"{nameof(IDocumentExtractionClient.ExtractAsync)} completed:") && entry.Message.Contains("blue whale"))); + } + else if (level is LogLevel.Debug) + { + Assert.Collection(logs, + entry => Assert.True(entry.Message.Contains($"{nameof(IDocumentExtractionClient.ExtractAsync)} invoked.") && !entry.Message.Contains("mistral-ocr-4-0")), + entry => Assert.True(entry.Message.Contains($"{nameof(IDocumentExtractionClient.ExtractAsync)} completed.") && !entry.Message.Contains("blue whale"))); + } + else + { + Assert.Empty(logs); + } + } +} diff --git a/test/Libraries/Microsoft.Extensions.DocumentExtraction.Tests/Microsoft.Extensions.DocumentExtraction.Tests.csproj b/test/Libraries/Microsoft.Extensions.DocumentExtraction.Tests/Microsoft.Extensions.DocumentExtraction.Tests.csproj new file mode 100644 index 00000000000..ad124253139 --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.DocumentExtraction.Tests/Microsoft.Extensions.DocumentExtraction.Tests.csproj @@ -0,0 +1,36 @@ + + + Microsoft.Extensions.DocumentExtraction + Unit tests for Microsoft.Extensions.DocumentExtraction. + + + + $(NoWarn);CA1063;CA1861;S104;SA1130;VSTHRD003;MEDE0001 + true + + + + true + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/Libraries/Microsoft.Extensions.DocumentExtraction.Tests/OpenTelemetryDocumentExtractionClientTests.cs b/test/Libraries/Microsoft.Extensions.DocumentExtraction.Tests/OpenTelemetryDocumentExtractionClientTests.cs new file mode 100644 index 00000000000..6d9b161aa2f --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.DocumentExtraction.Tests/OpenTelemetryDocumentExtractionClientTests.cs @@ -0,0 +1,98 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Threading.Tasks; +using OpenTelemetry.Trace; +using Xunit; + +namespace Microsoft.Extensions.DocumentExtraction; + +public class OpenTelemetryDocumentExtractionClientTests +{ + [Fact] + public void InvalidArgs_Throws() + { + Assert.Throws("innerClient", () => new OpenTelemetryDocumentExtractionClient(null!)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ExpectedInformationLogged_Async(bool enableSensitiveData) + { + var sourceName = Guid.NewGuid().ToString(); + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + using var innerClient = new TestDocumentExtractionClient + { + ExtractAsyncCallback = async (document, mediaType, options, cancellationToken) => + { + await Task.Yield(); + return new DocumentExtractionResult([new DocumentPage(1, "This is the recognized text.")]) + { + Usage = new() { PagesProcessed = 3 }, + }; + }, + + GetServiceCallback = (serviceType, serviceKey) => + serviceType == typeof(DocumentExtractionClientMetadata) ? new DocumentExtractionClientMetadata("testservice", new Uri("http://localhost:12345/something"), "amazingmodel") : + null, + }; + + using var client = innerClient + .AsBuilder() + .UseOpenTelemetry(null, sourceName, configure: instance => + { + instance.EnableSensitiveData = enableSensitiveData; + }) + .Build(); + + DocumentExtractionOptions options = new() + { + ModelId = "mycoolocrmodel", + AdditionalProperties = new() + { + ["service_tier"] = "value1", + ["SomethingElse"] = "value2", + }, + }; + + _ = await client.ExtractAsync(Stream.Null, "application/pdf", options); + + var activity = Assert.Single(activities); + + Assert.NotNull(activity.Id); + Assert.NotEmpty(activity.Id); + + Assert.Equal("localhost", activity.GetTagItem("server.address")); + Assert.Equal(12345, (int)activity.GetTagItem("server.port")!); + + Assert.Equal("generate_content mycoolocrmodel", activity.DisplayName); + Assert.Equal("testservice", activity.GetTagItem("gen_ai.provider.name")); + + Assert.Equal("mycoolocrmodel", activity.GetTagItem("gen_ai.request.model")); + Assert.Equal(enableSensitiveData ? "value1" : null, activity.GetTagItem("service_tier")); + Assert.Equal(enableSensitiveData ? "value2" : null, activity.GetTagItem("SomethingElse")); + + Assert.Equal(3, (int)activity.GetTagItem("gen_ai.usage.pages_processed")!); + + Assert.True(activity.Duration.TotalMilliseconds > 0); + } + + [Fact] + public void GetService_ReturnsActivitySource() + { + using var innerClient = new TestDocumentExtractionClient(); + using var client = innerClient.AsBuilder().UseOpenTelemetry().Build(); + + Assert.NotNull(client.GetService()); + } +}