diff --git a/.github/workflows/mevd.yml b/.github/workflows/mevd.yml new file mode 100644 index 0000000..b88d4b5 --- /dev/null +++ b/.github/workflows/mevd.yml @@ -0,0 +1,102 @@ +name: MEVD + +on: + push: + branches: [main] + paths: + - 'MEVD/**' + - 'Shared/**' + - 'eng/**' + - 'Directory.Build.props' + - 'Directory.Packages.props' + - 'global.json' + - '.github/workflows/mevd.yml' + pull_request: + branches: [main] + paths: + - 'MEVD/**' + - 'Shared/**' + - 'eng/**' + - 'Directory.Build.props' + - 'Directory.Packages.props' + - 'global.json' + - '.github/workflows/mevd.yml' + +env: + DOTNET_NOLOGO: true + DOTNET_CLI_TELEMETRY_OPTOUT: true + +jobs: + build: + name: Build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Setup .NET + uses: actions/setup-dotnet@v4 + with: + global-json-file: global.json + + - name: Build + run: dotnet build MEVD/MEVD.slnf + + unit-tests: + name: 'Unit: ${{ matrix.provider }} (${{ matrix.os }})' + needs: build + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + provider: + - InMemory + - AzureAISearch + - CosmosMongoDB + - PgVector + - Pinecone + - Qdrant + - Redis + - SqliteVec + - Weaviate + steps: + - uses: actions/checkout@v4 + + - name: Setup .NET + uses: actions/setup-dotnet@v4 + with: + global-json-file: global.json + + - name: Test + run: dotnet test MEVD/test/${{ matrix.provider }}.UnitTests + + # Conformance tests run on Ubuntu only: most providers use Testcontainers with Linux Docker + # images, which aren't available on Windows GitHub-hosted runners. + conformance-tests: + name: 'Conformance: ${{ matrix.provider }}' + needs: build + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + provider: + # AzureAISearch and CosmosMongoDB require cloud instances and are not run in CI. + # - AzureAISearch + # - CosmosMongoDB + - InMemory + - PgVector + - Pinecone + - Qdrant + - Redis + - SqliteVec + - Weaviate + steps: + - uses: actions/checkout@v4 + + - name: Setup .NET + uses: actions/setup-dotnet@v4 + with: + global-json-file: global.json + + - name: Test + run: dotnet test MEVD/test/${{ matrix.provider }}.ConformanceTests diff --git a/CommunityToolkit.AI.slnx b/CommunityToolkit.AI.slnx new file mode 100644 index 0000000..79732c7 --- /dev/null +++ b/CommunityToolkit.AI.slnx @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Directory.Build.props b/Directory.Build.props index 3c67d1d..c9a0319 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -9,4 +9,6 @@ true + + diff --git a/Directory.Packages.props b/Directory.Packages.props index 396aed1..7199d1e 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -1,4 +1,45 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/MEVD/MEVD.slnf b/MEVD/MEVD.slnf new file mode 100644 index 0000000..517fa78 --- /dev/null +++ b/MEVD/MEVD.slnf @@ -0,0 +1,36 @@ +{ + "solution": { + "path": "../CommunityToolkit.AI.slnx", + "projects": + [ + "MEVD/src/AzureAISearch/AzureAISearch.csproj", + "MEVD/src/CosmosMongoDB/CosmosMongoDB.csproj", + "MEVD/src/InMemory/InMemory.csproj", + "MEVD/src/Pinecone/Pinecone.csproj", + "MEVD/src/PgVector/PgVector.csproj", + "MEVD/src/Qdrant/Qdrant.csproj", + "MEVD/src/Redis/Redis.csproj", + "MEVD/src/SqliteVec/SqliteVec.csproj", + "MEVD/src/Weaviate/Weaviate.csproj", + + "MEVD/test/AzureAISearch.UnitTests/AzureAISearch.UnitTests.csproj", + "MEVD/test/AzureAISearch.ConformanceTests/AzureAISearch.ConformanceTests.csproj", + "MEVD/test/CosmosMongoDB.UnitTests/CosmosMongoDB.UnitTests.csproj", + "MEVD/test/CosmosMongoDB.ConformanceTests/CosmosMongoDB.ConformanceTests.csproj", + "MEVD/test/InMemory.UnitTests/InMemory.UnitTests.csproj", + "MEVD/test/InMemory.ConformanceTests/InMemory.ConformanceTests.csproj", + "MEVD/test/PgVector.UnitTests/PgVector.UnitTests.csproj", + "MEVD/test/PgVector.ConformanceTests/PgVector.ConformanceTests.csproj", + "MEVD/test/Pinecone.UnitTests/Pinecone.UnitTests.csproj", + "MEVD/test/Pinecone.ConformanceTests/Pinecone.ConformanceTests.csproj", + "MEVD/test/Qdrant.UnitTests/Qdrant.UnitTests.csproj", + "MEVD/test/Qdrant.ConformanceTests/Qdrant.ConformanceTests.csproj", + "MEVD/test/Redis.UnitTests/Redis.UnitTests.csproj", + "MEVD/test/Redis.ConformanceTests/Redis.ConformanceTests.csproj", + "MEVD/test/SqliteVec.UnitTests/SqliteVec.UnitTests.csproj", + "MEVD/test/SqliteVec.ConformanceTests/SqliteVec.ConformanceTests.csproj", + "MEVD/test/Weaviate.UnitTests/Weaviate.UnitTests.csproj", + "MEVD/test/Weaviate.ConformanceTests/Weaviate.ConformanceTests.csproj" + ] + } +} \ No newline at end of file diff --git a/MEVD/src/AzureAISearch/AssemblyInfo.cs b/MEVD/src/AzureAISearch/AssemblyInfo.cs new file mode 100644 index 0000000..d6395e8 --- /dev/null +++ b/MEVD/src/AzureAISearch/AssemblyInfo.cs @@ -0,0 +1,2 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. diff --git a/MEVD/src/AzureAISearch/AzureAISearch.csproj b/MEVD/src/AzureAISearch/AzureAISearch.csproj new file mode 100644 index 0000000..9b238b5 --- /dev/null +++ b/MEVD/src/AzureAISearch/AzureAISearch.csproj @@ -0,0 +1,27 @@ + + + 1.0.0-preview.1 + CommunityToolkit.VectorData.AzureAISearch + $(AssemblyName) + net10.0;net8.0;netstandard2.0;net462 + + Azure AI Search provider for Microsoft.Extensions.VectorData + Azure AI Search provider for Microsoft.Extensions.VectorData by Semantic Kernel + + + + + + + + + + + + + + + + + + diff --git a/MEVD/src/AzureAISearch/AzureAISearchCollection.cs b/MEVD/src/AzureAISearch/AzureAISearchCollection.cs new file mode 100644 index 0000000..8e42c09 --- /dev/null +++ b/MEVD/src/AzureAISearch/AzureAISearchCollection.cs @@ -0,0 +1,813 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Linq.Expressions; +using System.Runtime.CompilerServices; +using System.Text.Json; +using System.Text.Json.Nodes; +using Azure; +using Azure.Search.Documents; +using Azure.Search.Documents.Indexes; +using Azure.Search.Documents.Indexes.Models; +using Azure.Search.Documents.Models; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; +using MEAI = Microsoft.Extensions.AI; +using Microsoft.Shared.Diagnostics; + +namespace CommunityToolkit.VectorData.AzureAISearch; + +/// +/// Service for storing and retrieving vector records, that uses Azure AI Search as the underlying storage. +/// +/// The data type of the record key. Must be . +/// The data model to use for adding, updating and retrieving data from storage. +#pragma warning disable CA1711 // Identifiers should not have incorrect suffix +public class AzureAISearchCollection : VectorStoreCollection, IKeywordHybridSearchable + where TKey : notnull + where TRecord : class +#pragma warning restore CA1711 // Identifiers should not have incorrect suffix +{ + /// Metadata about vector store record collection. + private readonly VectorStoreCollectionMetadata _collectionMetadata; + + /// The default options for vector search. + private static readonly VectorSearchOptions s_defaultVectorSearchOptions = new(); + + /// The default options for hybrid vector search. + private static readonly HybridSearchOptions s_defaultKeywordVectorizedHybridSearchOptions = new(); + + /// Azure AI Search client that can be used to manage the list of indices in an Azure AI Search Service. + private readonly SearchIndexClient _searchIndexClient; + + /// Azure AI Search client that can be used to manage data in an Azure AI Search Service index. + private readonly SearchClient _searchClient; + + /// A mapper to use for converting between the data model and the Azure AI Search record. + private readonly IAzureAISearchMapper _mappper; + + /// The model for this collection. + private readonly CollectionModel _model; + + /// + /// Initializes a new instance of the class. + /// + /// Azure AI Search client that can be used to manage the list of indices in an Azure AI Search Service. + /// The name of the collection that this will access. + /// Optional configuration options for this class. + /// Thrown when is null. + /// Thrown when options are misconfigured. + [RequiresUnreferencedCode("The Azure AI Search provider is currently incompatible with trimming.")] + [RequiresDynamicCode("The Azure AI Search provider is currently incompatible with NativeAOT.")] + public AzureAISearchCollection(SearchIndexClient searchIndexClient, string name, AzureAISearchCollectionOptions? options = default) + : this( + searchIndexClient, + name, + static options => typeof(TRecord) == typeof(Dictionary) + ? throw new NotSupportedException(VectorDataStrings.NonDynamicCollectionWithDictionaryNotSupported(typeof(AzureAISearchDynamicCollection))) + : new AzureAISearchModelBuilder() + .Build(typeof(TRecord), typeof(TKey), options.Definition, options.EmbeddingGenerator, options.JsonSerializerOptions ?? JsonSerializerOptions.Default), + options) + { + } + + internal AzureAISearchCollection(SearchIndexClient searchIndexClient, string name, Func modelFactory, AzureAISearchCollectionOptions? options) + { + // Verify. + Throw.IfNull(searchIndexClient); + Throw.IfNullOrWhitespace(name); + + if (typeof(TKey) != typeof(string) && typeof(TKey) != typeof(Guid) && typeof(TKey) != typeof(object)) + { + throw new NotSupportedException("Only string and Guid keys are supported."); + } + + options ??= AzureAISearchCollectionOptions.Default; + + // Assign. + Name = name; + _model = modelFactory(options); + _searchIndexClient = searchIndexClient; + _searchClient = _searchIndexClient.GetSearchClient(name); + + _mappper = typeof(TRecord) == typeof(Dictionary) ? + (IAzureAISearchMapper)(object)new AzureAISearchDynamicMapper(_model, options.JsonSerializerOptions) : + new AzureAISearchMapper(_model, options.JsonSerializerOptions); + + _collectionMetadata = new() + { + VectorStoreSystemName = AzureAISearchConstants.VectorStoreSystemName, + VectorStoreName = searchIndexClient.ServiceName, + CollectionName = name + }; + } + + /// + public override string Name { get; } + + /// + public override async Task CollectionExistsAsync(CancellationToken cancellationToken = default) + { + try + { + await _searchIndexClient.GetIndexAsync(Name, cancellationToken).ConfigureAwait(false); + return true; + } + catch (RequestFailedException ex) when (ex.Status == 404) + { + return false; + } + catch (RequestFailedException ex) + { + throw new VectorStoreException("Call to vector store failed.", ex) + { + VectorStoreSystemName = AzureAISearchConstants.VectorStoreSystemName, + VectorStoreName = _collectionMetadata.VectorStoreName, + CollectionName = Name, + OperationName = "GetIndex" + }; + } + } + + /// + public override async Task EnsureCollectionExistsAsync(CancellationToken cancellationToken = default) + { + const string OperationName = "CreateIndex"; + + // Don't even try to create if the collection already exists. + if (await CollectionExistsAsync(cancellationToken).ConfigureAwait(false)) + { + return; + } + + var vectorSearchConfig = new VectorSearch(); + var searchFields = new List(); + + // Loop through all properties and create the search fields. + foreach (var property in _model.Properties) + { + switch (property) + { + case KeyPropertyModel p: + searchFields.Add(AzureAISearchCollectionCreateMapping.MapKeyField(p)); + break; + + case DataPropertyModel p: + searchFields.Add(AzureAISearchCollectionCreateMapping.MapDataField(p)); + break; + + case VectorPropertyModel p: + (VectorSearchField vectorSearchField, VectorSearchAlgorithmConfiguration algorithmConfiguration, VectorSearchProfile vectorSearchProfile) = AzureAISearchCollectionCreateMapping.MapVectorField(p); + + // Add the search field, plus its profile and algorithm configuration to the search config. + searchFields.Add(vectorSearchField); + vectorSearchConfig.Algorithms.Add(algorithmConfiguration); + vectorSearchConfig.Profiles.Add(vectorSearchProfile); + break; + + default: + throw new UnreachableException(); + } + } + + // Create the index definition. + var searchIndex = new SearchIndex(Name, searchFields); + searchIndex.VectorSearch = vectorSearchConfig; + + try + { + await _searchIndexClient.CreateIndexAsync(searchIndex, cancellationToken).ConfigureAwait(false); + } + catch (RequestFailedException ex) when (ex.ErrorCode == "ResourceNameAlreadyInUse") + { + // Index already exists, ignore. + } + catch (RequestFailedException ex) + { + throw new VectorStoreException("Call to vector store failed.", ex) + { + VectorStoreSystemName = AzureAISearchConstants.VectorStoreSystemName, + VectorStoreName = _collectionMetadata.VectorStoreName, + CollectionName = Name, + OperationName = OperationName + }; + } + catch (AggregateException ex) when (ex.InnerException is RequestFailedException innerEx) + { + throw new VectorStoreException("Call to vector store failed.", ex) + { + VectorStoreSystemName = AzureAISearchConstants.VectorStoreSystemName, + VectorStoreName = _collectionMetadata.VectorStoreName, + CollectionName = Name, + OperationName = OperationName + }; + } + } + + /// + public override Task EnsureCollectionDeletedAsync(CancellationToken cancellationToken = default) + { + return RunOperationAsync( + "DeleteIndex", + async () => + { + try + { + return await _searchIndexClient.DeleteIndexAsync(Name, cancellationToken).ConfigureAwait(false); + } + catch (RequestFailedException ex) when (ex.Status == 404) + { + return null!; + } + }); + } + + /// + public override Task GetAsync(TKey key, RecordRetrievalOptions? options = default, CancellationToken cancellationToken = default) + { + // Create Options. + var innerOptions = ConvertGetDocumentOptions(options); + var includeVectors = options?.IncludeVectors ?? false; + if (includeVectors && _model.EmbeddingGenerationRequired) + { + throw new NotSupportedException(VectorDataStrings.IncludeVectorsNotSupportedWithEmbeddingGeneration); + } + + // Get record. + return GetDocumentAndMapToDataModelAsync(key, includeVectors, innerOptions, cancellationToken); + } + + /// + public override async IAsyncEnumerable GetAsync(IEnumerable keys, RecordRetrievalOptions? options = default, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + Throw.IfNull(keys); + + // Create Options + var innerOptions = ConvertGetDocumentOptions(options); + var includeVectors = options?.IncludeVectors ?? false; + if (includeVectors && _model.EmbeddingGenerationRequired) + { + throw new NotSupportedException(VectorDataStrings.IncludeVectorsNotSupportedWithEmbeddingGeneration); + } + + foreach (var key in keys) + { + var record = await GetDocumentAndMapToDataModelAsync(key, includeVectors, innerOptions, cancellationToken).ConfigureAwait(false); + + if (record is not null) + { + yield return record; + } + } + } + + /// + public override Task DeleteAsync(TKey key, CancellationToken cancellationToken = default) + { + var stringKey = GetStringKey(key); + + // Remove record. + return RunOperationAsync( + "DeleteDocuments", + () => _searchClient.DeleteDocumentsAsync(_model.KeyProperty.StorageName, [stringKey], new IndexDocumentsOptions(), cancellationToken)); + } + + /// + public override Task DeleteAsync(IEnumerable keys, CancellationToken cancellationToken = default) + { + Throw.IfNull(keys); + if (!keys.Any()) + { + return Task.CompletedTask; + } + + var stringKeys = keys is IEnumerable k ? k : keys.Select(GetStringKey); + + // Remove records. + return RunOperationAsync( + "DeleteDocuments", + () => _searchClient.DeleteDocumentsAsync(_model.KeyProperty.StorageName, stringKeys, new IndexDocumentsOptions(), cancellationToken)); + } + + /// + public override async Task UpsertAsync(TRecord record, CancellationToken cancellationToken = default) + { + Throw.IfNull(record); + + // Create options. + var innerOptions = new IndexDocumentsOptions { ThrowOnAnyError = true }; + + // Upsert record. + await MapToStorageModelAndUploadDocumentAsync([record], innerOptions, cancellationToken).ConfigureAwait(false); + } + + /// + public override async Task UpsertAsync(IEnumerable records, CancellationToken cancellationToken = default) + { + Throw.IfNull(records); + if (!records.Any()) + { + return; + } + + // Create Options + var innerOptions = new IndexDocumentsOptions { ThrowOnAnyError = true }; + + // Upsert records + await MapToStorageModelAndUploadDocumentAsync(records, innerOptions, cancellationToken).ConfigureAwait(false); + } + + /// + public override IAsyncEnumerable GetAsync(Expression> filter, int top, + FilteredRecordRetrievalOptions? options = null, CancellationToken cancellationToken = default) + { + Throw.IfNull(filter); + Throw.IfLessThan(top, 1); + + options ??= new(); + + var includeVectors = options.IncludeVectors; + if (includeVectors && _model.EmbeddingGenerationRequired) + { + throw new NotSupportedException(VectorDataStrings.IncludeVectorsNotSupportedWithEmbeddingGeneration); + } + + SearchOptions searchOptions = new() + { + VectorSearch = new(), + Size = top, + Skip = options.Skip, + Filter = new AzureAISearchFilterTranslator().Translate(filter, _model) + }; + + // Filter out vector fields if requested. + if (!options.IncludeVectors) + { + searchOptions.Select.Add(_model.KeyProperty.StorageName); + + foreach (var dataProperty in _model.DataProperties) + { + searchOptions.Select.Add(dataProperty.StorageName); + } + } + + if (options.OrderBy is not null) + { + foreach (var pair in options.OrderBy(new()).Values) + { + PropertyModel property = _model.GetDataOrKeyProperty(pair.PropertySelector); + string name = property.StorageName; + // From https://learn.microsoft.com/dotnet/api/azure.search.documents.searchoptions.orderby: + // "Each expression can be followed by asc to indicate ascending, or desc to indicate descending". + // "The default is ascending order." + if (!pair.Ascending) + { + name += " desc"; + } + + searchOptions.OrderBy.Add(name); + } + } + + return SearchAndMapToDataModelAsync(null, searchOptions, options.IncludeVectors, cancellationToken) + .Select(result => result.Record); + } + + #region Search + + /// + public override async IAsyncEnumerable> SearchAsync( + TInput searchValue, + int top, + VectorSearchOptions? options = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + Throw.IfNull(searchValue); + Throw.IfLessThan(top, 1); + + options ??= s_defaultVectorSearchOptions; + var vectorProperty = _model.GetVectorPropertyOrSingle(options); + var floatVector = await GetSearchVectorAsync(searchValue, vectorProperty, cancellationToken).ConfigureAwait(false); + + var searchOptions = BuildSearchOptions( + _model, + options, + top, + floatVector is null + ? new VectorizableTextQuery((string)(object)searchValue) { KNearestNeighborsCount = top + options.Skip, Fields = { vectorProperty.StorageName } } + : new VectorizedQuery(floatVector.Value) { KNearestNeighborsCount = top + options.Skip, Fields = { vectorProperty.StorageName } }); + + await foreach (var record in SearchAndMapToDataModelAsync(null, searchOptions, options.IncludeVectors, cancellationToken).ConfigureAwait(false)) + { + // Azure AI Search threshold filtering is in preview: + // https://learn.microsoft.com/azure/search/vector-search-how-to-query#set-thresholds-to-exclude-low-scoring-results-preview + // See https://github.com/microsoft/semantic-kernel/issues/13500. + // For now, perform post-filtering on the client-side. + if (options.ScoreThreshold.HasValue && record.Score < options.ScoreThreshold.Value) + { + continue; + } + + yield return record; + } + } + + /// + public async IAsyncEnumerable> HybridSearchAsync( + TInput searchValue, + ICollection keywords, + int top, + HybridSearchOptions? options = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + where TInput : notnull + { + Throw.IfNull(keywords); + Throw.IfLessThan(top, 1); + + // Resolve options. + options ??= s_defaultKeywordVectorizedHybridSearchOptions; + var vectorProperty = _model.GetVectorPropertyOrSingle(new() { VectorProperty = options.VectorProperty }); + var floatVector = await GetSearchVectorAsync(searchValue, vectorProperty, cancellationToken).ConfigureAwait(false); + + var textDataProperty = _model.GetFullTextDataPropertyOrSingle(options.AdditionalProperty); + + // Build search options. + var searchOptions = BuildSearchOptions( + _model, + new() + { + Filter = options.Filter, + VectorProperty = options.VectorProperty, + Skip = options.Skip, + }, + top, + floatVector is null + ? new VectorizableTextQuery((string)(object)searchValue) { KNearestNeighborsCount = top + options.Skip, Fields = { vectorProperty.StorageName } } + : new VectorizedQuery(floatVector.Value) { KNearestNeighborsCount = top + options.Skip, Fields = { vectorProperty.StorageName } }); + + searchOptions.SearchFields.Add(textDataProperty.StorageName); + var keywordsCombined = string.Join(" ", keywords); + + await foreach (var record in SearchAndMapToDataModelAsync(keywordsCombined, searchOptions, options.IncludeVectors, cancellationToken).ConfigureAwait(false)) + { + // Azure AI Search returns scores where higher values indicate more relevant results. + if (options.ScoreThreshold.HasValue && record.Score < options.ScoreThreshold.Value) + { + continue; + } + + yield return record; + } + } + + private static async ValueTask?> GetSearchVectorAsync(TInput searchValue, VectorPropertyModel vectorProperty, CancellationToken cancellationToken) + where TInput : notnull + => searchValue switch + { + ReadOnlyMemory r => r, + float[] f => new ReadOnlyMemory(f), + Embedding e => e.Vector, + _ when vectorProperty.EmbeddingGenerationDispatcher is not null + => ((Embedding)await vectorProperty.GenerateEmbeddingAsync(searchValue, cancellationToken).ConfigureAwait(false)).Vector, + + // A string was passed without an embedding generator being configured; send the string to Azure AI Search for backend embedding generation. + string when vectorProperty.EmbeddingGenerator is null => (ReadOnlyMemory?)null, + + _ => vectorProperty.EmbeddingGenerator is null + ? throw new NotSupportedException(VectorDataStrings.InvalidSearchInputAndNoEmbeddingGeneratorWasConfigured(searchValue.GetType(), AzureAISearchModelBuilder.SupportedVectorTypes)) + : throw new InvalidOperationException(VectorDataStrings.IncompatibleEmbeddingGeneratorWasConfiguredForInputType(typeof(TInput), vectorProperty.EmbeddingGenerator.GetType())) + }; + + #endregion Search + + /// + public override object? GetService(Type serviceType, object? serviceKey = null) + { + Throw.IfNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreCollectionMetadata) ? _collectionMetadata : + serviceType == typeof(SearchIndexClient) ? _searchIndexClient : + serviceType == typeof(SearchClient) ? _searchClient : + serviceType.IsInstanceOfType(this) ? this : + null; + } + + /// + /// Get the document with the given key and map it to the data model using the configured mapper type. + /// + /// The key of the record to get. + /// A value indicating whether to include vectors in the result or not. + /// The Azure AI Search sdk options for getting a document. + /// The to monitor for cancellation requests. The default is . + /// The retrieved document, mapped to the consumer data model. + private async Task GetDocumentAndMapToDataModelAsync( + TKey key, + bool includeVectors, + GetDocumentOptions innerOptions, + CancellationToken cancellationToken) + { + const string OperationName = "GetDocument"; + + var stringKey = GetStringKey(key); + + var jsonObject = await RunOperationAsync( + OperationName, + () => GetDocumentWithNotFoundHandlingAsync(_searchClient, stringKey, innerOptions, cancellationToken)).ConfigureAwait(false); + + if (jsonObject is null) + { + return default; + } + + return (TRecord)(object)_mappper!.MapFromStorageToDataModel(jsonObject, includeVectors); + } + + /// + /// Search for the documents matching the given options and map them to the data model using the configured mapper type. + /// + /// Text to use if doing a hybrid search. Null for non-hybrid search. + /// The options controlling the behavior of the search operation. + /// A value indicating whether to include vectors in the result or not. + /// The to monitor for cancellation requests. The default is . + /// The mapped search results. + private async IAsyncEnumerable> SearchAndMapToDataModelAsync( + string? searchText, + SearchOptions searchOptions, + bool includeVectors, + [EnumeratorCancellation] CancellationToken cancellationToken) + { + const string OperationName = "Search"; + + var jsonObjectResults = await RunOperationAsync( + OperationName, + () => _searchClient.SearchAsync(searchText, searchOptions, cancellationToken)).ConfigureAwait(false); + + await foreach (var result in MapSearchResultsAsync(jsonObjectResults.Value.GetResultsAsync(), OperationName, includeVectors).ConfigureAwait(false)) + { + yield return result; + } + } + + /// + /// Map the data model to the storage model and upload the document. + /// + /// The records to upload. + /// The Azure AI Search sdk options for uploading a document. + /// The to monitor for cancellation requests. The default is . + /// The document upload result. + private async Task> MapToStorageModelAndUploadDocumentAsync( + IEnumerable records, + IndexDocumentsOptions innerOptions, + CancellationToken cancellationToken) + { + const string OperationName = "UploadDocuments"; + + (records, var generatedEmbeddings) = await ProcessEmbeddingsAsync(_model, records, cancellationToken).ConfigureAwait(false); + + // Handle auto-generated keys (client-side for Azure AI Search, which doesn't support server-side auto-generation) + var keyProperty = _model.KeyProperty; + var jsonObjects = new List(); + var recordIndex = 0; + foreach (var record in records) + { + if (keyProperty.IsAutoGenerated && keyProperty.GetValue(record) == Guid.Empty) + { + keyProperty.SetValue(record, Guid.NewGuid()); + } + + jsonObjects.Add(_mappper!.MapFromDataToStorageModel(record, recordIndex++, generatedEmbeddings)); + } + + return await RunOperationAsync( + OperationName, + () => _searchClient.UploadDocumentsAsync(jsonObjects, innerOptions, cancellationToken)).ConfigureAwait(false); + } + + /// + /// Map the search results from to objects using the configured mapper type. + /// + /// The search results to map. + /// The name of the current operation for telemetry purposes. + /// A value indicating whether to include vectors in the resultset or not. + /// The mapped results. + private async IAsyncEnumerable> MapSearchResultsAsync(IAsyncEnumerable> results, string operationName, bool includeVectors) + { + await foreach (var result in results.ConfigureAwait(false)) + { + var document = (TRecord)(object)_mappper!.MapFromStorageToDataModel(result.Document, includeVectors); + yield return new VectorSearchResult(document, result.Score); + } + } + + /// + /// Map the search results from to objects. + /// + /// The search results to map. + /// The mapped results. + private async IAsyncEnumerable> MapSearchResultsAsync(IAsyncEnumerable> results) + { + await foreach (var result in results.ConfigureAwait(false)) + { + yield return new VectorSearchResult(result.Document, result.Score); + } + } + + /// + /// Convert the public options model to the Azure AI Search options model. + /// + /// The public options model. + /// The Azure AI Search options model. + private GetDocumentOptions ConvertGetDocumentOptions(RecordRetrievalOptions? options) + { + var innerOptions = new GetDocumentOptions(); + if (options?.IncludeVectors is not true) + { + innerOptions.SelectedFields.Add(_model.KeyProperty.StorageName); + + foreach (var dataProperty in _model.DataProperties) + { + innerOptions.SelectedFields.Add(dataProperty.StorageName); + } + } + + return innerOptions; + } + + /// + /// Build the search options for a vector search, where the type of vector search can be provided as input. + /// E.g. VectorizedQuery or VectorizableTextQuery. + /// + private static SearchOptions BuildSearchOptions(CollectionModel model, VectorSearchOptions options, int top, VectorQuery? vectorQuery) + { + if (model.VectorProperties.Count == 0) + { + throw new InvalidOperationException("The collection does not have any vector fields, so vector search is not possible."); + } + + if (options.IncludeVectors && model.EmbeddingGenerationRequired) + { + throw new NotSupportedException(VectorDataStrings.IncludeVectorsNotSupportedWithEmbeddingGeneration); + } + + // Build filter object. + var filter = options.Filter is not null + ? new AzureAISearchFilterTranslator().Translate(options.Filter, model) + : null; + + // Build search options. + var searchOptions = new SearchOptions + { + VectorSearch = new(), + Size = top, + Skip = options.Skip, + }; + + if (filter is not null) + { + searchOptions.Filter = filter; + } + + searchOptions.VectorSearch.Queries.Add(vectorQuery); + + // Filter out vector fields if requested. + if (!options.IncludeVectors) + { + searchOptions.Select.Add(model.KeyProperty.StorageName); + + foreach (var dataProperty in model.DataProperties) + { + searchOptions.Select.Add(dataProperty.StorageName); + } + } + + return searchOptions; + } + + private static async ValueTask<(IEnumerable records, IReadOnlyList?[]?)> ProcessEmbeddingsAsync( + CollectionModel model, + IEnumerable records, + CancellationToken cancellationToken) + { + IReadOnlyList? recordsList = null; + + // If an embedding generator is defined, invoke it once per property for all records. + IReadOnlyList?[]? generatedEmbeddings = null; + + var vectorPropertyCount = model.VectorProperties.Count; + for (var i = 0; i < vectorPropertyCount; i++) + { + var vectorProperty = model.VectorProperties[i]; + + if (AzureAISearchModelBuilder.IsVectorPropertyTypeValidCore(vectorProperty.Type, out _)) + { + continue; + } + + // We have a vector property whose type isn't natively supported - we need to generate embeddings. + Debug.Assert(vectorProperty.EmbeddingGenerator is not null); + + // We have a property with embedding generation; materialize the records' enumerable if needed, to + // prevent multiple enumeration. + if (recordsList is null) + { + recordsList = records is IReadOnlyList r ? r : records.ToList(); + + if (recordsList.Count == 0) + { + return (records, null); + } + + records = recordsList; + } + + // TODO: Ideally we'd group together vector properties using the same generator (and with the same input and output properties), + // and generate embeddings for them in a single batch. That's some more complexity though. + generatedEmbeddings ??= new IReadOnlyList?[vectorPropertyCount]; + generatedEmbeddings[i] = await vectorProperty.GenerateEmbeddingsAsync(records.Select(r => vectorProperty.GetValueAsObject(r)), cancellationToken).ConfigureAwait(false); + } + + return (records, generatedEmbeddings); + } + + /// + /// Get a document with the given key, and return null if it is not found. + /// + /// The type to deserialize the document to. + /// The search client to use when fetching the document. + /// The key of the record to get. + /// The Azure AI Search sdk options for getting a document. + /// The to monitor for cancellation requests. The default is . + /// The retrieved document, mapped to the consumer data model, or null if not found. + private async Task GetDocumentWithNotFoundHandlingAsync( + SearchClient searchClient, + string key, + GetDocumentOptions innerOptions, + CancellationToken cancellationToken) + { + const string OperationName = "GetDocument"; + + try + { + return await searchClient.GetDocumentAsync(key, innerOptions, cancellationToken).ConfigureAwait(false); + } + catch (RequestFailedException ex) when (ex.Status == 404) + { + return default; + } + catch (AggregateException ex) when (ex.InnerException is RequestFailedException innerEx) + { + throw new VectorStoreException("Call to vector store failed.", ex) + { + VectorStoreSystemName = AzureAISearchConstants.VectorStoreSystemName, + VectorStoreName = _collectionMetadata.VectorStoreName, + CollectionName = Name, + OperationName = OperationName + }; + } + catch (RequestFailedException ex) + { + throw new VectorStoreException("Call to vector store failed.", ex) + { + VectorStoreSystemName = AzureAISearchConstants.VectorStoreSystemName, + VectorStoreName = _collectionMetadata.VectorStoreName, + CollectionName = Name, + OperationName = OperationName + }; + } + } + + /// + /// Run the given operation and wrap any with ."/> + /// + /// The response type of the operation. + /// The type of database operation being run. + /// The operation to run. + /// The result of the operation. + private Task RunOperationAsync(string operationName, Func> operation) => + VectorStoreErrorHandler.RunOperationAsync( + _collectionMetadata, + operationName, + operation); + + private static string GetStringKey(TKey key) + { + Throw.IfNull(key); + + var stringKey = key switch + { + string s => s, + Guid g => g.ToString(), + + _ => throw new UnreachableException("string key should have been validated during model building") + }; + + Throw.IfNullOrWhitespace(stringKey); + + return stringKey; + } +} diff --git a/MEVD/src/AzureAISearch/AzureAISearchCollectionCreateMapping.cs b/MEVD/src/AzureAISearch/AzureAISearchCollectionCreateMapping.cs new file mode 100644 index 0000000..575b8f5 --- /dev/null +++ b/MEVD/src/AzureAISearch/AzureAISearchCollectionCreateMapping.cs @@ -0,0 +1,159 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Azure.Search.Documents.Indexes.Models; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; + +namespace CommunityToolkit.VectorData.AzureAISearch; + +/// +/// Contains mapping helpers to use when creating a Azure AI Search vector collection. +/// +internal static class AzureAISearchCollectionCreateMapping +{ + /// + /// Map from a to an Azure AI Search . + /// + /// The key property definition. + /// The for the provided property definition. + public static SearchableField MapKeyField(KeyPropertyModel keyProperty) + { + return new SearchableField(keyProperty.StorageName) { IsKey = true, IsFilterable = true }; + } + + /// + /// Map from a to an Azure AI Search . + /// + /// The data property definition. + /// The for the provided property definition. + /// Throws when the definition is missing required information. + public static SimpleField MapDataField(DataPropertyModel dataProperty) + { + if (dataProperty.IsFullTextIndexed) + { + if (dataProperty.Type != typeof(string)) + { + throw new InvalidOperationException($"Property {nameof(dataProperty.IsFullTextIndexed)} on {nameof(VectorStoreDataProperty)} '{dataProperty.ModelName}' is set to true, but the property type is not a string. The Azure AI Search VectorStore supports {nameof(dataProperty.IsFullTextIndexed)} on string properties only."); + } + + return new SearchableField(dataProperty.StorageName) + { + IsFilterable = dataProperty.IsIndexed, + // Sometimes the users ask to also OrderBy given filterable property, so we make it sortable. + IsSortable = dataProperty.IsIndexed + }; + } + + var fieldType = AzureAISearchCollectionCreateMapping.GetSDKFieldDataType(dataProperty.Type); + return new SimpleField(dataProperty.StorageName, fieldType) + { + IsFilterable = dataProperty.IsIndexed, + // Sometimes the users ask to also OrderBy given filterable property, so we make it sortable. + IsSortable = dataProperty.IsIndexed && !fieldType.IsCollection + }; + } + + /// + /// Map form a to an Azure AI Search and generate the required index configuration. + /// + /// The vector property definition. + /// The and required index configuration. + /// Throws when the definition is missing required information, or unsupported options are configured. + public static (VectorSearchField vectorSearchField, VectorSearchAlgorithmConfiguration algorithmConfiguration, VectorSearchProfile vectorSearchProfile) MapVectorField(VectorPropertyModel vectorProperty) + { + // Build a name for the profile and algorithm configuration based on the property name + // since we'll just create a separate one for each vector property. + var vectorSearchProfileName = $"{vectorProperty.StorageName}Profile"; + var algorithmConfigName = $"{vectorProperty.StorageName}AlgoConfig"; + + // Read the vector index settings from the property definition and create the right index configuration. + var indexKind = AzureAISearchCollectionCreateMapping.GetSKIndexKind(vectorProperty); + var algorithmMetric = AzureAISearchCollectionCreateMapping.GetSDKDistanceAlgorithm(vectorProperty); + + VectorSearchAlgorithmConfiguration algorithmConfiguration = indexKind switch + { + IndexKind.Hnsw => new HnswAlgorithmConfiguration(algorithmConfigName) { Parameters = new HnswParameters { Metric = algorithmMetric } }, + IndexKind.Flat => new ExhaustiveKnnAlgorithmConfiguration(algorithmConfigName) { Parameters = new ExhaustiveKnnParameters { Metric = algorithmMetric } }, + + _ => throw new NotSupportedException($"Index kind '{indexKind}' on {nameof(VectorStoreVectorProperty)} '{vectorProperty.ModelName}' is not supported by the Azure AI Search VectorStore.") + }; + + var vectorSearchProfile = new VectorSearchProfile(vectorSearchProfileName, algorithmConfigName); + + return (new VectorSearchField(vectorProperty.StorageName, vectorProperty.Dimensions, vectorSearchProfileName), algorithmConfiguration, vectorSearchProfile); + } + + /// + /// Get the configured from the given . + /// If none is configured the default is . + /// + /// The vector property definition. + /// The configured or default . + public static string GetSKIndexKind(VectorPropertyModel vectorProperty) + => vectorProperty.IndexKind ?? IndexKind.Hnsw; + + /// + /// Get the configured from the given . + /// If none is configured, the default is . + /// + /// The vector property definition. + /// The chosen . + /// Thrown if a distance function is chosen that isn't supported by Azure AI Search. + public static VectorSearchAlgorithmMetric GetSDKDistanceAlgorithm(VectorPropertyModel vectorProperty) + => vectorProperty.DistanceFunction switch + { + DistanceFunction.CosineSimilarity or null => VectorSearchAlgorithmMetric.Cosine, + DistanceFunction.DotProductSimilarity => VectorSearchAlgorithmMetric.DotProduct, + DistanceFunction.EuclideanDistance => VectorSearchAlgorithmMetric.Euclidean, + + _ => throw new NotSupportedException($"Distance function '{vectorProperty.DistanceFunction}' for {nameof(VectorStoreVectorProperty)} '{vectorProperty.ModelName}' is not supported by the Azure AI Search VectorStore.") + }; + + /// + /// Maps the given property type to the corresponding . + /// + /// The property type to map. + /// The that corresponds to the given property type." + /// Thrown if the given type is not supported. + public static SearchFieldDataType GetSDKFieldDataType(Type propertyType) + => (Nullable.GetUnderlyingType(propertyType) ?? propertyType) switch + { + Type t when t == typeof(string) => SearchFieldDataType.String, + Type t when t == typeof(bool) => SearchFieldDataType.Boolean, + Type t when t == typeof(int) => SearchFieldDataType.Int32, + Type t when t == typeof(long) => SearchFieldDataType.Int64, + // We don't map float to SearchFieldDataType.Single, because Azure AI Search doesn't support it. + // Half is also listed by the SDK, but currently not supported. + Type t when t == typeof(float) => SearchFieldDataType.Double, + Type t when t == typeof(double) => SearchFieldDataType.Double, + Type t when t == typeof(DateTime) => SearchFieldDataType.DateTimeOffset, + Type t when t == typeof(DateTimeOffset) => SearchFieldDataType.DateTimeOffset, +#if NET + Type t when t == typeof(DateOnly) => SearchFieldDataType.DateTimeOffset, +#endif + + Type t when t == typeof(string[]) => SearchFieldDataType.Collection(SearchFieldDataType.String), + Type t when t == typeof(List) => SearchFieldDataType.Collection(SearchFieldDataType.String), + Type t when t == typeof(bool[]) => SearchFieldDataType.Collection(SearchFieldDataType.Boolean), + Type t when t == typeof(List) => SearchFieldDataType.Collection(SearchFieldDataType.Boolean), + Type t when t == typeof(int[]) => SearchFieldDataType.Collection(SearchFieldDataType.Int32), + Type t when t == typeof(List) => SearchFieldDataType.Collection(SearchFieldDataType.Int32), + Type t when t == typeof(long[]) => SearchFieldDataType.Collection(SearchFieldDataType.Int64), + Type t when t == typeof(List) => SearchFieldDataType.Collection(SearchFieldDataType.Int64), + Type t when t == typeof(float[]) => SearchFieldDataType.Collection(SearchFieldDataType.Double), + Type t when t == typeof(List) => SearchFieldDataType.Collection(SearchFieldDataType.Double), + Type t when t == typeof(double[]) => SearchFieldDataType.Collection(SearchFieldDataType.Double), + Type t when t == typeof(List) => SearchFieldDataType.Collection(SearchFieldDataType.Double), + Type t when t == typeof(DateTime[]) => SearchFieldDataType.Collection(SearchFieldDataType.DateTimeOffset), + Type t when t == typeof(List) => SearchFieldDataType.Collection(SearchFieldDataType.DateTimeOffset), + Type t when t == typeof(DateTimeOffset[]) => SearchFieldDataType.Collection(SearchFieldDataType.DateTimeOffset), + Type t when t == typeof(List) => SearchFieldDataType.Collection(SearchFieldDataType.DateTimeOffset), +#if NET + Type t when t == typeof(DateOnly[]) => SearchFieldDataType.Collection(SearchFieldDataType.DateTimeOffset), + Type t when t == typeof(List) => SearchFieldDataType.Collection(SearchFieldDataType.DateTimeOffset), +#endif + + _ => throw new NotSupportedException($"Data type '{propertyType}' for {nameof(VectorStoreDataProperty)} is not supported by the Azure AI Search VectorStore.") + }; +} diff --git a/MEVD/src/AzureAISearch/AzureAISearchCollectionOptions.cs b/MEVD/src/AzureAISearch/AzureAISearchCollectionOptions.cs new file mode 100644 index 0000000..fb15950 --- /dev/null +++ b/MEVD/src/AzureAISearch/AzureAISearchCollectionOptions.cs @@ -0,0 +1,35 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text.Json; +using Azure.Search.Documents.Indexes; +using Microsoft.Extensions.VectorData; + +namespace CommunityToolkit.VectorData.AzureAISearch; + +/// +/// Options when creating a . +/// +public sealed class AzureAISearchCollectionOptions : VectorStoreCollectionOptions +{ + internal static readonly AzureAISearchCollectionOptions Default = new(); + + /// + /// Initializes a new instance of the class. + /// + public AzureAISearchCollectionOptions() + { + } + + internal AzureAISearchCollectionOptions(AzureAISearchCollectionOptions? source) : base(source) + { + JsonSerializerOptions = source?.JsonSerializerOptions; + } + + /// + /// Gets or sets the JSON serializer options to use when converting between the data model and the Azure AI Search record. + /// Note that when using the default mapper and you are constructing your own , you will need + /// to provide the same set of both here and when constructing the . + /// + public JsonSerializerOptions? JsonSerializerOptions { get; set; } +} diff --git a/MEVD/src/AzureAISearch/AzureAISearchConstants.cs b/MEVD/src/AzureAISearch/AzureAISearchConstants.cs new file mode 100644 index 0000000..40a5c0d --- /dev/null +++ b/MEVD/src/AzureAISearch/AzureAISearchConstants.cs @@ -0,0 +1,9 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace CommunityToolkit.VectorData.AzureAISearch; + +internal static class AzureAISearchConstants +{ + internal const string VectorStoreSystemName = "azure.aisearch"; +} diff --git a/MEVD/src/AzureAISearch/AzureAISearchDynamicCollection.cs b/MEVD/src/AzureAISearch/AzureAISearchDynamicCollection.cs new file mode 100644 index 0000000..63d9174 --- /dev/null +++ b/MEVD/src/AzureAISearch/AzureAISearchDynamicCollection.cs @@ -0,0 +1,34 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using Azure.Search.Documents.Indexes; + +namespace CommunityToolkit.VectorData.AzureAISearch; + +/// +/// Represents a collection of vector store records in a AzureAISearch database, mapped to a dynamic Dictionary<string, object?>. +/// +#pragma warning disable CA1711 // Identifiers should not have incorrect suffix +public sealed class AzureAISearchDynamicCollection : AzureAISearchCollection> +#pragma warning restore CA1711 // Identifiers should not have incorrect suffix +{ + /// + /// Initializes a new instance of the class. + /// + /// Azure AI Search client that can be used to manage the list of indices in an Azure AI Search Service. + /// The name of the collection. + /// Optional configuration options for this class. + [RequiresUnreferencedCode("The Azure AI Search provider is currently incompatible with trimming.")] + [RequiresDynamicCode("The Azure AI Search provider is currently incompatible with NativeAOT.")] + public AzureAISearchDynamicCollection(SearchIndexClient searchIndexClient, string name, AzureAISearchCollectionOptions options) + : base( + searchIndexClient, + name, + static options => new AzureAISearchDynamicModelBuilder().BuildDynamic( + options.Definition ?? throw new ArgumentException("Definition is required for dynamic collections"), + options.EmbeddingGenerator), + options) + { + } +} diff --git a/MEVD/src/AzureAISearch/AzureAISearchDynamicMapper.cs b/MEVD/src/AzureAISearch/AzureAISearchDynamicMapper.cs new file mode 100644 index 0000000..7ecec53 --- /dev/null +++ b/MEVD/src/AzureAISearch/AzureAISearchDynamicMapper.cs @@ -0,0 +1,282 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using System.Text.Json; +using System.Text.Json.Nodes; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData.ProviderServices; +using MEAI = Microsoft.Extensions.AI; +using Microsoft.Shared.Diagnostics; + +namespace CommunityToolkit.VectorData.AzureAISearch; + +/// +/// A mapper that maps between the generic Semantic Kernel data model and the model that the data is stored under, within Azure AI Search. +/// +internal sealed class AzureAISearchDynamicMapper(CollectionModel model, JsonSerializerOptions? jsonSerializerOptions) : IAzureAISearchMapper> +{ + /// + public JsonObject MapFromDataToStorageModel(Dictionary dataModel, int recordIndex, IReadOnlyList?[]? generatedEmbeddings) + { + Throw.IfNull(dataModel); + + var jsonObject = new JsonObject + { + [model.KeyProperty.StorageName] = !dataModel.TryGetValue(model.KeyProperty.ModelName, out var keyValue) + ? throw new InvalidOperationException($"Missing value for key property '{model.KeyProperty.ModelName}") + : keyValue switch + { + string s => s, + Guid g => g.ToString(), + + null => throw new InvalidOperationException($"Key property '{model.KeyProperty.ModelName}' is null."), + _ => throw new InvalidCastException($"Key property '{model.KeyProperty.ModelName}' must be a string or Guid.") + } + }; + + foreach (var dataProperty in model.DataProperties) + { + if (dataModel.TryGetValue(dataProperty.ModelName, out var dataValue)) + { + jsonObject[dataProperty.StorageName] = dataValue is not null ? + JsonSerializer.SerializeToNode(ConvertToStorageValue(dataValue), GetStorageType(dataProperty.Type), jsonSerializerOptions) : + null; + } + } + + for (var i = 0; i < model.VectorProperties.Count; i++) + { + var property = model.VectorProperties[i]; + + // Don't create a property if it doesn't exist in the dictionary + if (dataModel.TryGetValue(property.ModelName, out var vectorValue)) + { + var vector = generatedEmbeddings?[i]?[recordIndex] is Embedding ge + ? ge + : vectorValue; + + if (vector is null) + { + jsonObject[property.StorageName] = null; + continue; + } + + var memory = vector switch + { + ReadOnlyMemory m => m, + Embedding e => e.Vector, + float[] a => a, + + _ => throw new UnreachableException() + }; + + var jsonArray = new JsonArray(); + + foreach (var item in memory.Span) + { + jsonArray.Add(JsonValue.Create(item)); + } + + jsonObject.Add(property.StorageName, jsonArray); + } + } + + return jsonObject; + } + + /// + public Dictionary MapFromStorageToDataModel(JsonObject storageModel, bool includeVectors) + { + Throw.IfNull(storageModel); + + // Create variables to store the response properties. + var result = new Dictionary(); + + // Loop through all known properties and map each from json to the data type. + foreach (var property in model.Properties) + { + switch (property) + { + case KeyPropertyModel keyProperty: + var key = (string?)storageModel[keyProperty.StorageName] + ?? throw new InvalidOperationException($"The key property '{keyProperty.StorageName}' is missing from the record retrieved from storage."); + + result[keyProperty.ModelName] = keyProperty.Type switch + { + var t when t == typeof(string) => key, + var t when t == typeof(Guid) => Guid.Parse(key), + _ => throw new UnreachableException() + }; + + continue; + + case DataPropertyModel dataProperty: + { + if (storageModel.TryGetPropertyValue(dataProperty.StorageName, out var value)) + { + result.Add(dataProperty.ModelName, value is null ? null : GetDataPropertyValue(property.Type, value)); + } + continue; + } + + case VectorPropertyModel vectorProperty when includeVectors: + { + if (storageModel.TryGetPropertyValue(vectorProperty.StorageName, out var value)) + { + if (value is null) + { + result.Add(vectorProperty.ModelName, null); + continue; + } + + var vector = value.AsArray().Select(x => (float)x!).ToArray(); + result.Add( + vectorProperty.ModelName, + (Nullable.GetUnderlyingType(vectorProperty.Type) ?? vectorProperty.Type) switch + { + Type t when t == typeof(ReadOnlyMemory) => new ReadOnlyMemory(vector), + Type t when t == typeof(Embedding) => new Embedding(vector), + Type t when t == typeof(float[]) => vector, + + _ => throw new UnreachableException() + }); + } + + continue; + } + + case VectorPropertyModel vectorProperty when !includeVectors: + break; + + default: + throw new UnreachableException(); + } + } + + return result; + } + + /// + /// Get the value of the given json node as the given property type. + /// + /// The type of property that is required. + /// The json node containing the property value. + /// The value of the json node as the required type. + private static object? GetDataPropertyValue(Type propertyType, JsonNode value) + { + var result = propertyType switch + { + Type t when t == typeof(string) => value.GetValue(), + Type t when t == typeof(int) => value.GetValue(), + Type t when t == typeof(int?) => value.GetValue(), + Type t when t == typeof(long) => value.GetValue(), + Type t when t == typeof(long?) => value.GetValue(), + Type t when t == typeof(float) => value.GetValue(), + Type t when t == typeof(float?) => value.GetValue(), + Type t when t == typeof(double) => value.GetValue(), + Type t when t == typeof(double?) => value.GetValue(), + Type t when t == typeof(bool) => value.GetValue(), + Type t when t == typeof(bool?) => value.GetValue(), + Type t when t == typeof(DateTime) => value.GetValue(), + Type t when t == typeof(DateTime?) => value.GetValue(), + Type t when t == typeof(DateTimeOffset) => value.GetValue(), + Type t when t == typeof(DateTimeOffset?) => value.GetValue(), +#if NET + Type t when t == typeof(DateOnly) => DateOnly.FromDateTime(value.GetValue().DateTime), + Type t when t == typeof(DateOnly?) => (DateOnly?)DateOnly.FromDateTime(value.GetValue().DateTime), +#endif + + _ => (object?)null + }; + + if (result is not null) + { + return result; + } + + if (propertyType.IsArray) + { + return propertyType switch + { + Type t when t == typeof(string[]) => value.AsArray().Select(x => (string?)x).ToArray(), + Type t when t == typeof(int[]) => value.AsArray().Select(x => (int)x!).ToArray(), + Type t when t == typeof(int?[]) => value.AsArray().Select(x => (int?)x).ToArray(), + Type t when t == typeof(long[]) => value.AsArray().Select(x => (long)x!).ToArray(), + Type t when t == typeof(long?[]) => value.AsArray().Select(x => (long?)x).ToArray(), + Type t when t == typeof(float[]) => value.AsArray().Select(x => (float)x!).ToArray(), + Type t when t == typeof(float?[]) => value.AsArray().Select(x => (float?)x).ToArray(), + Type t when t == typeof(double[]) => value.AsArray().Select(x => (double)x!).ToArray(), + Type t when t == typeof(double?[]) => value.AsArray().Select(x => (double?)x).ToArray(), + Type t when t == typeof(bool[]) => value.AsArray().Select(x => (bool)x!).ToArray(), + Type t when t == typeof(bool?[]) => value.AsArray().Select(x => (bool?)x).ToArray(), + Type t when t == typeof(DateTime[]) => value.AsArray().Select(x => (DateTime)x!).ToArray(), + Type t when t == typeof(DateTime?[]) => value.AsArray().Select(x => (DateTime?)x).ToArray(), + Type t when t == typeof(DateTimeOffset[]) => value.AsArray().Select(x => (DateTimeOffset)x!).ToArray(), + Type t when t == typeof(DateTimeOffset?[]) => value.AsArray().Select(x => (DateTimeOffset?)x).ToArray(), + + _ => throw new UnreachableException($"Unsupported property type '{propertyType.Name}'.") + }; + } + + if (propertyType.IsGenericType && propertyType.GetGenericTypeDefinition() == typeof(List<>)) + { + return propertyType switch + { + Type t when t == typeof(List) => value.AsArray().Select(x => (string?)x).ToList(), + Type t when t == typeof(List) => value.AsArray().Select(x => (int)x!).ToList(), + Type t when t == typeof(List) => value.AsArray().Select(x => (int?)x).ToList(), + Type t when t == typeof(List) => value.AsArray().Select(x => (long)x!).ToList(), + Type t when t == typeof(List) => value.AsArray().Select(x => (long?)x).ToList(), + Type t when t == typeof(List) => value.AsArray().Select(x => (float)x!).ToList(), + Type t when t == typeof(List) => value.AsArray().Select(x => (float?)x).ToList(), + Type t when t == typeof(List) => value.AsArray().Select(x => (double)x!).ToList(), + Type t when t == typeof(List) => value.AsArray().Select(x => (double?)x).ToList(), + Type t when t == typeof(List) => value.AsArray().Select(x => (bool)x!).ToList(), + Type t when t == typeof(List) => value.AsArray().Select(x => (bool?)x).ToList(), + Type t when t == typeof(List) => value.AsArray().Select(x => (DateTime)x!).ToList(), + Type t when t == typeof(List) => value.AsArray().Select(x => (DateTime?)x).ToList(), + Type t when t == typeof(List) => value.AsArray().Select(x => (DateTimeOffset)x!).ToList(), + Type t when t == typeof(List) => value.AsArray().Select(x => (DateTimeOffset?)x).ToList(), + + _ => throw new UnreachableException($"Unsupported property type '{propertyType.Name}'.") + }; + } + + throw new UnreachableException($"Unsupported property type '{propertyType.Name}'."); + } + + /// + /// Converts a value to its storage representation for Azure AI Search. + /// Azure AI Search only supports for date/time types, and always internally + /// converts to UTC for storage. + /// + /// + /// Gets the storage type for a given property type. Azure AI Search stores DateTime and DateOnly as DateTimeOffset. + /// + private static Type GetStorageType(Type propertyType) + { + var underlying = Nullable.GetUnderlyingType(propertyType) ?? propertyType; + + if (underlying == typeof(DateTime) +#if NET + || underlying == typeof(DateOnly) +#endif + ) + { + return propertyType == underlying ? typeof(DateTimeOffset) : typeof(DateTimeOffset?); + } + + return propertyType; + } + + private static object ConvertToStorageValue(object value) + => value switch + { + DateTime dateTime => new DateTimeOffset(dateTime, TimeSpan.Zero), +#if NET + DateOnly dateOnly => new DateTimeOffset(dateOnly.ToDateTime(TimeOnly.MinValue), TimeSpan.Zero), +#endif + _ => value + }; +} diff --git a/MEVD/src/AzureAISearch/AzureAISearchDynamicModelBuilder.cs b/MEVD/src/AzureAISearch/AzureAISearchDynamicModelBuilder.cs new file mode 100644 index 0000000..bf8e01d --- /dev/null +++ b/MEVD/src/AzureAISearch/AzureAISearchDynamicModelBuilder.cs @@ -0,0 +1,37 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; + +using Microsoft.Extensions.VectorData.ProviderServices; + +namespace CommunityToolkit.VectorData.AzureAISearch; + +internal class AzureAISearchDynamicModelBuilder() : CollectionModelBuilder(s_modelBuildingOptions) +{ + internal static readonly CollectionModelBuildingOptions s_modelBuildingOptions = new() + { + RequiresAtLeastOneVector = false, + SupportsMultipleVectors = true, + UsesExternalSerializer = true + }; + + protected override void ValidateKeyProperty(KeyPropertyModel keyProperty) + { + base.ValidateKeyProperty(keyProperty); + + var type = keyProperty.Type; + + if (type != typeof(string) && type != typeof(Guid)) + { + throw new NotSupportedException( + $"Property '{keyProperty.ModelName}' has unsupported type '{type.Name}'. Key properties must be one of the supported types: string, Guid."); + } + } + + protected override bool IsDataPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes) + => AzureAISearchModelBuilder.IsDataPropertyTypeValidCore(type, out supportedTypes); + + protected override bool IsVectorPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes) + => AzureAISearchModelBuilder.IsVectorPropertyTypeValidCore(type, out supportedTypes); +} diff --git a/MEVD/src/AzureAISearch/AzureAISearchFilterTranslator.cs b/MEVD/src/AzureAISearch/AzureAISearchFilterTranslator.cs new file mode 100644 index 0000000..49c917f --- /dev/null +++ b/MEVD/src/AzureAISearch/AzureAISearchFilterTranslator.cs @@ -0,0 +1,372 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections; +using System.Linq.Expressions; +using System.Text; +using Microsoft.Extensions.VectorData.ProviderServices; +using Microsoft.Extensions.VectorData.ProviderServices.Filter; + +namespace CommunityToolkit.VectorData.AzureAISearch; + +internal class AzureAISearchFilterTranslator : FilterTranslatorBase +{ + private readonly StringBuilder _filter = new(); + + private static readonly char[] s_searchInDefaultDelimiter = [' ', ',']; + + internal string Translate(LambdaExpression lambdaExpression, CollectionModel model) + { + var preprocessedExpression = PreprocessFilter(lambdaExpression, model, new FilterPreprocessingOptions()); + + Translate(preprocessedExpression); + + return _filter.ToString(); + } + + private void Translate(Expression? node) + { + switch (node) + { + case BinaryExpression binary: + TranslateBinary(binary); + return; + + case ConstantExpression constant: + TranslateConstant(constant); + return; + + case MemberExpression member: + TranslateMember(member); + return; + + case MethodCallExpression methodCall: + TranslateMethodCall(methodCall); + return; + + case UnaryExpression unary: + TranslateUnary(unary); + return; + + default: + throw new NotSupportedException("Unsupported NodeType in filter: " + node?.NodeType); + } + } + + private void TranslateBinary(BinaryExpression binary) + { + _filter.Append('('); + Translate(binary.Left); + + _filter.Append(binary.NodeType switch + { + ExpressionType.Equal => " eq ", + ExpressionType.NotEqual => " ne ", + + ExpressionType.GreaterThan => " gt ", + ExpressionType.GreaterThanOrEqual => " ge ", + ExpressionType.LessThan => " lt ", + ExpressionType.LessThanOrEqual => " le ", + + ExpressionType.AndAlso => " and ", + ExpressionType.OrElse => " or ", + + _ => throw new NotSupportedException("Unsupported binary expression node type: " + binary.NodeType) + }); + + Translate(binary.Right); + _filter.Append(')'); + } + + private void TranslateConstant(ConstantExpression constant) + => GenerateLiteral(constant.Value); + + private void GenerateLiteral(object? value) + { + switch (value) + { + case byte b: + _filter.Append(b); + return; + case short s: + _filter.Append(s); + return; + case int i: + _filter.Append(i); + return; + case long l: + _filter.Append(l); + return; + + case float f: + _filter.Append(f); + return; + case double d: + _filter.Append(d); + return; + + case string untrustedInput: + // This is the only place where we allow untrusted input to be passed in, so we need to quote and escape it. + _filter.Append('\'').Append(untrustedInput.Replace("'", "''")).Append('\''); + return; + case bool b: + _filter.Append(b ? "true" : "false"); + return; + case Guid g: + _filter.Append('\'').Append(g.ToString()).Append('\''); + return; + + case DateTime d: + _filter.Append(new DateTimeOffset(d, TimeSpan.Zero).ToString("o")); + return; + case DateTimeOffset d: + _filter.Append(d.ToString("o")); + return; +#if NET + case DateOnly d: + _filter.Append(new DateTimeOffset(d.ToDateTime(TimeOnly.MinValue), TimeSpan.Zero).ToString("o")); + return; +#endif + + case Array: + throw new NotImplementedException(); + + case null: + _filter.Append("null"); + return; + + default: + throw new NotSupportedException("Unsupported constant type: " + value.GetType().Name); + } + } + + private void TranslateMember(MemberExpression memberExpression) + { + if (TryBindProperty(memberExpression, out var property)) + { + // OData identifiers cannot be escaped; storage names are validated during model building. + _filter.Append(property.StorageName); + return; + } + + throw new NotSupportedException($"Member access for '{memberExpression.Member.Name}' is unsupported - only member access over the filter parameter are supported"); + } + + private void TranslateMethodCall(MethodCallExpression methodCall) + { + // Dictionary access for dynamic mapping (r => r["SomeString"] == "foo") + if (TryBindProperty(methodCall, out var property)) + { + // OData identifiers cannot be escaped; storage names are validated during model building. + _filter.Append(property.StorageName); + return; + } + + switch (methodCall) + { + // Enumerable.Contains(), List.Contains(), MemoryExtensions.Contains() + case var _ when TryMatchContains(methodCall, out var source, out var item): + TranslateContains(source, item); + return; + + // Enumerable.Any() with a Contains predicate (r => r.Strings.Any(s => array.Contains(s))) + case { Method.Name: nameof(Enumerable.Any), Arguments: [var anySource, LambdaExpression lambda] } any + when any.Method.DeclaringType == typeof(Enumerable): + TranslateAny(anySource, lambda); + return; + + default: + throw new NotSupportedException($"Unsupported method call: {methodCall.Method.DeclaringType?.Name}.{methodCall.Method.Name}"); + } + } + + private void TranslateContains(Expression source, Expression item) + { + switch (source) + { + // Contains over array field (r => r.Strings.Contains("foo")) + case var _ when TryBindProperty(source, out _): + Translate(source); + _filter.Append("/any(t: t eq "); + Translate(item); + _filter.Append(')'); + return; + + // Contains over inline enumerable + case NewArrayExpression newArray: + var elements = ExtractArrayValues(newArray); + _filter.Append("search.in("); + Translate(item); + GenerateSearchInValues(elements); + return; + + case ConstantExpression { Value: IEnumerable enumerable and not string }: + _filter.Append("search.in("); + Translate(item); + GenerateSearchInValues(enumerable); + return; + + default: + throw new NotSupportedException("Unsupported Contains expression"); + } + } + + /// + /// Translates an Any() call with a Contains predicate, e.g. r.Strings.Any(s => array.Contains(s)). + /// This checks whether any element in the array field is contained in the given values. + /// + private void TranslateAny(Expression source, LambdaExpression lambda) + { + // We only support the pattern: r.ArrayField.Any(x => values.Contains(x)) + // Translates to: Field/any(t: search.in(t, 'value1, value2, value3')) + if (!TryBindProperty(source, out var property) + || lambda.Body is not MethodCallExpression { Method.Name: "Contains" } containsCall + || !TryMatchContains(containsCall, out var valuesExpression, out var itemExpression)) + { + throw new NotSupportedException("Unsupported method call: Enumerable.Any"); + } + + // Verify that the item is the lambda parameter + if (itemExpression != lambda.Parameters[0]) + { + throw new NotSupportedException("Unsupported method call: Enumerable.Any"); + } + + // Extract the values and generate the OData filter + IEnumerable values = valuesExpression switch + { + NewArrayExpression newArray => ExtractArrayValues(newArray), + ConstantExpression { Value: IEnumerable enumerable and not string } => enumerable, + _ => throw new NotSupportedException("Unsupported method call: Enumerable.Any") + }; + + // Generate: Field/any(t: search.in(t, 'value1, value2, value3')) + // OData identifiers cannot be escaped; storage names are validated during model building. + _filter.Append(property.StorageName); + _filter.Append("/any(t: search.in(t"); + GenerateSearchInValues(values); + _filter.Append(')'); + } + + /// + /// Generates the values portion of a search.in() call, including the comma, quotes, and optional delimiter. + /// Appends: , 'value1, value2, value3') or , 'value1|value2|value3', '|') + /// + private void GenerateSearchInValues(IEnumerable values) + { + _filter.Append(", '"); + + string delimiter = ", "; + var startingPosition = _filter.Length; + +RestartLoop: + var isFirst = true; + foreach (var element in values) + { + if (element is not string stringElement) + { + throw new NotSupportedException("search.in() over non-string arrays is not supported"); + } + + if (isFirst) + { + isFirst = false; + } + else + { + _filter.Append(delimiter); + } + + // The default delimiter for search.in() is comma or space. + // If any element contains a comma or space, we switch to using pipe as the delimiter. + // If any contains a pipe, we throw (for now). + switch (delimiter) + { + case ", ": + if (stringElement.IndexOfAny(s_searchInDefaultDelimiter) > -1) + { + delimiter = "|"; + _filter.Length = startingPosition; + goto RestartLoop; + } + + break; + + case "|": + if (stringElement.Contains('|')) + { + throw new NotSupportedException("Some elements contain both commas/spaces and pipes, cannot translate to search.in()"); + } + + break; + } + + _filter.Append(stringElement.Replace("'", "''")); + } + + _filter.Append('\''); + + if (delimiter != ", ") + { + _filter + .Append(", '") + .Append(delimiter) + .Append('\''); + } + + _filter.Append(')'); + } + + private static object?[] ExtractArrayValues(NewArrayExpression newArray) + { + var result = new object?[newArray.Expressions.Count]; + for (var i = 0; i < newArray.Expressions.Count; i++) + { + if (newArray.Expressions[i] is not ConstantExpression { Value: var elementValue }) + { + throw new NotSupportedException("Invalid element in array"); + } + + result[i] = elementValue; + } + + return result; + } + + private void TranslateUnary(UnaryExpression unary) + { + switch (unary.NodeType) + { + case ExpressionType.Not: + // Special handling for !(a == b) and !(a != b) + if (unary.Operand is BinaryExpression { NodeType: ExpressionType.Equal or ExpressionType.NotEqual } binary) + { + TranslateBinary( + Expression.MakeBinary( + binary.NodeType is ExpressionType.Equal ? ExpressionType.NotEqual : ExpressionType.Equal, + binary.Left, + binary.Right)); + return; + } + + _filter.Append("(not "); + Translate(unary.Operand); + _filter.Append(')'); + return; + + // Handle converting non-nullable to nullable; such nodes are found in e.g. r => r.Int == nullableInt + case ExpressionType.Convert when Nullable.GetUnderlyingType(unary.Type) == unary.Operand.Type: + Translate(unary.Operand); + return; + + // Handle convert over member access, for dynamic dictionary access (r => (int)r["SomeInt"] == 8) + case ExpressionType.Convert when TryBindProperty(unary.Operand, out var property) && unary.Type == property.Type: + // OData identifiers cannot be escaped; storage names are validated during model building. + _filter.Append(property.StorageName); + return; + + default: + throw new NotSupportedException("Unsupported unary expression node type: " + unary.NodeType); + } + } +} diff --git a/MEVD/src/AzureAISearch/AzureAISearchMapper.cs b/MEVD/src/AzureAISearch/AzureAISearchMapper.cs new file mode 100644 index 0000000..401021d --- /dev/null +++ b/MEVD/src/AzureAISearch/AzureAISearchMapper.cs @@ -0,0 +1,134 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using System.Text.Json; +using System.Text.Json.Nodes; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData.ProviderServices; +using MEAI = Microsoft.Extensions.AI; +using Microsoft.Shared.Diagnostics; + +namespace CommunityToolkit.VectorData.AzureAISearch; + +internal sealed class AzureAISearchMapper(CollectionModel model, JsonSerializerOptions? jsonSerializerOptions) : IAzureAISearchMapper + where TRecord : class +{ + public JsonObject MapFromDataToStorageModel(TRecord dataModel, int recordIndex, IReadOnlyList?[]? generatedEmbeddings) + { + Throw.IfNull(dataModel); + + var jsonObject = JsonSerializer.SerializeToNode(dataModel, jsonSerializerOptions)!.AsObject(); + + // Azure AI Search only supports Edm.DateTimeOffset for date/time types. + // DateTime properties may be serialized by STJ without timezone info (when Kind=Unspecified), + // which Azure AI Search rejects. Convert them to DateTimeOffset (UTC). + // DateOnly properties are serialized as "yyyy-MM-dd", which also isn't accepted. + // Convert them to full DateTimeOffset representation (midnight UTC). + foreach (var dataProperty in model.DataProperties) + { + var propertyType = Nullable.GetUnderlyingType(dataProperty.Type) ?? dataProperty.Type; + + if (propertyType == typeof(DateTime) && jsonObject.TryGetPropertyValue(dataProperty.StorageName, out var dtNode) && dtNode is not null) + { + var dateTime = dtNode.Deserialize(jsonSerializerOptions); + jsonObject[dataProperty.StorageName] = JsonValue.Create(new DateTimeOffset(DateTime.SpecifyKind(dateTime, DateTimeKind.Utc), TimeSpan.Zero)); + } +#if NET + else if (propertyType == typeof(DateOnly) && jsonObject.TryGetPropertyValue(dataProperty.StorageName, out var dateNode) && dateNode is not null) + { + var dateOnly = dateNode.Deserialize(jsonSerializerOptions); + jsonObject[dataProperty.StorageName] = JsonValue.Create(new DateTimeOffset(dateOnly.ToDateTime(TimeOnly.MinValue), TimeSpan.Zero)); + } +#endif + } + + // Go over the vector properties; inject any generated embeddings to overwrite the JSON serialized above. + // Also, for Embedding properties we also need to overwrite with a simple array (since Embedding gets serialized as a complex object). + for (var i = 0; i < model.VectorProperties.Count; i++) + { + var property = model.VectorProperties[i]; + + Embedding? embedding = generatedEmbeddings?[i]?[recordIndex] is Embedding ge ? (Embedding)ge : null; + + if (embedding is null) + { + switch (Nullable.GetUnderlyingType(property.Type) ?? property.Type) + { + case var t when t == typeof(ReadOnlyMemory): + case var t2 when t2 == typeof(float[]): + // The .NET vector property is a ReadOnlyMemory or float[] (not an Embedding), which means that JsonSerializer + // already serialized it correctly above. + // In addition, there's no generated embedding (which would be an Embedding which we'd need to handle manually). + // So there's nothing for us to do. + continue; + + case var t when t == typeof(Embedding): + embedding = (Embedding)property.GetValueAsObject(dataModel)!; + break; + + default: + throw new UnreachableException(); + } + } + + var jsonArray = new JsonArray(); + + foreach (var item in embedding.Vector.Span) + { + jsonArray.Add(JsonValue.Create(item)); + } + + jsonObject[property.StorageName] = jsonArray; + } + + return jsonObject; + } + + public TRecord MapFromStorageToDataModel(JsonObject storageModel, bool includeVectors) + { + // Azure AI Search stores DateTime properties as Edm.DateTimeOffset. + // Convert them back to DateTime so STJ can deserialize them into the correct type. + // DateOnly properties also need to be converted back from DateTimeOffset. + foreach (var dataProperty in model.DataProperties) + { + var propertyType = Nullable.GetUnderlyingType(dataProperty.Type) ?? dataProperty.Type; + + if (propertyType == typeof(DateTime) && storageModel.TryGetPropertyValue(dataProperty.StorageName, out var dtNode) && dtNode is not null) + { + var dateTimeOffset = dtNode.Deserialize(jsonSerializerOptions); + storageModel[dataProperty.StorageName] = JsonValue.Create(dateTimeOffset.UtcDateTime); + } +#if NET + else if (propertyType == typeof(DateOnly) && storageModel.TryGetPropertyValue(dataProperty.StorageName, out var dateNode) && dateNode is not null) + { + var dateTimeOffset = dateNode.Deserialize(jsonSerializerOptions); + storageModel[dataProperty.StorageName] = JsonValue.Create(DateOnly.FromDateTime(dateTimeOffset.DateTime)); + } +#endif + } + + if (includeVectors) + { + foreach (var vectorProperty in model.VectorProperties) + { + // If the vector property .NET type is Embedding, we need to create the JSON structure for it + // (JSON array embedded inside an object representing the embedding), so that the deserialization below + // works correctly. + if (vectorProperty.Type == typeof(Embedding)) + { + var arrayNode = storageModel[vectorProperty.StorageName]; + if (arrayNode is not null) + { + storageModel[vectorProperty.StorageName] = new JsonObject + { + [nameof(Embedding.Vector)] = arrayNode.DeepClone() + }; + } + } + } + } + + return storageModel.Deserialize(jsonSerializerOptions)!; + } +} diff --git a/MEVD/src/AzureAISearch/AzureAISearchModelBuilder.cs b/MEVD/src/AzureAISearch/AzureAISearchModelBuilder.cs new file mode 100644 index 0000000..5723764 --- /dev/null +++ b/MEVD/src/AzureAISearch/AzureAISearchModelBuilder.cs @@ -0,0 +1,124 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; + +namespace CommunityToolkit.VectorData.AzureAISearch; + +internal class AzureAISearchModelBuilder() : CollectionJsonModelBuilder(s_modelBuildingOptions) +{ + internal const string SupportedVectorTypes = "ReadOnlyMemory, Embedding, float[]"; + + internal static readonly CollectionModelBuildingOptions s_modelBuildingOptions = new() + { + RequiresAtLeastOneVector = false, + SupportsMultipleVectors = true, + UsesExternalSerializer = true + }; + + protected override void ValidateKeyProperty(KeyPropertyModel keyProperty) + { + base.ValidateKeyProperty(keyProperty); + + var type = keyProperty.Type; + + if (type != typeof(string) && type != typeof(Guid)) + { + throw new NotSupportedException( + $"Property '{keyProperty.ModelName}' has unsupported type '{type.Name}'. Key properties must be one of the supported types: string, Guid."); + } + } + + protected override bool IsDataPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes) + => IsDataPropertyTypeValidCore(type, out supportedTypes); + + protected override bool IsVectorPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes) + => IsVectorPropertyTypeValidCore(type, out supportedTypes); + + internal static bool IsDataPropertyTypeValidCore(Type type, [NotNullWhen(false)] out string? supportedTypes) + { + supportedTypes = "string, int, long, double, float, bool, DateTime, DateTimeOffset," +#if NET + + " DateOnly," +#endif + + " or arrays/lists of these types"; + + if (Nullable.GetUnderlyingType(type) is Type underlyingType) + { + type = underlyingType; + } + + return IsValid(type) + || (type.IsArray && IsValid(type.GetElementType()!)) + || (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(List<>) && IsValid(type.GenericTypeArguments[0])); + + static bool IsValid(Type type) + => type == typeof(string) || + type == typeof(int) || + type == typeof(long) || + type == typeof(double) || + type == typeof(float) || + type == typeof(bool) || + type == typeof(DateTime) || +#if NET + type == typeof(DateOnly) || +#endif + type == typeof(DateTimeOffset); + } + + internal static bool IsVectorPropertyTypeValidCore(Type type, [NotNullWhen(false)] out string? supportedTypes) + { + // Azure AI Search is adding support for more types than just float32, but these are not available for use via the + // SDK yet. We will update this list as the SDK is updated. + // + supportedTypes = SupportedVectorTypes; + + return type == typeof(ReadOnlyMemory) + || type == typeof(ReadOnlyMemory?) + || type == typeof(Embedding) + || type == typeof(float[]); + } + + /// + protected override void ValidateProperty(PropertyModel propertyModel, VectorStoreCollectionDefinition? definition) + { + base.ValidateProperty(propertyModel, definition); + + // OData identifiers cannot be escaped; storage names are validated during model building. + // See https://docs.oasis-open.org/odata/odata/v4.01/odata-v4.01-part2-url-conventions.html#sec_ODataIdentifier + if (!IsValidIdentifier(propertyModel.StorageName)) + { + throw new InvalidOperationException( + $"Property '{propertyModel.ModelName}' has storage name '{propertyModel.StorageName}' which is not a valid OData identifier. " + + "OData identifiers must start with a letter or underscore, and contain only letters, digits, and underscores."); + } + } + + private static bool IsValidIdentifier(string name) + { + if (string.IsNullOrEmpty(name)) + { + return false; + } + + var first = name[0]; + if (!char.IsLetter(first) && first != '_') + { + return false; + } + + for (var i = 1; i < name.Length; i++) + { + var c = name[i]; + if (!char.IsLetterOrDigit(c) && c != '_') + { + return false; + } + } + + return true; + } +} diff --git a/MEVD/src/AzureAISearch/AzureAISearchServiceCollectionExtensions.cs b/MEVD/src/AzureAISearch/AzureAISearchServiceCollectionExtensions.cs new file mode 100644 index 0000000..a672582 --- /dev/null +++ b/MEVD/src/AzureAISearch/AzureAISearchServiceCollectionExtensions.cs @@ -0,0 +1,379 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using System.Text.Json; +using Azure; +using Azure.Core; +using Azure.Core.Serialization; +using Azure.Search.Documents; +using Azure.Search.Documents.Indexes; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData; +using CommunityToolkit.VectorData.AzureAISearch; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.DependencyInjection; + +/// +/// Extension methods to register and instances on an . +/// +public static class AzureAISearchServiceCollectionExtensions +{ + private const string ApplicationId = "CommunityToolkit.MEVD"; + private const string DynamicCodeMessage = "The Azure AI Search provider is currently incompatible with trimming."; + private const string UnreferencedCodeMessage = "The Azure AI Search provider is currently incompatible with NativeAOT."; + + /// + /// Registers a as + /// with returned by + /// or retrieved from the dependency injection container if was not provided. + /// + /// + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddAzureAISearchVectorStore( + this IServiceCollection services, + Func? clientProvider = default, + Func? optionsProvider = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + => AddKeyedAzureAISearchVectorStore(services, serviceKey: null, clientProvider, optionsProvider, lifetime); + + /// + /// Registers a keyed as + /// with returned by + /// or retrieved from the dependency injection container if was not provided. + /// + /// The to register the on. + /// The key with which to associate the vector store. + /// The provider. + /// Options provider to further configure the . + /// The service lifetime for the store. Defaults to . + /// Service collection. + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddKeyedAzureAISearchVectorStore( + this IServiceCollection services, + object? serviceKey, + Func? clientProvider = default, + Func? optionsProvider = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + { + Throw.IfNull(services); + + services.Add(new ServiceDescriptor(typeof(AzureAISearchVectorStore), serviceKey, (sp, _) => + { + var client = clientProvider is not null ? clientProvider(sp) : sp.GetRequiredService(); + var options = GetStoreOptions(sp, optionsProvider); + + return new AzureAISearchVectorStore(client, options); + }, lifetime)); + + services.Add(new ServiceDescriptor(typeof(VectorStore), serviceKey, + static (sp, key) => sp.GetRequiredKeyedService(key), lifetime)); + + return services; + } + + /// + /// Registers a as + /// using the provided and . + /// + /// + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddAzureAISearchVectorStore( + this IServiceCollection services, + Uri endpoint, + TokenCredential tokenCredential, + AzureAISearchVectorStoreOptions? options = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + => AddKeyedAzureAISearchVectorStore(services, serviceKey: null, endpoint, tokenCredential, options, lifetime); + + /// + /// Registers a keyed as + /// using the provided and . + /// + /// The to register the on. + /// The key with which to associate the vector store. + /// The service endpoint for Azure AI Search. + /// The credential to authenticate to Azure AI Search with. + /// Optional options to further configure the . + /// The service lifetime for the store. Defaults to . + /// Service collection. + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddKeyedAzureAISearchVectorStore( + this IServiceCollection services, + object? serviceKey, + Uri endpoint, + TokenCredential tokenCredential, + AzureAISearchVectorStoreOptions? options = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + { + Throw.IfNull(endpoint); + Throw.IfNull(tokenCredential); + + return AddKeyedAzureAISearchVectorStore(services, serviceKey, sp => + { + var searchClientOptions = BuildSearchClientOptions(options?.JsonSerializerOptions); + return new SearchIndexClient(endpoint, tokenCredential, searchClientOptions); + }, _ => options!, lifetime); + } + + /// + /// Registers a as + /// using the provided and . + /// + /// + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddAzureAISearchVectorStore( + this IServiceCollection services, + Uri endpoint, + AzureKeyCredential keyCredential, + AzureAISearchVectorStoreOptions? options = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + => AddKeyedAzureAISearchVectorStore(services, serviceKey: null, endpoint, keyCredential, options, lifetime); + + /// + /// Registers a keyed as + /// using the provided and . + /// + /// The to register the on. + /// The key with which to associate the vector store. + /// The service endpoint for Azure AI Search. + /// The credential to authenticate to Azure AI Search with. + /// Optional options to further configure the . + /// The service lifetime for the store. Defaults to . + /// Service collection. + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddKeyedAzureAISearchVectorStore( + this IServiceCollection services, + object? serviceKey, + Uri endpoint, + AzureKeyCredential keyCredential, + AzureAISearchVectorStoreOptions? options = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + { + Throw.IfNull(endpoint); + Throw.IfNull(keyCredential); + + return AddKeyedAzureAISearchVectorStore(services, serviceKey, sp => + { + var searchClientOptions = BuildSearchClientOptions(options?.JsonSerializerOptions); + return new SearchIndexClient(endpoint, keyCredential, searchClientOptions); + }, _ => options!, lifetime); + } + + /// + /// Registers a as + /// with returned by + /// or retrieved from the dependency injection container if was not provided. + /// + /// + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddAzureAISearchCollection( + this IServiceCollection services, + string name, + Func? clientProvider = default, + Func? optionsProvider = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + where TRecord : class + => AddKeyedAzureAISearchCollection(services, serviceKey: null, name, clientProvider, optionsProvider, lifetime); + + /// + /// Registers a keyed as + /// with returned by + /// or retrieved from the dependency injection container if was not provided. + /// + /// The to register the on. + /// The key with which to associate the collection. + /// The name of the collection. + /// The provider. + /// Options provider to further configure the . + /// The service lifetime for the store. Defaults to . + /// Service collection. + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddKeyedAzureAISearchCollection( + this IServiceCollection services, + object? serviceKey, + string name, + Func? clientProvider = default, + Func? optionsProvider = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + where TRecord : class + { + Throw.IfNull(services); + Throw.IfNullOrWhitespace(name); + + services.Add(new ServiceDescriptor(typeof(AzureAISearchCollection), serviceKey, (sp, _) => + { + var client = clientProvider is not null ? clientProvider(sp) : sp.GetRequiredService(); + var options = GetCollectionOptions(sp, optionsProvider); + + return new AzureAISearchCollection(client, name, options); + }, lifetime)); + + services.Add(new ServiceDescriptor(typeof(VectorStoreCollection), serviceKey, + static (sp, key) => sp.GetRequiredKeyedService>(key), lifetime)); + + services.Add(new ServiceDescriptor(typeof(IVectorSearchable), serviceKey, + static (sp, key) => sp.GetRequiredKeyedService>(key), lifetime)); + + services.Add(new ServiceDescriptor(typeof(IKeywordHybridSearchable), serviceKey, + static (sp, key) => sp.GetRequiredKeyedService>(key), lifetime)); + + return services; + } + + /// + /// Registers a as + /// using the provided and . + /// + /// + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddAzureAISearchCollection( + this IServiceCollection services, + string name, + Uri endpoint, + TokenCredential tokenCredential, + AzureAISearchCollectionOptions? options = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + where TRecord : class + => AddKeyedAzureAISearchCollection(services, serviceKey: null, name, endpoint, tokenCredential, options, lifetime); + + /// + /// Registers a keyed as + /// using the provided and . + /// + /// The to register the on. + /// The key with which to associate the collection. + /// The name of the collection. + /// The service endpoint for Azure AI Search. + /// The credential to authenticate to Azure AI Search with. + /// Optional options to further configure the . + /// The service lifetime for the store. Defaults to . + /// Service collection. + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddKeyedAzureAISearchCollection( + this IServiceCollection services, + object? serviceKey, + string name, + Uri endpoint, + TokenCredential tokenCredential, + AzureAISearchCollectionOptions? options = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + where TRecord : class + { + Throw.IfNull(endpoint); + Throw.IfNull(tokenCredential); + + return AddKeyedAzureAISearchCollection(services, serviceKey, name, sp => + { + var searchClientOptions = BuildSearchClientOptions(options?.JsonSerializerOptions); + return new SearchIndexClient(endpoint, tokenCredential, searchClientOptions); + }, _ => options!, lifetime); + } + + /// + /// Registers a as + /// using the provided and . + /// + /// + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddAzureAISearchCollection( + this IServiceCollection services, + string name, + Uri endpoint, + AzureKeyCredential keyCredential, + AzureAISearchCollectionOptions? options = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + where TRecord : class + => AddKeyedAzureAISearchCollection(services, serviceKey: null, name, endpoint, keyCredential, options, lifetime); + + /// + /// Registers a keyed as + /// using the provided and . + /// + /// The to register the on. + /// The key with which to associate the collection. + /// The name of the collection. + /// The service endpoint for Azure AI Search. + /// The credential to authenticate to Azure AI Search with. + /// Optional options to further configure the . + /// The service lifetime for the store. Defaults to . + /// Service collection. + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddKeyedAzureAISearchCollection( + this IServiceCollection services, + object? serviceKey, + string name, + Uri endpoint, + AzureKeyCredential keyCredential, + AzureAISearchCollectionOptions? options = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + where TRecord : class + { + Throw.IfNull(endpoint); + Throw.IfNull(keyCredential); + + return AddKeyedAzureAISearchCollection(services, serviceKey, name, sp => + { + var searchClientOptions = BuildSearchClientOptions(options?.JsonSerializerOptions); + return new SearchIndexClient(endpoint, keyCredential, searchClientOptions); + }, _ => options!, lifetime); + } + + /// + /// Build a instance, using the provided if it's not null and add the SK user agent string. + /// + /// Optional to add to the options if provided. + /// The . + private static SearchClientOptions BuildSearchClientOptions(JsonSerializerOptions? jsonSerializerOptions) + { + var searchClientOptions = new SearchClientOptions(); + searchClientOptions.Diagnostics.ApplicationId = ApplicationId; + if (jsonSerializerOptions != null) + { + searchClientOptions.Serializer = new JsonObjectSerializer(jsonSerializerOptions); + } + + return searchClientOptions; + } + + private static AzureAISearchVectorStoreOptions? GetStoreOptions(IServiceProvider sp, Func? optionsProvider) + { + var options = optionsProvider?.Invoke(sp); + if (options?.EmbeddingGenerator is not null) + { + return options; // The user has provided everything, there is nothing to change. + } + + var embeddingGenerator = sp.GetService(); + return embeddingGenerator is null + ? options // There is nothing to change. + : new(options) { EmbeddingGenerator = embeddingGenerator }; // Create a brand new copy in order to avoid modifying the original options. + } + + private static AzureAISearchCollectionOptions? GetCollectionOptions(IServiceProvider sp, Func? optionsProvider) + { + var options = optionsProvider?.Invoke(sp); + if (options?.EmbeddingGenerator is not null) + { + return options; // The user has provided everything, there is nothing to change. + } + + var embeddingGenerator = sp.GetService(); + return embeddingGenerator is null + ? options // There is nothing to change. + : new(options) { EmbeddingGenerator = embeddingGenerator }; // Create a brand new copy in order to avoid modifying the original options. + } +} diff --git a/MEVD/src/AzureAISearch/AzureAISearchVectorStore.cs b/MEVD/src/AzureAISearch/AzureAISearchVectorStore.cs new file mode 100644 index 0000000..b73866c --- /dev/null +++ b/MEVD/src/AzureAISearch/AzureAISearchVectorStore.cs @@ -0,0 +1,142 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; +using System.Text.Json; +using Azure; +using Azure.Search.Documents.Indexes; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; +using Microsoft.Shared.Diagnostics; +using static Microsoft.Extensions.VectorData.VectorStoreErrorHandler; + +namespace CommunityToolkit.VectorData.AzureAISearch; + +/// +/// Class for accessing the list of collections in a Azure AI Search vector store. +/// +/// +/// This class can be used with collections of any schema type, but requires you to provide schema information when getting a collection. +/// +public sealed class AzureAISearchVectorStore : VectorStore +{ + /// Metadata about vector store. + private readonly VectorStoreMetadata _metadata; + + /// Azure AI Search client that can be used to manage the list of indices in an Azure AI Search Service. + private readonly SearchIndexClient _searchIndexClient; + + private readonly IEmbeddingGenerator? _embeddingGenerator; + private readonly JsonSerializerOptions? _jsonSerializerOptions; + + /// A general purpose definition that can be used to construct a collection when needing to proxy schema agnostic operations. + private static readonly VectorStoreCollectionDefinition s_generalPurposeDefinition = new() { Properties = [new VectorStoreKeyProperty("Key", typeof(string))] }; + + /// + /// Initializes a new instance of the class. + /// + /// Azure AI Search client that can be used to manage the list of indices in an Azure AI Search Service. + /// Optional configuration options for this class. + [RequiresUnreferencedCode("The Azure AI Search provider is currently incompatible with trimming.")] + [RequiresDynamicCode("The Azure AI Search provider is currently incompatible with NativeAOT.")] + public AzureAISearchVectorStore(SearchIndexClient searchIndexClient, AzureAISearchVectorStoreOptions? options = default) + { + Throw.IfNull(searchIndexClient); + + _searchIndexClient = searchIndexClient; + _embeddingGenerator = options?.EmbeddingGenerator; + _jsonSerializerOptions = options?.JsonSerializerOptions; + + _metadata = new() + { + VectorStoreSystemName = AzureAISearchConstants.VectorStoreSystemName, + VectorStoreName = searchIndexClient.ServiceName + }; + } + +#pragma warning disable IDE0090 // Use 'new(...)' + /// + [RequiresDynamicCode("This overload of GetCollection() is incompatible with NativeAOT. For dynamic mapping via Dictionary, call GetDynamicCollection() instead.")] + [RequiresUnreferencedCode("This overload of GetCollecttion() is incompatible with trimming. For dynamic mapping via Dictionary, call GetDynamicCollection() instead.")] +#if NET + public override AzureAISearchCollection GetCollection(string name, VectorStoreCollectionDefinition? definition = null) +#else + public override VectorStoreCollection GetCollection(string name, VectorStoreCollectionDefinition? definition = null) +#endif + => typeof(TRecord) == typeof(Dictionary) + ? throw new ArgumentException(VectorDataStrings.GetCollectionWithDictionaryNotSupported) + : new AzureAISearchCollection( + _searchIndexClient, + name, + new AzureAISearchCollectionOptions() + { + JsonSerializerOptions = _jsonSerializerOptions, + Definition = definition, + EmbeddingGenerator = _embeddingGenerator + }); + + /// + [RequiresUnreferencedCode("The Azure AI Search provider is currently incompatible with trimming.")] + [RequiresDynamicCode("The Azure AI Search provider is currently incompatible with NativeAOT.")] +#if NET + public override AzureAISearchDynamicCollection GetDynamicCollection(string name, VectorStoreCollectionDefinition definition) +#else + public override VectorStoreCollection> GetDynamicCollection(string name, VectorStoreCollectionDefinition definition) +#endif + => new AzureAISearchDynamicCollection( + _searchIndexClient, + name, + new() + { + JsonSerializerOptions = _jsonSerializerOptions, + Definition = definition, + EmbeddingGenerator = _embeddingGenerator + } + ); +#pragma warning restore IDE0090 + + /// + public override async IAsyncEnumerable ListCollectionNamesAsync([EnumeratorCancellation] CancellationToken cancellationToken = default) + { + const string OperationName = "GetIndexNames"; + + var indexNamesEnumerable = _searchIndexClient.GetIndexNamesAsync(cancellationToken).ConfigureAwait(false); + var errorHandlingEnumerable = new ConfiguredCancelableErrorHandlingAsyncEnumerable(indexNamesEnumerable, _metadata, OperationName); + +#pragma warning disable CA2007 // Consider calling ConfigureAwait on the awaited task: False Positive + await foreach (var item in errorHandlingEnumerable.ConfigureAwait(false)) +#pragma warning restore CA2007 // Consider calling ConfigureAwait on the awaited task + { + yield return item; + } + } + + /// + public override Task CollectionExistsAsync(string name, CancellationToken cancellationToken = default) + { + var collection = GetDynamicCollection(name, s_generalPurposeDefinition); + return collection.CollectionExistsAsync(cancellationToken); + } + + /// + public override Task EnsureCollectionDeletedAsync(string name, CancellationToken cancellationToken = default) + { + var collection = GetDynamicCollection(name, s_generalPurposeDefinition); + return collection.EnsureCollectionDeletedAsync(cancellationToken); + } + + /// + public override object? GetService(Type serviceType, object? serviceKey = null) + { + Throw.IfNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreMetadata) ? _metadata : + serviceType == typeof(SearchIndexClient) ? _searchIndexClient : + serviceType.IsInstanceOfType(this) ? this : + null; + } +} diff --git a/MEVD/src/AzureAISearch/AzureAISearchVectorStoreOptions.cs b/MEVD/src/AzureAISearch/AzureAISearchVectorStoreOptions.cs new file mode 100644 index 0000000..5127cec --- /dev/null +++ b/MEVD/src/AzureAISearch/AzureAISearchVectorStoreOptions.cs @@ -0,0 +1,39 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text.Json; +using Azure.Search.Documents.Indexes; +using Microsoft.Extensions.AI; + +namespace CommunityToolkit.VectorData.AzureAISearch; + +/// +/// Options when creating a . +/// +public sealed class AzureAISearchVectorStoreOptions +{ + /// + /// Initializes a new instance of the class. + /// + public AzureAISearchVectorStoreOptions() + { + } + + internal AzureAISearchVectorStoreOptions(AzureAISearchVectorStoreOptions? source) + { + JsonSerializerOptions = source?.JsonSerializerOptions; + EmbeddingGenerator = source?.EmbeddingGenerator; + } + + /// + /// Gets or sets the JSON serializer options to use when converting between the data model and the Azure AI Search record. + /// Note that when using the default mapper and you are constructing your own , you will need + /// to provide the same set of both here and when constructing the . + /// + public JsonSerializerOptions? JsonSerializerOptions { get; set; } + + /// + /// Gets or sets the default embedding generator to use when generating vectors embeddings with this vector store. + /// + public IEmbeddingGenerator? EmbeddingGenerator { get; set; } +} diff --git a/MEVD/src/AzureAISearch/IAzureAISearchMapper.cs b/MEVD/src/AzureAISearch/IAzureAISearchMapper.cs new file mode 100644 index 0000000..88f95a7 --- /dev/null +++ b/MEVD/src/AzureAISearch/IAzureAISearchMapper.cs @@ -0,0 +1,20 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text.Json.Nodes; +using MEAI = Microsoft.Extensions.AI; + +namespace CommunityToolkit.VectorData.AzureAISearch; + +internal interface IAzureAISearchMapper +{ + /// + /// Maps from the consumer record data model to the storage model. + /// + JsonObject MapFromDataToStorageModel(TRecord dataModel, int recordIndex, IReadOnlyList?[]? generatedEmbeddings); + + /// + /// Maps from the storage model to the consumer record data model. + /// + TRecord MapFromStorageToDataModel(JsonObject storageModel, bool includeVectors); +} diff --git a/MEVD/src/CosmosMongoDB/AssemblyInfo.cs b/MEVD/src/CosmosMongoDB/AssemblyInfo.cs new file mode 100644 index 0000000..d6395e8 --- /dev/null +++ b/MEVD/src/CosmosMongoDB/AssemblyInfo.cs @@ -0,0 +1,2 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. diff --git a/MEVD/src/CosmosMongoDB/CosmosMongoCollection.cs b/MEVD/src/CosmosMongoDB/CosmosMongoCollection.cs new file mode 100644 index 0000000..766b533 --- /dev/null +++ b/MEVD/src/CosmosMongoDB/CosmosMongoCollection.cs @@ -0,0 +1,649 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Linq.Expressions; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; +using MongoDB.Bson; +using MongoDB.Bson.Serialization; +using MongoDB.Driver; +using MEVD = Microsoft.Extensions.VectorData; +using Microsoft.Shared.Diagnostics; + +namespace CommunityToolkit.VectorData.CosmosMongoDB; + +/// +/// Service for storing and retrieving vector records, that uses Azure CosmosDB MongoDB as the underlying storage. +/// +/// The data type of the record key. Must be either . +/// The data model to use for adding, updating and retrieving data from storage. +#pragma warning disable CA1711 // Identifiers should not have incorrect suffix +public class CosmosMongoCollection : VectorStoreCollection + where TKey : notnull + where TRecord : class +#pragma warning restore CA1711 // Identifiers should not have incorrect suffix +{ + /// Metadata about vector store record collection. + private readonly VectorStoreCollectionMetadata _collectionMetadata; + + /// Property name to be used for search similarity score value. + private const string ScorePropertyName = "similarityScore"; + + /// Property name to be used for search document value. + private const string DocumentPropertyName = "document"; + + /// The default options for vector search. + private static readonly MEVD.VectorSearchOptions s_defaultVectorSearchOptions = new(); + + /// that can be used to manage the collections in Azure CosmosDB MongoDB. + private readonly IMongoDatabase _mongoDatabase; + + /// Azure CosmosDB MongoDB collection to perform record operations. + private readonly IMongoCollection _mongoCollection; + + /// Interface for mapping between a storage model, and the consumer record data model. + private readonly IMongoMapper _mapper; + + /// The model for this collection. + private readonly CollectionModel _model; + + /// + public override string Name { get; } + + /// This integer is the number of clusters that the inverted file (IVF) index uses to group the vector data. + private readonly int _numLists; + + /// The size of the dynamic candidate list for constructing the graph. + private readonly int _efConstruction; + + /// The size of the dynamic candidate list for search. + private readonly int _efSearch; + + /// to use for serializing key values. + private readonly BsonSerializationInfo? _keySerializationInfo; + + private static readonly Type[] s_validKeyTypes = [typeof(string), typeof(Guid), typeof(ObjectId), typeof(int), typeof(long)]; + + /// + /// Initializes a new instance of the class. + /// + /// that can be used to manage the collections in Azure CosmosDB MongoDB. + /// The name of the collection that this will access. + /// Optional configuration options for this class. + [RequiresDynamicCode("This constructor is incompatible with NativeAOT. For dynamic mapping via Dictionary, instantiate CosmosMongoDynamicCollection instead.")] + [RequiresUnreferencedCode("This constructor is incompatible with trimming. For dynamic mapping via Dictionary, instantiate CosmosMongoDynamicCollection instead.")] + public CosmosMongoCollection( + IMongoDatabase mongoDatabase, + string name, + CosmosMongoCollectionOptions? options = default) + : this( + mongoDatabase, + name, + static options => typeof(TRecord) == typeof(Dictionary) + ? throw new NotSupportedException(VectorDataStrings.NonDynamicCollectionWithDictionaryNotSupported(typeof(CosmosMongoDynamicCollection))) + : new MongoModelBuilder().Build(typeof(TRecord), typeof(TKey), options.Definition, options.EmbeddingGenerator), + options) + { + } + + internal CosmosMongoCollection(IMongoDatabase mongoDatabase, string name, Func modelFactory, CosmosMongoCollectionOptions? options) + { + // Verify. + Throw.IfNull(mongoDatabase); + Throw.IfNullOrWhitespace(name); + + if (!s_validKeyTypes.Contains(typeof(TKey)) && typeof(TKey) != typeof(object)) + { + throw new NotSupportedException("Only ObjectID, string, Guid, int and long keys are supported."); + } + + options ??= CosmosMongoCollectionOptions.Default; + + // Assign. + _mongoDatabase = mongoDatabase; + _mongoCollection = mongoDatabase.GetCollection(name); + Name = name; + _model = modelFactory(options); + _numLists = options.NumLists; + _efConstruction = options.EfConstruction; + _efSearch = options.EfSearch; + + _mapper = typeof(TRecord) == typeof(Dictionary) + ? (new MongoDynamicMapper(_model) as IMongoMapper)! + : new MongoMapper(_model); + + _collectionMetadata = new() + { + VectorStoreSystemName = CosmosMongoConstants.VectorStoreSystemName, + VectorStoreName = mongoDatabase.DatabaseNamespace?.DatabaseName, + CollectionName = name + }; + + // Cache the key serialization info if possible + _keySerializationInfo = typeof(TKey) == typeof(object) + ? null + : GetKeySerializationInfo(); + } + + /// + public override Task CollectionExistsAsync(CancellationToken cancellationToken = default) + => RunOperationAsync("ListCollectionNames", () => InternalCollectionExistsAsync(cancellationToken)); + + /// + public override async Task EnsureCollectionExistsAsync(CancellationToken cancellationToken = default) + { + await RunOperationAsync("CreateCollection", + () => _mongoDatabase.CreateCollectionAsync(Name, cancellationToken: cancellationToken)).ConfigureAwait(false); + + await RunOperationAsync("CreateIndexes", + () => CreateIndexesAsync(Name, cancellationToken: cancellationToken)).ConfigureAwait(false); + } + + /// + public override async Task DeleteAsync(TKey key, CancellationToken cancellationToken = default) + { + Throw.IfNull(key); + + await RunOperationAsync("DeleteOne", () => _mongoCollection.DeleteOneAsync(GetFilterById(key), cancellationToken)) + .ConfigureAwait(false); + } + + /// + public override async Task DeleteAsync(IEnumerable keys, CancellationToken cancellationToken = default) + { + Throw.IfNull(keys); + + await RunOperationAsync("DeleteMany", () => _mongoCollection.DeleteManyAsync(GetFilterByIds(keys), cancellationToken)) + .ConfigureAwait(false); + } + + /// + public override Task EnsureCollectionDeletedAsync(CancellationToken cancellationToken = default) + => RunOperationAsync("DropCollection", () => _mongoDatabase.DropCollectionAsync(Name, cancellationToken)); + + /// + public override async Task GetAsync(TKey key, RecordRetrievalOptions? options = null, CancellationToken cancellationToken = default) + { + Throw.IfNull(key); + + var includeVectors = options?.IncludeVectors ?? false; + if (includeVectors && _model.EmbeddingGenerationRequired) + { + throw new NotSupportedException(VectorDataStrings.IncludeVectorsNotSupportedWithEmbeddingGeneration); + } + + using var cursor = await + FindAsync(GetFilterById(key), top: 1, skip: null, includeVectors, sortDefinition: null, cancellationToken) + .ConfigureAwait(false); + + var record = await cursor.SingleOrDefaultAsync(cancellationToken).ConfigureAwait(false); + + if (record is null) + { + return default; + } + + return _mapper.MapFromStorageToDataModel(record, includeVectors); + } + + /// + public override async IAsyncEnumerable GetAsync( + IEnumerable keys, + RecordRetrievalOptions? options = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + Throw.IfNull(keys); + + var includeVectors = options?.IncludeVectors ?? false; + if (includeVectors && _model.EmbeddingGenerationRequired) + { + throw new NotSupportedException(VectorDataStrings.IncludeVectorsNotSupportedWithEmbeddingGeneration); + } + + using var cursor = await + FindAsync(GetFilterByIds(keys), top: null, skip: null, includeVectors, sortDefinition: null, cancellationToken) + .ConfigureAwait(false); + + while (await cursor.MoveNextAsync(cancellationToken).ConfigureAwait(false)) + { + foreach (var record in cursor.Current) + { + if (record is not null) + { + yield return _mapper.MapFromStorageToDataModel(record, includeVectors); + } + } + } + } + + /// + public override async Task UpsertAsync(TRecord record, CancellationToken cancellationToken = default) + { + Throw.IfNull(record); + + (_, var generatedEmbeddings) = await ProcessEmbeddingsAsync(_model, [record], cancellationToken).ConfigureAwait(false); + + await UpsertCoreAsync(record, recordIndex: 0, generatedEmbeddings, cancellationToken).ConfigureAwait(false); + } + + /// + public override async Task UpsertAsync(IEnumerable records, CancellationToken cancellationToken = default) + { + Throw.IfNull(records); + + (records, var generatedEmbeddings) = await ProcessEmbeddingsAsync(_model, records, cancellationToken).ConfigureAwait(false); + + var i = 0; + + foreach (var record in records) + { + await UpsertCoreAsync(record, i++, generatedEmbeddings, cancellationToken).ConfigureAwait(false); + } + } + + private async Task UpsertCoreAsync(TRecord record, int recordIndex, IReadOnlyList?[]? generatedEmbeddings, CancellationToken cancellationToken = default) + { + const string OperationName = "ReplaceOne"; + + // Handle auto-generated keys + var keyProperty = _model.KeyProperty; + if (keyProperty.IsAutoGenerated) + { + switch (keyProperty.Type) + { + case var t when t == typeof(Guid): + if (keyProperty.GetValue(record) == Guid.Empty) + { + keyProperty.SetValue(record, Guid.NewGuid()); + } + break; + + case var t when t == typeof(ObjectId): + if (keyProperty.GetValue(record) == ObjectId.Empty) + { + keyProperty.SetValue(record, ObjectId.GenerateNewId()); + } + break; + + default: + throw new UnreachableException(); + } + } + + var replaceOptions = new ReplaceOptions { IsUpsert = true }; + var storageModel = _mapper.MapFromDataToStorageModel(record, recordIndex, generatedEmbeddings); + + var key = GetStorageKey(storageModel); + + await RunOperationAsync(OperationName, async () => + await _mongoCollection + .ReplaceOneAsync(GetFilterById(key), storageModel, replaceOptions, cancellationToken) + .ConfigureAwait(false)).ConfigureAwait(false); + } + + private static TKey GetStorageKey(BsonDocument document) + => (TKey)BsonTypeMapper.MapToDotNetValue(document[MongoConstants.MongoReservedKeyPropertyName]); + + private static async ValueTask<(IEnumerable records, IReadOnlyList?[]?)> ProcessEmbeddingsAsync( + CollectionModel model, + IEnumerable records, + CancellationToken cancellationToken) + { + IReadOnlyList? recordsList = null; + + // If an embedding generator is defined, invoke it once per property for all records. + IReadOnlyList?[]? generatedEmbeddings = null; + + var vectorPropertyCount = model.VectorProperties.Count; + for (var i = 0; i < vectorPropertyCount; i++) + { + var vectorProperty = model.VectorProperties[i]; + + if (MongoModelBuilder.IsVectorPropertyTypeValidCore(vectorProperty.Type, out _)) + { + continue; + } + + // We have a vector property whose type isn't natively supported - we need to generate embeddings. + Debug.Assert(vectorProperty.EmbeddingGenerator is not null); + + // We have a property with embedding generation; materialize the records' enumerable if needed, to + // prevent multiple enumeration. + if (recordsList is null) + { + recordsList = records is IReadOnlyList r ? r : records.ToList(); + + if (recordsList.Count == 0) + { + return (records, null); + } + + records = recordsList; + } + + // TODO: Ideally we'd group together vector properties using the same generator (and with the same input and output properties), + // and generate embeddings for them in a single batch. That's some more complexity though. + generatedEmbeddings ??= new IReadOnlyList?[vectorPropertyCount]; + generatedEmbeddings[i] = await vectorProperty.GenerateEmbeddingsAsync(records.Select(r => vectorProperty.GetValueAsObject(r)), cancellationToken).ConfigureAwait(false); + } + + return (records, generatedEmbeddings); + } + + #region Search + + /// + public override async IAsyncEnumerable> SearchAsync( + TInput searchValue, + int top, + MEVD.VectorSearchOptions? options = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + Throw.IfNull(searchValue); + Throw.IfLessThan(top, 1); + + options ??= s_defaultVectorSearchOptions; + if (options.IncludeVectors && _model.EmbeddingGenerationRequired) + { + throw new NotSupportedException(VectorDataStrings.IncludeVectorsNotSupportedWithEmbeddingGeneration); + } + + var vectorProperty = _model.GetVectorPropertyOrSingle(options); + + float[] vector = searchValue switch + { + ReadOnlyMemory r => Unwrap(r), + float[] f => f, + Embedding e => Unwrap(e.Vector), + + _ when vectorProperty.EmbeddingGenerationDispatcher is not null + => Unwrap(((Embedding)await vectorProperty.GenerateEmbeddingAsync(searchValue, cancellationToken).ConfigureAwait(false)).Vector), + + _ => vectorProperty.EmbeddingGenerator is null + ? throw new NotSupportedException(VectorDataStrings.InvalidSearchInputAndNoEmbeddingGeneratorWasConfigured(searchValue.GetType(), MongoModelBuilder.SupportedVectorTypes)) + : throw new InvalidOperationException(VectorDataStrings.IncompatibleEmbeddingGeneratorWasConfiguredForInputType(typeof(TInput), vectorProperty.EmbeddingGenerator.GetType())) + }; + + var filter = options.Filter is not null + ? new CosmosMongoFilterTranslator().Translate(options.Filter, _model) + : null; + + // Constructing a query to fetch "skip + top" total items + // to perform skip logic locally, since skip option is not part of API. + var itemsAmount = options.Skip + top; + + var vectorPropertyIndexKind = CosmosMongoCollectionSearchMapping.GetVectorPropertyIndexKind(vectorProperty.IndexKind); + + var searchQuery = vectorPropertyIndexKind switch + { + IndexKind.Hnsw => CosmosMongoCollectionSearchMapping.GetSearchQueryForHnswIndex( + vector, + vectorProperty.StorageName, + itemsAmount, + _efSearch, + filter), + IndexKind.IvfFlat => CosmosMongoCollectionSearchMapping.GetSearchQueryForIvfIndex( + vector, + vectorProperty.StorageName, + itemsAmount, + filter), + _ => throw new InvalidOperationException( + $"Index kind '{vectorProperty.IndexKind}' on {nameof(VectorStoreVectorProperty)} '{vectorProperty.StorageName}' is not supported by the Azure CosmosDB for MongoDB VectorStore. " + + $"Supported index kinds are: {string.Join(", ", [IndexKind.Hnsw, IndexKind.IvfFlat])}") + }; + + var projectionQuery = CosmosMongoCollectionSearchMapping.GetProjectionQuery( + ScorePropertyName, + DocumentPropertyName); + + List pipeline = [searchQuery, projectionQuery]; + + // Add score threshold filter as a $match stage if specified + if (options.ScoreThreshold.HasValue) + { + pipeline.Add(CosmosMongoCollectionSearchMapping.GetScoreThresholdMatchQuery(ScorePropertyName, options.ScoreThreshold.Value)); + } + + const string OperationName = "Aggregate"; + var cursor = await RunOperationAsync( + OperationName, + () => _mongoCollection.AggregateAsync(pipeline, cancellationToken: cancellationToken)).ConfigureAwait(false); + using var errorHandlingAsyncCursor = new ErrorHandlingAsyncCursor(cursor, _collectionMetadata, OperationName); + + await foreach (var result in EnumerateAndMapSearchResultsAsync(errorHandlingAsyncCursor, options, cancellationToken).ConfigureAwait(false)) + { + yield return result; + } + + static float[] Unwrap(ReadOnlyMemory memory) + => MemoryMarshal.TryGetArray(memory, out ArraySegment segment) && segment.Count == segment.Array!.Length + ? segment.Array + : memory.ToArray(); + } + + #endregion Search + + /// + public override object? GetService(Type serviceType, object? serviceKey = null) + { + Throw.IfNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreCollectionMetadata) ? _collectionMetadata : + serviceType == typeof(IMongoDatabase) ? _mongoDatabase : + serviceType == typeof(IMongoCollection) ? _mongoCollection : + serviceType.IsInstanceOfType(this) ? this : + null; + } + + /// + public override async IAsyncEnumerable GetAsync( + Expression> filter, + int top, + FilteredRecordRetrievalOptions? options = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + Throw.IfNull(filter); + Throw.IfLessThan(top, 1); + + options ??= new(); + + // Translate the filter now, so if it fails, we throw immediately. + var translatedFilter = new CosmosMongoFilterTranslator().Translate(filter, _model); + + SortDefinition? sortDefinition = null; + var orderBy = options.OrderBy?.Invoke(new()).Values; + if (orderBy is { Count: > 0 }) + { + sortDefinition = Builders.Sort.Combine( + orderBy.Select(pair => + { + var storageName = _model.GetDataOrKeyProperty(pair.PropertySelector).StorageName; + + return pair.Ascending + ? Builders.Sort.Ascending(storageName) + : Builders.Sort.Descending(storageName); + })); + } + + using IAsyncCursor cursor = await FindAsync( + translatedFilter, + top, + options.Skip, + options.IncludeVectors, + sortDefinition, + cancellationToken).ConfigureAwait(false); + + while (await cursor.MoveNextAsync(cancellationToken).ConfigureAwait(false)) + { + foreach (var response in cursor.Current) + { + var record = _mapper.MapFromStorageToDataModel(response, options.IncludeVectors); + + yield return record; + } + } + } + + #region private + + private async Task CreateIndexesAsync(string collectionName, CancellationToken cancellationToken) + { + const string OperationName = "CreateIndexes"; + + var indexCursor = await _mongoCollection.Indexes.ListAsync(cancellationToken).ConfigureAwait(false); + var indexes = indexCursor.ToList(cancellationToken).Select(index => index["name"].ToString()) ?? []; + var uniqueIndexes = new HashSet(indexes); + + var indexArray = new BsonArray(); + + indexArray.AddRange(CosmosMongoCollectionCreateMapping.GetVectorIndexes( + _model.VectorProperties, + uniqueIndexes, + _numLists, + _efConstruction)); + + indexArray.AddRange(CosmosMongoCollectionCreateMapping.GetFilterableDataIndexes( + _model.DataProperties, + uniqueIndexes)); + + if (indexArray.Count > 0) + { + var createIndexCommand = new BsonDocument + { + { "createIndexes", collectionName }, + { "indexes", indexArray } + }; + + var cursor = await RunOperationAsync(OperationName, () => + _mongoDatabase.RunCommandAsync(createIndexCommand, cancellationToken: cancellationToken)).ConfigureAwait(false); + } + } + + private async Task> FindAsync( + FilterDefinition filter, + int? top, + int? skip, + bool includeVectors, + SortDefinition? sortDefinition, + CancellationToken cancellationToken) + { + const string OperationName = "Find"; + + ProjectionDefinitionBuilder projectionBuilder = Builders.Projection; + ProjectionDefinition? projectionDefinition = null; + + if (!includeVectors && _model.VectorProperties.Count > 0) + { + foreach (var vectorProperty in _model.VectorProperties) + { + projectionDefinition = projectionDefinition is not null ? + projectionDefinition.Exclude(vectorProperty.StorageName) : + projectionBuilder.Exclude(vectorProperty.StorageName); + } + } + + var findOptions = projectionDefinition is not null ? + new FindOptions { Projection = projectionDefinition, Limit = top, Skip = skip, Sort = sortDefinition } : + new FindOptions { Limit = top, Skip = skip, Sort = sortDefinition }; + + var cursor = await RunOperationAsync(OperationName, () => + _mongoCollection.FindAsync(filter, findOptions, cancellationToken)).ConfigureAwait(false); + + return new ErrorHandlingAsyncCursor(cursor, _collectionMetadata, OperationName); + } + + private async IAsyncEnumerable> EnumerateAndMapSearchResultsAsync( + ErrorHandlingAsyncCursor cursor, + MEVD.VectorSearchOptions searchOptions, + [EnumeratorCancellation] CancellationToken cancellationToken) + { + var skipCounter = 0; + + while (await cursor.MoveNextAsync(cancellationToken).ConfigureAwait(false)) + { + foreach (var response in cursor.Current) + { + if (skipCounter >= searchOptions.Skip) + { + var score = response[ScorePropertyName].AsDouble; + var record = _mapper.MapFromStorageToDataModel(response[DocumentPropertyName].AsBsonDocument, includeVectors: searchOptions.IncludeVectors); + + yield return new VectorSearchResult(record, score); + } + + skipCounter++; + } + } + } + + private FilterDefinition GetFilterById(TKey id) + { + // Use cached key serialization info but fall back to BsonValueFactory for dynamic mapper. + var bsonValue = _keySerializationInfo?.SerializeValue(id) ?? BsonValueFactory.Create(id); + return Builders.Filter.Eq(MongoConstants.MongoReservedKeyPropertyName, bsonValue); + } + + private FilterDefinition GetFilterByIds(IEnumerable ids) + { + // Use cached key serialization info but fall back to BsonValueFactory for dynamic mapper. + var bsonValues = _keySerializationInfo?.SerializeValues(ids) ?? (BsonArray)BsonValueFactory.Create(ids); + return Builders.Filter.In(MongoConstants.MongoReservedKeyPropertyName, bsonValues); + } + + private BsonSerializationInfo GetKeySerializationInfo() + { + var documentSerializer = BsonSerializer.LookupSerializer(); + if (documentSerializer is null) + { + throw new InvalidOperationException($"BsonSerializer not found for type '{typeof(TRecord)}'"); + } + + if (documentSerializer is not IBsonDocumentSerializer bsonDocumentSerializer) + { + throw new InvalidOperationException($"BsonSerializer for type '{typeof(TRecord)}' does not implement IBsonDocumentSerializer"); + } + + if (!bsonDocumentSerializer.TryGetMemberSerializationInfo(_model.KeyProperty.ModelName, out var keySerializationInfo)) + { + throw new InvalidOperationException($"BsonSerializer for type '{typeof(TRecord)}' does not recognize key property {_model.KeyProperty.ModelName}"); + } + + return keySerializationInfo; + } + + private async Task InternalCollectionExistsAsync(CancellationToken cancellationToken) + { + var filter = new BsonDocument("name", Name); + var options = new ListCollectionNamesOptions { Filter = filter }; + + using var cursor = await _mongoDatabase.ListCollectionNamesAsync(options, cancellationToken: cancellationToken).ConfigureAwait(false); + + return await cursor.AnyAsync(cancellationToken).ConfigureAwait(false); + } + + private Task RunOperationAsync(string operationName, Func operation) + => VectorStoreErrorHandler.RunOperationAsync(_collectionMetadata, operationName, operation); + + private Task RunOperationAsync(string operationName, Func> operation) + => VectorStoreErrorHandler.RunOperationAsync(_collectionMetadata, operationName, operation); + + private string GetStringKey(TKey key) + { + Throw.IfNull(key); + + var stringKey = key as string ?? throw new UnreachableException("string key should have been validated during model building"); + + Throw.IfNullOrWhitespace(stringKey); + + return stringKey; + } + + #endregion +} diff --git a/MEVD/src/CosmosMongoDB/CosmosMongoCollectionCreateMapping.cs b/MEVD/src/CosmosMongoDB/CosmosMongoCollectionCreateMapping.cs new file mode 100644 index 0000000..8b3635c --- /dev/null +++ b/MEVD/src/CosmosMongoDB/CosmosMongoCollectionCreateMapping.cs @@ -0,0 +1,128 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; +using MongoDB.Bson; + +namespace CommunityToolkit.VectorData.CosmosMongoDB; + +/// +/// Contains mapping helpers to use when creating a collection in Azure CosmosDB MongoDB. +/// +internal static class CosmosMongoCollectionCreateMapping +{ + /// + /// Returns an array of indexes to create for vector properties. + /// + /// Collection of vector properties for index creation. + /// Collection of unique existing indexes to avoid creating duplicates. + /// Number of clusters that the inverted file (IVF) index uses to group the vector data. + /// The size of the dynamic candidate list for constructing the graph. + public static BsonArray GetVectorIndexes( + IReadOnlyList vectorProperties, + HashSet uniqueIndexes, + int numLists, + int efConstruction) + { + var indexArray = new BsonArray(); + + // Create separate index for each vector property + foreach (var property in vectorProperties) + { + var storageName = property.StorageName; + + // Use index name same as vector property name with underscore + var indexName = $"{storageName}_"; + + // If index already exists, proceed to the next vector property + if (uniqueIndexes.Contains(indexName)) + { + continue; + } + + // Otherwise, create a new index + var searchOptions = new BsonDocument + { + { "kind", GetIndexKind(property.IndexKind, storageName) }, + { "numLists", numLists }, + { "similarity", GetDistanceFunction(property.DistanceFunction, storageName) }, + { "dimensions", property.Dimensions }, + { "efConstruction", efConstruction } + }; + + var indexDocument = new BsonDocument + { + ["name"] = indexName, + ["key"] = new BsonDocument { [storageName] = "cosmosSearch" }, + ["cosmosSearchOptions"] = searchOptions + }; + + indexArray.Add(indexDocument); + } + + return indexArray; + } + + /// + /// Returns an array of indexes to create for filterable data properties. + /// + /// Collection of data properties for index creation. + /// Collection of unique existing indexes to avoid creating duplicates. + public static BsonArray GetFilterableDataIndexes( + IReadOnlyList dataProperties, + HashSet uniqueIndexes) + { + var indexArray = new BsonArray(); + + // Create separate index for each data property + foreach (var property in dataProperties) + { + if (property.IsIndexed) + { + // Use index name same as data property name with underscore + var indexName = $"{property.StorageName}_"; + + // If index already exists, proceed to the next data property + if (uniqueIndexes.Contains(indexName)) + { + continue; + } + + // Otherwise, create a new index + var indexDocument = new BsonDocument + { + ["name"] = indexName, + ["key"] = new BsonDocument { [property.StorageName] = 1 } + }; + + indexArray.Add(indexDocument); + } + } + + return indexArray; + } + + /// + /// More information about Azure CosmosDB for MongoDB index kinds here: . + /// + private static string GetIndexKind(string? indexKind, string vectorPropertyName) + => CosmosMongoCollectionSearchMapping.GetVectorPropertyIndexKind(indexKind) switch + { + IndexKind.Hnsw => "vector-hnsw", + IndexKind.IvfFlat => "vector-ivf", + _ => throw new NotSupportedException($"Index kind '{indexKind}' on {nameof(VectorStoreVectorProperty)} '{vectorPropertyName}' is not supported by the Azure CosmosDB for MongoDB VectorStore.") + }; + + /// + /// More information about Azure CosmosDB for MongoDB distance functions here: . + /// + private static string GetDistanceFunction(string? distanceFunction, string vectorPropertyName) + => CosmosMongoCollectionSearchMapping.GetVectorPropertyDistanceFunction(distanceFunction) switch + { + DistanceFunction.CosineDistance => "COS", + DistanceFunction.DotProductSimilarity => "IP", + DistanceFunction.EuclideanDistance => "L2", + _ => throw new NotSupportedException($"Distance function '{distanceFunction}' for {nameof(VectorStoreVectorProperty)} '{vectorPropertyName}' is not supported by the Azure CosmosDB for MongoDB VectorStore.") + }; +} diff --git a/MEVD/src/CosmosMongoDB/CosmosMongoCollectionOptions.cs b/MEVD/src/CosmosMongoDB/CosmosMongoCollectionOptions.cs new file mode 100644 index 0000000..0fc2d8e --- /dev/null +++ b/MEVD/src/CosmosMongoDB/CosmosMongoCollectionOptions.cs @@ -0,0 +1,49 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.VectorData; + +namespace CommunityToolkit.VectorData.CosmosMongoDB; + +/// +/// Options when creating a . +/// +public sealed class CosmosMongoCollectionOptions : VectorStoreCollectionOptions +{ + internal static readonly CosmosMongoCollectionOptions Default = new(); + + /// + /// Initializes a new instance of the class. + /// + public CosmosMongoCollectionOptions() + { + } + + internal CosmosMongoCollectionOptions(CosmosMongoCollectionOptions? source) : base(source) + { + NumLists = source?.NumLists ?? Default.NumLists; + EfConstruction = source?.EfConstruction ?? Default.EfConstruction; + EfSearch = source?.EfSearch ?? Default.EfSearch; + } + + /// + /// This integer is the number of clusters that the inverted file (IVF) index uses to group the vector data. Default is 1. + /// We recommend that numLists is set to documentCount/1000 for up to 1 million documents and to sqrt(documentCount) + /// for more than 1 million documents. Using a numLists value of 1 is akin to performing brute-force search, which has + /// limited performance. + /// + public int NumLists { get; set; } = 1; + + /// + /// The size of the dynamic candidate list for constructing the graph (64 by default, minimum value is 4, + /// maximum value is 1000). Higher ef_construction will result in better index quality and higher accuracy, but it will + /// also increase the time required to build the index. EfConstruction has to be at least 2 * m + /// + public int EfConstruction { get; set; } = 64; + + /// + /// The size of the dynamic candidate list for search (40 by default). A higher value provides better recall at + /// the cost of speed. + /// + public int EfSearch { get; set; } = 40; +} diff --git a/MEVD/src/CosmosMongoDB/CosmosMongoCollectionSearchMapping.cs b/MEVD/src/CosmosMongoDB/CosmosMongoCollectionSearchMapping.cs new file mode 100644 index 0000000..3aadb66 --- /dev/null +++ b/MEVD/src/CosmosMongoDB/CosmosMongoCollectionSearchMapping.cs @@ -0,0 +1,114 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.VectorData; +using MongoDB.Bson; + + +namespace CommunityToolkit.VectorData.CosmosMongoDB; + +/// +/// Contains mapping helpers to use when searching for documents using Azure CosmosDB MongoDB. +/// +internal static class CosmosMongoCollectionSearchMapping +{ + /// Returns index kind specified on vector property or default . + public static string GetVectorPropertyIndexKind(string? indexKind) => !string.IsNullOrWhiteSpace(indexKind) ? indexKind! : MongoConstants.DefaultIndexKind; + + /// Returns distance function specified on vector property or default . + public static string GetVectorPropertyDistanceFunction(string? distanceFunction) => !string.IsNullOrWhiteSpace(distanceFunction) ? distanceFunction! : MongoConstants.DefaultDistanceFunction; + + /// Returns search part of the search query for index kind. + public static BsonDocument GetSearchQueryForHnswIndex( + TVector vector, + string vectorPropertyName, + int limit, + int efSearch, + BsonDocument? filter) + { + var searchQuery = new BsonDocument + { + { "vector", BsonArray.Create(vector) }, + { "path", vectorPropertyName }, + { "k", limit }, + { "efSearch", efSearch } + }; + + if (filter is not null) + { + searchQuery["filter"] = filter; + } + + return new BsonDocument + { + { "$search", + new BsonDocument + { + { "cosmosSearch", searchQuery } + } + } + }; + } + + /// Returns search part of the search query for index kind. + public static BsonDocument GetSearchQueryForIvfIndex( + TVector vector, + string vectorPropertyName, + int limit, + BsonDocument? filter) + { + var searchQuery = new BsonDocument + { + { "vector", BsonArray.Create(vector) }, + { "path", vectorPropertyName }, + { "k", limit }, + }; + + if (filter is not null) + { + searchQuery["filter"] = filter; + } + + return new BsonDocument + { + { "$search", + new BsonDocument + { + { "cosmosSearch", searchQuery }, + { "returnStoredSource", true } + } + } + }; + } + + /// Returns projection part of the search query to return similarity score together with document. + public static BsonDocument GetProjectionQuery(string scorePropertyName, string documentPropertyName) + { + return new BsonDocument + { + { "$project", + new BsonDocument + { + { scorePropertyName, new BsonDocument { { "$meta", "searchScore" } } }, + { documentPropertyName, "$$ROOT" } + } + } + }; + } + + /// Returns a $match stage to filter results by score threshold. + /// + /// Cosmos MongoDB returns a similarity score where higher values mean more similar, + /// so we filter with $gte to keep results at or above the threshold. + /// + public static BsonDocument GetScoreThresholdMatchQuery(string scorePropertyName, double scoreThreshold) + => new() + { + { + "$match", new BsonDocument + { + { scorePropertyName, new BsonDocument { { "$gte", scoreThreshold } } } + } + } + }; +} diff --git a/MEVD/src/CosmosMongoDB/CosmosMongoConstants.cs b/MEVD/src/CosmosMongoDB/CosmosMongoConstants.cs new file mode 100644 index 0000000..83dd2e9 --- /dev/null +++ b/MEVD/src/CosmosMongoDB/CosmosMongoConstants.cs @@ -0,0 +1,10 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + + +namespace CommunityToolkit.VectorData.CosmosMongoDB; + +internal static class CosmosMongoConstants +{ + public const string VectorStoreSystemName = "azure.cosmosdbmongodb"; +} diff --git a/MEVD/src/CosmosMongoDB/CosmosMongoDB.csproj b/MEVD/src/CosmosMongoDB/CosmosMongoDB.csproj new file mode 100644 index 0000000..32af03c --- /dev/null +++ b/MEVD/src/CosmosMongoDB/CosmosMongoDB.csproj @@ -0,0 +1,26 @@ + + + + 1.0.0-preview.1 + CommunityToolkit.VectorData.CosmosMongoDB + $(AssemblyName) + net10.0;net8.0;netstandard2.1;net472 + true + + Azure CosmosDB MongoDB vCore provider for Microsoft.Extensions.VectorData + Azure CosmosDB MongoDB vCore provider for Microsoft.Extensions.VectorData by Semantic Kernel + + + + + + + + + + + + + + + diff --git a/MEVD/src/CosmosMongoDB/CosmosMongoDynamicCollection.cs b/MEVD/src/CosmosMongoDB/CosmosMongoDynamicCollection.cs new file mode 100644 index 0000000..b5b9a7d --- /dev/null +++ b/MEVD/src/CosmosMongoDB/CosmosMongoDynamicCollection.cs @@ -0,0 +1,33 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using MongoDB.Driver; + + +namespace CommunityToolkit.VectorData.CosmosMongoDB; + +/// +/// Represents a collection of vector store records in a Mongo database, mapped to a dynamic Dictionary<string, object?>. +/// +#pragma warning disable CA1711 // Identifiers should not have incorrect suffix +public sealed class CosmosMongoDynamicCollection : CosmosMongoCollection> +#pragma warning restore CA1711 // Identifiers should not have incorrect suffix +{ + /// + /// Initializes a new instance of the class. + /// + /// that can be used to manage the collections in MongoDB. + /// The name of the collection. + /// Optional configuration options for this class. + public CosmosMongoDynamicCollection(IMongoDatabase mongoDatabase, string name, CosmosMongoCollectionOptions options) + : base( + mongoDatabase, + name, + static options => new MongoModelBuilder() + .BuildDynamic( + options.Definition ?? throw new ArgumentException("Definition is required for dynamic collections"), + options.EmbeddingGenerator), + options) + { + } +} diff --git a/MEVD/src/CosmosMongoDB/CosmosMongoFilterTranslator.cs b/MEVD/src/CosmosMongoDB/CosmosMongoFilterTranslator.cs new file mode 100644 index 0000000..d3dc9e4 --- /dev/null +++ b/MEVD/src/CosmosMongoDB/CosmosMongoFilterTranslator.cs @@ -0,0 +1,208 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections; +using System.Diagnostics; +using System.Linq.Expressions; +using Microsoft.Extensions.VectorData.ProviderServices; +using Microsoft.Extensions.VectorData.ProviderServices.Filter; +using MongoDB.Bson; + +namespace CommunityToolkit.VectorData.CosmosMongoDB; + +// MongoDB query reference: https://www.mongodb.com/docs/manual/reference/operator/query +// Information specific to vector search pre-filter: https://www.mongodb.com/docs/atlas/atlas-vector-search/vector-search-stage/#atlas-vector-search-pre-filter +internal class CosmosMongoFilterTranslator : FilterTranslatorBase +{ + internal BsonDocument Translate(LambdaExpression lambdaExpression, CollectionModel model) + { + var preprocessedExpression = PreprocessFilter(lambdaExpression, model, new FilterPreprocessingOptions()); + + return Translate(preprocessedExpression); + } + + private BsonDocument Translate(Expression? node) + => node switch + { + BinaryExpression + { + NodeType: ExpressionType.Equal or ExpressionType.NotEqual + or ExpressionType.GreaterThan or ExpressionType.GreaterThanOrEqual + or ExpressionType.LessThan or ExpressionType.LessThanOrEqual + } binary + => TranslateEqualityComparison(binary), + + BinaryExpression { NodeType: ExpressionType.AndAlso or ExpressionType.OrElse } andOr + => TranslateAndOr(andOr), + UnaryExpression { NodeType: ExpressionType.Not } not + => TranslateNot(not), + // Handle converting non-nullable to nullable; such nodes are found in e.g. r => r.Int == nullableInt + UnaryExpression { NodeType: ExpressionType.Convert } convert when Nullable.GetUnderlyingType(convert.Type) == convert.Operand.Type + => Translate(convert.Operand), + + // Special handling for bool constant as the filter expression (r => r.Bool) + Expression when node.Type == typeof(bool) && TryBindProperty(node, out var property) + => GenerateEqualityComparison(property, value: true, ExpressionType.Equal), + // Handle true literal (r => true), which is useful for fetching all records + ConstantExpression { Value: true } + => [], + + MethodCallExpression methodCall => TranslateMethodCall(methodCall), + + _ => throw new NotSupportedException("The following NodeType is unsupported: " + node?.NodeType) + }; + + private BsonDocument TranslateEqualityComparison(BinaryExpression binary) + => TryBindProperty(binary.Left, out var property) && binary.Right is ConstantExpression { Value: var rightConstant } + ? GenerateEqualityComparison(property, rightConstant, binary.NodeType) + : TryBindProperty(binary.Right, out property) && binary.Left is ConstantExpression { Value: var leftConstant } + ? GenerateEqualityComparison(property, leftConstant, binary.NodeType) + : throw new NotSupportedException("Invalid equality/comparison"); + + private BsonDocument GenerateEqualityComparison(PropertyModel property, object? value, ExpressionType nodeType) + { + if (value is null) + { + throw new NotSupportedException("MongogDB does not support null checks in vector search pre-filters"); + } + + // Short form of equality (instead of $eq) + if (nodeType is ExpressionType.Equal) + { + return new BsonDocument { [property.StorageName] = BsonValueFactory.Create(value) }; + } + + var filterOperator = nodeType switch + { + ExpressionType.NotEqual => "$ne", + ExpressionType.GreaterThan => "$gt", + ExpressionType.GreaterThanOrEqual => "$gte", + ExpressionType.LessThan => "$lt", + ExpressionType.LessThanOrEqual => "$lte", + + _ => throw new UnreachableException() + }; + + return new BsonDocument { [property.StorageName] = new BsonDocument { [filterOperator] = BsonValueFactory.Create(value) } }; + } + + private BsonDocument TranslateAndOr(BinaryExpression andOr) + { + var mongoOperator = andOr.NodeType switch + { + ExpressionType.AndAlso => "$and", + ExpressionType.OrElse => "$or", + _ => throw new UnreachableException() + }; + + var (left, right) = (Translate(andOr.Left), Translate(andOr.Right)); + + var nestedLeft = left.ElementCount == 1 && left.Elements.First() is var leftElement && leftElement.Name == mongoOperator ? (BsonArray)leftElement.Value : null; + var nestedRight = right.ElementCount == 1 && right.Elements.First() is var rightElement && rightElement.Name == mongoOperator ? (BsonArray)rightElement.Value : null; + + switch ((nestedLeft, nestedRight)) + { + case (not null, not null): + nestedLeft.AddRange(nestedRight); + return left; + case (not null, null): + nestedLeft.Add(right); + return left; + case (null, not null): + nestedRight.Insert(0, left); + return right; + case (null, null): + return new BsonDocument { [mongoOperator] = new BsonArray([left, right]) }; + } + } + + private BsonDocument TranslateNot(UnaryExpression not) + { + switch (not.Operand) + { + // Special handling for !(a == b) and !(a != b) + case BinaryExpression { NodeType: ExpressionType.Equal or ExpressionType.NotEqual } binary: + return TranslateEqualityComparison( + Expression.MakeBinary( + binary.NodeType is ExpressionType.Equal ? ExpressionType.NotEqual : ExpressionType.Equal, + binary.Left, + binary.Right)); + + // Not over bool field (r => !r.Bool) + case var negated when negated.Type == typeof(bool) && TryBindProperty(negated, out var property): + return GenerateEqualityComparison(property, false, ExpressionType.Equal); + } + + var operand = Translate(not.Operand); + + // Identify NOT over $in, transform to $nin (https://www.mongodb.com/docs/manual/reference/operator/query/nin/#mongodb-query-op.-nin) + if (operand.ElementCount == 1 && operand.Elements.First() is { Name: var fieldName, Value: BsonDocument nested } && + nested.ElementCount == 1 && nested.Elements.First() is { Name: "$in", Value: BsonArray values }) + { + return new BsonDocument { [fieldName] = new BsonDocument { ["$nin"] = values } }; + } + + throw new NotSupportedException("MongogDB does not support the NOT operator in vector search pre-filters"); + } + + private BsonDocument TranslateMethodCall(MethodCallExpression methodCall) + { + return methodCall switch + { + // Enumerable.Contains(), List.Contains(), MemoryExtensions.Contains() + _ when TryMatchContains(methodCall, out var source, out var item) + => TranslateContains(source, item), + + _ => throw new NotSupportedException($"Unsupported method call: {methodCall.Method.DeclaringType?.Name}.{methodCall.Method.Name}") + }; + } + + private BsonDocument TranslateContains(Expression source, Expression item) + { + switch (source) + { + // Contains over array column (r => r.Strings.Contains("foo")) + case var _ when TryBindProperty(source, out _): + throw new NotSupportedException("MongoDB does not support Contains within array fields ($elemMatch) in vector search pre-filters"); + + // Contains over inline enumerable + case NewArrayExpression newArray: + var elements = new object?[newArray.Expressions.Count]; + + for (var i = 0; i < newArray.Expressions.Count; i++) + { + if (newArray.Expressions[i] is not ConstantExpression { Value: var elementValue }) + { + throw new NotSupportedException("Invalid element in array"); + } + + elements[i] = elementValue; + } + + return ProcessInlineEnumerable(elements, item); + + // Contains over captured enumerable (we inline) + case ConstantExpression { Value: IEnumerable enumerable and not string }: + return ProcessInlineEnumerable(enumerable, item); + + default: + throw new NotSupportedException("Unsupported Contains expression"); + } + + BsonDocument ProcessInlineEnumerable(IEnumerable elements, Expression item) + { + if (!TryBindProperty(item, out var property)) + { + throw new NotSupportedException("Unsupported item type in Contains"); + } + + return new BsonDocument + { + [property.StorageName] = new BsonDocument + { + ["$in"] = new BsonArray(from object? element in elements select BsonValueFactory.Create(element)) + } + }; + } + } +} diff --git a/MEVD/src/CosmosMongoDB/CosmosMongoServiceCollectionExtensions.cs b/MEVD/src/CosmosMongoDB/CosmosMongoServiceCollectionExtensions.cs new file mode 100644 index 0000000..3035de2 --- /dev/null +++ b/MEVD/src/CosmosMongoDB/CosmosMongoServiceCollectionExtensions.cs @@ -0,0 +1,332 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData; +using CommunityToolkit.VectorData.CosmosMongoDB; +using MongoDB.Driver; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.DependencyInjection; + +/// +/// Extension methods to register Azure CosmosDB MongoDB instances on an . +/// +public static class CosmosMongoServiceCollectionExtensions +{ + private const string DynamicCodeMessage = "This method is incompatible with NativeAOT, consult the documentation for adding collections in a way that's compatible with NativeAOT."; + private const string UnreferencedCodeMessage = "This method is incompatible with trimming, consult the documentation for adding collections in a way that's compatible with NativeAOT."; + private const string ApplicationId = "CommunityToolkit.MEVD"; + + /// + /// Registers a as + /// with retrieved from the dependency injection container. + /// + /// + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddCosmosMongoVectorStore( + this IServiceCollection services, + CosmosMongoVectorStoreOptions? options = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + => AddKeyedCosmosMongoVectorStore(services, serviceKey: null, options, lifetime); + + /// + /// Registers a keyed as + /// with retrieved from the dependency injection container. + /// + /// The to register the on. + /// The key with which to associate the vector store. + /// Optional options to further configure the . + /// The service lifetime for the store. Defaults to . + /// Service collection. + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddKeyedCosmosMongoVectorStore( + this IServiceCollection services, + object? serviceKey, + CosmosMongoVectorStoreOptions? options = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + { + Throw.IfNull(services); + + services.Add(new ServiceDescriptor(typeof(CosmosMongoVectorStore), serviceKey, (sp, _) => + { + var database = sp.GetRequiredService(); + options = GetStoreOptions(sp, _ => options); + + return new CosmosMongoVectorStore(database, options); + }, lifetime)); + + services.Add(new ServiceDescriptor(typeof(VectorStore), serviceKey, + static (sp, key) => sp.GetRequiredKeyedService(key), lifetime)); + + return services; + } + + /// + /// Registers a as + /// using the provided and . + /// + /// + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddCosmosMongoVectorStore( + this IServiceCollection services, + string connectionString, + string databaseName, + CosmosMongoVectorStoreOptions? options = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + => AddKeyedCosmosMongoVectorStore(services, serviceKey: null, connectionString, databaseName, options, lifetime); + + /// + /// Registers a keyed as + /// using the provided and . + /// + /// The to register the on. + /// The key with which to associate the vector store. + /// Connection string required to connect to Azure CosmosDB MongoDB. + /// Database name for Azure CosmosDB MongoDB. + /// Optional options to further configure the . + /// The service lifetime for the store. Defaults to . + /// Service collection. + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddKeyedCosmosMongoVectorStore( + this IServiceCollection services, + object? serviceKey, + string connectionString, + string databaseName, + CosmosMongoVectorStoreOptions? options = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + { + Throw.IfNull(services); + Throw.IfNullOrWhitespace(connectionString); + Throw.IfNullOrWhitespace(databaseName); + + services.Add(new ServiceDescriptor(typeof(CosmosMongoVectorStore), serviceKey, (sp, _) => + { + options = GetStoreOptions(sp, _ => options); + MongoClient mongoClient = new(CreateClientSettings(connectionString)); + var database = mongoClient.GetDatabase(databaseName); + + return new CosmosMongoVectorStore(database, options); + }, lifetime)); + + services.Add(new ServiceDescriptor(typeof(VectorStore), serviceKey, + static (sp, key) => sp.GetRequiredKeyedService(key), lifetime)); + + return services; + } + + /// + /// Registers a as + /// with retrieved from the dependency injection container. + /// + /// + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddCosmosMongoCollection( + this IServiceCollection services, + string name, + CosmosMongoCollectionOptions? options = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + where TRecord : class + => AddKeyedCosmosMongoCollection(services, serviceKey: null, name, options, lifetime); + + /// + /// Registers a keyed as + /// with retrieved from the dependency injection container. + /// + /// The type of the record. + /// The to register the on. + /// The key with which to associate the collection. + /// The name of the collection. + /// Optional options to further configure the . + /// The service lifetime for the store. Defaults to . + /// Service collection. + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddKeyedCosmosMongoCollection( + this IServiceCollection services, + object? serviceKey, + string name, + CosmosMongoCollectionOptions? options = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + where TRecord : class + { + Throw.IfNull(services); + Throw.IfNullOrWhitespace(name); + + services.Add(new ServiceDescriptor(typeof(CosmosMongoCollection), serviceKey, (sp, _) => + { + var database = sp.GetRequiredService(); + options = GetCollectionOptions(sp, _ => options); + + return new CosmosMongoCollection(database, name, options); + }, lifetime)); + + AddAbstractions(services, serviceKey, lifetime); + + return services; + } + + /// + /// Registers a as + /// using the provided and . + /// + /// + [RequiresUnreferencedCode(UnreferencedCodeMessage)] + [RequiresDynamicCode(DynamicCodeMessage)] + public static IServiceCollection AddCosmosMongoCollection( + this IServiceCollection services, + string name, + string connectionString, + string databaseName, + CosmosMongoCollectionOptions? options = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + where TRecord : class + => AddKeyedCosmosMongoCollection(services, serviceKey: null, name, connectionString, databaseName, options, lifetime); + + /// + /// Registers a keyed as + /// using the provided and . + /// + /// The type of the record. + /// The to register the on. + /// The key with which to associate the collection. + /// The name of the collection. + /// Connection string required to connect to Azure CosmosDB MongoDB. + /// Database name for Azure CosmosDB MongoDB. + /// Optional options to further configure the . + /// The service lifetime for the store. Defaults to . + /// Service collection. + [RequiresUnreferencedCode(UnreferencedCodeMessage)] + [RequiresDynamicCode(DynamicCodeMessage)] + public static IServiceCollection AddKeyedCosmosMongoCollection( + this IServiceCollection services, + object? serviceKey, + string name, + string connectionString, + string databaseName, + CosmosMongoCollectionOptions? options = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + where TRecord : class + { + Throw.IfNullOrWhitespace(connectionString); + Throw.IfNullOrWhitespace(databaseName); + + return AddKeyedCosmosMongoCollection(services, serviceKey, name, _ => connectionString, _ => databaseName, _ => options!, lifetime); + } + + /// + /// Registers a as + /// using the provided and . + /// + /// + [RequiresUnreferencedCode(UnreferencedCodeMessage)] + [RequiresDynamicCode(DynamicCodeMessage)] + public static IServiceCollection AddCosmosMongoCollection( + this IServiceCollection services, + string name, + Func connectionStringProvider, + Func databaseNameProvider, + Func? optionsProvider = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + where TRecord : class + => AddKeyedCosmosMongoCollection(services, serviceKey: null, name, connectionStringProvider, databaseNameProvider, optionsProvider, lifetime); + + /// + /// Registers a keyed as + /// using the provided and . + /// + /// The type of the record. + /// The to register the on. + /// The key with which to associate the collection. + /// The name of the collection. + /// The connection string provider. + /// The database name provider. + /// Options provider to further configure the collection. + /// The service lifetime for the store. Defaults to . + /// Service collection. + [RequiresUnreferencedCode(UnreferencedCodeMessage)] + [RequiresDynamicCode(DynamicCodeMessage)] + public static IServiceCollection AddKeyedCosmosMongoCollection( + this IServiceCollection services, + object? serviceKey, + string name, + Func connectionStringProvider, + Func databaseNameProvider, + Func? optionsProvider = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + where TRecord : class + { + Throw.IfNull(services); + Throw.IfNullOrWhitespace(name); + Throw.IfNull(connectionStringProvider); + Throw.IfNull(databaseNameProvider); + + services.Add(new ServiceDescriptor(typeof(CosmosMongoCollection), serviceKey, (sp, _) => + { + var options = GetCollectionOptions(sp, optionsProvider); + MongoClient mongoClient = new(CreateClientSettings(connectionStringProvider(sp))); + var database = mongoClient.GetDatabase(databaseNameProvider(sp)); + + return new CosmosMongoCollection(database, name, options); + }, lifetime)); + + AddAbstractions(services, serviceKey, lifetime); + + return services; + } + + private static void AddAbstractions(IServiceCollection services, object? serviceKey, ServiceLifetime lifetime) + where TKey : notnull + where TRecord : class + { + services.Add(new ServiceDescriptor(typeof(VectorStoreCollection), serviceKey, + static (sp, key) => sp.GetRequiredKeyedService>(key), lifetime)); + + services.Add(new ServiceDescriptor(typeof(IVectorSearchable), serviceKey, + static (sp, key) => sp.GetRequiredKeyedService>(key), lifetime)); + + // Once HybridSearch supports get implemented by CosmosMongoCollection, + // we need to add IKeywordHybridSearchable abstraction here as well. + } + + private static CosmosMongoVectorStoreOptions? GetStoreOptions(IServiceProvider sp, Func? optionsProvider) + { + var options = optionsProvider?.Invoke(sp); + if (options?.EmbeddingGenerator is not null) + { + return options; // The user has provided everything, there is nothing to change. + } + + var embeddingGenerator = sp.GetService(); + return embeddingGenerator is null + ? options // There is nothing to change. + : new(options) { EmbeddingGenerator = embeddingGenerator }; // Create a brand new copy in order to avoid modifying the original options. + } + + private static CosmosMongoCollectionOptions? GetCollectionOptions(IServiceProvider sp, Func? optionsProvider) + { + var options = optionsProvider?.Invoke(sp); + if (options?.EmbeddingGenerator is not null) + { + return options; // The user has provided everything, there is nothing to change. + } + + var embeddingGenerator = sp.GetService(); + return embeddingGenerator is null + ? options // There is nothing to change. + : new(options) { EmbeddingGenerator = embeddingGenerator }; // Create a brand new copy in order to avoid modifying the original options. + } + + private static MongoClientSettings CreateClientSettings(string connectionString) + { + var settings = MongoClientSettings.FromConnectionString(connectionString); + settings.ApplicationName = ApplicationId; + return settings; + } +} diff --git a/MEVD/src/CosmosMongoDB/CosmosMongoVectorStore.cs b/MEVD/src/CosmosMongoDB/CosmosMongoVectorStore.cs new file mode 100644 index 0000000..4d4b66f --- /dev/null +++ b/MEVD/src/CosmosMongoDB/CosmosMongoVectorStore.cs @@ -0,0 +1,136 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; +using MongoDB.Driver; +using Microsoft.Shared.Diagnostics; + + +namespace CommunityToolkit.VectorData.CosmosMongoDB; + +/// +/// Class for accessing the list of collections in a Azure CosmosDB MongoDB vector store. +/// +/// +/// This class can be used with collections of any schema type, but requires you to provide schema information when getting a collection. +/// +public sealed class CosmosMongoVectorStore : VectorStore +{ + /// Metadata about vector store. + private readonly VectorStoreMetadata _metadata; + + /// that can be used to manage the collections in Azure CosmosDB MongoDB. + private readonly IMongoDatabase _mongoDatabase; + + /// A general purpose definition that can be used to construct a collection when needing to proxy schema agnostic operations. + private static readonly VectorStoreCollectionDefinition s_generalPurposeDefinition = new() { Properties = [new VectorStoreKeyProperty("Key", typeof(string))] }; + + private readonly IEmbeddingGenerator? _embeddingGenerator; + + /// + /// Initializes a new instance of the class. + /// + /// that can be used to manage the collections in Azure CosmosDB MongoDB. + /// Optional configuration options for this class. + public CosmosMongoVectorStore(IMongoDatabase mongoDatabase, CosmosMongoVectorStoreOptions? options = default) + { + Throw.IfNull(mongoDatabase); + + _mongoDatabase = mongoDatabase; + _embeddingGenerator = options?.EmbeddingGenerator; + + _metadata = new() + { + VectorStoreSystemName = CosmosMongoConstants.VectorStoreSystemName, + VectorStoreName = mongoDatabase.DatabaseNamespace?.DatabaseName + }; + } + +#pragma warning disable IDE0090 // Use 'new(...)' + /// + [RequiresDynamicCode("This overload of GetCollection() is incompatible with NativeAOT. For dynamic mapping via Dictionary, call GetDynamicCollection() instead.")] + [RequiresUnreferencedCode("This overload of GetCollecttion() is incompatible with trimming. For dynamic mapping via Dictionary, call GetDynamicCollection() instead.")] +#if NET + public override CosmosMongoCollection GetCollection(string name, VectorStoreCollectionDefinition? definition = null) +#else + public override VectorStoreCollection GetCollection(string name, VectorStoreCollectionDefinition? definition = null) +#endif + => typeof(TRecord) == typeof(Dictionary) + ? throw new ArgumentException(VectorDataStrings.GetCollectionWithDictionaryNotSupported) + : new CosmosMongoCollection( + _mongoDatabase, + name, + new() + { + Definition = definition, + EmbeddingGenerator = _embeddingGenerator + }); + + /// +#if NET + public override CosmosMongoDynamicCollection GetDynamicCollection(string name, VectorStoreCollectionDefinition definition) +#else + public override VectorStoreCollection> GetDynamicCollection(string name, VectorStoreCollectionDefinition definition) +#endif + => new CosmosMongoDynamicCollection( + _mongoDatabase, + name, + new() + { + Definition = definition, + EmbeddingGenerator = _embeddingGenerator, + } + ); +#pragma warning restore IDE0090 + + /// + public override async IAsyncEnumerable ListCollectionNamesAsync([EnumeratorCancellation] CancellationToken cancellationToken = default) + { + const string OperationName = "ListCollectionNames"; + + using var cursor = await VectorStoreErrorHandler.RunOperationAsync, MongoException>( + _metadata, + OperationName, + () => _mongoDatabase.ListCollectionNamesAsync(cancellationToken: cancellationToken)).ConfigureAwait(false); + using var errorHandlingAsyncCursor = new ErrorHandlingAsyncCursor(cursor, _metadata, OperationName); + + while (await errorHandlingAsyncCursor.MoveNextAsync(cancellationToken).ConfigureAwait(false)) + { + foreach (var name in cursor.Current) + { + yield return name; + } + } + } + + /// + public override Task CollectionExistsAsync(string name, CancellationToken cancellationToken = default) + { + var collection = GetDynamicCollection(name, s_generalPurposeDefinition); + return collection.CollectionExistsAsync(cancellationToken); + } + + /// + public override Task EnsureCollectionDeletedAsync(string name, CancellationToken cancellationToken = default) + { + var collection = GetDynamicCollection(name, s_generalPurposeDefinition); + return collection.EnsureCollectionDeletedAsync(cancellationToken); + } + + /// + public override object? GetService(Type serviceType, object? serviceKey = null) + { + Throw.IfNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreMetadata) ? _metadata : + serviceType == typeof(IMongoDatabase) ? _mongoDatabase : + serviceType.IsInstanceOfType(this) ? this : + null; + } +} diff --git a/MEVD/src/CosmosMongoDB/CosmosMongoVectorStoreOptions.cs b/MEVD/src/CosmosMongoDB/CosmosMongoVectorStoreOptions.cs new file mode 100644 index 0000000..50e8a69 --- /dev/null +++ b/MEVD/src/CosmosMongoDB/CosmosMongoVectorStoreOptions.cs @@ -0,0 +1,29 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.AI; + +namespace CommunityToolkit.VectorData.CosmosMongoDB; + +/// +/// Options when creating a +/// +public sealed class CosmosMongoVectorStoreOptions +{ + /// + /// Initializes a new instance of the class. + /// + public CosmosMongoVectorStoreOptions() + { + } + + internal CosmosMongoVectorStoreOptions(CosmosMongoVectorStoreOptions? source) + { + EmbeddingGenerator = source?.EmbeddingGenerator; + } + + /// + /// Gets or sets the default embedding generator to use when generating vectors embeddings with this vector store. + /// + public IEmbeddingGenerator? EmbeddingGenerator { get; set; } +} diff --git a/MEVD/src/CosmosMongoDB/MongoShared/BsonValueFactory.cs b/MEVD/src/CosmosMongoDB/MongoShared/BsonValueFactory.cs new file mode 100644 index 0000000..7837ed1 --- /dev/null +++ b/MEVD/src/CosmosMongoDB/MongoShared/BsonValueFactory.cs @@ -0,0 +1,36 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using MongoDB.Bson; + +namespace CommunityToolkit.VectorData.CosmosMongoDB; + +/// +/// A class that constructs the correct BsonValue for a given CLR type. +/// +internal static class BsonValueFactory +{ + /// + /// Create a BsonValue for the given CLR type. + /// + /// The CLR object to create a BSON value for. + /// The appropriate for that CLR type. + public static BsonValue Create(object? value) + => value switch + { + null => BsonNull.Value, + Guid v => new BsonBinaryData(v, GuidRepresentation.Standard), + DateTimeOffset v => new BsonDateTime(v.UtcDateTime), +#if NET + DateOnly v => new BsonDateTime(v.ToDateTime(TimeOnly.MinValue, DateTimeKind.Utc)), +#endif + object[] v => new BsonArray(Array.ConvertAll(v, Create)), + Array v => new BsonArray(v), + IEnumerable v => new BsonArray(v.Select(Create)), + + _ => BsonValue.Create(value) + }; +} diff --git a/MEVD/src/CosmosMongoDB/MongoShared/ErrorHandlingAsyncCursor.cs b/MEVD/src/CosmosMongoDB/MongoShared/ErrorHandlingAsyncCursor.cs new file mode 100644 index 0000000..ff30423 --- /dev/null +++ b/MEVD/src/CosmosMongoDB/MongoShared/ErrorHandlingAsyncCursor.cs @@ -0,0 +1,63 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.VectorData; +using MongoDB.Driver; + +namespace CommunityToolkit.VectorData.CosmosMongoDB; + +/// +/// A decorator for that handles errors on move next. +/// +/// The type that the cursor returns. +internal class ErrorHandlingAsyncCursor : IAsyncCursor +{ + private readonly IAsyncCursor _cursor; + private readonly string _operationName; + private readonly VectorStoreCollectionMetadata _collectionMetadata; + + public ErrorHandlingAsyncCursor(IAsyncCursor cursor, VectorStoreCollectionMetadata collectionMetadata, string operationName) + { + this._cursor = cursor; + this._operationName = operationName; + this._collectionMetadata = collectionMetadata; + } + + public ErrorHandlingAsyncCursor(IAsyncCursor cursor, VectorStoreMetadata metadata, string operationName) + { + this._cursor = cursor; + this._operationName = operationName; + this._collectionMetadata = new VectorStoreCollectionMetadata() + { + CollectionName = null, + VectorStoreName = metadata.VectorStoreName, + VectorStoreSystemName = metadata.VectorStoreSystemName, + }; + } + + public IEnumerable Current => this._cursor.Current; + + public void Dispose() + { + this._cursor.Dispose(); + } + + public bool MoveNext(CancellationToken cancellationToken = default) + { + return VectorStoreErrorHandler.RunOperation( + this._collectionMetadata, + this._operationName, + () => this._cursor.MoveNext(cancellationToken)); + } + + public Task MoveNextAsync(CancellationToken cancellationToken = default) + { + return VectorStoreErrorHandler.RunOperationAsync( + this._collectionMetadata, + this._operationName, + () => this._cursor.MoveNextAsync(cancellationToken)); + } +} diff --git a/MEVD/src/CosmosMongoDB/MongoShared/IMongoMapper.cs b/MEVD/src/CosmosMongoDB/MongoShared/IMongoMapper.cs new file mode 100644 index 0000000..1d4363c --- /dev/null +++ b/MEVD/src/CosmosMongoDB/MongoShared/IMongoMapper.cs @@ -0,0 +1,21 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using Microsoft.Extensions.AI; +using MongoDB.Bson; + +namespace CommunityToolkit.VectorData.CosmosMongoDB; + +internal interface IMongoMapper +{ + /// + /// Maps from the consumer record data model to the storage model. + /// + BsonDocument MapFromDataToStorageModel(TRecord dataModel, int recordIndex, IReadOnlyList?[]? generatedEmbeddings); + + /// + /// Maps from the storage model to the consumer record data model. + /// + TRecord MapFromStorageToDataModel(BsonDocument storageModel, bool includeVectors); +} diff --git a/MEVD/src/CosmosMongoDB/MongoShared/MongoConstants.cs b/MEVD/src/CosmosMongoDB/MongoShared/MongoConstants.cs new file mode 100644 index 0000000..de970a2 --- /dev/null +++ b/MEVD/src/CosmosMongoDB/MongoShared/MongoConstants.cs @@ -0,0 +1,64 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.VectorData; + +namespace CommunityToolkit.VectorData.CosmosMongoDB; + +/// +/// Constants for MongoDB vector store implementation. +/// +internal static class MongoConstants +{ + internal const string VectorStoreSystemName = "mongodb"; + + /// Default ratio of number of nearest neighbors to number of documents to return. + internal const int DefaultNumCandidatesRatio = 10; + + /// Default vector index name. + internal const string DefaultVectorIndexName = "vector_index"; + + /// Default full text search index name. + internal const string DefaultFullTextSearchIndexName = "full_text_search_index"; + + /// Default index kind for vector search. + internal const string DefaultIndexKind = IndexKind.IvfFlat; + + /// Default distance function for vector search. + internal const string DefaultDistanceFunction = DistanceFunction.CosineDistance; + + /// Reserved key property name in MongoDB. + internal const string MongoReservedKeyPropertyName = "_id"; + + /// Reserved key property name in data model. + internal const string DataModelReservedKeyPropertyName = "Id"; + + /// A containing the supported key types. + internal static readonly HashSet SupportedKeyTypes = + [ + typeof(string) + ]; + + /// A containing the supported data property types. + internal static readonly HashSet SupportedDataTypes = + [ + typeof(bool), + typeof(string), + typeof(int), + typeof(long), + typeof(float), + typeof(double), + typeof(decimal), + typeof(DateTime), + ]; + + /// A containing the supported vector types. + internal static readonly HashSet SupportedVectorTypes = + [ + typeof(ReadOnlyMemory), + typeof(ReadOnlyMemory?) + ]; +} diff --git a/MEVD/src/CosmosMongoDB/MongoShared/MongoDynamicMapper.cs b/MEVD/src/CosmosMongoDB/MongoShared/MongoDynamicMapper.cs new file mode 100644 index 0000000..c959e6e --- /dev/null +++ b/MEVD/src/CosmosMongoDB/MongoShared/MongoDynamicMapper.cs @@ -0,0 +1,243 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Runtime.InteropServices; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData.ProviderServices; +using Microsoft.Shared.Diagnostics; +using MongoDB.Bson; + +namespace CommunityToolkit.VectorData.CosmosMongoDB; + +/// +/// A mapper that maps between the dynamic data model and the model that the data is stored under, within MongoDB. +/// +internal sealed class MongoDynamicMapper(CollectionModel model) : IMongoMapper> +{ + /// + public BsonDocument MapFromDataToStorageModel(Dictionary dataModel, int recordIndex, IReadOnlyList?[]? generatedEmbeddings) + { + Throw.IfNull(dataModel); + + var document = new BsonDocument + { + [MongoConstants.MongoReservedKeyPropertyName] = !dataModel.TryGetValue(model.KeyProperty.ModelName, out var keyValue) + ? throw new InvalidOperationException($"Missing value for key property '{model.KeyProperty.ModelName}") + : keyValue switch + { + string s => s, + Guid g => new BsonBinaryData(g, GuidRepresentation.Standard), + ObjectId o => o, + long i => i, + int i => i, + + null => throw new InvalidOperationException($"Key property '{model.KeyProperty.ModelName}' is null."), + _ => throw new InvalidCastException($"Key property '{model.KeyProperty.ModelName}' must be a string, Guid, ObjectID, long or int.") + } + }; + + foreach (var property in model.DataProperties) + { + if (dataModel.TryGetValue(property.ModelName, out var dataValue)) + { + document[property.StorageName] = BsonValueFactory.Create(dataValue); + } + } + + for (var i = 0; i < model.VectorProperties.Count; i++) + { + var property = model.VectorProperties[i]; + + // Don't create a property if it doesn't exist in the dictionary + if (dataModel.TryGetValue(property.ModelName, out var vectorValue)) + { + var vector = generatedEmbeddings?[i]?[recordIndex] is Embedding ge + ? ge + : vectorValue; + + document[property.StorageName] = BsonArray.Create( + vector switch + { + ReadOnlyMemory m + => MemoryMarshal.TryGetArray(m, out ArraySegment segment) && segment.Count == segment.Array!.Length ? segment.Array : m.ToArray(), + Embedding e + => MemoryMarshal.TryGetArray(e.Vector, out ArraySegment segment) && segment.Count == segment.Array!.Length ? segment.Array : e.Vector.ToArray(), + float[] a => a, + + null => Array.Empty(), + + _ => throw new UnreachableException() + }); + } + } + + return document; + } + + /// + public Dictionary MapFromStorageToDataModel(BsonDocument storageModel, bool includeVectors) + { + Throw.IfNull(storageModel); + + var result = new Dictionary(); + + // Loop through all known properties and map each from the storage model to the data model. + foreach (var property in model.Properties) + { + switch (property) + { + case KeyPropertyModel keyProperty: + if (!storageModel.TryGetValue(MongoConstants.MongoReservedKeyPropertyName, out var keyValue)) + { + throw new InvalidOperationException("No key property was found in the record retrieved from storage."); + } + + result[keyProperty.ModelName] = keyProperty.Type switch + { + var t when t == typeof(string) => keyValue.AsString, + var t when t == typeof(Guid) => keyValue.AsGuid, + var t when t == typeof(ObjectId) => keyValue.AsObjectId, + var t when t == typeof(long) => keyValue.AsInt64, + var t when t == typeof(int) => keyValue.AsInt32, + + _ => throw new UnreachableException() + }; + + continue; + + case DataPropertyModel dataProperty: + if (storageModel.TryGetValue(dataProperty.StorageName, out var dataValue)) + { + result.Add(dataProperty.ModelName, GetDataPropertyValue(property.ModelName, property.Type, dataValue)); + } + continue; + + case VectorPropertyModel vectorProperty: + if (includeVectors && storageModel.TryGetValue(vectorProperty.StorageName, out var vectorValue)) + { + result.Add( + vectorProperty.ModelName, + vectorValue.IsBsonNull + ? null + : (Nullable.GetUnderlyingType(property.Type) ?? property.Type) switch + { + Type t when t == typeof(ReadOnlyMemory) => new ReadOnlyMemory(vectorValue.AsBsonArray.Select(item => (float)item.AsDouble).ToArray()), + Type t when t == typeof(Embedding) => new Embedding(vectorValue.AsBsonArray.Select(item => (float)item.AsDouble).ToArray()), + Type t when t == typeof(float[]) => vectorValue.AsBsonArray.Select(item => (float)item.AsDouble).ToArray(), + + _ => throw new UnreachableException() + }); + } + continue; + + default: + throw new UnreachableException(); + } + } + + return result; + } + + #region private + + private static object? GetDataPropertyValue(string propertyName, Type propertyType, BsonValue value) + { + if (value.IsBsonNull) + { + return null; + } + + var result = propertyType switch + { + Type t when t == typeof(bool) => value.AsBoolean, + Type t when t == typeof(bool?) => value.AsNullableBoolean, + Type t when t == typeof(string) => value.AsString, + Type t when t == typeof(int) => value.AsInt32, + Type t when t == typeof(int?) => value.AsNullableInt32, + Type t when t == typeof(long) => value.AsInt64, + Type t when t == typeof(long?) => value.AsNullableInt64, + Type t when t == typeof(float) => (float)value.AsDouble, + Type t when t == typeof(float?) => ((float?)value.AsNullableDouble), + Type t when t == typeof(double) => value.AsDouble, + Type t when t == typeof(double?) => value.AsNullableDouble, + Type t when t == typeof(decimal) => value.AsDecimal, + Type t when t == typeof(decimal?) => value.AsNullableDecimal, + Type t when t == typeof(DateTime) => value.ToUniversalTime(), + Type t when t == typeof(DateTime?) => value.ToNullableUniversalTime(), + Type t when t == typeof(DateTimeOffset) => new DateTimeOffset(value.ToUniversalTime(), TimeSpan.Zero), + Type t when t == typeof(DateTimeOffset?) => value.ToNullableUniversalTime() is DateTime dateTime + ? new DateTimeOffset(dateTime, TimeSpan.Zero) + : null, +#if NET + Type t when t == typeof(DateOnly) => DateOnly.FromDateTime(value.ToUniversalTime()), + Type t when t == typeof(DateOnly?) => value.ToNullableUniversalTime() is DateTime dateTime + ? DateOnly.FromDateTime(dateTime) + : null, +#endif + + _ => (object?)null + }; + + if (result is not null) + { + return result; + } + + if (propertyType.IsArray) + { + return propertyType switch + { + Type t when t == typeof(bool[]) => value.AsBsonArray.Select(x => x.AsBoolean).ToArray(), + Type t when t == typeof(bool?[]) => value.AsBsonArray.Select(x => x.AsNullableBoolean).ToArray(), + Type t when t == typeof(string[]) => value.AsBsonArray.Select(x => x.AsString).ToArray(), + Type t when t == typeof(int[]) => value.AsBsonArray.Select(x => x.AsInt32).ToArray(), + Type t when t == typeof(int?[]) => value.AsBsonArray.Select(x => x.AsNullableInt32).ToArray(), + Type t when t == typeof(long[]) => value.AsBsonArray.Select(x => x.AsInt64).ToArray(), + Type t when t == typeof(long?[]) => value.AsBsonArray.Select(x => x.AsNullableInt64).ToArray(), + Type t when t == typeof(float[]) => value.AsBsonArray.Select(x => (float)x.AsDouble).ToArray(), + Type t when t == typeof(float?[]) => value.AsBsonArray.Select(x => (float?)x.AsNullableDouble).ToArray(), + Type t when t == typeof(double[]) => value.AsBsonArray.Select(x => x.AsDouble).ToArray(), + Type t when t == typeof(double?[]) => value.AsBsonArray.Select(x => x.AsNullableDouble).ToArray(), + Type t when t == typeof(decimal[]) => value.AsBsonArray.Select(x => x.AsDecimal).ToArray(), + Type t when t == typeof(decimal?[]) => value.AsBsonArray.Select(x => x.AsNullableDecimal).ToArray(), + Type t when t == typeof(DateTime[]) => value.AsBsonArray.Select(x => x.ToUniversalTime()).ToArray(), + Type t when t == typeof(DateTime?[]) => value.AsBsonArray.Select(x => x.ToNullableUniversalTime()).ToArray(), + + _ => (object?)null + }; + } + + if (propertyType.IsGenericType && propertyType.GetGenericTypeDefinition() == typeof(List<>)) + { + return propertyType switch + { + Type t when t == typeof(List) => value.AsBsonArray.Select(x => x.AsBoolean).ToList(), + Type t when t == typeof(List) => value.AsBsonArray.Select(x => x.AsNullableBoolean).ToList(), + Type t when t == typeof(List) => value.AsBsonArray.Select(x => x.AsString).ToList(), + Type t when t == typeof(List) => value.AsBsonArray.Select(x => x.AsInt32).ToList(), + Type t when t == typeof(List) => value.AsBsonArray.Select(x => x.AsNullableInt32).ToList(), + Type t when t == typeof(List) => value.AsBsonArray.Select(x => x.AsInt64).ToList(), + Type t when t == typeof(List) => value.AsBsonArray.Select(x => x.AsNullableInt64).ToList(), + Type t when t == typeof(List) => value.AsBsonArray.Select(x => (float)x.AsDouble).ToList(), + Type t when t == typeof(List) => value.AsBsonArray.Select(x => (float?)x.AsNullableDouble).ToList(), + Type t when t == typeof(List) => value.AsBsonArray.Select(x => x.AsDouble).ToList(), + Type t when t == typeof(List) => value.AsBsonArray.Select(x => x.AsNullableDouble).ToList(), + Type t when t == typeof(List) => value.AsBsonArray.Select(x => x.AsDecimal).ToList(), + Type t when t == typeof(List) => value.AsBsonArray.Select(x => x.AsNullableDecimal).ToList(), + Type t when t == typeof(List) => value.AsBsonArray.Select(x => x.ToUniversalTime()).ToList(), + Type t when t == typeof(List) => value.AsBsonArray.Select(x => x.ToNullableUniversalTime()).ToList(), + + _ => (object?)null + }; + } + + throw new NotSupportedException($"Mapping for property {propertyName} with type {propertyType.FullName} is not supported in dynamic data model."); + } + + #endregion +} diff --git a/MEVD/src/CosmosMongoDB/MongoShared/MongoMapper.cs b/MEVD/src/CosmosMongoDB/MongoShared/MongoMapper.cs new file mode 100644 index 0000000..2e26256 --- /dev/null +++ b/MEVD/src/CosmosMongoDB/MongoShared/MongoMapper.cs @@ -0,0 +1,155 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Reflection; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData.ProviderServices; +using MongoDB.Bson; +using MongoDB.Bson.Serialization; +using MongoDB.Bson.Serialization.Attributes; +using MongoDB.Bson.Serialization.Conventions; +using MongoDB.Bson.Serialization.Serializers; + +namespace CommunityToolkit.VectorData.CosmosMongoDB; + +internal sealed class MongoMapper : IMongoMapper + where TRecord : class +{ + private readonly CollectionModel _model; + + /// A key property info of the data model. + private readonly PropertyInfo? _keyClrProperty; + + /// A key property name of the data model. + private readonly string _keyPropertyModelName; + + /// + /// Initializes a new instance of the class. + /// + /// The model. + public MongoMapper(CollectionModel model) + { + this._model = model; + + var keyProperty = model.KeyProperty; + this._keyPropertyModelName = keyProperty.ModelName; + this._keyClrProperty = keyProperty.PropertyInfo; + + var conventionPack = new ConventionPack + { + new IgnoreExtraElementsConvention(ignoreExtraElements: true), + new GuidStandardRepresentationConvention() + }; + + ConventionRegistry.Register( + nameof(MongoMapper), + conventionPack, + type => type == typeof(TRecord)); + } + + public BsonDocument MapFromDataToStorageModel(TRecord dataModel, int recordIndex, IReadOnlyList?[]? generatedEmbeddings) + { + var document = dataModel.ToBsonDocument(); + + // Handle key property mapping due to reserved key name in Mongo. + if (!document.Contains(MongoConstants.MongoReservedKeyPropertyName)) + { + var value = document[this._keyPropertyModelName]; + + document.Remove(this._keyPropertyModelName); + + document[MongoConstants.MongoReservedKeyPropertyName] = value; + } + + // Go over the vector properties; those which have an embedding generator configured on them will have embedding generators, overwrite + // the value in the JSON object with that. + for (var i = 0; i < this._model.VectorProperties.Count; i++) + { + var property = this._model.VectorProperties[i]; + + Embedding? embedding = generatedEmbeddings?[i]?[recordIndex] is Embedding e ? (Embedding)e : null; + + if (embedding is null) + { + switch (Nullable.GetUnderlyingType(property.Type) ?? property.Type) + { + case var t when t == typeof(ReadOnlyMemory): + case var t2 when t2 == typeof(float[]): + // The .NET vector property is a ReadOnlyMemory or float[] (not an Embedding), which means that ToBsonDocument() + // already serialized it correctly above. + // In addition, there's no generated embedding (which would be an Embedding which we'd need to handle manually). + // So there's nothing for us to do. + continue; + + case var t when t == typeof(Embedding): + embedding = (Embedding)property.GetValueAsObject(dataModel)!; + break; + + default: + throw new UnreachableException(); + } + } + + document[property.StorageName] = BsonArray.Create(embedding.Vector.ToArray()); + } + + return document; + } + + public TRecord MapFromStorageToDataModel(BsonDocument storageModel, bool includeVectors) + { + // Handle key property mapping due to reserved key name in Mongo. + if (!this._keyPropertyModelName.Equals(MongoConstants.DataModelReservedKeyPropertyName, StringComparison.OrdinalIgnoreCase) && + this._keyClrProperty?.GetCustomAttribute() is null) + { + var value = storageModel[MongoConstants.MongoReservedKeyPropertyName]; + + storageModel.Remove(MongoConstants.MongoReservedKeyPropertyName); + + storageModel[this._keyPropertyModelName] = value; + } + + if (includeVectors) + { + foreach (var vectorProperty in this._model.VectorProperties) + { + // If the vector property .NET type is Embedding, we need to create the BSON structure for it + // (BSON array embedded inside an object representing the embedding), so that the deserialization below + // works correctly. + if (vectorProperty.Type == typeof(Embedding)) + { + storageModel[vectorProperty.StorageName] = new BsonDocument + { + [nameof(Embedding.Vector)] = BsonArray.Create(storageModel[vectorProperty.StorageName]) + }; + } + } + } + else + { + // If includeVectors is false, remove the values; this allows us to not project them out of Mongo in the future + // (more efficient) without introducing a breaking change. + foreach (var vectorProperty in this._model.VectorProperties) + { + storageModel.Remove(vectorProperty.StorageName); + } + } + + return BsonSerializer.Deserialize(storageModel); + } + + private class GuidStandardRepresentationConvention : ConventionBase, IMemberMapConvention + { + public void Apply(BsonMemberMap memberMap) + { + if (memberMap.MemberType == typeof(Guid) && memberMap.MemberInfo.GetCustomAttribute() is null) + { + memberMap.SetSerializer(new GuidSerializer(GuidRepresentation.Standard)); + } + } + } +} diff --git a/MEVD/src/CosmosMongoDB/MongoShared/MongoModelBuilder.cs b/MEVD/src/CosmosMongoDB/MongoShared/MongoModelBuilder.cs new file mode 100644 index 0000000..abff7b0 --- /dev/null +++ b/MEVD/src/CosmosMongoDB/MongoShared/MongoModelBuilder.cs @@ -0,0 +1,115 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Reflection; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; + +namespace CommunityToolkit.VectorData.CosmosMongoDB; + +/// +/// Customized MongoDB model builder that adds specialized configuration of property storage names +/// (Mongo's reserve key property name and [BsonElement]). +/// +internal class MongoModelBuilder() : CollectionModelBuilder(s_validationOptions) +{ + internal const string SupportedVectorTypes = "ReadOnlyMemory, Embedding, float[]"; + + private static readonly CollectionModelBuildingOptions s_validationOptions = new() + { + RequiresAtLeastOneVector = false, + SupportsMultipleVectors = true, + UsesExternalSerializer = true, + }; + + protected override void ProcessProperty(PropertyInfo? clrProperty, VectorStoreProperty? definitionProperty) + { + base.ProcessProperty(clrProperty!, definitionProperty!); + + // TODO: Reenable this after https://github.com/dotnet/extensions/pull/7475 + // if (clrProperty?.GetCustomAttribute() is { } bsonElementAttribute + // && this.PropertyMap.TryGetValue(clrProperty.Name, out var property)) + // { + // property.StorageName = bsonElementAttribute.ElementName; + // } + + // TODO: Workaround for https://github.com/dotnet/extensions/pull/7475 + // The BSON serializer determines field names from the CLR property name (or [BsonElement] if present). + // Since UsesExternalSerializer is true, the base CollectionModelBuilder is supposed to ignore StorageName + // from VectorStoreProperty attributes to stay in sync with the serializer. However, some versions of the + // base class don't do this correctly, causing StorageName to diverge from what the serializer uses. + // Override here to always keep StorageName in sync with the serializer's behavior. + if (clrProperty is not null && this.PropertyMap.TryGetValue(clrProperty.Name, out var property)) + { + property.StorageName = clrProperty.GetCustomAttribute()?.ElementName ?? clrProperty.Name; + } + } + + protected override bool SupportsKeyAutoGeneration(Type keyPropertyType) + => keyPropertyType == typeof(Guid) || keyPropertyType == typeof(ObjectId); + + protected override void ValidateKeyProperty(KeyPropertyModel keyProperty) + { + base.ValidateKeyProperty(keyProperty); + + var type = keyProperty.Type; + + if (type != typeof(string) && type != typeof(int) && type != typeof(long) && type != typeof(Guid) && type != typeof(ObjectId)) + { + throw new NotSupportedException( + $"Property '{keyProperty.ModelName}' has unsupported type '{type.Name}'. Key properties must be one of the supported types: string, int, long, Guid, ObjectId."); + } + } + + protected override bool IsDataPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes) + { + supportedTypes = "string, int, long, double, float, bool, decimal, DateTime, DateTimeOffset," +#if NET + + " DateOnly," +#endif + + " or arrays/lists of these types"; + + if (Nullable.GetUnderlyingType(type) is Type underlyingType) + { + type = underlyingType; + } + + return IsValid(type) + || (type.IsArray && IsValid(type.GetElementType()!)) + || (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(List<>) && IsValid(type.GenericTypeArguments[0])); + + static bool IsValid(Type type) + => type == typeof(bool) || + type == typeof(string) || + type == typeof(int) || + type == typeof(long) || + type == typeof(float) || + type == typeof(double) || + type == typeof(decimal) || + type == typeof(DateTime) || + type == typeof(DateTimeOffset) || +#if NET + type == typeof(DateOnly) || +#endif + false; + } + + protected override bool IsVectorPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes) + => IsVectorPropertyTypeValidCore(type, out supportedTypes); + + internal static bool IsVectorPropertyTypeValidCore(Type type, [NotNullWhen(false)] out string? supportedTypes) + { + supportedTypes = SupportedVectorTypes; + + return type == typeof(ReadOnlyMemory) + || type == typeof(ReadOnlyMemory?) + || type == typeof(Embedding) + || type == typeof(float[]); + } +} diff --git a/MEVD/src/CosmosMongoDB/MongoShared/README.md b/MEVD/src/CosmosMongoDB/MongoShared/README.md new file mode 100644 index 0000000..60c2992 --- /dev/null +++ b/MEVD/src/CosmosMongoDB/MongoShared/README.md @@ -0,0 +1,5 @@ +# MongoShared + +These files are shared utilities that are identical across the Cosmos MongoDB and the regular MongoDB vector store providers. They were originally shared via linked compilation in the Semantic Kernel repository. + +If a regular MongoDB provider is added in the future, these files should be extracted to a shared location and linked into both projects. diff --git a/MEVD/src/Directory.Build.props b/MEVD/src/Directory.Build.props new file mode 100644 index 0000000..d8c6476 --- /dev/null +++ b/MEVD/src/Directory.Build.props @@ -0,0 +1,24 @@ + + + + + + $(NoWarn);MEVD9000,MEVD9001 + true + true + true + true + true + true + + + true + + + + + + + + + diff --git a/MEVD/src/InMemory/AssemblyInfo.cs b/MEVD/src/InMemory/AssemblyInfo.cs new file mode 100644 index 0000000..d6395e8 --- /dev/null +++ b/MEVD/src/InMemory/AssemblyInfo.cs @@ -0,0 +1,2 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. diff --git a/MEVD/src/InMemory/InMemory.csproj b/MEVD/src/InMemory/InMemory.csproj new file mode 100644 index 0000000..1e70c16 --- /dev/null +++ b/MEVD/src/InMemory/InMemory.csproj @@ -0,0 +1,27 @@ + + + 1.0.0-preview.1 + CommunityToolkit.VectorData.InMemory + $(AssemblyName) + net10.0;net8.0;netstandard2.0;net462 + + In-Memory provider for Microsoft.Extensions.VectorData + In-Memory provider for Microsoft.Extensions.VectorData by Semantic Kernel + + + + + + + + + + + + + + + + + + diff --git a/MEVD/src/InMemory/InMemoryCollection.cs b/MEVD/src/InMemory/InMemoryCollection.cs new file mode 100644 index 0000000..272001f --- /dev/null +++ b/MEVD/src/InMemory/InMemoryCollection.cs @@ -0,0 +1,458 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Concurrent; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Linq.Expressions; +using System.Runtime.CompilerServices; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; +using Microsoft.Shared.Diagnostics; + +namespace CommunityToolkit.VectorData.InMemory; + +/// +/// Service for storing and retrieving vector records, that uses an in memory dictionary as the underlying storage. +/// +/// The data type of the record key. +/// The data model to use for adding, updating and retrieving data from storage. +#pragma warning disable CA1711 // Identifiers should not have incorrect suffix +public class InMemoryCollection : VectorStoreCollection +#pragma warning restore CA1711 // Identifiers should not have incorrect suffix + where TKey : notnull + where TRecord : class +{ + /// Metadata about vector store record collection. + private readonly VectorStoreCollectionMetadata _collectionMetadata; + + /// The default options for vector search. + private static readonly VectorSearchOptions s_defaultVectorSearchOptions = new(); + + /// Internal storage for all of the record collections. + private readonly ConcurrentDictionary> _internalCollections; + + /// The data type of each collection, to enforce a single type per collection. + private readonly ConcurrentDictionary _internalCollectionTypes; + + /// The model for this collection. + private readonly CollectionModel _model; + + /// + /// Initializes a new instance of the class. + /// + /// The name of the collection that this will access. + /// Optional configuration options for this class. + [RequiresUnreferencedCode("The InMemory provider is incompatible with trimming.")] + [RequiresDynamicCode("The InMemory provider is incompatible with NativeAOT.")] + public InMemoryCollection(string name, InMemoryCollectionOptions? options = default) + : this( + internalCollection: null, + internalCollectionTypes: null, + name, + static options => typeof(TRecord) == typeof(Dictionary) + ? throw new NotSupportedException(VectorDataStrings.NonDynamicCollectionWithDictionaryNotSupported(typeof(InMemoryDynamicCollection))) + : new InMemoryModelBuilder().Build(typeof(TRecord), typeof(TKey), options.Definition, options.EmbeddingGenerator), + options) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// Internal storage for the record collection. + /// The data type of each collection, to enforce a single type per collection. + /// The name of the collection that this will access. + /// Optional configuration options for this class. + [RequiresUnreferencedCode("The InMemory provider is incompatible with trimming.")] + [RequiresDynamicCode("The InMemory provider is incompatible with NativeAOT.")] + internal InMemoryCollection( + ConcurrentDictionary> internalCollection, + ConcurrentDictionary internalCollectionTypes, + string name, + InMemoryCollectionOptions? options = default) + : this(name, options) + { + _internalCollections = internalCollection; + _internalCollectionTypes = internalCollectionTypes; + } + + internal InMemoryCollection( + ConcurrentDictionary>? internalCollection, + ConcurrentDictionary? internalCollectionTypes, + string name, + Func modelFactory, + InMemoryCollectionOptions? options) + { + // Verify. + Throw.IfNullOrWhitespace(name); + + options ??= new InMemoryCollectionOptions(); + + // Assign. + Name = name; + _model = modelFactory(options); + + _internalCollections = internalCollection ?? new(); + _internalCollectionTypes = internalCollectionTypes ?? new(); + + _collectionMetadata = new() + { + VectorStoreSystemName = InMemoryConstants.VectorStoreSystemName, + CollectionName = name + }; + } + + /// + public override string Name { get; } + + /// + public override Task CollectionExistsAsync(CancellationToken cancellationToken = default) + { + return _internalCollections.ContainsKey(Name) ? Task.FromResult(true) : Task.FromResult(false); + } + + /// + public override Task EnsureCollectionExistsAsync(CancellationToken cancellationToken = default) + { + if (!_internalCollections.ContainsKey(Name)) + { + _internalCollections.TryAdd(Name, new ConcurrentDictionary()); + _internalCollectionTypes.TryAdd(Name, typeof(TRecord)); + } + + return Task.CompletedTask; + } + + /// + public override Task EnsureCollectionDeletedAsync(CancellationToken cancellationToken = default) + { + _internalCollections.TryRemove(Name, out _); + _internalCollectionTypes.TryRemove(Name, out _); + return Task.CompletedTask; + } + + /// + public override Task GetAsync(TKey key, RecordRetrievalOptions? options = null, CancellationToken cancellationToken = default) + { + if (options?.IncludeVectors == true && _model.EmbeddingGenerationRequired) + { + throw new NotSupportedException(VectorDataStrings.IncludeVectorsNotSupportedWithEmbeddingGeneration); + } + + var collectionDictionary = GetCollectionDictionary(); + + if (collectionDictionary.TryGetValue(key, out var record)) + { + return Task.FromResult(((InMemoryRecordWrapper)record).Record); + } + + return Task.FromResult(default); + } + + /// + public override Task DeleteAsync(TKey key, CancellationToken cancellationToken = default) + { + var collectionDictionary = GetCollectionDictionary(); + + collectionDictionary.TryRemove(key, out _); + return Task.CompletedTask; + } + + /// + public override Task UpsertAsync(TRecord record, CancellationToken cancellationToken = default) + => UpsertAsync([record], cancellationToken); + + /// + public override async Task UpsertAsync(IEnumerable records, CancellationToken cancellationToken = default) + { + Throw.IfNull(records); + + IReadOnlyList? recordsList = null; + + // If an embedding generator is defined, invoke it once per property for all records. + IReadOnlyList?[]? generatedEmbeddings = null; + + var vectorPropertyCount = _model.VectorProperties.Count; + for (var i = 0; i < vectorPropertyCount; i++) + { + var vectorProperty = _model.VectorProperties[i]; + + if (InMemoryModelBuilder.IsVectorPropertyTypeValidCore(vectorProperty.Type, out _)) + { + continue; + } + + // We have a vector property whose type isn't natively supported - we need to generate embeddings. + Debug.Assert(vectorProperty.EmbeddingGenerator is not null); + + // We have a property with embedding generation; materialize the records' enumerable if needed, to + // prevent multiple enumeration. + if (recordsList is null) + { + recordsList = records is IReadOnlyList r ? r : records.ToList(); + + if (recordsList.Count == 0) + { + return; + } + + records = recordsList; + } + + // TODO: Ideally we'd group together vector properties using the same generator (and with the same input and output properties), + // and generate embeddings for them in a single batch. That's some more complexity though. + generatedEmbeddings ??= new IReadOnlyList?[vectorPropertyCount]; + generatedEmbeddings[i] = await vectorProperty.GenerateEmbeddingsAsync(records.Select(r => vectorProperty.GetValueAsObject(r)), cancellationToken).ConfigureAwait(false); + } + + var collectionDictionary = GetCollectionDictionary(); + + var recordIndex = 0; + var keyProperty = _model.KeyProperty; + foreach (var record in records) + { + var key = (TKey)keyProperty.GetValueAsObject(record)!; + if (keyProperty.IsAutoGenerated && (Guid)(object)key == Guid.Empty) + { + var generatedGuid = Guid.NewGuid(); + keyProperty.SetValue(record, generatedGuid); + key = (TKey)(object)generatedGuid; + } + + var wrappedRecord = new InMemoryRecordWrapper(record); + + if (generatedEmbeddings is not null) + { + for (var i = 0; i < _model.VectorProperties.Count; i++) + { + if (generatedEmbeddings![i] is IReadOnlyList propertyEmbeddings) + { + var property = _model.VectorProperties[i]; + + wrappedRecord.EmbeddingGeneratedVectors[property.ModelName] = propertyEmbeddings[recordIndex] switch + { + Embedding e => e.Vector, + _ => throw new UnreachableException() + }; + } + } + } + + collectionDictionary.AddOrUpdate(key!, wrappedRecord, (key, currentValue) => wrappedRecord); + + recordIndex++; + } + } + + #region Search + + /// + public override async IAsyncEnumerable> SearchAsync( + TInput searchValue, + int top, + VectorSearchOptions? options = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + Throw.IfNull(searchValue); + Throw.IfLessThan(top, 1); + + options ??= s_defaultVectorSearchOptions; + if (options.IncludeVectors && _model.EmbeddingGenerationRequired) + { + throw new NotSupportedException(VectorDataStrings.IncludeVectorsNotSupportedWithEmbeddingGeneration); + } + + var vectorProperty = _model.GetVectorPropertyOrSingle(options); + + ReadOnlyMemory inputVector = searchValue switch + { + ReadOnlyMemory r => r, + float[] f => new ReadOnlyMemory(f), + Embedding e => e.Vector, + _ when vectorProperty.EmbeddingGenerationDispatcher is not null + => ((Embedding)await vectorProperty.GenerateEmbeddingAsync(searchValue, cancellationToken).ConfigureAwait(false)).Vector, + + _ => vectorProperty.EmbeddingGenerator is null + ? throw new NotSupportedException(VectorDataStrings.InvalidSearchInputAndNoEmbeddingGeneratorWasConfigured(searchValue.GetType(), InMemoryModelBuilder.SupportedVectorTypes)) + : throw new InvalidOperationException(VectorDataStrings.IncompatibleEmbeddingGeneratorWasConfiguredForInputType(typeof(TInput), vectorProperty.EmbeddingGenerator.GetType())) + }; + + // Filter records using the provided filter before doing the vector comparison. + var allValues = GetCollectionDictionary().Values.Cast>(); + var filteredRecords = options.Filter is not null + ? allValues.AsQueryable().Where(ConvertFilter(options.Filter)) + : allValues; + + // Compare each vector in the filtered results with the provided vector. + var results = filteredRecords.Select, (TRecord record, float score)?>(wrapper => + { + ReadOnlySpan vector = null; + + if (InMemoryModelBuilder.IsVectorPropertyTypeValidCore(vectorProperty.Type, out _)) + { + // No embedding generation - just get the the vector property directly from the stored instance. + var value = vectorProperty.GetValueAsObject(wrapper.Record); + if (value is null) + { + return null; + } + + vector = value switch + { + ReadOnlyMemory m => m.Span, + Embedding e => e.Vector.Span, + float[] a => a, + + _ => throw new UnreachableException() + }; + } + else + { + // The property requires embedding generation; the generated embedding is stored outside the instance, in the wrapper. + vector = wrapper.EmbeddingGeneratedVectors[vectorProperty.ModelName].Span; + } + + var score = InMemoryCollectionSearchMapping.CompareVectors(inputVector.Span, vector, vectorProperty.DistanceFunction); + var convertedscore = InMemoryCollectionSearchMapping.ConvertScore(score, vectorProperty.DistanceFunction); + return (wrapper.Record, convertedscore); + }); + + // Get the non-null results since any record with a null vector results in a null result. + var nonNullResults = results.Where(x => x.HasValue).Select(x => x!.Value); + + // Filter by score threshold if specified. + if (options.ScoreThreshold is double scoreThreshold) + { + nonNullResults = InMemoryCollectionSearchMapping.ShouldSortDescending(vectorProperty.DistanceFunction) + ? nonNullResults.Where(x => x.score >= scoreThreshold) + : nonNullResults.Where(x => x.score <= scoreThreshold); + } + + // Sort the results appropriately for the selected distance function and get the right page of results . + var sortedScoredResults = InMemoryCollectionSearchMapping.ShouldSortDescending(vectorProperty.DistanceFunction) ? + nonNullResults.OrderByDescending(x => x.score) : + nonNullResults.OrderBy(x => x.score); + var resultsPage = sortedScoredResults.Skip(options.Skip).Take(top); + + // Build the response. + foreach (var record in resultsPage.Select(x => new VectorSearchResult((TRecord)x.record, x.score))) + { + yield return record; + } + } + + #endregion Search + + /// + public override object? GetService(Type serviceType, object? serviceKey = null) + { + Throw.IfNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreCollectionMetadata) ? _collectionMetadata : + serviceType == typeof(ConcurrentDictionary>) ? _internalCollections : + serviceType.IsInstanceOfType(this) ? this : + null; + } + + /// + public override IAsyncEnumerable GetAsync(Expression> filter, int top, + FilteredRecordRetrievalOptions? options = null, CancellationToken cancellationToken = default) + { + Throw.IfNull(filter); + Throw.IfLessThan(top, 1); + + options ??= new(); + + if (options.IncludeVectors && _model.EmbeddingGenerationRequired) + { + throw new NotSupportedException(VectorDataStrings.IncludeVectorsNotSupportedWithEmbeddingGeneration); + } + + var records = GetCollectionDictionary() + .Values + .Cast>() + .Select(x => x.Record) + .AsQueryable() + .Where(filter); + + var orderBy = options.OrderBy?.Invoke(new()).Values; + if (orderBy is { Count: > 0 }) + { + var first = orderBy[0]; + var sorted = first.Ascending + ? records.OrderBy(first.PropertySelector) + : records.OrderByDescending(first.PropertySelector); + + for (int i = 1; i < orderBy.Count; i++) + { + var next = orderBy[i]; + sorted = next.Ascending + ? sorted.ThenBy(next.PropertySelector) + : sorted.ThenByDescending(next.PropertySelector); + } + + records = sorted; + } + + return records + .Skip(options.Skip) + .Take(top) + .ToAsyncEnumerable(); + } + + /// + /// Get the collection dictionary from the internal storage, throws if it does not exist. + /// + /// The retrieved collection dictionary. + internal ConcurrentDictionary GetCollectionDictionary() + { + if (!_internalCollections.TryGetValue(Name, out var collectionDictionary)) + { + throw new VectorStoreException($"Call to vector store failed. Collection '{Name}' does not exist."); + } + + return collectionDictionary; + } + + /// + /// Updates the collection dictionary with any matches values from the provided dictionary. + /// + /// Updates to apply to the collection dictionary. + internal void UpdateCollectionDictionary(Dictionary updates) + { + if (!_internalCollections.TryGetValue(Name, out var collectionDictionary)) + { + throw new VectorStoreException($"Call to vector store failed. Collection '{Name}' does not exist."); + } + + foreach (var update in updates) + { + collectionDictionary.AddOrUpdate(update.Key, update.Value, (key, currentValue) => update.Value); + } + } + + /// + /// The user provides a filter expression accepting a Record, but we internally store it wrapped in an InMemoryVectorRecordWrapper. + /// This method converts a filter expression accepting a Record to one accepting an InMemoryVectorRecordWrapper. + /// + [RequiresUnreferencedCode("Filtering isn't supported with trimming.")] + private Expression, bool>> ConvertFilter(Expression> recordFilter) + { + var wrapperParameter = Expression.Parameter(typeof(InMemoryRecordWrapper), "w"); + var replacement = Expression.Property(wrapperParameter, nameof(InMemoryRecordWrapper.Record)); + + return Expression.Lambda, bool>>( + new ParameterReplacer(recordFilter.Parameters.Single(), replacement).Visit(recordFilter.Body), + wrapperParameter); + } + + private sealed class ParameterReplacer(ParameterExpression originalRecordParameter, Expression replacementExpression) : ExpressionVisitor + { + protected override Expression VisitParameter(ParameterExpression node) + => node == originalRecordParameter ? replacementExpression : base.VisitParameter(node); + } +} diff --git a/MEVD/src/InMemory/InMemoryCollectionOptions.cs b/MEVD/src/InMemory/InMemoryCollectionOptions.cs new file mode 100644 index 0000000..69a616a --- /dev/null +++ b/MEVD/src/InMemory/InMemoryCollectionOptions.cs @@ -0,0 +1,13 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.VectorData; + +namespace CommunityToolkit.VectorData.InMemory; + +/// +/// Options when creating a . +/// +public sealed class InMemoryCollectionOptions : VectorStoreCollectionOptions +{ +} diff --git a/MEVD/src/InMemory/InMemoryCollectionSearchMapping.cs b/MEVD/src/InMemory/InMemoryCollectionSearchMapping.cs new file mode 100644 index 0000000..b05f21d --- /dev/null +++ b/MEVD/src/InMemory/InMemoryCollectionSearchMapping.cs @@ -0,0 +1,86 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Numerics.Tensors; +using Microsoft.Extensions.VectorData; + +namespace CommunityToolkit.VectorData.InMemory; + +/// +/// Contains mapping helpers to use when searching for documents using the InMemory store. +/// +internal static class InMemoryCollectionSearchMapping +{ + /// + /// Compare the two vectors using the specified distance function. + /// + /// The first vector to compare. + /// The second vector to compare. + /// The distance function to use for comparison. + /// The score of the comparison. + /// Thrown when the distance function is not supported. + public static float CompareVectors(ReadOnlySpan x, ReadOnlySpan y, string? distanceFunction) + { + switch (distanceFunction) + { + case null: + case DistanceFunction.CosineSimilarity: + case DistanceFunction.CosineDistance: + return TensorPrimitives.CosineSimilarity(x, y); + case DistanceFunction.DotProductSimilarity: + return TensorPrimitives.Dot(x, y); + case DistanceFunction.EuclideanDistance: + return TensorPrimitives.Distance(x, y); + default: + throw new NotSupportedException($"The distance function '{distanceFunction}' is not supported by the InMemory connector."); + } + } + + /// + /// Indicates whether result ordering should be descending or ascending, to get most similar results at the top, based on the distance function. + /// + /// The distance function to use for comparison. + /// Whether to order descending or ascending. + /// Thrown when the distance function is not supported. + public static bool ShouldSortDescending(string? distanceFunction) + { + switch (distanceFunction) + { + case null: + case DistanceFunction.CosineSimilarity: + case DistanceFunction.DotProductSimilarity: + return true; + case DistanceFunction.CosineDistance: + case DistanceFunction.EuclideanDistance: + return false; + default: + throw new NotSupportedException($"The distance function '{distanceFunction}' is not supported by the InMemory connector."); + } + } + + /// + /// Converts the provided score into the correct result depending on the distance function. + /// The main purpose here is to convert from cosine similarity to cosine distance if cosine distance is requested, + /// since the two are inversely related and the only supports cosine similarity so + /// we are using cosine similarity for both similarity and distance. + /// + /// The score to convert. + /// The distance function to use for comparison. + /// Whether to order descending or ascending. + /// Thrown when the distance function is not supported. + public static float ConvertScore(float score, string? distanceFunction) + { + switch (distanceFunction) + { + case DistanceFunction.CosineDistance: + return 1 - score; + case null: + case DistanceFunction.CosineSimilarity: + case DistanceFunction.DotProductSimilarity: + case DistanceFunction.EuclideanDistance: + return score; + default: + throw new NotSupportedException($"The distance function '{distanceFunction}' is not supported by the InMemory connector."); + } + } +} diff --git a/MEVD/src/InMemory/InMemoryConstants.cs b/MEVD/src/InMemory/InMemoryConstants.cs new file mode 100644 index 0000000..f743363 --- /dev/null +++ b/MEVD/src/InMemory/InMemoryConstants.cs @@ -0,0 +1,9 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace CommunityToolkit.VectorData.InMemory; + +internal static class InMemoryConstants +{ + internal const string VectorStoreSystemName = "inmemory"; +} diff --git a/MEVD/src/InMemory/InMemoryDynamicCollection.cs b/MEVD/src/InMemory/InMemoryDynamicCollection.cs new file mode 100644 index 0000000..05ad511 --- /dev/null +++ b/MEVD/src/InMemory/InMemoryDynamicCollection.cs @@ -0,0 +1,50 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Concurrent; +using System.Diagnostics.CodeAnalysis; + +namespace CommunityToolkit.VectorData.InMemory; + +/// +/// Represents a collection of vector store records in a InMemory database, mapped to a dynamic Dictionary<string, object?>. +/// +#pragma warning disable CA1711 // Identifiers should not have incorrect suffix +public sealed class InMemoryDynamicCollection : InMemoryCollection> +#pragma warning restore CA1711 // Identifiers should not have incorrect suffix +{ + /// + /// Initializes a new instance of the class. + /// + /// The name of the collection. + /// Optional configuration options for this class. + [RequiresUnreferencedCode("The InMemory provider is incompatible with trimming.")] + [RequiresDynamicCode("The InMemory provider is incompatible with NativeAOT.")] + public InMemoryDynamicCollection(string name, InMemoryCollectionOptions options) + : base( + internalCollection: null, + internalCollectionTypes: null, + name, + static options => new InMemoryModelBuilder().BuildDynamic( + options.Definition ?? throw new ArgumentException("Definition is required for dynamic collections"), + options.EmbeddingGenerator), + options) + { + } + + internal InMemoryDynamicCollection( + ConcurrentDictionary> internalCollection, + ConcurrentDictionary internalCollectionTypes, + string name, + InMemoryCollectionOptions options) + : base( + internalCollection, + internalCollectionTypes, + name, + static options => new InMemoryModelBuilder().BuildDynamic( + options.Definition ?? throw new ArgumentException("Definition is required for dynamic collections"), + options.EmbeddingGenerator), + options) + { + } +} diff --git a/MEVD/src/InMemory/InMemoryModelBuilder.cs b/MEVD/src/InMemory/InMemoryModelBuilder.cs new file mode 100644 index 0000000..b11420c --- /dev/null +++ b/MEVD/src/InMemory/InMemoryModelBuilder.cs @@ -0,0 +1,52 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData.ProviderServices; + +namespace CommunityToolkit.VectorData.InMemory; + +internal class InMemoryModelBuilder() : CollectionModelBuilder(ValidationOptions) +{ + internal const string SupportedVectorTypes = "ReadOnlyMemory, Embedding, float[]"; + + internal static readonly CollectionModelBuildingOptions ValidationOptions = new() + { + RequiresAtLeastOneVector = false, + SupportsMultipleVectors = true + }; + + protected override void ValidateKeyProperty(KeyPropertyModel keyProperty) + { + base.ValidateKeyProperty(keyProperty); + + // All .NET types are supported by the InMemory provider, but we support auto-generation of keys only for GUIDs + if (keyProperty.IsAutoGenerated && keyProperty.Type != typeof(Guid)) + { + throw new NotSupportedException( + $"Auto-generation is only supported for key properties of type Guid. Property '{keyProperty.ModelName}' has type '{keyProperty.Type.Name}'."); + } + } + + protected override bool IsDataPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes) + { + supportedTypes = ""; + + // All .NET types are supported by the InMemory provider + return true; + } + + protected override bool IsVectorPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes) + => IsVectorPropertyTypeValidCore(type, out supportedTypes); + + internal static bool IsVectorPropertyTypeValidCore(Type type, [NotNullWhen(false)] out string? supportedTypes) + { + supportedTypes = SupportedVectorTypes; + + return type == typeof(ReadOnlyMemory) + || type == typeof(ReadOnlyMemory?) + || type == typeof(Embedding) + || type == typeof(float[]); + } +} diff --git a/MEVD/src/InMemory/InMemoryRecordWrapper.cs b/MEVD/src/InMemory/InMemoryRecordWrapper.cs new file mode 100644 index 0000000..f561fb9 --- /dev/null +++ b/MEVD/src/InMemory/InMemoryRecordWrapper.cs @@ -0,0 +1,24 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text.Json.Serialization; + +namespace CommunityToolkit.VectorData.InMemory; + +internal readonly struct InMemoryRecordWrapper +{ + public InMemoryRecordWrapper(TRecord record) + { + Record = record; + } + + [JsonConstructor] + public InMemoryRecordWrapper(TRecord record, Dictionary> embeddingGeneratedVectors) + { + Record = record; + EmbeddingGeneratedVectors = embeddingGeneratedVectors; + } + + public TRecord Record { get; } + public Dictionary> EmbeddingGeneratedVectors { get; } = []; +} diff --git a/MEVD/src/InMemory/InMemoryServiceCollectionExtensions.cs b/MEVD/src/InMemory/InMemoryServiceCollectionExtensions.cs new file mode 100644 index 0000000..b4413bb --- /dev/null +++ b/MEVD/src/InMemory/InMemoryServiceCollectionExtensions.cs @@ -0,0 +1,85 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.VectorData; +using CommunityToolkit.VectorData.InMemory; + +namespace Microsoft.SemanticKernel; + +/// +/// Extension methods to register Data services on an . +/// +public static class InMemoryServiceCollectionExtensions +{ + /// + /// Register an InMemory with the specified service ID. + /// + /// The to register the on. + /// Optional options to further configure the . + /// An optional service id to use as the service key. + /// The service collection. + [RequiresUnreferencedCode("The InMemory provider is incompatible with trimming.")] + [RequiresDynamicCode("The InMemory provider is incompatible with NativeAOT.")] + public static IServiceCollection AddInMemoryVectorStore(this IServiceCollection services, InMemoryVectorStoreOptions? options = default, string? serviceId = default) + { + services.AddKeyedTransient( + serviceId, + (sp, obj) => + { + options ??= sp.GetService() ?? new() + { + EmbeddingGenerator = sp.GetService() + }; + + return new InMemoryVectorStore(options); + }); + + services.AddKeyedSingleton(serviceId); + services.AddKeyedSingleton(serviceId, (sp, obj) => sp.GetRequiredKeyedService(serviceId)); + return services; + } + + /// + /// Register an InMemory and with the specified service ID. + /// + /// The type of the key. + /// The type of the record. + /// The to register the on. + /// The name of the collection. + /// Optional options to further configure the . + /// An optional service id to use as the service key. + /// The service collection. + [RequiresUnreferencedCode("The InMemory provider is incompatible with trimming.")] + [RequiresDynamicCode("The InMemory provider is incompatible with NativeAOT.")] + public static IServiceCollection AddInMemoryVectorStoreRecordCollection( + this IServiceCollection services, + string collectionName, + InMemoryCollectionOptions? options = default, + string? serviceId = default) + where TKey : notnull + where TRecord : class + { + services.AddKeyedSingleton>( + serviceId, + (sp, obj) => + { + options ??= sp.GetService() ?? new() + { + EmbeddingGenerator = sp.GetService() + }; + return (new InMemoryCollection(collectionName, options) as VectorStoreCollection)!; + }); + + services.AddKeyedSingleton>( + serviceId, + (sp, obj) => + { + return sp.GetRequiredKeyedService>(serviceId); + }); + + return services; + } +} diff --git a/MEVD/src/InMemory/InMemoryVectorStore.cs b/MEVD/src/InMemory/InMemoryVectorStore.cs new file mode 100644 index 0000000..3ce2c8f --- /dev/null +++ b/MEVD/src/InMemory/InMemoryVectorStore.cs @@ -0,0 +1,128 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Concurrent; +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; +using Microsoft.Shared.Diagnostics; + +namespace CommunityToolkit.VectorData.InMemory; + +/// +/// Service for storing and retrieving vector records, and managing vector record collections, that uses an in memory dictionary as the underlying storage. +/// +public sealed class InMemoryVectorStore : VectorStore +{ + /// Metadata about vector store. + private readonly VectorStoreMetadata _metadata; + + /// Internal storage for the record collection. + private readonly ConcurrentDictionary> _internalCollections; + + /// The data type of each collection, to enforce a single type per collection. + private readonly ConcurrentDictionary _internalCollectionTypes = new(); + + private readonly IEmbeddingGenerator? _embeddingGenerator; + + /// + /// Initializes a new instance of the class. + /// + /// Optional configuration options for this class + public InMemoryVectorStore(InMemoryVectorStoreOptions? options = default) + { + _internalCollections = new(); + _embeddingGenerator = options?.EmbeddingGenerator; + + _metadata = new() + { + VectorStoreSystemName = InMemoryConstants.VectorStoreSystemName, + }; + } + +#pragma warning disable IDE0090 // Use 'new(...)' + /// + [RequiresUnreferencedCode("The InMemory provider is incompatible with trimming.")] + [RequiresDynamicCode("The InMemory provider is incompatible with NativeAOT.")] +#if NET + public override InMemoryCollection GetCollection(string name, VectorStoreCollectionDefinition? definition = null) +#else + public override VectorStoreCollection GetCollection(string name, VectorStoreCollectionDefinition? definition = null) +#endif + { + if (typeof(TRecord) == typeof(Dictionary)) + { + throw new ArgumentException(VectorDataStrings.GetCollectionWithDictionaryNotSupported); + } + + if (_internalCollectionTypes.TryGetValue(name, out var existingCollectionDataType) && existingCollectionDataType != typeof(TRecord)) + { + throw new InvalidOperationException($"Collection '{name}' already exists and with data type '{existingCollectionDataType.Name}' so cannot be re-created with data type '{typeof(TRecord).Name}'."); + } + + var collection = new InMemoryCollection( + _internalCollections, + _internalCollectionTypes, + name, + new() + { + Definition = definition, + EmbeddingGenerator = _embeddingGenerator + }); + return collection!; + } + + /// + [RequiresUnreferencedCode("The InMemory provider is incompatible with trimming.")] + [RequiresDynamicCode("The InMemory provider is incompatible with NativeAOT.")] +#if NET + public override InMemoryDynamicCollection GetDynamicCollection(string name, VectorStoreCollectionDefinition definition) +#else + public override VectorStoreCollection> GetDynamicCollection(string name, VectorStoreCollectionDefinition definition) +#endif + => new InMemoryDynamicCollection( + _internalCollections, + _internalCollectionTypes, + name, + new() + { + Definition = definition, + EmbeddingGenerator = _embeddingGenerator, + } + ); +#pragma warning restore IDE0090 + + /// + public override IAsyncEnumerable ListCollectionNamesAsync(CancellationToken cancellationToken = default) + { + return _internalCollections.Keys.ToAsyncEnumerable(); + } + + /// + public override Task CollectionExistsAsync(string name, CancellationToken cancellationToken = default) + { + return _internalCollections.ContainsKey(name) ? Task.FromResult(true) : Task.FromResult(false); + } + + /// + public override Task EnsureCollectionDeletedAsync(string name, CancellationToken cancellationToken = default) + { + _internalCollections.TryRemove(name, out _); + _internalCollectionTypes.TryRemove(name, out _); + return Task.CompletedTask; + } + + /// + public override object? GetService(Type serviceType, object? serviceKey = null) + { + Throw.IfNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreMetadata) ? _metadata : + serviceType == typeof(ConcurrentDictionary>) ? _internalCollections : + serviceType.IsInstanceOfType(this) ? this : + null; + } +} diff --git a/MEVD/src/InMemory/InMemoryVectorStoreExtensions.cs b/MEVD/src/InMemory/InMemoryVectorStoreExtensions.cs new file mode 100644 index 0000000..96e3204 --- /dev/null +++ b/MEVD/src/InMemory/InMemoryVectorStoreExtensions.cs @@ -0,0 +1,99 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using System.Text.Json; +using Microsoft.Extensions.VectorData; + +namespace CommunityToolkit.VectorData.InMemory; + +/// +/// Extension methods for which allow: +/// 1. Serializing an instance of to a stream. +/// 2. Deserializing an instance of from a stream. +/// +public static class InMemoryVectorStoreExtensions +{ + /// + /// Serialize a to a stream as JSON. + /// + /// Type of the record key. + /// Type of the record. + /// Instance of used to retrieve the collection. + /// The collection name. + /// The stream to write the serialized JSON to. + /// The JSON serializer options to use. + [RequiresDynamicCode("This method is incompatible with NativeAOT.")] + [RequiresUnreferencedCode("This method is incompatible with trimming.")] + public static async Task SerializeCollectionAsJsonAsync( + this InMemoryVectorStore vectorStore, + string collectionName, + Stream stream, + JsonSerializerOptions? jsonSerializerOptions = null) + where TKey : notnull + where TRecord : class + { + // Get collection and verify that it exists. + var collection = vectorStore.GetCollection(collectionName); + var exists = await collection.CollectionExistsAsync().ConfigureAwait(false); + if (!exists) + { + throw new InvalidOperationException($"Collection '{collectionName}' does not exist."); + } + + var inMemoryCollection = collection as InMemoryCollection; + var records = inMemoryCollection!.GetCollectionDictionary(); + InMemoryRecordCollection recordCollection = new(collectionName, records); + + await JsonSerializer.SerializeAsync(stream, recordCollection, jsonSerializerOptions).ConfigureAwait(false); + } + + /// + /// Deserialize a to a stream as JSON. + /// + /// Type of the record key. + /// Type of the record. + /// Instance of used to retrieve the collection. + /// The stream to read the serialized JSON from. + [RequiresDynamicCode("This method is incompatible with NativeAOT.")] + [RequiresUnreferencedCode("This method is incompatible with trimming.")] + public static async Task?> DeserializeCollectionFromJsonAsync( + this InMemoryVectorStore vectorStore, + Stream stream) + where TKey : notnull + where TRecord : class + { + VectorStoreCollection? collection = null; + + using (StreamReader streamReader = new(stream)) + { + string result = streamReader.ReadToEnd(); + var recordCollection = JsonSerializer.Deserialize>>(result); + if (recordCollection is null) + { + throw new InvalidOperationException("Stream does not contain valid record collection JSON."); + } + + // Get and create collection if it doesn't exist. + collection = vectorStore.GetCollection(recordCollection.Name); + await collection.EnsureCollectionExistsAsync().ConfigureAwait(false); + var inMemoryCollection = collection as InMemoryCollection; + + // Upsert records. + inMemoryCollection!.UpdateCollectionDictionary(recordCollection.Records.ToDictionary(x => x.Key as object, x => x.Value as object)); + } + + return collection; + } + + #region private + /// Model class used when storing a . + private sealed class InMemoryRecordCollection(string name, IDictionary records) + where TKey : notnull + { + public string Name { get; init; } = name; + public IDictionary Records { get; init; } = records; + } + #endregion + +} diff --git a/MEVD/src/InMemory/InMemoryVectorStoreOptions.cs b/MEVD/src/InMemory/InMemoryVectorStoreOptions.cs new file mode 100644 index 0000000..1816884 --- /dev/null +++ b/MEVD/src/InMemory/InMemoryVectorStoreOptions.cs @@ -0,0 +1,17 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.AI; + +namespace CommunityToolkit.VectorData.InMemory; + +/// +/// Options when creating a . +/// +public sealed class InMemoryVectorStoreOptions +{ + /// + /// Gets or sets the default embedding generator to use when generating vectors embeddings with this vector store. + /// + public IEmbeddingGenerator? EmbeddingGenerator { get; set; } +} diff --git a/MEVD/src/PgVector/AssemblyInfo.cs b/MEVD/src/PgVector/AssemblyInfo.cs new file mode 100644 index 0000000..d6395e8 --- /dev/null +++ b/MEVD/src/PgVector/AssemblyInfo.cs @@ -0,0 +1,2 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. diff --git a/MEVD/src/PgVector/PgVector.csproj b/MEVD/src/PgVector/PgVector.csproj new file mode 100644 index 0000000..4a96ffe --- /dev/null +++ b/MEVD/src/PgVector/PgVector.csproj @@ -0,0 +1,32 @@ + + + + 1.0.0-preview.1 + CommunityToolkit.VectorData.PgVector + $(AssemblyName) + net10.0;net8.0;netstandard2.0;net462 + true + $(NoWarn);CS0436 + + Postgres provider for Microsoft.Extensions.VectorData + Postgres (with pgvector extension) provider for Microsoft.Extensions.VectorData by Semantic Kernel + + + + + + + + + 8.0.7 + + + + + + + + + + + diff --git a/MEVD/src/PgVector/PostgresCollection.cs b/MEVD/src/PgVector/PostgresCollection.cs new file mode 100644 index 0000000..96025fe --- /dev/null +++ b/MEVD/src/PgVector/PostgresCollection.cs @@ -0,0 +1,591 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Linq.Expressions; +using System.Runtime.CompilerServices; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; +using Npgsql; +using Pgvector; +using Microsoft.Shared.Diagnostics; + +namespace CommunityToolkit.VectorData.PgVector; + +/// +/// Represents a collection of vector store records in a Postgres database. +/// +/// The type of the key. +/// The type of the record. +#pragma warning disable CA1711 // Identifiers should not have incorrect suffix +public class PostgresCollection : VectorStoreCollection, IKeywordHybridSearchable +#pragma warning restore CA1711 // Identifiers should not have incorrect suffix + where TKey : notnull + where TRecord : class +{ + /// + public override string Name { get; } + + /// Metadata about vector store record collection. + private readonly VectorStoreCollectionMetadata _collectionMetadata; + + /// Data source used to interact with the database. + private readonly NpgsqlDataSource _dataSource; + private readonly NpgsqlDataSourceArc? _dataSourceArc; + private readonly string _databaseName; + + /// The database schema. + private readonly string? _schema; + + /// The model for this collection. + private readonly CollectionModel _model; + + /// A mapper to use for converting between the data model and the Azure AI Search record. + private readonly PostgresMapper _mapper; + + /// The default options for vector search. + private static readonly VectorSearchOptions s_defaultVectorSearchOptions = new(); + + /// The default options for hybrid search. + private static readonly HybridSearchOptions s_defaultHybridSearchOptions = new(); + + /// + /// Initializes a new instance of the class. + /// + /// The data source to use for connecting to the database. + /// The name of the collection. + /// A value indicating whether is disposed when the collection is disposed. + /// Optional configuration options for this class. + [RequiresDynamicCode("This constructor is incompatible with NativeAOT. For dynamic mapping via Dictionary, instantiate PostgresDynamicCollection instead.")] + [RequiresUnreferencedCode("This constructor is incompatible with trimming. For dynamic mapping via Dictionary, instantiate PostgresDynamicCollection instead")] + public PostgresCollection(NpgsqlDataSource dataSource, string name, bool ownsDataSource, PostgresCollectionOptions? options = default) + : this(dataSource, ownsDataSource ? new NpgsqlDataSourceArc(dataSource) : null, name, options) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// Postgres database connection string. + /// The name of the collection. + /// Optional configuration options for this class. + [RequiresDynamicCode("This constructor is incompatible with NativeAOT. For dynamic mapping via Dictionary, instantiate PostgresDynamicCollection instead.")] + [RequiresUnreferencedCode("This constructor is incompatible with trimming. For dynamic mapping via Dictionary, instantiate PostgresDynamicCollection instead")] + public PostgresCollection(string connectionString, string name, PostgresCollectionOptions? options = default) + : this(PostgresUtils.CreateDataSource(connectionString), name, ownsDataSource: true, options) + { + Throw.IfNullOrWhitespace(connectionString); + } + + [RequiresDynamicCode("This constructor is incompatible with NativeAOT. For dynamic mapping via Dictionary, instantiate PostgresDynamicCollection instead.")] + [RequiresUnreferencedCode("This constructor is incompatible with trimming. For dynamic mapping via Dictionary, instantiate PostgresDynamicCollection instead.")] + internal PostgresCollection(NpgsqlDataSource dataSource, NpgsqlDataSourceArc? dataSourceArc, string name, PostgresCollectionOptions? options) + : this( + dataSource, + dataSourceArc, + name, + static options => typeof(TRecord) == typeof(Dictionary) + ? throw new NotSupportedException(VectorDataStrings.NonDynamicCollectionWithDictionaryNotSupported(typeof(PostgresDynamicCollection))) + : new PostgresModelBuilder().Build(typeof(TRecord), typeof(TKey), options.Definition, options.EmbeddingGenerator), + options) + { + } + + internal PostgresCollection(NpgsqlDataSource dataSource, NpgsqlDataSourceArc? dataSourceArc, string name, Func modelFactory, PostgresCollectionOptions? options) + { + Throw.IfNullOrWhitespace(name); + + options ??= PostgresCollectionOptions.Default; + + Name = name; + _model = modelFactory(options); + _mapper = new PostgresMapper(_model); + + _dataSource = dataSource; + _dataSourceArc = dataSourceArc; + _databaseName = new NpgsqlConnectionStringBuilder(dataSource.ConnectionString).Database!; + _schema = options.Schema; + + _collectionMetadata = new() + { + VectorStoreSystemName = PostgresConstants.VectorStoreSystemName, + VectorStoreName = _databaseName, + CollectionName = name + }; + + // Don't add any lines after this - an exception thrown afterwards would leave the reference count wrongly incremented. + _dataSourceArc?.IncrementReferenceCount(); + } + + /// + protected override void Dispose(bool disposing) + { + _dataSourceArc?.Dispose(); + base.Dispose(disposing); + } + + /// + public override Task CollectionExistsAsync(CancellationToken cancellationToken = default) + => RunOperationAsync("DoesTableExists", async () => + { + NpgsqlConnection connection = await _dataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false); + + await using (connection) + using (var command = connection.CreateCommand()) + { + PostgresSqlBuilder.BuildDoesTableExistCommand(command, _schema, Name); + using NpgsqlDataReader dataReader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + + if (await dataReader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + return dataReader.GetString(dataReader.GetOrdinal("table_name")) == Name; + } + + return false; + } + }); + + /// + public override Task EnsureCollectionExistsAsync(CancellationToken cancellationToken = default) + { + return RunOperationAsync("EnsureCollectionExists", () => + InternalCreateCollectionAsync(true, cancellationToken)); + } + + /// + public override async Task EnsureCollectionDeletedAsync(CancellationToken cancellationToken = default) + { + using var connection = await _dataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false); + using var command = connection.CreateCommand(); + PostgresSqlBuilder.BuildDropTableCommand(command, _schema, Name); + + await RunOperationAsync("DeleteCollection", () => command.ExecuteNonQueryAsync()).ConfigureAwait(false); + } + + /// + public override Task UpsertAsync(TRecord record, CancellationToken cancellationToken = default) + => UpsertAsync([record], cancellationToken); + + /// + public override async Task UpsertAsync(IEnumerable records, CancellationToken cancellationToken = default) + { + Throw.IfNull(records); + IReadOnlyList? recordsList = null; + + // If an embedding generator is defined, invoke it once per property for all records. + Dictionary>? generatedEmbeddings = null; + + var vectorPropertyCount = _model.VectorProperties.Count; + for (var i = 0; i < vectorPropertyCount; i++) + { + var vectorProperty = _model.VectorProperties[i]; + + if (PostgresModelBuilder.IsVectorPropertyTypeValidCore(vectorProperty.Type, out _)) + { + continue; + } + + // We have a vector property whose type isn't natively supported - we need to generate embeddings. + Debug.Assert(vectorProperty.EmbeddingGenerator is not null); + + // Materialize the records' enumerable if needed, to prevent multiple enumeration. + if (recordsList is null) + { + recordsList = records is IReadOnlyList r ? r : records.ToList(); + + if (recordsList.Count == 0) + { + return; + } + + records = recordsList; + } + + // TODO: Ideally we'd group together vector properties using the same generator (and with the same input and output properties), + // and generate embeddings for them in a single batch. That's some more complexity though. + generatedEmbeddings ??= new Dictionary>(vectorPropertyCount); + generatedEmbeddings[vectorProperty] = await vectorProperty.GenerateEmbeddingsAsync(records.Select(r => vectorProperty.GetValueAsObject(r)), cancellationToken).ConfigureAwait(false); + } + + using var connection = await _dataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false); + using var batch = connection.CreateBatch(); + + // If key auto-generation is enabled, we'll need to enumerate over the records multiple times: + // once to generate the upsert batch, and again to populate the retrieved keys into the records. + // To prevent multiple enumeration of the input enumerable, we materialize it here if needed. + if (_model.KeyProperty.IsAutoGenerated && recordsList is null) + { + recordsList = records is IReadOnlyList r ? r : records.ToList(); + + if (recordsList.Count == 0) + { + return; + } + + records = recordsList; + } + + if (PostgresSqlBuilder.BuildUpsertCommand(batch, _schema, Name, _model, records, generatedEmbeddings)) + { + await RunOperationAsync("Upsert", async () => + { + var reader = await batch.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + try + { + var keyProperty = _model.KeyProperty; + if (keyProperty.IsAutoGenerated) + { + int? keyOrdinal = null; + + foreach (var record in recordsList!) + { + if (Equals(keyProperty.GetValueAsObject(record), default(TKey))) + { + keyOrdinal ??= reader.GetOrdinal(keyProperty.StorageName); + + await reader.ReadAsync(cancellationToken).ConfigureAwait(false); + var keyValue = reader.GetFieldValue(keyOrdinal.Value); + _model.KeyProperty.SetValue(record, keyValue); + await reader.NextResultAsync(cancellationToken).ConfigureAwait(false); + } + } + } + } + finally + { + await reader.DisposeAsync().ConfigureAwait(false); + } + }).ConfigureAwait(false); + } + } + + /// + public override async Task GetAsync(TKey key, RecordRetrievalOptions? options = null, CancellationToken cancellationToken = default) + { + Throw.IfNull(key); + + bool includeVectors = options?.IncludeVectors is true; + if (includeVectors && _model.EmbeddingGenerationRequired) + { + throw new NotSupportedException(VectorDataStrings.IncludeVectorsNotSupportedWithEmbeddingGeneration); + } + + using var connection = await _dataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false); + using var command = connection.CreateCommand(); + PostgresSqlBuilder.BuildGetCommand(command, _schema, Name, _model, key, includeVectors); + + return await connection.ExecuteWithErrorHandlingAsync( + _collectionMetadata, + operationName: "Get", + async () => + { + using NpgsqlDataReader reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + await reader.ReadAsync(cancellationToken).ConfigureAwait(false); + return reader.HasRows + ? _mapper.MapFromStorageToDataModel(reader, includeVectors) + : null; + }, + cancellationToken).ConfigureAwait(false); + } + + /// + public override async IAsyncEnumerable GetAsync( + IEnumerable keys, + RecordRetrievalOptions? options = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + const string OperationName = "GetBatch"; + + Throw.IfNull(keys); + + bool includeVectors = options?.IncludeVectors is true; + if (includeVectors && _model.EmbeddingGenerationRequired) + { + throw new NotSupportedException(VectorDataStrings.IncludeVectorsNotSupportedWithEmbeddingGeneration); + } + + List listOfKeys = keys.ToList(); + if (listOfKeys.Count == 0) + { + yield break; + } + + using var connection = await _dataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false); + using var command = connection.CreateCommand(); + PostgresSqlBuilder.BuildGetBatchCommand(command, _schema, Name, _model, listOfKeys, includeVectors); + + using var reader = await connection.ExecuteWithErrorHandlingAsync( + _collectionMetadata, + OperationName, + () => command.ExecuteReaderAsync(cancellationToken), + cancellationToken).ConfigureAwait(false); + + while (await reader.ReadWithErrorHandlingAsync(_collectionMetadata, OperationName, cancellationToken).ConfigureAwait(false)) + { + yield return _mapper.MapFromStorageToDataModel(reader, includeVectors); + } + } + + /// + public override async Task DeleteAsync(TKey key, CancellationToken cancellationToken = default) + { + using var connection = await _dataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false); + using var command = connection.CreateCommand(); + PostgresSqlBuilder.BuildDeleteCommand(command, _schema, Name, _model.KeyProperty.StorageName, key); + + await RunOperationAsync("Delete", () => command.ExecuteNonQueryAsync(cancellationToken)).ConfigureAwait(false); + } + + /// + public override async Task DeleteAsync(IEnumerable keys, CancellationToken cancellationToken = default) + { + Throw.IfNull(keys); + + var listOfKeys = keys.ToList(); + if (listOfKeys.Count == 0) + { + return; + } + + using var connection = await _dataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false); + using var command = connection.CreateCommand(); + PostgresSqlBuilder.BuildDeleteBatchCommand(command, _schema, Name, _model.KeyProperty, listOfKeys); + + await RunOperationAsync("DeleteBatch", () => command.ExecuteNonQueryAsync(cancellationToken)).ConfigureAwait(false); + } + + #region Search + + /// + public override async IAsyncEnumerable> SearchAsync( + TInput searchValue, + int top, + VectorSearchOptions? options = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + Throw.IfNull(searchValue); + Throw.IfLessThan(top, 1); + + options ??= s_defaultVectorSearchOptions; + if (options.IncludeVectors && _model.EmbeddingGenerationRequired) + { + throw new NotSupportedException(VectorDataStrings.IncludeVectorsNotSupportedWithEmbeddingGeneration); + } + + var vectorProperty = _model.GetVectorPropertyOrSingle(options); + var pgVector = await ConvertSearchInputToVectorAsync(searchValue, vectorProperty, cancellationToken).ConfigureAwait(false); + + // Simulating skip/offset logic locally, since OFFSET can work only with LIMIT in combination + // and LIMIT is not supported in vector search extension, instead of LIMIT - "k" parameter is used. + var limit = top + options.Skip; + + using var connection = await _dataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false); + using var command = connection.CreateCommand(); + PostgresSqlBuilder.BuildGetNearestMatchCommand(command, _schema, Name, _model, vectorProperty, pgVector, + options.Filter, options.Skip, options.IncludeVectors, top, options.ScoreThreshold); + + using var reader = await connection.ExecuteWithErrorHandlingAsync( + _collectionMetadata, + "Search", + () => command.ExecuteReaderAsync(cancellationToken), + cancellationToken).ConfigureAwait(false); + + while (await reader.ReadWithErrorHandlingAsync(_collectionMetadata, "Search", cancellationToken).ConfigureAwait(false)) + { + yield return new VectorSearchResult( + _mapper.MapFromStorageToDataModel(reader, options.IncludeVectors), + reader.GetDouble(reader.GetOrdinal(PostgresConstants.DistanceColumnName))); + } + } + + /// + public async IAsyncEnumerable> HybridSearchAsync( + TInput searchValue, + ICollection keywords, + int top, + HybridSearchOptions? options = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + where TInput : notnull + { + Throw.IfNull(searchValue); + Throw.IfNull(keywords); + Throw.IfLessThan(top, 1); + + options ??= s_defaultHybridSearchOptions; + if (options.IncludeVectors && _model.EmbeddingGenerationRequired) + { + throw new NotSupportedException(VectorDataStrings.IncludeVectorsNotSupportedWithEmbeddingGeneration); + } + + var vectorProperty = _model.GetVectorPropertyOrSingle(new() { VectorProperty = options.VectorProperty }); + var textProperty = _model.GetFullTextDataPropertyOrSingle(options.AdditionalProperty); + var pgVector = await ConvertSearchInputToVectorAsync(searchValue, vectorProperty, cancellationToken).ConfigureAwait(false); + + using var connection = await _dataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false); + using var command = connection.CreateCommand(); + PostgresSqlBuilder.BuildHybridSearchCommand(command, _schema, Name, _model, vectorProperty, textProperty, pgVector, keywords, + options.Filter, options.Skip, options.IncludeVectors, top, options.ScoreThreshold); + + using var reader = await connection.ExecuteWithErrorHandlingAsync( + _collectionMetadata, + "HybridSearch", + () => command.ExecuteReaderAsync(cancellationToken), + cancellationToken).ConfigureAwait(false); + + while (await reader.ReadWithErrorHandlingAsync(_collectionMetadata, "HybridSearch", cancellationToken).ConfigureAwait(false)) + { + yield return new VectorSearchResult( + _mapper.MapFromStorageToDataModel(reader, options.IncludeVectors), + reader.GetDouble(reader.GetOrdinal(PostgresConstants.DistanceColumnName))); + } + } + + #endregion Search + + /// + public override async IAsyncEnumerable GetAsync( + Expression> filter, + int top, + FilteredRecordRetrievalOptions? options = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + Throw.IfNull(filter); + Throw.IfLessThan(top, 1); + + options ??= new(); + + using var connection = await _dataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false); + using var command = connection.CreateCommand(); + PostgresSqlBuilder.BuildSelectWhereCommand(command, _schema, Name, _model, filter, top, options); + using NpgsqlDataReader reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + + while (await reader.ReadWithErrorHandlingAsync(_collectionMetadata, "Get", cancellationToken).ConfigureAwait(false)) + { + yield return _mapper.MapFromStorageToDataModel(reader, options.IncludeVectors); + } + } + + /// + public override object? GetService(Type serviceType, object? serviceKey = null) + { + Throw.IfNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreCollectionMetadata) ? _collectionMetadata : + serviceType == typeof(NpgsqlDataSource) ? _dataSource : + serviceType.IsInstanceOfType(this) ? this : + null; + } + + private async Task InternalCreateCollectionAsync(bool ifNotExists, CancellationToken cancellationToken = default) + { + // Execute the commands in a transaction. + NpgsqlConnection connection = await _dataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false); + + await using (connection) + { + var pgVersion = connection.PostgreSqlVersion; + + // Prepare the SQL commands. + using var batch = connection.CreateBatch(); + + // First, check if the pgvector extension is already installed in PostgreSQL, and then install it if not. + // Note that we do a separate check before doing CREATE EXTENSION IF EXISTS in order to know if it was actually created, + // since in that case we must also must call ReloadTypesAsync() at the Npgsql level + batch.BatchCommands.Add(new NpgsqlBatchCommand("SELECT EXISTS(SELECT * FROM pg_extension WHERE extname='vector')")); + batch.BatchCommands.Add(new NpgsqlBatchCommand("CREATE EXTENSION IF NOT EXISTS vector")); + + bool extensionAlreadyExisted; + + try + { + extensionAlreadyExisted = (bool)(await batch.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false))!; + } + catch (PostgresException e) when (e.SqlState == PostgresErrorCodes.UniqueViolation) + { + // CREATE EXTENSION IF NOT EXISTS is not atomic in PG, so concurrent sessions doing this at the same time + // may trigger a unique constraint violation. We ignore it and interpret it to mean that the extension + // already exists. + extensionAlreadyExisted = true; + } + + if (!extensionAlreadyExisted) + { + await connection.ReloadTypesAsync().ConfigureAwait(false); + } + + batch.BatchCommands.Clear(); + + // Now create the table and any indexes. + // Note that this must be done in a separate batch/transaction as the creation of the extension above. + batch.BatchCommands.Add( + new NpgsqlBatchCommand(PostgresSqlBuilder.BuildCreateTableSql(_schema, Name, _model, pgVersion, ifNotExists))); + + foreach (var (column, kind, function, isVector, isFullText, fullTextLanguage) in PostgresPropertyMapping.GetIndexInfo(_model.Properties)) + { + batch.BatchCommands.Add( + new NpgsqlBatchCommand( + PostgresSqlBuilder.BuildCreateIndexSql(_schema, Name, column, kind, function, isVector, isFullText, fullTextLanguage, ifNotExists))); + } + + await batch.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + } + + private Task RunOperationAsync(string operationName, Func operation) + => VectorStoreErrorHandler.RunOperationAsync( + _collectionMetadata, + operationName, + operation); + + private Task RunOperationAsync(string operationName, Func> operation) + => VectorStoreErrorHandler.RunOperationAsync( + _collectionMetadata, + operationName, + operation); + + /// + /// Converts a search input value to a PostgreSQL vector representation, generating embeddings if necessary. + /// + private async Task ConvertSearchInputToVectorAsync(TInput searchValue, VectorPropertyModel vectorProperty, CancellationToken cancellationToken) + where TInput : notnull + { + object vector = searchValue switch + { + // Dense float32 + ReadOnlyMemory r => r, + float[] f => new ReadOnlyMemory(f), + Embedding e => e.Vector, + +#if NET + // Dense float16 + ReadOnlyMemory r => r, + Half[] f => new ReadOnlyMemory(f), + Embedding e => e.Vector, +#endif + + // Dense Binary + BitArray b => b, + BinaryEmbedding e => e.Vector, + + // Sparse + SparseVector sv => sv, + // TODO: Add a PG-specific SparseVectorEmbedding type + + _ when vectorProperty.EmbeddingGenerationDispatcher is not null + => await vectorProperty.GenerateEmbeddingAsync(searchValue, cancellationToken).ConfigureAwait(false), + + _ => vectorProperty.EmbeddingGenerator is null + ? throw new NotSupportedException(VectorDataStrings.InvalidSearchInputAndNoEmbeddingGeneratorWasConfigured(searchValue.GetType(), PostgresModelBuilder.SupportedVectorTypes)) + : throw new InvalidOperationException(VectorDataStrings.IncompatibleEmbeddingGeneratorWasConfiguredForInputType(typeof(TInput), vectorProperty.EmbeddingGenerator.GetType())) + }; + + var pgVector = PostgresPropertyMapping.MapVectorForStorageModel(vector); + Throw.IfNull(pgVector); + return pgVector; + } +} diff --git a/MEVD/src/PgVector/PostgresCollectionOptions.cs b/MEVD/src/PgVector/PostgresCollectionOptions.cs new file mode 100644 index 0000000..f0f2772 --- /dev/null +++ b/MEVD/src/PgVector/PostgresCollectionOptions.cs @@ -0,0 +1,31 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.VectorData; + +namespace CommunityToolkit.VectorData.PgVector; + +/// +/// Options when creating a . +/// +public sealed class PostgresCollectionOptions : VectorStoreCollectionOptions +{ + internal static readonly PostgresCollectionOptions Default = new(); + + /// + /// Initializes a new instance of the class. + /// + public PostgresCollectionOptions() + { + } + + internal PostgresCollectionOptions(PostgresCollectionOptions? source) : base(source) + { + Schema = source?.Schema; + } + + /// + /// Gets or sets the database schema. + /// + public string? Schema { get; set; } +} diff --git a/MEVD/src/PgVector/PostgresConstants.cs b/MEVD/src/PgVector/PostgresConstants.cs new file mode 100644 index 0000000..318a214 --- /dev/null +++ b/MEVD/src/PgVector/PostgresConstants.cs @@ -0,0 +1,34 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.VectorData; + +namespace CommunityToolkit.VectorData.PgVector; + +internal static class PostgresConstants +{ + /// The name of this vector store for telemetry purposes. + public const string VectorStoreSystemName = "postgresql"; + + /// The default schema name. + public const string DefaultSchema = "public"; + + /// The name of the column that returns distance value in the database. + /// It is used in the similarity search query. Must not conflict with model property. + public const string DistanceColumnName = "sk_pg_distance"; + + /// The default index kind. + /// Defaults to "Flat", which means no indexing. + public const string DefaultIndexKind = IndexKind.Flat; + + /// The default distance function. + public const string DefaultDistanceFunction = DistanceFunction.CosineDistance; + + /// The default full-text search language for PostgreSQL. + public const string DefaultFullTextSearchLanguage = "english"; + + public static readonly Dictionary IndexMaxDimensions = new() + { + { IndexKind.Hnsw, 2000 }, + }; +} diff --git a/MEVD/src/PgVector/PostgresDbClient.cs b/MEVD/src/PgVector/PostgresDbClient.cs new file mode 100644 index 0000000..9d15da9 --- /dev/null +++ b/MEVD/src/PgVector/PostgresDbClient.cs @@ -0,0 +1,29 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Npgsql; + +namespace CommunityToolkit.VectorData.PgVector; + +/// +/// A reference-counting wrapper around an instance. +/// +internal sealed class NpgsqlDataSourceArc(NpgsqlDataSource dataSource) : IDisposable +{ + private int _referenceCount = 1; + + public void Dispose() + { + if (Interlocked.Decrement(ref _referenceCount) == 0) + { + dataSource.Dispose(); + } + } + + internal NpgsqlDataSourceArc IncrementReferenceCount() + { + Interlocked.Increment(ref _referenceCount); + + return this; + } +} diff --git a/MEVD/src/PgVector/PostgresDynamicCollection.cs b/MEVD/src/PgVector/PostgresDynamicCollection.cs new file mode 100644 index 0000000..00482dc --- /dev/null +++ b/MEVD/src/PgVector/PostgresDynamicCollection.cs @@ -0,0 +1,49 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Npgsql; + +namespace CommunityToolkit.VectorData.PgVector; + +/// +/// Represents a collection of vector store records in a Postgres database, mapped to a dynamic Dictionary<string, object?>. +/// +#pragma warning disable CA1711 // Identifiers should not have incorrect suffix +public sealed class PostgresDynamicCollection : PostgresCollection> +#pragma warning restore CA1711 // Identifiers should not have incorrect suffix +{ + /// + /// Initializes a new instance of the class. + /// + /// The data source to use for connecting to the database. + /// The name of the collection. + /// A value indicating whether the data source should be disposed when the collection is disposed. + /// Optional configuration options for this class. + public PostgresDynamicCollection(NpgsqlDataSource dataSource, string name, bool ownsDataSource, PostgresCollectionOptions options) + : this(dataSource, ownsDataSource ? new NpgsqlDataSourceArc(dataSource) : null, name, options) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// Postgres database connection string. + /// The name of the collection. + /// Optional configuration options for this class. + public PostgresDynamicCollection(string connectionString, string name, PostgresCollectionOptions options) + : this(PostgresUtils.CreateDataSource(connectionString), name, ownsDataSource: true, options) + { + } + + internal PostgresDynamicCollection(NpgsqlDataSource dataSource, NpgsqlDataSourceArc? dataSourceArc, string name, PostgresCollectionOptions options) + : base( + dataSource, + dataSourceArc, + name, + static options => new PostgresModelBuilder().BuildDynamic( + options.Definition ?? throw new ArgumentException("Definition is required for dynamic collections"), + options.EmbeddingGenerator), + options) + { + } +} diff --git a/MEVD/src/PgVector/PostgresFilterTranslator.cs b/MEVD/src/PgVector/PostgresFilterTranslator.cs new file mode 100644 index 0000000..a2a99b5 --- /dev/null +++ b/MEVD/src/PgVector/PostgresFilterTranslator.cs @@ -0,0 +1,425 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections; +using System.Collections.Generic; +using System.Diagnostics; +using System.Globalization; +using System.Linq; +using System.Linq.Expressions; +using System.Text; +using Microsoft.Extensions.VectorData.ProviderServices; +using Microsoft.Extensions.VectorData.ProviderServices.Filter; + +namespace CommunityToolkit.VectorData.PgVector; + +internal sealed class PostgresFilterTranslator : FilterTranslatorBase +{ + private readonly StringBuilder _sql; + private readonly Expression _preprocessedExpression; + private int _parameterIndex; + + internal PostgresFilterTranslator( + CollectionModel model, + LambdaExpression lambdaExpression, + int startParamIndex, + StringBuilder? sql = null) + { + Debug.Assert(lambdaExpression.Parameters.Count == 1); + _sql = sql ?? new(); + _parameterIndex = startParamIndex; + + _preprocessedExpression = PreprocessFilter(lambdaExpression, model, new FilterPreprocessingOptions { SupportsParameterization = true }); + } + + internal StringBuilder Clause => _sql; + + internal List ParameterValues { get; } = []; + + internal void Translate(bool appendWhere) + { + if (appendWhere) + { + _sql.Append("WHERE "); + } + + Translate(_preprocessedExpression); + } + + private void Translate(Expression? node) + { + switch (node) + { + case BinaryExpression binary: + TranslateBinary(binary); + return; + + case ConstantExpression constant: + TranslateConstant(constant.Value); + return; + + case QueryParameterExpression { Name: var name, Value: var value }: + TranslateQueryParameter(value); + return; + + case MemberExpression member: + TranslateMember(member); + return; + + case MethodCallExpression methodCall: + TranslateMethodCall(methodCall); + return; + + case UnaryExpression unary: + TranslateUnary(unary); + return; + + default: + throw new NotSupportedException("Unsupported NodeType in filter: " + node?.NodeType); + } + } + + private void TranslateBinary(BinaryExpression binary) + { + // Special handling for null comparisons + switch (binary.NodeType) + { + case ExpressionType.Equal when IsNull(binary.Right): + _sql.Append('('); + Translate(binary.Left); + _sql.Append(" IS NULL)"); + return; + case ExpressionType.NotEqual when IsNull(binary.Right): + _sql.Append('('); + Translate(binary.Left); + _sql.Append(" IS NOT NULL)"); + return; + + case ExpressionType.Equal when IsNull(binary.Left): + _sql.Append('('); + Translate(binary.Right); + _sql.Append(" IS NULL)"); + return; + case ExpressionType.NotEqual when IsNull(binary.Left): + _sql.Append('('); + Translate(binary.Right); + _sql.Append(" IS NOT NULL)"); + return; + } + + _sql.Append('('); + Translate(binary.Left); + + _sql.Append(binary.NodeType switch + { + ExpressionType.Equal => " = ", + ExpressionType.NotEqual => " <> ", + + ExpressionType.GreaterThan => " > ", + ExpressionType.GreaterThanOrEqual => " >= ", + ExpressionType.LessThan => " < ", + ExpressionType.LessThanOrEqual => " <= ", + + ExpressionType.AndAlso => " AND ", + ExpressionType.OrElse => " OR ", + + _ => throw new NotSupportedException("Unsupported binary expression node type: " + binary.NodeType) + }); + + Translate(binary.Right); + + _sql.Append(')'); + + static bool IsNull(Expression expression) + => expression is ConstantExpression { Value: null } or QueryParameterExpression { Value: null }; + } + + private void TranslateConstant(object? value) + { + switch (value) + { + case byte b: + _sql.Append(b); + return; + case short s: + _sql.Append(s); + return; + case int i: + _sql.Append(i); + return; + case long l: + _sql.Append(l); + return; + + case float f: + _sql.Append(f); + return; + case double d: + _sql.Append(d); + return; + case decimal d: + _sql.Append(d); + return; + + case string untrustedInput: + _sql.Append('\'').Append(untrustedInput.Replace("'", "''")).Append('\''); + return; + case bool b: + _sql.Append(b ? "TRUE" : "FALSE"); + return; + case Guid g: + _sql.Append('\'').Append(g.ToString()).Append('\''); + return; + + case DateTime dateTime: + switch (dateTime.Kind) + { + case DateTimeKind.Utc: + _sql.Append('\'').Append(dateTime.ToString("yyyy-MM-ddTHH:mm:ss.FFFFFFZ", CultureInfo.InvariantCulture)).Append('\''); + return; + + case DateTimeKind.Unspecified: + case DateTimeKind.Local: + _sql.Append('\'').Append(dateTime.ToString("yyyy-MM-ddTHH:mm:ss.FFFFFF", CultureInfo.InvariantCulture)).Append('\''); + return; + + default: + throw new UnreachableException(); + } + + case DateTimeOffset dateTimeOffset: + if (dateTimeOffset.Offset != TimeSpan.Zero) + { + throw new NotSupportedException("DateTimeOffset with non-zero offset is not supported with PostgreSQL. Use DateTimeOffset.UtcNow or convert to UTC."); + } + + _sql.Append('\'').Append(dateTimeOffset.ToString("yyyy-MM-ddTHH:mm:ss.FFFFFFZ", CultureInfo.InvariantCulture)).Append('\''); + return; + +#if NET + case DateOnly dateOnly: + _sql.Append('\'').Append(dateOnly.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture)).Append('\''); + return; + + case TimeOnly timeOnly: + _sql.Append('\'').Append(timeOnly.ToString("HH:mm:ss.FFFFFFF", CultureInfo.InvariantCulture)).Append('\''); + return; +#endif + + // Array constants (ARRAY[1, 2, 3]) + case IEnumerable v when v.GetType() is var type && (type.IsArray || type.IsGenericType && type.GetGenericTypeDefinition() == typeof(List<>)): + _sql.Append("ARRAY["); + + var arrayIndex = 0; + foreach (var element in v) + { + if (arrayIndex++ > 0) + { + _sql.Append(','); + } + + TranslateConstant(element); + } + + _sql.Append(']'); + return; + + case null: + _sql.Append("NULL"); + return; + + default: + throw new NotSupportedException("Unsupported constant type: " + value.GetType().Name); + } + } + + private void TranslateMember(MemberExpression memberExpression) + { + if (TryBindProperty(memberExpression, out var property)) + { + GenerateColumn(property); + return; + } + + throw new NotSupportedException($"Member access for '{memberExpression.Member.Name}' is unsupported - only member access over the filter parameter are supported"); + } + + private void GenerateColumn(PropertyModel property) + => _sql.Append('"').Append(property.StorageName.Replace("\"", "\"\"")).Append('"'); + + private void TranslateQueryParameter(object? value) + { + // For null values, simply inline rather than parameterize; parameterized NULLs require setting NpgsqlDbType which is a bit more complicated, + // plus in any case equality with NULL requires different SQL (x IS NULL rather than x = y) + if (value is null) + { + _sql.Append("NULL"); + } + else + { + ParameterValues.Add(value); + _sql.Append('$').Append(_parameterIndex++); + } + } + + private void TranslateMethodCall(MethodCallExpression methodCall) + { + if (TryBindProperty(methodCall, out var property)) + { + GenerateColumn(property); + return; + } + + switch (methodCall) + { + case var _ when TryMatchContains(methodCall, out var source, out var item): + TranslateContains(source, item); + return; + + case { Method.Name: nameof(Enumerable.Any), Arguments: [var anySource, LambdaExpression lambda] } any + when any.Method.DeclaringType == typeof(Enumerable): + TranslateAny(anySource, lambda); + return; + + default: + throw new NotSupportedException($"Unsupported method call: {methodCall.Method.DeclaringType?.Name}.{methodCall.Method.Name}"); + } + } + + private void TranslateContains(Expression source, Expression item) + { + switch (source) + { + // Contains over array column (r => r.Strings.Contains("foo")) + case var _ when TryBindProperty(source, out _): + Translate(source); + _sql.Append(" @> ARRAY["); + Translate(item); + _sql.Append(']'); + return; + + // Contains over inline array (r => new[] { "foo", "bar" }.Contains(r.String)) + case NewArrayExpression newArray: + Translate(item); + _sql.Append(" IN ("); + + var isFirst = true; + foreach (var element in newArray.Expressions) + { + if (isFirst) + { + isFirst = false; + } + else + { + _sql.Append(", "); + } + + Translate(element); + } + + _sql.Append(')'); + return; + + // Contains over captured array (r => arrayLocalVariable.Contains(r.String)) + case QueryParameterExpression { Value: var value }: + Translate(item); + _sql.Append(" = ANY ("); + Translate(source); + _sql.Append(')'); + return; + + default: + throw new NotSupportedException("Unsupported Contains expression"); + } + } + + private void TranslateAny(Expression source, LambdaExpression lambda) + { + if (!TryBindProperty(source, out var property) + || lambda.Body is not MethodCallExpression containsCall + || !TryMatchContains(containsCall, out var valuesExpression, out var itemExpression)) + { + throw new NotSupportedException("Unsupported method call: Enumerable.Any"); + } + + if (itemExpression != lambda.Parameters[0]) + { + throw new NotSupportedException("Unsupported method call: Enumerable.Any"); + } + + switch (valuesExpression) + { + case NewArrayExpression newArray: + { + var values = new object?[newArray.Expressions.Count]; + for (var i = 0; i < newArray.Expressions.Count; i++) + { + values[i] = newArray.Expressions[i] switch + { + ConstantExpression { Value: var v } => v, + QueryParameterExpression { Value: var v } => v, + _ => throw new NotSupportedException("Unsupported method call: Enumerable.Any") + }; + } + + TranslateAnyContainsOverArrayColumn(property, values); + return; + } + + case QueryParameterExpression { Value: var value }: + TranslateAnyContainsOverArrayColumn(property, value); + return; + + case ConstantExpression { Value: var value }: + TranslateAnyContainsOverArrayColumn(property, value); + return; + + default: + throw new NotSupportedException("Unsupported method call: Enumerable.Any"); + } + } + + private void TranslateAnyContainsOverArrayColumn(PropertyModel property, object? values) + { + // Translate r.Strings.Any(s => array.Contains(s)) to: column && ARRAY[values] + // The && operator checks if the two arrays have any elements in common + GenerateColumn(property); + _sql.Append(" && "); + TranslateConstant(values); + } + + private void TranslateUnary(UnaryExpression unary) + { + switch (unary.NodeType) + { + case ExpressionType.Not: + if (unary.Operand is BinaryExpression { NodeType: ExpressionType.Equal or ExpressionType.NotEqual } binary) + { + TranslateBinary( + Expression.MakeBinary( + binary.NodeType is ExpressionType.Equal ? ExpressionType.NotEqual : ExpressionType.Equal, + binary.Left, + binary.Right)); + return; + } + + _sql.Append("(NOT "); + Translate(unary.Operand); + _sql.Append(')'); + return; + + case ExpressionType.Convert when Nullable.GetUnderlyingType(unary.Type) == unary.Operand.Type: + Translate(unary.Operand); + return; + + case ExpressionType.Convert when TryBindProperty(unary.Operand, out var property) && unary.Type == property.Type: + GenerateColumn(property); + return; + + default: + throw new NotSupportedException("Unsupported unary expression node type: " + unary.NodeType); + } + } +} diff --git a/MEVD/src/PgVector/PostgresMapper.cs b/MEVD/src/PgVector/PostgresMapper.cs new file mode 100644 index 0000000..51bb083 --- /dev/null +++ b/MEVD/src/PgVector/PostgresMapper.cs @@ -0,0 +1,236 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections; +using System.Diagnostics; +using System.Runtime.InteropServices; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData.ProviderServices; +using Npgsql; + +namespace CommunityToolkit.VectorData.PgVector; + +/// +/// A mapper class that handles the conversion between data models and storage models for Postgres vector store. +/// +/// The type of the data model record. +internal sealed class PostgresMapper(CollectionModel model) + where TRecord : class +{ + public TRecord MapFromStorageToDataModel(NpgsqlDataReader reader, bool includeVectors) + { + var record = model.CreateRecord()!; + + PopulateProperty(model.KeyProperty, reader, record); + + foreach (var dataProperty in model.DataProperties) + { + PopulateProperty(dataProperty, reader, record); + } + + if (includeVectors) + { + foreach (var vectorProperty in model.VectorProperties) + { + int ordinal = reader.GetOrdinal(vectorProperty.StorageName); + + if (reader.IsDBNull(ordinal)) + { + vectorProperty.SetValueAsObject(record, null); + continue; + } + + switch (reader.GetValue(ordinal)) + { + case Pgvector.Vector { Memory: ReadOnlyMemory memory }: + { + vectorProperty.SetValueAsObject(record, (Nullable.GetUnderlyingType(vectorProperty.Type) ?? vectorProperty.Type) switch + { + var t when t == typeof(ReadOnlyMemory) => memory, + var t when t == typeof(Embedding) => new Embedding(memory), + var t when t == typeof(float[]) + => MemoryMarshal.TryGetArray(memory, out ArraySegment segment) && segment.Count == segment.Array!.Length + ? segment.Array + : memory.ToArray(), + + _ => throw new UnreachableException() + }); + continue; + } + +#if NET + case Pgvector.HalfVector { Memory: ReadOnlyMemory memory }: + { + vectorProperty.SetValueAsObject(record, (Nullable.GetUnderlyingType(vectorProperty.Type) ?? vectorProperty.Type) switch + { + var t when t == typeof(ReadOnlyMemory) => memory, + var t when t == typeof(Embedding) => new Embedding(memory), + var t when t == typeof(Half[]) + => MemoryMarshal.TryGetArray(memory, out ArraySegment segment) && segment.Count == segment.Array!.Length + ? segment.Array + : memory.ToArray(), + + _ => throw new UnreachableException() + }); + continue; + } +#endif + + case BitArray bitArray when vectorProperty.Type == typeof(BinaryEmbedding): + vectorProperty.SetValueAsObject(record, new BinaryEmbedding(bitArray)); + continue; + + case BitArray bitArray: + vectorProperty.SetValueAsObject(record, bitArray); + continue; + + case Pgvector.SparseVector pgSparseVector: + vectorProperty.SetValueAsObject(record, pgSparseVector); + continue; + + // TODO: We currently allow round-tripping null for the vector property; this is not supported for most (?) dedicated databases; think about it. + case null: + vectorProperty.SetValueAsObject(record, null); + continue; + + case var value: + throw new InvalidOperationException($"Embedding vector read back from PostgreSQL is of type '{value.GetType().Name}' instead of the expected Pgvector.Vector type for property '{vectorProperty.ModelName}'."); + } + } + } + + return record; + } + + private static void PopulateProperty(PropertyModel property, NpgsqlDataReader reader, TRecord record) + { + int ordinal = reader.GetOrdinal(property.StorageName); + + if (reader.IsDBNull(ordinal)) + { + property.SetValueAsObject(record, null); + return; + } + + var type = Nullable.GetUnderlyingType(property.Type) ?? property.Type; + + // First try an efficient switch over the TypeCode enum for common base types + switch (Type.GetTypeCode(type)) + { + case TypeCode.Int32: + property.SetValue(record, reader.GetInt32(ordinal)); + return; + case TypeCode.String: + property.SetValue(record, reader.GetString(ordinal)); + return; + case TypeCode.Boolean: + property.SetValue(record, reader.GetBoolean(ordinal)); + return; + case TypeCode.Int16: + property.SetValue(record, reader.GetInt16(ordinal)); + return; + case TypeCode.Int64: + property.SetValue(record, reader.GetInt64(ordinal)); + return; + case TypeCode.Single: + property.SetValue(record, reader.GetFloat(ordinal)); + return; + case TypeCode.Double: + property.SetValue(record, reader.GetDouble(ordinal)); + return; + case TypeCode.Decimal: + property.SetValue(record, reader.GetDecimal(ordinal)); + return; + case TypeCode.DateTime: + property.SetValue(record, reader.GetDateTime(ordinal)); + return; + + case TypeCode.Object: + switch (type) + { + // Some more base types + case var t when t == typeof(Guid): + property.SetValue(record, reader.GetGuid(ordinal)); + return; + case var t when t == typeof(DateTimeOffset): + property.SetValue(record, reader.GetFieldValue(ordinal)); + return; +#if NET + case var t when t == typeof(DateOnly): + property.SetValue(record, reader.GetFieldValue(ordinal)); + return; + case var t when t == typeof(TimeOnly): + property.SetValue(record, reader.GetFieldValue(ordinal)); + return; +#endif + case var t when t == typeof(byte[]): + property.SetValue(record, reader.GetFieldValue(ordinal)); + return; + + // Array types are returned by default from GetValue(), and since they're reference types + // there's no boxing involved. + case var t when t.IsArray: + property.SetValueAsObject(record, reader.GetValue(ordinal)); + return; + + // For List, we need to call GetFieldValue>() to get the right type + case Type lt when lt.IsGenericType && lt.GetGenericTypeDefinition() == typeof(List<>): + switch (type) + { + case Type t when t == typeof(List): + property.SetValueAsObject(record, reader.GetFieldValue>(ordinal)); + return; + case Type t when t == typeof(List): + property.SetValueAsObject(record, reader.GetFieldValue>(ordinal)); + return; + case Type t when t == typeof(List): + property.SetValueAsObject(record, reader.GetFieldValue>(ordinal)); + return; + case Type t when t == typeof(List): + property.SetValueAsObject(record, reader.GetFieldValue>(ordinal)); + return; + case Type t when t == typeof(List): + property.SetValueAsObject(record, reader.GetFieldValue>(ordinal)); + return; + case Type t when t == typeof(List): + property.SetValueAsObject(record, reader.GetFieldValue>(ordinal)); + return; + case Type t when t == typeof(List): + property.SetValueAsObject(record, reader.GetFieldValue>(ordinal)); + return; + case Type t when t == typeof(List): + property.SetValueAsObject(record, reader.GetFieldValue>(ordinal)); + return; + case Type t when t == typeof(List): + property.SetValueAsObject(record, reader.GetFieldValue>(ordinal)); + return; + case Type t when t == typeof(List): + property.SetValueAsObject(record, reader.GetFieldValue>(ordinal)); + return; + case Type t when t == typeof(List): + property.SetValueAsObject(record, reader.GetFieldValue>(ordinal)); + return; +#if NET + case Type t when t == typeof(List): + property.SetValueAsObject(record, reader.GetFieldValue>(ordinal)); + return; + case Type t when t == typeof(List): + property.SetValueAsObject(record, reader.GetFieldValue>(ordinal)); + return; +#endif + case Type t when t == typeof(List): + property.SetValueAsObject(record, reader.GetFieldValue>(ordinal)); + return; + default: + throw new UnreachableException("Unsupported property type: " + property.Type.Name); + } + + default: + throw new UnreachableException("Unsupported property type: " + property.Type.Name); + } + + default: + throw new UnreachableException("Unsupported property type: " + property.Type.Name); + } + } +} diff --git a/MEVD/src/PgVector/PostgresModelBuilder.cs b/MEVD/src/PgVector/PostgresModelBuilder.cs new file mode 100644 index 0000000..ff23b67 --- /dev/null +++ b/MEVD/src/PgVector/PostgresModelBuilder.cs @@ -0,0 +1,125 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections; +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; +using Pgvector; + +namespace CommunityToolkit.VectorData.PgVector; + +internal class PostgresModelBuilder() : CollectionModelBuilder(PostgresModelBuilder.ModelBuildingOptions) +{ + internal const string SupportedVectorTypes = "ReadOnlyMemory, Embedding, float[], ReadOnlyMemory, Embedding, Half[], BinaryEmbedding, BitArray, or SparseVector"; + + public static readonly CollectionModelBuildingOptions ModelBuildingOptions = new() + { + RequiresAtLeastOneVector = false, + SupportsMultipleVectors = true, + }; + + protected override bool SupportsKeyAutoGeneration(Type keyPropertyType) + => keyPropertyType == typeof(Guid) || keyPropertyType == typeof(int) || keyPropertyType == typeof(long); + + protected override void ValidateKeyProperty(KeyPropertyModel keyProperty) + { + base.ValidateKeyProperty(keyProperty); + + var type = keyProperty.Type; + + if (type != typeof(short) + && type != typeof(int) + && type != typeof(long) + && type != typeof(string) + && type != typeof(Guid)) + { + throw new NotSupportedException( + $"Property '{keyProperty.ModelName}' has unsupported type '{type.Name}'. Key properties must be one of the supported types: short, int, long, string, Guid."); + } + } + + protected override bool IsDataPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes) + { + supportedTypes = "bool, short, int, long, float, double, decimal, string, DateTime, DateTimeOffset, Guid, or arrays/lists of these types"; + + if (Nullable.GetUnderlyingType(type) is Type underlyingType) + { + type = underlyingType; + } + + return IsValid(type) + || (type.IsArray && IsValid(type.GetElementType()!)) + || (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(List<>) && IsValid(type.GenericTypeArguments[0])); + + static bool IsValid(Type type) + => type == typeof(bool) || + type == typeof(short) || + type == typeof(int) || + type == typeof(long) || + type == typeof(float) || + type == typeof(double) || + type == typeof(decimal) || + type == typeof(string) || + type == typeof(byte[]) || + type == typeof(DateTime) || + type == typeof(DateTimeOffset) || +#if NET + type == typeof(DateOnly) || + type == typeof(TimeOnly) || +#endif + type == typeof(Guid); + } + + protected override bool IsVectorPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes) + => IsVectorPropertyTypeValidCore(type, out supportedTypes); + + internal static bool IsVectorPropertyTypeValidCore(Type type, [NotNullWhen(false)] out string? supportedTypes) + { + supportedTypes = SupportedVectorTypes; + + if (Nullable.GetUnderlyingType(type) is Type underlyingType) + { + type = underlyingType; + } + + return type == typeof(ReadOnlyMemory) || + type == typeof(Embedding) || + type == typeof(float[]) || +#if NET + type == typeof(ReadOnlyMemory) || + type == typeof(Embedding) || + type == typeof(Half[]) || +#endif + type == typeof(BinaryEmbedding) || + type == typeof(BitArray) || + type == typeof(SparseVector); + } + + /// + protected override void ValidateProperty(PropertyModel propertyModel, VectorStoreCollectionDefinition? definition) + { + base.ValidateProperty(propertyModel, definition); + + if (propertyModel.IsTimestampWithoutTimezone()) + { + var type = Nullable.GetUnderlyingType(propertyModel.Type) ?? propertyModel.Type; + if (type != typeof(DateTime)) + { + throw new NotSupportedException( + $"Property '{propertyModel.ModelName}' has store type 'timestamp' configured, but this is only supported for DateTime properties. The property type is '{propertyModel.Type.Name}'."); + } + } + } + + /// + protected override IReadOnlyList EmbeddingGenerationDispatchers { get; } = + [ + EmbeddingGenerationDispatcher.Create>(), +#if NET + EmbeddingGenerationDispatcher.Create>(), +#endif + EmbeddingGenerationDispatcher.Create() + ]; +} diff --git a/MEVD/src/PgVector/PostgresPropertyExtensions.cs b/MEVD/src/PgVector/PostgresPropertyExtensions.cs new file mode 100644 index 0000000..2166120 --- /dev/null +++ b/MEVD/src/PgVector/PostgresPropertyExtensions.cs @@ -0,0 +1,122 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; + +namespace CommunityToolkit.VectorData.PgVector; + +/// +/// Extension methods for configuring PostgreSQL-specific properties on vector store property definitions. +/// +public static class PostgresPropertyExtensions +{ + private const string FullTextSearchLanguageKey = "Postgres:FullTextSearchLanguage"; + private const string StoreTypeKey = "Postgres:StoreType"; + + #region Full-text search language + + /// + /// Sets the PostgreSQL full-text search language for a data property. + /// + /// The data property to configure. + /// The PostgreSQL text search language name (e.g., "english", "spanish", "german"). + /// The same property instance for method chaining. + /// + /// This language is used with PostgreSQL's to_tsvector and plainto_tsquery functions + /// when creating GIN indexes and performing full-text search operations. + /// Common language options include: "simple", "english", "spanish", "german", "french", etc. + /// See PostgreSQL documentation for the full list of available text search configurations. + /// + public static VectorStoreDataProperty WithFullTextSearchLanguage(this VectorStoreDataProperty property, string? language) + { + property.ProviderAnnotations ??= []; + property.ProviderAnnotations[FullTextSearchLanguageKey] = language; + return property; + } + + /// + /// Gets the PostgreSQL full-text search language configured for a data property. + /// + /// The data property to read from. + /// The configured language, or if not set. + public static string? GetFullTextSearchLanguage(this VectorStoreDataProperty property) + => property.ProviderAnnotations?.TryGetValue(FullTextSearchLanguageKey, out var value) == true + ? value as string + : null; + + /// + /// Gets the PostgreSQL full-text search language configured for a data property model. + /// + /// The data property model to read from. + /// The configured language, or the default language ("english") if not set. + internal static string GetFullTextSearchLanguageOrDefault(this DataPropertyModel property) + => property.ProviderAnnotations?.TryGetValue(FullTextSearchLanguageKey, out var value) == true && value is string language + ? language + : PostgresConstants.DefaultFullTextSearchLanguage; + + #endregion Full-text search language + + #region Store type + + /// + /// Sets the PostgreSQL store type for a property, overriding the default type mapping. + /// + /// The property to configure. + /// + /// The PostgreSQL type name. Currently, only "timestamp" and "timestamp without time zone" + /// are supported, and only on properties. This causes the property to be stored as + /// PostgreSQL timestamp (without time zone) instead of the default timestamptz. + /// + /// The same property instance for method chaining. + /// + /// + /// By default, .NET properties are mapped to PostgreSQL timestamptz (timestamp with time zone), + /// which requires UTC values. Use this method to map to timestamp (without time zone) instead, which stores + /// local/unspecified date-time values without time zone information. + /// + /// + /// When using timestamp, values with or + /// kind should be used. Values read back from the database will have + /// . + /// + /// + /// Thrown if the is not a supported value. + public static TProperty WithStoreType(this TProperty property, string storeType) + where TProperty : VectorStoreProperty + { + if (!IsTimestampStoreType(storeType)) + { + throw new NotSupportedException( + $"Store type '{storeType}' is not supported. Only 'timestamp' and 'timestamp without time zone' are supported."); + } + + property.ProviderAnnotations ??= []; + property.ProviderAnnotations[StoreTypeKey] = storeType; + return property; + } + + /// + /// Gets the PostgreSQL store type configured for a property. + /// + /// The property to read from. + /// The configured store type, or if not set. + public static string? GetStoreType(this VectorStoreProperty property) + => property.ProviderAnnotations?.TryGetValue(StoreTypeKey, out var value) == true + ? value as string + : null; + + /// + /// Gets whether the property model has been configured with a timestamp (without time zone) store type. + /// + internal static bool IsTimestampWithoutTimezone(this PropertyModel property) + => property.ProviderAnnotations?.TryGetValue(StoreTypeKey, out var value) == true + && value is string storeType + && IsTimestampStoreType(storeType); + + private static bool IsTimestampStoreType(string storeType) + => string.Equals(storeType, "timestamp", StringComparison.OrdinalIgnoreCase) + || string.Equals(storeType, "timestamp without time zone", StringComparison.OrdinalIgnoreCase); + + #endregion Store type +} diff --git a/MEVD/src/PgVector/PostgresPropertyMapping.cs b/MEVD/src/PgVector/PostgresPropertyMapping.cs new file mode 100644 index 0000000..e227852 --- /dev/null +++ b/MEVD/src/PgVector/PostgresPropertyMapping.cs @@ -0,0 +1,237 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; +using Npgsql; +using NpgsqlTypes; +using Pgvector; + +namespace CommunityToolkit.VectorData.PgVector; + +internal static class PostgresPropertyMapping +{ + public static object? MapVectorForStorageModel(object? vector) + => vector switch + { + ReadOnlyMemory m => new Pgvector.Vector(m), + Embedding e => new Pgvector.Vector(e.Vector), + float[] a => new Pgvector.Vector(a), + +#if NET + ReadOnlyMemory m => new Pgvector.HalfVector(m), + Embedding e => new Pgvector.HalfVector(e.Vector), + Half[] a => new Pgvector.HalfVector(a), +#endif + + BitArray bitArray => bitArray, + BinaryEmbedding binaryEmbedding => binaryEmbedding.Vector, + SparseVector sparseVector => sparseVector, + + null => null, + + var value => throw new NotSupportedException($"Mapping for type '{value.GetType().Name}' to a vector is not supported.") + }; + + /// + /// Gets the NpgsqlDbType for a property, taking into account any store type annotation. + /// + internal static NpgsqlDbType? GetNpgsqlDbType(PropertyModel property) + => (Nullable.GetUnderlyingType(property.Type) ?? property.Type) switch + { + Type t when t == typeof(bool) => NpgsqlDbType.Boolean, + Type t when t == typeof(short) => NpgsqlDbType.Smallint, + Type t when t == typeof(int) => NpgsqlDbType.Integer, + Type t when t == typeof(long) => NpgsqlDbType.Bigint, + Type t when t == typeof(float) => NpgsqlDbType.Real, + Type t when t == typeof(double) => NpgsqlDbType.Double, + Type t when t == typeof(decimal) => NpgsqlDbType.Numeric, + Type t when t == typeof(string) => NpgsqlDbType.Text, + Type t when t == typeof(byte[]) => NpgsqlDbType.Bytea, + Type t when t == typeof(Guid) => NpgsqlDbType.Uuid, + Type t when t == typeof(DateTimeOffset) => NpgsqlDbType.TimestampTz, + + // DateTime properties map to PG's "timestamp with time zone" (UTC timestamps) by default, aligning with Npgsql/EF/etc. + // Users can explicitly opt into "timestamp without time zone". + Type t when t == typeof(DateTime) && property.IsTimestampWithoutTimezone() => NpgsqlDbType.Timestamp, + Type t when t == typeof(DateTime) => NpgsqlDbType.TimestampTz, + +#if NET + Type t when t == typeof(DateOnly) => NpgsqlDbType.Date, + Type t when t == typeof(TimeOnly) => NpgsqlDbType.Time, +#endif + + _ => null + }; + + /// + /// Maps a .NET type to a PostgreSQL type name, taking into account any store type annotation on the property. + /// + internal static (string PgType, bool IsNullable) GetPostgresTypeName(PropertyModel property) + { + static bool TryGetBaseType(Type type, [NotNullWhen(true)] out string? typeName) + { + typeName = type switch + { + Type t when t == typeof(bool) => "BOOLEAN", + Type t when t == typeof(short) => "SMALLINT", + Type t when t == typeof(int) => "INTEGER", + Type t when t == typeof(long) => "BIGINT", + Type t when t == typeof(float) => "REAL", + Type t when t == typeof(double) => "DOUBLE PRECISION", + Type t when t == typeof(decimal) => "NUMERIC", + Type t when t == typeof(string) => "TEXT", + Type t when t == typeof(byte[]) => "BYTEA", + Type t when t == typeof(DateTime) => "TIMESTAMPTZ", + Type t when t == typeof(DateTimeOffset) => "TIMESTAMPTZ", +#if NET + Type t when t == typeof(DateOnly) => "DATE", + Type t when t == typeof(TimeOnly) => "TIME", +#endif + Type t when t == typeof(Guid) => "UUID", + _ => null + }; + + return typeName is not null; + } + + var propertyType = property.Type; + + (string PgType, bool IsNullable) result; + + if (TryGetBaseType(propertyType, out string? pgType)) + { + result = (pgType, property.IsNullable); + } + // Handle nullable types (e.g. Nullable) + else if (Nullable.GetUnderlyingType(propertyType) is Type unwrappedType + && TryGetBaseType(unwrappedType, out string? underlyingPgType)) + { + result = (underlyingPgType, property.IsNullable); + } + // Handle collections + else if ((propertyType.IsArray && TryGetBaseType(propertyType.GetElementType()!, out string? elementPgType)) + || (propertyType.IsGenericType + && propertyType.GetGenericTypeDefinition() == typeof(List<>) + && TryGetBaseType(propertyType.GetGenericArguments()[0], out elementPgType))) + { + result = (elementPgType + "[]", property.IsNullable); + } + else + { + throw new NotSupportedException($"Type {propertyType.Name} is not supported by this store."); + } + + if (property.IsTimestampWithoutTimezone()) + { + // Replace TIMESTAMPTZ with TIMESTAMP in the PG type name. + // This handles both "TIMESTAMPTZ" and "TIMESTAMPTZ[]" cases. + result = ("TIMESTAMP", result.IsNullable); + } + + return result; + } + + /// + /// Gets the PostgreSQL vector type name based on the dimensions of the vector property. + /// + /// The vector property. + /// The PostgreSQL vector type name. + public static (string PgType, bool IsNullable) GetPgVectorTypeName(VectorPropertyModel vectorProperty) + { + var unwrappedEmbeddingType = Nullable.GetUnderlyingType(vectorProperty.EmbeddingType) ?? vectorProperty.EmbeddingType; + + var pgType = unwrappedEmbeddingType switch + { + Type t when t == typeof(ReadOnlyMemory) + || t == typeof(Embedding) + || t == typeof(float[]) + => "VECTOR", + +#if NET + Type t when t == typeof(ReadOnlyMemory) + || t == typeof(Embedding) + || t == typeof(Half[]) + => "HALFVEC", +#endif + + Type t when t == typeof(SparseVector) => "SPARSEVEC", + Type t when t == typeof(BitArray) => "BIT", + Type t when t == typeof(BinaryEmbedding) => "BIT", + + _ => throw new NotSupportedException($"Type {vectorProperty.EmbeddingType.Name} is not supported by this store.") + }; + + return ($"{pgType}({vectorProperty.Dimensions})", vectorProperty.IsNullable); + } + + public static NpgsqlParameter GetNpgsqlParameter(object? value) + => new() { Value = value ?? DBNull.Value }; + + /// + /// Returns information about indexes to create, validating that the dimensions of the vector are supported. + /// + /// The properties of the vector store record. + /// A list of tuples containing the column name, index kind, distance function, and full-text language for each property. + /// + /// The default index kind is "Flat", which prevents the creation of an index. + /// + public static List<(string column, string kind, string function, bool isVector, bool isFullText, string? fullTextLanguage)> GetIndexInfo(IReadOnlyList properties) + { + var vectorIndexesToCreate = new List<(string column, string kind, string function, bool isVector, bool isFullText, string? fullTextLanguage)>(); + foreach (var property in properties) + { + switch (property) + { + case KeyPropertyModel: + // There is no need to create a separate index for the key property. + break; + + case VectorPropertyModel vectorProperty: + var indexKind = vectorProperty.IndexKind ?? PostgresConstants.DefaultIndexKind; + var distanceFunction = vectorProperty.DistanceFunction ?? PostgresConstants.DefaultDistanceFunction; + + // Index kind of "Flat" to prevent the creation of an index. This is the default behavior. + // Otherwise, the index will be created with the specified index kind and distance function, if supported. + if (indexKind != IndexKind.Flat) + { + // Ensure the dimensionality of the vector is supported for indexing. + if (PostgresConstants.IndexMaxDimensions.TryGetValue(indexKind, out int maxDimensions) && vectorProperty.Dimensions > maxDimensions) + { + throw new NotSupportedException( + $"The provided vector property {vectorProperty.ModelName} has {vectorProperty.Dimensions} dimensions, " + + $"which is not supported by the {indexKind} index. The maximum number of dimensions supported by the {indexKind} index " + + $"is {maxDimensions}. Please reduce the number of dimensions or use a different index." + ); + } + + vectorIndexesToCreate.Add((vectorProperty.StorageName, indexKind, distanceFunction, isVector: true, isFullText: false, fullTextLanguage: null)); + } + + break; + + case DataPropertyModel dataProperty: + if (dataProperty.IsIndexed) + { + vectorIndexesToCreate.Add((dataProperty.StorageName, kind: "", function: "", isVector: false, isFullText: false, fullTextLanguage: null)); + } + + if (dataProperty.IsFullTextIndexed) + { + var language = dataProperty.GetFullTextSearchLanguageOrDefault(); + vectorIndexesToCreate.Add((dataProperty.StorageName, kind: "", function: "", isVector: false, isFullText: true, fullTextLanguage: language)); + } + break; + + default: + throw new UnreachableException(); + } + } + + return vectorIndexesToCreate; + } +} diff --git a/MEVD/src/PgVector/PostgresServiceCollectionExtensions.cs b/MEVD/src/PgVector/PostgresServiceCollectionExtensions.cs new file mode 100644 index 0000000..84afd6c --- /dev/null +++ b/MEVD/src/PgVector/PostgresServiceCollectionExtensions.cs @@ -0,0 +1,320 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData; +using CommunityToolkit.VectorData.PgVector; +using Npgsql; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.DependencyInjection; + +/// +/// Extension methods to register Postgres instances on an . +/// +public static class PostgresServiceCollectionExtensions +{ + private const string DynamicCodeMessage = "This method is incompatible with NativeAOT, consult the documentation for adding collections in a way that's compatible with NativeAOT."; + private const string UnreferencedCodeMessage = "This method is incompatible with trimming, consult the documentation for adding collections in a way that's compatible with NativeAOT."; + + /// + /// Register a as , where the is retrieved from the dependency injection container. + /// + /// The to register the on. + /// Optional options to further configure the . + /// The service lifetime for the store. Defaults to . + /// The service collection. + public static IServiceCollection AddPostgresVectorStore( + this IServiceCollection services, + PostgresVectorStoreOptions? options = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + { + Throw.IfNull(services); + + services.Add(new ServiceDescriptor(typeof(PostgresVectorStore), (sp) => + { + var dataSource = sp.GetRequiredService(); + options = GetStoreOptions(sp, _ => options); + + // The data source has been solved from the DI container, so we do not own it. + return new PostgresVectorStore(dataSource, ownsDataSource: false, options); + }, lifetime)); + + services.Add(new ServiceDescriptor(typeof(VectorStore), + static (sp) => sp.GetRequiredService(), lifetime)); + + return services; + } + + /// + /// Registers a as , with the specified connection string and service lifetime. + /// + /// + public static IServiceCollection AddPostgresVectorStore( + this IServiceCollection services, + string connectionString, + PostgresVectorStoreOptions? options = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + { + Throw.IfNullOrWhitespace(connectionString); + + return AddVectorStore(services, serviceKey: null, sp => connectionString, sp => options, lifetime); + } + + /// + /// Registers a keyed as , with the specified connection string and service lifetime. + /// + /// The to register the on. + /// The key with which to associate the vector store. + /// Postgres database connection string. + /// Optional options to further configure the . + /// The service lifetime for the store. Defaults to . + /// The service collection. + public static IServiceCollection AddKeyedPostgresVectorStore( + this IServiceCollection services, + object? serviceKey, + string connectionString, + PostgresVectorStoreOptions? options = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + { + Throw.IfNullOrWhitespace(connectionString); + + return AddVectorStore(services, serviceKey, sp => connectionString, sp => options, lifetime); + } + + /// + /// Registers a as , with the specified connection string and service lifetime. + /// + /// + public static IServiceCollection AddPostgresVectorStore( + this IServiceCollection services, + Func connectionStringProvider, + Func? optionsProvider = null, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + => AddVectorStore(services, serviceKey: null, connectionStringProvider, optionsProvider, lifetime); + + /// + public static IServiceCollection AddKeyedPostgresVectorStore( + this IServiceCollection services, + object? serviceKey, + Func connectionStringProvider, + Func? optionsProvider = null, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + => AddVectorStore(services, serviceKey, connectionStringProvider, optionsProvider, lifetime); + + /// + /// Registers a keyed as , with the specified connection string and service lifetime. + /// + /// The to register the on. + /// The key with which to associate the store. + /// The connection string provider. + /// Options provider to further configure the . + /// The service lifetime for the store. Defaults to . + /// The service collection. + private static IServiceCollection AddVectorStore( + IServiceCollection services, + object? serviceKey, + Func connectionStringProvider, + Func? optionsProvider, + ServiceLifetime lifetime) + { + Throw.IfNull(services); + Throw.IfNull(connectionStringProvider); + + services.Add(new ServiceDescriptor(typeof(PostgresVectorStore), serviceKey, (sp, _) => + { + var connectionString = connectionStringProvider(sp); + var options = GetStoreOptions(sp, optionsProvider); + + return new PostgresVectorStore(connectionString, options); + }, lifetime)); + + services.Add(new ServiceDescriptor(typeof(VectorStore), serviceKey, + static (sp, key) => sp.GetRequiredKeyedService(key), lifetime)); + + return services; + } + + /// + /// Register a where the is retrieved from the dependency injection container. + /// + /// The type of the key. + /// The type of the record. + /// The to register the on. + /// The name of the collection. + /// Optional options to further configure the . + /// The service lifetime for the store. It needs to match lifetime. Defaults to . + /// Service collection. + [RequiresDynamicCode(DynamicCodeMessage)] + [RequiresUnreferencedCode(UnreferencedCodeMessage)] + public static IServiceCollection AddPostgresCollection( + this IServiceCollection services, + string name, + PostgresCollectionOptions? options = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + where TKey : notnull + where TRecord : class + { + Throw.IfNull(services); + Throw.IfNullOrWhitespace(name); + + services.Add(new ServiceDescriptor(typeof(PostgresCollection), (sp) => + { + var dataSource = sp.GetRequiredService(); + var copy = GetCollectionOptions(sp, _ => options); + + // The data source has been solved from the DI container, so we do not own it. + return new PostgresCollection(dataSource, name, ownsDataSource: false, copy); + }, lifetime)); + + AddAbstractions(services, serviceKey: null, lifetime); + + return services; + } + + /// + /// Registers a as , with the specified connection string and service lifetime. + /// + /// + [RequiresDynamicCode(DynamicCodeMessage)] + [RequiresUnreferencedCode(UnreferencedCodeMessage)] + public static IServiceCollection AddPostgresCollection( + this IServiceCollection services, + string name, + string connectionString, + PostgresCollectionOptions? options = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + where TKey : notnull + where TRecord : class + { + Throw.IfNullOrWhitespace(connectionString); + + return AddKeyedPostgresCollection(services, serviceKey: null, name, sp => connectionString, sp => options, lifetime); + } + + /// + /// Registers a keyed as , with the specified connection string and service lifetime. + /// + /// The type of the key. + /// The type of the record. + /// The to register the on. + /// The key with which to associate the collection. + /// The name of the collection. + /// Postgres database connection string. + /// Optional options to further configure the . + /// The service lifetime for the store. Defaults to . + /// Service collection. + [RequiresDynamicCode(DynamicCodeMessage)] + [RequiresUnreferencedCode(UnreferencedCodeMessage)] + public static IServiceCollection AddKeyedPostgresCollection( + this IServiceCollection services, + object? serviceKey, + string name, + string connectionString, + PostgresCollectionOptions? options = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + where TKey : notnull + where TRecord : class + { + Throw.IfNullOrWhitespace(connectionString); + + return AddKeyedPostgresCollection(services, serviceKey, name, sp => connectionString, sp => options, lifetime); + } + + /// + /// Registers a as , with the specified connection string and service lifetime. + /// + /// + [RequiresDynamicCode(DynamicCodeMessage)] + [RequiresUnreferencedCode(UnreferencedCodeMessage)] + public static IServiceCollection AddPostgresCollection( + this IServiceCollection services, + string name, + Func connectionStringProvider, + Func? optionsProvider = null, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + where TKey : notnull + where TRecord : class + => AddKeyedPostgresCollection(services, serviceKey: null, name, connectionStringProvider, optionsProvider, lifetime); + + /// + /// Registers a keyed as , with the specified connection string and service lifetime. + /// + /// The to register the on. + /// The key with which to associate the collection. + /// The name of the collection. + /// The connection string provider. + /// Options provider to further configure the collection. + /// The service lifetime for the store. Defaults to . + /// The service collection. + [RequiresDynamicCode(DynamicCodeMessage)] + [RequiresUnreferencedCode(UnreferencedCodeMessage)] + public static IServiceCollection AddKeyedPostgresCollection( + this IServiceCollection services, + object? serviceKey, + string name, + Func connectionStringProvider, + Func? optionsProvider = null, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + where TKey : notnull + where TRecord : class + { + Throw.IfNull(services); + Throw.IfNullOrWhitespace(name); + Throw.IfNull(connectionStringProvider); + + services.Add(new ServiceDescriptor(typeof(PostgresCollection), serviceKey, (sp, _) => + { + var connectionString = connectionStringProvider(sp); + var options = GetCollectionOptions(sp, optionsProvider); + return new PostgresCollection(connectionString, name, options); + }, lifetime)); + + AddAbstractions(services, serviceKey, lifetime); + + return services; + } + + private static void AddAbstractions(IServiceCollection services, object? serviceKey, ServiceLifetime lifetime) + where TKey : notnull + where TRecord : class + { + services.Add(new ServiceDescriptor(typeof(VectorStoreCollection), serviceKey, + static (sp, key) => sp.GetRequiredKeyedService>(key), lifetime)); + + services.Add(new ServiceDescriptor(typeof(IVectorSearchable), serviceKey, + static (sp, key) => sp.GetRequiredKeyedService>(key), lifetime)); + + services.Add(new ServiceDescriptor(typeof(IKeywordHybridSearchable), serviceKey, + static (sp, key) => sp.GetRequiredKeyedService>(key), lifetime)); + } + + private static PostgresVectorStoreOptions? GetStoreOptions(IServiceProvider sp, Func? optionsProvider) + { + var options = optionsProvider?.Invoke(sp); + if (options?.EmbeddingGenerator is not null) + { + return options; // The user has provided everything, there is nothing to change. + } + + var embeddingGenerator = sp.GetService(); + return embeddingGenerator is null + ? options // There is nothing to change. + : new(options) { EmbeddingGenerator = embeddingGenerator }; // Create a brand new copy in order to avoid modifying the original options. + } + + private static PostgresCollectionOptions? GetCollectionOptions(IServiceProvider sp, Func? optionsProvider) + { + var options = optionsProvider?.Invoke(sp); + if (options?.EmbeddingGenerator is not null) + { + return options; // The user has provided everything, there is nothing to change. + } + + var embeddingGenerator = sp.GetService(); + return embeddingGenerator is null + ? options // There is nothing to change. + : new(options) { EmbeddingGenerator = embeddingGenerator }; // Create a brand new copy in order to avoid modifying the original options. + } +} diff --git a/MEVD/src/PgVector/PostgresSqlBuilder.cs b/MEVD/src/PgVector/PostgresSqlBuilder.cs new file mode 100644 index 0000000..bce97e4 --- /dev/null +++ b/MEVD/src/PgVector/PostgresSqlBuilder.cs @@ -0,0 +1,814 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using System.Linq.Expressions; +using System.Text; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; +using Npgsql; +using NpgsqlTypes; + +namespace CommunityToolkit.VectorData.PgVector; + +/// +/// Provides methods to build SQL commands for managing vector store collections in PostgreSQL. +/// +[System.Diagnostics.CodeAnalysis.SuppressMessage("Security", "CA2100:Review SQL queries for security vulnerabilities", Justification = "We need to build the full table name using schema and collection, it does not support parameterized passing.")] +internal static class PostgresSqlBuilder +{ + internal static void BuildDoesTableExistCommand(NpgsqlCommand command, string? schema, string tableName) + { + Debug.Assert(command.Parameters.Count == 0); + + command.CommandText = """ + SELECT table_name + FROM information_schema.tables + WHERE table_schema = $1 + AND table_type = 'BASE TABLE' + AND table_name = $2 + """; + + command.Parameters.Add(new() { Value = schema ?? "public" }); + command.Parameters.Add(new() { Value = tableName }); + } + + internal static void BuildGetTablesCommand(NpgsqlCommand command, string? schema) + { + Debug.Assert(command.Parameters.Count == 0); + + command.CommandText = """ + SELECT table_name + FROM information_schema.tables + WHERE table_schema = $1 AND table_type = 'BASE TABLE' + """; + + command.Parameters.Add(new() { Value = schema ?? "public" }); + } + + internal static string BuildCreateTableSql(string? schema, string tableName, CollectionModel model, Version pgVersion, bool ifNotExists = true) + { + if (string.IsNullOrWhiteSpace(tableName)) + { + throw new ArgumentException("Table name cannot be null or whitespace", nameof(tableName)); + } + + var keyName = model.KeyProperty.StorageName; + + StringBuilder createTableCommand = new(); + createTableCommand.Append("CREATE TABLE "); + if (ifNotExists) + { + createTableCommand.Append("IF NOT EXISTS "); + } + createTableCommand.AppendTableName(schema, tableName).AppendLine(" ("); + + // Add the key column + var keyStoreType = PostgresPropertyMapping.GetPostgresTypeName(model.KeyProperty).PgType; + createTableCommand.Append(" ").AppendIdentifier(keyName).Append(' ').Append(keyStoreType); + if (model.KeyProperty.IsAutoGenerated) + { + switch (keyStoreType.ToUpperInvariant()) + { + case "INTEGER": + case "BIGINT": + createTableCommand.Append(" GENERATED BY DEFAULT AS IDENTITY"); + break; + case "UUID": + // UUIDv7 is superior for PostgreSQL indexing, but generating it was only added in PG18. + // Fall back to UUIDv4 in older PG versions. + createTableCommand.Append(pgVersion >= new Version(18, 0) + ? " DEFAULT uuidv7()" + : " DEFAULT gen_random_uuid()"); + break; + default: + throw new NotSupportedException($"Auto-generation of keys is not supported for key type '{keyStoreType}'. Only INTEGER, BIGINT and UUID are supported."); + } + } + createTableCommand.AppendLine(","); + + // Add the data columns + foreach (var dataProperty in model.DataProperties) + { + var dataPgTypeInfo = PostgresPropertyMapping.GetPostgresTypeName(dataProperty); + createTableCommand.Append(" ").AppendIdentifier(dataProperty.StorageName).Append(' ').Append(dataPgTypeInfo.PgType); + if (!dataPgTypeInfo.IsNullable) + { + createTableCommand.Append(" NOT NULL"); + } + createTableCommand.AppendLine(","); + } + + // Add the vector columns + foreach (var vectorProperty in model.VectorProperties) + { + var vectorPgTypeInfo = PostgresPropertyMapping.GetPgVectorTypeName(vectorProperty); + createTableCommand.Append(" ").AppendIdentifier(vectorProperty.StorageName).Append(' ').Append(vectorPgTypeInfo.PgType); + if (!vectorPgTypeInfo.IsNullable) + { + createTableCommand.Append(" NOT NULL"); + } + createTableCommand.AppendLine(","); + } + + createTableCommand.Append(" PRIMARY KEY (").AppendIdentifier(keyName).AppendLine(")"); + + createTableCommand.AppendLine(");"); + + return createTableCommand.ToString(); + } + + /// + internal static string BuildCreateIndexSql(string? schema, string tableName, string columnName, string indexKind, string distanceFunction, bool isVector, bool isFullText, string? fullTextLanguage, bool ifNotExists) + { + var indexName = $"{tableName}_{columnName}_index"; + + StringBuilder sql = new(); + sql.Append("CREATE INDEX "); + if (ifNotExists) + { + sql.Append("IF NOT EXISTS "); + } + + if (isFullText) + { + // Create a GIN index for full-text search + var language = fullTextLanguage ?? PostgresConstants.DefaultFullTextSearchLanguage; + sql.AppendIdentifier(indexName).Append(" ON ").AppendTableName(schema, tableName) + .Append(" USING GIN (to_tsvector(").AppendLiteral(language).Append(", ").AppendIdentifier(columnName).Append("))"); + return sql.ToString(); + } + + if (!isVector) + { + sql.AppendIdentifier(indexName).Append(" ON ").AppendTableName(schema, tableName) + .Append(" (").AppendIdentifier(columnName).Append(')'); + return sql.ToString(); + } + + // Only support creating HNSW index creation through the connector. + var indexTypeName = indexKind switch + { + IndexKind.Hnsw => "hnsw", + _ => throw new NotSupportedException($"Index kind '{indexKind}' is not supported for table creation. If you need to create an index of this type, please do so manually. Only HNSW indexes are supported through the vector store.") + }; + + distanceFunction ??= PostgresConstants.DefaultDistanceFunction; // Default to Cosine distance + + var indexOps = distanceFunction switch + { + DistanceFunction.CosineDistance => "vector_cosine_ops", + DistanceFunction.CosineSimilarity => "vector_cosine_ops", + DistanceFunction.DotProductSimilarity => "vector_ip_ops", + DistanceFunction.EuclideanDistance => "vector_l2_ops", + DistanceFunction.ManhattanDistance => "vector_l1_ops", + DistanceFunction.HammingDistance => "bit_hamming_ops", + + _ => throw new NotSupportedException($"Distance function {distanceFunction} is not supported.") + }; + + sql.AppendIdentifier(indexName).Append(" ON ").AppendTableName(schema, tableName) + .Append(" USING ").Append(indexTypeName).Append(" (").AppendIdentifier(columnName).Append(' ').Append(indexOps).Append(')'); + return sql.ToString(); + } + + /// + internal static void BuildDropTableCommand(NpgsqlCommand command, string? schema, string tableName) + { + StringBuilder sql = new(); + sql.Append("DROP TABLE IF EXISTS ").AppendTableName(schema, tableName); + command.CommandText = sql.ToString(); + } + + /// + internal static bool BuildUpsertCommand( + NpgsqlBatch batch, + string? schema, + string tableName, + CollectionModel model, + IEnumerable records, + Dictionary>? generatedEmbeddings) + { + // Note: since keys may be auto-generated, we can't use a single multi-value INSERT statement, since that would return + // the generated keys in random order. Use a batch of single-value INSERT statements instead. + + string? upsertSql = null, upsertSqlWithAutoGeneratedKey = null; + var keyProperty = model.KeyProperty; + var recordIndex = 0; + + foreach (var record in records) + { + var batchCommand = batch.CreateBatchCommand(); + + var autoGeneratedKey = keyProperty.IsAutoGenerated && Equals(keyProperty.GetValueAsObject(record), default(TKey)); + batchCommand.CommandText = autoGeneratedKey + ? upsertSqlWithAutoGeneratedKey ??= GenerateSingleUpsertSql(autoGeneratedKey: true) + : upsertSql ??= GenerateSingleUpsertSql(autoGeneratedKey: false); + + foreach (var property in model.Properties) + { + if (property is KeyPropertyModel && autoGeneratedKey) + { + continue; + } + + var value = property.GetValueAsObject(record); + + if (property is VectorPropertyModel vectorProperty) + { + if (generatedEmbeddings?[vectorProperty] is IReadOnlyList ge) + { + value = ge[recordIndex]; + } + + value = PostgresPropertyMapping.MapVectorForStorageModel(value); + } + + NpgsqlDbType? npgsqlDbType = null; + + if (property.Type == typeof(DateTime)) + { + npgsqlDbType = property.IsTimestampWithoutTimezone() + ? NpgsqlDbType.Timestamp + // DateTime properties map to PG's "timestamp with time zone" (timestamptz) by default. + // Npgsql would silently send non-UTC DateTimes as 'timestamp' (without timezone), and PG would + // implicitly cast to timestamptz using the session timezone, producing incorrect results. + // Explicitly set NpgsqlDbType.TimestampTz to ensure Npgsql rejects non-UTC DateTimes. + : NpgsqlDbType.TimestampTz; + } + + batchCommand.Parameters.Add(npgsqlDbType.HasValue + ? new() { Value = value ?? DBNull.Value, NpgsqlDbType = npgsqlDbType.Value } + : new() { Value = value ?? DBNull.Value }); + } + + batch.BatchCommands.Add(batchCommand); + recordIndex++; + } + + // No records to insert, return false to indicate no command was built. + if (recordIndex == 0) + { + return false; + } + + return true; + + string GenerateSingleUpsertSql(bool autoGeneratedKey) + { + StringBuilder sqlBuilder = new(); + + sqlBuilder + .Append("INSERT INTO ") + .AppendTableName(schema, tableName) + .Append(" ("); + + var i = 0; + foreach (var property in model.Properties) + { + if (property is KeyPropertyModel && autoGeneratedKey) + { + continue; + } + + if (i++ > 0) + { + sqlBuilder.Append(", "); + } + + sqlBuilder.AppendIdentifier(property.StorageName); + } + + sqlBuilder + .AppendLine(")") + .Append("VALUES ("); + + i = 0; + foreach (var property in model.Properties) + { + if (property is KeyPropertyModel && autoGeneratedKey) + { + continue; + } + + if (i++ > 0) + { + sqlBuilder.Append(", "); + } + + sqlBuilder.Append('$').Append(i); + } + + sqlBuilder.Append(')'); + + if (autoGeneratedKey) + { + sqlBuilder + .AppendLine() + .Append("RETURNING ") + .AppendIdentifier(keyProperty.StorageName); + } + else + { + sqlBuilder + .AppendLine() + .Append("ON CONFLICT (") + .AppendIdentifier(keyProperty.StorageName) + .AppendLine(")") + .AppendLine("DO UPDATE SET "); + + i = 0; + foreach (var property in model.Properties) + { + if (property is KeyPropertyModel) + { + continue; + } + + if (i++ > 0) + { + sqlBuilder.AppendLine(", "); + } + + sqlBuilder + .Append(" ") + .AppendIdentifier(property.StorageName) + .Append(" = EXCLUDED.") + .AppendIdentifier(property.StorageName); + } + } + + return sqlBuilder.ToString(); + } + } + + /// + internal static void BuildGetCommand(NpgsqlCommand command, string? schema, string tableName, CollectionModel model, TKey key, bool includeVectors = false) + where TKey : notnull + { + StringBuilder sql = new(); + sql.Append("SELECT "); + + for (var i = 0; i < model.Properties.Count; i++) + { + if (i > 0) + { + sql.Append(", "); + } + sql.AppendIdentifier(model.Properties[i].StorageName); + } + + sql.AppendLine().Append("FROM ").AppendTableName(schema, tableName).AppendLine() + .Append("WHERE ").AppendIdentifier(model.KeyProperty.StorageName).Append(" = $1;"); + + command.CommandText = sql.ToString(); + Debug.Assert(command.Parameters.Count == 0); + command.Parameters.Add(new() { Value = key }); + } + + /// + internal static void BuildGetBatchCommand(NpgsqlCommand command, string? schema, string tableName, CollectionModel model, List keys, bool includeVectors = false) + where TKey : notnull + { + NpgsqlDbType? keyType = PostgresPropertyMapping.GetNpgsqlDbType(model.KeyProperty) ?? throw new UnreachableException($"Unsupported key type {model.KeyProperty.Type.Name}"); + + StringBuilder sql = new(); + sql.Append("SELECT "); + + var first = true; + foreach (var property in model.Properties) + { + if (!includeVectors && property is VectorPropertyModel) + { + continue; + } + + if (!first) + { + sql.Append(", "); + } + first = false; + sql.AppendIdentifier(property.StorageName); + } + + sql.AppendLine().Append("FROM ").AppendTableName(schema, tableName).AppendLine() + .Append("WHERE ").AppendIdentifier(model.KeyProperty.StorageName).Append(" = ANY($1);"); + + command.CommandText = sql.ToString(); + Debug.Assert(command.Parameters.Count == 0); + command.Parameters.Add(new() + { + Value = keys.ToArray(), + NpgsqlDbType = NpgsqlDbType.Array | keyType.Value + }); + } + + /// + internal static void BuildDeleteCommand(NpgsqlCommand command, string? schema, string tableName, string keyColumn, TKey key) + { + StringBuilder sql = new(); + sql.Append("DELETE FROM ").AppendTableName(schema, tableName).AppendLine() + .Append("WHERE ").AppendIdentifier(keyColumn).Append(" = $1;"); + + command.CommandText = sql.ToString(); + Debug.Assert(command.Parameters.Count == 0); + command.Parameters.Add(new() { Value = key }); + } + + /// + internal static void BuildDeleteBatchCommand(NpgsqlCommand command, string? schema, string tableName, KeyPropertyModel keyProperty, List keys) + { + NpgsqlDbType? keyType = PostgresPropertyMapping.GetNpgsqlDbType(keyProperty) ?? throw new ArgumentException($"Unsupported key type {typeof(TKey).Name}"); + + for (int i = 0; i < keys.Count; i++) + { + if (keys[i] == null) + { + throw new ArgumentException("Keys cannot contain null values", nameof(keys)); + } + } + + StringBuilder sql = new(); + sql.Append("DELETE FROM ").AppendTableName(schema, tableName).AppendLine() + .Append("WHERE ").AppendIdentifier(keyProperty.StorageName).Append(" = ANY($1);"); + + command.CommandText = sql.ToString(); + Debug.Assert(command.Parameters.Count == 0); + command.Parameters.Add(new() { Value = keys, NpgsqlDbType = NpgsqlDbType.Array | keyType.Value }); + } + + /// + /// Appends a properly quoted and escaped PostgreSQL identifier to the StringBuilder. + /// In PostgreSQL, identifiers are quoted with double quotes, and embedded double quotes are escaped by doubling them. + /// + internal static StringBuilder AppendIdentifier(this StringBuilder sb, string identifier) + => sb.Append('"').Append(identifier.Replace("\"", "\"\"")).Append('"'); + + /// + /// Appends a properly quoted and escaped PostgreSQL string literal to the StringBuilder. + /// In PostgreSQL, string literals are quoted with single quotes, and embedded single quotes are escaped by doubling them. + /// + internal static StringBuilder AppendLiteral(this StringBuilder sb, string value) + => sb.Append('\'').Append(value.Replace("'", "''")).Append('\''); + + /// + /// Gets the PostgreSQL distance operator for the specified distance function. + /// + private static string GetDistanceOperator(string? distanceFunction) + => distanceFunction switch + { + DistanceFunction.EuclideanDistance or null => "<->", + DistanceFunction.CosineDistance or DistanceFunction.CosineSimilarity => "<=>", + DistanceFunction.ManhattanDistance => "<+>", + DistanceFunction.DotProductSimilarity => "<#>", + DistanceFunction.HammingDistance => "<~>", + _ => throw new NotSupportedException($"Distance function {distanceFunction} is not supported.") + }; + + /// + /// Generates filter clause from the provided filter, returning condition and parameters. + /// + private static (string Clause, List Parameters) GenerateFilterClause( + CollectionModel model, + Expression>? filter, + int startParamIndex) + => filter is not null + ? TranslateFilterWhereClause(model, filter, startParamIndex) + : (Clause: string.Empty, Parameters: new List()); + + /// + /// Generates filter condition (without WHERE keyword) from the provided filter. + /// + private static (string Condition, List Parameters) GenerateFilterCondition( + CollectionModel model, + Expression>? filter, + int startParamIndex) + => filter is not null + ? TranslateFilterCondition(model, filter, startParamIndex) + : (Condition: string.Empty, Parameters: new List()); + + /// + internal static void BuildGetNearestMatchCommand( + NpgsqlCommand command, string? schema, string tableName, CollectionModel model, VectorPropertyModel vectorProperty, object vectorValue, + Expression>? filter, int? skip, bool includeVectors, int limit, + double? scoreThreshold = null) + { + // Build column list with proper escaping + StringBuilder columns = new(); + for (var i = 0; i < model.Properties.Count; i++) + { + if (i > 0) + { + columns.Append(", "); + } + columns.AppendIdentifier(model.Properties[i].StorageName); + } + + var distanceFunction = vectorProperty.DistanceFunction ?? PostgresConstants.DefaultDistanceFunction; + var distanceOp = GetDistanceOperator(distanceFunction); + + // Start where clause params at 2, vector takes param 1. + var (where, parameters) = GenerateFilterClause(model, filter, startParamIndex: 2); + + StringBuilder sql = new(); + sql.Append("SELECT ").Append(columns).Append(", ").AppendIdentifier(vectorProperty.StorageName) + .Append(' ').Append(distanceOp).Append(" $1 AS ").AppendIdentifier(PostgresConstants.DistanceColumnName).AppendLine() + .Append("FROM ").AppendTableName(schema, tableName) + .Append(' ').AppendLine(where) + .Append("ORDER BY ").AppendLine(PostgresConstants.DistanceColumnName) + .Append("LIMIT ").Append(limit); + + if (skip.HasValue) + { + sql.Append(" OFFSET ").Append(skip.Value); + } + + var commandText = sql.ToString(); + + // For cosine similarity, we need to take 1 - cosine distance. + // However, we can't use an expression in the ORDER BY clause or else the index won't be used. + // Instead we'll wrap the query in a subquery and modify the distance in the outer query. + if (vectorProperty.DistanceFunction == DistanceFunction.CosineSimilarity) + { + StringBuilder outerSql = new(); + outerSql.Append("SELECT ").Append(columns).Append(", 1 - ").AppendIdentifier(PostgresConstants.DistanceColumnName) + .Append(" AS ").AppendIdentifier(PostgresConstants.DistanceColumnName).AppendLine() + .Append("FROM (").Append(commandText).Append(") AS subquery"); + commandText = outerSql.ToString(); + } + + // For inner product, we need to take -1 * inner product. + // However, we can't use an expression in the ORDER BY clause or else the index won't be used. + // Instead we'll wrap the query in a subquery and modify the distance in the outer query. + if (vectorProperty.DistanceFunction == DistanceFunction.DotProductSimilarity) + { + StringBuilder outerSql = new(); + outerSql.Append("SELECT ").Append(columns).Append(", -1 * ").AppendIdentifier(PostgresConstants.DistanceColumnName) + .Append(" AS ").AppendIdentifier(PostgresConstants.DistanceColumnName).AppendLine() + .Append("FROM (").Append(commandText).Append(") AS subquery"); + commandText = outerSql.ToString(); + } + + // Apply score threshold filter if specified. + // For similarity functions (higher = more similar), filter out results below the threshold. + // For distance functions (lower = more similar), filter out results above the threshold. + if (scoreThreshold.HasValue) + { + var scoreThresholdParamIndex = parameters.Count + 2; + var comparisonOp = distanceFunction switch + { + DistanceFunction.CosineSimilarity or DistanceFunction.DotProductSimilarity + => ">=", + + DistanceFunction.EuclideanDistance + or DistanceFunction.CosineDistance + or DistanceFunction.ManhattanDistance + or DistanceFunction.HammingDistance + => "<=", + + _ => throw new UnreachableException($"Unexpected distance function: {distanceFunction}") + }; + + StringBuilder outerSql = new(); + outerSql.Append("SELECT * FROM (").Append(commandText).Append(") AS scored WHERE ") + .AppendIdentifier(PostgresConstants.DistanceColumnName).Append(' ').Append(comparisonOp) + .Append(" $").Append(scoreThresholdParamIndex); + commandText = outerSql.ToString(); + } + + command.CommandText = commandText; + + Debug.Assert(command.Parameters.Count == 0); + command.Parameters.Add(new NpgsqlParameter { Value = vectorValue }); + + foreach (var parameter in parameters) + { + command.Parameters.Add(new NpgsqlParameter { Value = parameter }); + } + + if (scoreThreshold.HasValue) + { + command.Parameters.Add(new NpgsqlParameter { Value = scoreThreshold.Value }); + } + } + + internal static void BuildSelectWhereCommand( + NpgsqlCommand command, string? schema, string tableName, CollectionModel model, + Expression> filter, int top, FilteredRecordRetrievalOptions options) + { + StringBuilder query = new(200); + query.Append("SELECT "); + var first = true; + foreach (var property in model.Properties) + { + if (options.IncludeVectors || property is not VectorPropertyModel) + { + if (!first) + { + query.Append(','); + } + first = false; + query.AppendIdentifier(property.StorageName); + } + } + query.AppendLine(); + query.Append("FROM ").AppendTableName(schema, tableName).AppendLine(); + + PostgresFilterTranslator translator = new(model, filter, startParamIndex: 1, query); + translator.Translate(appendWhere: true); + query.AppendLine(); + + var orderBy = options.OrderBy?.Invoke(new()).Values; + if (orderBy is { Count: > 0 }) + { + query.Append("ORDER BY "); + + var firstOrderBy = true; + foreach (var sortInfo in orderBy) + { + if (!firstOrderBy) + { + query.Append(','); + } + firstOrderBy = false; + query.AppendIdentifier(model.GetDataOrKeyProperty(sortInfo.PropertySelector).StorageName) + .Append(sortInfo.Ascending ? " ASC" : " DESC"); + } + + query.AppendLine(); + } + + query.AppendFormat("OFFSET {0}", options.Skip).AppendLine(); + query.AppendFormat("LIMIT {0}", top).AppendLine(); + + command.CommandText = query.ToString(); + + Debug.Assert(command.Parameters.Count == 0); + foreach (var parameter in translator.ParameterValues) + { + command.Parameters.Add(new NpgsqlParameter { Value = parameter }); + } + } + + private static (string Clause, List Parameters) TranslateFilterWhereClause(CollectionModel model, LambdaExpression filter, int startParamIndex) + { + var (condition, parameters) = TranslateFilterCondition(model, filter, startParamIndex); + return (string.IsNullOrEmpty(condition) ? string.Empty : $"WHERE {condition}", parameters); + } + + private static (string Condition, List Parameters) TranslateFilterCondition(CollectionModel model, LambdaExpression filter, int startParamIndex) + { + PostgresFilterTranslator translator = new(model, filter, startParamIndex); + translator.Translate(appendWhere: false); + return (translator.Clause.ToString(), translator.ParameterValues); + } + + /// + /// Appends a schema-qualified table name. If schema is null or empty, omits the schema prefix. + /// + private static StringBuilder AppendTableName(this StringBuilder sb, string? schema, string tableName) + { + if (!string.IsNullOrEmpty(schema)) + { + sb.AppendIdentifier(schema!).Append('.'); + } + + return sb.AppendIdentifier(tableName); + } + + /// + /// Builds a hybrid search command that combines vector similarity search with full-text keyword search using RRF (Reciprocal Rank Fusion). + /// + internal static void BuildHybridSearchCommand( + NpgsqlCommand command, string? schema, string tableName, CollectionModel model, + VectorPropertyModel vectorProperty, DataPropertyModel textProperty, + object vectorValue, ICollection keywords, + Expression>? filter, + int? skip, bool includeVectors, int top, double? scoreThreshold = null) + { + // RRF constant - higher values give more weight to lower-ranked results + const int RrfConstant = 60; + + // Build column list with proper escaping (for the final SELECT) + StringBuilder columns = new(); + for (var i = 0; i < model.Properties.Count; i++) + { + if (!includeVectors && model.Properties[i] is VectorPropertyModel) + { + continue; + } + + if (columns.Length > 0) + { + columns.Append(", "); + } + + columns.Append("t.").AppendIdentifier(model.Properties[i].StorageName); + } + + var distanceOp = GetDistanceOperator(vectorProperty.DistanceFunction ?? PostgresConstants.DefaultDistanceFunction); + + // Parameters: $1 = keywords, $2 = vector, $3 = RRF constant + // Additional parameters start at $4 for filters + var (filterCondition, filterParameters) = GenerateFilterCondition(model, filter, startParamIndex: 4); + + // Build the full table name + var fullTableName = new StringBuilder() + .AppendTableName(schema, tableName) + .ToString(); + + // Use a larger internal limit for the CTEs to get better ranking, then limit final results + var internalLimit = (top + (skip ?? 0)) * 2; + if (internalLimit < 20) + { + internalLimit = 20; + } + + // Get the full-text search language from the text property + var language = textProperty.GetFullTextSearchLanguageOrDefault(); + + StringBuilder sql = new(); + sql.AppendLine() + .Append("WITH semantic_search AS (").AppendLine() + .Append(" SELECT ").AppendIdentifier(model.KeyProperty.StorageName).Append(", RANK () OVER (ORDER BY ").AppendIdentifier(vectorProperty.StorageName) + .Append(' ').Append(distanceOp).Append(" $2) AS rank").AppendLine() + .Append(" FROM ").Append(fullTableName); + + // Add filter for semantic search + if (!string.IsNullOrEmpty(filterCondition)) + { + sql.AppendLine().Append(" WHERE ").Append(filterCondition); + } + + sql.AppendLine() + .Append(" ORDER BY ").AppendIdentifier(vectorProperty.StorageName).Append(' ').Append(distanceOp).Append(" $2").AppendLine() + .Append(" LIMIT ").Append(internalLimit).AppendLine() + .Append("),").AppendLine() + .Append("keyword_search AS (").AppendLine() + .Append(" SELECT ").AppendIdentifier(model.KeyProperty.StorageName).Append(", RANK () OVER (ORDER BY ts_rank_cd(to_tsvector(").AppendLiteral(language).Append(", ").AppendIdentifier(textProperty.StorageName).Append("), query) DESC) AS rank").AppendLine() + .Append(" FROM ").Append(fullTableName).Append(", plainto_tsquery(").AppendLiteral(language).Append(", $1) query").AppendLine() + .Append(" WHERE to_tsvector(").AppendLiteral(language).Append(", ").AppendIdentifier(textProperty.StorageName).Append(") @@ query"); + + // Add filter for keyword search (using AND since we already have a WHERE clause) + if (!string.IsNullOrEmpty(filterCondition)) + { + sql.Append(" AND ").Append(filterCondition); + } + + sql.AppendLine() + .Append(" ORDER BY ts_rank_cd(to_tsvector(").AppendLiteral(language).Append(", ").AppendIdentifier(textProperty.StorageName).Append("), query) DESC").AppendLine() + .Append(" LIMIT ").Append(internalLimit).AppendLine() + .Append(')').AppendLine() + .Append("SELECT ").Append(columns).Append(',').AppendLine() + .Append(" COALESCE(1.0 / ($3 + semantic_search.rank), 0.0) +").AppendLine() + .Append(" COALESCE(1.0 / ($3 + keyword_search.rank), 0.0) AS ").AppendIdentifier(PostgresConstants.DistanceColumnName).AppendLine() + .Append("FROM semantic_search").AppendLine() + .Append("FULL OUTER JOIN keyword_search ON semantic_search.").AppendIdentifier(model.KeyProperty.StorageName).Append(" = keyword_search.").AppendIdentifier(model.KeyProperty.StorageName).AppendLine() + .Append("JOIN ").Append(fullTableName).Append(" t ON t.").AppendIdentifier(model.KeyProperty.StorageName).Append(" = COALESCE(semantic_search.").AppendIdentifier(model.KeyProperty.StorageName).Append(", keyword_search.").AppendIdentifier(model.KeyProperty.StorageName).Append(')').AppendLine() + .Append("ORDER BY ").AppendIdentifier(PostgresConstants.DistanceColumnName).Append(" DESC"); + + var commandText = sql.ToString(); + + // Apply score threshold filter if specified (higher RRF scores = better match) + if (scoreThreshold.HasValue) + { + var scoreThresholdParamIndex = filterParameters.Count + 4; + StringBuilder outerSql = new(); + outerSql.Append("SELECT * FROM (").Append(commandText).Append(") AS scored WHERE ") + .AppendIdentifier(PostgresConstants.DistanceColumnName).Append(" >= $").Append(scoreThresholdParamIndex); + commandText = outerSql.ToString(); + } + + // Apply LIMIT and OFFSET + StringBuilder finalSql = new(); + finalSql.Append("SELECT * FROM (").Append(commandText).Append(") AS results LIMIT ").Append(top); + if (skip > 0) + { + finalSql.Append(" OFFSET ").Append(skip.Value); + } + + command.CommandText = finalSql.ToString(); + + Debug.Assert(command.Parameters.Count == 0); + + // $1 = keywords (joined as single string for plainto_tsquery) + command.Parameters.Add(new NpgsqlParameter { Value = string.Join(" ", keywords) }); + // $2 = vector + command.Parameters.Add(new NpgsqlParameter { Value = vectorValue }); + // $3 = RRF constant + command.Parameters.Add(new NpgsqlParameter { Value = RrfConstant }); + + // Filter parameters starting at $4 + foreach (var parameter in filterParameters) + { + command.Parameters.Add(new NpgsqlParameter { Value = parameter }); + } + + // Score threshold parameter + if (scoreThreshold.HasValue) + { + command.Parameters.Add(new NpgsqlParameter { Value = scoreThreshold.Value }); + } + } +} diff --git a/MEVD/src/PgVector/PostgresUtils.cs b/MEVD/src/PgVector/PostgresUtils.cs new file mode 100644 index 0000000..c8c5d58 --- /dev/null +++ b/MEVD/src/PgVector/PostgresUtils.cs @@ -0,0 +1,75 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.VectorData; +using Npgsql; +using Microsoft.Shared.Diagnostics; +using static Microsoft.Extensions.VectorData.VectorStoreErrorHandler; + +namespace CommunityToolkit.VectorData.PgVector; + +internal static class PostgresUtils +{ + /// + /// Wraps an in an that will throw a + /// if an exception is thrown while iterating over the original enumerator. + /// + /// The type of the items in the async enumerable. + /// The async enumerable to wrap. + /// The name of the operation being performed. + /// The vector store metadata to describe the type of database. + /// An async enumerable that will throw a if an exception is thrown while iterating over the original enumerator. + public static async IAsyncEnumerable WrapAsyncEnumerableAsync( + IAsyncEnumerable asyncEnumerable, + string operationName, + VectorStoreMetadata metadata) + { + var errorHandlingEnumerable = new ConfiguredCancelableErrorHandlingAsyncEnumerable( + asyncEnumerable.ConfigureAwait(false), + metadata, + operationName); + +#pragma warning disable CA2007 // Consider calling ConfigureAwait on the awaited task: False Positive + await foreach (var item in errorHandlingEnumerable.ConfigureAwait(false)) +#pragma warning restore CA2007 // Consider calling ConfigureAwait on the awaited task + { + yield return item; + } + } + + /// + /// Wraps an in an that will throw a + /// if an exception is thrown while iterating over the original enumerator. + /// + /// The type of the items in the async enumerable. + /// The async enumerable to wrap. + /// The name of the operation being performed. + /// The collection metadata to describe the type of database. + /// An async enumerable that will throw a if an exception is thrown while iterating over the original enumerator. + public static async IAsyncEnumerable WrapAsyncEnumerableAsync( + IAsyncEnumerable asyncEnumerable, + string operationName, + VectorStoreCollectionMetadata metadata) + { + var errorHandlingEnumerable = new ConfiguredCancelableErrorHandlingAsyncEnumerable( + asyncEnumerable.ConfigureAwait(false), + metadata, + operationName); + +#pragma warning disable CA2007 // Consider calling ConfigureAwait on the awaited task: False Positive + await foreach (var item in errorHandlingEnumerable.ConfigureAwait(false)) +#pragma warning restore CA2007 // Consider calling ConfigureAwait on the awaited task + { + yield return item; + } + } + + internal static NpgsqlDataSource CreateDataSource(string connectionString) + { + Throw.IfNullOrWhitespace(connectionString); + + NpgsqlDataSourceBuilder sourceBuilder = new(connectionString); + sourceBuilder.UseVector(); + return sourceBuilder.Build(); + } +} diff --git a/MEVD/src/PgVector/PostgresVectorStore.cs b/MEVD/src/PgVector/PostgresVectorStore.cs new file mode 100644 index 0000000..f9ad83f --- /dev/null +++ b/MEVD/src/PgVector/PostgresVectorStore.cs @@ -0,0 +1,171 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; +using Npgsql; +using Microsoft.Shared.Diagnostics; + +namespace CommunityToolkit.VectorData.PgVector; + +/// +/// Represents a vector store implementation using PostgreSQL. +/// +public sealed class PostgresVectorStore : VectorStore +{ + /// Metadata about vector store. + private readonly VectorStoreMetadata _metadata; + + /// A general purpose definition that can be used to construct a collection when needing to proxy schema agnostic operations. + private static readonly VectorStoreCollectionDefinition s_generalPurposeDefinition = new() { Properties = [new VectorStoreKeyProperty("Key", typeof(string))] }; + + /// Data source used to interact with the database. + private readonly NpgsqlDataSource _dataSource; + private readonly NpgsqlDataSourceArc? _dataSourceArc; + private readonly string _databaseName; + + /// The database schema. + private readonly string? _schema; + + private readonly IEmbeddingGenerator? _embeddingGenerator; + + /// + /// Initializes a new instance of the class. + /// + /// Postgres data source. + /// A value indicating whether is disposed when this instance of is disposed. + /// Optional configuration options for this class + public PostgresVectorStore(NpgsqlDataSource dataSource, bool ownsDataSource, PostgresVectorStoreOptions? options = default) + { + Throw.IfNull(dataSource); + + _schema = options?.Schema; + _embeddingGenerator = options?.EmbeddingGenerator; + _dataSource = dataSource; + _dataSourceArc = ownsDataSource ? new NpgsqlDataSourceArc(dataSource) : null; + _databaseName = new NpgsqlConnectionStringBuilder(dataSource.ConnectionString).Database!; + + _metadata = new() + { + VectorStoreSystemName = PostgresConstants.VectorStoreSystemName, + VectorStoreName = _databaseName + }; + } + + /// + /// Initializes a new instance of the class. + /// + /// Postgres database connection string. + /// Optional configuration options for this class. + public PostgresVectorStore(string connectionString, PostgresVectorStoreOptions? options = default) +#pragma warning disable CA2000 // Dispose objects before losing scope + : this(PostgresUtils.CreateDataSource(connectionString), ownsDataSource: true, options) +#pragma warning restore CA2000 // Dispose objects before losing scope + { + } + + /// + protected override void Dispose(bool disposing) + { + _dataSourceArc?.Dispose(); + base.Dispose(disposing); + } + + /// + public override IAsyncEnumerable ListCollectionNamesAsync(CancellationToken cancellationToken = default) + { + return PostgresUtils.WrapAsyncEnumerableAsync( + GetTablesAsync(cancellationToken), + "ListCollectionNames", + _metadata); + + async IAsyncEnumerable GetTablesAsync([EnumeratorCancellation] CancellationToken cancellationToken = default) + { + NpgsqlConnection connection = await _dataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false); + + await using (connection) + using (var command = connection.CreateCommand()) + { + PostgresSqlBuilder.BuildGetTablesCommand(command, _schema); + using NpgsqlDataReader dataReader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + + while (await dataReader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + yield return dataReader.GetString(dataReader.GetOrdinal("table_name")); + } + } + } + } + +#pragma warning disable IDE0090 // Use 'new(...)' + /// + [RequiresDynamicCode("This overload of GetCollection() is incompatible with NativeAOT. For dynamic mapping via Dictionary, call GetDynamicCollection() instead.")] + [RequiresUnreferencedCode("This overload of GetCollecttion() is incompatible with trimming. For dynamic mapping via Dictionary, call GetDynamicCollection() instead.")] +#if NET + public override PostgresCollection GetCollection(string name, VectorStoreCollectionDefinition? definition = null) +#else + public override VectorStoreCollection GetCollection(string name, VectorStoreCollectionDefinition? definition = null) +#endif + => typeof(TRecord) == typeof(Dictionary) + ? throw new ArgumentException(VectorDataStrings.GetCollectionWithDictionaryNotSupported) + : new PostgresCollection( + _dataSource, + _dataSourceArc, + name, + new() + { + Schema = _schema, + Definition = definition, + EmbeddingGenerator = _embeddingGenerator, + } + ); + + /// +#if NET + public override PostgresDynamicCollection GetDynamicCollection(string name, VectorStoreCollectionDefinition definition) +#else + public override VectorStoreCollection> GetDynamicCollection(string name, VectorStoreCollectionDefinition definition) +#endif + => new PostgresDynamicCollection( + _dataSource, + _dataSourceArc, + name, + new() + { + Schema = _schema, + Definition = definition, + EmbeddingGenerator = _embeddingGenerator, + } + ); +#pragma warning restore IDE0090 // Use 'new(...)' + + /// + public override Task CollectionExistsAsync(string name, CancellationToken cancellationToken = default) + { + var collection = GetDynamicCollection(name, s_generalPurposeDefinition); + return collection.CollectionExistsAsync(cancellationToken); + } + + /// + public override Task EnsureCollectionDeletedAsync(string name, CancellationToken cancellationToken = default) + { + var collection = GetDynamicCollection(name, s_generalPurposeDefinition); + return collection.EnsureCollectionDeletedAsync(cancellationToken); + } + + /// + public override object? GetService(Type serviceType, object? serviceKey = null) + { + Throw.IfNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreMetadata) ? _metadata : + serviceType == typeof(NpgsqlDataSource) ? _dataSource : + serviceType.IsInstanceOfType(this) ? this : + null; + } +} diff --git a/MEVD/src/PgVector/PostgresVectorStoreOptions.cs b/MEVD/src/PgVector/PostgresVectorStoreOptions.cs new file mode 100644 index 0000000..5c8b7d6 --- /dev/null +++ b/MEVD/src/PgVector/PostgresVectorStoreOptions.cs @@ -0,0 +1,37 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.AI; + +namespace CommunityToolkit.VectorData.PgVector; + +/// +/// Options when creating a . +/// +public sealed class PostgresVectorStoreOptions +{ + internal static readonly PostgresVectorStoreOptions Default = new(); + + /// + /// Initializes a new instance of the class. + /// + public PostgresVectorStoreOptions() + { + } + + internal PostgresVectorStoreOptions(PostgresVectorStoreOptions? source) + { + Schema = source?.Schema; + EmbeddingGenerator = source?.EmbeddingGenerator; + } + + /// + /// Gets or sets the database schema. + /// + public string? Schema { get; set; } + + /// + /// Gets or sets the default embedding generator to use when generating vectors embeddings with this vector store. + /// + public IEmbeddingGenerator? EmbeddingGenerator { get; set; } +} diff --git a/MEVD/src/PgVector/README.md b/MEVD/src/PgVector/README.md new file mode 100644 index 0000000..4d98642 --- /dev/null +++ b/MEVD/src/PgVector/README.md @@ -0,0 +1,42 @@ +# Microsoft.SemanticKernel.Connectors.Postgres + +This connector uses Postgres to implement Semantic Memory. It requires the [pgvector](https://github.com/pgvector/pgvector) extension to be installed on Postgres to implement vector similarity search. + +## What is pgvector? + +[pgvector](https://github.com/pgvector/pgvector) is an open-source vector similarity search engine for Postgres. It supports exact and approximate nearest neighbor search, L2 distance, inner product, and cosine distance. + +How to install the pgvector extension, please refer to its [documentation](https://github.com/pgvector/pgvector#installation). + +This extension is also available for **Azure Database for PostgreSQL - Flexible Server** and **Azure Cosmos DB for PostgreSQL**. + +- [Azure Database for Postgres](https://learn.microsoft.com/en-us/azure/postgresql/flexible-server/how-to-use-pgvector) +- [Azure Cosmos DB for PostgreSQL](https://learn.microsoft.com/en-us/azure/cosmos-db/postgresql/howto-use-pgvector) + +## Quick start + +1. To install pgvector using Docker: + +```bash +docker run -d --name postgres-pgvector -p 5432:5432 -e POSTGRES_PASSWORD=mysecretpassword pgvector/pgvector +``` + +2. Create a database and enable pgvector extension on this database + +```bash +docker exec -it postgres-pgvector psql -U postgres + +postgres=# CREATE DATABASE sk_demo; +postgres=# \c sk_demo +sk_demo=# CREATE EXTENSION vector; +``` + +> Note, "Azure Cosmos DB for PostgreSQL" uses `SELECT CREATE_EXTENSION('vector');` to enable the extension. + +### Using PostgresVectorStore + +See [this sample](../../../samples/Concepts/Memory/VectorStore_VectorSearch_MultiStore_Postgres.cs) for an example of using the vector store. + +For more information on using Postgres as a vector store, see the [PostgresVectorStore](https://learn.microsoft.com/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/postgres-connector) documentation. + +Use the [getting started instructions](https://learn.microsoft.com/semantic-kernel/concepts/vector-store-connectors/?pivots=programming-language-csharp#getting-started-with-vector-store-connectors) on the Microsoft Leearn site to learn more about using the vector store. diff --git a/MEVD/src/Pinecone/AssemblyInfo.cs b/MEVD/src/Pinecone/AssemblyInfo.cs new file mode 100644 index 0000000..d6395e8 --- /dev/null +++ b/MEVD/src/Pinecone/AssemblyInfo.cs @@ -0,0 +1,2 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. diff --git a/MEVD/src/Pinecone/Pinecone.csproj b/MEVD/src/Pinecone/Pinecone.csproj new file mode 100644 index 0000000..63c378d --- /dev/null +++ b/MEVD/src/Pinecone/Pinecone.csproj @@ -0,0 +1,29 @@ + + + + 1.0.0-preview.1 + CommunityToolkit.VectorData.Pinecone + $(AssemblyName) + net10.0;net8.0;netstandard2.0;net462 + true + + Pinecone provider for Microsoft.Extensions.VectorData + Pinecone provider for Microsoft.Extensions.VectorData by Semantic Kernel + + + + + + + + + + + + + + + + + + diff --git a/MEVD/src/Pinecone/PineconeCollection.cs b/MEVD/src/Pinecone/PineconeCollection.cs new file mode 100644 index 0000000..ad5e2ec --- /dev/null +++ b/MEVD/src/Pinecone/PineconeCollection.cs @@ -0,0 +1,635 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Linq.Expressions; +using System.Runtime.CompilerServices; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; +using Pinecone; +using Microsoft.Shared.Diagnostics; +using VectorDataCollectionModel = Microsoft.Extensions.VectorData.ProviderServices.CollectionModel; + +namespace CommunityToolkit.VectorData.Pinecone; + +/// +/// Service for storing and retrieving vector records, that uses Pinecone as the underlying storage. +/// +/// The data type of the record key. Must be . +/// The data model to use for adding, updating and retrieving data from storage. +#pragma warning disable CA1711 // Identifiers should not have incorrect suffix +public class PineconeCollection : VectorStoreCollection + where TKey : notnull + where TRecord : class +#pragma warning restore CA1711 // Identifiers should not have incorrect suffix +{ + private static readonly VectorSearchOptions s_defaultVectorSearchOptions = new(); + + /// Metadata about vector store record collection. + private readonly VectorStoreCollectionMetadata _collectionMetadata; + + private readonly PineconeClient _pineconeClient; + private readonly VectorDataCollectionModel _model; + private readonly PineconeMapper _mapper; + private IndexClient? _indexClient; + + /// + public override string Name { get; } + + /// The namespace within the Pinecone index that will be used for operations involving records (Get, Upsert, Delete). + private readonly string? _indexNamespace; + + /// The public cloud where the serverless index is hosted. + private readonly string _serverlessIndexCloud; + + /// The region where the serverless index is created. + private readonly string _serverlessIndexRegion; + + /// + /// Initializes a new instance of the class. + /// + /// Pinecone client that can be used to manage the collections and vectors in a Pinecone store. + /// Optional configuration options for this class. + /// Thrown if the is null. + /// The name of the collection that this will access. + /// Thrown for any misconfigured options. + [RequiresDynamicCode("This constructor is incompatible with NativeAOT. For dynamic mapping via Dictionary, instantiate PineconeDynamicCollection instead.")] + [RequiresUnreferencedCode("This constructor is incompatible with trimming. For dynamic mapping via Dictionary, instantiate PineconeDynamicCollection instead.")] + public PineconeCollection(PineconeClient pineconeClient, string name, PineconeCollectionOptions? options = null) + : this( + pineconeClient, + name, + static options => typeof(TRecord) == typeof(Dictionary) + ? throw new NotSupportedException(VectorDataStrings.NonDynamicCollectionWithDictionaryNotSupported(typeof(PineconeDynamicCollection))) + : new PineconeModelBuilder().Build(typeof(TRecord), typeof(TKey), options.Definition, options.EmbeddingGenerator), + options) + { + } + + internal PineconeCollection(PineconeClient pineconeClient, string name, Func modelFactory, PineconeCollectionOptions? options) + { + Throw.IfNull(pineconeClient); + VerifyCollectionName(name); + + if (typeof(TKey) != typeof(string) && typeof(TKey) != typeof(Guid) && typeof(TKey) != typeof(object)) + { + throw new NotSupportedException("Only string and Guid keys are supported."); + } + + options ??= PineconeCollectionOptions.Default; + + _pineconeClient = pineconeClient; + Name = name; + _model = modelFactory(options); + + _indexNamespace = options.IndexNamespace; + _serverlessIndexCloud = options.ServerlessIndexCloud; + _serverlessIndexRegion = options.ServerlessIndexRegion; + + _mapper = new PineconeMapper(_model); + + _collectionMetadata = new() + { + VectorStoreSystemName = PineconeConstants.VectorStoreSystemName, + CollectionName = name + }; + } + + /// + public override Task CollectionExistsAsync(CancellationToken cancellationToken = default) + => RunCollectionOperationAsync( + "CollectionExists", + async () => + { + var collections = await _pineconeClient.ListIndexesAsync(cancellationToken: cancellationToken).ConfigureAwait(false); + + return collections.Indexes?.Any(x => x.Name == Name) is true; + }); + + /// + public override async Task EnsureCollectionExistsAsync(CancellationToken cancellationToken = default) + { + // Don't even try to create if the collection already exists. + if (await CollectionExistsAsync(cancellationToken).ConfigureAwait(false)) + { + return; + } + + // we already run through record property validation, so a single VectorStoreRecordVectorProperty is guaranteed. + var vectorProperty = _model.VectorProperty!; + + if (!string.IsNullOrEmpty(vectorProperty.IndexKind) && vectorProperty.IndexKind != "PGA") + { + throw new NotSupportedException( + $"IndexKind of '{vectorProperty.IndexKind}' for property '{vectorProperty.ModelName}' is not supported. Pinecone only supports 'PGA' (Pinecone Graph Algorithm), which is always enabled."); + } + + CreateIndexRequest request = new() + { + Name = Name, + Dimension = vectorProperty.Dimensions, + Metric = MapDistanceFunction(vectorProperty), + Spec = new ServerlessIndexSpec + { + Serverless = new ServerlessSpec + { + Cloud = MapCloud(_serverlessIndexCloud), + Region = _serverlessIndexRegion, + } + }, + }; + + try + { + await _pineconeClient.CreateIndexAsync(request, cancellationToken: cancellationToken).ConfigureAwait(false); + } + catch (ConflictError) + { + // Do nothing, since the index is already created. + } + catch (PineconeApiException other) + { + throw new VectorStoreException("Call to vector store failed.", other) + { + VectorStoreSystemName = PineconeConstants.VectorStoreSystemName, + VectorStoreName = _collectionMetadata.VectorStoreName, + CollectionName = Name, + OperationName = "EnsureCollectionExists" + }; + } + } + + /// + public override async Task EnsureCollectionDeletedAsync(CancellationToken cancellationToken = default) + { + try + { + await _pineconeClient.DeleteIndexAsync(Name, cancellationToken: cancellationToken).ConfigureAwait(false); + } + catch (NotFoundError) + { + // If the collection does not exist, we should ignore the exception. + } + catch (PineconeApiException other) + { + throw new VectorStoreException("Call to vector store failed.", other) + { + VectorStoreSystemName = PineconeConstants.VectorStoreSystemName, + VectorStoreName = _collectionMetadata.VectorStoreName, + CollectionName = Name, + OperationName = "DeleteCollection" + }; + } + } + + /// + public override async Task GetAsync(TKey key, RecordRetrievalOptions? options = null, CancellationToken cancellationToken = default) + { + var includeVectors = options?.IncludeVectors is true; + if (includeVectors && _model.EmbeddingGenerationRequired) + { + throw new NotSupportedException(VectorDataStrings.IncludeVectorsNotSupportedWithEmbeddingGeneration); + } + + FetchRequest request = new() + { + Namespace = _indexNamespace, + Ids = [GetStringKey(key)] + }; + + var response = await RunIndexOperationAsync( + "Get", + indexClient => indexClient.FetchAsync(request, cancellationToken: cancellationToken)).ConfigureAwait(false); + + var result = response.Vectors?.Values.FirstOrDefault(); + if (result is null) + { + return default; + } + + return _mapper.MapFromStorageToDataModel(result, includeVectors); + } + + /// + public override async IAsyncEnumerable GetAsync( + IEnumerable keys, + RecordRetrievalOptions? options = default, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + Throw.IfNull(keys); + + var includeVectors = options?.IncludeVectors is true; + if (includeVectors && _model.EmbeddingGenerationRequired) + { + throw new NotSupportedException(VectorDataStrings.IncludeVectorsNotSupportedWithEmbeddingGeneration); + } + +#pragma warning disable CA1851 // Bogus: Possible multiple enumerations of 'IEnumerable' collection + var keysList = keys switch + { + IEnumerable k => k.ToList(), + IEnumerable k => k.Select(x => x.ToString()).ToList(), + IEnumerable k => k.Select(x => x.ToString()!).ToList(), + _ => throw new UnreachableException("string key should have been validated during model building") + }; +#pragma warning restore CA1851 + + if (keysList.Count == 0) + { + yield break; + } + + FetchRequest request = new() + { + Namespace = _indexNamespace, + Ids = keysList + }; + + var response = await RunIndexOperationAsync( + "GetBatch", + indexClient => indexClient.FetchAsync(request, cancellationToken: cancellationToken)).ConfigureAwait(false); + if (response.Vectors is null || response.Vectors.Count == 0) + { + yield break; + } + + var records = response.Vectors.Values.Select(x => _mapper.MapFromStorageToDataModel(x, includeVectors)); + + foreach (var record in records) + { + yield return record; + } + } + + /// + public override Task DeleteAsync(TKey key, CancellationToken cancellationToken = default) + { + DeleteRequest request = new() + { + Namespace = _indexNamespace, + Ids = [GetStringKey(key)] + }; + + return RunIndexOperationAsync( + "Delete", + indexClient => indexClient.DeleteAsync(request, cancellationToken: cancellationToken)); + } + + /// + public override Task DeleteAsync(IEnumerable keys, CancellationToken cancellationToken = default) + { + Throw.IfNull(keys); + + var keysList = keys switch + { + IEnumerable k => k.ToList(), + IEnumerable k => k.Select(x => x.ToString()).ToList(), + IEnumerable k => k.Select(x => x.ToString()!).ToList(), + _ => throw new UnreachableException("string key should have been validated during model building") + }; + + if (keysList.Count == 0) + { + return Task.CompletedTask; + } + + DeleteRequest request = new() + { + Namespace = _indexNamespace, + Ids = keysList + }; + + return RunIndexOperationAsync( + "DeleteBatch", + indexClient => indexClient.DeleteAsync(request, cancellationToken: cancellationToken)); + } + + /// + public override async Task UpsertAsync(TRecord record, CancellationToken cancellationToken = default) + { + Throw.IfNull(record); + + // Handle auto-generated keys (client-side for Pinecone, which doesn't support server-side auto-generation) + var keyProperty = _model.KeyProperty; + if (keyProperty.IsAutoGenerated && keyProperty.GetValue(record) == Guid.Empty) + { + keyProperty.SetValue(record, Guid.NewGuid()); + } + + // If an embedding generator is defined, invoke it once for all records. + Embedding? generatedEmbedding = null; + + Debug.Assert(_model.VectorProperties.Count <= 1); + if (_model.VectorProperties is [var vectorProperty] && !PineconeModelBuilder.IsVectorPropertyTypeValidCore(vectorProperty.Type, out _)) + { + // We have a vector property whose type isn't natively supported - we need to generate embeddings. + Debug.Assert(vectorProperty.EmbeddingGenerator is not null); + + generatedEmbedding = (Embedding)await vectorProperty.GenerateEmbeddingAsync(vectorProperty.GetValueAsObject(record), cancellationToken).ConfigureAwait(false); + } + + var vector = _mapper.MapFromDataToStorageModel(record, generatedEmbedding); + + UpsertRequest request = new() + { + Namespace = _indexNamespace, + Vectors = [vector], + }; + + await RunIndexOperationAsync( + "Upsert", + indexClient => indexClient.UpsertAsync(request, cancellationToken: cancellationToken)).ConfigureAwait(false); + } + + /// + public override async Task UpsertAsync(IEnumerable records, CancellationToken cancellationToken = default) + { + Throw.IfNull(records); + + // If an embedding generator is defined, invoke it once for all records. + GeneratedEmbeddings>? generatedEmbeddings = null; + + if (_model.VectorProperties is [var vectorProperty] && !PineconeModelBuilder.IsVectorPropertyTypeValidCore(vectorProperty.Type, out _)) + { + // We have a vector property whose type isn't natively supported - we need to generate embeddings. + Debug.Assert(vectorProperty.EmbeddingGenerator is not null); + + var recordsList = records is IReadOnlyList r ? r : records.ToList(); + + if (recordsList.Count == 0) + { + return; + } + + records = recordsList; + + generatedEmbeddings = (GeneratedEmbeddings>)await vectorProperty.GenerateEmbeddingsAsync(records.Select(r => vectorProperty.GetValueAsObject(r)), cancellationToken).ConfigureAwait(false); + + Debug.Assert(generatedEmbeddings.Count == recordsList.Count); + } + + // Handle auto-generated keys (client-side for Pinecone, which doesn't support server-side auto-generation) + var keyProperty = _model.KeyProperty; + var vectors = new List(); + var recordIndex = 0; + foreach (var record in records) + { + if (keyProperty.IsAutoGenerated && keyProperty.GetValue(record) == Guid.Empty) + { + keyProperty.SetValue(record, Guid.NewGuid()); + } + + vectors.Add(_mapper.MapFromDataToStorageModel(record, generatedEmbeddings?[recordIndex++])); + } + + if (vectors.Count == 0) + { + return; + } + + UpsertRequest request = new() + { + Namespace = _indexNamespace, + Vectors = vectors, + }; + + await RunIndexOperationAsync( + "UpsertBatch", + indexClient => indexClient.UpsertAsync(request, cancellationToken: cancellationToken)).ConfigureAwait(false); + } + + #region Search + + /// + public override async IAsyncEnumerable> SearchAsync( + TInput searchValue, + int top, + VectorSearchOptions? options = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + Throw.IfNull(searchValue); + Throw.IfLessThan(top, 1); + + options ??= s_defaultVectorSearchOptions; + + if (options.IncludeVectors && _model.EmbeddingGenerationRequired) + { + throw new NotSupportedException(VectorDataStrings.IncludeVectorsNotSupportedWithEmbeddingGeneration); + } + + var vectorProperty = _model.GetVectorPropertyOrSingle(options); + + ReadOnlyMemory vector = searchValue switch + { + ReadOnlyMemory r => r, + float[] f => new ReadOnlyMemory(f), + Embedding e => e.Vector, + _ when vectorProperty.EmbeddingGenerationDispatcher is not null + => ((Embedding)await vectorProperty.GenerateEmbeddingAsync(searchValue, cancellationToken).ConfigureAwait(false)).Vector, + + _ => vectorProperty.EmbeddingGenerator is null + ? throw new NotSupportedException(VectorDataStrings.InvalidSearchInputAndNoEmbeddingGeneratorWasConfigured(searchValue.GetType(), PineconeModelBuilder.SupportedVectorTypes)) + : throw new InvalidOperationException(VectorDataStrings.IncompatibleEmbeddingGeneratorWasConfiguredForInputType(typeof(TInput), vectorProperty.EmbeddingGenerator.GetType())) + }; + + var filter = options.Filter is not null + ? new PineconeFilterTranslator().Translate(options.Filter, _model) + : null; + + QueryRequest request = new() + { + TopK = (uint)(top + options.Skip), + Namespace = _indexNamespace, + IncludeValues = options.IncludeVectors, + IncludeMetadata = true, + Vector = vector, + Filter = filter, + }; + + QueryResponse response = await RunIndexOperationAsync( + "VectorizedSearch", + indexClient => indexClient.QueryAsync(request, cancellationToken: cancellationToken)).ConfigureAwait(false); + + if (response.Matches is null) + { + yield break; + } + + // Pinecone does not provide a way to skip results, so we need to do it manually. + var skippedResults = response.Matches + .Skip(options.Skip); + + var records = skippedResults.Select( + x => new VectorSearchResult( + _mapper.MapFromStorageToDataModel( + new Vector() + { + Id = x.Id, + Values = x.Values ?? Array.Empty(), + Metadata = x.Metadata, + SparseValues = x.SparseValues + }, + options.IncludeVectors), + x.Score)); + + foreach (var record in records) + { + // Pinecone returns similarity scores where higher values indicate more similar results. + if (options.ScoreThreshold.HasValue && record.Score < options.ScoreThreshold.Value) + { + continue; + } + + yield return record; + } + } + + #endregion Search + + /// + public override async IAsyncEnumerable GetAsync(Expression> filter, int top, FilteredRecordRetrievalOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + Throw.IfNull(filter); + Throw.IfLessThan(top, 1); + + if (options?.OrderBy is not null) + { + throw new NotSupportedException("Pinecone does not support ordering."); + } + + options ??= new(); + + if (options.IncludeVectors && _model.EmbeddingGenerationRequired) + { + throw new NotSupportedException(VectorDataStrings.IncludeVectorsNotSupportedWithEmbeddingGeneration); + } + + QueryRequest request = new() + { + TopK = (uint)(top + options.Skip), + Namespace = _indexNamespace, + IncludeValues = options.IncludeVectors, + IncludeMetadata = true, + // "Either 'vector' or 'ID' must be provided" + // Since we are doing a query, we don't have a vector to provide, so we fake one. + // When https://github.com/pinecone-io/pinecone-dotnet-client/issues/43 gets implemented, we need to switch. + Vector = new ReadOnlyMemory(new float[_model.VectorProperty.Dimensions]), + Filter = new PineconeFilterTranslator().Translate(filter, _model), + }; + + QueryResponse response = await RunIndexOperationAsync( + "Get", + indexClient => indexClient.QueryAsync(request, cancellationToken: cancellationToken)).ConfigureAwait(false); + + if (response.Matches is null) + { + yield break; + } + + var records = response + .Matches + .Skip(options.Skip) + .Select( + x => _mapper.MapFromStorageToDataModel( + new Vector() + { + Id = x.Id, + Values = x.Values ?? Array.Empty(), + Metadata = x.Metadata, + SparseValues = x.SparseValues + }, + options.IncludeVectors)); + + foreach (var record in records) + { + yield return record; + } + } + + /// + public override object? GetService(Type serviceType, object? serviceKey = null) + { + Throw.IfNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreCollectionMetadata) ? _collectionMetadata : + serviceType == typeof(PineconeClient) ? _pineconeClient : + serviceType.IsInstanceOfType(this) ? this : + null; + } + + private Task RunIndexOperationAsync(string operationName, Func> operation) + => VectorStoreErrorHandler.RunOperationAsync( + _collectionMetadata, + operationName, + async () => + { + if (_indexClient is null) + { + // If we don't provide "host" to the Index method, it's going to perform + // a blocking call to DescribeIndexAsync!! + string hostName = (await _pineconeClient.DescribeIndexAsync(Name).ConfigureAwait(false)).Host; + _indexClient = _pineconeClient.Index(host: hostName); + } + + return await operation.Invoke(_indexClient).ConfigureAwait(false); + }); + + private Task RunCollectionOperationAsync(string operationName, Func> operation) + => VectorStoreErrorHandler.RunOperationAsync( + _collectionMetadata, + operationName, + operation); + + private static ServerlessSpecCloud MapCloud(string serverlessIndexCloud) + => serverlessIndexCloud switch + { + "aws" => ServerlessSpecCloud.Aws, + "azure" => ServerlessSpecCloud.Azure, + "gcp" => ServerlessSpecCloud.Gcp, + _ => throw new ArgumentException($"Invalid serverless index cloud: {serverlessIndexCloud}.", nameof(serverlessIndexCloud)) + }; + + private static CreateIndexRequestMetric MapDistanceFunction(VectorPropertyModel vectorProperty) + => vectorProperty.DistanceFunction switch + { + DistanceFunction.CosineSimilarity or null => CreateIndexRequestMetric.Cosine, + DistanceFunction.DotProductSimilarity => CreateIndexRequestMetric.Dotproduct, + DistanceFunction.EuclideanSquaredDistance => CreateIndexRequestMetric.Euclidean, + _ => throw new NotSupportedException($"Distance function '{vectorProperty.DistanceFunction}' is not supported.") + }; + + private static void VerifyCollectionName(string collectionName) + { + Throw.IfNullOrWhitespace(collectionName); + + // Based on https://docs.pinecone.io/troubleshooting/restrictions-on-index-names + foreach (char character in collectionName) + { + if (character is not (>= 'a' and <= 'z' or '-' or >= '0' and <= '9')) + { + throw new ArgumentException("Collection name must contain only ASCII lowercase letters, digits and dashes.", nameof(collectionName)); + } + } + } + + private string GetStringKey(TKey key) + { + Throw.IfNull(key); + + var stringKey = key switch + { + string s => s, + Guid g => g.ToString(), + + _ => throw new UnreachableException() + }; + + Throw.IfNullOrWhitespace(stringKey); + + return stringKey; + } +} diff --git a/MEVD/src/Pinecone/PineconeCollectionOptions.cs b/MEVD/src/Pinecone/PineconeCollectionOptions.cs new file mode 100644 index 0000000..6a42eca --- /dev/null +++ b/MEVD/src/Pinecone/PineconeCollectionOptions.cs @@ -0,0 +1,49 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.VectorData; + +namespace CommunityToolkit.VectorData.Pinecone; + +/// +/// Options when creating a . +/// +public sealed class PineconeCollectionOptions : VectorStoreCollectionOptions +{ + internal static readonly PineconeCollectionOptions Default = new(); + + /// + /// Initializes a new instance of the class. + /// + public PineconeCollectionOptions() + { + } + + internal PineconeCollectionOptions(PineconeCollectionOptions? source) : base(source) + { + IndexNamespace = source?.IndexNamespace; + ServerlessIndexCloud = source?.ServerlessIndexCloud ?? Default.ServerlessIndexCloud; + ServerlessIndexRegion = source?.ServerlessIndexRegion ?? Default.ServerlessIndexRegion; + } + + /// + /// Gets or sets the value for a namespace within the Pinecone index that will be used for operations involving records (Get, Upsert, Delete)."/> + /// + public string? IndexNamespace { get; set; } + + /// + /// Gets or sets the value for public cloud where the serverless index is hosted. + /// + /// + /// This value is only used when creating a new Pinecone index. Default value is 'aws'. + /// + public string ServerlessIndexCloud { get; set; } = "aws"; + + /// + /// Gets or sets the value for region where the serverless index is created. + /// + /// + /// This option is only used when creating a new Pinecone index. Default value is 'us-east-1'. + /// + public string ServerlessIndexRegion { get; set; } = "us-east-1"; +} diff --git a/MEVD/src/Pinecone/PineconeConstants.cs b/MEVD/src/Pinecone/PineconeConstants.cs new file mode 100644 index 0000000..a692b9f --- /dev/null +++ b/MEVD/src/Pinecone/PineconeConstants.cs @@ -0,0 +1,9 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace CommunityToolkit.VectorData.Pinecone; + +internal static class PineconeConstants +{ + internal const string VectorStoreSystemName = "pinecone"; +} diff --git a/MEVD/src/Pinecone/PineconeDynamicCollection.cs b/MEVD/src/Pinecone/PineconeDynamicCollection.cs new file mode 100644 index 0000000..be0c09e --- /dev/null +++ b/MEVD/src/Pinecone/PineconeDynamicCollection.cs @@ -0,0 +1,32 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Pinecone; + +namespace CommunityToolkit.VectorData.Pinecone; + +/// +/// Represents a collection of vector store records in a Pinecone database, mapped to a dynamic Dictionary<string, object?>. +/// +#pragma warning disable CA1711 // Identifiers should not have incorrect suffix +public sealed class PineconeDynamicCollection : PineconeCollection> +#pragma warning restore CA1711 // Identifiers should not have incorrect suffix +{ + /// + /// Initializes a new instance of the class. + /// + /// Pinecone client that can be used to manage the collections and vectors in a Pinecone store. + /// The name of the collection. + /// Optional configuration options for this class. + public PineconeDynamicCollection(PineconeClient pineconeClient, string name, PineconeCollectionOptions options) + : base( + pineconeClient, + name, + static options => new PineconeModelBuilder() + .BuildDynamic( + options.Definition ?? throw new ArgumentException("Definition is required for dynamic collections"), + options.EmbeddingGenerator), + options) + { + } +} diff --git a/MEVD/src/Pinecone/PineconeFieldMapping.cs b/MEVD/src/Pinecone/PineconeFieldMapping.cs new file mode 100644 index 0000000..deb2bca --- /dev/null +++ b/MEVD/src/Pinecone/PineconeFieldMapping.cs @@ -0,0 +1,76 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using Pinecone; + +namespace CommunityToolkit.VectorData.Pinecone; + +/// +/// Contains helper methods for mapping fields to and from the format required by the Pinecone client sdk. +/// +internal static class PineconeFieldMapping +{ + public static object? ConvertFromMetadataValueToNativeType(MetadataValue metadataValue, Type targetType) + => metadataValue.Value switch + { + null => null, + bool v => v, + string v => v, + + // Numeric values are not always coming from the SDK in the desired type + // that the data model requires, so we need to convert them. + int v => ConvertToNumericValue(v, targetType), + long v => ConvertToNumericValue(v, targetType), + float v => ConvertToNumericValue(v, targetType), + double v => ConvertToNumericValue(v, targetType), + + IEnumerable enumerable => DeserializeCollection(enumerable, targetType), + + _ => throw new InvalidOperationException($"Unsupported metadata type: '{metadataValue.Value?.GetType().FullName}'."), + }; + + private static object? DeserializeCollection(IEnumerable collection, Type targetType) + => targetType switch + { + Type t when t == typeof(List) + => collection.Select(v => (string)v.Value).ToList(), + Type t when t == typeof(string[]) + => collection.Select(v => (string)v.Value).ToArray(), + + _ => throw new UnreachableException($"Unsupported collection type {targetType.Name}"), + }; + + public static MetadataValue ConvertToMetadataValue(object? sourceValue) + => sourceValue switch + { + bool boolValue => boolValue, + bool[] bools => bools, + List bools => bools, + string stringValue => stringValue, + string[] stringArray => stringArray, + List stringList => stringList, + double doubleValue => doubleValue, + double[] doubles => doubles, + List doubles => doubles, + // Other numeric types are simply cast into double in implicit way. + // We could consider supporting arrays of these types. + int intValue => intValue, + long longValue => longValue, + float floatValue => floatValue, + _ => throw new InvalidOperationException($"Unsupported source value type '{sourceValue?.GetType().FullName}'.") + }; + + private static object? ConvertToNumericValue(object? number, Type targetType) + => number is null + ? null + : (Nullable.GetUnderlyingType(targetType) ?? targetType) switch + { + Type t when t == typeof(int) => (object)Convert.ToInt32(number), + Type t when t == typeof(long) => Convert.ToInt64(number), + Type t when t == typeof(float) => Convert.ToSingle(number), + Type t when t == typeof(double) => Convert.ToDouble(number), + + _ => throw new InvalidOperationException($"Unsupported target numeric type '{targetType.FullName}'."), + }; +} diff --git a/MEVD/src/Pinecone/PineconeFilterTranslator.cs b/MEVD/src/Pinecone/PineconeFilterTranslator.cs new file mode 100644 index 0000000..493dabb --- /dev/null +++ b/MEVD/src/Pinecone/PineconeFilterTranslator.cs @@ -0,0 +1,222 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections; +using System.Diagnostics; +using System.Linq.Expressions; +using Microsoft.Extensions.VectorData.ProviderServices; +using Microsoft.Extensions.VectorData.ProviderServices.Filter; +using Pinecone; +using VectorDataCollectionModel = Microsoft.Extensions.VectorData.ProviderServices.CollectionModel; + +namespace CommunityToolkit.VectorData.Pinecone; + +// This class is a modification of MongoDBFilterTranslator that uses the same query language +// (https://docs.pinecone.io/guides/data/understanding-metadata#metadata-query-language), +// with the difference of representing everything as Metadata rather than BsonDocument. +// For representing collections of any kinds, we use List, +// as we sometimes need to extend the collection (with for example another condition). +internal class PineconeFilterTranslator : FilterTranslatorBase +{ + internal Metadata? Translate(LambdaExpression lambdaExpression, VectorDataCollectionModel model) + { + // Pinecone doesn't seem to have a native way of expressing "always true" filters; since this scenario is important for fetching + // all records (via GetAsync with filter), we special-case and support it here. Note that false isn't supported (useless), + // nor is 'x && true'. + if (lambdaExpression.Body is ConstantExpression { Value: true }) + { + return null; + } + + var preprocessedExpression = this.PreprocessFilter(lambdaExpression, model, new FilterPreprocessingOptions()); + + return Translate(preprocessedExpression); + } + + private Metadata Translate(Expression? node) + => node switch + { + BinaryExpression + { + NodeType: ExpressionType.Equal or ExpressionType.NotEqual + or ExpressionType.GreaterThan or ExpressionType.GreaterThanOrEqual + or ExpressionType.LessThan or ExpressionType.LessThanOrEqual + } binary + => TranslateEqualityComparison(binary), + + BinaryExpression { NodeType: ExpressionType.AndAlso or ExpressionType.OrElse } andOr + => TranslateAndOr(andOr), + UnaryExpression { NodeType: ExpressionType.Not } not + => TranslateNot(not), + // Handle converting non-nullable to nullable; such nodes are found in e.g. r => r.Int == nullableInt + UnaryExpression { NodeType: ExpressionType.Convert } convert when Nullable.GetUnderlyingType(convert.Type) == convert.Operand.Type + => Translate(convert.Operand), + + // Special handling for bool constant as the filter expression (r => r.Bool) + Expression when node.Type == typeof(bool) && TryBindProperty(node, out var property) + => GenerateEqualityComparison(property, true, ExpressionType.Equal), + + MethodCallExpression methodCall => TranslateMethodCall(methodCall), + + _ => throw new NotSupportedException("The following NodeType is unsupported: " + node?.NodeType) + }; + + private Metadata TranslateEqualityComparison(BinaryExpression binary) + => TryBindProperty(binary.Left, out var property) && binary.Right is ConstantExpression { Value: var rightConstant } + ? GenerateEqualityComparison(property, rightConstant, binary.NodeType) + : TryBindProperty(binary.Right, out property) && binary.Left is ConstantExpression { Value: var leftConstant } + ? GenerateEqualityComparison(property, leftConstant, binary.NodeType) + : throw new NotSupportedException("Invalid equality/comparison"); + + private Metadata GenerateEqualityComparison(PropertyModel property, object? value, ExpressionType nodeType) + { + if (value is null) + { + throw new NotSupportedException("Pincone does not support null checks in vector search pre-filters"); + } + + // Short form of equality (instead of $eq) + if (nodeType is ExpressionType.Equal) + { + return new Metadata { [property.StorageName] = ToMetadata(value) }; + } + + var filterOperator = nodeType switch + { + ExpressionType.NotEqual => "$ne", + ExpressionType.GreaterThan => "$gt", + ExpressionType.GreaterThanOrEqual => "$gte", + ExpressionType.LessThan => "$lt", + ExpressionType.LessThanOrEqual => "$lte", + + _ => throw new UnreachableException() + }; + + return new Metadata { [property.StorageName] = new Metadata { [filterOperator] = ToMetadata(value) } }; + } + + private Metadata TranslateAndOr(BinaryExpression andOr) + { + var mongoOperator = andOr.NodeType switch + { + ExpressionType.AndAlso => "$and", + ExpressionType.OrElse => "$or", + _ => throw new UnreachableException() + }; + + var (left, right) = (Translate(andOr.Left), Translate(andOr.Right)); + + List? nestedLeft = GetListOrNull(left, mongoOperator); + List? nestedRight = GetListOrNull(right, mongoOperator); + + switch ((nestedLeft, nestedRight)) + { + case (not null, not null): + nestedLeft.AddRange(nestedRight); + return left; + case (not null, null): + nestedLeft.Add(right); + return left; + case (null, not null): + nestedRight.Insert(0, left); + return right; + case (null, null): + return new Metadata { [mongoOperator] = new MetadataValue(new List { left, right }) }; + } + } + + private Metadata TranslateNot(UnaryExpression not) + { + switch (not.Operand) + { + // Special handling for !(a == b) and !(a != b) + case BinaryExpression { NodeType: ExpressionType.Equal or ExpressionType.NotEqual } binary: + return TranslateEqualityComparison( + Expression.MakeBinary( + binary.NodeType is ExpressionType.Equal ? ExpressionType.NotEqual : ExpressionType.Equal, + binary.Left, + binary.Right)); + + // Not over bool field (Filter => r => !r.Bool) + case Expression when not.Operand.Type == typeof(bool) && TryBindProperty(not.Operand, out var property): + return GenerateEqualityComparison(property, false, ExpressionType.Equal); + } + + var operand = Translate(not.Operand); + + // Identify NOT over $in, transform to $nin (https://www.mongodb.com/docs/manual/reference/operator/query/nin/#mongodb-query-op.-nin) + if (operand.Count == 1 && operand.First() is { Key: var fieldName, Value: MetadataValue nested } && nested.Value is Metadata nestedMetadata + && GetListOrNull(nestedMetadata, "$in") is List values) + { + return new Metadata { [fieldName] = new Metadata { ["$nin"] = values } }; + } + + throw new NotSupportedException("Pinecone does not support the NOT operator in vector search pre-filters"); + } + + private Metadata TranslateMethodCall(MethodCallExpression methodCall) + { + return methodCall switch + { + // Enumerable.Contains(), List.Contains(), MemoryExtensions.Contains() + _ when TryMatchContains(methodCall, out var source, out var item) + => TranslateContains(source, item), + + _ => throw new NotSupportedException($"Unsupported method call: {methodCall.Method.DeclaringType?.Name}.{methodCall.Method.Name}") + }; + } + + private Metadata TranslateContains(Expression source, Expression item) + { + switch (source) + { + // Contains over array column (r => r.Strings.Contains("foo")) + case var _ when TryBindProperty(source, out _): + throw new NotSupportedException("Pinecone does not support Contains within array fields ($elemMatch) in vector search pre-filters"); + + // Contains over inline enumerable + case NewArrayExpression newArray: + var elements = new object?[newArray.Expressions.Count]; + + for (var i = 0; i < newArray.Expressions.Count; i++) + { + if (newArray.Expressions[i] is not ConstantExpression { Value: var elementValue }) + { + throw new NotSupportedException("Invalid element in array"); + } + + elements[i] = elementValue; + } + + return ProcessInlineEnumerable(elements, item); + + case ConstantExpression { Value: IEnumerable enumerable and not string }: + return ProcessInlineEnumerable(enumerable, item); + + default: + throw new NotSupportedException("Unsupported Contains expression"); + } + + Metadata ProcessInlineEnumerable(IEnumerable elements, Expression item) + { + if (!TryBindProperty(item, out var property)) + { + throw new NotSupportedException("Unsupported item type in Contains"); + } + + return new Metadata + { + [property.StorageName] = new Metadata + { + ["$in"] = new MetadataValue(elements.Cast().Select(ToMetadata).ToList()) + } + }; + } + } + + private static MetadataValue? ToMetadata(object? value) + => value is null ? null : PineconeFieldMapping.ConvertToMetadataValue(value); + + private static List? GetListOrNull(Metadata value, string mongoOperator) + => value.Count == 1 && value.First() is var element && element.Key == mongoOperator ? element.Value?.Value as List : null; +} diff --git a/MEVD/src/Pinecone/PineconeMapper.cs b/MEVD/src/Pinecone/PineconeMapper.cs new file mode 100644 index 0000000..ec62fb6 --- /dev/null +++ b/MEVD/src/Pinecone/PineconeMapper.cs @@ -0,0 +1,108 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData.ProviderServices; +using Pinecone; +using VectorDataCollectionModel = Microsoft.Extensions.VectorData.ProviderServices.CollectionModel; + +namespace CommunityToolkit.VectorData.Pinecone; + +/// +/// Mapper between a Pinecone record and the consumer data model that uses json as an intermediary to allow supporting a wide range of models. +/// +/// The consumer data model to map to or from. +internal sealed class PineconeMapper(VectorDataCollectionModel model) +{ + /// + public Vector MapFromDataToStorageModel(TRecord dataModel, Embedding? generatedEmbedding) + { + var keyObject = model.KeyProperty.GetValueAsObject(dataModel!); + if (keyObject is null) + { + throw new InvalidOperationException($"Key property '{model.KeyProperty.ModelName}' on provided record of type '{typeof(TRecord).Name}' may not be null."); + } + + var metadata = new Metadata(); + foreach (var property in model.DataProperties) + { + if (property.GetValueAsObject(dataModel!) is { } value) + { + metadata[property.StorageName] = PineconeFieldMapping.ConvertToMetadataValue(value); + } + } + + var values = (generatedEmbedding ?? model.VectorProperty!.GetValueAsObject(dataModel!)) switch + { + ReadOnlyMemory m => m, + Embedding e => e.Vector, + float[] a => a, + + null => throw new InvalidOperationException($"Vector property '{model.VectorProperty.ModelName}' on provided record of type '{typeof(TRecord).Name}' may not be null."), + _ => throw new InvalidOperationException($"Unsupported vector type '{model.VectorProperty.Type.Name}' for vector property '{model.VectorProperty.ModelName}' on provided record of type '{typeof(TRecord).Name}'.") + }; + + // TODO: what about sparse values? + var result = new Vector + { + Id = keyObject switch + { + string s => s, + Guid g => g.ToString(), + _ => throw new UnreachableException() + }, + Values = values, + Metadata = metadata, + SparseValues = null + }; + + return result; + } + + /// + public TRecord MapFromStorageToDataModel(Vector storageModel, bool includeVectors) + { + var outputRecord = model.CreateRecord()!; + + model.KeyProperty.SetValueAsObject(outputRecord, model.KeyProperty.Type switch + { + var t when t == typeof(string) => storageModel.Id, + var t when t == typeof(Guid) => Guid.Parse(storageModel.Id), + + _ => throw new UnreachableException() + }); + + if (includeVectors is true) + { + var vectorProperty = model.VectorProperty; + + vectorProperty.SetValueAsObject( + outputRecord, + storageModel.Values is null + ? null + : (Nullable.GetUnderlyingType(vectorProperty.Type) ?? vectorProperty.Type) switch + { + var t when t == typeof(ReadOnlyMemory) => storageModel.Values, + var t when t == typeof(Embedding) => new Embedding(storageModel.Values.Value), + var t when t == typeof(float[]) => storageModel.Values.Value.ToArray(), + + _ => throw new UnreachableException() + }); + } + + if (storageModel.Metadata != null) + { + foreach (var property in model.DataProperties) + { + property.SetValueAsObject( + outputRecord, + storageModel.Metadata.TryGetValue(property.StorageName, out var metadataValue) && metadataValue is not null + ? PineconeFieldMapping.ConvertFromMetadataValueToNativeType(metadataValue, property.Type) + : null); + } + } + + return outputRecord; + } +} diff --git a/MEVD/src/Pinecone/PineconeModelBuilder.cs b/MEVD/src/Pinecone/PineconeModelBuilder.cs new file mode 100644 index 0000000..b0f8f78 --- /dev/null +++ b/MEVD/src/Pinecone/PineconeModelBuilder.cs @@ -0,0 +1,64 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData.ProviderServices; + +namespace CommunityToolkit.VectorData.Pinecone; + +internal class PineconeModelBuilder() : CollectionModelBuilder(s_validationOptions) +{ + internal const string SupportedVectorTypes = "ReadOnlyMemory, Embedding, float[]"; + + private static readonly CollectionModelBuildingOptions s_validationOptions = new() + { + RequiresAtLeastOneVector = true, + SupportsMultipleVectors = false, + }; + + protected override void ValidateKeyProperty(KeyPropertyModel keyProperty) + { + base.ValidateKeyProperty(keyProperty); + + var type = keyProperty.Type; + + if (type != typeof(string) && type != typeof(Guid)) + { + throw new NotSupportedException( + $"Property '{keyProperty.ModelName}' has unsupported type '{type.Name}'. Key properties must be one of the supported types: string, Guid."); + } + } + + protected override bool IsDataPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes) + { + supportedTypes = "bool, string, int, long, float, double, string[]/List"; + + if (Nullable.GetUnderlyingType(type) is Type underlyingType) + { + type = underlyingType; + } + + return type == typeof(bool) + || type == typeof(string) + || type == typeof(int) + || type == typeof(long) + || type == typeof(float) + || type == typeof(double) + || type == typeof(string[]) + || type == typeof(List); + } + + protected override bool IsVectorPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes) + => IsVectorPropertyTypeValidCore(type, out supportedTypes); + + internal static bool IsVectorPropertyTypeValidCore(Type type, [NotNullWhen(false)] out string? supportedTypes) + { + supportedTypes = SupportedVectorTypes; + + return type == typeof(ReadOnlyMemory) + || type == typeof(ReadOnlyMemory?) + || type == typeof(Embedding) + || type == typeof(float[]); + } +} diff --git a/MEVD/src/Pinecone/PineconeServiceCollectionExtensions.cs b/MEVD/src/Pinecone/PineconeServiceCollectionExtensions.cs new file mode 100644 index 0000000..5837a30 --- /dev/null +++ b/MEVD/src/Pinecone/PineconeServiceCollectionExtensions.cs @@ -0,0 +1,255 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData; +using CommunityToolkit.VectorData.Pinecone; +using Pinecone; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.DependencyInjection; + +/// +/// Extension methods to register and instances on an . +/// +public static class PineconeServiceCollectionExtensions +{ + private const string DynamicCodeMessage = "This method is incompatible with NativeAOT, consult the documentation for adding collections in a way that's compatible with NativeAOT."; + private const string UnreferencedCodeMessage = "This method is incompatible with trimming, consult the documentation for adding collections in a way that's compatible with NativeAOT."; + + /// + /// Registers a as + /// with returned by + /// or retrieved from the dependency injection container if was not provided. + /// + /// + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddPineconeVectorStore( + this IServiceCollection services, + Func? clientProvider = default, + Func? optionsProvider = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + => AddKeyedPineconeVectorStore(services, serviceKey: null, clientProvider, optionsProvider, lifetime); + + /// + /// Registers a keyed as + /// with returned by + /// or retrieved from the dependency injection container if was not provided. + /// + /// The to register the on. + /// The key with which to associate the vector store. + /// The provider. + /// Options provider to further configure the . + /// The service lifetime for the store. Defaults to . + /// Service collection. + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddKeyedPineconeVectorStore( + this IServiceCollection services, + object? serviceKey, + Func? clientProvider = default, + Func? optionsProvider = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + { + Throw.IfNull(services); + + services.Add(new ServiceDescriptor(typeof(PineconeVectorStore), serviceKey, (sp, _) => + { + var database = clientProvider is not null ? clientProvider(sp) : sp.GetRequiredService(); + var options = GetStoreOptions(sp, optionsProvider); + + return new PineconeVectorStore(database, options); + }, lifetime)); + + services.Add(new ServiceDescriptor(typeof(VectorStore), serviceKey, + static (sp, key) => sp.GetRequiredKeyedService(key), lifetime)); + + return services; + } + + /// + /// Registers a as + /// using the provided and . + /// + /// + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddPineconeVectorStore( + this IServiceCollection services, + string apiKey, + ClientOptions? clientOptions = default, + PineconeVectorStoreOptions? options = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + => AddKeyedPineconeVectorStore(services, serviceKey: null, apiKey, clientOptions, options, lifetime); + + /// + /// Registers a keyed as + /// using the provided and . + /// + /// The to register the on. + /// The key with which to associate the vector store. + /// API Key required to connect to PineconeDB. + /// The to configure . + /// Optional options to further configure the . + /// The service lifetime for the store. Defaults to . + /// Service collection. + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddKeyedPineconeVectorStore( + this IServiceCollection services, + object? serviceKey, + string apiKey, + ClientOptions? clientOptions = default, + PineconeVectorStoreOptions? options = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + { + Throw.IfNull(services); + Throw.IfNullOrWhitespace(apiKey); + + return AddKeyedPineconeVectorStore(services, serviceKey, _ => new PineconeClient(apiKey, clientOptions), _ => options!, lifetime); + } + + /// + /// Registers a as + /// with retrieved from the dependency injection container. + /// + /// + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddPineconeCollection( + this IServiceCollection services, + string name, + Func? clientProvider = default, + Func? optionsProvider = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + where TRecord : class + => AddKeyedPineconeCollection(services, serviceKey: null, name, clientProvider, optionsProvider, lifetime); + + /// + /// Registers a keyed as + /// with returned by + /// or retrieved from the dependency injection container if was not provided. + /// + /// The type of the record. + /// The to register the on. + /// The key with which to associate the collection. + /// The name of the collection. + /// The provider. + /// Options provider to further configure the . + /// The service lifetime for the store. Defaults to . + /// Service collection. + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddKeyedPineconeCollection( + this IServiceCollection services, + object? serviceKey, + string name, + Func? clientProvider = default, + Func? optionsProvider = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + where TRecord : class + { + Throw.IfNull(services); + Throw.IfNullOrWhitespace(name); + + services.Add(new ServiceDescriptor(typeof(PineconeCollection), serviceKey, (sp, _) => + { + var client = clientProvider is not null ? clientProvider(sp) : sp.GetRequiredService(); + var options = GetCollectionOptions(sp, optionsProvider); + + return new PineconeCollection(client, name, options); + }, lifetime)); + + AddAbstractions(services, serviceKey, lifetime); + + return services; + } + + /// + /// Registers a as + /// using the provided and . + /// + /// + [RequiresUnreferencedCode(UnreferencedCodeMessage)] + [RequiresDynamicCode(DynamicCodeMessage)] + public static IServiceCollection AddPineconeCollection( + this IServiceCollection services, + string name, + string apiKey, + ClientOptions? clientOptions = default, + PineconeCollectionOptions? options = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + where TRecord : class + => AddKeyedPineconeCollection(services, serviceKey: null, name, apiKey, clientOptions, options, lifetime); + + /// + /// Registers a keyed as + /// using the provided and . + /// + /// The type of the record. + /// The to register the on. + /// The key with which to associate the collection. + /// The name of the collection. + /// API Key required to connect to PineconeDB. + /// The to configure . + /// Optional options to further configure the . + /// The service lifetime for the store. Defaults to . + /// Service collection. + [RequiresUnreferencedCode(UnreferencedCodeMessage)] + [RequiresDynamicCode(DynamicCodeMessage)] + public static IServiceCollection AddKeyedPineconeCollection( + this IServiceCollection services, + object? serviceKey, + string name, + string apiKey, + ClientOptions? clientOptions = default, + PineconeCollectionOptions? options = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + where TRecord : class + { + Throw.IfNullOrWhitespace(apiKey); + + return AddKeyedPineconeCollection(services, serviceKey, name, _ => new PineconeClient(apiKey, clientOptions), _ => options!, lifetime); + } + + private static void AddAbstractions(IServiceCollection services, object? serviceKey, ServiceLifetime lifetime) + where TKey : notnull + where TRecord : class + { + services.Add(new ServiceDescriptor(typeof(VectorStoreCollection), serviceKey, + static (sp, key) => sp.GetRequiredKeyedService>(key), lifetime)); + + services.Add(new ServiceDescriptor(typeof(IVectorSearchable), serviceKey, + static (sp, key) => sp.GetRequiredKeyedService>(key), lifetime)); + } + + private static PineconeVectorStoreOptions? GetStoreOptions(IServiceProvider sp, Func? optionsProvider) + { + var options = optionsProvider?.Invoke(sp); + if (options?.EmbeddingGenerator is not null) + { + return options; // The user has provided everything, there is nothing to change. + } + + var embeddingGenerator = sp.GetService(); + return embeddingGenerator is null + ? options // There is nothing to change. + : new(options) { EmbeddingGenerator = embeddingGenerator }; // Create a brand new copy in order to avoid modifying the original options. + } + + private static PineconeCollectionOptions? GetCollectionOptions(IServiceProvider sp, Func? optionsProvider) + { + var options = optionsProvider?.Invoke(sp); + if (options?.EmbeddingGenerator is not null) + { + return options; // The user has provided everything, there is nothing to change. + } + + var embeddingGenerator = sp.GetService(); + return embeddingGenerator is null + ? options // There is nothing to change. + : new(options) { EmbeddingGenerator = embeddingGenerator }; // Create a brand new copy in order to avoid modifying the original options. + } +} diff --git a/MEVD/src/Pinecone/PineconeVectorStore.cs b/MEVD/src/Pinecone/PineconeVectorStore.cs new file mode 100644 index 0000000..0b1cf4d --- /dev/null +++ b/MEVD/src/Pinecone/PineconeVectorStore.cs @@ -0,0 +1,130 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; +using Pinecone; +using Microsoft.Shared.Diagnostics; + +namespace CommunityToolkit.VectorData.Pinecone; + +/// +/// Class for accessing the list of collections in a Pinecone vector store. +/// +/// +/// This class can be used with collections of any schema type, but requires you to provide schema information when getting a collection. +/// +public sealed class PineconeVectorStore : VectorStore +{ + private readonly PineconeClient _pineconeClient; + + /// Metadata about vector store. + private readonly VectorStoreMetadata _metadata; + + /// A general purpose definition that can be used to construct a collection when needing to proxy schema agnostic operations. + private static readonly VectorStoreCollectionDefinition s_generalPurposeDefinition = new() { Properties = [new VectorStoreKeyProperty("Key", typeof(string)), new VectorStoreVectorProperty("Vector", typeof(ReadOnlyMemory), 1)] }; + + private readonly IEmbeddingGenerator? _embeddingGenerator; + + /// + /// Initializes a new instance of the class. + /// + /// Pinecone client that can be used to manage the collections and points in a Pinecone store. + /// Optional configuration options for this class. + public PineconeVectorStore(PineconeClient pineconeClient, PineconeVectorStoreOptions? options = default) + { + Throw.IfNull(pineconeClient); + + _pineconeClient = pineconeClient; + _embeddingGenerator = options?.EmbeddingGenerator; + + _metadata = new() + { + VectorStoreSystemName = PineconeConstants.VectorStoreSystemName + }; + } + +#pragma warning disable IDE0090 // Use 'new(...)' + /// + [RequiresDynamicCode("This overload of GetCollection() is incompatible with NativeAOT. For dynamic mapping via Dictionary, call GetDynamicCollection() instead.")] + [RequiresUnreferencedCode("This overload of GetCollecttion() is incompatible with trimming. For dynamic mapping via Dictionary, call GetDynamicCollection() instead.")] +#if NET + public override PineconeCollection GetCollection(string name, VectorStoreCollectionDefinition? definition = null) +#else + public override VectorStoreCollection GetCollection(string name, VectorStoreCollectionDefinition? definition = null) +#endif + => typeof(TRecord) == typeof(Dictionary) + ? throw new ArgumentException(VectorDataStrings.GetCollectionWithDictionaryNotSupported) + : new PineconeCollection( + _pineconeClient, + name, + new PineconeCollectionOptions() + { + Definition = definition, + EmbeddingGenerator = _embeddingGenerator + }); + + /// +#if NET + public override PineconeDynamicCollection GetDynamicCollection(string name, VectorStoreCollectionDefinition definition) +#else + public override VectorStoreCollection> GetDynamicCollection(string name, VectorStoreCollectionDefinition definition) +#endif + => new PineconeDynamicCollection( + _pineconeClient, + name, + new() + { + Definition = definition, + EmbeddingGenerator = _embeddingGenerator + } + ); +#pragma warning restore IDE0090 + + /// + public override async IAsyncEnumerable ListCollectionNamesAsync([EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var indexList = await VectorStoreErrorHandler.RunOperationAsync( + _metadata, + "ListCollections", + () => _pineconeClient.ListIndexesAsync(cancellationToken: cancellationToken)).ConfigureAwait(false); + + if (indexList.Indexes is not null) + { + foreach (var index in indexList.Indexes) + { + yield return index.Name; + } + } + } + + /// + public override Task CollectionExistsAsync(string name, CancellationToken cancellationToken = default) + { + var collection = GetDynamicCollection(name, s_generalPurposeDefinition); + return collection.CollectionExistsAsync(cancellationToken); + } + + /// + public override Task EnsureCollectionDeletedAsync(string name, CancellationToken cancellationToken = default) + { + var collection = GetDynamicCollection(name, s_generalPurposeDefinition); + return collection.EnsureCollectionDeletedAsync(cancellationToken); + } + + /// + public override object? GetService(Type serviceType, object? serviceKey = null) + { + Throw.IfNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreMetadata) ? _metadata : + serviceType == typeof(PineconeClient) ? _pineconeClient : + serviceType.IsInstanceOfType(this) ? this : + null; + } +} diff --git a/MEVD/src/Pinecone/PineconeVectorStoreOptions.cs b/MEVD/src/Pinecone/PineconeVectorStoreOptions.cs new file mode 100644 index 0000000..56177b6 --- /dev/null +++ b/MEVD/src/Pinecone/PineconeVectorStoreOptions.cs @@ -0,0 +1,29 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.AI; + +namespace CommunityToolkit.VectorData.Pinecone; + +/// +/// Options when creating a . +/// +public sealed class PineconeVectorStoreOptions +{ + /// + /// Gets or sets the default embedding generator for vector properties in this collection. + /// + public IEmbeddingGenerator? EmbeddingGenerator { get; set; } + + /// + /// Initializes a new instance of the class. + /// + public PineconeVectorStoreOptions() + { + } + + internal PineconeVectorStoreOptions(PineconeVectorStoreOptions? source) + { + EmbeddingGenerator = source?.EmbeddingGenerator; + } +} diff --git a/MEVD/src/Qdrant/AssemblyInfo.cs b/MEVD/src/Qdrant/AssemblyInfo.cs new file mode 100644 index 0000000..d6395e8 --- /dev/null +++ b/MEVD/src/Qdrant/AssemblyInfo.cs @@ -0,0 +1,2 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. diff --git a/MEVD/src/Qdrant/MockableQdrantClient.cs b/MEVD/src/Qdrant/MockableQdrantClient.cs new file mode 100644 index 0000000..ac6c243 --- /dev/null +++ b/MEVD/src/Qdrant/MockableQdrantClient.cs @@ -0,0 +1,358 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Qdrant.Client; +using Qdrant.Client.Grpc; +using Microsoft.Shared.Diagnostics; + +namespace CommunityToolkit.VectorData.Qdrant; + +/// +/// Decorator class for that exposes the required methods as virtual allowing for mocking in unit tests. +/// +internal class MockableQdrantClient : IDisposable +{ + /// Qdrant client that can be used to manage the collections and points in a Qdrant store. + private readonly QdrantClient _qdrantClient; + private readonly bool _ownsClient; + private int _referenceCount = 1; + + /// + /// Initializes a new instance of the class. + /// + /// Qdrant client that can be used to manage the collections and points in a Qdrant store. + /// A value indicating whether is disposed when the vector store is disposed. + public MockableQdrantClient(QdrantClient qdrantClient, bool ownsClient = true) + { + Throw.IfNull(qdrantClient); + + _qdrantClient = qdrantClient; + _ownsClient = ownsClient; + } + +#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable. + + /// + /// Constructor for mocking purposes only. + /// + internal MockableQdrantClient() + { + } + +#pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable. + + /// + /// Gets the internal that this mockable instance wraps. + /// + public QdrantClient QdrantClient => _qdrantClient; + + public void Dispose() + { + if (_ownsClient) + { + if (Interlocked.Decrement(ref _referenceCount) == 0) + { + _qdrantClient.Dispose(); + } + } + } + + /// + /// Check if a collection exists. + /// + /// The name of the collection. + /// + /// The token to monitor for cancellation requests. The default value is . + /// + public virtual Task CollectionExistsAsync( + string collectionName, + CancellationToken cancellationToken = default) + => _qdrantClient.CollectionExistsAsync(collectionName, cancellationToken); + + /// + /// Creates a new collection with the given parameters. + /// + /// The name of the collection to be created. + /// + /// Configuration of the vector storage. Vector params contains size and distance for the vector storage. + /// This overload creates a single anonymous vector storage. + /// + /// + /// The token to monitor for cancellation requests. The default value is . + /// + public virtual Task CreateCollectionAsync( + string collectionName, + VectorParams vectorsConfig, + CancellationToken cancellationToken = default) + => _qdrantClient.CreateCollectionAsync( + collectionName, + vectorsConfig, + cancellationToken: cancellationToken); + + /// + /// Creates a new collection with the given parameters. + /// + /// The name of the collection to be created. + /// + /// Configuration of the vector storage. Vector params contains size and distance for the vector storage. + /// This overload creates a vector storage for each key in the provided map. + /// + /// + /// The token to monitor for cancellation requests. The default value is . + /// + public virtual Task CreateCollectionAsync( + string collectionName, + VectorParamsMap? vectorsConfig = null, + CancellationToken cancellationToken = default) + => _qdrantClient.CreateCollectionAsync( + collectionName, + vectorsConfig, + cancellationToken: cancellationToken); + + /// + /// Creates a payload field index in a collection. + /// + /// The name of the collection. + /// Field name to index. + /// The schema type of the field. + /// + /// The token to monitor for cancellation requests. The default value is . + /// + public virtual Task CreatePayloadIndexAsync( + string collectionName, + string fieldName, + PayloadSchemaType schemaType = PayloadSchemaType.Keyword, + CancellationToken cancellationToken = default) + => _qdrantClient.CreatePayloadIndexAsync(collectionName, fieldName, schemaType, cancellationToken: cancellationToken); + + /// + /// Drop a collection and all its associated data. + /// + /// The name of the collection. + /// Wait timeout for operation commit in seconds, if not specified - default value will be supplied + /// + /// The token to monitor for cancellation requests. The default value is . + /// + public virtual Task DeleteCollectionAsync( + string collectionName, + TimeSpan? timeout = null, + CancellationToken cancellationToken = default) + => _qdrantClient.DeleteCollectionAsync(collectionName, timeout, cancellationToken); + + /// + /// Gets the names of all existing collections. + /// + /// + /// The token to monitor for cancellation requests. The default value is . + /// + public virtual Task> ListCollectionsAsync(CancellationToken cancellationToken = default) + => _qdrantClient.ListCollectionsAsync(cancellationToken); + + /// + /// Delete a point. + /// + /// The name of the collection. + /// The ID to delete. + /// Whether to wait until the changes have been applied. Defaults to true. + /// Write ordering guarantees. Defaults to Weak. + /// Option for custom sharding to specify used shard keys. + /// + /// The token to monitor for cancellation requests. The default value is . + /// + public virtual Task DeleteAsync( + string collectionName, + ulong id, + bool wait = true, + WriteOrderingType? ordering = null, + ShardKeySelector? shardKeySelector = null, + CancellationToken cancellationToken = default) + => _qdrantClient.DeleteAsync(collectionName, id, wait, ordering, shardKeySelector, cancellationToken: cancellationToken); + + /// + /// Delete a point. + /// + /// The name of the collection. + /// The ID to delete. + /// Whether to wait until the changes have been applied. Defaults to true. + /// Write ordering guarantees. Defaults to Weak. + /// Option for custom sharding to specify used shard keys. + /// + /// The token to monitor for cancellation requests. The default value is . + /// + public virtual Task DeleteAsync( + string collectionName, + Guid id, + bool wait = true, + WriteOrderingType? ordering = null, + ShardKeySelector? shardKeySelector = null, + CancellationToken cancellationToken = default) + => _qdrantClient.DeleteAsync(collectionName, id, wait, ordering, shardKeySelector, cancellationToken: cancellationToken); + + /// + /// Delete a point. + /// + /// The name of the collection. + /// The IDs to delete. + /// Whether to wait until the changes have been applied. Defaults to true. + /// Write ordering guarantees. Defaults to Weak. + /// Option for custom sharding to specify used shard keys. + /// + /// The token to monitor for cancellation requests. The default value is . + /// + public virtual Task DeleteAsync( + string collectionName, + IReadOnlyList ids, + bool wait = true, + WriteOrderingType? ordering = null, + ShardKeySelector? shardKeySelector = null, + CancellationToken cancellationToken = default) + => _qdrantClient.DeleteAsync(collectionName, ids, wait, ordering, shardKeySelector, cancellationToken: cancellationToken); + + /// + /// Delete a point. + /// + /// The name of the collection. + /// The IDs to delete. + /// Whether to wait until the changes have been applied. Defaults to true. + /// Write ordering guarantees. Defaults to Weak. + /// Option for custom sharding to specify used shard keys. + /// + /// The token to monitor for cancellation requests. The default value is . + /// + public virtual Task DeleteAsync( + string collectionName, + IReadOnlyList ids, + bool wait = true, + WriteOrderingType? ordering = null, + ShardKeySelector? shardKeySelector = null, + CancellationToken cancellationToken = default) + => _qdrantClient.DeleteAsync(collectionName, ids, wait, ordering, shardKeySelector, cancellationToken: cancellationToken); + + /// + /// Perform insert and updates on points. If a point with a given ID already exists, it will be overwritten. + /// + /// The name of the collection. + /// The points to be upserted. + /// Whether to wait until the changes have been applied. Defaults to true. + /// Write ordering guarantees. + /// Option for custom sharding to specify used shard keys. + /// + /// The token to monitor for cancellation requests. The default value is . + /// + public virtual Task UpsertAsync( + string collectionName, + IReadOnlyList points, + bool wait = true, + WriteOrderingType? ordering = null, + ShardKeySelector? shardKeySelector = null, + CancellationToken cancellationToken = default) + => _qdrantClient.UpsertAsync(collectionName, points, wait, ordering, shardKeySelector, cancellationToken); + + /// + /// Retrieve points. + /// + /// The name of the collection. + /// List of points to retrieve. + /// Whether to include the payload or not. + /// Whether to include the vectors or not. + /// Options for specifying read consistency guarantees. + /// Option for custom sharding to specify used shard keys. + /// + /// The token to monitor for cancellation requests. The default value is . + /// + public virtual Task> RetrieveAsync( + string collectionName, + IReadOnlyList ids, + bool withPayload = true, + bool withVectors = false, + ReadConsistency? readConsistency = null, + ShardKeySelector? shardKeySelector = null, + CancellationToken cancellationToken = default) + => _qdrantClient.RetrieveAsync(collectionName, ids, withPayload, withVectors, readConsistency, shardKeySelector, cancellationToken); + + /// + /// Universally query points. + /// Covers all capabilities of search, recommend, discover, filters. + /// Also enables hybrid and multi-stage queries. + /// + /// The name of the collection. + /// Query to perform. If missing, returns points ordered by their IDs. + /// Sub-requests to perform first. If present, the query will be performed on the results of the prefetches. + /// Name of the vector to use for querying. If missing, the default vector is used.. + /// Filter conditions - return only those points that satisfy the specified conditions. + /// Return points with scores better than this threshold. + /// Search config. + /// Max number of results. + /// Offset of the result. + /// Options for specifying which payload to include or not. + /// Options for specifying which vectors to include into the response. + /// Options for specifying read consistency guarantees. + /// Specify in which shards to look for the points, if not specified - look in all shards. + /// The location to use for IDs lookup, if not specified - use the current collection and the 'usingVector' vector + /// If set, overrides global timeout setting for this request. + /// The token to monitor for cancellation requests. The default value is . + /// + public virtual Task> QueryAsync( + string collectionName, + Query? query = null, + IReadOnlyList? prefetch = null, + string? usingVector = null, + Filter? filter = null, + float? scoreThreshold = null, + SearchParams? searchParams = null, + ulong limit = 10, + ulong offset = 0, + WithPayloadSelector? payloadSelector = null, + WithVectorsSelector? vectorsSelector = null, + ReadConsistency? readConsistency = null, + ShardKeySelector? shardKeySelector = null, + LookupLocation? lookupFrom = null, + TimeSpan? timeout = null, + CancellationToken cancellationToken = default) + => _qdrantClient.QueryAsync( + collectionName, + query, + prefetch, + usingVector, + filter, + scoreThreshold, + searchParams, + limit, + offset, + payloadSelector, + vectorsSelector, + readConsistency, + shardKeySelector, + lookupFrom, + timeout, + cancellationToken); + + public virtual Task ScrollAsync( + string collectionName, + Filter filter, + WithVectorsSelector vectorsSelector, + uint limit = 10, + OrderBy? orderBy = null, + CancellationToken cancellationToken = default) + => _qdrantClient.ScrollAsync( + collectionName, + filter, + limit, + offset: null, + payloadSelector: null, + vectorsSelector, + readConsistency: null, + shardKeySelector: null, + orderBy, + cancellationToken); + + internal MockableQdrantClient Share() + { + if (_ownsClient) + { + Interlocked.Increment(ref _referenceCount); + } + + return this; + } +} diff --git a/MEVD/src/Qdrant/Qdrant.csproj b/MEVD/src/Qdrant/Qdrant.csproj new file mode 100644 index 0000000..294dae0 --- /dev/null +++ b/MEVD/src/Qdrant/Qdrant.csproj @@ -0,0 +1,30 @@ + + + + 1.0.0-preview.1 + CommunityToolkit.VectorData.Qdrant + $(AssemblyName) + net10.0;net8.0;netstandard2.0;net462 + true + + Qdrant provider for Microsoft.Extensions.VectorData + Qdrant provider for Microsoft.Extensions.VectorData by Semantic Kernel + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/MEVD/src/Qdrant/QdrantCollection.cs b/MEVD/src/Qdrant/QdrantCollection.cs new file mode 100644 index 0000000..f1aa9aa --- /dev/null +++ b/MEVD/src/Qdrant/QdrantCollection.cs @@ -0,0 +1,781 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Linq.Expressions; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using Grpc.Core; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; +using Qdrant.Client; +using Qdrant.Client.Grpc; +using Microsoft.Shared.Diagnostics; + +namespace CommunityToolkit.VectorData.Qdrant; + +/// +/// Service for storing and retrieving vector records, that uses Qdrant as the underlying storage. +/// +/// The data type of the record key. Can be either or . +/// The data model to use for adding, updating and retrieving data from storage. +#pragma warning disable CA1711 // Identifiers should not have incorrect suffix +public class QdrantCollection : VectorStoreCollection, IKeywordHybridSearchable + where TKey : notnull + where TRecord : class +#pragma warning restore CA1711 // Identifiers should not have incorrect suffix +{ + /// Metadata about vector store record collection. + private readonly VectorStoreCollectionMetadata _collectionMetadata; + + /// The default options for vector search. + private static readonly VectorSearchOptions s_defaultVectorSearchOptions = new(); + + /// The default options for hybrid vector search. + private static readonly HybridSearchOptions s_defaultKeywordVectorizedHybridSearchOptions = new(); + + /// The name of the upsert operation for telemetry purposes. + private const string UpsertName = "Upsert"; + + /// The name of the Delete operation for telemetry purposes. + private const string DeleteName = "Delete"; + + /// Qdrant client that can be used to manage the collections and points in a Qdrant store. + private readonly MockableQdrantClient _qdrantClient; + + /// The model for this collection. + private readonly CollectionModel _model; + + /// A mapper to use for converting between qdrant point and consumer models. + private readonly QdrantMapper _mapper; + + /// Whether the vectors in the store are named and multiple vectors are supported, or whether there is just a single unnamed vector per qdrant point. + private readonly bool _hasNamedVectors; + + /// + /// Initializes a new instance of the class. + /// + /// Qdrant client that can be used to manage the collections and points in a Qdrant store. + /// The name of the collection that this will access. + /// A value indicating whether is disposed when the collection is disposed. + /// Optional configuration options for this class. + /// Thrown if the is null. + /// Thrown for any misconfigured options. + [RequiresDynamicCode("This constructor is incompatible with NativeAOT. For dynamic mapping via Dictionary, instantiate QdrantDynamicCollection instead.")] + [RequiresUnreferencedCode("This constructor is incompatible with trimming. For dynamic mapping via Dictionary, instantiate QdrantDynamicCollection instead")] + public QdrantCollection(QdrantClient qdrantClient, string name, bool ownsClient, QdrantCollectionOptions? options = null) + : this(() => new MockableQdrantClient(qdrantClient, ownsClient), name, options) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// Qdrant client factory. + /// The name of the collection that this will access. + /// Optional configuration options for this class. + /// Thrown if the is null. + /// Thrown for any misconfigured options. + [RequiresDynamicCode("This constructor is incompatible with NativeAOT. For dynamic mapping via Dictionary, instantiate QdrantDynamicCollection instead.")] + [RequiresUnreferencedCode("This constructor is incompatible with trimming. For dynamic mapping via Dictionary, instantiate QdrantDynamicCollection instead")] + internal QdrantCollection(Func clientFactory, string name, QdrantCollectionOptions? options = null) + : this( + clientFactory, + name, + static options => typeof(TRecord) == typeof(Dictionary) + ? throw new NotSupportedException(VectorDataStrings.NonDynamicCollectionWithDictionaryNotSupported(typeof(QdrantDynamicCollection))) + : new QdrantModelBuilder(options.HasNamedVectors).Build(typeof(TRecord), typeof(TKey), options.Definition, options.EmbeddingGenerator), + options) + { + } + + internal QdrantCollection(Func clientFactory, string name, Func modelFactory, QdrantCollectionOptions? options) + { + // Verify. + Throw.IfNull(clientFactory); + Throw.IfNullOrWhitespace(name); + + if (typeof(TKey) != typeof(ulong) && typeof(TKey) != typeof(Guid) && typeof(TKey) != typeof(object)) + { + throw new NotSupportedException("Only ulong and Guid keys are supported."); + } + + options ??= QdrantCollectionOptions.Default; + + // Assign. + Name = name; + _model = modelFactory(options); + + _hasNamedVectors = options.HasNamedVectors; + _mapper = new QdrantMapper(_model, options.HasNamedVectors); + + _collectionMetadata = new() + { + VectorStoreSystemName = QdrantConstants.VectorStoreSystemName, + CollectionName = name + }; + + // The code above can throw, so we need to create the client after the model is built and verified. + // In case an exception is thrown, we don't need to dispose any resources. + _qdrantClient = clientFactory(); + } + + /// + protected override void Dispose(bool disposing) + { + _qdrantClient.Dispose(); + base.Dispose(disposing); + } + + /// + public override string Name { get; } + + /// + public override Task CollectionExistsAsync(CancellationToken cancellationToken = default) + { + return RunOperationAsync( + "CollectionExists", + () => _qdrantClient.CollectionExistsAsync(Name, cancellationToken)); + } + + /// + public override async Task EnsureCollectionExistsAsync(CancellationToken cancellationToken = default) + { + // Don't even try to create if the collection already exists. + if (await CollectionExistsAsync(cancellationToken).ConfigureAwait(false)) + { + return; + } + + try + { + if (!_hasNamedVectors) + { + // If we are not using named vectors, we can only have one vector property. We can assume we have exactly one, since this is already verified in the constructor. + var singleVectorProperty = _model.VectorProperty; + + // Map the single vector property to the qdrant config. + var vectorParams = QdrantCollectionCreateMapping.MapSingleVector(singleVectorProperty!); + + // Create the collection with the single unnamed vector. + await _qdrantClient.CreateCollectionAsync( + Name, + vectorParams, + cancellationToken: cancellationToken).ConfigureAwait(false); + } + else + { + // Since we are using named vectors, iterate over all vector properties. + var vectorProperties = _model.VectorProperties; + + // Map the named vectors to the qdrant config. + var vectorParamsMap = QdrantCollectionCreateMapping.MapNamedVectors(vectorProperties); + + // Create the collection with named vectors. + await _qdrantClient.CreateCollectionAsync( + Name, + vectorParamsMap, + cancellationToken: cancellationToken).ConfigureAwait(false); + } + + // Add indexes for each of the data properties that require filtering. + var dataProperties = _model.DataProperties.Where(x => x.IsIndexed); + foreach (var dataProperty in dataProperties) + { + // Note that the schema type doesn't distinguish between array and scalar type (so PayloadSchemaType.Integer is used for both integer and array of integers) + if (QdrantCollectionCreateMapping.s_schemaTypeMap.TryGetValue(dataProperty.Type, out PayloadSchemaType schemaType) + || dataProperty.Type.IsArray + && QdrantCollectionCreateMapping.s_schemaTypeMap.TryGetValue(dataProperty.Type.GetElementType()!, out schemaType) + || dataProperty.Type.IsGenericType + && dataProperty.Type.GetGenericTypeDefinition() == typeof(List<>) + && QdrantCollectionCreateMapping.s_schemaTypeMap.TryGetValue(dataProperty.Type.GenericTypeArguments[0], out schemaType)) + { + await _qdrantClient.CreatePayloadIndexAsync( + Name, + dataProperty.StorageName, + schemaType, + cancellationToken: cancellationToken).ConfigureAwait(false); + } + else + { + // TODO: This should move to model validation + throw new InvalidOperationException($"Property {nameof(VectorStoreDataProperty.IsIndexed)} on {nameof(VectorStoreDataProperty)} '{dataProperty.ModelName}' is set to true, but the property type {dataProperty.Type.Name} is not supported for filtering. The Qdrant VectorStore supports filtering on {string.Join(", ", QdrantCollectionCreateMapping.s_schemaTypeMap.Keys.Select(x => x.Name))} properties only."); + } + } + + // Add indexes for each of the data properties that require full text search. + dataProperties = _model.DataProperties.Where(x => x.IsFullTextIndexed); + foreach (var dataProperty in dataProperties) + { + // TODO: This should move to model validation + if (dataProperty.Type != typeof(string)) + { + throw new InvalidOperationException($"Property {nameof(dataProperty.IsFullTextIndexed)} on {nameof(VectorStoreDataProperty)} '{dataProperty.ModelName}' is set to true, but the property type is not a string. The Qdrant VectorStore supports {nameof(dataProperty.IsFullTextIndexed)} on string properties only."); + } + + await _qdrantClient.CreatePayloadIndexAsync( + Name, + dataProperty.StorageName, + PayloadSchemaType.Text, + cancellationToken: cancellationToken).ConfigureAwait(false); + } + } + catch (RpcException ex) when (ex.StatusCode == StatusCode.AlreadyExists) + { + // Do nothing, since the collection is already created. + } + catch (RpcException ex) + { + throw new VectorStoreException("Call to vector store failed.", ex) + { + VectorStoreSystemName = QdrantConstants.VectorStoreSystemName, + VectorStoreName = _collectionMetadata.VectorStoreName, + CollectionName = Name, + OperationName = "EnsureCollectionExists" + }; + } + } + + /// + public override Task EnsureCollectionDeletedAsync(CancellationToken cancellationToken = default) + => RunOperationAsync("DeleteCollection", + async () => + { + try + { + await _qdrantClient.DeleteCollectionAsync(Name, null, cancellationToken).ConfigureAwait(false); + } + catch (QdrantException) + { + // There is no reliable way to check if the operation failed because the + // collection does not exist based on the exception itself. + // So we just check here if it exists, and if not, ignore the exception. + if (!await CollectionExistsAsync(cancellationToken).ConfigureAwait(false)) + { + return; + } + + throw; + } + }); + + /// + public override async Task GetAsync(TKey key, RecordRetrievalOptions? options = null, CancellationToken cancellationToken = default) + { + Throw.IfNull(key); + + var retrievedPoints = await GetAsync([key], options, cancellationToken).ToListAsync(cancellationToken).ConfigureAwait(false); + return retrievedPoints.FirstOrDefault(); + } + + /// + public override async IAsyncEnumerable GetAsync( + IEnumerable keys, + RecordRetrievalOptions? options = default, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + const string OperationName = "Retrieve"; + + Throw.IfNull(keys); + + // Create options. + var pointsIds = new List(); + + Type? keyType = null; + + foreach (var key in keys) + { + switch (key) + { + case ulong id: + if (keyType == typeof(Guid)) + { + throw new NotSupportedException("Mixing ulong and Guid keys is not supported"); + } + + keyType = typeof(ulong); + pointsIds.Add(new PointId { Num = id }); + break; + + case Guid id: + if (keyType == typeof(ulong)) + { + throw new NotSupportedException("Mixing ulong and Guid keys is not supported"); + } + + pointsIds.Add(new PointId { Uuid = id.ToString("D") }); + keyType = typeof(Guid); + break; + + default: + throw new NotSupportedException($"The provided key type '{key.GetType().Name}' is not supported by Qdrant."); + } + } + + var includeVectors = options?.IncludeVectors ?? false; + if (includeVectors && _model.EmbeddingGenerationRequired) + { + throw new NotSupportedException(VectorDataStrings.IncludeVectorsNotSupportedWithEmbeddingGeneration); + } + + // Retrieve data points. + var retrievedPoints = await RunOperationAsync( + OperationName, + () => _qdrantClient.RetrieveAsync(Name, pointsIds, true, includeVectors, cancellationToken: cancellationToken)).ConfigureAwait(false); + + // Convert the retrieved points to the target data model. + foreach (var retrievedPoint in retrievedPoints) + { + yield return _mapper.MapFromStorageToDataModel(retrievedPoint.Id, retrievedPoint.Payload, retrievedPoint.Vectors, includeVectors); + } + } + + /// + public override Task DeleteAsync(TKey key, CancellationToken cancellationToken = default) + { + Throw.IfNull(key); + + return RunOperationAsync( + DeleteName, + () => key switch + { + ulong id => _qdrantClient.DeleteAsync(Name, id, wait: true, cancellationToken: cancellationToken), + Guid id => _qdrantClient.DeleteAsync(Name, id, wait: true, cancellationToken: cancellationToken), + _ => throw new NotSupportedException($"The provided key type '{key.GetType().Name}' is not supported by Qdrant.") + }); + } + + /// + public override Task DeleteAsync(IEnumerable keys, CancellationToken cancellationToken = default) + { + Throw.IfNull(keys); + + IList? keyList = null; + + switch (keys) + { + case IEnumerable k: + keyList = k.ToList(); + break; + + case IEnumerable k: + keyList = k.ToList(); + break; + + case IEnumerable objectKeys: + { + // We need to cast the keys to a list of the same type as the first element. + List? guidKeys = null; + List? ulongKeys = null; + + var isFirst = true; + foreach (var key in objectKeys) + { + if (isFirst) + { + switch (key) + { + case ulong l: + ulongKeys = [l]; + keyList = ulongKeys; + break; + + case Guid g: + guidKeys = [g]; + keyList = guidKeys; + break; + + default: + throw new NotSupportedException($"The provided key type '{key.GetType().Name}' is not supported by Qdrant."); + } + + isFirst = false; + continue; + } + + switch (key) + { + case ulong u when ulongKeys is not null: + ulongKeys.Add(u); + continue; + + case Guid g when guidKeys is not null: + guidKeys.Add(g); + continue; + + case Guid or ulong: + throw new NotSupportedException("Mixing ulong and Guid keys is not supported"); + + default: + throw new NotSupportedException($"The provided key type '{key.GetType().Name}' is not supported by Qdrant."); + } + } + + break; + } + } + + if (keyList is { Count: 0 }) + { + return Task.CompletedTask; + } + + return RunOperationAsync( + DeleteName, + () => keyList switch + { + List keysList => _qdrantClient.DeleteAsync( + Name, + keysList, + wait: true, + cancellationToken: cancellationToken), + + List keysList => _qdrantClient.DeleteAsync( + Name, + keysList, + wait: true, + cancellationToken: cancellationToken), + + _ => throw new UnreachableException() + }); + } + + /// + public override async Task UpsertAsync(TRecord record, CancellationToken cancellationToken = default) + { + Throw.IfNull(record); + + await UpsertAsync([record], cancellationToken).ConfigureAwait(false); + } + + /// + public override async Task UpsertAsync(IEnumerable records, CancellationToken cancellationToken = default) + { + Throw.IfNull(records); + + IReadOnlyList? recordsList = null; + + // If an embedding generator is defined, invoke it once per property for all records. + GeneratedEmbeddings>?[]? generatedEmbeddings = null; + + var vectorPropertyCount = _model.VectorProperties.Count; + for (var i = 0; i < vectorPropertyCount; i++) + { + var vectorProperty = _model.VectorProperties[i]; + + if (QdrantModelBuilder.IsVectorPropertyTypeValidCore(vectorProperty.Type, out _)) + { + continue; + } + + // We have a vector property whose type isn't natively supported - we need to generate embeddings. + Debug.Assert(vectorProperty.EmbeddingGenerator is not null); + + if (recordsList is null) + { + recordsList = records is IReadOnlyList r ? r : records.ToList(); + + if (recordsList.Count == 0) + { + return; + } + + records = recordsList; + } + + // TODO: Ideally we'd group together vector properties using the same generator (and with the same input and output properties), + // and generate embeddings for them in a single batch. That's some more complexity though. + generatedEmbeddings ??= new GeneratedEmbeddings>?[vectorPropertyCount]; + generatedEmbeddings[i] = (GeneratedEmbeddings>)await vectorProperty.GenerateEmbeddingsAsync(records.Select(r => vectorProperty.GetValueAsObject(r)), cancellationToken).ConfigureAwait(false); + } + + // Create points from records. + var keyProperty = _model.KeyProperty; + var pointStructs = new List(); + var recordIndex = 0; + foreach (var record in records) + { + if (keyProperty.IsAutoGenerated && keyProperty.GetValue(record) == Guid.Empty) + { + keyProperty.SetValue(record, Guid.NewGuid()); + } + + pointStructs.Add(_mapper.MapFromDataToStorageModel(record, recordIndex++, generatedEmbeddings)); + } + + if (pointStructs.Count == 0) + { + return; + } + + // Upsert. + await RunOperationAsync( + UpsertName, + () => _qdrantClient.UpsertAsync(Name, pointStructs, true, cancellationToken: cancellationToken)).ConfigureAwait(false); + } + + #region Search + + /// + public override async IAsyncEnumerable> SearchAsync( + TInput searchValue, + int top, + VectorSearchOptions? options = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + Throw.IfNull(searchValue); + Throw.IfLessThan(top, 1); + + options ??= s_defaultVectorSearchOptions; + if (options.IncludeVectors && _model.EmbeddingGenerationRequired) + { + throw new NotSupportedException(VectorDataStrings.IncludeVectorsNotSupportedWithEmbeddingGeneration); + } + + var vectorProperty = _model.GetVectorPropertyOrSingle(options); + var vectorArray = await GetSearchVectorArrayAsync(searchValue, vectorProperty, cancellationToken).ConfigureAwait(false); + + // Build filter object. + var filter = options.Filter is not null + ? new QdrantFilterTranslator().Translate(options.Filter, _model) + : new Filter(); + + // Specify whether to include vectors in the search results. + var vectorsSelector = new WithVectorsSelector { Enable = options.IncludeVectors }; + var query = new Query { Nearest = new VectorInput(vectorArray) }; + + // Execute Search. + var points = await RunOperationAsync( + "Query", + () => _qdrantClient.QueryAsync( + Name, + query: query, + usingVector: _hasNamedVectors ? vectorProperty.StorageName : null, + filter: filter, + scoreThreshold: (float?)options.ScoreThreshold, + limit: (ulong)top, + offset: (ulong)options.Skip, + vectorsSelector: vectorsSelector, + cancellationToken: cancellationToken)).ConfigureAwait(false); + + // Map to data model. + var mappedResults = points.Select(point => QdrantCollectionSearchMapping.MapScoredPointToVectorSearchResult( + point, + _mapper, + options.IncludeVectors, + QdrantConstants.VectorStoreSystemName, + _collectionMetadata.VectorStoreName, + Name, + "Query")); + + foreach (var result in mappedResults) + { + yield return result; + } + } + + private static async ValueTask GetSearchVectorArrayAsync(TInput searchValue, VectorPropertyModel vectorProperty, CancellationToken cancellationToken) + where TInput : notnull + { + if (searchValue is float[] array) + { + return array; + } + + var memory = searchValue switch + { + ReadOnlyMemory r => r, + Embedding e => e.Vector, + _ when vectorProperty.EmbeddingGenerationDispatcher is not null + => ((Embedding)await vectorProperty.GenerateEmbeddingAsync(searchValue, cancellationToken).ConfigureAwait(false)).Vector, + + _ => vectorProperty.EmbeddingGenerator is null + ? throw new NotSupportedException(VectorDataStrings.InvalidSearchInputAndNoEmbeddingGeneratorWasConfigured(searchValue.GetType(), QdrantModelBuilder.SupportedVectorTypes)) + : throw new InvalidOperationException(VectorDataStrings.IncompatibleEmbeddingGeneratorWasConfiguredForInputType(typeof(TInput), vectorProperty.EmbeddingGenerator.GetType())) + }; + + return MemoryMarshal.TryGetArray(memory, out ArraySegment segment) && segment.Count == segment.Array!.Length + ? segment.Array + : memory.ToArray(); + } + + #endregion Search + + /// + public override async IAsyncEnumerable GetAsync(Expression> filter, int top, + FilteredRecordRetrievalOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + Throw.IfNull(filter); + Throw.IfLessThan(top, 1); + + options ??= new(); + + var translatedFilter = new QdrantFilterTranslator().Translate(filter, _model); + + // Specify whether to include vectors in the search results. + WithVectorsSelector vectorsSelector = new() { Enable = options.IncludeVectors }; + + var orderByValues = options.OrderBy?.Invoke(new()).Values; + var sortInfo = orderByValues switch + { + null => null, + _ when orderByValues.Count == 1 => orderByValues[0], + _ => throw new NotSupportedException("Qdrant does not support ordering by more than one property.") + }; + + OrderBy? orderBy = null; + if (sortInfo is not null) + { + var orderByName = _model.GetDataOrKeyProperty(sortInfo.PropertySelector).StorageName; + orderBy = new(orderByName) + { + Direction = sortInfo.Ascending ? global::Qdrant.Client.Grpc.Direction.Asc : global::Qdrant.Client.Grpc.Direction.Desc + }; + } + + var scrollResponse = await RunOperationAsync( + "Scroll", + () => _qdrantClient.ScrollAsync( + Name, + translatedFilter, + vectorsSelector, + limit: (uint)(top + options.Skip), + orderBy, + cancellationToken: cancellationToken)).ConfigureAwait(false); + + var mappedResults = scrollResponse.Result.Skip(options.Skip).Select(point => QdrantCollectionSearchMapping.MapRetrievedPointToRecord( + point, + _mapper, + options.IncludeVectors, + QdrantConstants.VectorStoreSystemName, + _collectionMetadata.VectorStoreName, + Name, + "Scroll")); + + foreach (var mappedResult in mappedResults) + { + yield return mappedResult; + } + } + + /// + public async IAsyncEnumerable> HybridSearchAsync(TInput searchValue, ICollection keywords, int top, HybridSearchOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + where TInput : notnull + { + Throw.IfLessThan(top, 1); + + // Resolve options. + options ??= s_defaultKeywordVectorizedHybridSearchOptions; + var vectorProperty = _model.GetVectorPropertyOrSingle(new() { VectorProperty = options.VectorProperty }); + var vectorArray = await GetSearchVectorArrayAsync(searchValue, vectorProperty, cancellationToken).ConfigureAwait(false); + var textDataProperty = _model.GetFullTextDataPropertyOrSingle(options.AdditionalProperty); + + // Build filter object. + var filter = options.Filter is not null + ? new QdrantFilterTranslator().Translate(options.Filter, _model) + : new Filter(); + + // Specify whether to include vectors in the search results. + var vectorsSelector = new WithVectorsSelector { Enable = options.IncludeVectors }; + + // Build the vector query. + var vectorQuery = new PrefetchQuery + { + Filter = filter, + Query = new Query { Nearest = new VectorInput(vectorArray) } + }; + + if (_hasNamedVectors) + { + vectorQuery.Using = _hasNamedVectors ? vectorProperty.StorageName : null; + } + + // Build the keyword query. + var keywordFilter = filter.Clone(); + var keywordSubFilter = new Filter(); + foreach (string keyword in keywords) + { + keywordSubFilter.Should.Add(new Condition() { Field = new FieldCondition() { Key = textDataProperty.StorageName, Match = new Match { Text = keyword } } }); + } + keywordFilter.Must.Add(new Condition() { Filter = keywordSubFilter }); + var keywordQuery = new PrefetchQuery + { + Filter = keywordFilter, + }; + + // Build the fusion query. + var fusionQuery = new Query + { + Fusion = Fusion.Rrf, + }; + + // Execute Search. + var points = await RunOperationAsync( + "Query", + () => _qdrantClient.QueryAsync( + Name, + prefetch: [vectorQuery, keywordQuery], + query: fusionQuery, + scoreThreshold: (float?)options.ScoreThreshold, + limit: (ulong)top, + offset: (ulong)options.Skip, + vectorsSelector: vectorsSelector, + cancellationToken: cancellationToken)).ConfigureAwait(false); + + // Map to data model. + var mappedResults = points.Select(point => QdrantCollectionSearchMapping.MapScoredPointToVectorSearchResult( + point, + _mapper, + options.IncludeVectors, + QdrantConstants.VectorStoreSystemName, + _collectionMetadata.VectorStoreName, + Name, + "Query")); + + foreach (var result in mappedResults) + { + yield return result; + } + } + + /// + public override object? GetService(Type serviceType, object? serviceKey = null) + { + Throw.IfNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreCollectionMetadata) ? _collectionMetadata : + serviceType == typeof(QdrantClient) ? _qdrantClient.QdrantClient : + serviceType.IsInstanceOfType(this) ? this : + null; + } + + /// + /// Run the given operation and wrap any with ."/> + /// + /// The type of database operation being run. + /// The operation to run. + /// The result of the operation. + private Task RunOperationAsync(string operationName, Func operation) + => VectorStoreErrorHandler.RunOperationAsync( + _collectionMetadata, + operationName, + operation); + + /// + /// Run the given operation and wrap any with ."/> + /// + /// The response type of the operation. + /// The type of database operation being run. + /// The operation to run. + /// The result of the operation. + private Task RunOperationAsync(string operationName, Func> operation) + => VectorStoreErrorHandler.RunOperationAsync( + _collectionMetadata, + operationName, + operation); +} diff --git a/MEVD/src/Qdrant/QdrantCollectionCreateMapping.cs b/MEVD/src/Qdrant/QdrantCollectionCreateMapping.cs new file mode 100644 index 0000000..6f9f4a7 --- /dev/null +++ b/MEVD/src/Qdrant/QdrantCollectionCreateMapping.cs @@ -0,0 +1,109 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; +using Qdrant.Client.Grpc; + +namespace CommunityToolkit.VectorData.Qdrant; + +/// +/// Contains mapping helpers to use when creating a qdrant vector collection. +/// +internal static class QdrantCollectionCreateMapping +{ + /// A dictionary of types and their matching qdrant index schema type. + public static readonly Dictionary s_schemaTypeMap = new() + { + { typeof(short), PayloadSchemaType.Integer }, + { typeof(sbyte), PayloadSchemaType.Integer }, + { typeof(byte), PayloadSchemaType.Integer }, + { typeof(ushort), PayloadSchemaType.Integer }, + { typeof(int), PayloadSchemaType.Integer }, + { typeof(uint), PayloadSchemaType.Integer }, + { typeof(long), PayloadSchemaType.Integer }, + { typeof(ulong), PayloadSchemaType.Integer }, + { typeof(float), PayloadSchemaType.Float }, + { typeof(double), PayloadSchemaType.Float }, + { typeof(decimal), PayloadSchemaType.Float }, + + { typeof(short?), PayloadSchemaType.Integer }, + { typeof(sbyte?), PayloadSchemaType.Integer }, + { typeof(byte?), PayloadSchemaType.Integer }, + { typeof(ushort?), PayloadSchemaType.Integer }, + { typeof(int?), PayloadSchemaType.Integer }, + { typeof(uint?), PayloadSchemaType.Integer }, + { typeof(long?), PayloadSchemaType.Integer }, + { typeof(ulong?), PayloadSchemaType.Integer }, + { typeof(float?), PayloadSchemaType.Float }, + { typeof(double?), PayloadSchemaType.Float }, + { typeof(decimal?), PayloadSchemaType.Float }, + + { typeof(string), PayloadSchemaType.Keyword }, + { typeof(bool), PayloadSchemaType.Bool }, + { typeof(bool?), PayloadSchemaType.Bool }, + + { typeof(DateTime), PayloadSchemaType.Datetime }, + { typeof(DateTimeOffset), PayloadSchemaType.Datetime }, + { typeof(DateTime?), PayloadSchemaType.Datetime }, + { typeof(DateTimeOffset?), PayloadSchemaType.Datetime }, + +#if NET + { typeof(DateOnly), PayloadSchemaType.Datetime }, + { typeof(DateOnly?), PayloadSchemaType.Datetime }, +#endif + }; + + /// + /// Maps a single to a qdrant . + /// + /// The property to map. + /// The mapped . + /// Thrown if the property is missing information or has unsupported options specified. + public static VectorParams MapSingleVector(VectorPropertyModel vectorProperty) + { + if (vectorProperty!.IndexKind is not null and not IndexKind.Hnsw) + { + throw new NotSupportedException($"Index kind '{vectorProperty!.IndexKind}' for {nameof(VectorStoreVectorProperty)} '{vectorProperty.ModelName}' is not supported by the Qdrant VectorStore."); + } + + return new VectorParams { Size = (ulong)vectorProperty.Dimensions, Distance = QdrantCollectionCreateMapping.GetSDKDistanceAlgorithm(vectorProperty) }; + } + + /// + /// Maps a collection of to a qdrant . + /// + /// The properties to map. + /// THe mapped . + /// Thrown if the property is missing information or has unsupported options specified. + public static VectorParamsMap MapNamedVectors(IEnumerable vectorProperties) + { + var vectorParamsMap = new VectorParamsMap(); + + foreach (var vectorProperty in vectorProperties) + { + // Add each vector property to the vectors map. + vectorParamsMap.Map.Add(vectorProperty.StorageName, MapSingleVector(vectorProperty)); + } + + return vectorParamsMap; + } + + /// + /// Get the configured from the given . + /// If none is configured, the default is . + /// + /// The vector property definition. + /// The chosen . + /// Thrown if a distance function is chosen that isn't supported by qdrant. + public static Distance GetSDKDistanceAlgorithm(VectorPropertyModel vectorProperty) + => vectorProperty.DistanceFunction switch + { + DistanceFunction.CosineSimilarity or null => Distance.Cosine, + DistanceFunction.DotProductSimilarity => Distance.Dot, + DistanceFunction.EuclideanDistance => Distance.Euclid, + DistanceFunction.ManhattanDistance => Distance.Manhattan, + + _ => throw new NotSupportedException($"Distance function '{vectorProperty.DistanceFunction}' for {nameof(VectorStoreVectorProperty)} '{vectorProperty.ModelName}' is not supported by the Qdrant VectorStore.") + }; +} diff --git a/MEVD/src/Qdrant/QdrantCollectionOptions.cs b/MEVD/src/Qdrant/QdrantCollectionOptions.cs new file mode 100644 index 0000000..e22b451 --- /dev/null +++ b/MEVD/src/Qdrant/QdrantCollectionOptions.cs @@ -0,0 +1,32 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.VectorData; + +namespace CommunityToolkit.VectorData.Qdrant; + +/// +/// Options when creating a . +/// +public sealed class QdrantCollectionOptions : VectorStoreCollectionOptions +{ + internal static readonly QdrantCollectionOptions Default = new(); + + /// + /// Initializes a new instance of the class. + /// + public QdrantCollectionOptions() + { + } + + internal QdrantCollectionOptions(QdrantCollectionOptions? source) : base(source) + { + HasNamedVectors = source?.HasNamedVectors ?? Default.HasNamedVectors; + } + + /// + /// Gets or sets a value indicating whether the vectors in the store are named and multiple vectors are supported, or whether there is just a single unnamed vector per qdrant point. + /// Defaults to single vector per point. + /// + public bool HasNamedVectors { get; set; } +} diff --git a/MEVD/src/Qdrant/QdrantCollectionSearchMapping.cs b/MEVD/src/Qdrant/QdrantCollectionSearchMapping.cs new file mode 100644 index 0000000..4784863 --- /dev/null +++ b/MEVD/src/Qdrant/QdrantCollectionSearchMapping.cs @@ -0,0 +1,55 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.VectorData; +using Qdrant.Client.Grpc; + +namespace CommunityToolkit.VectorData.Qdrant; + +/// +/// Contains mapping helpers to use when searching for documents using Qdrant. +/// +internal static class QdrantCollectionSearchMapping +{ + /// + /// Map the given to a . + /// + /// The type of the record to map to. + /// The point to map to a . + /// The mapper to perform the main mapping operation with. + /// A value indicating whether to include vectors in the mapped result. + /// The name of the vector store system the operation is being run on. + /// The name of the vector store the operation is being run on. + /// The name of the collection the operation is being run on. + /// The type of database operation being run. + /// The mapped . + public static VectorSearchResult MapScoredPointToVectorSearchResult( + ScoredPoint point, + QdrantMapper mapper, + bool includeVectors, + string vectorStoreSystemName, + string? vectorStoreName, + string collectionName, + string operationName) + where TRecord : class + { + // Do the mapping with error handling. + return new VectorSearchResult( + mapper.MapFromStorageToDataModel(point.Id, point.Payload, point.Vectors, includeVectors), + point.Score); + } + + internal static TRecord MapRetrievedPointToRecord( + RetrievedPoint point, + QdrantMapper mapper, + bool includeVectors, + string vectorStoreSystemName, + string? vectorStoreName, + string collectionName, + string operationName) + where TRecord : class + { + // Do the mapping with error handling. + return mapper.MapFromStorageToDataModel(point.Id, point.Payload, point.Vectors, includeVectors); + } +} diff --git a/MEVD/src/Qdrant/QdrantConstants.cs b/MEVD/src/Qdrant/QdrantConstants.cs new file mode 100644 index 0000000..ac3c669 --- /dev/null +++ b/MEVD/src/Qdrant/QdrantConstants.cs @@ -0,0 +1,9 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace CommunityToolkit.VectorData.Qdrant; + +internal static class QdrantConstants +{ + internal const string VectorStoreSystemName = "qdrant"; +} diff --git a/MEVD/src/Qdrant/QdrantDynamicCollection.cs b/MEVD/src/Qdrant/QdrantDynamicCollection.cs new file mode 100644 index 0000000..c2bef2c --- /dev/null +++ b/MEVD/src/Qdrant/QdrantDynamicCollection.cs @@ -0,0 +1,38 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Qdrant.Client; + +namespace CommunityToolkit.VectorData.Qdrant; + +/// +/// Represents a collection of vector store records in a Qdrant database, mapped to a dynamic Dictionary<string, object?>. +/// +#pragma warning disable CA1711 // Identifiers should not have incorrect suffix +public sealed class QdrantDynamicCollection : QdrantCollection> +#pragma warning restore CA1711 // Identifiers should not have incorrect suffix +{ + /// + /// Initializes a new instance of the class. + /// + /// Qdrant client that can be used to manage the collections and points in a Qdrant store. + /// The name of the collection. + /// A value indicating whether is disposed when the collection is disposed. + /// Optional configuration options for this class. + public QdrantDynamicCollection(QdrantClient qdrantClient, string name, bool ownsClient, QdrantCollectionOptions options) + : this(() => new MockableQdrantClient(qdrantClient, ownsClient), name, options) + { + } + + internal QdrantDynamicCollection(Func clientFactory, string name, QdrantCollectionOptions options) + : base( + clientFactory, + name, + static options => new QdrantModelBuilder(options.HasNamedVectors) + .BuildDynamic( + options.Definition ?? throw new ArgumentException("Definition is required for dynamic collections"), + options.EmbeddingGenerator), + options) + { + } +} diff --git a/MEVD/src/Qdrant/QdrantFieldMapping.cs b/MEVD/src/Qdrant/QdrantFieldMapping.cs new file mode 100644 index 0000000..9372794 --- /dev/null +++ b/MEVD/src/Qdrant/QdrantFieldMapping.cs @@ -0,0 +1,182 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections; +using System.Diagnostics; +using System.Globalization; +using Qdrant.Client.Grpc; + +namespace CommunityToolkit.VectorData.Qdrant; + +/// +/// Contains helper methods for mapping fields to and from the format required by the Qdrant client sdk. +/// +internal static class QdrantFieldMapping +{ + /// + /// Convert the given to the correct native type based on its properties. + /// + /// The value to convert to a native type. + /// The target type to convert the value to. + /// The converted native value. + /// Thrown when an unsupported type is encountered. + public static object? Deserialize(Value payloadValue, Type targetType) + { + if (Nullable.GetUnderlyingType(targetType) is Type unwrapped) + { + targetType = unwrapped; + } + + return payloadValue.KindCase switch + { + Value.KindOneofCase.NullValue => null, + + Value.KindOneofCase.IntegerValue + => targetType == typeof(int) ? (object)(int)payloadValue.IntegerValue : (object)payloadValue.IntegerValue, + + Value.KindOneofCase.StringValue when targetType == typeof(DateTimeOffset) + => DateTimeOffset.Parse(payloadValue.StringValue, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind), + + Value.KindOneofCase.StringValue when targetType == typeof(DateTime) + => DateTime.Parse(payloadValue.StringValue, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind), + +#if NET + Value.KindOneofCase.StringValue when targetType == typeof(DateOnly) + => DateOnly.Parse(payloadValue.StringValue, CultureInfo.InvariantCulture), +#endif + + Value.KindOneofCase.StringValue + => payloadValue.StringValue, + + Value.KindOneofCase.DoubleValue + => targetType == typeof(float) ? (object)(float)payloadValue.DoubleValue : (object)payloadValue.DoubleValue, + + Value.KindOneofCase.BoolValue + => payloadValue.BoolValue, + + Value.KindOneofCase.ListValue => DeserializeCollection(payloadValue, targetType), + + _ => throw new InvalidOperationException($"Unsupported grpc value kind {payloadValue.KindCase}."), + }; + + static object? DeserializeCollection(Value payloadValue, Type targetType) + => targetType switch + { + Type t when t == typeof(List) + => payloadValue.ListValue.Values.Select(v => (int)v.IntegerValue).ToList(), + Type t when t == typeof(int[]) + => payloadValue.ListValue.Values.Select(v => (int)v.IntegerValue).ToArray(), + Type t when t == typeof(List) + => payloadValue.ListValue.Values.Select(v => v.IntegerValue).ToList(), + Type t when t == typeof(long[]) + => payloadValue.ListValue.Values.Select(v => v.IntegerValue).ToArray(), + + Type t when t == typeof(List) + => payloadValue.ListValue.Values.Select(v => v.StringValue).ToList(), + Type t when t == typeof(string[]) + => payloadValue.ListValue.Values.Select(v => v.StringValue).ToArray(), + + Type t when t == typeof(List) + => payloadValue.ListValue.Values.Select(v => v.DoubleValue).ToList(), + Type t when t == typeof(double[]) + => payloadValue.ListValue.Values.Select(v => v.DoubleValue).ToArray(), + Type t when t == typeof(List) + => payloadValue.ListValue.Values.Select(v => (float)v.DoubleValue).ToList(), + Type t when t == typeof(float[]) + => payloadValue.ListValue.Values.Select(v => (float)v.DoubleValue).ToArray(), + + Type t when t == typeof(List) + => payloadValue.ListValue.Values.Select(v => v.BoolValue).ToList(), + Type t when t == typeof(bool[]) + => payloadValue.ListValue.Values.Select(v => v.BoolValue).ToArray(), + + Type t when t == typeof(List) + => payloadValue.ListValue.Values.Select(v => DateTimeOffset.Parse(v.StringValue, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind)).ToList(), + Type t when t == typeof(DateTimeOffset[]) + => payloadValue.ListValue.Values.Select(v => DateTimeOffset.Parse(v.StringValue, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind)).ToArray(), + + Type t when t == typeof(List) + => payloadValue.ListValue.Values.Select(v => DateTime.Parse(v.StringValue, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind)).ToList(), + Type t when t == typeof(DateTime[]) + => payloadValue.ListValue.Values.Select(v => DateTime.Parse(v.StringValue, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind)).ToArray(), + +#if NET + Type t when t == typeof(List) + => payloadValue.ListValue.Values.Select(v => DateOnly.Parse(v.StringValue, CultureInfo.InvariantCulture)).ToList(), + Type t when t == typeof(DateOnly[]) + => payloadValue.ListValue.Values.Select(v => DateOnly.Parse(v.StringValue, CultureInfo.InvariantCulture)).ToArray(), +#endif + + _ => throw new UnreachableException($"Unsupported collection type {targetType.Name}"), + }; + } + + /// + /// Convert the given to a object that can be stored in Qdrant. + /// + /// The object to convert. + /// The converted Qdrant value. + /// Thrown when an unsupported type is encountered. + public static Value ConvertToGrpcFieldValue(object? sourceValue) + { + var value = new Value(); + switch (sourceValue) + { + case null: + value.NullValue = NullValue.NullValue; + break; + case int intValue: + value.IntegerValue = intValue; + break; + case long longValue: + value.IntegerValue = longValue; + break; + case string stringValue: + value.StringValue = stringValue; + break; + case float floatValue: + value.DoubleValue = floatValue; + break; + case double doubleValue: + value.DoubleValue = doubleValue; + break; + case bool boolValue: + value.BoolValue = boolValue; + break; + case DateTimeOffset dateTimeOffsetValue: + value.StringValue = dateTimeOffsetValue.ToString("O"); + break; + case DateTime dateTimeValue: + value.StringValue = dateTimeValue.ToString("O"); + break; +#if NET + case DateOnly dateOnlyValue: + value.StringValue = dateOnlyValue.ToString("O"); + break; +#endif + case IEnumerable or + IEnumerable or + IEnumerable or + IEnumerable or + IEnumerable or + IEnumerable or + IEnumerable or + IEnumerable: + { + var listValue = sourceValue as IEnumerable; + value.ListValue = new ListValue(); + foreach (var item in listValue!) + { + value.ListValue.Values.Add(ConvertToGrpcFieldValue(item)); + } + + break; + } + + default: + throw new InvalidOperationException($"Unsupported source value type {sourceValue?.GetType().FullName}."); + } + + return value; + } +} diff --git a/MEVD/src/Qdrant/QdrantFilterTranslator.cs b/MEVD/src/Qdrant/QdrantFilterTranslator.cs new file mode 100644 index 0000000..9823e86 --- /dev/null +++ b/MEVD/src/Qdrant/QdrantFilterTranslator.cs @@ -0,0 +1,461 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections; +using System.Diagnostics.CodeAnalysis; +using System.Linq.Expressions; +using Google.Protobuf.Collections; +using Google.Protobuf.WellKnownTypes; +using Microsoft.Extensions.VectorData.ProviderServices; +using Microsoft.Extensions.VectorData.ProviderServices.Filter; +using Qdrant.Client.Grpc; +using Expression = System.Linq.Expressions.Expression; +using Range = Qdrant.Client.Grpc.Range; + +namespace CommunityToolkit.VectorData.Qdrant; + +// https://qdrant.tech/documentation/concepts/filtering +internal class QdrantFilterTranslator : FilterTranslatorBase +{ + internal Filter Translate(LambdaExpression lambdaExpression, CollectionModel model) + { + var preprocessedExpression = PreprocessFilter(lambdaExpression, model, new FilterPreprocessingOptions()); + + return Translate(preprocessedExpression); + } + + private Filter Translate(Expression? node) + => node switch + { + BinaryExpression { NodeType: ExpressionType.Equal } equal => TranslateEqual(equal.Left, equal.Right), + BinaryExpression { NodeType: ExpressionType.NotEqual } notEqual => TranslateEqual(notEqual.Left, notEqual.Right, negated: true), + + BinaryExpression + { + NodeType: ExpressionType.GreaterThan or ExpressionType.GreaterThanOrEqual or ExpressionType.LessThan or ExpressionType.LessThanOrEqual + } comparison + => TranslateComparison(comparison), + + BinaryExpression { NodeType: ExpressionType.AndAlso } andAlso => TranslateAndAlso(andAlso.Left, andAlso.Right), + BinaryExpression { NodeType: ExpressionType.OrElse } orElse => TranslateOrElse(orElse.Left, orElse.Right), + + UnaryExpression { NodeType: ExpressionType.Not } not => TranslateNot(not.Operand), + // Handle converting non-nullable to nullable; such nodes are found in e.g. r => r.Int == nullableInt + UnaryExpression { NodeType: ExpressionType.Convert } convert when Nullable.GetUnderlyingType(convert.Type) == convert.Operand.Type + => Translate(convert.Operand), + + // Special handling for bool constant as the filter expression (r => r.Bool) + Expression when node.Type == typeof(bool) && TryBindProperty(node, out var property) + => GenerateEqual(property.StorageName, value: true), + // Handle true literal (r => true), which is useful for fetching all records + ConstantExpression { Value: true } => new Filter(), + + MethodCallExpression methodCall => TranslateMethodCall(methodCall), + + _ => throw new NotSupportedException("Qdrant does not support the following NodeType in filters: " + node?.NodeType) + }; + + private Filter TranslateEqual(Expression left, Expression right, bool negated = false) + => TryBindProperty(left, out var property) && right is ConstantExpression { Value: var rightConstant } + ? GenerateEqual(property.StorageName, rightConstant, negated) + : TryBindProperty(right, out property) && left is ConstantExpression { Value: var leftConstant } + ? GenerateEqual(property.StorageName, leftConstant, negated) + : throw new NotSupportedException("Invalid equality/comparison"); + + private Filter GenerateEqual(string propertyStorageName, object? value, bool negated = false) + { + var condition = value is null + ? new Condition { IsNull = new() { Key = propertyStorageName } } + : new Condition + { + Field = new FieldCondition + { + Key = propertyStorageName, + Match = value switch + { + string v => new Match { Keyword = v }, + int v => new Match { Integer = v }, + long v => new Match { Integer = v }, + bool v => new Match { Boolean = v }, + DateTime v => new Match { Keyword = v.ToString("o") }, + DateTimeOffset v => new Match { Keyword = v.ToString("o") }, +#if NET + DateOnly v => new Match { Keyword = v.ToString("O") }, +#endif + + _ => throw new NotSupportedException($"Unsupported filter value type '{value.GetType().Name}'.") + } + } + }; + + var result = new Filter(); + + if (negated) + { + result.MustNot.Add(condition); + } + else + { + result.Must.Add(condition); + } + + return result; + } + + private Filter TranslateComparison(BinaryExpression comparison) + { + return TryProcessComparison(comparison.Left, comparison.Right, out var result) + ? result + : TryProcessComparison(comparison.Right, comparison.Left, out result) + ? result + : throw new NotSupportedException("Comparison expression not supported by Qdrant"); + + bool TryProcessComparison(Expression first, Expression second, [NotNullWhen(true)] out Filter? result) + { + if (TryBindProperty(first, out var property) && second is ConstantExpression { Value: var constantValue }) + { + result = new Filter(); + result.Must.Add(new Condition + { + Field = constantValue switch + { + double v => DoubleFieldCondition(v), + int v => DoubleFieldCondition(v), + long v => DoubleFieldCondition(v), + + DateTimeOffset v => new FieldCondition + { + Key = property.StorageName, + DatetimeRange = new DatetimeRange + { + Gt = comparison.NodeType == ExpressionType.GreaterThan ? Timestamp.FromDateTimeOffset(v) : null, + Gte = comparison.NodeType == ExpressionType.GreaterThanOrEqual ? Timestamp.FromDateTimeOffset(v) : null, + Lt = comparison.NodeType == ExpressionType.LessThan ? Timestamp.FromDateTimeOffset(v) : null, + Lte = comparison.NodeType == ExpressionType.LessThanOrEqual ? Timestamp.FromDateTimeOffset(v) : null + } + }, + + DateTime v => new FieldCondition + { + Key = property.StorageName, + DatetimeRange = new DatetimeRange + { + Gt = comparison.NodeType == ExpressionType.GreaterThan ? Timestamp.FromDateTime(DateTime.SpecifyKind(v, DateTimeKind.Utc)) : null, + Gte = comparison.NodeType == ExpressionType.GreaterThanOrEqual ? Timestamp.FromDateTime(DateTime.SpecifyKind(v, DateTimeKind.Utc)) : null, + Lt = comparison.NodeType == ExpressionType.LessThan ? Timestamp.FromDateTime(DateTime.SpecifyKind(v, DateTimeKind.Utc)) : null, + Lte = comparison.NodeType == ExpressionType.LessThanOrEqual ? Timestamp.FromDateTime(DateTime.SpecifyKind(v, DateTimeKind.Utc)) : null + } + }, + +#if NET + DateOnly v => new FieldCondition + { + Key = property.StorageName, + DatetimeRange = new DatetimeRange + { + Gt = comparison.NodeType == ExpressionType.GreaterThan ? Timestamp.FromDateTimeOffset(new DateTimeOffset(v.ToDateTime(TimeOnly.MinValue), TimeSpan.Zero)) : null, + Gte = comparison.NodeType == ExpressionType.GreaterThanOrEqual ? Timestamp.FromDateTimeOffset(new DateTimeOffset(v.ToDateTime(TimeOnly.MinValue), TimeSpan.Zero)) : null, + Lt = comparison.NodeType == ExpressionType.LessThan ? Timestamp.FromDateTimeOffset(new DateTimeOffset(v.ToDateTime(TimeOnly.MinValue), TimeSpan.Zero)) : null, + Lte = comparison.NodeType == ExpressionType.LessThanOrEqual ? Timestamp.FromDateTimeOffset(new DateTimeOffset(v.ToDateTime(TimeOnly.MinValue), TimeSpan.Zero)) : null + } + }, +#endif + + _ => throw new NotSupportedException($"Can't perform comparison on type '{constantValue?.GetType().Name}'") + } + }); + + return true; + + FieldCondition DoubleFieldCondition(double d) + => new() + { + Key = property.StorageName, + Range = comparison.NodeType switch + { + ExpressionType.GreaterThan => new Range { Gt = d }, + ExpressionType.GreaterThanOrEqual => new Range { Gte = d }, + ExpressionType.LessThan => new Range { Lt = d }, + ExpressionType.LessThanOrEqual => new Range { Lte = d }, + + _ => throw new InvalidOperationException("Unreachable") + } + }; + } + + result = null; + return false; + } + } + + #region Logical operators + + private Filter TranslateAndAlso(Expression left, Expression right) + { + var leftFilter = Translate(left); + var rightFilter = Translate(right); + + // As long as there are only AND conditions (Must or MustNot), we can simply combine both filters into a single flat one. + // The moment there's a Should, things become a bit more complicated: + // 1. If a side contains both a Should and a Must/MustNot, it must be pushed down. + // 2. Otherwise, if the left's Should is empty, and the right side is only Should, we can just copy the right Should into the left's. + // 3. Finally, if both sides have a Should, we push down the right side and put the result in the left's Must. + if (leftFilter.Should.Count > 0 && (leftFilter.Must.Count > 0 || leftFilter.MustNot.Count > 0)) + { + leftFilter = new Filter { Must = { new Condition { Filter = leftFilter } } }; + } + + if (rightFilter.Should.Count > 0 && (rightFilter.Must.Count > 0 || rightFilter.MustNot.Count > 0)) + { + rightFilter = new Filter { Must = { new Condition { Filter = rightFilter } } }; + } + + if (rightFilter.Should.Count > 0) + { + if (leftFilter.Should.Count == 0) + { + leftFilter.Should.AddRange(rightFilter.Should); + } + else + { + rightFilter = new Filter { Must = { new Condition { Filter = rightFilter } } }; + } + } + + leftFilter.Must.AddRange(rightFilter.Must); + leftFilter.MustNot.AddRange(rightFilter.MustNot); + + return leftFilter; + } + + private Filter TranslateOrElse(Expression left, Expression right) + { + var leftFilter = Translate(left); + var rightFilter = Translate(right); + + var result = new Filter(); + result.Should.AddRange(GetShouldConditions(leftFilter)); + result.Should.AddRange(GetShouldConditions(rightFilter)); + return result; + + static RepeatedField GetShouldConditions(Filter filter) + => filter switch + { + // If the filter only contains Should conditions (string of ORs), those can be directly added to the result + // (concatenated into the Should with whatever comes out of the other side) + { Must.Count: 0, MustNot.Count: 0 } => filter.Should, + + // If the filter is just a single Must condition, it can also be directly added to the result. + { Must.Count: 1, MustNot.Count: 0, Should.Count: 0 } => [filter.Must[0]], + + // For all other cases, we need to wrap the filter in a condition and return that, to preserve the logical structure. + _ => [new Condition { Filter = filter }] + }; + } + + private Filter TranslateNot(Expression expression) + { + // Special handling for !(a == b) and !(a != b) + if (expression is BinaryExpression { NodeType: ExpressionType.Equal or ExpressionType.NotEqual } binary) + { + return TranslateEqual(binary.Left, binary.Right, negated: binary.NodeType is ExpressionType.Equal); + } + + var filter = Translate(expression); + + switch (filter) + { + case { Must.Count: 1, MustNot.Count: 0, Should.Count: 0 }: + filter.MustNot.Add(filter.Must[0]); + filter.Must.RemoveAt(0); + return filter; + + case { Must.Count: 0, MustNot.Count: 1, Should.Count: 0 }: + filter.Must.Add(filter.MustNot[0]); + filter.MustNot.RemoveAt(0); + return filter; + + case { Must.Count: 0, MustNot.Count: 0, Should.Count: > 0 }: + filter.MustNot.AddRange(filter.Should); + filter.Should.Clear(); + return filter; + + default: + return new Filter { MustNot = { new Condition { Filter = filter } } }; + } + } + + #endregion Logical operators + + private Filter TranslateMethodCall(MethodCallExpression methodCall) + { + return methodCall switch + { + // Enumerable.Contains(), List.Contains(), MemoryExtensions.Contains() + _ when TryMatchContains(methodCall, out var source, out var item) + => TranslateContains(source, item), + + // Enumerable.Any() with a Contains predicate (r => r.Strings.Any(s => array.Contains(s))) + { Method.Name: nameof(Enumerable.Any), Arguments: [var anySource, LambdaExpression lambda] } any + when any.Method.DeclaringType == typeof(Enumerable) + => TranslateAny(anySource, lambda), + + _ => throw new NotSupportedException($"Unsupported method call: {methodCall.Method.DeclaringType?.Name}.{methodCall.Method.Name}") + }; + } + + private Filter TranslateContains(Expression source, Expression item) + { + switch (source) + { + // Contains over field enumerable + case var _ when TryBindProperty(source, out _): + // Oddly, in Qdrant, tag list contains is handled using a Match condition, just like equality. + return TranslateEqual(source, item); + + // Contains over inline enumerable + case NewArrayExpression newArray: + var elements = new object?[newArray.Expressions.Count]; + + for (var i = 0; i < newArray.Expressions.Count; i++) + { + if (newArray.Expressions[i] is not ConstantExpression { Value: var elementValue }) + { + throw new NotSupportedException("Inline array elements must be constants"); + } + + elements[i] = elementValue; + } + + return ProcessInlineEnumerable(elements, item); + + case ConstantExpression { Value: IEnumerable enumerable and not string }: + return ProcessInlineEnumerable(enumerable, item); + + default: + throw new NotSupportedException("Unsupported Contains"); + } + + Filter ProcessInlineEnumerable(IEnumerable elements, Expression item) + { + if (!TryBindProperty(item, out var property)) + { + throw new NotSupportedException("Unsupported item type in Contains"); + } + + switch (property.Type) + { + case var t when t == typeof(string): + var strings = new RepeatedStrings(); + + foreach (var value in elements) + { + strings.Strings.Add(value is string or null + ? (string?)value + : throw new ArgumentException("Non-string element in string Contains array")); + } + + return new Filter { Must = { new Condition { Field = new FieldCondition { Key = property.StorageName, Match = new Match { Keywords = strings } } } } }; + + case var t when t == typeof(int): + var ints = new RepeatedIntegers(); + + foreach (var value in elements) + { + ints.Integers.Add(value is int intValue + ? intValue + : throw new ArgumentException("Non-int element in string Contains array")); + } + + return new Filter { Must = { new Condition { Field = new FieldCondition { Key = property.StorageName, Match = new Match { Integers = ints } } } } }; + + default: + throw new NotSupportedException("Contains only supported over array of ints or strings"); + } + } + } + + /// + /// Translates an Any() call with a Contains predicate, e.g. r.Strings.Any(s => array.Contains(s)). + /// This checks whether any element in the array field is contained in the given values. + /// + private Filter TranslateAny(Expression source, LambdaExpression lambda) + { + // We only support the pattern: r.ArrayField.Any(x => values.Contains(x)) + if (!TryBindProperty(source, out var property) + || lambda.Body is not MethodCallExpression containsCall + || !TryMatchContains(containsCall, out var valuesExpression, out var itemExpression)) + { + throw new NotSupportedException("Unsupported method call: Enumerable.Any"); + } + + // Verify that the item is the lambda parameter + if (itemExpression != lambda.Parameters[0]) + { + throw new NotSupportedException("Unsupported method call: Enumerable.Any"); + } + + // Extract the values + IEnumerable values = valuesExpression switch + { + NewArrayExpression newArray => ExtractArrayValues(newArray), + ConstantExpression { Value: IEnumerable enumerable and not string } => enumerable, + _ => throw new NotSupportedException("Unsupported method call: Enumerable.Any") + }; + + // Generate a Match condition with RepeatedStrings or RepeatedIntegers + // This works because in Qdrant, matching against an array field with multiple values + // returns records where ANY element matches ANY of the values + var elementType = property.Type.IsArray + ? property.Type.GetElementType() + : property.Type.IsGenericType && property.Type.GetGenericTypeDefinition() == typeof(List<>) + ? property.Type.GetGenericArguments()[0] + : null; + + switch (elementType) + { + case var t when t == typeof(string): + var strings = new RepeatedStrings(); + + foreach (var value in values) + { + strings.Strings.Add(value is string or null + ? (string?)value + : throw new ArgumentException("Non-string element in string Any array")); + } + + return new Filter { Must = { new Condition { Field = new FieldCondition { Key = property.StorageName, Match = new Match { Keywords = strings } } } } }; + + case var t when t == typeof(int): + var ints = new RepeatedIntegers(); + + foreach (var value in values) + { + ints.Integers.Add(value is int intValue + ? intValue + : throw new ArgumentException("Non-int element in int Any array")); + } + + return new Filter { Must = { new Condition { Field = new FieldCondition { Key = property.StorageName, Match = new Match { Integers = ints } } } } }; + + default: + throw new NotSupportedException("Any with Contains only supported over array of ints or strings"); + } + + static object?[] ExtractArrayValues(NewArrayExpression newArray) + { + var result = new object?[newArray.Expressions.Count]; + for (var i = 0; i < newArray.Expressions.Count; i++) + { + if (newArray.Expressions[i] is not ConstantExpression { Value: var elementValue }) + { + throw new NotSupportedException("Invalid element in array"); + } + + result[i] = elementValue; + } + + return result; + } + } +} diff --git a/MEVD/src/Qdrant/QdrantMapper.cs b/MEVD/src/Qdrant/QdrantMapper.cs new file mode 100644 index 0000000..376998a --- /dev/null +++ b/MEVD/src/Qdrant/QdrantMapper.cs @@ -0,0 +1,168 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using Google.Protobuf.Collections; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData.ProviderServices; +using Qdrant.Client.Grpc; + +namespace CommunityToolkit.VectorData.Qdrant; + +/// +/// Mapper between a Qdrant record and the consumer data model that uses json as an intermediary to allow supporting a wide range of models. +/// +/// The consumer data model to map to or from. +internal sealed class QdrantMapper(CollectionModel model, bool hasNamedVectors) + where TRecord : class +{ + /// + public PointStruct MapFromDataToStorageModel(TRecord dataModel, int recordIndex, GeneratedEmbeddings>?[]? generatedEmbeddings) + { + var keyProperty = model.KeyProperty; + + var pointId = keyProperty.Type switch + { + var t when t == typeof(ulong) => new PointId + { + Num = (ulong?)keyProperty.GetValueAsObject(dataModel!) ?? throw new InvalidOperationException($"Missing key property '{keyProperty.ModelName}' on provided record of type '{typeof(TRecord).Name}'.") + }, + + var t when t == typeof(Guid) => new PointId + { + Uuid = ((Guid?)keyProperty.GetValueAsObject(dataModel!))?.ToString("D") ?? throw new InvalidOperationException($"Missing key property '{keyProperty.ModelName}' on provided record of type '{typeof(TRecord).Name}'.") + }, + _ => throw new InvalidOperationException($"Unsupported key type '{keyProperty.Type.Name}' for key property '{keyProperty.ModelName}' on provided record of type '{typeof(TRecord).Name}'.") + }; + + // Create point. + var pointStruct = new PointStruct + { + Id = pointId, + Vectors = new Vectors(), + Payload = { }, + }; + + // Add point payload. + foreach (var property in model.DataProperties) + { + var propertyValue = property.GetValueAsObject(dataModel!); + pointStruct.Payload.Add(property.StorageName, QdrantFieldMapping.ConvertToGrpcFieldValue(propertyValue)); + } + + // Add vectors. + if (hasNamedVectors) + { + var namedVectors = new NamedVectors(); + + for (var i = 0; i < model.VectorProperties.Count; i++) + { + var property = model.VectorProperties[i]; + + namedVectors.Vectors.Add( + property.StorageName, + GetVector( + property, + generatedEmbeddings?[i] is GeneratedEmbeddings> e + ? e[recordIndex] + : property.GetValueAsObject(dataModel!))); + } + + pointStruct.Vectors.Vectors_ = namedVectors; + } + else + { + // We already verified in the constructor via FindProperties that there is exactly one vector property when not using named vectors. + Debug.Assert( + generatedEmbeddings is null || generatedEmbeddings.Length == 1 && generatedEmbeddings[0] is not null, + "There should be exactly one generated embedding when not using named vectors (single vector property)."); + pointStruct.Vectors.Vector = GetVector( + model.VectorProperty, + generatedEmbeddings is null + ? model.VectorProperty.GetValueAsObject(dataModel!) + : generatedEmbeddings[0]![recordIndex]); + } + + return pointStruct; + + Vector GetVector(PropertyModel property, object? embedding) + => embedding switch + { + ReadOnlyMemory m => m.ToArray(), + Embedding e => e.Vector.ToArray(), + float[] a => a, + + null => throw new InvalidOperationException($"Vector property '{property.ModelName}' on provided record of type '{typeof(TRecord).Name}' may not be null when not using named vectors."), + var unknownEmbedding => throw new InvalidOperationException($"Vector property '{property.ModelName}' on provided record of type '{typeof(TRecord).Name}' has unsupported embedding type '{unknownEmbedding.GetType().Name}'.") + }; + } + + /// + public TRecord MapFromStorageToDataModel(PointId pointId, MapField payload, VectorsOutput vectorsOutput, bool includeVectors) + { + var outputRecord = model.CreateRecord()!; + + // TODO: Set the following generically to avoid boxing + model.KeyProperty.SetValueAsObject(outputRecord, pointId switch + { + { HasNum: true } => pointId.Num, + { HasUuid: true } => Guid.Parse(pointId.Uuid), + _ => throw new UnreachableException() + }); + + // Set each vector property if embeddings are included in the point. + if (includeVectors) + { + if (hasNamedVectors) + { + var storageVectors = vectorsOutput.Vectors.Vectors; + + foreach (var vectorProperty in model.VectorProperties) + { + PopulateVectorProperty(outputRecord, storageVectors[vectorProperty.StorageName], vectorProperty); + } + } + else + { + PopulateVectorProperty(outputRecord, vectorsOutput.Vector, model.VectorProperty); + } + + static void PopulateVectorProperty(TRecord record, VectorOutput value, VectorPropertyModel property) + { + RepeatedField data = value switch + { + // qdrant 1.16 and newer, return the new union type + { Dense: not null } => value.Dense.Data, + // Required for qdrant < 1.16.0, but deprecated in client >=1.16.0 and is empty with qdrant server 1.17.0 +#pragma warning disable CS0612 // Type or member is obsolete + { Data: not null } => value.Data, +#pragma warning restore CS0612 + _ => throw new UnreachableException() + + }; + property.SetValueAsObject( + record, + (Nullable.GetUnderlyingType(property.Type) ?? property.Type) switch + { + var t when t == typeof(ReadOnlyMemory) => new ReadOnlyMemory(data.ToArray()), + var t when t == typeof(Embedding) => new Embedding(data.ToArray()), + var t when t == typeof(float[]) => data.ToArray(), + + _ => throw new UnreachableException() + }); + } + } + + foreach (var dataProperty in model.DataProperties) + { + if (payload.TryGetValue(dataProperty.StorageName, out var fieldValue)) + { + dataProperty.SetValueAsObject( + outputRecord, + QdrantFieldMapping.Deserialize(fieldValue, dataProperty.Type)); + } + } + + return outputRecord; + } +} diff --git a/MEVD/src/Qdrant/QdrantModelBuilder.cs b/MEVD/src/Qdrant/QdrantModelBuilder.cs new file mode 100644 index 0000000..cfa4e1e --- /dev/null +++ b/MEVD/src/Qdrant/QdrantModelBuilder.cs @@ -0,0 +1,77 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData.ProviderServices; + +namespace CommunityToolkit.VectorData.Qdrant; + +internal class QdrantModelBuilder(bool hasNamedVectors) : CollectionModelBuilder(GetModelBuildOptions(hasNamedVectors)) +{ + internal const string SupportedVectorTypes = "ReadOnlyMemory, Embedding, float[]"; + + private static CollectionModelBuildingOptions GetModelBuildOptions(bool hasNamedVectors) + => new() + { + RequiresAtLeastOneVector = !hasNamedVectors, + SupportsMultipleVectors = hasNamedVectors, + }; + + protected override void ValidateKeyProperty(KeyPropertyModel keyProperty) + { + base.ValidateKeyProperty(keyProperty); + + var type = keyProperty.Type; + + if (type != typeof(ulong) && type != typeof(Guid)) + { + throw new NotSupportedException( + $"Property '{keyProperty.ModelName}' has unsupported type '{type.Name}'. Key properties must be either ulong or Guid."); + } + } + + protected override bool IsDataPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes) + { + supportedTypes = "string, int, long, double, float, bool, DateTime, DateTimeOffset," +#if NET + + " DateOnly," +#endif + + " or arrays/lists of these types"; + + if (Nullable.GetUnderlyingType(type) is Type underlyingType) + { + type = underlyingType; + } + + return IsValid(type) + || (type.IsArray && IsValid(type.GetElementType()!)) + || (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(List<>) && IsValid(type.GenericTypeArguments[0])); + + static bool IsValid(Type type) + => type == typeof(string) || + type == typeof(int) || + type == typeof(long) || + type == typeof(double) || + type == typeof(float) || + type == typeof(bool) || + type == typeof(DateTime) || +#if NET + type == typeof(DateOnly) || +#endif + type == typeof(DateTimeOffset); + } + + protected override bool IsVectorPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes) + => IsVectorPropertyTypeValidCore(type, out supportedTypes); + + internal static bool IsVectorPropertyTypeValidCore(Type type, [NotNullWhen(false)] out string? supportedTypes) + { + supportedTypes = "ReadOnlyMemory, Embedding, or float[]"; + + return type == typeof(ReadOnlyMemory) + || type == typeof(ReadOnlyMemory?) + || type == typeof(Embedding) + || type == typeof(float[]); + } +} diff --git a/MEVD/src/Qdrant/QdrantServiceCollectionExtensions.cs b/MEVD/src/Qdrant/QdrantServiceCollectionExtensions.cs new file mode 100644 index 0000000..41322fa --- /dev/null +++ b/MEVD/src/Qdrant/QdrantServiceCollectionExtensions.cs @@ -0,0 +1,267 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData; +using CommunityToolkit.VectorData.Qdrant; +using Qdrant.Client; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.DependencyInjection; + +/// +/// Extension methods to register and instances on an . +/// +public static class QdrantServiceCollectionExtensions +{ + private const string DynamicCodeMessage = "This method is incompatible with NativeAOT, consult the documentation for adding collections in a way that's compatible with NativeAOT."; + private const string UnreferencedCodeMessage = "This method is incompatible with trimming, consult the documentation for adding collections in a way that's compatible with NativeAOT."; + + /// + /// Registers a as + /// with returned by + /// or retrieved from the dependency injection container if was not provided. + /// + /// + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddQdrantVectorStore( + this IServiceCollection services, + Func? clientProvider = default, + Func? optionsProvider = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + => AddKeyedQdrantVectorStore(services, serviceKey: null, clientProvider, optionsProvider, lifetime); + + /// + /// Registers a keyed as + /// with returned by or retrieved from the dependency injection container if was not provided. + /// + /// The to register the on. + /// The key with which to associate the vector store. + /// The provider. + /// Options provider to further configure the . + /// The service lifetime for the store. Defaults to . + /// Service collection. + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddKeyedQdrantVectorStore( + this IServiceCollection services, + object? serviceKey, + Func? clientProvider = default, + Func? optionsProvider = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + { + Throw.IfNull(services); + + services.Add(new ServiceDescriptor(typeof(QdrantVectorStore), serviceKey, (sp, _) => + { + var client = clientProvider is null ? sp.GetRequiredService() : clientProvider(sp); + var options = GetStoreOptions(sp, optionsProvider); + + // The client was restored from the DI container, so we do not own it. + return new QdrantVectorStore(client, ownsClient: false, options); + }, lifetime)); + + services.Add(new ServiceDescriptor(typeof(VectorStore), serviceKey, + static (sp, key) => sp.GetRequiredKeyedService(key), lifetime)); + + return services; + } + + /// + /// Registers a as + /// with created with , , + /// and . + /// + /// + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddQdrantVectorStore( + this IServiceCollection services, + string host, + int port = 6334, + bool https = true, + string? apiKey = default, + QdrantVectorStoreOptions? options = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + => AddKeyedQdrantVectorStore(services, serviceKey: null, host, port, https, apiKey, options, lifetime); + + /// + /// Registers a keyed as + /// with created with , , + /// and . + /// + /// The to register the on. + /// The key with which to associate the vector store. + /// The host to connect to. + /// The port to connect to. Defaults to 6334. + /// Whether to encrypt the connection using HTTPS. Defaults to true. + /// The API key to use. + /// Options to further configure the . + /// The service lifetime for the store. Defaults to . + /// Service collection. + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddKeyedQdrantVectorStore( + this IServiceCollection services, + object? serviceKey, + string host, + int port = 6334, + bool https = true, + string? apiKey = default, + QdrantVectorStoreOptions? options = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + { + Throw.IfNullOrWhitespace(host); + + return AddKeyedQdrantVectorStore(services, serviceKey, _ => new QdrantClient(host, port, https, apiKey), sp => options!, lifetime); + } + + /// + /// Registers a as + /// with returned by or retrieved from the dependency injection container if was not provided. + /// + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddQdrantCollection( + this IServiceCollection services, + string name, + Func? clientProvider = default, + Func? optionsProvider = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + where TKey : notnull + where TRecord : class + => AddKeyedQdrantCollection(services, serviceKey: null, name, clientProvider, optionsProvider, lifetime); + + /// + /// Registers a keyed as + /// with returned by or retrieved from the dependency injection container if was not provided. + /// + /// The to register the on. + /// The key with which to associate the collection. + /// The name of the collection. + /// The provider. + /// Options provider to further configure the . + /// The service lifetime for the store. Defaults to . + /// Service collection. + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddKeyedQdrantCollection( + this IServiceCollection services, + object? serviceKey, + string name, + Func? clientProvider = default, + Func? optionsProvider = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + where TKey : notnull + where TRecord : class + { + Throw.IfNull(services); + Throw.IfNullOrWhitespace(name); + + services.Add(new ServiceDescriptor(typeof(QdrantCollection), serviceKey, (sp, _) => + { + var client = clientProvider is null ? sp.GetRequiredService() : clientProvider(sp); + var options = GetCollectionOptions(sp, optionsProvider); + + // The client was restored from the DI container, so we do not own it. + return new QdrantCollection(client, name, ownsClient: false, options); + }, lifetime)); + + services.Add(new ServiceDescriptor(typeof(VectorStoreCollection), serviceKey, + static (sp, key) => sp.GetRequiredKeyedService>(key), lifetime)); + + services.Add(new ServiceDescriptor(typeof(IVectorSearchable), serviceKey, + static (sp, key) => sp.GetRequiredKeyedService>(key), lifetime)); + + services.Add(new ServiceDescriptor(typeof(IKeywordHybridSearchable), serviceKey, + static (sp, key) => sp.GetRequiredKeyedService>(key), lifetime)); + + return services; + } + + /// + /// Registers a as + /// with created with , , + /// and . + /// + /// + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddQdrantCollection( + this IServiceCollection services, + string name, + string host, + int port = 6334, + bool https = true, + string? apiKey = default, + QdrantCollectionOptions? options = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + where TKey : notnull + where TRecord : class + => AddKeyedQdrantCollection(services, serviceKey: null, name, host, port, https, apiKey, options, lifetime); + + /// + /// Registers a keyed as + /// with created with , , + /// and . + /// + /// The to register the on. + /// The key with which to associate the collection. + /// The name of the collection. + /// The host to connect to. + /// The port to connect to. Defaults to 6334. + /// Whether to encrypt the connection using HTTPS. Defaults to true. + /// The API key to use. + /// Options to further configure the . + /// The service lifetime for the store. Defaults to . + /// Service collection. + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddKeyedQdrantCollection( + this IServiceCollection services, + object? serviceKey, + string name, + string host, + int port = 6334, + bool https = true, + string? apiKey = default, + QdrantCollectionOptions? options = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + where TKey : notnull + where TRecord : class + { + Throw.IfNullOrWhitespace(host); + + return AddKeyedQdrantCollection(services, serviceKey, name, _ => new QdrantClient(host, port, https, apiKey), sp => options!, lifetime); + } + + private static QdrantVectorStoreOptions? GetStoreOptions(IServiceProvider sp, Func? optionsProvider) + { + var options = optionsProvider?.Invoke(sp); + if (options?.EmbeddingGenerator is not null) + { + return options; // The user has provided everything, there is nothing to change. + } + + var embeddingGenerator = sp.GetService(); + return embeddingGenerator is null + ? options // There is nothing to change. + : new(options) { EmbeddingGenerator = embeddingGenerator }; // Create a brand new copy in order to avoid modifying the original options. + } + + private static QdrantCollectionOptions? GetCollectionOptions(IServiceProvider sp, Func? optionsProvider) + { + var options = optionsProvider?.Invoke(sp); + if (options?.EmbeddingGenerator is not null) + { + return options; // The user has provided everything, there is nothing to change. + } + + var embeddingGenerator = sp.GetService(); + return embeddingGenerator is null + ? options // There is nothing to change. + : new(options) { EmbeddingGenerator = embeddingGenerator }; // Create a brand new copy in order to avoid modifying the original options. + } +} diff --git a/MEVD/src/Qdrant/QdrantVectorStore.cs b/MEVD/src/Qdrant/QdrantVectorStore.cs new file mode 100644 index 0000000..1b41b3f --- /dev/null +++ b/MEVD/src/Qdrant/QdrantVectorStore.cs @@ -0,0 +1,148 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; +using Grpc.Core; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; +using Qdrant.Client; +using Microsoft.Shared.Diagnostics; + +namespace CommunityToolkit.VectorData.Qdrant; + +/// +/// Class for accessing the list of collections in a Qdrant vector store. +/// +/// +/// This class can be used with collections of any schema type, but requires you to provide schema information when getting a collection. +/// +public sealed class QdrantVectorStore : VectorStore +{ + /// Metadata about vector store. + private readonly VectorStoreMetadata _metadata; + + /// Qdrant client that can be used to manage the collections and points in a Qdrant store. + private readonly MockableQdrantClient _qdrantClient; + + /// A general purpose definition that can be used to construct a collection when needing to proxy schema agnostic operations. + private static readonly VectorStoreCollectionDefinition s_generalPurposeDefinition = new() { Properties = [new VectorStoreKeyProperty("Key", typeof(ulong)), new VectorStoreVectorProperty("Vector", typeof(ReadOnlyMemory), 1)] }; + + /// Whether the vectors in the store are named and multiple vectors are supported, or whether there is just a single unnamed vector per qdrant point. + private readonly bool _hasNamedVectors; + + private readonly IEmbeddingGenerator? _embeddingGenerator; + + /// + /// Initializes a new instance of the class. + /// + /// Qdrant client that can be used to manage the collections and points in a Qdrant store. + /// A value indicating whether is disposed after the vector store is disposed. + /// Optional configuration options for this class. + public QdrantVectorStore(QdrantClient qdrantClient, bool ownsClient, QdrantVectorStoreOptions? options = default) + : this(new MockableQdrantClient(qdrantClient, ownsClient), options) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// Qdrant client that can be used to manage the collections and points in a Qdrant store. + /// Optional configuration options for this class. + internal QdrantVectorStore(MockableQdrantClient qdrantClient, QdrantVectorStoreOptions? options = default) + { + Throw.IfNull(qdrantClient); + + _qdrantClient = qdrantClient; + + options ??= QdrantVectorStoreOptions.Default; + _hasNamedVectors = options.HasNamedVectors; + _embeddingGenerator = options.EmbeddingGenerator; + + _metadata = new() + { + VectorStoreSystemName = QdrantConstants.VectorStoreSystemName + }; + } + + /// + protected override void Dispose(bool disposing) + { + _qdrantClient.Dispose(); + base.Dispose(disposing); + } + +#pragma warning disable IDE0090 // Use 'new(...)' + /// + [RequiresDynamicCode("This overload of GetCollection() is incompatible with NativeAOT. For dynamic mapping via Dictionary, call GetDynamicCollection() instead.")] + [RequiresUnreferencedCode("This overload of GetCollecttion() is incompatible with trimming. For dynamic mapping via Dictionary, call GetDynamicCollection() instead.")] +#if NET + public override QdrantCollection GetCollection(string name, VectorStoreCollectionDefinition? definition = null) +#else + public override VectorStoreCollection GetCollection(string name, VectorStoreCollectionDefinition? definition = null) +#endif + => typeof(TRecord) == typeof(Dictionary) + ? throw new ArgumentException(VectorDataStrings.GetCollectionWithDictionaryNotSupported) + : new QdrantCollection(_qdrantClient.Share, name, new() + { + HasNamedVectors = _hasNamedVectors, + Definition = definition, + EmbeddingGenerator = _embeddingGenerator + }); + + /// +#if NET + public override QdrantDynamicCollection GetDynamicCollection(string name, VectorStoreCollectionDefinition definition) +#else + public override VectorStoreCollection> GetDynamicCollection(string name, VectorStoreCollectionDefinition definition) +#endif + => new QdrantDynamicCollection(_qdrantClient.Share, name, new QdrantCollectionOptions() + { + HasNamedVectors = _hasNamedVectors, + Definition = definition, + EmbeddingGenerator = _embeddingGenerator + }); +#pragma warning restore IDE0090 + + /// + public override async IAsyncEnumerable ListCollectionNamesAsync([EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var collections = await VectorStoreErrorHandler.RunOperationAsync, RpcException>( + _metadata, + "ListCollections", + () => _qdrantClient.ListCollectionsAsync(cancellationToken)).ConfigureAwait(false); + + foreach (var collection in collections) + { + yield return collection; + } + } + + /// + public override Task CollectionExistsAsync(string name, CancellationToken cancellationToken = default) + { + var collection = GetDynamicCollection(name, s_generalPurposeDefinition); + return collection.CollectionExistsAsync(cancellationToken); + } + + /// + public override Task EnsureCollectionDeletedAsync(string name, CancellationToken cancellationToken = default) + { + var collection = GetDynamicCollection(name, s_generalPurposeDefinition); + return collection.EnsureCollectionDeletedAsync(cancellationToken); + } + + /// + public override object? GetService(Type serviceType, object? serviceKey = null) + { + Throw.IfNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreMetadata) ? _metadata : + serviceType == typeof(QdrantClient) ? _qdrantClient.QdrantClient : + serviceType.IsInstanceOfType(this) ? this : + null; + } +} diff --git a/MEVD/src/Qdrant/QdrantVectorStoreOptions.cs b/MEVD/src/Qdrant/QdrantVectorStoreOptions.cs new file mode 100644 index 0000000..9ab0510 --- /dev/null +++ b/MEVD/src/Qdrant/QdrantVectorStoreOptions.cs @@ -0,0 +1,38 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.AI; + +namespace CommunityToolkit.VectorData.Qdrant; + +/// +/// Options when creating a . +/// +public sealed class QdrantVectorStoreOptions +{ + internal static readonly QdrantVectorStoreOptions Default = new(); + + /// + /// Initializes a new instance of the class. + /// + public QdrantVectorStoreOptions() + { + } + + internal QdrantVectorStoreOptions(QdrantVectorStoreOptions? source) + { + HasNamedVectors = source?.HasNamedVectors ?? Default.HasNamedVectors; + EmbeddingGenerator = source?.EmbeddingGenerator; + } + + /// + /// Gets or sets a value indicating whether the vectors in the store are named and multiple vectors are supported, or whether there is just a single unnamed vector per qdrant point. + /// Defaults to single vector per point. + /// + public bool HasNamedVectors { get; set; } + + /// + /// Gets or sets the default embedding generator to use when generating vectors embeddings with this vector store. + /// + public IEmbeddingGenerator? EmbeddingGenerator { get; set; } +} diff --git a/MEVD/src/Redis/AssemblyInfo.cs b/MEVD/src/Redis/AssemblyInfo.cs new file mode 100644 index 0000000..d6395e8 --- /dev/null +++ b/MEVD/src/Redis/AssemblyInfo.cs @@ -0,0 +1,2 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. diff --git a/MEVD/src/Redis/IRedisJsonMapper.cs b/MEVD/src/Redis/IRedisJsonMapper.cs new file mode 100644 index 0000000..bb13f89 --- /dev/null +++ b/MEVD/src/Redis/IRedisJsonMapper.cs @@ -0,0 +1,20 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text.Json.Nodes; +using Microsoft.Extensions.AI; + +namespace CommunityToolkit.VectorData.Redis; + +internal interface IRedisJsonMapper +{ + /// + /// Maps from the consumer record data model to the storage model. + /// + (string Key, JsonNode Node) MapFromDataToStorageModel(TRecord dataModel, int recordIndex, IReadOnlyList?[]? generatedEmbeddings); + + /// + /// Maps from the storage model to the consumer record data model. + /// + TRecord MapFromStorageToDataModel((string Key, JsonNode Node) storageModel, bool includeVectors); +} diff --git a/MEVD/src/Redis/README.md b/MEVD/src/Redis/README.md new file mode 100644 index 0000000..c0feab4 --- /dev/null +++ b/MEVD/src/Redis/README.md @@ -0,0 +1,27 @@ +# Microsoft.SemanticKernel.Connectors.Redis + +This connector uses Redis to implement Semantic Memory. It requires the [RediSearch](https://redis.io/docs/latest/develop/interact/search-and-query/advanced-concepts/vectors/) module to be enabled on Redis to implement vector similarity search. + +## What is RediSearch? + +[RediSearch](https://redis.io/docs/latest/develop/interact/search-and-query/advanced-concepts/vectors/) is a source-available Redis module that enables querying, secondary indexing, and full-text search for Redis. These features enable multi-field queries, aggregation, exact phrase matching, numeric filtering, geo filtering and vector similarity semantic search on top of text queries. + +Ways to get RediSearch: + +1. You can create an [Azure Cache for Redis Enterpise instance](https://learn.microsoft.com/azure/azure-cache-for-redis/quickstart-create-redis-enterprise) and [enable RediSearch module](https://learn.microsoft.com/azure/azure-cache-for-redis/cache-redis-modules). + +1. Set up the RediSearch on your self-managed Redis, please refer to its [documentation](https://redis.io/docs/latest/develop/interact/search-and-query/advanced-concepts/vectors/). + +1. Use the [Redis Enterprise](https://redis.io/docs/latest/operate/rs/), see [Azure Marketplace](https://azuremarketplace.microsoft.com/en-us/marketplace/apps/garantiadata.redis_enterprise_1sp_public_preview?tab=Overview), [AWS Marketplace](https://aws.amazon.com/marketplace/pp/prodview-e6y7ork67pjwg?sr=0-2&ref_=beagle&applicationId=AWSMPContessa), or [Google Marketplace](https://console.cloud.google.com/marketplace/details/redislabs-public/redis-enterprise?pli=1). + +## Quick start + +1. Run with Docker: + +```bash +docker run -d --name redis-stack-server -p 6379:6379 redis/redis-stack-server:latest +``` + +2. Create a Redis Vector Store using instructions on the [Microsoft Learn site](https://learn.microsoft.com/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/redis-connector). + +3. Use the [getting started instructions](https://learn.microsoft.com/semantic-kernel/concepts/vector-store-connectors/?pivots=programming-language-csharp#getting-started-with-vector-store-connectors) on the Microsoft Leearn site to learn more about using the vector store. diff --git a/MEVD/src/Redis/Redis.csproj b/MEVD/src/Redis/Redis.csproj new file mode 100644 index 0000000..ad7ff16 --- /dev/null +++ b/MEVD/src/Redis/Redis.csproj @@ -0,0 +1,24 @@ + + + + 1.0.0-preview.1 + CommunityToolkit.VectorData.Redis + $(AssemblyName) + net10.0;net8.0;netstandard2.0;net462 + + Redis provider for Microsoft.Extensions.VectorData + Redis provider for Microsoft.Extensions.VectorData by Semantic Kernel + + + + + + + + + + + + + + \ No newline at end of file diff --git a/MEVD/src/Redis/RedisCollectionCreateMapping.cs b/MEVD/src/Redis/RedisCollectionCreateMapping.cs new file mode 100644 index 0000000..ebf3e19 --- /dev/null +++ b/MEVD/src/Redis/RedisCollectionCreateMapping.cs @@ -0,0 +1,186 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using System.Globalization; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; +using NRedisStack.Search; + +namespace CommunityToolkit.VectorData.Redis; + +/// +/// Contains mapping helpers to use when creating a redis vector collection. +/// +internal static class RedisCollectionCreateMapping +{ + /// A set of number types that are supported for filtering. + public static readonly HashSet s_supportedFilterableNumericDataTypes = + [ + typeof(short), + typeof(sbyte), + typeof(byte), + typeof(ushort), + typeof(int), + typeof(uint), + typeof(long), + typeof(ulong), + typeof(float), + typeof(double), + typeof(decimal), + + typeof(short?), + typeof(sbyte?), + typeof(byte?), + typeof(ushort?), + typeof(int?), + typeof(uint?), + typeof(long?), + typeof(ulong?), + typeof(float?), + typeof(double?), + typeof(decimal?), + ]; + + /// + /// Map from the given list of items to the Redis . + /// + /// The property definitions to map from. + /// A value indicating whether to include $. prefix for field names as required in JSON mode. + /// The mapped Redis . + /// Thrown if there are missing required or unsupported configuration options set. + public static Schema MapToSchema(IEnumerable properties, bool useDollarPrefix) + { + var schema = new Schema(); + var fieldNamePrefix = useDollarPrefix ? "$." : string.Empty; + + // Loop through all properties and create the index fields. + foreach (var property in properties) + { + var storageName = property.StorageName; + + switch (property) + { + case KeyPropertyModel keyProperty: + // Do nothing, since key is not stored as part of the payload and therefore doesn't have to be added to the index. + continue; + + case DataPropertyModel dataProperty when dataProperty.IsIndexed || dataProperty.IsFullTextIndexed: + if (dataProperty.IsIndexed && dataProperty.IsFullTextIndexed) + { + throw new InvalidOperationException($"Property '{dataProperty.ModelName}' has both {nameof(VectorStoreDataProperty.IsIndexed)} and {nameof(VectorStoreDataProperty.IsFullTextIndexed)} set to true, and this is not supported by the Redis VectorStore."); + } + + // Add full text search field index. + if (dataProperty.IsFullTextIndexed) + { + if (dataProperty.Type == typeof(string) || IsTagsType(dataProperty.Type)) + { + schema.AddTextField(new FieldName($"{fieldNamePrefix}{storageName}", storageName)); + } + else + { + throw new InvalidOperationException($"Property {nameof(dataProperty.IsFullTextIndexed)} on {nameof(VectorStoreDataProperty)} '{dataProperty.ModelName}' is set to true, but the property type is not a string or IEnumerable. The Redis VectorStore supports {nameof(dataProperty.IsFullTextIndexed)} on string or IEnumerable properties only."); + } + } + + // Add filter field index. + if (dataProperty.IsIndexed) + { + if (dataProperty.Type == typeof(string)) + { + schema.AddTagField(new FieldName($"{fieldNamePrefix}{storageName}", storageName)); + } + else if (IsTagsType(dataProperty.Type)) + { + schema.AddTagField(new FieldName($"{fieldNamePrefix}{storageName}.*", storageName)); + } + else if (RedisCollectionCreateMapping.s_supportedFilterableNumericDataTypes.Contains(dataProperty.Type)) + { + schema.AddNumericField(new FieldName($"{fieldNamePrefix}{storageName}", storageName)); + } + else + { + throw new InvalidOperationException($"Property '{dataProperty.ModelName}' is marked as {nameof(VectorStoreDataProperty.IsIndexed)}, but the property type '{dataProperty.Type}' is not supported. Only string, IEnumerable and numeric properties are supported for filtering by the Redis VectorStore."); + } + } + + continue; + + case VectorPropertyModel vectorProperty: + var indexKind = GetSDKIndexKind(vectorProperty); + var vectorType = GetSDKVectorType(vectorProperty); + var dimensions = vectorProperty.Dimensions.ToString(CultureInfo.InvariantCulture); + var distanceAlgorithm = GetSDKDistanceAlgorithm(vectorProperty); + schema.AddVectorField(new FieldName($"{fieldNamePrefix}{storageName}", storageName), indexKind, new Dictionary() + { + ["TYPE"] = vectorType, + ["DIM"] = dimensions, + ["DISTANCE_METRIC"] = distanceAlgorithm + }); + continue; + } + } + + return schema; + + static bool IsTagsType(Type type) + => (type.IsArray && type.GetElementType() == typeof(string)) + || (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(List<>) && type.GenericTypeArguments[0] == typeof(string)) + || (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(HashSet<>) && type.GenericTypeArguments[0] == typeof(string)); + } + + /// + /// Get the configured from the given . + /// If none is configured the default is . + /// + /// The vector property definition. + /// The chosen . + /// Thrown if a index type was chosen that isn't supported by Redis. + public static Schema.VectorField.VectorAlgo GetSDKIndexKind(VectorPropertyModel vectorProperty) + => vectorProperty.IndexKind switch + { + IndexKind.Hnsw or null => Schema.VectorField.VectorAlgo.HNSW, + IndexKind.Flat => Schema.VectorField.VectorAlgo.FLAT, + _ => throw new InvalidOperationException($"Index kind '{vectorProperty.IndexKind}' for {nameof(VectorStoreVectorProperty)} '{vectorProperty.ModelName}' is not supported by the Redis VectorStore.") + }; + + /// + /// Get the configured distance metric from the given . + /// If none is configured, the default is cosine. + /// + /// The vector property definition. + /// The chosen distance metric. + /// Thrown if a distance function is chosen that isn't supported by Redis. + public static string GetSDKDistanceAlgorithm(VectorPropertyModel vectorProperty) + => vectorProperty.DistanceFunction switch + { + DistanceFunction.CosineSimilarity or null => "COSINE", + DistanceFunction.CosineDistance => "COSINE", + DistanceFunction.DotProductSimilarity => "IP", + DistanceFunction.EuclideanSquaredDistance => "L2", + + _ => throw new NotSupportedException($"Distance function '{vectorProperty.DistanceFunction}' for {nameof(VectorStoreVectorProperty)} '{vectorProperty.ModelName}' is not supported by the Redis VectorStore.") + }; + + /// + /// Get the vector type to pass to the SDK based on the data type of the vector property. + /// + /// The vector property definition. + /// The SDK required vector type. + /// Thrown if the property data type is not supported by the connector. + public static string GetSDKVectorType(VectorPropertyModel vectorProperty) + => (Nullable.GetUnderlyingType(vectorProperty.EmbeddingType) ?? vectorProperty.EmbeddingType) switch + { + Type t when t == typeof(ReadOnlyMemory) => "FLOAT32", + Type t when t == typeof(Embedding) => "FLOAT32", + Type t when t == typeof(float[]) => "FLOAT32", + Type t when t == typeof(ReadOnlyMemory) => "FLOAT64", + Type t when t == typeof(Embedding) => "FLOAT64", + Type t when t == typeof(double[]) => "FLOAT64", + + null => throw new UnreachableException("null embedding type"), + _ => throw new InvalidOperationException($"Vector data type '{vectorProperty.Type.Name}' for {nameof(VectorStoreVectorProperty)} '{vectorProperty.ModelName}' is not supported by the Redis VectorStore.") + }; +} diff --git a/MEVD/src/Redis/RedisCollectionSearchMapping.cs b/MEVD/src/Redis/RedisCollectionSearchMapping.cs new file mode 100644 index 0000000..60e1a3d --- /dev/null +++ b/MEVD/src/Redis/RedisCollectionSearchMapping.cs @@ -0,0 +1,131 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Linq.Expressions; +using System.Runtime.InteropServices; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; +using NRedisStack.Search; + +namespace CommunityToolkit.VectorData.Redis; + +/// +/// Contains mapping helpers to use when searching in a redis vector collection. +/// +internal static class RedisCollectionSearchMapping +{ + /// + /// Validate that the given vector is one of the types supported by the Redis connector and convert it to a byte array. + /// + /// The vector type. + /// The vector to validate and convert. + /// The type of connector, HashSet or JSON, to use for error reporting. + /// The vector converted to a byte array. + /// Thrown if the vector type is not supported. + public static byte[] ValidateVectorAndConvertToBytes(TVector vector, string connectorTypeName) + => vector switch + { + ReadOnlyMemory m => MemoryMarshal.AsBytes(m.Span).ToArray(), + Embedding e => MemoryMarshal.AsBytes(e.Vector.Span).ToArray(), + float[] a => MemoryMarshal.AsBytes(a.AsSpan()).ToArray(), + + ReadOnlyMemory m => MemoryMarshal.AsBytes(m.Span).ToArray(), + Embedding e => MemoryMarshal.AsBytes(e.Vector.Span).ToArray(), + double[] a => MemoryMarshal.AsBytes(a.AsSpan()).ToArray(), + + _ => throw new NotSupportedException($"The provided vector type {vector?.GetType().FullName} is not supported by the Redis {connectorTypeName} connector.") + }; + + /// + /// Build a Redis object from the given vector and options. + /// + /// The vector to search the database with as a byte array. + /// The maximum number of elements to return. + /// The options to configure the behavior of the search. + /// The model. + /// The vector property. + /// The set of fields to limit the results to. Null for all. + /// The . + public static Query BuildQuery(byte[] vectorBytes, int top, VectorSearchOptions options, CollectionModel model, VectorPropertyModel vectorProperty, string[]? selectFields) + { + // Build search query. + var redisLimit = top + options.Skip; + + var filter = options.Filter is not null + ? new RedisFilterTranslator().Translate(options.Filter, model) + : "*"; + + var query = new Query($"{filter}=>[KNN {redisLimit} @{vectorProperty.StorageName} $embedding AS vector_score]") + .AddParam("embedding", vectorBytes) + .SetSortBy("vector_score") + .Limit(options.Skip, redisLimit) + .SetWithScores(true) + .Dialect(2); + + if (selectFields != null) + { + query.ReturnFields(selectFields); + } + + return query; + } + + internal static Query BuildQuery(Expression> filter, int top, FilteredRecordRetrievalOptions options, CollectionModel model) + { + var translatedFilter = new RedisFilterTranslator().Translate(filter, model); + Query query = new Query(translatedFilter) + .Limit(options.Skip, top) + .Dialect(2); + + var orderByValues = options.OrderBy?.Invoke(new()).Values; + var sortInfo = orderByValues switch + { + null => null, + _ when orderByValues.Count == 1 => orderByValues[0], + _ => throw new NotSupportedException("Redis does not support ordering by more than one property.") + }; + + if (sortInfo is not null) + { + string storageName = model.GetDataOrKeyProperty(sortInfo.PropertySelector).StorageName; + query = query.SetSortBy(field: storageName, ascending: sortInfo.Ascending); + } + + return query; + } + + /// + /// Resolve the distance function to use for a search by checking the distance function of the vector property specified in options + /// or by falling back to the distance function of the first vector property, or by falling back to the default distance function. + /// + /// The vector property to be used. + /// The distance function for the vector we want to search. + public static string ResolveDistanceFunction(VectorPropertyModel vectorProperty) + => vectorProperty.DistanceFunction ?? DistanceFunction.CosineSimilarity; + + /// + /// Convert the score from redis into the appropriate output score based on the distance function. + /// Redis doesn't support Cosine Similarity, so we need to convert from distance to similarity if it was chosen. + /// + /// The redis score to convert. + /// The distance function used in the search. + /// The converted score. + /// Thrown if the provided distance function is not supported by redis. + public static float? GetOutputScoreFromRedisScore(float? redisScore, string distanceFunction) + { + if (redisScore is null) + { + return null; + } + + return distanceFunction switch + { + DistanceFunction.CosineSimilarity => 1 - redisScore, + DistanceFunction.CosineDistance => redisScore, + DistanceFunction.DotProductSimilarity => redisScore, + DistanceFunction.EuclideanSquaredDistance => redisScore, + _ => throw new InvalidOperationException($"The distance function '{distanceFunction}' is not supported."), + }; + } +} diff --git a/MEVD/src/Redis/RedisConstants.cs b/MEVD/src/Redis/RedisConstants.cs new file mode 100644 index 0000000..98e8a83 --- /dev/null +++ b/MEVD/src/Redis/RedisConstants.cs @@ -0,0 +1,9 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace CommunityToolkit.VectorData.Redis; + +internal static class RedisConstants +{ + internal const string VectorStoreSystemName = "redis"; +} diff --git a/MEVD/src/Redis/RedisFieldMapping.cs b/MEVD/src/Redis/RedisFieldMapping.cs new file mode 100644 index 0000000..9827e41 --- /dev/null +++ b/MEVD/src/Redis/RedisFieldMapping.cs @@ -0,0 +1,82 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using System.Runtime.InteropServices; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData.ProviderServices; + +namespace CommunityToolkit.VectorData.Redis; + +/// +/// Contains helper methods for mapping fields to and from the format required by the Redis client sdk. +/// +internal static class RedisFieldMapping +{ + /// + /// Convert a vector to a byte array as required by the Redis client sdk when using hashsets. + /// + /// The vector to convert. + /// The byte array. + public static byte[] ConvertVectorToBytes(ReadOnlySpan vector) + { + return MemoryMarshal.AsBytes(vector).ToArray(); + } + + /// + /// Convert a vector to a byte array as required by the Redis client sdk when using hashsets. + /// + /// The vector to convert. + /// The byte array. + public static byte[] ConvertVectorToBytes(ReadOnlySpan vector) + { + return MemoryMarshal.AsBytes(vector).ToArray(); + } + + internal static async ValueTask<(IEnumerable records, IReadOnlyList?[]?)> ProcessEmbeddingsAsync( + CollectionModel model, + IEnumerable records, + CancellationToken cancellationToken) + where TRecord : class + { + IReadOnlyList? recordsList = null; + + // If an embedding generator is defined, invoke it once per property for all records. + IReadOnlyList?[]? generatedEmbeddings = null; + + var vectorPropertyCount = model.VectorProperties.Count; + for (var i = 0; i < vectorPropertyCount; i++) + { + var vectorProperty = model.VectorProperties[i]; + + if (RedisModelBuilder.IsVectorPropertyTypeValidCore(vectorProperty.Type, out _)) + { + continue; + } + + // We have a vector property whose type isn't natively supported - we need to generate embeddings. + Debug.Assert(vectorProperty.EmbeddingGenerator is not null); + + // We have a property with embedding generation; materialize the records' enumerable if needed, to + // prevent multiple enumeration. + if (recordsList is null) + { + recordsList = records is IReadOnlyList r ? r : records.ToList(); + + if (recordsList.Count == 0) + { + return (records, null); + } + + records = recordsList; + } + + // TODO: Ideally we'd group together vector properties using the same generator (and with the same input and output properties), + // and generate embeddings for them in a single batch. That's some more complexity though. + generatedEmbeddings ??= new IReadOnlyList?[vectorPropertyCount]; + generatedEmbeddings[i] = await vectorProperty.GenerateEmbeddingsAsync(records.Select(r => vectorProperty.GetValueAsObject(r)), cancellationToken).ConfigureAwait(false); + } + + return (records, generatedEmbeddings); + } +} diff --git a/MEVD/src/Redis/RedisFilterTranslator.cs b/MEVD/src/Redis/RedisFilterTranslator.cs new file mode 100644 index 0000000..e80eb25 --- /dev/null +++ b/MEVD/src/Redis/RedisFilterTranslator.cs @@ -0,0 +1,262 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections; +using System.Linq.Expressions; +using System.Text; +using Microsoft.Extensions.VectorData.ProviderServices; +using Microsoft.Extensions.VectorData.ProviderServices.Filter; + +namespace CommunityToolkit.VectorData.Redis; + +internal class RedisFilterTranslator : FilterTranslatorBase +{ + private readonly StringBuilder _filter = new(); + + internal string Translate(LambdaExpression lambdaExpression, CollectionModel model) + { + // Redis doesn't seem to have a native way of expressing "always true" filters; since this scenario is important for fetching + // all records (via GetAsync with filter), we special-case and support it here. Note that false isn't supported (useless), + // nor is 'x && true'. + if (lambdaExpression.Body is ConstantExpression { Value: true }) + { + return "*"; + } + + var preprocessedExpression = PreprocessFilter(lambdaExpression, model, new FilterPreprocessingOptions()); + + Translate(preprocessedExpression); + return _filter.ToString(); + } + + private void Translate(Expression? node) + { + switch (node) + { + case BinaryExpression + { + NodeType: ExpressionType.Equal or ExpressionType.NotEqual + or ExpressionType.GreaterThan or ExpressionType.GreaterThanOrEqual + or ExpressionType.LessThan or ExpressionType.LessThanOrEqual + } binary: + TranslateEqualityComparison(binary); + return; + + case BinaryExpression { NodeType: ExpressionType.AndAlso } andAlso: + // https://redis.io/docs/latest/develop/interact/search-and-query/query/combined/#and + _filter.Append('('); + Translate(andAlso.Left); + _filter.Append(' '); + Translate(andAlso.Right); + _filter.Append(')'); + return; + + case BinaryExpression { NodeType: ExpressionType.OrElse } orElse: + // https://redis.io/docs/latest/develop/interact/search-and-query/query/combined/#or + _filter.Append('('); + Translate(orElse.Left); + _filter.Append(" | "); + Translate(orElse.Right); + _filter.Append(')'); + return; + + case UnaryExpression { NodeType: ExpressionType.Not } not: + TranslateNot(not.Operand); + return; + + // Handle converting non-nullable to nullable; such nodes are found in e.g. r => r.Int == nullableInt + case UnaryExpression { NodeType: ExpressionType.Convert } convert when Nullable.GetUnderlyingType(convert.Type) == convert.Operand.Type: + Translate(convert.Operand); + return; + + // MemberExpression is generally handled within e.g. TranslateEqual; this is used to translate direct bool inside filter (e.g. Filter => r => r.Bool) + case MemberExpression member when member.Type == typeof(bool) && TryBindProperty(member, out _): + { + TranslateEqualityComparison(Expression.Equal(member, Expression.Constant(true))); + return; + } + + case MethodCallExpression methodCall: + TranslateMethodCall(methodCall); + return; + + default: + throw new NotSupportedException("Redis does not support the following NodeType in filters: " + node?.NodeType); + } + } + + private void TranslateEqualityComparison(BinaryExpression binary) + { + if (!TryProcessEqualityComparison(binary.Left, binary.Right) && !TryProcessEqualityComparison(binary.Right, binary.Left)) + { + throw new NotSupportedException("Binary expression not supported by Redis"); + } + + bool TryProcessEqualityComparison(Expression first, Expression second) + { + if (TryBindProperty(first, out var property) && second is ConstantExpression { Value: var constantValue }) + { + // Numeric negation has a special syntax (!=), for the rest we nest in a NOT + if (binary.NodeType is ExpressionType.NotEqual && constantValue is not (int or long or float or double)) + { + TranslateNot(Expression.Equal(first, second)); + return true; + } + + // Redis field names cannot be escaped in all contexts; storage names are validated during model building. + // https://redis.io/docs/latest/develop/interact/search-and-query/query/exact-match + _filter.Append('@').Append(property.StorageName); + + _filter.Append( + binary.NodeType switch + { + ExpressionType.Equal when constantValue is byte or short or int or long or float or double => $" == {constantValue}", + ExpressionType.Equal when constantValue is string stringValue + => $$""":{"{{SanitizeStringConstant(stringValue)}}"}""", + ExpressionType.Equal when constantValue is null => throw new NotSupportedException("Null value type not supported"), // TODO + + ExpressionType.NotEqual when constantValue is int or long or float or double => $" != {constantValue}", + ExpressionType.NotEqual => throw new InvalidOperationException("Unreachable"), // Handled above + + ExpressionType.GreaterThan => $" > {constantValue}", + ExpressionType.GreaterThanOrEqual => $" >= {constantValue}", + ExpressionType.LessThan => $" < {constantValue}", + ExpressionType.LessThanOrEqual => $" <= {constantValue}", + + _ => throw new InvalidOperationException("Unsupported equality/comparison") + }); + + return true; + } + + return false; + } + } + + private void TranslateNot(Expression expression) + { + // https://redis.io/docs/latest/develop/interact/search-and-query/query/combined/#not + _filter.Append("(-"); + Translate(expression); + _filter.Append(')'); + } + + private void TranslateMethodCall(MethodCallExpression methodCall) + { + switch (methodCall) + { + // Enumerable.Contains(), List.Contains(), MemoryExtensions.Contains() + case var _ when TryMatchContains(methodCall, out var source, out var item): + TranslateContains(source, item); + return; + + // Enumerable.Any() with a Contains predicate (r => r.Strings.Any(s => array.Contains(s))) + case { Method.Name: nameof(Enumerable.Any), Arguments: [var anySource, LambdaExpression lambda] } any + when any.Method.DeclaringType == typeof(Enumerable): + TranslateAny(anySource, lambda); + return; + + default: + throw new NotSupportedException($"Unsupported method call: {methodCall.Method.DeclaringType?.Name}.{methodCall.Method.Name}"); + } + } + + private void TranslateContains(Expression source, Expression item) + { + // Contains over tag field + if (TryBindProperty(source, out var property) && item is ConstantExpression { Value: string stringConstant }) + { + // Redis field names cannot be escaped in all contexts; storage names are validated during model building. + _filter + .Append('@') + .Append(property.StorageName) + .Append(":{\"") + .Append(SanitizeStringConstant(stringConstant)) + .Append("\"}"); + return; + } + + throw new NotSupportedException("Contains supported only over tag field"); + } + + /// + /// Translates an Any() call with a Contains predicate, e.g. r.Strings.Any(s => array.Contains(s)). + /// This checks whether any element in the array field is contained in the given values. + /// + private void TranslateAny(Expression source, LambdaExpression lambda) + { + // We only support the pattern: r.ArrayField.Any(x => values.Contains(x)) + // Translates to: @Field:{value1 | value2 | value3} + if (!TryBindProperty(source, out var property) + || lambda.Body is not MethodCallExpression containsCall + || !TryMatchContains(containsCall, out var valuesExpression, out var itemExpression)) + { + throw new NotSupportedException("Unsupported method call: Enumerable.Any"); + } + + // Verify that the item is the lambda parameter + if (itemExpression != lambda.Parameters[0]) + { + throw new NotSupportedException("Unsupported method call: Enumerable.Any"); + } + + // Extract the values + IEnumerable values = valuesExpression switch + { + NewArrayExpression newArray => ExtractArrayValues(newArray), + ConstantExpression { Value: IEnumerable enumerable and not string } => enumerable, + _ => throw new NotSupportedException("Unsupported method call: Enumerable.Any") + }; + + // Generate: @Field:{value1 | value2 | value3} + _filter + .Append('@') + .Append(property.StorageName) + .Append(":{"); + + var isFirst = true; + foreach (var element in values) + { + if (element is not string stringElement) + { + throw new NotSupportedException("Any with Contains over non-string arrays is not supported"); + } + + if (isFirst) + { + isFirst = false; + } + else + { + _filter.Append(" | "); + } + + _filter.Append('"').Append(SanitizeStringConstant(stringElement)).Append('"'); + } + + _filter.Append('}'); + + static object?[] ExtractArrayValues(NewArrayExpression newArray) + { + var result = new object?[newArray.Expressions.Count]; + for (var i = 0; i < newArray.Expressions.Count; i++) + { + if (newArray.Expressions[i] is not ConstantExpression { Value: var elementValue }) + { + throw new NotSupportedException("Invalid element in array"); + } + + result[i] = elementValue; + } + + return result; + } + } + + private static string SanitizeStringConstant(string value) +#if NET + => value.Replace("\\", "\\\\", StringComparison.Ordinal).Replace("\"", "\\\"", StringComparison.Ordinal); +#else + => value.Replace("\\", "\\\\").Replace("\"", "\\\""); +#endif +} diff --git a/MEVD/src/Redis/RedisHashSetCollection.cs b/MEVD/src/Redis/RedisHashSetCollection.cs new file mode 100644 index 0000000..b20bb9a --- /dev/null +++ b/MEVD/src/Redis/RedisHashSetCollection.cs @@ -0,0 +1,547 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Linq.Expressions; +using System.Runtime.CompilerServices; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; +using NRedisStack.RedisStackCommands; +using NRedisStack.Search; +using NRedisStack.Search.Literals.Enums; +using StackExchange.Redis; +using Microsoft.Shared.Diagnostics; + +namespace CommunityToolkit.VectorData.Redis; + +/// +/// Service for storing and retrieving vector records, that uses Redis HashSets as the underlying storage. +/// +/// The data type of the record key. Must be . +/// The data model to use for adding, updating and retrieving data from storage. +#pragma warning disable CA1711 // Identifiers should not have incorrect suffix +public class RedisHashSetCollection : VectorStoreCollection + where TKey : notnull + where TRecord : class +#pragma warning restore CA1711 // Identifiers should not have incorrect suffix +{ + /// Metadata about vector store record collection. + private readonly VectorStoreCollectionMetadata _collectionMetadata; + + internal static readonly CollectionModelBuildingOptions ModelBuildingOptions = new() + { + RequiresAtLeastOneVector = false, + SupportsMultipleVectors = true + }; + + /// The default options for vector search. + private static readonly VectorSearchOptions s_defaultVectorSearchOptions = new(); + + /// The Redis database to read/write records from. + private readonly IDatabase _database; + + /// The model. + private readonly CollectionModel _model; + + /// An array of the names of all the data properties that are part of the Redis payload as RedisValue objects, i.e. all properties except the key and vector properties. + private readonly RedisValue[] _dataStoragePropertyNameRedisValues; + + /// An array of the names of all the data properties that are part of the Redis payload, i.e. all properties except the key and vector properties, plus the generated score property. + private readonly string[] _dataStoragePropertyNamesWithScore; + + /// The mapper to use when mapping between the consumer data model and the Redis record. + private readonly RedisHashSetMapper _mapper; + + /// whether the collection name should be prefixed to the key names before reading or writing to the Redis store. + private readonly bool _prefixCollectionNameToKeyNames; + + /// + /// Initializes a new instance of the class. + /// + /// The Redis database to read/write records from. + /// The name of the collection that this will access. + /// Optional configuration options for this class. + /// Throw when parameters are invalid. + // TODO: The provider uses unsafe JSON serialization in many places, #11963 + [RequiresUnreferencedCode("The Weaviate provider is currently incompatible with trimming.")] + [RequiresDynamicCode("The Weaviate provider is currently incompatible with NativeAOT.")] + public RedisHashSetCollection(IDatabase database, string name, RedisHashSetCollectionOptions? options = null) + : this( + database, + name, + static options => typeof(TRecord) == typeof(Dictionary) + ? throw new NotSupportedException(VectorDataStrings.NonDynamicCollectionWithDictionaryNotSupported(typeof(RedisHashSetDynamicCollection))) + : new RedisModelBuilder(ModelBuildingOptions).Build(typeof(TRecord), typeof(TKey), options.Definition, options.EmbeddingGenerator), + options) + { + } + + internal RedisHashSetCollection(IDatabase database, string name, Func modelFactory, RedisHashSetCollectionOptions? options) + { + // Verify. + Throw.IfNull(database); + Throw.IfNullOrWhitespace(name); + + if (typeof(TKey) != typeof(string) && typeof(TKey) != typeof(Guid) && typeof(TKey) != typeof(object)) + { + throw new NotSupportedException("Only string and Guid keys are supported."); + } + + options ??= RedisHashSetCollectionOptions.Default; + + // Assign. + _database = database; + Name = name; + _model = modelFactory(options); + + _prefixCollectionNameToKeyNames = options.PrefixCollectionNameToKeyNames; + + // Lookup storage property names. + _dataStoragePropertyNameRedisValues = _model.DataProperties.Select(p => RedisValue.Unbox(p.StorageName)).ToArray(); + _dataStoragePropertyNamesWithScore = [.. _model.DataProperties.Select(p => p.StorageName), "vector_score"]; + + // Assign Mapper. + _mapper = new RedisHashSetMapper(_model); + + _collectionMetadata = new() + { + VectorStoreSystemName = RedisConstants.VectorStoreSystemName, + VectorStoreName = database.Database.ToString(), + CollectionName = name + }; + } + + /// + public override string Name { get; } + + /// + public override async Task CollectionExistsAsync(CancellationToken cancellationToken = default) + { + try + { + await _database.FT().InfoAsync(Name).ConfigureAwait(false); + return true; + } + // "Unknown index name" is returned in Redis Stack + // "no such index" is returned in Redis Alpine + catch (RedisServerException ex) when (ex.Message.Contains("Unknown index name") || ex.Message.Contains("no such index")) + { + return false; + } + catch (RedisConnectionException ex) + { + throw new VectorStoreException("Call to vector store failed.", ex) + { + VectorStoreSystemName = RedisConstants.VectorStoreSystemName, + VectorStoreName = _collectionMetadata.VectorStoreName, + CollectionName = Name, + OperationName = "FT.INFO" + }; + } + } + + /// + public override async Task EnsureCollectionExistsAsync(CancellationToken cancellationToken = default) + { + const string OperationName = "FT.CREATE"; + + // Don't even try to create if the collection already exists. + if (await CollectionExistsAsync(cancellationToken).ConfigureAwait(false)) + { + return; + } + + try + { + // Map the record definition to a schema. + var schema = RedisCollectionCreateMapping.MapToSchema(_model.Properties, useDollarPrefix: false); + + // Create the index creation params. + // Add the collection name and colon as the index prefix, which means that any record where the key is prefixed with this text will be indexed by this index + var createParams = new FTCreateParams() + .AddPrefix($"{Name}:") + .On(IndexDataType.HASH); + + // Create the index. + await _database.FT().CreateAsync(Name, createParams, schema).ConfigureAwait(false); + } + catch (RedisException ex) + { + // Since redis only returns textual error messages, we can check here if the index already exists. + // If it does, we can ignore the error. +#pragma warning disable CA1031 // Do not catch general exception types + try + { + if (await CollectionExistsAsync(cancellationToken).ConfigureAwait(false)) + { + return; + } + } + catch + { + } +#pragma warning restore CA1031 // Do not catch general exception types + + throw new VectorStoreException("Call to vector store failed.", ex) + { + VectorStoreSystemName = RedisConstants.VectorStoreSystemName, + VectorStoreName = _collectionMetadata.VectorStoreName, + CollectionName = Name, + OperationName = OperationName + }; + } + } + + /// + public override async Task EnsureCollectionDeletedAsync(CancellationToken cancellationToken = default) + { + try + { + await RunOperationAsync("FT.DROPINDEX", + () => _database.FT().DropIndexAsync(Name)).ConfigureAwait(false); + } + catch (VectorStoreException ex) when (ex.InnerException is RedisServerException) + { + // The RedisServerException does not expose any reliable way of checking if the index does not exist. + // It just sets the message to "Unknown index name". + // We catch the exception and ignore it, but only after checking that the index does not exist. + if (!await CollectionExistsAsync(cancellationToken).ConfigureAwait(false)) + { + return; + } + + throw; + } + } + + /// + public override async Task GetAsync(TKey key, RecordRetrievalOptions? options = null, CancellationToken cancellationToken = default) + { + var stringKey = GetStringKey(key); + + // Create Options + var maybePrefixedKey = PrefixKeyIfNeeded(stringKey); + + var includeVectors = options?.IncludeVectors ?? false; + if (includeVectors && _model.EmbeddingGenerationRequired) + { + throw new NotSupportedException(VectorDataStrings.IncludeVectorsNotSupportedWithEmbeddingGeneration); + } + + var operationName = includeVectors ? "HGETALL" : "HMGET"; + + // Get the Redis value. + HashEntry[] retrievedHashEntries; + if (includeVectors) + { + retrievedHashEntries = await RunOperationAsync( + operationName, + () => _database.HashGetAllAsync(maybePrefixedKey)).ConfigureAwait(false); + } + else + { + var fieldKeys = _dataStoragePropertyNameRedisValues; + var retrievedValues = await RunOperationAsync( + operationName, + () => _database.HashGetAsync(maybePrefixedKey, fieldKeys)).ConfigureAwait(false); + retrievedHashEntries = fieldKeys.Zip(retrievedValues, (field, value) => new HashEntry(field, value)).Where(x => x.Value.HasValue).ToArray(); + } + + // Return null if we found nothing. + if (retrievedHashEntries == null || retrievedHashEntries.Length == 0) + { + return default; + } + + // Convert to the caller's data model. + return _mapper.MapFromStorageToDataModel((stringKey, retrievedHashEntries), includeVectors); + } + + /// + public override Task DeleteAsync(TKey key, CancellationToken cancellationToken = default) + { + var stringKey = GetStringKey(key); + + // Create Options + var maybePrefixedKey = PrefixKeyIfNeeded(stringKey); + + // Remove. + return RunOperationAsync( + "DEL", + () => _database + .KeyDeleteAsync(maybePrefixedKey)); + } + + /// + public override async Task UpsertAsync(TRecord record, CancellationToken cancellationToken = default) + { + (_, var generatedEmbeddings) = await RedisFieldMapping.ProcessEmbeddingsAsync(_model, [record], cancellationToken).ConfigureAwait(false); + + await UpsertCoreAsync(record, 0, generatedEmbeddings, cancellationToken).ConfigureAwait(false); + } + + /// + public override async Task UpsertAsync(IEnumerable records, CancellationToken cancellationToken = default) + { + Throw.IfNull(records); + + (records, var generatedEmbeddings) = await RedisFieldMapping.ProcessEmbeddingsAsync(_model, records, cancellationToken).ConfigureAwait(false); + + var i = 0; + + foreach (var record in records) + { + await UpsertCoreAsync(record, i++, generatedEmbeddings, cancellationToken).ConfigureAwait(false); + } + } + + private async Task UpsertCoreAsync(TRecord record, int recordIndex, IReadOnlyList?[]? generatedEmbeddings, CancellationToken cancellationToken = default) + { + Throw.IfNull(record); + + // Auto-generate key if needed (client-side for Redis) + var keyProperty = _model.KeyProperty; + if (keyProperty.IsAutoGenerated && keyProperty.GetValue(record) == Guid.Empty) + { + keyProperty.SetValue(record, Guid.NewGuid()); + } + + // Map. + var redisHashSetRecord = _mapper.MapFromDataToStorageModel(record, recordIndex, generatedEmbeddings); + + // Upsert. + var maybePrefixedKey = PrefixKeyIfNeeded(redisHashSetRecord.Key); + + await RunOperationAsync( + "HSET", + () => _database + .HashSetAsync( + maybePrefixedKey, + redisHashSetRecord.HashEntries)).ConfigureAwait(false); + } + + #region Search + + /// + public override async IAsyncEnumerable> SearchAsync( + TInput searchValue, + int top, + VectorSearchOptions? options = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + Throw.IfNull(searchValue); + Throw.IfLessThan(top, 1); + + options ??= s_defaultVectorSearchOptions; + + if (options.IncludeVectors && _model.EmbeddingGenerationRequired) + { + throw new NotSupportedException(VectorDataStrings.IncludeVectorsNotSupportedWithEmbeddingGeneration); + } + + var vectorProperty = _model.GetVectorPropertyOrSingle(options); + + object vector = searchValue switch + { + // float32 + ReadOnlyMemory r => r, + float[] f => new ReadOnlyMemory(f), + Embedding e => e.Vector, + + // float64 + ReadOnlyMemory r => r, + double[] f => new ReadOnlyMemory(f), + Embedding e => e.Vector, + + _ when vectorProperty.EmbeddingGenerationDispatcher is not null + => await vectorProperty.GenerateEmbeddingAsync(searchValue, cancellationToken).ConfigureAwait(false), + + _ => vectorProperty.EmbeddingGenerator is null + ? throw new NotSupportedException(VectorDataStrings.InvalidSearchInputAndNoEmbeddingGeneratorWasConfigured(searchValue.GetType(), RedisModelBuilder.SupportedVectorTypes)) + : throw new InvalidOperationException(VectorDataStrings.IncompatibleEmbeddingGeneratorWasConfiguredForInputType(typeof(TInput), vectorProperty.EmbeddingGenerator.GetType())) + }; + + // Build query & search. + var selectFields = options.IncludeVectors ? null : _dataStoragePropertyNamesWithScore; + byte[] vectorBytes = RedisCollectionSearchMapping.ValidateVectorAndConvertToBytes(vector, "HashSet"); + var query = RedisCollectionSearchMapping.BuildQuery( + vectorBytes, + top, + options, + _model, + vectorProperty, + selectFields); + var results = await RunOperationAsync( + "FT.SEARCH", + () => _database + .FT() + .SearchAsync(Name, query)).ConfigureAwait(false); + + // Loop through result and convert to the caller's data model. + var mappedResults = results.Documents.Select(result => + { + var retrievedHashEntries = _model.DataProperties.Select(p => p.StorageName) + .Concat(_model.VectorProperties.Select(p => p.StorageName)) + .Select(propertyName => new HashEntry(propertyName, result[propertyName])) + .ToArray(); + + // Convert to the caller's data model. + var dataModel = _mapper.MapFromStorageToDataModel((RemoveKeyPrefixIfNeeded(result.Id), retrievedHashEntries), options.IncludeVectors); + + // Process the score of the result item. + var vectorProperty = _model.GetVectorPropertyOrSingle(options); + var distanceFunction = RedisCollectionSearchMapping.ResolveDistanceFunction(vectorProperty); + var score = RedisCollectionSearchMapping.GetOutputScoreFromRedisScore(result["vector_score"].HasValue ? (float)result["vector_score"] : null, distanceFunction); + + return new VectorSearchResult(dataModel, score); + }); + + foreach (var result in mappedResults) + { + // Apply score threshold filtering. The score semantics depend on the distance function: + // - For similarity functions (CosineSimilarity, DotProductSimilarity): higher = more similar, filter out below threshold + // - For distance functions (CosineDistance, EuclideanSquaredDistance): lower = more similar, filter out above threshold + if (options.ScoreThreshold.HasValue && result.Score.HasValue) + { + var distanceFunction = RedisCollectionSearchMapping.ResolveDistanceFunction(vectorProperty); + var passesThreshold = distanceFunction switch + { + DistanceFunction.CosineSimilarity or DistanceFunction.DotProductSimilarity => result.Score.Value >= options.ScoreThreshold.Value, + DistanceFunction.CosineDistance or DistanceFunction.EuclideanSquaredDistance => result.Score.Value <= options.ScoreThreshold.Value, + _ => throw new InvalidOperationException($"Unexpected distance function: {distanceFunction}") + }; + + if (!passesThreshold) + { + continue; + } + } + + yield return result; + } + } + + #endregion Search + + /// + public override async IAsyncEnumerable GetAsync(Expression> filter, int top, + FilteredRecordRetrievalOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + Throw.IfNull(filter); + Throw.IfLessThan(top, 1); + + options ??= new(); + if (options.IncludeVectors && _model.EmbeddingGenerationRequired) + { + throw new NotSupportedException(VectorDataStrings.IncludeVectorsNotSupportedWithEmbeddingGeneration); + } + + Query query = RedisCollectionSearchMapping.BuildQuery(filter, top, options, _model); + + var results = await RunOperationAsync( + "FT.SEARCH", + () => _database + .FT() + .SearchAsync(Name, query)).ConfigureAwait(false); + + foreach (var document in results.Documents) + { + var retrievedHashEntries = _model.DataProperties.Select(p => p.StorageName) + .Concat(_model.VectorProperties.Select(p => p.StorageName)) + .Select(propertyName => new HashEntry(propertyName, document[propertyName])) + .ToArray(); + + // Convert to the caller's data model. + yield return _mapper.MapFromStorageToDataModel((RemoveKeyPrefixIfNeeded(document.Id), retrievedHashEntries), options.IncludeVectors); + } + } + + /// + public override object? GetService(Type serviceType, object? serviceKey = null) + { + Throw.IfNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreCollectionMetadata) ? _collectionMetadata : + serviceType == typeof(IDatabase) ? _database : + serviceType.IsInstanceOfType(this) ? this : + null; + } + + /// + /// Prefix the key with the collection name if the option is set. + /// + /// The key to prefix. + /// The updated key if updating is required, otherwise the input key. + private string PrefixKeyIfNeeded(string key) + { + if (_prefixCollectionNameToKeyNames) + { + return $"{Name}:{key}"; + } + + return key; + } + + /// + /// Remove the prefix of the given key if the option is set. + /// + /// The key to remove a prefix from. + /// The updated key if updating is required, otherwise the input key. + private string RemoveKeyPrefixIfNeeded(string key) + { + var prefixLength = Name.Length + 1; + + if (_prefixCollectionNameToKeyNames && key.Length > prefixLength) + { + return key.Substring(prefixLength); + } + + return key; + } + + /// + /// Run the given operation and wrap any Redis exceptions with ."/> + /// + /// The response type of the operation. + /// The type of database operation being run. + /// The operation to run. + /// The result of the operation. + private Task RunOperationAsync(string operationName, Func> operation) + => VectorStoreErrorHandler.RunOperationAsync( + _collectionMetadata, + operationName, + operation); + + /// + /// Run the given operation and wrap any Redis exceptions with ."/> + /// + /// The type of database operation being run. + /// The operation to run. + /// The result of the operation. + private Task RunOperationAsync(string operationName, Func operation) + => VectorStoreErrorHandler.RunOperationAsync( + _collectionMetadata, + operationName, + operation); + + private string GetStringKey(TKey key) + { + Throw.IfNull(key); + + var stringKey = key switch + { + string s => s, + Guid g => g.ToString(), + + _ => throw new UnreachableException("string key should have been validated during model building") + }; + + Throw.IfNullOrWhitespace(stringKey); + + return stringKey; + } +} diff --git a/MEVD/src/Redis/RedisHashSetCollectionOptions.cs b/MEVD/src/Redis/RedisHashSetCollectionOptions.cs new file mode 100644 index 0000000..851dfc9 --- /dev/null +++ b/MEVD/src/Redis/RedisHashSetCollectionOptions.cs @@ -0,0 +1,36 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.VectorData; + +namespace CommunityToolkit.VectorData.Redis; + +/// +/// Options when creating a . +/// +public sealed class RedisHashSetCollectionOptions : VectorStoreCollectionOptions +{ + internal static readonly RedisHashSetCollectionOptions Default = new(); + + /// + /// Initializes a new instance of the class. + /// + public RedisHashSetCollectionOptions() + { + } + + internal RedisHashSetCollectionOptions(RedisHashSetCollectionOptions? source) : base(source) + { + PrefixCollectionNameToKeyNames = source?.PrefixCollectionNameToKeyNames ?? Default.PrefixCollectionNameToKeyNames; + } + + /// + /// Gets or sets a value indicating whether the collection name should be prefixed to the + /// key names before reading or writing to the Redis store. Default is true. + /// + /// + /// For a record to be indexed by a specific Redis index, the key name must be prefixed with the matching prefix configured on the Redis index. + /// You can either pass in keys that are already prefixed, or set this option to true to have the collection name prefixed to the key names automatically. + /// + public bool PrefixCollectionNameToKeyNames { get; set; } = true; +} diff --git a/MEVD/src/Redis/RedisHashSetDynamicCollection.cs b/MEVD/src/Redis/RedisHashSetDynamicCollection.cs new file mode 100644 index 0000000..1775ba0 --- /dev/null +++ b/MEVD/src/Redis/RedisHashSetDynamicCollection.cs @@ -0,0 +1,36 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using StackExchange.Redis; + +namespace CommunityToolkit.VectorData.Redis; + +/// +/// Represents a collection of vector store records in a Redis HashSet database, mapped to a dynamic Dictionary<string, object?>. +/// +#pragma warning disable CA1711 // Identifiers should not have incorrect suffix +public sealed class RedisHashSetDynamicCollection : RedisHashSetCollection> +#pragma warning restore CA1711 // Identifiers should not have incorrect suffix +{ + /// + /// Initializes a new instance of the class. + /// + /// The Redis database to read/write records from. + /// The name of the collection. + /// Optional configuration options for this class. + // TODO: The provider uses unsafe JSON serialization in many places, #11963 + [RequiresUnreferencedCode("The Redis provider is currently incompatible with trimming.")] + [RequiresDynamicCode("The Redis provider is currently incompatible with NativeAOT.")] + public RedisHashSetDynamicCollection(IDatabase database, string name, RedisHashSetCollectionOptions options) + : base( + database, + name, + static options => new RedisModelBuilder(ModelBuildingOptions) + .BuildDynamic( + options.Definition ?? throw new ArgumentException("Definition is required for dynamic collections"), + options.EmbeddingGenerator), + options) + { + } +} diff --git a/MEVD/src/Redis/RedisHashSetMapper.cs b/MEVD/src/Redis/RedisHashSetMapper.cs new file mode 100644 index 0000000..4d98ce2 --- /dev/null +++ b/MEVD/src/Redis/RedisHashSetMapper.cs @@ -0,0 +1,139 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using System.Runtime.InteropServices; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData.ProviderServices; +using StackExchange.Redis; + +namespace CommunityToolkit.VectorData.Redis; + +/// +/// Class for mapping between a hashset stored in redis, and the consumer data model. +/// +/// The consumer data model to map to or from. +internal sealed class RedisHashSetMapper(CollectionModel model) +{ + /// + public (string Key, HashEntry[] HashEntries) MapFromDataToStorageModel(TConsumerDataModel dataModel, int recordIndex, IReadOnlyList?[]? generatedEmbeddings) + { + var keyValue = model.KeyProperty.GetValueAsObject(dataModel!) switch + { + string s => s, + Guid g => g.ToString(), + + _ => throw new InvalidOperationException($"Missing key property {model.KeyProperty.ModelName} on provided record of type '{typeof(TConsumerDataModel).Name}'.") + }; + + var hashEntries = new List(); + foreach (var property in model.DataProperties) + { + var value = property.GetValueAsObject(dataModel!); + hashEntries.Add(new HashEntry(property.StorageName, RedisValue.Unbox(value))); + } + + for (var i = 0; i < model.VectorProperties.Count; i++) + { + var property = model.VectorProperties[i]; + + var value = generatedEmbeddings?[i]?[recordIndex] ?? property.GetValueAsObject(dataModel!); + + if (value is not null) + { + // Convert the vector to a byte array and store it in the hash entry. + // We only support float and double vectors and we do checking in the + // collection constructor to ensure that the model has no other vector types. + hashEntries.Add(new HashEntry(property.StorageName, value switch + { + ReadOnlyMemory m => MemoryMarshal.AsBytes(m.Span).ToArray(), + Embedding e => MemoryMarshal.AsBytes(e.Vector.Span).ToArray(), + float[] a => MemoryMarshal.AsBytes(a.AsSpan()).ToArray(), + + ReadOnlyMemory m => MemoryMarshal.AsBytes(m.Span).ToArray(), + Embedding e => MemoryMarshal.AsBytes(e.Vector.Span).ToArray(), + double[] a => MemoryMarshal.AsBytes(a.AsSpan()).ToArray(), + + _ => throw new InvalidOperationException($"Unsupported vector type '{value.GetType()}'. Only float and double vectors are supported.") + })); + } + } + + return (keyValue, hashEntries.ToArray()); + } + + /// + public TConsumerDataModel MapFromStorageToDataModel((string Key, HashEntry[] HashEntries) storageModel, bool includeVectors) + { + var hashEntriesDictionary = storageModel.HashEntries.ToDictionary(x => (string)x.Name!, x => x.Value); + + // Construct the output record. + var outputRecord = model.CreateRecord()!; + + // Set Key. + model.KeyProperty.SetValueAsObject(outputRecord, model.KeyProperty.Type switch + { + Type t when t == typeof(string) + => storageModel.Key, + Type t when t == typeof(Guid) + => Guid.Parse(storageModel.Key), + + _ => throw new UnreachableException() + }); + + // Set each vector property if embeddings should be returned. + if (includeVectors) + { + foreach (var property in model.VectorProperties) + { + if (hashEntriesDictionary.TryGetValue(property.StorageName, out var value)) + { + if (value.IsNull) + { + property.SetValueAsObject(outputRecord!, null); + continue; + } + + var vector = (byte[])value!; + + property.SetValueAsObject(outputRecord!, (Nullable.GetUnderlyingType(property.Type) ?? property.Type) switch + { + Type t when t == typeof(ReadOnlyMemory) + => new ReadOnlyMemory(MemoryMarshal.Cast(vector).ToArray()), + Type t when t == typeof(Embedding) + => new Embedding(MemoryMarshal.Cast(vector).ToArray()), + Type t when t == typeof(float[]) + => MemoryMarshal.Cast(vector).ToArray(), + + Type t when t == typeof(ReadOnlyMemory) + => new ReadOnlyMemory(MemoryMarshal.Cast(vector).ToArray()), + Type t when t == typeof(Embedding) + => new Embedding(MemoryMarshal.Cast(vector).ToArray()), + Type t when t == typeof(double[]) + => MemoryMarshal.Cast(vector).ToArray(), + + _ => throw new InvalidOperationException($"Unsupported vector type '{property.Type}'. Only float and double vectors are supported.") + }); + } + } + } + + foreach (var property in model.DataProperties) + { + if (hashEntriesDictionary.TryGetValue(property.StorageName, out var hashValue)) + { + if (hashValue.IsNull) + { + property.SetValueAsObject(outputRecord!, null); + continue; + } + + var typeOrNullableType = Nullable.GetUnderlyingType(property.Type) ?? property.Type; + var value = Convert.ChangeType(hashValue, typeOrNullableType); + property.SetValueAsObject(outputRecord!, value); + } + } + + return outputRecord; + } +} diff --git a/MEVD/src/Redis/RedisJsonCollection.cs b/MEVD/src/Redis/RedisJsonCollection.cs new file mode 100644 index 0000000..e9e391c --- /dev/null +++ b/MEVD/src/Redis/RedisJsonCollection.cs @@ -0,0 +1,642 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Linq.Expressions; +using System.Runtime.CompilerServices; +using System.Text.Json; +using System.Text.Json.Nodes; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; +using NRedisStack.Json.DataTypes; +using NRedisStack.RedisStackCommands; +using NRedisStack.Search; +using NRedisStack.Search.Literals.Enums; +using StackExchange.Redis; +using Microsoft.Shared.Diagnostics; + +namespace CommunityToolkit.VectorData.Redis; + +/// +/// Service for storing and retrieving vector records, that uses Redis JSON as the underlying storage. +/// +/// The data type of the record key. Must be . +/// The data model to use for adding, updating and retrieving data from storage. +#pragma warning disable CA1711 // Identifiers should not have incorrect suffix +public class RedisJsonCollection : VectorStoreCollection + where TKey : notnull + where TRecord : class +#pragma warning restore CA1711 // Identifiers should not have incorrect suffix +{ + /// Metadata about vector store record collection. + private readonly VectorStoreCollectionMetadata _collectionMetadata; + + internal static readonly CollectionModelBuildingOptions ModelBuildingOptions = new() + { + RequiresAtLeastOneVector = false, + SupportsMultipleVectors = true, + UsesExternalSerializer = true + }; + + /// The default options for vector search. + private static readonly VectorSearchOptions s_defaultVectorSearchOptions = new(); + + /// The Redis database to read/write records from. + private readonly IDatabase _database; + + /// The model. + private readonly CollectionModel _model; + + /// An array of the storage names of all the data properties that are part of the Redis payload, i.e. all properties except the key and vector properties. + private readonly string[] _dataStoragePropertyNames; + + /// The mapper to use when mapping between the consumer data model and the Redis record. + private readonly IRedisJsonMapper _mapper; + + /// The JSON serializer options to use when converting between the data model and the Redis record. + private readonly JsonSerializerOptions _jsonSerializerOptions; + + /// whether the collection name should be prefixed to the key names before reading or writing to the Redis store. + private readonly bool _prefixCollectionNameToKeyNames; + + /// + /// Initializes a new instance of the class. + /// + /// The Redis database to read/write records from. + /// The name of the collection that this will access. + /// Optional configuration options for this class. + /// Throw when parameters are invalid. + // TODO: The provider uses unsafe JSON serialization in many places, #11963 + [RequiresUnreferencedCode("The Weaviate provider is currently incompatible with trimming.")] + [RequiresDynamicCode("The Weaviate provider is currently incompatible with NativeAOT.")] + public RedisJsonCollection(IDatabase database, string name, RedisJsonCollectionOptions? options = null) + : this( + database, + name, + static options => typeof(TRecord) == typeof(Dictionary) + ? throw new NotSupportedException(VectorDataStrings.NonDynamicCollectionWithDictionaryNotSupported(typeof(RedisJsonDynamicCollection))) + : new RedisJsonModelBuilder(ModelBuildingOptions) + .Build( + typeof(TRecord), + typeof(TKey), + options.Definition, + options.EmbeddingGenerator, + options.JsonSerializerOptions ?? JsonSerializerOptions.Default), + options) + { + } + + internal RedisJsonCollection(IDatabase database, string name, Func modelFactory, RedisJsonCollectionOptions? options) + { + // Verify. + Throw.IfNull(database); + Throw.IfNullOrWhitespace(name); + + if (typeof(TKey) != typeof(string) && typeof(TKey) != typeof(Guid) && typeof(TKey) != typeof(object)) + { + throw new NotSupportedException("Only string or Guid keys are supported."); + } + + var isDynamic = typeof(TRecord) == typeof(Dictionary); + + options ??= RedisJsonCollectionOptions.Default; + + // Assign. + _database = database; + Name = name; + _model = modelFactory(options); + + _prefixCollectionNameToKeyNames = options.PrefixCollectionNameToKeyNames; + _jsonSerializerOptions = options.JsonSerializerOptions ?? JsonSerializerOptions.Default; + + // Lookup storage property names. + _dataStoragePropertyNames = _model.DataProperties.Select(p => p.StorageName).ToArray(); + + // Assign Mapper. + _mapper = isDynamic + ? (IRedisJsonMapper)new RedisJsonDynamicMapper(_model, _jsonSerializerOptions) + : new RedisJsonMapper(_model, _jsonSerializerOptions); + + _collectionMetadata = new() + { + VectorStoreSystemName = RedisConstants.VectorStoreSystemName, + VectorStoreName = database.Database.ToString(), + CollectionName = name + }; + } + + /// + public override string Name { get; } + + /// + public override async Task CollectionExistsAsync(CancellationToken cancellationToken = default) + { + try + { + await _database.FT().InfoAsync(Name).ConfigureAwait(false); + return true; + } + // "Unknown index name" is returned in Redis Stack + // "no such index" is returned in Redis Alpine + catch (RedisServerException ex) when (ex.Message.Contains("Unknown index name") || ex.Message.Contains("no such index")) + { + return false; + } + catch (RedisConnectionException ex) + { + throw new VectorStoreException("Call to vector store failed.", ex) + { + VectorStoreSystemName = RedisConstants.VectorStoreSystemName, + VectorStoreName = _collectionMetadata.VectorStoreName, + CollectionName = Name, + OperationName = "FT.INFO" + }; + } + } + + /// + public override async Task EnsureCollectionExistsAsync(CancellationToken cancellationToken = default) + { + const string OperationName = "FT.CREATE"; + + // Don't even try to create if the collection already exists. + if (await CollectionExistsAsync(cancellationToken).ConfigureAwait(false)) + { + return; + } + + try + { + // Map the record definition to a schema. + var schema = RedisCollectionCreateMapping.MapToSchema(_model.Properties, useDollarPrefix: true); + + // Create the index creation params. + // Add the collection name and colon as the index prefix, which means that any record where the key is prefixed with this text will be indexed by this index + var createParams = new FTCreateParams() + .AddPrefix($"{Name}:") + .On(IndexDataType.JSON); + + // Create the index. + await _database.FT().CreateAsync(Name, createParams, schema).ConfigureAwait(false); + } + catch (RedisException ex) + { + // Since redis only returns textual error messages, we can check here if the index already exists. + // If it does, we can ignore the error. +#pragma warning disable CA1031 // Do not catch general exception types + try + { + if (await CollectionExistsAsync(cancellationToken).ConfigureAwait(false)) + { + return; + } + } + catch + { + } +#pragma warning restore CA1031 // Do not catch general exception types + + throw new VectorStoreException("Call to vector store failed.", ex) + { + VectorStoreSystemName = RedisConstants.VectorStoreSystemName, + VectorStoreName = _collectionMetadata.VectorStoreName, + CollectionName = Name, + OperationName = OperationName + }; + } + } + + /// + public override async Task EnsureCollectionDeletedAsync(CancellationToken cancellationToken = default) + { + try + { + await RunOperationAsync("FT.DROPINDEX", + () => _database.FT().DropIndexAsync(Name)).ConfigureAwait(false); + } + catch (VectorStoreException ex) when (ex.InnerException is RedisServerException) + { + // The RedisServerException does not expose any reliable way of checking if the index does not exist. + // It just sets the message to "Unknown index name". + // We catch the exception and ignore it, but only after checking that the index does not exist. + if (!await CollectionExistsAsync(cancellationToken).ConfigureAwait(false)) + { + return; + } + + throw; + } + } + + /// + public override async Task GetAsync(TKey key, RecordRetrievalOptions? options = null, CancellationToken cancellationToken = default) + { + var stringKey = GetStringKey(key); + + // Create Options + var maybePrefixedKey = PrefixKeyIfNeeded(stringKey); + var includeVectors = options?.IncludeVectors ?? false; + if (includeVectors && _model.EmbeddingGenerationRequired) + { + throw new NotSupportedException(VectorDataStrings.IncludeVectorsNotSupportedWithEmbeddingGeneration); + } + + // Get the Redis value. + var redisResult = await RunOperationAsync( + "GET", + () => options?.IncludeVectors is true ? + _database + .JSON() + .GetAsync(maybePrefixedKey) : + _database + .JSON() + .GetAsync(maybePrefixedKey, _dataStoragePropertyNames)).ConfigureAwait(false); + + // Check if the key was found before trying to parse the result. + if (redisResult.IsNull || redisResult is null) + { + return default; + } + + // Check if the value contained any JSON text before trying to parse the result. + var redisResultString = redisResult.ToString(); + if (redisResultString is null) + { + throw new InvalidOperationException($"Document with key '{key}' does not contain any json."); + } + + // Convert to the caller's data model. + var node = JsonSerializer.Deserialize(redisResultString, _jsonSerializerOptions)!; + return _mapper.MapFromStorageToDataModel((stringKey, node), includeVectors); + } + + /// + public override async IAsyncEnumerable GetAsync(IEnumerable keys, RecordRetrievalOptions? options = default, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + Throw.IfNull(keys); + +#pragma warning disable CA1851 // Possible multiple enumerations of 'IEnumerable' collection + var keysList = keys switch + { + IEnumerable k => k.ToList(), + IEnumerable k => k.Select(x => x.ToString()).ToList(), + IEnumerable k => k.Select(x => x.ToString()!).ToList(), + _ => throw new UnreachableException() + }; +#pragma warning restore CA1851 // Possible multiple enumerations of 'IEnumerable' collection + + if (keysList.Count == 0) + { + yield break; + } + + // Create Options + var maybePrefixedKeys = keysList.Select(key => PrefixKeyIfNeeded(key)); + var redisKeys = maybePrefixedKeys.Select(x => new RedisKey(x)).ToArray(); + var includeVectors = options?.IncludeVectors ?? false; + if (includeVectors && _model.EmbeddingGenerationRequired) + { + throw new NotSupportedException(VectorDataStrings.IncludeVectorsNotSupportedWithEmbeddingGeneration); + } + + // Get the list of Redis results. + var redisResults = await RunOperationAsync( + "MGET", + () => _database + .JSON() + .MGetAsync(redisKeys, "$")).ConfigureAwait(false); + + // Loop through each key and result and convert to the caller's data model. + for (int i = 0; i < keysList.Count; i++) + { + var key = keysList[i]; + var redisResult = redisResults[i]; + + // Check if the key was found before trying to parse the result. + if (redisResult.IsNull || redisResult is null) + { + continue; + } + + // Check if the value contained any JSON text before trying to parse the result. + var redisResultString = redisResult.ToString(); + if (redisResultString is null) + { + throw new InvalidOperationException($"Document with key '{key}' does not contain any json."); + } + + // Convert to the caller's data model. + var node = JsonSerializer.Deserialize(redisResultString, _jsonSerializerOptions)!; + yield return _mapper.MapFromStorageToDataModel((key, node), includeVectors); + } + } + + /// + public override Task DeleteAsync(TKey key, CancellationToken cancellationToken = default) + { + var stringKey = GetStringKey(key); + + // Create Options + var maybePrefixedKey = PrefixKeyIfNeeded(stringKey); + + // Remove. + return RunOperationAsync( + "DEL", + () => _database + .JSON() + .DelAsync(maybePrefixedKey)); + } + + /// + public override async Task UpsertAsync(TRecord record, CancellationToken cancellationToken = default) + { + Throw.IfNull(record); + + // Auto-generate key if needed (client-side for Redis) + var keyProperty = _model.KeyProperty; + if (keyProperty.IsAutoGenerated && keyProperty.GetValue(record) == Guid.Empty) + { + keyProperty.SetValue(record, Guid.NewGuid()); + } + + // Map. + (_, var generatedEmbeddings) = await RedisFieldMapping.ProcessEmbeddingsAsync(_model, [record], cancellationToken).ConfigureAwait(false); + + var mapResult = _mapper.MapFromDataToStorageModel(record, recordIndex: 0, generatedEmbeddings); + var serializedRecord = JsonSerializer.Serialize(mapResult.Node, _jsonSerializerOptions); + var redisJsonRecord = new { Key = mapResult.Key, SerializedRecord = serializedRecord }; + + // Upsert. + var maybePrefixedKey = PrefixKeyIfNeeded(redisJsonRecord.Key); + await RunOperationAsync( + "SET", + () => _database + .JSON() + .SetAsync( + maybePrefixedKey, + "$", + redisJsonRecord.SerializedRecord)).ConfigureAwait(false); + } + + /// + public override async Task UpsertAsync(IEnumerable records, CancellationToken cancellationToken = default) + { + Throw.IfNull(records); + + // Map. + (records, var generatedEmbeddings) = await RedisFieldMapping.ProcessEmbeddingsAsync(_model, records, cancellationToken).ConfigureAwait(false); + + var redisRecords = new List<(string maybePrefixedKey, string originalKey, string serializedRecord)>(); + var keyProperty = _model.KeyProperty; + + var recordIndex = 0; + + foreach (var record in records) + { + // Auto-generate key if needed (client-side for Redis) + if (keyProperty.IsAutoGenerated && keyProperty.GetValue(record) == Guid.Empty) + { + keyProperty.SetValue(record, Guid.NewGuid()); + } + + var mapResult = _mapper.MapFromDataToStorageModel(record, recordIndex++, generatedEmbeddings); + var serializedRecord = JsonSerializer.Serialize(mapResult.Node, _jsonSerializerOptions); + var redisJsonRecord = new { Key = mapResult.Key, SerializedRecord = serializedRecord }; + + var maybePrefixedKey = PrefixKeyIfNeeded(redisJsonRecord.Key); + redisRecords.Add((maybePrefixedKey, redisJsonRecord.Key, redisJsonRecord.SerializedRecord)); + } + + if (redisRecords.Count == 0) + { + return; + } + + // Upsert. + var keyPathValues = redisRecords.Select(x => new KeyPathValue(x.maybePrefixedKey, "$", x.serializedRecord)).ToArray(); + await RunOperationAsync( + "MSET", + () => _database + .JSON() + .MSetAsync(keyPathValues)).ConfigureAwait(false); + } + + #region Search + + /// + public override async IAsyncEnumerable> SearchAsync( + TInput searchValue, + int top, + VectorSearchOptions? options = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + Throw.IfNull(searchValue); + Throw.IfLessThan(top, 1); + + options ??= s_defaultVectorSearchOptions; + + if (options.IncludeVectors && _model.EmbeddingGenerationRequired) + { + throw new NotSupportedException(VectorDataStrings.IncludeVectorsNotSupportedWithEmbeddingGeneration); + } + + var vectorProperty = _model.GetVectorPropertyOrSingle(options); + + object vector = searchValue switch + { + // float32 + ReadOnlyMemory r => r, + float[] f => new ReadOnlyMemory(f), + Embedding e => e.Vector, + + // float64 + ReadOnlyMemory r => r, + double[] f => new ReadOnlyMemory(f), + Embedding e => e.Vector, + + _ when vectorProperty.EmbeddingGenerationDispatcher is not null + => await vectorProperty.GenerateEmbeddingAsync(searchValue, cancellationToken).ConfigureAwait(false), + + _ => vectorProperty.EmbeddingGenerator is null + ? throw new NotSupportedException(VectorDataStrings.InvalidSearchInputAndNoEmbeddingGeneratorWasConfigured(searchValue.GetType(), RedisModelBuilder.SupportedVectorTypes)) + : throw new InvalidOperationException(VectorDataStrings.IncompatibleEmbeddingGeneratorWasConfiguredForInputType(typeof(TInput), vectorProperty.EmbeddingGenerator.GetType())) + }; + + // Build query & search. + byte[] vectorBytes = RedisCollectionSearchMapping.ValidateVectorAndConvertToBytes(vector, "JSON"); + var query = RedisCollectionSearchMapping.BuildQuery( + vectorBytes, + top, + options, + _model, + vectorProperty, + null); + var results = await RunOperationAsync( + "FT.SEARCH", + () => _database + .FT() + .SearchAsync(Name, query)).ConfigureAwait(false); + + // Loop through result and convert to the caller's data model. + var mappedResults = results.Documents.Select(result => + { + var redisResultString = result["json"].ToString(); + var node = JsonSerializer.Deserialize(redisResultString, _jsonSerializerOptions)!; + var mappedRecord = _mapper.MapFromStorageToDataModel( + (RemoveKeyPrefixIfNeeded(result.Id), node), + options.IncludeVectors); + + // Process the score of the result item. + var vectorProperty = _model.GetVectorPropertyOrSingle(options); + var distanceFunction = RedisCollectionSearchMapping.ResolveDistanceFunction(vectorProperty); + var score = RedisCollectionSearchMapping.GetOutputScoreFromRedisScore(result["vector_score"].HasValue ? (float)result["vector_score"] : null, distanceFunction); + + return new VectorSearchResult(mappedRecord, score); + }); + + foreach (var result in mappedResults) + { + // Apply score threshold filtering. The score semantics depend on the distance function: + // - For similarity functions (CosineSimilarity, DotProductSimilarity): higher = more similar, filter out below threshold + // - For distance functions (CosineDistance, EuclideanSquaredDistance): lower = more similar, filter out above threshold + if (options.ScoreThreshold.HasValue && result.Score.HasValue) + { + var distanceFunction = RedisCollectionSearchMapping.ResolveDistanceFunction(vectorProperty); + var passesThreshold = distanceFunction switch + { + DistanceFunction.CosineSimilarity or DistanceFunction.DotProductSimilarity => result.Score.Value >= options.ScoreThreshold.Value, + DistanceFunction.CosineDistance or DistanceFunction.EuclideanSquaredDistance => result.Score.Value <= options.ScoreThreshold.Value, + _ => throw new InvalidOperationException($"Unexpected distance function: {distanceFunction}") + }; + + if (!passesThreshold) + { + continue; + } + } + + yield return result; + } + } + + #endregion Search + + /// + public override async IAsyncEnumerable GetAsync(Expression> filter, int top, + FilteredRecordRetrievalOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + Throw.IfNull(filter); + Throw.IfLessThan(top, 1); + + if (options?.IncludeVectors == true && _model.EmbeddingGenerationRequired) + { + throw new NotSupportedException(VectorDataStrings.IncludeVectorsNotSupportedWithEmbeddingGeneration); + } + + Query query = RedisCollectionSearchMapping.BuildQuery(filter, top, options ??= new(), _model); + + var results = await RunOperationAsync( + "FT.SEARCH", + () => _database + .FT() + .SearchAsync(Name, query)).ConfigureAwait(false); + + foreach (var document in results.Documents) + { + var redisResultString = document["json"].ToString(); + var node = JsonSerializer.Deserialize(redisResultString, _jsonSerializerOptions)!; + yield return _mapper.MapFromStorageToDataModel( + (RemoveKeyPrefixIfNeeded(document.Id), node), + options.IncludeVectors); + } + } + + /// + public override object? GetService(Type serviceType, object? serviceKey = null) + { + Throw.IfNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreCollectionMetadata) ? _collectionMetadata : + serviceType == typeof(IDatabase) ? _database : + serviceType.IsInstanceOfType(this) ? this : + null; + } + + /// + /// Prefix the key with the collection name if the option is set. + /// + /// The key to prefix. + /// The updated key if updating is required, otherwise the input key. + private string PrefixKeyIfNeeded(string key) + { + if (_prefixCollectionNameToKeyNames) + { + return $"{Name}:{key}"; + } + + return key; + } + + /// + /// Remove the prefix of the given key if the option is set. + /// + /// The key to remove a prefix from. + /// The updated key if updating is required, otherwise the input key. + private string RemoveKeyPrefixIfNeeded(string key) + { + var prefixLength = Name.Length + 1; + + if (_prefixCollectionNameToKeyNames && key.Length > prefixLength) + { + return key.Substring(prefixLength); + } + + return key; + } + + /// + /// Run the given operation and wrap any Redis exceptions with ."/> + /// + /// The response type of the operation. + /// The type of database operation being run. + /// The operation to run. + /// The result of the operation. + private Task RunOperationAsync(string operationName, Func> operation) + => VectorStoreErrorHandler.RunOperationAsync( + _collectionMetadata, + operationName, + operation); + + /// + /// Run the given operation and wrap any Redis exceptions with ."/> + /// + /// The type of database operation being run. + /// The operation to run. + /// The result of the operation. + private Task RunOperationAsync(string operationName, Func operation) + => VectorStoreErrorHandler.RunOperationAsync( + _collectionMetadata, + operationName, + operation); + + private string GetStringKey(TKey key) + { + Throw.IfNull(key); + + var stringKey = key switch + { + string s => s, + Guid g => g.ToString(), + + _ => throw new UnreachableException("string key should have been validated during model building") + }; + + Throw.IfNullOrWhitespace(stringKey); + + return stringKey; + } +} diff --git a/MEVD/src/Redis/RedisJsonCollectionOptions.cs b/MEVD/src/Redis/RedisJsonCollectionOptions.cs new file mode 100644 index 0000000..510b755 --- /dev/null +++ b/MEVD/src/Redis/RedisJsonCollectionOptions.cs @@ -0,0 +1,43 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text.Json; +using Microsoft.Extensions.VectorData; + +namespace CommunityToolkit.VectorData.Redis; + +/// +/// Options when creating a . +/// +public sealed class RedisJsonCollectionOptions : VectorStoreCollectionOptions +{ + internal static readonly RedisJsonCollectionOptions Default = new(); + + /// + /// Initializes a new instance of the class. + /// + public RedisJsonCollectionOptions() + { + } + + internal RedisJsonCollectionOptions(RedisJsonCollectionOptions? source) : base(source) + { + PrefixCollectionNameToKeyNames = source?.PrefixCollectionNameToKeyNames ?? Default.PrefixCollectionNameToKeyNames; + JsonSerializerOptions = source?.JsonSerializerOptions; + } + + /// + /// Gets or sets a value indicating whether the collection name should be prefixed to the + /// key names before reading or writing to the Redis store. Default is true. + /// + /// + /// For a record to be indexed by a specific Redis index, the key name must be prefixed with the matching prefix configured on the Redis index. + /// You can either pass in keys that are already prefixed, or set this option to true to have the collection name prefixed to the key names automatically. + /// + public bool PrefixCollectionNameToKeyNames { get; set; } = true; + + /// + /// Gets or sets the JSON serializer options to use when converting between the data model and the Redis record. + /// + public JsonSerializerOptions? JsonSerializerOptions { get; set; } +} diff --git a/MEVD/src/Redis/RedisJsonDynamicCollection.cs b/MEVD/src/Redis/RedisJsonDynamicCollection.cs new file mode 100644 index 0000000..802a828 --- /dev/null +++ b/MEVD/src/Redis/RedisJsonDynamicCollection.cs @@ -0,0 +1,36 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using StackExchange.Redis; + +namespace CommunityToolkit.VectorData.Redis; + +/// +/// Represents a collection of vector store records in a Redis JSON database, mapped to a dynamic Dictionary<string, object?>. +/// +#pragma warning disable CA1711 // Identifiers should not have incorrect suffix +public sealed class RedisJsonDynamicCollection : RedisJsonCollection> +#pragma warning restore CA1711 // Identifiers should not have incorrect suffix +{ + /// + /// Initializes a new instance of the class. + /// + /// The Redis database to read/write records from. + /// The name of the collection. + /// Optional configuration options for this class. + // TODO: The provider uses unsafe JSON serialization in many places, #11963 + [RequiresUnreferencedCode("The Redis provider is currently incompatible with trimming.")] + [RequiresDynamicCode("The Redis provider is currently incompatible with NativeAOT.")] + public RedisJsonDynamicCollection(IDatabase database, string name, RedisJsonCollectionOptions options) + : base( + database, + name, + static options => new RedisJsonDynamicModelBuilder(ModelBuildingOptions) + .BuildDynamic( + options.Definition ?? throw new ArgumentException("Definition is required for dynamic collections"), + options.EmbeddingGenerator), + options) + { + } +} diff --git a/MEVD/src/Redis/RedisJsonDynamicMapper.cs b/MEVD/src/Redis/RedisJsonDynamicMapper.cs new file mode 100644 index 0000000..f008e30 --- /dev/null +++ b/MEVD/src/Redis/RedisJsonDynamicMapper.cs @@ -0,0 +1,183 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using System.Text.Json; +using System.Text.Json.Nodes; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData.ProviderServices; + +namespace CommunityToolkit.VectorData.Redis; + +/// +/// A mapper that maps between the generic Semantic Kernel data model and the model that the data is stored under, within Redis when using JSON. +/// +internal class RedisJsonDynamicMapper(CollectionModel model, JsonSerializerOptions jsonSerializerOptions) : IRedisJsonMapper> +{ + /// + public (string Key, JsonNode Node) MapFromDataToStorageModel(Dictionary dataModel, int recordIndex, IReadOnlyList?[]? generatedEmbeddings) + { + var jsonObject = new JsonObject(); + + // Key handled below, outside of the JsonNode + + foreach (var dataProperty in model.DataProperties) + { + if (dataModel.TryGetValue(dataProperty.ModelName, out var sourceValue)) + { + jsonObject.Add(dataProperty.StorageName, sourceValue is null + ? null + : JsonSerializer.SerializeToNode(sourceValue, dataProperty.Type, jsonSerializerOptions)); + } + } + + for (var i = 0; i < model.VectorProperties.Count; i++) + { + var property = model.VectorProperties[i]; + + // Don't create a property if it doesn't exist in the dictionary + if (dataModel.TryGetValue(property.ModelName, out var vectorValue)) + { + var vector = generatedEmbeddings?[i]?[recordIndex] is Embedding ge + ? ge + : vectorValue; + + if (vector is null) + { + jsonObject[property.StorageName] = null; + continue; + } + + var jsonArray = new JsonArray(); + + if (vector switch + { + ReadOnlyMemory m => m, + Embedding e => e.Vector, + float[] a => new ReadOnlyMemory(a), + _ => (ReadOnlyMemory?)null + } is ReadOnlyMemory floatMemory) + { + foreach (var item in floatMemory.Span) + { + jsonArray.Add(JsonValue.Create(item)); + } + } + else if (vector switch + { + ReadOnlyMemory m => m, + Embedding e => e.Vector, + double[] a => new ReadOnlyMemory(a), + _ => null + } is ReadOnlyMemory doubleMemory) + { + foreach (var item in doubleMemory.Span) + { + jsonArray.Add(JsonValue.Create(item)); + } + } + else + { + throw new UnreachableException(); + } + + jsonObject.Add(property.StorageName, jsonArray); + } + } + + var storageKey = dataModel[model.KeyProperty.ModelName] switch + { + string s => s, + Guid g => g.ToString(), + + _ => throw new UnreachableException() + }; + + return (storageKey, jsonObject); + } + + /// + public Dictionary MapFromStorageToDataModel((string Key, JsonNode Node) storageModel, bool includeVectors) + { + var dataModel = new Dictionary + { + [model.KeyProperty.ModelName] = model.KeyProperty.Type switch + { + Type t when t == typeof(string) => storageModel.Key, + Type t when t == typeof(Guid) => Guid.Parse(storageModel.Key), + + _ => throw new UnreachableException() + }, + }; + + // The redis result can be either a single object or an array with a single object in the case where we are doing an MGET. + // If there's a single data property, we get a simple value (no object wrapper). + var jsonObject = storageModel.Node switch + { + JsonValue v when model.DataProperties is [var singleDataProperty] => new JsonObject([new(singleDataProperty.StorageName, v)]), + JsonObject o => o, + JsonArray a and [JsonObject arrayEntryJsonObject] => arrayEntryJsonObject, + + _ => throw new InvalidOperationException($"Invalid data format for document with key '{storageModel.Key}'"), + }; + + // The key was handled above + + foreach (var dataProperty in model.DataProperties) + { + // Replicate null if the property exists but is null. + if (jsonObject.TryGetPropertyValue(dataProperty.StorageName, out var sourceValue)) + { + dataModel.Add(dataProperty.ModelName, sourceValue is null + ? null + : JsonSerializer.Deserialize(sourceValue, dataProperty.Type, jsonSerializerOptions)); + } + } + + if (includeVectors) + { + foreach (var vectorProperty in model.VectorProperties) + { + // Replicate null if the property exists but is null. + if (jsonObject.TryGetPropertyValue(vectorProperty.StorageName, out var sourceValue)) + { + if (sourceValue is null) + { + dataModel.Add(vectorProperty.ModelName, null); + continue; + } + + dataModel.Add( + vectorProperty.ModelName, + (Nullable.GetUnderlyingType(vectorProperty.Type) ?? vectorProperty.Type) switch + { + Type t when t == typeof(ReadOnlyMemory) => new ReadOnlyMemory(ToArray(sourceValue)), + Type t when t == typeof(Embedding) => new Embedding(ToArray(sourceValue)), + Type t when t == typeof(float[]) => ToArray(sourceValue), + + Type t when t == typeof(ReadOnlyMemory) => new ReadOnlyMemory(ToArray(sourceValue)), + Type t when t == typeof(Embedding) => new Embedding(ToArray(sourceValue)), + Type t when t == typeof(double[]) => ToArray(sourceValue), + + _ => throw new UnreachableException() + }); + } + } + } + + return dataModel; + + static T[] ToArray(JsonNode jsonNode) + { + var jsonArray = jsonNode.AsArray(); + var array = new T[jsonArray.Count]; + + for (var i = 0; i < jsonArray.Count; i++) + { + array[i] = jsonArray[i]!.GetValue(); + } + + return array; + } + } +} diff --git a/MEVD/src/Redis/RedisJsonDynamicModelBuilder.cs b/MEVD/src/Redis/RedisJsonDynamicModelBuilder.cs new file mode 100644 index 0000000..575573d --- /dev/null +++ b/MEVD/src/Redis/RedisJsonDynamicModelBuilder.cs @@ -0,0 +1,60 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData.ProviderServices; + +namespace CommunityToolkit.VectorData.Redis; + +internal class RedisJsonDynamicModelBuilder(CollectionModelBuildingOptions options) : CollectionModelBuilder(options) +{ + /// + protected override IReadOnlyList EmbeddingGenerationDispatchers { get; } = + [ + EmbeddingGenerationDispatcher.Create>(), + EmbeddingGenerationDispatcher.Create>() + ]; + + protected override void ValidateKeyProperty(KeyPropertyModel keyProperty) + { + base.ValidateKeyProperty(keyProperty); + + var type = keyProperty.Type; + + if (type != typeof(string) && type != typeof(Guid)) + { + throw new NotSupportedException( + $"Property '{keyProperty.ModelName}' has unsupported type '{type.Name}'. Key properties must be one of the supported types: string, Guid."); + } + } + + protected override bool IsDataPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes) + { + // TODO: Validate data property types + + supportedTypes = ""; + + return true; + } + + protected override bool IsVectorPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes) + => IsVectorPropertyTypeValidCore(type, out supportedTypes); + + internal static bool IsVectorPropertyTypeValidCore(Type type, [NotNullWhen(false)] out string? supportedTypes) + { + supportedTypes = RedisModelBuilder.SupportedVectorTypes; + + if (Nullable.GetUnderlyingType(type) is Type underlyingType) + { + type = underlyingType; + } + + return type == typeof(ReadOnlyMemory) + || type == typeof(Embedding) + || type == typeof(float[]) + || type == typeof(ReadOnlyMemory) + || type == typeof(Embedding) + || type == typeof(double[]); + } +} diff --git a/MEVD/src/Redis/RedisJsonMapper.cs b/MEVD/src/Redis/RedisJsonMapper.cs new file mode 100644 index 0000000..ea0af1b --- /dev/null +++ b/MEVD/src/Redis/RedisJsonMapper.cs @@ -0,0 +1,159 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using System.Text.Json; +using System.Text.Json.Nodes; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData.ProviderServices; + +namespace CommunityToolkit.VectorData.Redis; + +/// +/// Class for mapping between a json node stored in redis, and the consumer data model. +/// +/// The consumer data model to map to or from. +internal sealed class RedisJsonMapper( + CollectionModel model, + JsonSerializerOptions jsonSerializerOptions) + : IRedisJsonMapper + where TConsumerDataModel : class +{ + /// The key property. + private readonly string _keyPropertyStorageName = model.KeyProperty.StorageName; + + /// + public (string Key, JsonNode Node) MapFromDataToStorageModel(TConsumerDataModel dataModel, int recordIndex, IReadOnlyList?[]? generatedEmbeddings) + { + // Convert the provided record into a JsonNode object and try to get the key field for it. + // Since we already checked that the key field is a string in the constructor, and that it exists on the model, + // the only edge case we have to be concerned about is if the key field is null. + var jsonNode = JsonSerializer.SerializeToNode(dataModel, jsonSerializerOptions)!.AsObject(); + + if (!(jsonNode.TryGetPropertyValue(_keyPropertyStorageName, out var keyField) && keyField is JsonValue jsonValue)) + { + throw new InvalidOperationException($"Missing key field '{_keyPropertyStorageName}' on provided record of type {typeof(TConsumerDataModel).FullName}."); + } + + // Remove the key field from the JSON object since we don't want to store it in the redis payload. + var keyValue = jsonValue.ToString(); + jsonNode.Remove(_keyPropertyStorageName); + + // Go over the vector properties; inject any generated embeddings to overwrite the JSON serialized above. + // Also, for Embedding properties we also need to overwrite with a simple array (since Embedding gets serialized as a complex object). + for (var i = 0; i < model.VectorProperties.Count; i++) + { + var property = model.VectorProperties[i]; + + Embedding? embedding = generatedEmbeddings?[i]?[recordIndex] is Embedding ge ? ge : null; + + if (embedding is null) + { + switch (Nullable.GetUnderlyingType(property.Type) ?? property.Type) + { + case var t when t == typeof(ReadOnlyMemory): + case var t2 when t2 == typeof(float[]): + case var t3 when t3 == typeof(ReadOnlyMemory): + case var t4 when t4 == typeof(double[]): + // The .NET vector property is a ReadOnlyMemory or T[] (not an Embedding), which means that JsonSerializer + // already serialized it correctly above. + // In addition, there's no generated embedding (which would be an Embedding which we'd need to handle manually). + // So there's nothing for us to do. + continue; + + case var t when t == typeof(Embedding): + case var t1 when t1 == typeof(Embedding): + embedding = (Embedding)property.GetValueAsObject(dataModel)!; + break; + + default: + throw new UnreachableException(); + } + } + + var jsonArray = new JsonArray(); + + switch (embedding) + { + case Embedding e: + foreach (var item in e.Vector.Span) + { + jsonArray.Add(JsonValue.Create(item)); + } + break; + + case Embedding e: + foreach (var item in e.Vector.Span) + { + jsonArray.Add(JsonValue.Create(item)); + } + break; + + default: + throw new UnreachableException(); + } + + jsonNode[property.StorageName] = jsonArray; + } + + return (keyValue, jsonNode); + } + + /// + public TConsumerDataModel MapFromStorageToDataModel((string Key, JsonNode Node) storageModel, bool includeVectors) + { + // The redis result can have one of three different formats: + // 1. a single object + // 2. an array with a single object in the case where we are doing an MGET + // 3. a single value (string, number, etc.) in the case where there is only one property being requested because the model has only one property apart from the key + var jsonObject = storageModel.Node switch + { + JsonObject topLevelJsonObject => topLevelJsonObject, + JsonArray and [JsonObject arrayEntryJsonObject] => arrayEntryJsonObject, + JsonValue when model.DataProperties.Count + (includeVectors ? model.VectorProperties.Count : 0) == 1 => new JsonObject + { + [model.DataProperties.Concat(model.VectorProperties).First().StorageName] = storageModel.Node + }, + _ => throw new InvalidOperationException($"Invalid data format for document with key '{storageModel.Key}'") + }; + + // Check that the key field is not already present in the redis value. + if (jsonObject.ContainsKey(_keyPropertyStorageName)) + { + throw new InvalidOperationException($"Invalid data format for document with key '{storageModel.Key}'. Key property '{_keyPropertyStorageName}' is already present on retrieved object."); + } + + // Since the key is not stored in the redis value, add it back in before deserializing into the data model. + jsonObject.Add(_keyPropertyStorageName, storageModel.Key); + + // For vector properties which have embedding generation configured, we need to remove the embeddings before deserializing + // (we can't go back from an embedding to e.g. string). + if (includeVectors) + { + foreach (var vectorProperty in model.VectorProperties) + { + if (vectorProperty.Type == typeof(Embedding) || vectorProperty.Type == typeof(Embedding)) + { + var arrayNode = jsonObject[vectorProperty.StorageName]; + if (arrayNode is not null) + { + var embeddingNode = new JsonObject + { + [nameof(Embedding.Vector)] = arrayNode.DeepClone() + }; + jsonObject[vectorProperty.StorageName] = embeddingNode; + } + } + } + } + else + { + foreach (var vectorProperty in model.VectorProperties) + { + jsonObject.Remove(vectorProperty.StorageName); + } + } + + return JsonSerializer.Deserialize(jsonObject, jsonSerializerOptions)!; + } +} diff --git a/MEVD/src/Redis/RedisJsonModelBuilder.cs b/MEVD/src/Redis/RedisJsonModelBuilder.cs new file mode 100644 index 0000000..dc0c48a --- /dev/null +++ b/MEVD/src/Redis/RedisJsonModelBuilder.cs @@ -0,0 +1,69 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; + +namespace CommunityToolkit.VectorData.Redis; + +internal class RedisJsonModelBuilder(CollectionModelBuildingOptions options) : CollectionJsonModelBuilder(options) +{ + /// + protected override IReadOnlyList EmbeddingGenerationDispatchers { get; } = + [ + EmbeddingGenerationDispatcher.Create>(), + EmbeddingGenerationDispatcher.Create>() + ]; + + protected override void ValidateKeyProperty(KeyPropertyModel keyProperty) + { + base.ValidateKeyProperty(keyProperty); + + var type = keyProperty.Type; + + if (type != typeof(string) && type != typeof(Guid)) + { + throw new NotSupportedException( + $"Property '{keyProperty.ModelName}' has unsupported type '{type.Name}'. Key properties must be one of the supported types: string, Guid."); + } + } + + protected override bool IsDataPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes) + { + // TODO: Validate data property types + + supportedTypes = ""; + + return true; + } + + protected override bool IsVectorPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes) + => IsVectorPropertyTypeValidCore(type, out supportedTypes); + + internal static bool IsVectorPropertyTypeValidCore(Type type, [NotNullWhen(false)] out string? supportedTypes) + { + supportedTypes = RedisModelBuilder.SupportedVectorTypes; + + if (Nullable.GetUnderlyingType(type) is Type underlyingType) + { + type = underlyingType; + } + + return type == typeof(ReadOnlyMemory) + || type == typeof(Embedding) + || type == typeof(float[]) + || type == typeof(ReadOnlyMemory) + || type == typeof(Embedding) + || type == typeof(double[]); + } + + /// + protected override void ValidateProperty(PropertyModel propertyModel, VectorStoreCollectionDefinition? definition) + { + base.ValidateProperty(propertyModel, definition); + + RedisModelBuilder.ValidateStorageName(propertyModel); + } +} diff --git a/MEVD/src/Redis/RedisModelBuilder.cs b/MEVD/src/Redis/RedisModelBuilder.cs new file mode 100644 index 0000000..58e6f22 --- /dev/null +++ b/MEVD/src/Redis/RedisModelBuilder.cs @@ -0,0 +1,116 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; + +namespace CommunityToolkit.VectorData.Redis; + +internal class RedisModelBuilder(CollectionModelBuildingOptions options) : CollectionModelBuilder(options) +{ + internal const string SupportedVectorTypes = "ReadOnlyMemory, Embedding, float[], ReadOnlyMemory, Embedding, double[]"; + + /// + protected override IReadOnlyList EmbeddingGenerationDispatchers { get; } = + [ + EmbeddingGenerationDispatcher.Create>(), + EmbeddingGenerationDispatcher.Create>() + ]; + + protected override void ValidateKeyProperty(KeyPropertyModel keyProperty) + { + base.ValidateKeyProperty(keyProperty); + + var type = keyProperty.Type; + + if (type != typeof(string) && type != typeof(Guid)) + { + throw new NotSupportedException( + $"Property '{keyProperty.ModelName}' has unsupported type '{type.Name}'. Key properties must be one of the supported types: string, Guid."); + } + } + + protected override bool IsDataPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes) + { + supportedTypes = "string, int, uint, long, ulong, double, float"; + + if (Nullable.GetUnderlyingType(type) is Type underlyingType) + { + type = underlyingType; + } + + return type == typeof(string) + || type == typeof(int) + || type == typeof(uint) + || type == typeof(long) + || type == typeof(ulong) + || type == typeof(double) + || type == typeof(float); + } + + protected override bool IsVectorPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes) + => IsVectorPropertyTypeValidCore(type, out supportedTypes); + + internal static bool IsVectorPropertyTypeValidCore(Type type, [NotNullWhen(false)] out string? supportedTypes) + { + supportedTypes = SupportedVectorTypes; + + if (Nullable.GetUnderlyingType(type) is Type underlyingType) + { + type = underlyingType; + } + + return type == typeof(ReadOnlyMemory) + || type == typeof(Embedding) + || type == typeof(float[]) + || type == typeof(ReadOnlyMemory) + || type == typeof(Embedding) + || type == typeof(double[]); + } + + /// + protected override void ValidateProperty(PropertyModel propertyModel, VectorStoreCollectionDefinition? definition) + { + base.ValidateProperty(propertyModel, definition); + + ValidateStorageName(propertyModel); + } + + internal static void ValidateStorageName(PropertyModel propertyModel) + { + // RediSearch field names cannot be escaped in all contexts; storage names are validated during model building. + if (!IsValidIdentifier(propertyModel.StorageName)) + { + throw new InvalidOperationException( + $"Property '{propertyModel.ModelName}' has storage name '{propertyModel.StorageName}' which is not a valid RediSearch field name. " + + "RediSearch field names must start with a letter or underscore, and contain only letters, digits, and underscores."); + } + } + + internal static bool IsValidIdentifier(string name) + { + if (string.IsNullOrEmpty(name)) + { + return false; + } + + var first = name[0]; + if (!char.IsLetter(first) && first != '_') + { + return false; + } + + for (var i = 1; i < name.Length; i++) + { + var c = name[i]; + if (!char.IsLetterOrDigit(c) && c != '_') + { + return false; + } + } + + return true; + } +} diff --git a/MEVD/src/Redis/RedisServiceCollectionExtensions.cs b/MEVD/src/Redis/RedisServiceCollectionExtensions.cs new file mode 100644 index 0000000..3fe264c --- /dev/null +++ b/MEVD/src/Redis/RedisServiceCollectionExtensions.cs @@ -0,0 +1,363 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData; +using CommunityToolkit.VectorData.Redis; +using StackExchange.Redis; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.DependencyInjection; + +/// +/// Extension methods to register , and instances on an . +/// +public static class RedisServiceCollectionExtensions +{ + private const string DynamicCodeMessage = "This method is incompatible with NativeAOT, consult the documentation for adding collections in a way that's compatible with NativeAOT."; + private const string UnreferencedCodeMessage = "This method is incompatible with trimming, consult the documentation for adding collections in a way that's compatible with NativeAOT."; + + /// + /// Registers a as + /// with returned by or retrieved from the dependency injection container if was not provided. + /// + /// + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddRedisVectorStore( + this IServiceCollection services, + Func? clientProvider = default, + Func? optionsProvider = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + => AddKeyedRedisVectorStore(services, serviceKey: null, clientProvider, optionsProvider, lifetime); + + /// + /// Registers a keyed as + /// with returned by or retrieved from the dependency injection container if was not provided. + /// + /// The to register the on. + /// The key with which to associate the vector store. + /// The provider. + /// Options provider to further configure the . + /// The service lifetime for the store. Defaults to . + /// Service collection. + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddKeyedRedisVectorStore( + this IServiceCollection services, + object? serviceKey, + Func? clientProvider = default, + Func? optionsProvider = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + { + Throw.IfNull(services); + + services.Add(new ServiceDescriptor(typeof(RedisVectorStore), serviceKey, (sp, _) => + { + var client = clientProvider is null ? sp.GetRequiredService() : clientProvider(sp); + var options = GetStoreOptions(sp, optionsProvider); + + return new RedisVectorStore(client, options); + }, lifetime)); + + services.Add(new ServiceDescriptor(typeof(VectorStore), serviceKey, + static (sp, key) => sp.GetRequiredKeyedService(key), lifetime)); + + return services; + } + + /// + /// Registers a as + /// with created with . + /// + /// + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddRedisVectorStore( + this IServiceCollection services, + string connectionConfiguration, + RedisVectorStoreOptions? options = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + => AddKeyedRedisVectorStore(services, serviceKey: null, connectionConfiguration, options, lifetime); + + /// + /// Registers a keyed as + /// with created with . + /// + /// The to register the on. + /// The key with which to associate the vector store. + /// The connectionConfiguration passed to . + /// Options to further configure the . + /// The service lifetime for the store. Defaults to . + /// Service collection. + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddKeyedRedisVectorStore( + this IServiceCollection services, + object? serviceKey, + string connectionConfiguration, + RedisVectorStoreOptions? options = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + { + Throw.IfNullOrWhitespace(connectionConfiguration); + + return AddKeyedRedisVectorStore(services, serviceKey, _ => ConnectionMultiplexer.Connect(connectionConfiguration).GetDatabase(), sp => options!, lifetime); + } + + /// + /// Registers a as + /// with returned by or retrieved from the dependency injection container if was not provided. + /// + /// + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddRedisJsonCollection( + this IServiceCollection services, + string name, + Func? clientProvider = default, + Func? optionsProvider = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + where TRecord : class + => AddKeyedRedisJsonCollection(services, serviceKey: null, name, clientProvider, optionsProvider, lifetime); + + /// + /// Registers a keyed as + /// with returned by or retrieved from the dependency injection container if was not provided. + /// + /// The to register the on. + /// The key with which to associate the collection. + /// The name of the collection. + /// The provider. + /// Options provider to further configure the . + /// The service lifetime for the store. Defaults to . + /// Service collection. + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddKeyedRedisJsonCollection( + this IServiceCollection services, + object? serviceKey, + string name, + Func? clientProvider = default, + Func? optionsProvider = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + where TRecord : class + { + Throw.IfNull(services); + Throw.IfNullOrWhitespace(name); + + services.Add(new ServiceDescriptor(typeof(RedisJsonCollection), serviceKey, (sp, _) => + { + var client = clientProvider is null ? sp.GetRequiredService() : clientProvider(sp); + var options = GetCollectionOptions(sp, optionsProvider); + + return new RedisJsonCollection(client, name, options); + }, lifetime)); + + services.Add(new ServiceDescriptor(typeof(VectorStoreCollection), serviceKey, + static (sp, key) => sp.GetRequiredKeyedService>(key), lifetime)); + + services.Add(new ServiceDescriptor(typeof(IVectorSearchable), serviceKey, + static (sp, key) => sp.GetRequiredKeyedService>(key), lifetime)); + + return services; + } + + /// + /// Registers a as + /// with created with . + /// + /// + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddRedisJsonCollection( + this IServiceCollection services, + string name, + string connectionConfiguration, + RedisJsonCollectionOptions? options = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + where TRecord : class + => AddKeyedRedisJsonCollection(services, serviceKey: null, name, connectionConfiguration, options, lifetime); + + /// + /// Registers a keyed as + /// with created with . + /// + /// The to register the on. + /// The key with which to associate the collection. + /// The name of the collection. + /// The connectionConfiguration passed to . + /// Options to further configure the . + /// The service lifetime for the store. Defaults to . + /// Service collection. + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddKeyedRedisJsonCollection( + this IServiceCollection services, + object? serviceKey, + string name, + string connectionConfiguration, + RedisJsonCollectionOptions? options = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + where TRecord : class + { + Throw.IfNullOrWhitespace(connectionConfiguration); + + return AddKeyedRedisJsonCollection( + services, + serviceKey, + name, + _ => ConnectionMultiplexer.Connect(connectionConfiguration).GetDatabase(), + sp => options!, + lifetime); + } + + /// + /// Registers a as + /// with returned by or retrieved from the dependency injection container if was not provided. + /// + /// + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddRedisHashSetCollection( + this IServiceCollection services, + string name, + Func? clientProvider = default, + Func? optionsProvider = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + where TRecord : class + => AddKeyedRedisHashSetCollection(services, serviceKey: null, name, clientProvider, optionsProvider, lifetime); + + /// + /// Registers a keyed as + /// with returned by or retrieved from the dependency injection container if was not provided. + /// + /// The to register the on. + /// The key with which to associate the collection. + /// The name of the collection. + /// The provider. + /// Options provider to further configure the . + /// The service lifetime for the store. Defaults to . + /// Service collection. + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddKeyedRedisHashSetCollection( + this IServiceCollection services, + object? serviceKey, + string name, + Func? clientProvider = default, + Func? optionsProvider = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + where TRecord : class + { + Throw.IfNull(services); + Throw.IfNullOrWhitespace(name); + + services.Add(new ServiceDescriptor(typeof(RedisHashSetCollection), serviceKey, (sp, _) => + { + var client = clientProvider is null ? sp.GetRequiredService() : clientProvider(sp); + var options = GetCollectionOptions(sp, optionsProvider); + + return new RedisHashSetCollection(client, name, options); + }, lifetime)); + + services.Add(new ServiceDescriptor(typeof(VectorStoreCollection), serviceKey, + static (sp, key) => sp.GetRequiredKeyedService>(key), lifetime)); + + services.Add(new ServiceDescriptor(typeof(IVectorSearchable), serviceKey, + static (sp, key) => sp.GetRequiredKeyedService>(key), lifetime)); + + return services; + } + + /// + /// Registers a as + /// with created with . + /// + /// + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddRedisHashSetCollection( + this IServiceCollection services, + string name, + string connectionConfiguration, + RedisHashSetCollectionOptions? options = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + where TRecord : class + => AddKeyedRedisHashSetCollection(services, serviceKey: null, name, connectionConfiguration, options, lifetime); + + /// + /// Registers a keyed as + /// with created with . + /// + /// The to register the on. + /// The key with which to associate the collection. + /// The name of the collection. + /// The connectionConfiguration passed to . + /// Options to further configure the . + /// The service lifetime for the store. Defaults to . + /// Service collection. + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddKeyedRedisHashSetCollection( + this IServiceCollection services, + object? serviceKey, + string name, + string connectionConfiguration, + RedisHashSetCollectionOptions? options = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + where TRecord : class + { + Throw.IfNullOrWhitespace(connectionConfiguration); + + return AddKeyedRedisHashSetCollection( + services, + serviceKey, + name, + _ => ConnectionMultiplexer.Connect(connectionConfiguration).GetDatabase(), + sp => options!, + lifetime); + } + + private static RedisVectorStoreOptions? GetStoreOptions(IServiceProvider sp, Func? optionsProvider) + { + var options = optionsProvider?.Invoke(sp); + if (options?.EmbeddingGenerator is not null) + { + return options; // The user has provided everything, there is nothing to change. + } + + var embeddingGenerator = sp.GetService(); + return embeddingGenerator is null + ? options // There is nothing to change. + : new(options) { EmbeddingGenerator = embeddingGenerator }; // Create a brand new copy in order to avoid modifying the original options. + } + + private static RedisJsonCollectionOptions? GetCollectionOptions(IServiceProvider sp, Func? optionsProvider) + { + var options = optionsProvider?.Invoke(sp); + if (options?.EmbeddingGenerator is not null) + { + return options; // The user has provided everything, there is nothing to change. + } + + var embeddingGenerator = sp.GetService(); + return embeddingGenerator is null + ? options // There is nothing to change. + : new(options) { EmbeddingGenerator = embeddingGenerator }; // Create a brand new copy in order to avoid modifying the original options. + } + + private static RedisHashSetCollectionOptions? GetCollectionOptions(IServiceProvider sp, Func? optionsProvider) + { + var options = optionsProvider?.Invoke(sp); + if (options?.EmbeddingGenerator is not null) + { + return options; // The user has provided everything, there is nothing to change. + } + + var embeddingGenerator = sp.GetService(); + return embeddingGenerator is null + ? options // There is nothing to change. + : new(options) { EmbeddingGenerator = embeddingGenerator }; // Create a brand new copy in order to avoid modifying the original options. + } +} diff --git a/MEVD/src/Redis/RedisStorageType.cs b/MEVD/src/Redis/RedisStorageType.cs new file mode 100644 index 0000000..af45051 --- /dev/null +++ b/MEVD/src/Redis/RedisStorageType.cs @@ -0,0 +1,20 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace CommunityToolkit.VectorData.Redis; + +/// +/// Indicates the way in which data is stored in redis. +/// +public enum RedisStorageType +{ + /// + /// Data is stored as JSON. + /// + Json, + + /// + /// Data is stored as collections of field-value pairs. + /// + HashSet +} diff --git a/MEVD/src/Redis/RedisVectorStore.cs b/MEVD/src/Redis/RedisVectorStore.cs new file mode 100644 index 0000000..156d98b --- /dev/null +++ b/MEVD/src/Redis/RedisVectorStore.cs @@ -0,0 +1,159 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; +using NRedisStack.RedisStackCommands; +using StackExchange.Redis; +using Microsoft.Shared.Diagnostics; + +namespace CommunityToolkit.VectorData.Redis; + +/// +/// Class for accessing the list of collections in a Redis vector store. +/// +/// +/// This class can be used with collections of any schema type, but requires you to provide schema information when getting a collection. +/// +public sealed class RedisVectorStore : VectorStore +{ + /// Metadata about vector store. + private readonly VectorStoreMetadata _metadata; + + /// The redis database to read/write indices from. + private readonly IDatabase _database; + + /// A general purpose definition that can be used to construct a collection when needing to proxy schema agnostic operations. + private static readonly VectorStoreCollectionDefinition s_generalPurposeDefinition = new() { Properties = [new VectorStoreKeyProperty("Key", typeof(string))] }; + + /// The way in which data should be stored in redis.. + private readonly RedisStorageType? _storageType; + + private readonly IEmbeddingGenerator? _embeddingGenerator; + + /// + /// Initializes a new instance of the class. + /// + /// The redis database to read/write indices from. + /// Optional configuration options for this class. + [RequiresUnreferencedCode("The Redis provider is currently incompatible with trimming.")] + [RequiresDynamicCode("The Redis provider is currently incompatible with NativeAOT.")] + public RedisVectorStore(IDatabase database, RedisVectorStoreOptions? options = default) + { + Throw.IfNull(database); + + _database = database; + + options ??= RedisVectorStoreOptions.Default; + _storageType = options.StorageType; + _embeddingGenerator = options.EmbeddingGenerator; + + _metadata = new() + { + VectorStoreSystemName = RedisConstants.VectorStoreSystemName, + VectorStoreName = database.Database.ToString() + }; + } + + /// + // TODO: The provider uses unsafe JSON serialization in many places, #11963 + [RequiresUnreferencedCode("The Weaviate provider is currently incompatible with trimming.")] + [RequiresDynamicCode("The Weaviate provider is currently incompatible with NativeAOT.")] + public override VectorStoreCollection GetCollection(string name, VectorStoreCollectionDefinition? definition = null) + { + if (typeof(TRecord) == typeof(Dictionary)) + { + throw new ArgumentException(VectorDataStrings.GetCollectionWithDictionaryNotSupported); + } + + return _storageType switch + { + RedisStorageType.HashSet => new RedisHashSetCollection(_database, name, new RedisHashSetCollectionOptions() + { + Definition = definition, + EmbeddingGenerator = _embeddingGenerator + }), + + RedisStorageType.Json => new RedisJsonCollection(_database, name, new RedisJsonCollectionOptions() + { + Definition = definition, + EmbeddingGenerator = _embeddingGenerator + }), + + _ => throw new UnreachableException() + }; + } + + /// + // TODO: The provider uses unsafe JSON serialization in many places, #11963 + [RequiresUnreferencedCode("The Redis provider is currently incompatible with trimming.")] + [RequiresDynamicCode("The Redis provider is currently incompatible with NativeAOT.")] + public override VectorStoreCollection> GetDynamicCollection(string name, VectorStoreCollectionDefinition definition) + => _storageType switch + { + RedisStorageType.HashSet => new RedisHashSetDynamicCollection(_database, name, new RedisHashSetCollectionOptions() + { + Definition = definition, + EmbeddingGenerator = _embeddingGenerator + }), + + RedisStorageType.Json => new RedisJsonDynamicCollection(_database, name, new RedisJsonCollectionOptions() + { + Definition = definition, + EmbeddingGenerator = _embeddingGenerator + }), + + _ => throw new UnreachableException() + }; + + /// + public override async IAsyncEnumerable ListCollectionNamesAsync([EnumeratorCancellation] CancellationToken cancellationToken = default) + { + const string OperationName = "FT._LIST"; + + var listResult = await VectorStoreErrorHandler.RunOperationAsync( + _metadata, + OperationName, + () => _database.FT()._ListAsync()).ConfigureAwait(false); + + foreach (var item in listResult) + { + var name = item.ToString(); + if (name != null) + { + yield return name; + } + } + } + + /// + public override Task CollectionExistsAsync(string name, CancellationToken cancellationToken = default) + { + var collection = GetDynamicCollection(name, s_generalPurposeDefinition); + return collection.CollectionExistsAsync(cancellationToken); + } + + /// + public override Task EnsureCollectionDeletedAsync(string name, CancellationToken cancellationToken = default) + { + var collection = GetDynamicCollection(name, s_generalPurposeDefinition); + return collection.EnsureCollectionDeletedAsync(cancellationToken); + } + + /// + public override object? GetService(Type serviceType, object? serviceKey = null) + { + Throw.IfNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreMetadata) ? _metadata : + serviceType == typeof(IDatabase) ? _metadata : + serviceType.IsInstanceOfType(this) ? this : + null; + } +} diff --git a/MEVD/src/Redis/RedisVectorStoreOptions.cs b/MEVD/src/Redis/RedisVectorStoreOptions.cs new file mode 100644 index 0000000..b5f4f70 --- /dev/null +++ b/MEVD/src/Redis/RedisVectorStoreOptions.cs @@ -0,0 +1,37 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.AI; + +namespace CommunityToolkit.VectorData.Redis; + +/// +/// Options when creating a . +/// +public sealed class RedisVectorStoreOptions +{ + internal static readonly RedisVectorStoreOptions Default = new(); + + /// + /// Initializes a new instance of the class. + /// + public RedisVectorStoreOptions() + { + } + + internal RedisVectorStoreOptions(RedisVectorStoreOptions? source) + { + StorageType = source?.StorageType ?? Default.StorageType; + EmbeddingGenerator = source?.EmbeddingGenerator; + } + + /// + /// Indicates the way in which data should be stored in redis. Default is . + /// + public RedisStorageType? StorageType { get; set; } = RedisStorageType.Json; + + /// + /// Gets or sets the default embedding generator to use when generating vectors embeddings with this vector store. + /// + public IEmbeddingGenerator? EmbeddingGenerator { get; set; } +} diff --git a/MEVD/src/Shared/VectorStoreErrorHandler.cs b/MEVD/src/Shared/VectorStoreErrorHandler.cs new file mode 100644 index 0000000..529a13d --- /dev/null +++ b/MEVD/src/Shared/VectorStoreErrorHandler.cs @@ -0,0 +1,470 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Data.Common; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.Extensions.VectorData; + +#pragma warning disable MEVD9000 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + +/// +/// Contains helpers for reading vector store model properties and their attributes. +/// +[ExcludeFromCodeCoverage] +internal static class VectorStoreErrorHandler +{ + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Task RunOperationAsync( + VectorStoreMetadata metadata, + string operationName, + Func> operation) + where TException : Exception + { + return RunOperationAsync( + new VectorStoreCollectionMetadata() + { + CollectionName = null, + VectorStoreName = metadata.VectorStoreName, + VectorStoreSystemName = metadata.VectorStoreSystemName, + }, + operationName, + operation); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static async Task RunOperationAsync( + VectorStoreCollectionMetadata metadata, + string operationName, + Func> operation) + where TException : Exception + { + try + { + return await operation.Invoke().ConfigureAwait(false); + } + catch (AggregateException ex) when (ex.InnerException is TException innerEx) + { + throw new VectorStoreException("Call to vector store failed.", ex) + { + VectorStoreSystemName = metadata.VectorStoreSystemName, + VectorStoreName = metadata.VectorStoreName, + CollectionName = metadata.CollectionName, + OperationName = operationName + }; + } + catch (TException ex) + { + throw new VectorStoreException("Call to vector store failed.", ex) + { + VectorStoreSystemName = metadata.VectorStoreSystemName, + VectorStoreName = metadata.VectorStoreName, + CollectionName = metadata.CollectionName, + OperationName = operationName + }; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static TResult RunOperation( + VectorStoreMetadata metadata, + string operationName, + Func operation) + where TException : Exception + { + return RunOperation( + new VectorStoreCollectionMetadata() + { + CollectionName = null, + VectorStoreName = metadata.VectorStoreName, + VectorStoreSystemName = metadata.VectorStoreSystemName, + }, + operationName, + operation); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static TResult RunOperation( + VectorStoreCollectionMetadata metadata, + string operationName, + Func operation) + where TException : Exception + { + try + { + return operation.Invoke(); + } + catch (AggregateException ex) when (ex.InnerException is TException innerEx) + { + throw new VectorStoreException("Call to vector store failed.", ex) + { + VectorStoreSystemName = metadata.VectorStoreSystemName, + VectorStoreName = metadata.VectorStoreName, + CollectionName = metadata.CollectionName, + OperationName = operationName + }; + } + catch (TException ex) + { + throw new VectorStoreException("Call to vector store failed.", ex) + { + VectorStoreSystemName = metadata.VectorStoreSystemName, + VectorStoreName = metadata.VectorStoreName, + CollectionName = metadata.CollectionName, + OperationName = operationName + }; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static async Task RunOperationWithRetryAsync( + VectorStoreCollectionMetadata metadata, + string operationName, + int maxRetries, + int delayInMilliseconds, + Func> operation, + CancellationToken cancellationToken) + where TException : Exception + { + var retries = 0; + + var exceptions = new List(); + + while (retries < maxRetries) + { + try + { + return await operation.Invoke().ConfigureAwait(false); + } + catch (AggregateException ex) when (ex.InnerException is TException innerEx) + { + retries++; + exceptions.Add(ex); + + if (retries >= maxRetries) + { + throw new VectorStoreException("Call to vector store failed.", new AggregateException(exceptions)) + { + VectorStoreSystemName = metadata.VectorStoreSystemName, + VectorStoreName = metadata.VectorStoreName, + CollectionName = metadata.CollectionName, + OperationName = operationName + }; + } + + await Task.Delay(delayInMilliseconds, cancellationToken).ConfigureAwait(false); + } + catch (TException ex) + { + retries++; + exceptions.Add(ex); + + if (retries >= maxRetries) + { + throw new VectorStoreException("Call to vector store failed.", new AggregateException(exceptions)) + { + VectorStoreSystemName = metadata.VectorStoreSystemName, + VectorStoreName = metadata.VectorStoreName, + CollectionName = metadata.CollectionName, + OperationName = operationName + }; + } + + await Task.Delay(delayInMilliseconds, cancellationToken).ConfigureAwait(false); + } + } + + throw new VectorStoreException("Call to vector store failed.", new AggregateException(exceptions)) + { + VectorStoreSystemName = metadata.VectorStoreSystemName, + VectorStoreName = metadata.VectorStoreName, + CollectionName = metadata.CollectionName, + OperationName = operationName + }; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static async Task RunOperationAsync( + VectorStoreCollectionMetadata metadata, + string operationName, + Func operation) + where TException : Exception + { + try + { + await operation.Invoke().ConfigureAwait(false); + } + catch (AggregateException ex) when (ex.InnerException is TException innerEx) + { + throw new VectorStoreException("Call to vector store failed.", ex) + { + VectorStoreSystemName = metadata.VectorStoreSystemName, + VectorStoreName = metadata.VectorStoreName, + CollectionName = metadata.CollectionName, + OperationName = operationName + }; + } + catch (TException ex) + { + throw new VectorStoreException("Call to vector store failed.", ex) + { + VectorStoreSystemName = metadata.VectorStoreSystemName, + VectorStoreName = metadata.VectorStoreName, + CollectionName = metadata.CollectionName, + OperationName = operationName + }; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static async Task RunOperationWithRetryAsync( + VectorStoreCollectionMetadata metadata, + string operationName, + int maxRetries, + int delayInMilliseconds, + Func operation, + CancellationToken cancellationToken) + where TException : Exception + { + var retries = 0; + + var exceptions = new List(); + + while (retries < maxRetries) + { + try + { + await operation.Invoke().ConfigureAwait(false); + return; + } + catch (AggregateException ex) when (ex.InnerException is TException innerEx) + { + retries++; + exceptions.Add(ex); + + if (retries >= maxRetries) + { + throw new VectorStoreException("Call to vector store failed.", new AggregateException(exceptions)) + { + VectorStoreSystemName = metadata.VectorStoreSystemName, + VectorStoreName = metadata.VectorStoreName, + CollectionName = metadata.CollectionName, + OperationName = operationName + }; + } + + await Task.Delay(delayInMilliseconds, cancellationToken).ConfigureAwait(false); + } + catch (TException ex) + { + retries++; + exceptions.Add(ex); + + if (retries >= maxRetries) + { + throw new VectorStoreException("Call to vector store failed.", new AggregateException(exceptions)) + { + VectorStoreSystemName = metadata.VectorStoreSystemName, + VectorStoreName = metadata.VectorStoreName, + CollectionName = metadata.CollectionName, + OperationName = operationName + }; + } + + await Task.Delay(delayInMilliseconds, cancellationToken).ConfigureAwait(false); + } + } + + throw new VectorStoreException("Call to vector store failed.", new AggregateException(exceptions)) + { + VectorStoreSystemName = metadata.VectorStoreSystemName, + VectorStoreName = metadata.VectorStoreName, + CollectionName = metadata.CollectionName, + OperationName = operationName + }; + } + + public struct ConfiguredCancelableErrorHandlingAsyncEnumerable + where TException : Exception + { + private readonly ConfiguredCancelableAsyncEnumerable _enumerable; + private readonly VectorStoreCollectionMetadata _metadata; + private readonly string _operationName; + + public ConfiguredCancelableErrorHandlingAsyncEnumerable( + ConfiguredCancelableAsyncEnumerable enumerable, + VectorStoreCollectionMetadata metadata, + string operationName) + { + this._enumerable = enumerable; + this._metadata = metadata; + this._operationName = operationName; + } + + public ConfiguredCancelableErrorHandlingAsyncEnumerable( + ConfiguredCancelableAsyncEnumerable enumerable, + VectorStoreMetadata metadata, + string operationName) + { + this._enumerable = enumerable; + this._metadata = new() + { + CollectionName = null, + VectorStoreName = metadata.VectorStoreName, + VectorStoreSystemName = metadata.VectorStoreSystemName, + }; + this._operationName = operationName; + } + + public ConfiguredCancelableErrorHandlingAsyncEnumerable.Enumerator GetAsyncEnumerator(CancellationToken cancellationToken = default) + { + return new Enumerator(this._enumerable.WithCancellation(cancellationToken).GetAsyncEnumerator(), this._metadata, this._operationName); + } + + public ConfiguredCancelableErrorHandlingAsyncEnumerable ConfigureAwait(bool continueOnCapturedContext) + { + return new ConfiguredCancelableErrorHandlingAsyncEnumerable(this._enumerable.ConfigureAwait(continueOnCapturedContext), this._metadata, this._operationName); + } + + public struct Enumerator( + ConfiguredCancelableAsyncEnumerable.Enumerator enumerator, + VectorStoreCollectionMetadata metadata, + string operationName) + { + public async ValueTask MoveNextAsync() + { + try + { + return await enumerator.MoveNextAsync(); + } + catch (AggregateException ex) when (ex.InnerException is TException innerEx) + { + throw new VectorStoreException("Call to vector store failed.", ex) + { + VectorStoreSystemName = metadata.VectorStoreSystemName, + VectorStoreName = metadata.VectorStoreName, + CollectionName = metadata.CollectionName, + OperationName = operationName + }; + } + catch (TException ex) + { + throw new VectorStoreException("Call to vector store failed.", ex) + { + VectorStoreSystemName = metadata.VectorStoreSystemName, + VectorStoreName = metadata.VectorStoreName, + CollectionName = metadata.CollectionName, + OperationName = operationName + }; + } + } + + public TResult Current => enumerator.Current; + } + } + + internal static Task ReadWithErrorHandlingAsync( + this DbDataReader reader, + VectorStoreCollectionMetadata metadata, + string operationName, + CancellationToken cancellationToken) + => VectorStoreErrorHandler.RunOperationAsync( + metadata, + operationName, + () => reader.ReadAsync(cancellationToken)); + + internal static Task ReadWithErrorHandlingAsync( + this DbDataReader reader, + VectorStoreMetadata metadata, + string operationName, + CancellationToken cancellationToken) + => VectorStoreErrorHandler.RunOperationAsync( + metadata, + operationName, + () => reader.ReadAsync(cancellationToken)); + + internal static async Task ExecuteWithErrorHandlingAsync( + this DbConnection connection, + VectorStoreMetadata metadata, + string operationName, + Func> operation, + CancellationToken cancellationToken) + { + return await ExecuteWithErrorHandlingAsync( + connection, + new VectorStoreCollectionMetadata + { + VectorStoreSystemName = metadata.VectorStoreSystemName, + VectorStoreName = metadata.VectorStoreName, + CollectionName = null + }, + operationName, + operation, + cancellationToken).ConfigureAwait(false); + } + + internal static async Task ExecuteWithErrorHandlingAsync( + this DbConnection connection, + VectorStoreCollectionMetadata metadata, + string operationName, + Func> operation, + CancellationToken cancellationToken) + { + if (connection.State != System.Data.ConnectionState.Open) + { + await connection.OpenAsync(cancellationToken).ConfigureAwait(false); + } + + try + { + return await operation().ConfigureAwait(false); + } + catch (DbException ex) + { +#if NET + await connection.DisposeAsync().ConfigureAwait(false); +#else + connection.Dispose(); +#endif + + throw new VectorStoreException("Call to vector store failed.", ex) + { + VectorStoreSystemName = metadata.VectorStoreSystemName, + VectorStoreName = metadata.VectorStoreName, + CollectionName = metadata.CollectionName, + OperationName = operationName + }; + } + catch (IOException ex) + { +#if NET + await connection.DisposeAsync().ConfigureAwait(false); +#else + connection.Dispose(); +#endif + + throw new VectorStoreException("Call to vector store failed.", ex) + { + VectorStoreSystemName = metadata.VectorStoreSystemName, + VectorStoreName = metadata.VectorStoreName, + CollectionName = metadata.CollectionName, + OperationName = operationName + }; + } + catch (Exception) + { +#if NET + await connection.DisposeAsync().ConfigureAwait(false); +#else + connection.Dispose(); +#endif + throw; + } + } +} diff --git a/MEVD/src/SqliteVec/AssemblyInfo.cs b/MEVD/src/SqliteVec/AssemblyInfo.cs new file mode 100644 index 0000000..d6395e8 --- /dev/null +++ b/MEVD/src/SqliteVec/AssemblyInfo.cs @@ -0,0 +1,2 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. diff --git a/MEVD/src/SqliteVec/Conditions/SqliteWhereCondition.cs b/MEVD/src/SqliteVec/Conditions/SqliteWhereCondition.cs new file mode 100644 index 0000000..a643592 --- /dev/null +++ b/MEVD/src/SqliteVec/Conditions/SqliteWhereCondition.cs @@ -0,0 +1,19 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace CommunityToolkit.VectorData.SqliteVec; + +internal abstract class SqliteWhereCondition(string operand, List values) +{ + public string Operand { get; set; } = operand; + + public List Values { get; set; } = values; + + public string? TableName { get; set; } + + public abstract string BuildQuery(List parameterNames); + + protected string GetOperand() => !string.IsNullOrWhiteSpace(TableName) ? + $"\"{TableName}\".\"{Operand}\"" : + $"\"{Operand}\""; +} diff --git a/MEVD/src/SqliteVec/Conditions/SqliteWhereEqualsCondition.cs b/MEVD/src/SqliteVec/Conditions/SqliteWhereEqualsCondition.cs new file mode 100644 index 0000000..5afca0b --- /dev/null +++ b/MEVD/src/SqliteVec/Conditions/SqliteWhereEqualsCondition.cs @@ -0,0 +1,17 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace CommunityToolkit.VectorData.SqliteVec; + +internal sealed class SqliteWhereEqualsCondition(string operand, object value) + : SqliteWhereCondition(operand, [value]) +{ + public override string BuildQuery(List parameterNames) + { + const string EqualsOperator = "="; + + if (parameterNames.Count == 0) { throw new ArgumentException($"Cannot build '{nameof(SqliteWhereEqualsCondition)}' condition without parameter name."); } + + return $"{GetOperand()} {EqualsOperator} {parameterNames[0]}"; + } +} diff --git a/MEVD/src/SqliteVec/Conditions/SqliteWhereInCondition.cs b/MEVD/src/SqliteVec/Conditions/SqliteWhereInCondition.cs new file mode 100644 index 0000000..8dfbf3e --- /dev/null +++ b/MEVD/src/SqliteVec/Conditions/SqliteWhereInCondition.cs @@ -0,0 +1,17 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace CommunityToolkit.VectorData.SqliteVec; + +internal sealed class SqliteWhereInCondition(string operand, List values) + : SqliteWhereCondition(operand, values) +{ + public override string BuildQuery(List parameterNames) + { + const string InOperator = "IN"; + + if (parameterNames.Count == 0) { throw new ArgumentException($"Cannot build '{nameof(SqliteWhereInCondition)}' condition without parameter names."); } + + return $"{GetOperand()} {InOperator} ({string.Join(", ", parameterNames)})"; + } +} diff --git a/MEVD/src/SqliteVec/Conditions/SqliteWhereMatchCondition.cs b/MEVD/src/SqliteVec/Conditions/SqliteWhereMatchCondition.cs new file mode 100644 index 0000000..77956f5 --- /dev/null +++ b/MEVD/src/SqliteVec/Conditions/SqliteWhereMatchCondition.cs @@ -0,0 +1,17 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace CommunityToolkit.VectorData.SqliteVec; + +internal sealed class SqliteWhereMatchCondition(string operand, object value) + : SqliteWhereCondition(operand, [value]) +{ + public override string BuildQuery(List parameterNames) + { + const string MatchOperator = "MATCH"; + + if (parameterNames.Count == 0) { throw new ArgumentException($"Cannot build '{nameof(SqliteWhereMatchCondition)}' condition without parameter name."); } + + return $"{GetOperand()} {MatchOperator} {parameterNames[0]}"; + } +} diff --git a/MEVD/src/SqliteVec/SqliteCollection.cs b/MEVD/src/SqliteVec/SqliteCollection.cs new file mode 100644 index 0000000..847d534 --- /dev/null +++ b/MEVD/src/SqliteVec/SqliteCollection.cs @@ -0,0 +1,700 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Linq.Expressions; +using System.Runtime.CompilerServices; +using Microsoft.Data.Sqlite; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; +using Microsoft.Shared.Diagnostics; + +namespace CommunityToolkit.VectorData.SqliteVec; + +/// +/// Service for storing and retrieving vector records, that uses SQLite as the underlying storage. +/// +/// The data type of the record key. Can be , or . +/// The data model to use for adding, updating and retrieving data from storage. +#pragma warning disable CA1711 // Identifiers should not have incorrect suffix +public class SqliteCollection : VectorStoreCollection + where TKey : notnull + where TRecord : class +#pragma warning restore CA1711 // Identifiers should not have incorrect +{ + /// Metadata about vector store record collection. + private readonly VectorStoreCollectionMetadata _collectionMetadata; + + /// The connection string for the SQLite database represented by this . + private readonly string _connectionString; + + /// The mapper to use when mapping between the consumer data model and the SQLite record. + private readonly SqliteMapper _mapper; + + /// The default options for vector search. + private static readonly VectorSearchOptions s_defaultVectorSearchOptions = new(); + + /// The model for this collection. + private readonly CollectionModel _model; + + /// Flag which indicates whether vector properties exist in the consumer data model. + private readonly bool _vectorPropertiesExist; + + /// The storage name of the key property. + private readonly string _keyStorageName; + + /// Table name in SQLite for data properties. + private readonly string _dataTableName; + + /// Table name in SQLite for vector properties. + private readonly string _vectorTableName; + + /// + public override string Name { get; } + + /// + /// Initializes a new instance of the class. + /// + /// The connection string for the SQLite database represented by this . + /// The name of the collection/table that this will access. + /// Optional configuration options for this class. + [RequiresDynamicCode("This constructor is incompatible with NativeAOT. For dynamic mapping via Dictionary, instantiate SqliteDynamicCollection instead.")] + [RequiresUnreferencedCode("This constructor is incompatible with trimming. For dynamic mapping via Dictionary, instantiate SqliteDynamicCollection instead")] + public SqliteCollection( + string connectionString, + string name, + SqliteCollectionOptions? options = default) + : this( + connectionString, + name, + static options => typeof(TRecord) == typeof(Dictionary) + ? throw new NotSupportedException(VectorDataStrings.NonDynamicCollectionWithDictionaryNotSupported(typeof(SqliteDynamicCollection))) + : new SqliteModelBuilder().Build(typeof(TRecord), typeof(TKey), options.Definition, options.EmbeddingGenerator), + options) + { + } + + internal SqliteCollection(string connectionString, string name, Func modelFactory, SqliteCollectionOptions? options) + { + // Verify. + Throw.IfNull(connectionString); + Throw.IfNullOrWhitespace(name); + + if (typeof(TKey) != typeof(string) + && typeof(TKey) != typeof(int) + && typeof(TKey) != typeof(long) + && typeof(TKey) != typeof(Guid) + && typeof(TKey) != typeof(object)) + { + throw new NotSupportedException("Only string, int, long and Guid keys are supported."); + } + + options ??= SqliteCollectionOptions.Default; + + // Assign. + _connectionString = connectionString; + Name = name; + _model = modelFactory(options); + + _dataTableName = name; + _vectorTableName = GetVectorTableName(name, options); + + _vectorPropertiesExist = _model.VectorProperties.Count > 0; + + // Populate some collections of properties + _keyStorageName = _model.KeyProperty.StorageName; + _mapper = new SqliteMapper(_model); + + var connectionStringBuilder = new SqliteConnectionStringBuilder(connectionString); + + _collectionMetadata = new() + { + VectorStoreSystemName = SqliteConstants.VectorStoreSystemName, + VectorStoreName = connectionStringBuilder.DataSource, + CollectionName = name + }; + } + + /// + public override async Task CollectionExistsAsync(CancellationToken cancellationToken = default) + { + const string OperationName = "TableCount"; + + using var connection = await GetConnectionAsync(cancellationToken).ConfigureAwait(false); + using var command = SqliteCommandBuilder.BuildTableCountCommand(connection, _dataTableName); + + var result = await connection.ExecuteWithErrorHandlingAsync( + _collectionMetadata, + OperationName, + () => command.ExecuteScalarAsync(cancellationToken), + cancellationToken).ConfigureAwait(false); + + long count = result is not null ? (long)result : 0; + + return count > 0; + } + + /// + public override async Task EnsureCollectionExistsAsync(CancellationToken cancellationToken = default) + { + using var connection = await GetConnectionAsync(cancellationToken).ConfigureAwait(false); + await InternalCreateCollectionAsync(connection, ifNotExists: true, cancellationToken) + .ConfigureAwait(false); + } + + /// + public override async Task EnsureCollectionDeletedAsync(CancellationToken cancellationToken = default) + { + using var connection = await GetConnectionAsync(cancellationToken).ConfigureAwait(false); + + await DropTableAsync(connection, _dataTableName, cancellationToken).ConfigureAwait(false); + + if (_vectorPropertiesExist) + { + await DropTableAsync(connection, _vectorTableName, cancellationToken).ConfigureAwait(false); + } + } + + #region Search + + /// + public override async IAsyncEnumerable> SearchAsync( + TInput searchValue, + int top, + VectorSearchOptions? options = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + Throw.IfNull(searchValue); + Throw.IfLessThan(top, 1); + + const string LimitPropertyName = "k"; + + options ??= s_defaultVectorSearchOptions; + if (options.IncludeVectors && _model.EmbeddingGenerationRequired) + { + throw new NotSupportedException(VectorDataStrings.IncludeVectorsNotSupportedWithEmbeddingGeneration); + } + + var vectorProperty = _model.GetVectorPropertyOrSingle(options); + + ReadOnlyMemory vector = searchValue switch + { + ReadOnlyMemory r => r, + float[] f => new ReadOnlyMemory(f), + Embedding e => e.Vector, + _ when vectorProperty.EmbeddingGenerationDispatcher is not null + => ((Embedding)await vectorProperty.GenerateEmbeddingAsync(searchValue, cancellationToken).ConfigureAwait(false)).Vector, + + _ => vectorProperty.EmbeddingGenerator is null + ? throw new NotSupportedException(VectorDataStrings.InvalidSearchInputAndNoEmbeddingGeneratorWasConfigured(searchValue.GetType(), SqliteModelBuilder.SupportedVectorTypes)) + : throw new InvalidOperationException(VectorDataStrings.IncompatibleEmbeddingGeneratorWasConfiguredForInputType(typeof(TInput), vectorProperty.EmbeddingGenerator.GetType())) + }; + + var mappedArray = SqlitePropertyMapping.MapVectorForStorageModel(vector); + + // Simulating skip/offset logic locally, since OFFSET can work only with LIMIT in combination + // and LIMIT is not supported in vector search extension, instead of LIMIT - "k" parameter is used. + var limit = top + options.Skip; + + var conditions = new List() + { + new SqliteWhereMatchCondition(vectorProperty.StorageName, mappedArray), + new SqliteWhereEqualsCondition(LimitPropertyName, limit) + }; + + string? extraWhereFilter = null; + Dictionary? extraParameters = null; + + if (options.Filter is not null) + { + SqliteFilterTranslator translator = new(_model, options.Filter); + translator.Translate(appendWhere: false); + extraWhereFilter = translator.Clause.ToString(); + extraParameters = translator.Parameters; + } + + await foreach (var record in EnumerateAndMapSearchResultsAsync( + conditions, + extraWhereFilter, + extraParameters, + options, + cancellationToken) + .ConfigureAwait(false)) + { + yield return record; + } + } + + #endregion Search + + /// + public override async IAsyncEnumerable GetAsync(Expression> filter, int top, FilteredRecordRetrievalOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + Throw.IfNull(filter); + Throw.IfLessThan(top, 1); + + options ??= new(); + if (options.IncludeVectors && _model.EmbeddingGenerationRequired) + { + throw new NotSupportedException(VectorDataStrings.IncludeVectorsNotSupportedWithEmbeddingGeneration); + } + + SqliteFilterTranslator translator = new(_model, filter); + translator.Translate(appendWhere: false); + + using var connection = await GetConnectionAsync(cancellationToken).ConfigureAwait(false); + + using var command = options.IncludeVectors + ? SqliteCommandBuilder.BuildSelectInnerJoinCommand( + connection, + _vectorTableName, + _dataTableName, + _keyStorageName, + _model, + conditions: [], + includeDistance: false, + filterOptions: options, + translator.Clause.ToString(), + translator.Parameters, + top: top, + skip: options.Skip) + : SqliteCommandBuilder.BuildSelectDataCommand( + connection, + _dataTableName, + _model, + conditions: [], + filterOptions: options, + translator.Clause.ToString(), + translator.Parameters, + top: top, + skip: options.Skip); + + const string OperationName = "Get"; + + using var reader = await connection.ExecuteWithErrorHandlingAsync( + _collectionMetadata, + OperationName, + () => command.ExecuteReaderAsync(cancellationToken), + cancellationToken).ConfigureAwait(false); + + while (await reader.ReadWithErrorHandlingAsync( + _collectionMetadata, + OperationName, + cancellationToken).ConfigureAwait(false)) + { + yield return _mapper.MapFromStorageToDataModel(reader, options.IncludeVectors); + } + } + + /// + public override async Task GetAsync(TKey key, RecordRetrievalOptions? options = null, CancellationToken cancellationToken = default) + { + Throw.IfNull(key); + + using var connection = await GetConnectionAsync(cancellationToken).ConfigureAwait(false); + + var condition = new SqliteWhereEqualsCondition(_keyStorageName, key) + { + TableName = _dataTableName + }; + + return await InternalGetBatchAsync(connection, condition, options, cancellationToken) + .FirstOrDefaultAsync(cancellationToken) + .ConfigureAwait(false); + } + + /// + public override async IAsyncEnumerable GetAsync(IEnumerable keys, RecordRetrievalOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + Throw.IfNull(keys); + var keysList = keys.Cast().ToList(); + if (keysList.Count == 0) + { + yield break; + } + + using var connection = await GetConnectionAsync(cancellationToken).ConfigureAwait(false); + + var condition = new SqliteWhereInCondition(_keyStorageName, keysList) + { + TableName = _dataTableName + }; + + await foreach (var record in InternalGetBatchAsync(connection, condition, options, cancellationToken).ConfigureAwait(false)) + { + yield return record; + } + } + + /// + public override Task UpsertAsync(TRecord record, CancellationToken cancellationToken = default) + => DoUpsertAsync([record], cancellationToken); + + /// + public override Task UpsertAsync(IEnumerable records, CancellationToken cancellationToken = default) + { + Throw.IfNull(records); + + return DoUpsertAsync(records, cancellationToken); + } + + /// + public override async Task DeleteAsync(TKey key, CancellationToken cancellationToken = default) + { + Throw.IfNull(key); + + using var connection = await GetConnectionAsync(cancellationToken).ConfigureAwait(false); + + var condition = new SqliteWhereEqualsCondition(_keyStorageName, key); + + await InternalDeleteBatchAsync(connection, condition, cancellationToken).ConfigureAwait(false); + } + + /// + public override async Task DeleteAsync(IEnumerable keys, CancellationToken cancellationToken = default) + { + Throw.IfNull(keys); + var keysList = keys.Cast().ToList(); + if (keysList.Count == 0) + { + return; + } + + using var connection = await GetConnectionAsync(cancellationToken).ConfigureAwait(false); + + var condition = new SqliteWhereInCondition( + _keyStorageName, + keysList); + + await InternalDeleteBatchAsync(connection, condition, cancellationToken).ConfigureAwait(false); + } + + /// + public override object? GetService(Type serviceType, object? serviceKey = null) + { + Throw.IfNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreCollectionMetadata) ? _collectionMetadata : + serviceType.IsInstanceOfType(this) ? this : + null; + } + + #region private + + private async ValueTask GetConnectionAsync(CancellationToken cancellationToken = default) + { + var connection = new SqliteConnection(_connectionString); + await connection.OpenAsync(cancellationToken).ConfigureAwait(false); + connection.LoadVector(); + return connection; + } + + private async IAsyncEnumerable> EnumerateAndMapSearchResultsAsync( + List conditions, + string? extraWhereFilter, + Dictionary? extraParameters, + VectorSearchOptions searchOptions, + [EnumeratorCancellation] CancellationToken cancellationToken) + { + const string OperationName = "VectorizedSearch"; + + using var connection = await GetConnectionAsync(cancellationToken).ConfigureAwait(false); + using var command = SqliteCommandBuilder.BuildSelectInnerJoinCommand( + connection, + _vectorTableName, + _dataTableName, + _keyStorageName, + _model, + conditions, + includeDistance: true, + extraWhereFilter: extraWhereFilter, + extraParameters: extraParameters, + scoreThreshold: searchOptions.ScoreThreshold); + + using var reader = await connection.ExecuteWithErrorHandlingAsync( + _collectionMetadata, + OperationName, + () => command.ExecuteReaderAsync(cancellationToken), + cancellationToken).ConfigureAwait(false); + + for (var recordCounter = 0; await reader.ReadAsync(cancellationToken).ConfigureAwait(false); recordCounter++) + { + if (recordCounter >= searchOptions.Skip) + { + var score = SqlitePropertyMapping.GetPropertyValue(reader, SqliteCommandBuilder.DistancePropertyName); + var record = _mapper.MapFromStorageToDataModel(reader, searchOptions.IncludeVectors); + + yield return new VectorSearchResult(record, score); + } + } + } + + private async Task InternalCreateCollectionAsync(SqliteConnection connection, bool ifNotExists, CancellationToken cancellationToken) + { + List dataTableColumns = SqlitePropertyMapping.GetColumns(_model.Properties, data: true); + + await CreateTableAsync(connection, _dataTableName, dataTableColumns, ifNotExists, cancellationToken) + .ConfigureAwait(false); + + if (_vectorPropertiesExist) + { + List vectorTableColumns = SqlitePropertyMapping.GetColumns(_model.Properties, data: false); + + await CreateVirtualTableAsync(connection, _vectorTableName, vectorTableColumns, ifNotExists, cancellationToken) + .ConfigureAwait(false); + } + } + + private Task CreateTableAsync(SqliteConnection connection, string tableName, List columns, bool ifNotExists, CancellationToken cancellationToken) + { + const string OperationName = "CreateTable"; + + using var command = SqliteCommandBuilder.BuildCreateTableCommand(connection, tableName, columns, ifNotExists); + + return connection.ExecuteWithErrorHandlingAsync( + _collectionMetadata, + OperationName, + () => command.ExecuteNonQueryAsync(cancellationToken), + cancellationToken); + } + + private Task CreateVirtualTableAsync(SqliteConnection connection, string tableName, List columns, bool ifNotExists, CancellationToken cancellationToken) + { + const string OperationName = "CreateVirtualTable"; + + using var command = SqliteCommandBuilder.BuildCreateVirtualTableCommand(connection, tableName, columns, ifNotExists); + + return connection.ExecuteWithErrorHandlingAsync( + _collectionMetadata, + OperationName, + () => command.ExecuteNonQueryAsync(cancellationToken), + cancellationToken); + } + + private Task DropTableAsync(SqliteConnection connection, string tableName, CancellationToken cancellationToken) + { + const string OperationName = "DropTable"; + + using var command = SqliteCommandBuilder.BuildDropTableCommand(connection, tableName); + + return connection.ExecuteWithErrorHandlingAsync( + _collectionMetadata, + OperationName, + () => command.ExecuteNonQueryAsync(cancellationToken), + cancellationToken); + } + + private async IAsyncEnumerable InternalGetBatchAsync( + SqliteConnection connection, + SqliteWhereCondition condition, + RecordRetrievalOptions? options, + [EnumeratorCancellation] CancellationToken cancellationToken) + { + const string OperationName = "Select"; + + bool includeVectors = options?.IncludeVectors is true && _vectorPropertiesExist; + if (includeVectors && _model.EmbeddingGenerationRequired) + { + throw new NotSupportedException(VectorDataStrings.IncludeVectorsNotSupportedWithEmbeddingGeneration); + } + + var command = includeVectors + ? SqliteCommandBuilder.BuildSelectInnerJoinCommand( + connection, + _vectorTableName, + _dataTableName, + _keyStorageName, + _model, + [condition], + includeDistance: false) + : SqliteCommandBuilder.BuildSelectDataCommand( + connection, + _dataTableName, + _model, + [condition]); + + using var reader = await connection.ExecuteWithErrorHandlingAsync( + _collectionMetadata, + OperationName, + () => command.ExecuteReaderAsync(cancellationToken), + cancellationToken).ConfigureAwait(false); + + while (await reader.ReadWithErrorHandlingAsync(_collectionMetadata, OperationName, cancellationToken).ConfigureAwait(false)) + { + yield return _mapper.MapFromStorageToDataModel(reader, includeVectors); + } + } + + private async Task DoUpsertAsync(IEnumerable records, CancellationToken cancellationToken) + { + Throw.IfNull(records); + + // With SQLite, we'll need to enumerate the records multiple times in almost all cases (e.g. because of the existence + // of two separate tables for data and vectors). To avoid multiple enumerations, we materialize the records into a list here. + var recordsList = records is IReadOnlyList r ? r : records.ToList(); + if (recordsList.Count == 0) + { + return; + } + records = recordsList; + + // If an embedding generator is defined, invoke it once per property for all records. + Dictionary>>? generatedEmbeddings = null; + + var vectorPropertyCount = _model.VectorProperties.Count; + for (var i = 0; i < vectorPropertyCount; i++) + { + var vectorProperty = _model.VectorProperties[i]; + + if (SqliteModelBuilder.IsVectorPropertyTypeValidCore(vectorProperty.Type, out _)) + { + continue; + } + + // We have a vector property whose type isn't natively supported - we need to generate embeddings. + Debug.Assert(vectorProperty.EmbeddingGenerator is not null); + + // TODO: Ideally we'd group together vector properties using the same generator (and with the same input and output properties), + // and generate embeddings for them in a single batch. That's some more complexity though. + generatedEmbeddings ??= new Dictionary>>(vectorPropertyCount); + generatedEmbeddings[vectorProperty] = (IReadOnlyList>)await vectorProperty.GenerateEmbeddingsAsync(records.Select(r => vectorProperty.GetValueAsObject(r)), cancellationToken).ConfigureAwait(false); + } + + var keyProperty = _model.KeyProperty; + + using var connection = await GetConnectionAsync(cancellationToken).ConfigureAwait(false); + + using var dataCommand = SqliteCommandBuilder.BuildInsertCommand( + connection, + _dataTableName, + _model, + recordsList, + generatedEmbeddings, + data: true, + replaceIfExists: true); + + using (var reader = await connection.ExecuteWithErrorHandlingAsync( + _collectionMetadata, + "updateData", + () => dataCommand.ExecuteReaderAsync(cancellationToken), + cancellationToken).ConfigureAwait(false)) + { + // If the key property is auto-generated, we need to read the generated keys from the database and inject them into the records + // (except for GUIDs which are generated client-side and have already been injected). + if (keyProperty is KeyPropertyModel { IsAutoGenerated: true } && keyProperty.Type != typeof(Guid)) + { + int? keyOrdinal = null; + + foreach (var record in recordsList) + { + switch (keyProperty.Type) + { + case var t when t == typeof(int) && keyProperty.GetValue(record) == 0: + keyOrdinal ??= reader.GetOrdinal(keyProperty.StorageName); + await reader.ReadAsync(cancellationToken).ConfigureAwait(false); + keyProperty.SetValue(record, reader.GetFieldValue(keyOrdinal.Value)); + await reader.NextResultAsync(cancellationToken).ConfigureAwait(false); + continue; + case var t when t == typeof(long) && keyProperty.GetValue(record) == 0L: + keyOrdinal ??= reader.GetOrdinal(keyProperty.StorageName); + await reader.ReadAsync(cancellationToken).ConfigureAwait(false); + keyProperty.SetValue(record, reader.GetFieldValue(keyOrdinal.Value)); + await reader.NextResultAsync(cancellationToken).ConfigureAwait(false); + continue; + } + } + } + } + + // We've inserted the main data records, now insert the records into the vector virtual table as well. + if (_vectorPropertiesExist) + { + var keys = recordsList.Select(r => keyProperty.GetValueAsObject(r)!).ToList(); + + // Deleting vector records first since current version of vector search extension + // doesn't support Upsert operation, only Delete/Insert. + using var vectorDeleteCommand = SqliteCommandBuilder.BuildDeleteCommand( + connection, + _vectorTableName, + [new SqliteWhereInCondition(_keyStorageName, keys)]); + + await connection.ExecuteWithErrorHandlingAsync( + _collectionMetadata, + "VectorDelete", + () => vectorDeleteCommand.ExecuteNonQueryAsync(cancellationToken), + cancellationToken).ConfigureAwait(false); + + using var vectorInsertCommand = SqliteCommandBuilder.BuildInsertCommand( + connection, + _vectorTableName, + _model, + recordsList, + generatedEmbeddings, + data: false); + + await connection.ExecuteWithErrorHandlingAsync( + _collectionMetadata, + "VectorInsert", + () => vectorInsertCommand.ExecuteNonQueryAsync(cancellationToken), + cancellationToken).ConfigureAwait(false); + } + } + + private Task InternalDeleteBatchAsync(SqliteConnection connection, SqliteWhereCondition condition, CancellationToken cancellationToken) + { + var tasks = new List(); + + if (_vectorPropertiesExist) + { + using var vectorCommand = SqliteCommandBuilder.BuildDeleteCommand( + connection, + _vectorTableName, + [condition]); + + tasks.Add(connection.ExecuteWithErrorHandlingAsync( + _collectionMetadata, + "VectorDelete", + () => vectorCommand.ExecuteNonQueryAsync(cancellationToken), + cancellationToken)); + } + + using var dataCommand = SqliteCommandBuilder.BuildDeleteCommand( + connection, + _dataTableName, + [condition]); + + tasks.Add(connection.ExecuteWithErrorHandlingAsync( + _collectionMetadata, + "DataDelete", + () => dataCommand.ExecuteNonQueryAsync(cancellationToken), + cancellationToken)); + + return Task.WhenAll(tasks); + } + + /// + /// Gets vector table name. + /// + /// + /// If custom vector table name is not provided, default one will be generated with a prefix to avoid name collisions. + /// + private static string GetVectorTableName( + string dataTableName, + SqliteCollectionOptions options) + { + const string DefaultVirtualTableNamePrefix = "vec_"; + + if (!string.IsNullOrWhiteSpace(options.VectorVirtualTableName)) + { + return options.VectorVirtualTableName!; + } + + return $"{DefaultVirtualTableNamePrefix}{dataTableName}"; + } + + #endregion +} diff --git a/MEVD/src/SqliteVec/SqliteCollectionOptions.cs b/MEVD/src/SqliteVec/SqliteCollectionOptions.cs new file mode 100644 index 0000000..e613f59 --- /dev/null +++ b/MEVD/src/SqliteVec/SqliteCollectionOptions.cs @@ -0,0 +1,34 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.VectorData; + +namespace CommunityToolkit.VectorData.SqliteVec; + +/// +/// Options when creating a . +/// +public sealed class SqliteCollectionOptions : VectorStoreCollectionOptions +{ + internal static readonly SqliteCollectionOptions Default = new(); + + /// + /// Initializes a new instance of the class. + /// + public SqliteCollectionOptions() + { + } + + internal SqliteCollectionOptions(SqliteCollectionOptions? source) : base(source) + { + VectorVirtualTableName = source?.VectorVirtualTableName; + } + + /// + /// Custom virtual table name to store vectors. + /// + /// + /// If not provided, collection name with prefix will be used as virtual table name. + /// + public string? VectorVirtualTableName { get; set; } +} diff --git a/MEVD/src/SqliteVec/SqliteColumn.cs b/MEVD/src/SqliteVec/SqliteColumn.cs new file mode 100644 index 0000000..05f6b1f --- /dev/null +++ b/MEVD/src/SqliteVec/SqliteColumn.cs @@ -0,0 +1,25 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace CommunityToolkit.VectorData.SqliteVec; + +/// +/// Representation of SQLite column. +/// +internal sealed class SqliteColumn( + string name, + string type, + bool isPrimary) +{ + public string Name { get; set; } = name; + + public string Type { get; set; } = type; + + public bool IsPrimary { get; set; } = isPrimary; + + public bool IsNullable { get; set; } + + public bool HasIndex { get; set; } + + public Dictionary? Configuration { get; set; } +} diff --git a/MEVD/src/SqliteVec/SqliteCommandBuilder.cs b/MEVD/src/SqliteVec/SqliteCommandBuilder.cs new file mode 100644 index 0000000..29eebf5 --- /dev/null +++ b/MEVD/src/SqliteVec/SqliteCommandBuilder.cs @@ -0,0 +1,561 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Data.Common; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Text; +using Microsoft.Data.Sqlite; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; +using Microsoft.Shared.Diagnostics; + +namespace CommunityToolkit.VectorData.SqliteVec; + +/// +/// Command builder for queries in SQLite database. +/// +[SuppressMessage("Security", "CA2100:Review SQL queries for security vulnerabilities", Justification = "User input is passed using command parameters.")] +internal static class SqliteCommandBuilder +{ + internal const string DistancePropertyName = "distance"; + + public static DbCommand BuildTableCountCommand(SqliteConnection connection, string tableName) + { + Throw.IfNullOrWhitespace(tableName); + + const string SystemTable = "sqlite_master"; + const string ParameterName = "@tableName"; + + var query = $"SELECT count(*) FROM {SystemTable} WHERE type='table' AND name={ParameterName};"; + + var command = connection.CreateCommand(); + + command.CommandText = query; + + command.Parameters.Add(new SqliteParameter(ParameterName, tableName)); + + return command; + } + + public static DbCommand BuildCreateTableCommand(SqliteConnection connection, string tableName, IReadOnlyList columns, bool ifNotExists) + { + var builder = new StringBuilder(); + + builder.Append("CREATE TABLE "); + if (ifNotExists) + { + builder.Append("IF NOT EXISTS "); + } + builder.AppendIdentifier(tableName).AppendLine(" ("); + + builder.AppendLine(string.Join(",\n", columns.Select(column => GetColumnDefinition(column, quote: true, includeNullability: true)))); + builder.AppendLine(");"); + + foreach (var column in columns) + { + if (column.HasIndex) + { + builder.Append("CREATE INDEX "); + if (ifNotExists) + { + builder.Append("IF NOT EXISTS "); + } + builder.AppendIdentifier($"{tableName}_{column.Name}_index").Append(" ON ") + .AppendIdentifier(tableName).Append('(').AppendIdentifier(column.Name).AppendLine(");"); + } + } + + var command = connection.CreateCommand(); + + command.CommandText = builder.ToString(); + + return command; + } + + public static DbCommand BuildCreateVirtualTableCommand( + SqliteConnection connection, + string tableName, + IReadOnlyList columns, + bool ifNotExists) + { + var builder = new StringBuilder(); + + builder.Append("CREATE VIRTUAL TABLE "); + if (ifNotExists) + { + builder.Append("IF NOT EXISTS "); + } + builder.AppendIdentifier(tableName).AppendLine(" USING vec0("); + + // The vector extension is currently uncapable of handling quoted identifiers. + builder.AppendLine(string.Join(",\n", columns.Select(column => GetColumnDefinition(column, quote: false, includeNullability: false)))); + builder.Append(");"); + + var command = connection.CreateCommand(); + + command.CommandText = builder.ToString(); + + return command; + } + + public static DbCommand BuildDropTableCommand(SqliteConnection connection, string tableName) + { + var builder = new StringBuilder(); + builder.Append("DROP TABLE IF EXISTS ").AppendIdentifier(tableName).Append(';'); + + var command = connection.CreateCommand(); + command.CommandText = builder.ToString(); + return command; + } + + public static DbCommand BuildInsertCommand( + SqliteConnection connection, + string tableName, + CollectionModel model, + IReadOnlyList records, + Dictionary>>? generatedEmbeddings, + bool data, + bool replaceIfExists = false) + { + var sql = new StringBuilder(); + var command = connection.CreateCommand(); + + var recordIndex = 0; + + var properties = model.KeyProperties.Concat(data ? model.DataProperties : (IEnumerable)model.VectorProperties).ToList(); + var keyProperty = model.KeyProperty; + var isKeyPossiblyDatabaseGenerated = keyProperty.IsAutoGenerated && (keyProperty.Type == typeof(int) || keyProperty.Type == typeof(long)); + + foreach (var record in records) + { + var isRecordKeyDatabaseGenerated = isKeyPossiblyDatabaseGenerated + ? (keyProperty.Type == typeof(int) && keyProperty.GetValue(record) is var i && i == 0) + || (keyProperty.Type == typeof(long) && keyProperty.GetValue(record) is var l && l == 0L) + : false; + + sql.Append("INSERT"); + + if (replaceIfExists && !isRecordKeyDatabaseGenerated) + { + sql.Append(" OR REPLACE"); + } + + sql.Append(" INTO ").AppendIdentifier(tableName).Append(" ("); + + var propertyIndex = 0; + foreach (var property in properties) + { + if (property is KeyPropertyModel && isRecordKeyDatabaseGenerated) + { + continue; + } + + if (propertyIndex++ > 0) + { + sql.Append(", "); + } + + sql.AppendIdentifier(property.StorageName); + } + + sql.AppendLine(")"); + + sql.Append("VALUES ("); + + propertyIndex = 0; + foreach (var property in properties) + { + var value = property.GetValueAsObject(record); + + switch (property) + { + case KeyPropertyModel { IsAutoGenerated: true }: + { + switch (value) + { + // Database ROWID generation. We don't specify the value in INSERT, and read it back later via RETURNING. + case int or long when isRecordKeyDatabaseGenerated: + continue; + + case Guid g when g == Guid.Empty: + // As SQLite has no built-in GUID generation, we generate client-side + // If the key is a Guid and auto-generated, generate a new Guid for any record where the key is empty. + if (keyProperty.IsAutoGenerated && keyProperty.Type == typeof(Guid)) + { +#if NET9_0_OR_GREATER + g = Guid.CreateVersion7(); +#else + g = Guid.NewGuid(); +#endif + value = g; + keyProperty.SetValue(record, g); + } + break; + + // We're configured for auto-generation but the user specified an explicit value. + case int or long or Guid: + break; + + default: + throw new UnreachableException(); + } + + break; + } + + case VectorPropertyModel vectorProperty: + { + if (generatedEmbeddings?[vectorProperty] is IReadOnlyList ge) + { + value = ((Embedding)ge[recordIndex]).Vector; + } + + value = value switch + { + ReadOnlyMemory m => SqlitePropertyMapping.MapVectorForStorageModel(m), + Embedding e => SqlitePropertyMapping.MapVectorForStorageModel(e.Vector), + float[] a => SqlitePropertyMapping.MapVectorForStorageModel(a), + null => null, + + _ => throw new InvalidOperationException($"Retrieved value for vector property '{property.StorageName}' which is not a ReadOnlyMemory ('{value?.GetType().Name}').") + }; + break; + } + } + + var parameterName = GetParameterName(property.StorageName, recordIndex); + + if (propertyIndex++ > 0) + { + sql.Append(", "); + } + + sql.Append(parameterName); + + command.Parameters.Add(new SqliteParameter(parameterName, value ?? DBNull.Value)); + } + + sql.AppendLine(")"); + + if (isRecordKeyDatabaseGenerated) + { + sql.Append("RETURNING \"").Append(keyProperty.StorageName).Append('"'); + } + + sql.AppendLine(";"); + + recordIndex++; + } + + command.CommandText = sql.ToString(); + + return command; + } + + public static DbCommand BuildSelectDataCommand( + SqliteConnection connection, + string tableName, + CollectionModel model, + List conditions, + FilteredRecordRetrievalOptions? filterOptions = null, + string? extraWhereFilter = null, + Dictionary? extraParameters = null, + int top = 0, + int skip = 0) + { + var builder = new StringBuilder(); + + var (command, whereClause) = GetCommandWithWhereClause(connection, conditions, extraWhereFilter, extraParameters); + + builder.Append("SELECT "); + builder.AppendColumnNames(includeVectors: false, model.Properties); + builder.Append("FROM ").AppendIdentifier(tableName).AppendLine(); + builder.AppendWhereClause(whereClause); + + if (filterOptions is not null) + { + builder.AppendOrderBy(model, filterOptions); + } + + builder.AppendLimits(top, skip); + + command.CommandText = builder.ToString(); + + return command; + } + + public static DbCommand BuildSelectInnerJoinCommand( + SqliteConnection connection, + string vectorTableName, + string dataTableName, + string keyColumnName, + CollectionModel model, + IReadOnlyList conditions, + bool includeDistance, + FilteredRecordRetrievalOptions? filterOptions = null, + string? extraWhereFilter = null, + Dictionary? extraParameters = null, + int top = 0, + int skip = 0, + double? scoreThreshold = null) + { + const string SubqueryName = "subquery"; + + var builder = new StringBuilder(); + + var subqueryCommand = BuildSelectDataCommand( + connection, + dataTableName, + model, + [], + filterOptions, + extraWhereFilter, + extraParameters, + top, + skip); + + var queryExtraFilter = new StringBuilder() + .AppendIdentifier(vectorTableName).Append('.').AppendIdentifier(keyColumnName) + .Append(" IN (SELECT ").AppendIdentifier(keyColumnName).Append(" FROM ").Append(SubqueryName).Append(')') + .ToString(); + var (command, whereClause) = GetCommandWithWhereClause(connection, conditions, queryExtraFilter, []); + + foreach (var parameter in subqueryCommand.Parameters) + { + command.Parameters.Add(parameter); + } + + builder.Append("WITH ").Append(SubqueryName).Append(" AS (").Append(subqueryCommand.CommandText).AppendLine(") "); + + builder.Append("SELECT "); + builder.AppendColumnNames(includeVectors: true, model.Properties, vectorTableName, dataTableName); + if (includeDistance) + { + builder.Append(", ").AppendIdentifier(vectorTableName).Append('.').AppendIdentifier(DistancePropertyName).AppendLine(); + } + builder.Append("FROM ").AppendIdentifier(vectorTableName).AppendLine(); + builder.Append("INNER JOIN ").AppendIdentifier(dataTableName).Append(" ON ") + .AppendIdentifier(vectorTableName).Append('.').AppendIdentifier(keyColumnName).Append(" = ") + .AppendIdentifier(dataTableName).Append('.').AppendIdentifier(keyColumnName).AppendLine(); + + // SQLite vec only supports distance metrics (lower = more similar), so filter with <= + if (scoreThreshold.HasValue) + { + var scoreThresholdClause = new StringBuilder() + .AppendIdentifier(vectorTableName).Append('.').AppendIdentifier(DistancePropertyName).Append(" <= @scoreThreshold") + .ToString(); + whereClause = string.IsNullOrEmpty(whereClause) + ? scoreThresholdClause + : $"{whereClause} AND {scoreThresholdClause}"; + command.Parameters.Add(new SqliteParameter("@scoreThreshold", scoreThreshold.Value)); + } + + builder.AppendWhereClause(whereClause); + + if (filterOptions is not null) + { + builder.AppendOrderBy(model, filterOptions, dataTableName); + } + else if (includeDistance) + { + builder.Append("ORDER BY ").AppendIdentifier(vectorTableName).Append('.').AppendIdentifier(DistancePropertyName).AppendLine(); + } + + builder.AppendLimits(top, skip); + + command.CommandText = builder.ToString(); + + return command; + } + + public static DbCommand BuildDeleteCommand( + SqliteConnection connection, + string tableName, + IReadOnlyList conditions) + { + var builder = new StringBuilder(); + + var (command, whereClause) = GetCommandWithWhereClause(connection, conditions); + + builder.Append("DELETE FROM ").AppendIdentifier(tableName).AppendLine(); + builder.AppendWhereClause(whereClause); + + command.CommandText = builder.ToString(); + + return command; + } + + /// + /// Appends a properly quoted and escaped SQLite identifier to the StringBuilder. + /// In SQLite, identifiers are quoted with double quotes, and embedded double quotes are escaped by doubling them. + /// + internal static StringBuilder AppendIdentifier(this StringBuilder sb, string identifier) + => sb.Append('"').Append(identifier.Replace("\"", "\"\"")).Append('"'); + + #region private + + private static StringBuilder AppendColumnNames(this StringBuilder builder, bool includeVectors, IReadOnlyList properties, + string? vectorTableName = null, string? dataTableName = null) + { + foreach (var property in properties) + { + string? tableName = dataTableName; + if (property is VectorPropertyModel) + { + if (!includeVectors) + { + continue; + } + tableName = vectorTableName; + } + + if (tableName is not null) + { + builder.AppendIdentifier(tableName).Append('.').AppendIdentifier(property.StorageName).Append(','); + } + else + { + builder.AppendIdentifier(property.StorageName).Append(','); + } + } + + builder.Length--; // Remove the trailing comma + builder.AppendLine(); + return builder; + } + + private static StringBuilder AppendOrderBy(this StringBuilder builder, CollectionModel model, + FilteredRecordRetrievalOptions options, string? tableName = null) + { + var orderBy = options.OrderBy?.Invoke(new()).Values; + if (orderBy is { Count: > 0 }) + { + builder.Append("ORDER BY "); + + foreach (var sortInfo in orderBy) + { + var storageName = model.GetDataOrKeyProperty(sortInfo.PropertySelector).StorageName; + + if (tableName is not null) + { + builder.AppendIdentifier(tableName).Append('.'); + } + + builder.AppendIdentifier(storageName).Append(sortInfo.Ascending ? " ASC," : " DESC,"); + } + + builder.Length--; // remove the last comma + builder.AppendLine(); + } + + return builder; + } + + private static StringBuilder AppendLimits(this StringBuilder builder, int top, int skip) + { + if (top > 0) + { + builder.AppendFormat("LIMIT {0}", top).AppendLine(); + } + + if (skip > 0) + { + builder.AppendFormat("OFFSET {0}", skip).AppendLine(); + } + + return builder; + } + + private static StringBuilder AppendWhereClause(this StringBuilder builder, string? whereClause) + { + if (!string.IsNullOrWhiteSpace(whereClause)) + { + builder.AppendLine($"WHERE {whereClause}"); + } + + return builder; + } + + private static string GetColumnDefinition(SqliteColumn column, bool quote, bool includeNullability) + { + const string PrimaryKeyIdentifier = "PRIMARY KEY"; + + List columnDefinitionParts = [quote ? QuoteIdentifier(column.Name) : column.Name, column.Type]; + + if (column.IsPrimary) + { + columnDefinitionParts.Add(PrimaryKeyIdentifier); + } + + if (includeNullability && !column.IsPrimary && !column.IsNullable) + { + columnDefinitionParts.Add("NOT NULL"); + } + + if (column.Configuration is { Count: > 0 }) + { + columnDefinitionParts.AddRange(column.Configuration + .Select(configuration => $"{configuration.Key}={configuration.Value}")); + } + + return string.Join(" ", columnDefinitionParts); + } + + private static string QuoteIdentifier(string identifier) + => $"\"{identifier.Replace("\"", "\"\"")}\""; + + private static (DbCommand Command, string WhereClause) GetCommandWithWhereClause( + SqliteConnection connection, + IReadOnlyList conditions, + string? extraWhereFilter = null, + Dictionary? extraParameters = null) + { + const string WhereClauseOperator = " AND "; + + var command = connection.CreateCommand(); + var whereClauseParts = new List(); + + foreach (var condition in conditions) + { + var parameterNames = new List(); + + for (var parameterIndex = 0; parameterIndex < condition.Values.Count; parameterIndex++) + { + var parameterName = GetParameterName(condition.Operand, parameterIndex); + + parameterNames.Add(parameterName); + + command.Parameters.Add(new SqliteParameter(parameterName, condition.Values[parameterIndex])); + } + + whereClauseParts.Add(condition.BuildQuery(parameterNames)); + } + + var whereClause = string.Join(WhereClauseOperator, whereClauseParts); + + if (extraWhereFilter is not null) + { + if (conditions.Count > 0) + { + whereClause += " AND "; + } + + whereClause += extraWhereFilter; + + Debug.Assert(extraParameters is not null, "extraParameters must be provided when extraWhereFilter is provided."); + foreach (var p in extraParameters!) + { + command.Parameters.Add(new SqliteParameter(p.Key, p.Value)); + } + } + + return (command, whereClause); + } + + private static string GetParameterName(string propertyName, int index) + => $"@{propertyName}{index}"; + + #endregion +} diff --git a/MEVD/src/SqliteVec/SqliteConstants.cs b/MEVD/src/SqliteVec/SqliteConstants.cs new file mode 100644 index 0000000..fb753f2 --- /dev/null +++ b/MEVD/src/SqliteVec/SqliteConstants.cs @@ -0,0 +1,9 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace CommunityToolkit.VectorData.SqliteVec; + +internal static class SqliteConstants +{ + internal const string VectorStoreSystemName = "sqlite"; +} diff --git a/MEVD/src/SqliteVec/SqliteDynamicCollection.cs b/MEVD/src/SqliteVec/SqliteDynamicCollection.cs new file mode 100644 index 0000000..d9c7ab2 --- /dev/null +++ b/MEVD/src/SqliteVec/SqliteDynamicCollection.cs @@ -0,0 +1,30 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace CommunityToolkit.VectorData.SqliteVec; + +/// +/// Represents a collection of vector store records in a Sqlite database, mapped to a dynamic Dictionary<string, object?>. +/// +#pragma warning disable CA1711 // Identifiers should not have incorrect suffix +public sealed class SqliteDynamicCollection : SqliteCollection> +#pragma warning restore CA1711 // Identifiers should not have incorrect suffix +{ + /// + /// Initializes a new instance of the class. + /// + /// The connection string for the SQLite database represented by this . + /// The name of the collection. + /// Optional configuration options for this class. + public SqliteDynamicCollection(string connectionString, string name, SqliteCollectionOptions options) + : base( + connectionString, + name, + static options => new SqliteModelBuilder() + .BuildDynamic( + options.Definition ?? throw new ArgumentException("Definition is required for dynamic collections"), + options.EmbeddingGenerator), + options) + { + } +} diff --git a/MEVD/src/SqliteVec/SqliteExtensions.cs b/MEVD/src/SqliteVec/SqliteExtensions.cs new file mode 100644 index 0000000..1bb7a92 --- /dev/null +++ b/MEVD/src/SqliteVec/SqliteExtensions.cs @@ -0,0 +1,21 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Data.Sqlite; + +namespace CommunityToolkit.VectorData.SqliteVec; + +internal static class SqliteExtensions +{ + public static T GetFieldValue(this SqliteDataReader reader, string fieldName) + { + int ordinal = reader.GetOrdinal(fieldName); + return reader.GetFieldValue(ordinal); + } + + public static string GetString(this SqliteDataReader reader, string fieldName) + { + int ordinal = reader.GetOrdinal(fieldName); + return reader.GetString(ordinal); + } +} diff --git a/MEVD/src/SqliteVec/SqliteFilterTranslator.cs b/MEVD/src/SqliteVec/SqliteFilterTranslator.cs new file mode 100644 index 0000000..6c7c4b1 --- /dev/null +++ b/MEVD/src/SqliteVec/SqliteFilterTranslator.cs @@ -0,0 +1,348 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Linq.Expressions; +using System.Text; +using Microsoft.Extensions.VectorData.ProviderServices; +using Microsoft.Extensions.VectorData.ProviderServices.Filter; + +namespace CommunityToolkit.VectorData.SqliteVec; + +internal sealed class SqliteFilterTranslator : FilterTranslatorBase +{ + private readonly StringBuilder _sql; + private readonly Expression _preprocessedExpression; + private readonly Dictionary _parameters = []; + + internal SqliteFilterTranslator(CollectionModel model, LambdaExpression lambdaExpression) + { + Debug.Assert(lambdaExpression.Parameters.Count == 1); + _sql = new(); + + _preprocessedExpression = PreprocessFilter(lambdaExpression, model, new FilterPreprocessingOptions { SupportsParameterization = true }); + } + + internal StringBuilder Clause => _sql; + + internal Dictionary Parameters => _parameters; + + internal void Translate(bool appendWhere) + { + if (appendWhere) + { + _sql.Append("WHERE "); + } + + Translate(_preprocessedExpression); + } + + private void Translate(Expression? node) + { + switch (node) + { + case BinaryExpression binary: + TranslateBinary(binary); + return; + + case ConstantExpression constant: + TranslateConstant(constant.Value); + return; + + case QueryParameterExpression { Name: var name, Value: var value }: + TranslateQueryParameter(value); + return; + + case MemberExpression member: + TranslateMember(member); + return; + + case MethodCallExpression methodCall: + TranslateMethodCall(methodCall); + return; + + case UnaryExpression unary: + TranslateUnary(unary); + return; + + default: + throw new NotSupportedException("Unsupported NodeType in filter: " + node?.NodeType); + } + } + + private void TranslateBinary(BinaryExpression binary) + { + // Special handling for null comparisons + switch (binary.NodeType) + { + case ExpressionType.Equal when IsNull(binary.Right): + _sql.Append('('); + Translate(binary.Left); + _sql.Append(" IS NULL)"); + return; + case ExpressionType.NotEqual when IsNull(binary.Right): + _sql.Append('('); + Translate(binary.Left); + _sql.Append(" IS NOT NULL)"); + return; + + case ExpressionType.Equal when IsNull(binary.Left): + _sql.Append('('); + Translate(binary.Right); + _sql.Append(" IS NULL)"); + return; + case ExpressionType.NotEqual when IsNull(binary.Left): + _sql.Append('('); + Translate(binary.Right); + _sql.Append(" IS NOT NULL)"); + return; + } + + _sql.Append('('); + Translate(binary.Left); + + _sql.Append(binary.NodeType switch + { + ExpressionType.Equal => " = ", + ExpressionType.NotEqual => " <> ", + + ExpressionType.GreaterThan => " > ", + ExpressionType.GreaterThanOrEqual => " >= ", + ExpressionType.LessThan => " < ", + ExpressionType.LessThanOrEqual => " <= ", + + ExpressionType.AndAlso => " AND ", + ExpressionType.OrElse => " OR ", + + _ => throw new NotSupportedException("Unsupported binary expression node type: " + binary.NodeType) + }); + + Translate(binary.Right); + + _sql.Append(')'); + + static bool IsNull(Expression expression) + => expression is ConstantExpression { Value: null } or QueryParameterExpression { Value: null }; + } + + private void TranslateConstant(object? value) + { + switch (value) + { + case byte b: + _sql.Append(b); + return; + case short s: + _sql.Append(s); + return; + case int i: + _sql.Append(i); + return; + case long l: + _sql.Append(l); + return; + + case float f: + _sql.Append(f); + return; + case double d: + _sql.Append(d); + return; + case decimal d: + _sql.Append(d); + return; + + case string untrustedInput: + _sql.Append('\'').Append(untrustedInput.Replace("'", "''")).Append('\''); + return; + case bool b: + _sql.Append(b ? "TRUE" : "FALSE"); + return; + case Guid g: + // Microsoft.Data.Sqlite writes GUIDs as upper-case strings, align our constant formatting with that. + _sql.Append('\'').Append(g.ToString().ToUpperInvariant()).Append('\''); + return; + + case DateTime dateTime: + _sql.Append('\'').Append(dateTime.ToString("yyyy-MM-dd HH:mm:ss.FFFFFFF", System.Globalization.CultureInfo.InvariantCulture)).Append('\''); + return; + case DateTimeOffset dateTimeOffset: + _sql.Append('\'').Append(dateTimeOffset.ToString("yyyy-MM-dd HH:mm:ss.FFFFFFFzzz", System.Globalization.CultureInfo.InvariantCulture)).Append('\''); + return; +#if NET + case DateOnly dateOnly: + _sql.Append('\'').Append(dateOnly.ToString("yyyy-MM-dd", System.Globalization.CultureInfo.InvariantCulture)).Append('\''); + return; + case TimeOnly timeOnly: + _sql.Append('\'').Append(timeOnly.ToString("HH:mm:ss.FFFFFFF", System.Globalization.CultureInfo.InvariantCulture)).Append('\''); + return; +#endif + + case null: + _sql.Append("NULL"); + return; + + default: + throw new NotSupportedException("Unsupported constant type: " + value.GetType().Name); + } + } + + private void TranslateMember(MemberExpression memberExpression) + { + if (TryBindProperty(memberExpression, out var property)) + { + GenerateColumn(property); + return; + } + + throw new NotSupportedException($"Member access for '{memberExpression.Member.Name}' is unsupported - only member access over the filter parameter are supported"); + } + + private void GenerateColumn(PropertyModel property) + => _sql.Append('"').Append(property.StorageName.Replace("\"", "\"\"")).Append('"'); + + private void TranslateQueryParameter(object? value) + { + // For null values, simply inline rather than parameterize + if (value is null) + { + _sql.Append("NULL"); + } + else + { + int index = _sql.Length; + _sql.Append('@').Append(_parameters.Count + 1); + string paramName = _sql.ToString(index, _sql.Length - index); + _parameters.Add(paramName, value); + } + } + + private void TranslateMethodCall(MethodCallExpression methodCall) + { + if (TryBindProperty(methodCall, out var property)) + { + GenerateColumn(property); + return; + } + + switch (methodCall) + { + case var _ when TryMatchContains(methodCall, out var source, out var item): + TranslateContains(source, item); + return; + + case { Method.Name: nameof(Enumerable.Any), Arguments: [var anySource, LambdaExpression lambda] } any + when any.Method.DeclaringType == typeof(Enumerable): + TranslateAny(anySource, lambda); + return; + + default: + throw new NotSupportedException($"Unsupported method call: {methodCall.Method.DeclaringType?.Name}.{methodCall.Method.Name}"); + } + } + + private void TranslateContains(Expression source, Expression item) + { + switch (source) + { + // TODO: support Contains over array fields (#10343) + case var _ when TryBindProperty(source, out _): + throw new NotSupportedException("Unsupported Contains expression"); + + // Contains over inline array (r => new[] { "foo", "bar" }.Contains(r.String)) + case NewArrayExpression newArray: + Translate(item); + _sql.Append(" IN ("); + + var isFirst = true; + foreach (var element in newArray.Expressions) + { + if (isFirst) + { + isFirst = false; + } + else + { + _sql.Append(", "); + } + + Translate(element); + } + + _sql.Append(')'); + return; + + // Contains over captured array (r => arrayLocalVariable.Contains(r.String)) + case QueryParameterExpression { Value: var value }: + if (value is not IEnumerable elements) + { + throw new NotSupportedException("Unsupported Contains expression"); + } + + Translate(item); + _sql.Append(" IN ("); + + isFirst = true; + foreach (var element in elements) + { + if (isFirst) + { + isFirst = false; + } + else + { + _sql.Append(", "); + } + + TranslateConstant(element); + } + + _sql.Append(')'); + return; + + default: + throw new NotSupportedException("Unsupported Contains expression"); + } + } + + // TODO: support Any over array fields (#10343) + private void TranslateAny(Expression source, LambdaExpression lambda) + => throw new NotSupportedException("Unsupported method call: Enumerable.Any"); + + private void TranslateUnary(UnaryExpression unary) + { + switch (unary.NodeType) + { + case ExpressionType.Not: + if (unary.Operand is BinaryExpression { NodeType: ExpressionType.Equal or ExpressionType.NotEqual } binary) + { + TranslateBinary( + Expression.MakeBinary( + binary.NodeType is ExpressionType.Equal ? ExpressionType.NotEqual : ExpressionType.Equal, + binary.Left, + binary.Right)); + return; + } + + _sql.Append("(NOT "); + Translate(unary.Operand); + _sql.Append(')'); + return; + + case ExpressionType.Convert when Nullable.GetUnderlyingType(unary.Type) == unary.Operand.Type: + Translate(unary.Operand); + return; + + case ExpressionType.Convert when TryBindProperty(unary.Operand, out var property) && unary.Type == property.Type: + GenerateColumn(property); + return; + + default: + throw new NotSupportedException("Unsupported unary expression node type: " + unary.NodeType); + } + } +} diff --git a/MEVD/src/SqliteVec/SqliteMapper.cs b/MEVD/src/SqliteVec/SqliteMapper.cs new file mode 100644 index 0000000..4160e50 --- /dev/null +++ b/MEVD/src/SqliteVec/SqliteMapper.cs @@ -0,0 +1,114 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Data.Common; +using System.Diagnostics; +using System.Runtime.InteropServices; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData.ProviderServices; + +namespace CommunityToolkit.VectorData.SqliteVec; + +/// +/// Class for mapping between a dictionary and the consumer data model. +/// +/// The consumer data model to map to or from. +internal sealed class SqliteMapper(CollectionModel model) +{ + public TRecord MapFromStorageToDataModel(DbDataReader reader, bool includeVectors) + { + var record = model.CreateRecord()!; + + var keyProperty = model.KeyProperty; + keyProperty.SetValueAsObject( + record, + GetPropertyValue(reader, keyProperty.StorageName, keyProperty.Type)); + + foreach (var property in model.DataProperties) + { + property.SetValueAsObject( + record, + GetPropertyValue(reader, property.StorageName, property.Type)); + } + + if (includeVectors) + { + foreach (var property in model.VectorProperties) + { + int ordinal = reader.GetOrdinal(property.StorageName); + + if (reader.IsDBNull(ordinal)) + { + continue; + } + + // SqliteVec provides the vector data as a byte[], which we need to convert to a float[]. + // In modern .NET, we allocate a float[] of the right size, reinterpret-cast it into byte[], + // and then read the data into that via Stream. + // In .NET Framework, which doesn't have Span APIs on Stream, we just create a copy (inefficient). +#if NET + using var stream = reader.GetStream(ordinal); + + var length = stream.Length; + if (length % 4 != 0) + { + throw new InvalidOperationException($"Retrieved value for vector property '{property.StorageName}' which is not a valid byte array length (expected multiple of 4, got {stream.Length})."); + } + + var floats = new float[length / 4]; + var bytes = MemoryMarshal.Cast(floats.AsSpan()); + stream.ReadExactly(bytes); +#else + var floats = MemoryMarshal.Cast((byte[])reader[ordinal]).ToArray(); +#endif + + property.SetValueAsObject( + record, + (Nullable.GetUnderlyingType(property.Type) ?? property.Type) switch + { + var t when t == typeof(ReadOnlyMemory) => new ReadOnlyMemory(floats), + var t when t == typeof(Embedding) => new Embedding(floats), + var t when t == typeof(float[]) => floats, + + _ => throw new UnreachableException() + }); + } + } + + return record; + } + + private static object? GetPropertyValue(DbDataReader reader, string propertyName, Type propertyType) + { + int ordinal = reader.GetOrdinal(propertyName); + + if (reader.IsDBNull(ordinal)) + { + return null; + } + + return (Nullable.GetUnderlyingType(propertyType) ?? propertyType) switch + { + Type t when t == typeof(int) => reader.GetInt32(ordinal), + Type t when t == typeof(long) => reader.GetInt64(ordinal), + Type t when t == typeof(short) => reader.GetInt16(ordinal), + Type t when t == typeof(bool) => reader.GetBoolean(ordinal), + Type t when t == typeof(float) => reader.GetFloat(ordinal), + Type t when t == typeof(double) => reader.GetDouble(ordinal), + Type t when t == typeof(string) => reader.GetString(ordinal), + Type t when t == typeof(Guid) => reader.GetGuid(ordinal), + Type t when t == typeof(DateTime) => reader.GetDateTime(ordinal), + Type t when t == typeof(DateTimeOffset) => reader.GetFieldValue(ordinal), +#if NET + Type t when t == typeof(DateOnly) => reader.GetFieldValue(ordinal), + Type t when t == typeof(TimeOnly) => reader.GetFieldValue(ordinal), +#endif + Type t when t == typeof(byte[]) => (byte[])reader[ordinal], + Type t when t == typeof(ReadOnlyMemory) => (byte[])reader[ordinal], + Type t when t == typeof(Embedding) => (byte[])reader[ordinal], + Type t when t == typeof(float[]) => (byte[])reader[ordinal], + + _ => throw new NotSupportedException($"Unsupported type: {propertyType} for property: {propertyName}") + }; + } +} diff --git a/MEVD/src/SqliteVec/SqliteModelBuilder.cs b/MEVD/src/SqliteVec/SqliteModelBuilder.cs new file mode 100644 index 0000000..2bb0732 --- /dev/null +++ b/MEVD/src/SqliteVec/SqliteModelBuilder.cs @@ -0,0 +1,78 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData.ProviderServices; + +namespace CommunityToolkit.VectorData.SqliteVec; + +internal class SqliteModelBuilder() : CollectionModelBuilder(s_modelBuildingOptions) +{ + internal const string SupportedVectorTypes = "ReadOnlyMemory, Embedding, float[]"; + + private static readonly CollectionModelBuildingOptions s_modelBuildingOptions = new() + { + RequiresAtLeastOneVector = false, + SupportsMultipleVectors = true, + }; + + protected override bool SupportsKeyAutoGeneration(Type keyPropertyType) + => keyPropertyType == typeof(Guid) || keyPropertyType == typeof(int) || keyPropertyType == typeof(long); + + protected override void ValidateKeyProperty(KeyPropertyModel keyProperty) + { + base.ValidateKeyProperty(keyProperty); + + var type = keyProperty.Type; + + if (type != typeof(int) && type != typeof(long) && type != typeof(string) && type != typeof(Guid)) + { + throw new NotSupportedException($"The property type '{type.FullName}' is not supported for key properties by the SqliteVec provider. Supported types are: int, long, string, Guid."); + } + } + + protected override bool IsDataPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes) + { + supportedTypes = "int, long, short, string, bool, float, double, byte[], Guid, DateTime, DateTimeOffset" +#if NET + + ", DateOnly, TimeOnly" +#endif + ; + + if (Nullable.GetUnderlyingType(type) is Type underlyingType) + { + type = underlyingType; + } + + return type == typeof(int) + || type == typeof(long) + || type == typeof(short) + || type == typeof(string) + || type == typeof(bool) + || type == typeof(float) + || type == typeof(double) + || type == typeof(byte[]) + || type == typeof(Guid) + || type == typeof(DateTime) + || type == typeof(DateTimeOffset) +#if NET + || type == typeof(DateOnly) + || type == typeof(TimeOnly) +#endif + ; + } + + protected override bool IsVectorPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes) + => IsVectorPropertyTypeValidCore(type, out supportedTypes); + + internal static bool IsVectorPropertyTypeValidCore(Type type, [NotNullWhen(false)] out string? supportedTypes) + { + supportedTypes = SupportedVectorTypes; + + return type == typeof(ReadOnlyMemory) + || type == typeof(ReadOnlyMemory?) + || type == typeof(Embedding) + || type == typeof(float[]); + } +} diff --git a/MEVD/src/SqliteVec/SqlitePropertyMapping.cs b/MEVD/src/SqliteVec/SqlitePropertyMapping.cs new file mode 100644 index 0000000..41782d0 --- /dev/null +++ b/MEVD/src/SqliteVec/SqlitePropertyMapping.cs @@ -0,0 +1,144 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Data.Common; +using System.Diagnostics; +using System.Runtime.InteropServices; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; + +namespace CommunityToolkit.VectorData.SqliteVec; + +/// +/// Contains helper methods with property mapping for SQLite. +/// +internal static class SqlitePropertyMapping +{ + public static byte[] MapVectorForStorageModel(ReadOnlyMemory memory) + { + ReadOnlySpan floatSpan = memory.Span; + byte[] byteArray = new byte[floatSpan.Length * sizeof(float)]; + MemoryMarshal.AsBytes(floatSpan).CopyTo(byteArray); + + return byteArray; + } + + public static List GetColumns(IReadOnlyList properties, bool data) + { + const string DistanceMetricConfigurationName = "distance_metric"; + + var columns = new List(); + + foreach (var property in properties) + { + var isPrimary = false; + + string propertyType; + Dictionary? configuration = null; + + if (property is VectorPropertyModel vectorProperty) + { + if (data) + { + continue; + } + + propertyType = GetStorageVectorPropertyType(vectorProperty); + configuration = new() + { + [DistanceMetricConfigurationName] = GetDistanceMetric(vectorProperty) + }; + } + else if (property is DataPropertyModel dataProperty) + { + if (!data) + { + continue; + } + + propertyType = GetStorageDataPropertyType(property); + } + else + { + // The Key column in included in both Vector and Data tables. + Debug.Assert(property is KeyPropertyModel, "property is not a KeyPropertyModel"); + + propertyType = GetStorageDataPropertyType(property); + isPrimary = true; + } + + var column = new SqliteColumn(property.StorageName, propertyType, isPrimary) + { + IsNullable = property.IsNullable, + Configuration = configuration, + HasIndex = property is DataPropertyModel { IsIndexed: true } + }; + + columns.Add(column); + } + + return columns; + } + + public static TPropertyType? GetPropertyValue(DbDataReader reader, string propertyName) + { + int propertyIndex = reader.GetOrdinal(propertyName); + + // TODO: Check this + return reader.IsDBNull(propertyIndex) + ? default + : reader.GetFieldValue(propertyIndex); + } + + #region private + + private static string GetStorageDataPropertyType(PropertyModel property) + => property.Type switch + { + // Integer types + Type t when t == typeof(int) || t == typeof(int?) => "INTEGER", + Type t when t == typeof(long) || t == typeof(long?) => "INTEGER", + Type t when t == typeof(short) || t == typeof(short?) => "INTEGER", + + // Floating-point types + Type t when t == typeof(float) || t == typeof(float?) => "REAL", + Type t when t == typeof(double) || t == typeof(double?) => "REAL", + + // String type + Type t when t == typeof(string) => "TEXT", + + // Boolean type - represent it as INTEGER (0/1 (this is standard SQLite) + Type t when t == typeof(bool) || t == typeof(bool?) => "INTEGER", + + // Guid type - represent as TEXT + Type t when t == typeof(Guid) || t == typeof(Guid?) => "TEXT", + + // Date/time types - represent as TEXT (ISO 8601) + Type t when t == typeof(DateTime) || t == typeof(DateTime?) => "TEXT", + Type t when t == typeof(DateTimeOffset) || t == typeof(DateTimeOffset?) => "TEXT", +#if NET + Type t when t == typeof(DateOnly) || t == typeof(DateOnly?) => "TEXT", + Type t when t == typeof(TimeOnly) || t == typeof(TimeOnly?) => "TEXT", +#endif + + // Byte array (BLOB) + Type t when t == typeof(byte[]) => "BLOB", + + // Default fallback for unknown types + _ => throw new NotSupportedException($"Property '{property.ModelName}' has type '{property.Type.Name}', which is not supported by SQLite connector.") + }; + + private static string GetDistanceMetric(VectorPropertyModel vectorProperty) + => vectorProperty.DistanceFunction switch + { + DistanceFunction.CosineDistance or null => "cosine", + DistanceFunction.ManhattanDistance => "l1", + DistanceFunction.EuclideanDistance => "l2", + _ => throw new NotSupportedException($"Distance function '{vectorProperty.DistanceFunction}' for {nameof(VectorStoreVectorProperty)} '{vectorProperty.ModelName}' is not supported by the SQLite connector.") + }; + + private static string GetStorageVectorPropertyType(VectorPropertyModel vectorProperty) + => $"FLOAT[{vectorProperty.Dimensions}]"; + + #endregion +} diff --git a/MEVD/src/SqliteVec/SqliteServiceCollectionExtensions.cs b/MEVD/src/SqliteVec/SqliteServiceCollectionExtensions.cs new file mode 100644 index 0000000..286d784 --- /dev/null +++ b/MEVD/src/SqliteVec/SqliteServiceCollectionExtensions.cs @@ -0,0 +1,194 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData; +using CommunityToolkit.VectorData.SqliteVec; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.DependencyInjection; + +/// +/// Extension methods to register SQLite instances on an . +/// +public static class SqliteServiceCollectionExtensions +{ + private const string DynamicCodeMessage = "This method is incompatible with NativeAOT, consult the documentation for adding collections in a way that's compatible with NativeAOT."; + private const string UnreferencedCodeMessage = "This method is incompatible with trimming, consult the documentation for adding collections in a way that's compatible with NativeAOT."; + + /// + /// Registers a as , with the specified connection string and service lifetime. + /// + /// + public static IServiceCollection AddSqliteVectorStore( + this IServiceCollection services, + Func connectionStringProvider, + Func? optionsProvider = null, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + => AddKeyedSqliteVectorStore(services, serviceKey: null, connectionStringProvider, optionsProvider, lifetime); + + /// + /// Registers a keyed as , with the specified connection string and service lifetime. + /// + /// The to register the on. + /// The key with which to associate the vector store. + /// The connection string provider. + /// Options provider to further configure the vector store. + /// The service lifetime for the store. Defaults to . + /// The service collection. + public static IServiceCollection AddKeyedSqliteVectorStore( + this IServiceCollection services, + object? serviceKey, + Func connectionStringProvider, + Func? optionsProvider = null, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + { + Throw.IfNull(services); + Throw.IfNull(connectionStringProvider); + + services.Add(new ServiceDescriptor(typeof(SqliteVectorStore), serviceKey, (sp, _) => + { + var connectionString = connectionStringProvider(sp); + var options = GetStoreOptions(sp, optionsProvider); + return new SqliteVectorStore(connectionString, options); + }, lifetime)); + + services.Add(new ServiceDescriptor(typeof(VectorStore), serviceKey, + static (sp, key) => sp.GetRequiredKeyedService(key), lifetime)); + + return services; + } + + /// + /// Registers a as , with the specified connection string and service lifetime. + /// + /// + [RequiresDynamicCode(DynamicCodeMessage)] + [RequiresUnreferencedCode(UnreferencedCodeMessage)] + public static IServiceCollection AddSqliteCollection( + this IServiceCollection services, + string name, + Func connectionStringProvider, + Func? optionsProvider = null, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + where TKey : notnull + where TRecord : class + => AddKeyedSqliteCollection(services, serviceKey: null, name, connectionStringProvider, optionsProvider, lifetime); + + /// + /// Registers a keyed as , with the specified connection string and service lifetime. + /// + /// The to register the on. + /// The key with which to associate the collection. + /// The name of the collection. + /// The connection string provider. + /// Options provider to further configure the collection. + /// The service lifetime for the store. Defaults to . + /// The service collection. + [RequiresDynamicCode(DynamicCodeMessage)] + [RequiresUnreferencedCode(UnreferencedCodeMessage)] + public static IServiceCollection AddKeyedSqliteCollection( + this IServiceCollection services, + object? serviceKey, + string name, + Func connectionStringProvider, + Func? optionsProvider = null, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + where TKey : notnull + where TRecord : class + { + Throw.IfNull(services); + Throw.IfNullOrWhitespace(name); + Throw.IfNull(connectionStringProvider); + + services.Add(new ServiceDescriptor(typeof(SqliteCollection), serviceKey, (sp, _) => + { + var connectionString = connectionStringProvider(sp); + var options = GetCollectionOptions(sp, optionsProvider); + return new SqliteCollection(connectionString, name, options); + }, lifetime)); + + services.Add(new ServiceDescriptor(typeof(VectorStoreCollection), serviceKey, + static (sp, key) => sp.GetRequiredKeyedService>(key), lifetime)); + + services.Add(new ServiceDescriptor(typeof(IVectorSearchable), serviceKey, + static (sp, key) => sp.GetRequiredKeyedService>(key), lifetime)); + + // Once HybridSearch supports get implemented by SqliteCollection + // we need to add IKeywordHybridSearchable abstraction here as well. + + return services; + } + + /// + /// Registers a as , with the specified connection string and service lifetime. + /// + /// /> + [RequiresDynamicCode(DynamicCodeMessage)] + [RequiresUnreferencedCode(UnreferencedCodeMessage)] + public static IServiceCollection AddSqliteCollection( + this IServiceCollection services, + string name, + string connectionString, + SqliteCollectionOptions? options = null, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + where TKey : notnull + where TRecord : class + => AddKeyedSqliteCollection(services, serviceKey: null, name, connectionString, options, lifetime); + + /// + /// Registers a keyed as , with the specified connection string and service lifetime. + /// + /// The to register the on. + /// The key with which to associate the collection. + /// The name of the collection. + /// The connection string. + /// Options to further configure the collection. + /// The service lifetime for the store. Defaults to . + /// The service collection. + [RequiresDynamicCode(DynamicCodeMessage)] + [RequiresUnreferencedCode(UnreferencedCodeMessage)] + public static IServiceCollection AddKeyedSqliteCollection( + this IServiceCollection services, + object? serviceKey, + string name, + string connectionString, + SqliteCollectionOptions? options = null, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + where TKey : notnull + where TRecord : class + { + Throw.IfNullOrWhitespace(connectionString); + + return AddKeyedSqliteCollection(services, serviceKey, name, _ => connectionString, _ => options!, lifetime); + } + + private static SqliteVectorStoreOptions? GetStoreOptions(IServiceProvider sp, Func? optionsProvider) + { + var options = optionsProvider?.Invoke(sp); + if (options?.EmbeddingGenerator is not null) + { + return options; // The user has provided everything, there is nothing to change. + } + + var embeddingGenerator = sp.GetService(); + return embeddingGenerator is null + ? options // There is nothing to change. + : new(options) { EmbeddingGenerator = embeddingGenerator }; // Create a brand new copy in order to avoid modifying the original options. + } + + private static SqliteCollectionOptions? GetCollectionOptions(IServiceProvider sp, Func? optionsProvider) + { + var options = optionsProvider?.Invoke(sp); + if (options?.EmbeddingGenerator is not null) + { + return options; // The user has provided everything, there is nothing to change. + } + + var embeddingGenerator = sp.GetService(); + return embeddingGenerator is null + ? options // There is nothing to change. + : new(options) { EmbeddingGenerator = embeddingGenerator }; // Create a brand new copy in order to avoid modifying the original options. + } +} diff --git a/MEVD/src/SqliteVec/SqliteVec.csproj b/MEVD/src/SqliteVec/SqliteVec.csproj new file mode 100644 index 0000000..50615c6 --- /dev/null +++ b/MEVD/src/SqliteVec/SqliteVec.csproj @@ -0,0 +1,28 @@ + + + + 1.0.0-preview.1 + CommunityToolkit.VectorData.SqliteVec + $(AssemblyName) + net10.0;net8.0;netstandard2.0;net462 + true + + SQLite provider for Microsoft.Extensions.VectorData + SQLite (sqlitevec) provider for Microsoft.Extensions.VectorData by Semantic Kernel + + + + + + + + + + + + + + + + + diff --git a/MEVD/src/SqliteVec/SqliteVectorStore.cs b/MEVD/src/SqliteVec/SqliteVectorStore.cs new file mode 100644 index 0000000..5e149f7 --- /dev/null +++ b/MEVD/src/SqliteVec/SqliteVectorStore.cs @@ -0,0 +1,153 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; +using Microsoft.Data.Sqlite; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; +using Microsoft.Shared.Diagnostics; + +namespace CommunityToolkit.VectorData.SqliteVec; + +/// +/// Class for accessing the list of collections in a SQLite vector store. +/// +/// +/// This class can be used with collections of any schema type, but requires you to provide schema information when getting a collection. +/// +public sealed class SqliteVectorStore : VectorStore +{ + /// Metadata about vector store. + private readonly VectorStoreMetadata _metadata; + + /// The connection string for the SQLite database represented by this . + private readonly string _connectionString; + + /// A general purpose definition that can be used to construct a collection when needing to proxy schema agnostic operations. + private static readonly VectorStoreCollectionDefinition s_generalPurposeDefinition = new() { Properties = [new VectorStoreKeyProperty("Key", typeof(string))] }; + + /// Custom virtual table name to store vectors. + private readonly string? _vectorVirtualTableName; + + private readonly IEmbeddingGenerator? _embeddingGenerator; + + /// + /// Initializes a new instance of the class. + /// + /// The connection string for the SQLite database represented by this . + /// Optional configuration options for this class. + public SqliteVectorStore(string connectionString, SqliteVectorStoreOptions? options = default) + { + Throw.IfNull(connectionString); + + _connectionString = connectionString; + + options ??= SqliteVectorStoreOptions.Default; + _vectorVirtualTableName = options.VectorVirtualTableName; + _embeddingGenerator = options.EmbeddingGenerator; + + var connectionStringBuilder = new SqliteConnectionStringBuilder(connectionString); + + _metadata = new() + { + VectorStoreSystemName = SqliteConstants.VectorStoreSystemName, + VectorStoreName = connectionStringBuilder.DataSource + }; + } + +#pragma warning disable IDE0090 // Use 'new(...)' + /// + [RequiresDynamicCode("This overload of GetCollection() is incompatible with NativeAOT. For dynamic mapping via Dictionary, call GetDynamicCollection() instead.")] + [RequiresUnreferencedCode("This overload of GetCollecttion() is incompatible with trimming. For dynamic mapping via Dictionary, call GetDynamicCollection() instead.")] +#if NET + public override SqliteCollection GetCollection(string name, VectorStoreCollectionDefinition? definition = null) +#else + public override VectorStoreCollection GetCollection(string name, VectorStoreCollectionDefinition? definition = null) +#endif + => typeof(TRecord) == typeof(Dictionary) + ? throw new ArgumentException(VectorDataStrings.GetCollectionWithDictionaryNotSupported) + : new SqliteCollection( + _connectionString, + name, + new() + { + Definition = definition, + VectorVirtualTableName = _vectorVirtualTableName, + EmbeddingGenerator = _embeddingGenerator + }); + + /// +#if NET + public override SqliteDynamicCollection GetDynamicCollection(string name, VectorStoreCollectionDefinition definition) +#else + public override VectorStoreCollection> GetDynamicCollection(string name, VectorStoreCollectionDefinition definition) +#endif + => new SqliteDynamicCollection( + _connectionString, + name, + new() + { + Definition = definition, + VectorVirtualTableName = _vectorVirtualTableName, + EmbeddingGenerator = _embeddingGenerator + } + ); +#pragma warning restore IDE0090 + + /// + public override async IAsyncEnumerable ListCollectionNamesAsync([EnumeratorCancellation] CancellationToken cancellationToken = default) + { + const string OperationName = "ListCollectionNames"; + const string TablePropertyName = "name"; + const string Query = $"SELECT {TablePropertyName} FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%';"; + + using var connection = new SqliteConnection(_connectionString); + await connection.OpenAsync(cancellationToken).ConfigureAwait(false); + using var command = connection.CreateCommand(); + + command.CommandText = Query; + + using var reader = await connection.ExecuteWithErrorHandlingAsync( + _metadata, + OperationName, + () => command.ExecuteReaderAsync(cancellationToken), + cancellationToken).ConfigureAwait(false); + + while (await reader.ReadWithErrorHandlingAsync( + _metadata, + OperationName, + cancellationToken).ConfigureAwait(false)) + { + var ordinal = reader.GetOrdinal(TablePropertyName); + yield return reader.GetString(ordinal); + } + } + + /// + public override Task CollectionExistsAsync(string name, CancellationToken cancellationToken = default) + { + var collection = GetDynamicCollection(name, s_generalPurposeDefinition); + return collection.CollectionExistsAsync(cancellationToken); + } + + /// + public override Task EnsureCollectionDeletedAsync(string name, CancellationToken cancellationToken = default) + { + var collection = GetDynamicCollection(name, s_generalPurposeDefinition); + return collection.EnsureCollectionDeletedAsync(cancellationToken); + } + + /// + public override object? GetService(Type serviceType, object? serviceKey = null) + { + Throw.IfNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreMetadata) ? _metadata : + serviceType.IsInstanceOfType(this) ? this : + null; + } +} diff --git a/MEVD/src/SqliteVec/SqliteVectorStoreOptions.cs b/MEVD/src/SqliteVec/SqliteVectorStoreOptions.cs new file mode 100644 index 0000000..4835b67 --- /dev/null +++ b/MEVD/src/SqliteVec/SqliteVectorStoreOptions.cs @@ -0,0 +1,40 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.AI; + +namespace CommunityToolkit.VectorData.SqliteVec; + +/// +/// Options when creating a . +/// +public sealed class SqliteVectorStoreOptions +{ + internal static readonly SqliteVectorStoreOptions Default = new(); + + /// + /// Initializes a new instance of the class. + /// + public SqliteVectorStoreOptions() + { + } + + internal SqliteVectorStoreOptions(SqliteVectorStoreOptions? source) + { + VectorVirtualTableName = source?.VectorVirtualTableName; + EmbeddingGenerator = source?.EmbeddingGenerator; + } + + /// + /// Custom virtual table name to store vectors. + /// + /// + /// If not provided, collection name with prefix "vec_" will be used as virtual table name. + /// + public string? VectorVirtualTableName { get; set; } + + /// + /// Gets or sets the default embedding generator to use when generating vectors embeddings with this vector store. + /// + public IEmbeddingGenerator? EmbeddingGenerator { get; set; } +} diff --git a/MEVD/src/Weaviate/AssemblyInfo.cs b/MEVD/src/Weaviate/AssemblyInfo.cs new file mode 100644 index 0000000..d6395e8 --- /dev/null +++ b/MEVD/src/Weaviate/AssemblyInfo.cs @@ -0,0 +1,2 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. diff --git a/MEVD/src/Weaviate/Converters/WeaviateDateOnlyConverter.cs b/MEVD/src/Weaviate/Converters/WeaviateDateOnlyConverter.cs new file mode 100644 index 0000000..aec852b --- /dev/null +++ b/MEVD/src/Weaviate/Converters/WeaviateDateOnlyConverter.cs @@ -0,0 +1,41 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#if NET + +using System; +using System.Globalization; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace CommunityToolkit.VectorData.Weaviate; + +/// +/// Converts to RFC 3339 formatted string for Weaviate. +/// DateOnly values are stored as midnight UTC. +/// +internal sealed class WeaviateDateOnlyConverter : JsonConverter +{ + private const string DateTimeFormat = "yyyy-MM-dd'T'HH:mm:ss.fffZ"; + + public override DateOnly Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + var dateString = reader.GetString(); + + if (string.IsNullOrWhiteSpace(dateString)) + { + return default; + } + + var dateTimeOffset = DateTimeOffset.Parse(dateString, CultureInfo.InvariantCulture); + return DateOnly.FromDateTime(dateTimeOffset.DateTime); + } + + public override void Write(Utf8JsonWriter writer, DateOnly value, JsonSerializerOptions options) + { + var dateTimeOffset = new DateTimeOffset(value.ToDateTime(TimeOnly.MinValue), TimeSpan.Zero); + writer.WriteStringValue(dateTimeOffset.ToString(DateTimeFormat, CultureInfo.InvariantCulture)); + } +} + +#endif diff --git a/MEVD/src/Weaviate/Converters/WeaviateDateTimeConverter.cs b/MEVD/src/Weaviate/Converters/WeaviateDateTimeConverter.cs new file mode 100644 index 0000000..123657a --- /dev/null +++ b/MEVD/src/Weaviate/Converters/WeaviateDateTimeConverter.cs @@ -0,0 +1,43 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Globalization; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace CommunityToolkit.VectorData.Weaviate; + +/// +/// Converts to RFC 3339 formatted string for Weaviate. +/// +internal sealed class WeaviateDateTimeConverter : JsonConverter +{ + private const string DateTimeFormat = "yyyy-MM-dd'T'HH:mm:ss.fffK"; + + public override DateTime Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + var dateString = reader.GetString(); + + if (string.IsNullOrWhiteSpace(dateString)) + { + return default; + } + + // Parse as DateTimeOffset to properly handle timezone, then convert to UTC DateTime. + // Weaviate may return the timestamp in a different timezone than it was stored in. + return DateTimeOffset.Parse(dateString, CultureInfo.InvariantCulture).UtcDateTime; + } + + public override void Write(Utf8JsonWriter writer, DateTime value, JsonSerializerOptions options) + { + // When DateTime.Kind is Unspecified, the 'K' format specifier produces an empty string (no timezone), + // which violates RFC 3339. Treat Unspecified as UTC so 'K' produces 'Z'. + if (value.Kind == DateTimeKind.Unspecified) + { + value = DateTime.SpecifyKind(value, DateTimeKind.Utc); + } + + writer.WriteStringValue(value.ToString(DateTimeFormat, CultureInfo.InvariantCulture)); + } +} diff --git a/MEVD/src/Weaviate/Converters/WeaviateDateTimeOffsetConverter.cs b/MEVD/src/Weaviate/Converters/WeaviateDateTimeOffsetConverter.cs new file mode 100644 index 0000000..e65111b --- /dev/null +++ b/MEVD/src/Weaviate/Converters/WeaviateDateTimeOffsetConverter.cs @@ -0,0 +1,34 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Globalization; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace CommunityToolkit.VectorData.Weaviate; + +/// +/// Converts datetime type to RFC 3339 formatted string. +/// +internal sealed class WeaviateDateTimeOffsetConverter : JsonConverter +{ + private const string DateTimeFormat = "yyyy-MM-dd'T'HH:mm:ss.fffK"; + + public override DateTimeOffset Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + var dateString = reader.GetString(); + + if (string.IsNullOrWhiteSpace(dateString)) + { + return default; + } + + return DateTimeOffset.Parse(dateString, CultureInfo.InvariantCulture); + } + + public override void Write(Utf8JsonWriter writer, DateTimeOffset value, JsonSerializerOptions options) + { + writer.WriteStringValue(value.ToString(DateTimeFormat, CultureInfo.InvariantCulture)); + } +} diff --git a/MEVD/src/Weaviate/Converters/WeaviateNullableDateOnlyConverter.cs b/MEVD/src/Weaviate/Converters/WeaviateNullableDateOnlyConverter.cs new file mode 100644 index 0000000..3168960 --- /dev/null +++ b/MEVD/src/Weaviate/Converters/WeaviateNullableDateOnlyConverter.cs @@ -0,0 +1,48 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#if NET + +using System; +using System.Globalization; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace CommunityToolkit.VectorData.Weaviate; + +/// +/// Converts nullable to RFC 3339 formatted string for Weaviate. +/// DateOnly values are stored as midnight UTC. +/// +internal sealed class WeaviateNullableDateOnlyConverter : JsonConverter +{ + private const string DateTimeFormat = "yyyy-MM-dd'T'HH:mm:ss.fffZ"; + + public override DateOnly? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + var dateString = reader.GetString(); + + if (string.IsNullOrWhiteSpace(dateString)) + { + return null; + } + + var dateTimeOffset = DateTimeOffset.Parse(dateString, CultureInfo.InvariantCulture); + return DateOnly.FromDateTime(dateTimeOffset.DateTime); + } + + public override void Write(Utf8JsonWriter writer, DateOnly? value, JsonSerializerOptions options) + { + if (value.HasValue) + { + var dateTimeOffset = new DateTimeOffset(value.Value.ToDateTime(TimeOnly.MinValue), TimeSpan.Zero); + writer.WriteStringValue(dateTimeOffset.ToString(DateTimeFormat, CultureInfo.InvariantCulture)); + } + else + { + writer.WriteNullValue(); + } + } +} + +#endif diff --git a/MEVD/src/Weaviate/Converters/WeaviateNullableDateTimeConverter.cs b/MEVD/src/Weaviate/Converters/WeaviateNullableDateTimeConverter.cs new file mode 100644 index 0000000..35c637f --- /dev/null +++ b/MEVD/src/Weaviate/Converters/WeaviateNullableDateTimeConverter.cs @@ -0,0 +1,51 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Globalization; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace CommunityToolkit.VectorData.Weaviate; + +/// +/// Converts nullable to RFC 3339 formatted string for Weaviate. +/// +internal sealed class WeaviateNullableDateTimeConverter : JsonConverter +{ + private const string DateTimeFormat = "yyyy-MM-dd'T'HH:mm:ss.fffK"; + + public override DateTime? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + var dateString = reader.GetString(); + + if (string.IsNullOrWhiteSpace(dateString)) + { + return null; + } + + // Parse as DateTimeOffset to properly handle timezone, then convert to UTC DateTime. + // Weaviate may return the timestamp in a different timezone than it was stored in. + return DateTimeOffset.Parse(dateString, CultureInfo.InvariantCulture).UtcDateTime; + } + + public override void Write(Utf8JsonWriter writer, DateTime? value, JsonSerializerOptions options) + { + if (value.HasValue) + { + // When DateTime.Kind is Unspecified, the 'K' format specifier produces an empty string (no timezone), + // which violates RFC 3339. Treat Unspecified as UTC so 'K' produces 'Z'. + var v = value.Value; + if (v.Kind == DateTimeKind.Unspecified) + { + v = DateTime.SpecifyKind(v, DateTimeKind.Utc); + } + + writer.WriteStringValue(v.ToString(DateTimeFormat, CultureInfo.InvariantCulture)); + } + else + { + writer.WriteNullValue(); + } + } +} diff --git a/MEVD/src/Weaviate/Converters/WeaviateNullableDateTimeOffsetConverter.cs b/MEVD/src/Weaviate/Converters/WeaviateNullableDateTimeOffsetConverter.cs new file mode 100644 index 0000000..1b1673b --- /dev/null +++ b/MEVD/src/Weaviate/Converters/WeaviateNullableDateTimeOffsetConverter.cs @@ -0,0 +1,46 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Globalization; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace CommunityToolkit.VectorData.Weaviate; + +/// +/// Converts datetime type to RFC 3339 formatted string. +/// +internal sealed class WeaviateNullableDateTimeOffsetConverter : JsonConverter +{ + private const string DateTimeFormat = "yyyy-MM-dd'T'HH:mm:ss.fffK"; + + public override DateTimeOffset? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + if (reader.TokenType == JsonTokenType.Null) + { + return null; + } + + var dateString = reader.GetString(); + + if (string.IsNullOrWhiteSpace(dateString)) + { + return null; + } + + return DateTimeOffset.Parse(dateString, CultureInfo.InvariantCulture); + } + + public override void Write(Utf8JsonWriter writer, DateTimeOffset? value, JsonSerializerOptions options) + { + if (value.HasValue) + { + writer.WriteStringValue(value.Value.ToString(DateTimeFormat, CultureInfo.InvariantCulture)); + } + else + { + writer.WriteNullValue(); + } + } +} diff --git a/MEVD/src/Weaviate/Http/HttpRequest.cs b/MEVD/src/Weaviate/Http/HttpRequest.cs new file mode 100644 index 0000000..a861a39 --- /dev/null +++ b/MEVD/src/Weaviate/Http/HttpRequest.cs @@ -0,0 +1,61 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Net.Http; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace CommunityToolkit.VectorData.Weaviate; + +internal static class HttpRequest +{ + private static readonly JsonSerializerOptions s_jsonOptionsCache = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + }; + + public static HttpRequestMessage CreateGetRequest(string url, object? payload = null) + { + return new(HttpMethod.Get, url) + { + Content = GetJsonContent(payload) + }; + } + + public static HttpRequestMessage CreatePostRequest(string url, object? payload = null) + { + return new(HttpMethod.Post, url) + { + Content = GetJsonContent(payload) + }; + } + + public static HttpRequestMessage CreateDeleteRequest(string url, object? payload = null) + { + return new(HttpMethod.Delete, url) + { + Content = GetJsonContent(payload) + }; + } + + public static HttpRequestMessage CreatePutRequest(string url, object? payload = null) + { + return new(HttpMethod.Put, url) + { + Content = GetJsonContent(payload) + }; + } + + private static StringContent? GetJsonContent(object? payload) + { + if (payload is null) + { + return null; + } + + string strPayload = payload as string ?? JsonSerializer.Serialize(payload, s_jsonOptionsCache); + return new(strPayload, Encoding.UTF8, "application/json"); + } +} diff --git a/MEVD/src/Weaviate/HttpV2/WeaviateCreateCollectionSchemaRequest.cs b/MEVD/src/Weaviate/HttpV2/WeaviateCreateCollectionSchemaRequest.cs new file mode 100644 index 0000000..389df3c --- /dev/null +++ b/MEVD/src/Weaviate/HttpV2/WeaviateCreateCollectionSchemaRequest.cs @@ -0,0 +1,37 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Net.Http; +using System.Text.Json.Serialization; + +namespace CommunityToolkit.VectorData.Weaviate; + +internal sealed class WeaviateCreateCollectionSchemaRequest +{ + private const string ApiRoute = "schema"; + + [JsonConstructor] + public WeaviateCreateCollectionSchemaRequest() { } + + public WeaviateCreateCollectionSchemaRequest(WeaviateCollectionSchema collectionSchema) + { + CollectionName = collectionSchema.CollectionName; + VectorConfigurations = collectionSchema.VectorConfigurations; + Properties = collectionSchema.Properties; + } + + [JsonPropertyName("class")] + public string? CollectionName { get; set; } + + [JsonPropertyName("vectorConfig")] + public Dictionary? VectorConfigurations { get; set; } + + [JsonPropertyName("properties")] + public List? Properties { get; set; } + + public HttpRequestMessage Build() + { + return HttpRequest.CreatePostRequest(ApiRoute, this); + } +} diff --git a/MEVD/src/Weaviate/HttpV2/WeaviateDeleteCollectionSchemaRequest.cs b/MEVD/src/Weaviate/HttpV2/WeaviateDeleteCollectionSchemaRequest.cs new file mode 100644 index 0000000..b8e098f --- /dev/null +++ b/MEVD/src/Weaviate/HttpV2/WeaviateDeleteCollectionSchemaRequest.cs @@ -0,0 +1,20 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Net.Http; +using System.Text.Json.Serialization; + +namespace CommunityToolkit.VectorData.Weaviate; + +internal sealed class WeaviateDeleteCollectionSchemaRequest(string collectionName) +{ + private const string ApiRoute = "schema"; + + [JsonIgnore] + public string CollectionName { get; set; } = collectionName; + + public HttpRequestMessage Build() + { + return HttpRequest.CreateDeleteRequest($"{ApiRoute}/{CollectionName}"); + } +} diff --git a/MEVD/src/Weaviate/HttpV2/WeaviateDeleteObjectBatchRequest.cs b/MEVD/src/Weaviate/HttpV2/WeaviateDeleteObjectBatchRequest.cs new file mode 100644 index 0000000..512f1f1 --- /dev/null +++ b/MEVD/src/Weaviate/HttpV2/WeaviateDeleteObjectBatchRequest.cs @@ -0,0 +1,28 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Net.Http; +using System.Text.Json.Serialization; + +namespace CommunityToolkit.VectorData.Weaviate; + +internal sealed class WeaviateDeleteObjectBatchRequest +{ + private const string ApiRoute = "batch/objects"; + + [JsonConstructor] + public WeaviateDeleteObjectBatchRequest() { } + + public WeaviateDeleteObjectBatchRequest(WeaviateQueryMatch match) + { + Match = match; + } + + [JsonPropertyName("match")] + public WeaviateQueryMatch? Match { get; set; } + + public HttpRequestMessage Build() + { + return HttpRequest.CreateDeleteRequest(ApiRoute, this); + } +} diff --git a/MEVD/src/Weaviate/HttpV2/WeaviateDeleteObjectRequest.cs b/MEVD/src/Weaviate/HttpV2/WeaviateDeleteObjectRequest.cs new file mode 100644 index 0000000..7ed94c6 --- /dev/null +++ b/MEVD/src/Weaviate/HttpV2/WeaviateDeleteObjectRequest.cs @@ -0,0 +1,24 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Net.Http; +using System.Text.Json.Serialization; + +namespace CommunityToolkit.VectorData.Weaviate; + +internal sealed class WeaviateDeleteObjectRequest(string collectionName, Guid id) +{ + private const string ApiRoute = "objects"; + + [JsonIgnore] + public string CollectionName { get; set; } = collectionName; + + [JsonIgnore] + public Guid Id { get; set; } = id; + + public HttpRequestMessage Build() + { + return HttpRequest.CreateDeleteRequest($"{ApiRoute}/{CollectionName}/{Id}"); + } +} diff --git a/MEVD/src/Weaviate/HttpV2/WeaviateGetCollectionObjectRequest.cs b/MEVD/src/Weaviate/HttpV2/WeaviateGetCollectionObjectRequest.cs new file mode 100644 index 0000000..40ef74f --- /dev/null +++ b/MEVD/src/Weaviate/HttpV2/WeaviateGetCollectionObjectRequest.cs @@ -0,0 +1,36 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Net.Http; +using System.Text.Json.Serialization; + +namespace CommunityToolkit.VectorData.Weaviate; + +internal sealed class WeaviateGetCollectionObjectRequest(string collectionName, Guid id, bool includeVectors) +{ + private const string ApiRoute = "objects"; + private const string IncludeQueryParameterName = "include"; + private const string IncludeVectorQueryParameterValue = "vector"; + + [JsonIgnore] + public string CollectionName { get; set; } = collectionName; + + [JsonIgnore] + public Guid Id { get; set; } = id; + + [JsonIgnore] + public bool IncludeVectors { get; set; } = includeVectors; + + public HttpRequestMessage Build() + { + var uri = $"{ApiRoute}/{CollectionName}/{Id}"; + + if (IncludeVectors) + { + uri += $"?{IncludeQueryParameterName}={IncludeVectorQueryParameterValue}"; + } + + return HttpRequest.CreateGetRequest(uri); + } +} diff --git a/MEVD/src/Weaviate/HttpV2/WeaviateGetCollectionSchemaRequest.cs b/MEVD/src/Weaviate/HttpV2/WeaviateGetCollectionSchemaRequest.cs new file mode 100644 index 0000000..bdbab78 --- /dev/null +++ b/MEVD/src/Weaviate/HttpV2/WeaviateGetCollectionSchemaRequest.cs @@ -0,0 +1,20 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Net.Http; +using System.Text.Json.Serialization; + +namespace CommunityToolkit.VectorData.Weaviate; + +internal sealed class WeaviateGetCollectionSchemaRequest(string collectionName) +{ + private const string ApiRoute = "schema"; + + [JsonIgnore] + public string CollectionName { get; set; } = collectionName; + + public HttpRequestMessage Build() + { + return HttpRequest.CreateGetRequest($"{ApiRoute}/{CollectionName}"); + } +} diff --git a/MEVD/src/Weaviate/HttpV2/WeaviateGetCollectionSchemaResponse.cs b/MEVD/src/Weaviate/HttpV2/WeaviateGetCollectionSchemaResponse.cs new file mode 100644 index 0000000..3832e03 --- /dev/null +++ b/MEVD/src/Weaviate/HttpV2/WeaviateGetCollectionSchemaResponse.cs @@ -0,0 +1,12 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text.Json.Serialization; + +namespace CommunityToolkit.VectorData.Weaviate; + +internal sealed class WeaviateGetCollectionSchemaResponse +{ + [JsonPropertyName("class")] + public string? CollectionName { get; set; } +} diff --git a/MEVD/src/Weaviate/HttpV2/WeaviateGetCollectionsRequest.cs b/MEVD/src/Weaviate/HttpV2/WeaviateGetCollectionsRequest.cs new file mode 100644 index 0000000..6504923 --- /dev/null +++ b/MEVD/src/Weaviate/HttpV2/WeaviateGetCollectionsRequest.cs @@ -0,0 +1,16 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Net.Http; + +namespace CommunityToolkit.VectorData.Weaviate; + +internal sealed class WeaviateGetCollectionsRequest +{ + private const string ApiRoute = "schema"; + + public HttpRequestMessage Build() + { + return HttpRequest.CreateGetRequest(ApiRoute); + } +} diff --git a/MEVD/src/Weaviate/HttpV2/WeaviateGetCollectionsResponse.cs b/MEVD/src/Weaviate/HttpV2/WeaviateGetCollectionsResponse.cs new file mode 100644 index 0000000..56ef349 --- /dev/null +++ b/MEVD/src/Weaviate/HttpV2/WeaviateGetCollectionsResponse.cs @@ -0,0 +1,13 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace CommunityToolkit.VectorData.Weaviate; + +internal sealed class WeaviateGetCollectionsResponse +{ + [JsonPropertyName("classes")] + public List? Collections { get; set; } +} diff --git a/MEVD/src/Weaviate/HttpV2/WeaviateUpsertCollectionObjectBatchRequest.cs b/MEVD/src/Weaviate/HttpV2/WeaviateUpsertCollectionObjectBatchRequest.cs new file mode 100644 index 0000000..8d87dad --- /dev/null +++ b/MEVD/src/Weaviate/HttpV2/WeaviateUpsertCollectionObjectBatchRequest.cs @@ -0,0 +1,33 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Net.Http; +using System.Text.Json.Nodes; +using System.Text.Json.Serialization; + +namespace CommunityToolkit.VectorData.Weaviate; + +internal sealed class WeaviateUpsertCollectionObjectBatchRequest +{ + private const string ApiRoute = "batch/objects"; + + [JsonConstructor] + public WeaviateUpsertCollectionObjectBatchRequest() { } + + public WeaviateUpsertCollectionObjectBatchRequest(List collectionObjects) + { + CollectionObjects = collectionObjects; + } + + [JsonPropertyName("fields")] + public List Fields { get; set; } = [WeaviateConstants.ReservedKeyPropertyName]; + + [JsonPropertyName("objects")] + public List? CollectionObjects { get; set; } + + public HttpRequestMessage Build() + { + return HttpRequest.CreatePostRequest(ApiRoute, this); + } +} diff --git a/MEVD/src/Weaviate/HttpV2/WeaviateUpsertCollectionObjectBatchResponse.cs b/MEVD/src/Weaviate/HttpV2/WeaviateUpsertCollectionObjectBatchResponse.cs new file mode 100644 index 0000000..fc1f25d --- /dev/null +++ b/MEVD/src/Weaviate/HttpV2/WeaviateUpsertCollectionObjectBatchResponse.cs @@ -0,0 +1,16 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Text.Json.Serialization; + +namespace CommunityToolkit.VectorData.Weaviate; + +internal sealed class WeaviateUpsertCollectionObjectBatchResponse +{ + [JsonPropertyName("id")] + public Guid Id { get; set; } + + [JsonPropertyName("result")] + public WeaviateOperationResult? Result { get; set; } +} diff --git a/MEVD/src/Weaviate/HttpV2/WeaviateVectorSearchRequest.cs b/MEVD/src/Weaviate/HttpV2/WeaviateVectorSearchRequest.cs new file mode 100644 index 0000000..abaaf53 --- /dev/null +++ b/MEVD/src/Weaviate/HttpV2/WeaviateVectorSearchRequest.cs @@ -0,0 +1,24 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Net.Http; +using System.Text.Json.Serialization; + +namespace CommunityToolkit.VectorData.Weaviate; + +/// +/// Vector search request. +/// More information here: . +/// +internal sealed class WeaviateVectorSearchRequest(string query) +{ + private const string ApiRoute = "graphql"; + + [JsonPropertyName("query")] + public string Query { get; set; } = query; + + public HttpRequestMessage Build() + { + return HttpRequest.CreatePostRequest(ApiRoute, this); + } +} diff --git a/MEVD/src/Weaviate/HttpV2/WeaviateVectorSearchResponse.cs b/MEVD/src/Weaviate/HttpV2/WeaviateVectorSearchResponse.cs new file mode 100644 index 0000000..aab3992 --- /dev/null +++ b/MEVD/src/Weaviate/HttpV2/WeaviateVectorSearchResponse.cs @@ -0,0 +1,16 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text.Json.Serialization; + +namespace CommunityToolkit.VectorData.Weaviate; + +/// +/// Vector search response. +/// More information here: . +/// +internal sealed class WeaviateVectorSearchResponse +{ + [JsonPropertyName("data")] + public WeaviateVectorSearchData? Data { get; set; } +} diff --git a/MEVD/src/Weaviate/ModelV2/WeaviateCollectionSchema.cs b/MEVD/src/Weaviate/ModelV2/WeaviateCollectionSchema.cs new file mode 100644 index 0000000..efdc6f9 --- /dev/null +++ b/MEVD/src/Weaviate/ModelV2/WeaviateCollectionSchema.cs @@ -0,0 +1,34 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace CommunityToolkit.VectorData.Weaviate; + +internal sealed class WeaviateCollectionSchema +{ + [JsonConstructor] + public WeaviateCollectionSchema(string collectionName) + { + CollectionName = collectionName; + } + + [JsonPropertyName("class")] + public string CollectionName { get; set; } + + [JsonPropertyName("vectorConfig")] + public Dictionary VectorConfigurations { get; set; } = []; + + [JsonPropertyName("properties")] + public List Properties { get; set; } = []; + + [JsonPropertyName("vectorizer")] + public string Vectorizer { get; set; } = WeaviateConstants.DefaultVectorizer; + + [JsonPropertyName("vectorIndexType")] + public string? VectorIndexType { get; set; } + + [JsonPropertyName("vectorIndexConfig")] + public WeaviateCollectionSchemaVectorIndexConfig? VectorIndexConfig { get; set; } +} diff --git a/MEVD/src/Weaviate/ModelV2/WeaviateCollectionSchemaProperty.cs b/MEVD/src/Weaviate/ModelV2/WeaviateCollectionSchemaProperty.cs new file mode 100644 index 0000000..d14affb --- /dev/null +++ b/MEVD/src/Weaviate/ModelV2/WeaviateCollectionSchemaProperty.cs @@ -0,0 +1,22 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace CommunityToolkit.VectorData.Weaviate; + +internal sealed class WeaviateCollectionSchemaProperty +{ + [JsonPropertyName("name")] + public string? Name { get; set; } + + [JsonPropertyName("dataType")] + public List DataType { get; set; } = []; + + [JsonPropertyName("indexFilterable")] + public bool IndexFilterable { get; set; } + + [JsonPropertyName("indexSearchable")] + public bool IndexSearchable { get; set; } +} diff --git a/MEVD/src/Weaviate/ModelV2/WeaviateCollectionSchemaVectorConfig.cs b/MEVD/src/Weaviate/ModelV2/WeaviateCollectionSchemaVectorConfig.cs new file mode 100644 index 0000000..ba6a845 --- /dev/null +++ b/MEVD/src/Weaviate/ModelV2/WeaviateCollectionSchemaVectorConfig.cs @@ -0,0 +1,19 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace CommunityToolkit.VectorData.Weaviate; + +internal sealed class WeaviateCollectionSchemaVectorConfig +{ + [JsonPropertyName("vectorizer")] + public Dictionary Vectorizer { get; set; } = new() { [WeaviateConstants.DefaultVectorizer] = null }; + + [JsonPropertyName("vectorIndexType")] + public string? VectorIndexType { get; set; } + + [JsonPropertyName("vectorIndexConfig")] + public WeaviateCollectionSchemaVectorIndexConfig? VectorIndexConfig { get; set; } +} diff --git a/MEVD/src/Weaviate/ModelV2/WeaviateCollectionSchemaVectorIndexConfig.cs b/MEVD/src/Weaviate/ModelV2/WeaviateCollectionSchemaVectorIndexConfig.cs new file mode 100644 index 0000000..69ad6c0 --- /dev/null +++ b/MEVD/src/Weaviate/ModelV2/WeaviateCollectionSchemaVectorIndexConfig.cs @@ -0,0 +1,12 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text.Json.Serialization; + +namespace CommunityToolkit.VectorData.Weaviate; + +internal sealed class WeaviateCollectionSchemaVectorIndexConfig +{ + [JsonPropertyName("distance")] + public string? Distance { get; set; } +} diff --git a/MEVD/src/Weaviate/ModelV2/WeaviateOperationResult.cs b/MEVD/src/Weaviate/ModelV2/WeaviateOperationResult.cs new file mode 100644 index 0000000..5e3e367 --- /dev/null +++ b/MEVD/src/Weaviate/ModelV2/WeaviateOperationResult.cs @@ -0,0 +1,21 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Text.Json.Serialization; + +namespace CommunityToolkit.VectorData.Weaviate; + +internal sealed class WeaviateOperationResult +{ + private const string Success = nameof(Success); + + [JsonPropertyName("errors")] + public WeaviateOperationResultErrors? Errors { get; set; } + + [JsonPropertyName("status")] + public string? Status { get; set; } + + [JsonIgnore] + public bool? IsSuccess => Status?.Equals(Success, StringComparison.OrdinalIgnoreCase); +} diff --git a/MEVD/src/Weaviate/ModelV2/WeaviateOperationResultError.cs b/MEVD/src/Weaviate/ModelV2/WeaviateOperationResultError.cs new file mode 100644 index 0000000..0ce3bb6 --- /dev/null +++ b/MEVD/src/Weaviate/ModelV2/WeaviateOperationResultError.cs @@ -0,0 +1,12 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text.Json.Serialization; + +namespace CommunityToolkit.VectorData.Weaviate; + +internal sealed class WeaviateOperationResultError +{ + [JsonPropertyName("message")] + public string? Message { get; set; } +} diff --git a/MEVD/src/Weaviate/ModelV2/WeaviateOperationResultErrors.cs b/MEVD/src/Weaviate/ModelV2/WeaviateOperationResultErrors.cs new file mode 100644 index 0000000..4d1211f --- /dev/null +++ b/MEVD/src/Weaviate/ModelV2/WeaviateOperationResultErrors.cs @@ -0,0 +1,13 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace CommunityToolkit.VectorData.Weaviate; + +internal class WeaviateOperationResultErrors +{ + [JsonPropertyName("error")] + public List? Errors { get; set; } +} diff --git a/MEVD/src/Weaviate/ModelV2/WeaviateQueryMatch.cs b/MEVD/src/Weaviate/ModelV2/WeaviateQueryMatch.cs new file mode 100644 index 0000000..63224fe --- /dev/null +++ b/MEVD/src/Weaviate/ModelV2/WeaviateQueryMatch.cs @@ -0,0 +1,15 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text.Json.Serialization; + +namespace CommunityToolkit.VectorData.Weaviate; + +internal sealed class WeaviateQueryMatch +{ + [JsonPropertyName("class")] + public string? CollectionName { get; set; } + + [JsonPropertyName("where")] + public WeaviateQueryMatchWhereClause? WhereClause { get; set; } +} diff --git a/MEVD/src/Weaviate/ModelV2/WeaviateQueryMatchWhereClause.cs b/MEVD/src/Weaviate/ModelV2/WeaviateQueryMatchWhereClause.cs new file mode 100644 index 0000000..a50075b --- /dev/null +++ b/MEVD/src/Weaviate/ModelV2/WeaviateQueryMatchWhereClause.cs @@ -0,0 +1,19 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace CommunityToolkit.VectorData.Weaviate; + +internal sealed class WeaviateQueryMatchWhereClause +{ + [JsonPropertyName("operator")] + public string? Operator { get; set; } + + [JsonPropertyName("path")] + public List Path { get; set; } = []; + + [JsonPropertyName("valueTextArray")] + public List Values { get; set; } = []; +} diff --git a/MEVD/src/Weaviate/ModelV2/WeaviateVectorSearchData.cs b/MEVD/src/Weaviate/ModelV2/WeaviateVectorSearchData.cs new file mode 100644 index 0000000..2e0486f --- /dev/null +++ b/MEVD/src/Weaviate/ModelV2/WeaviateVectorSearchData.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Text.Json.Nodes; +using System.Text.Json.Serialization; + +namespace CommunityToolkit.VectorData.Weaviate; + +/// +/// Vector search data model. +/// More information here: . +/// +internal sealed class WeaviateVectorSearchData +{ + [JsonPropertyName("Get")] + public Dictionary? GetOperation { get; set; } +} diff --git a/MEVD/src/Weaviate/Weaviate.csproj b/MEVD/src/Weaviate/Weaviate.csproj new file mode 100644 index 0000000..1aedd80 --- /dev/null +++ b/MEVD/src/Weaviate/Weaviate.csproj @@ -0,0 +1,30 @@ + + + + + 1.0.0-preview.1 + CommunityToolkit.VectorData.Weaviate + $(AssemblyName) + net10.0;net8.0;netstandard2.0;net462 + + + Weaviate provider for Microsoft.Extensions.VectorData + Weaviate provider for Microsoft.Extensions.VectorData by Semantic Kernel + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/MEVD/src/Weaviate/WeaviateCollection.cs b/MEVD/src/Weaviate/WeaviateCollection.cs new file mode 100644 index 0000000..4599cd4 --- /dev/null +++ b/MEVD/src/Weaviate/WeaviateCollection.cs @@ -0,0 +1,589 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Linq.Expressions; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Runtime.CompilerServices; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; +using Microsoft.Shared.Diagnostics; + +namespace CommunityToolkit.VectorData.Weaviate; + +/// +/// Service for storing and retrieving vector records, that uses Weaviate as the underlying storage. +/// +/// The data type of the record key. Must be . +/// The data model to use for adding, updating and retrieving data from storage. +#pragma warning disable CA1711 // Identifiers should not have incorrect suffix +public class WeaviateCollection : VectorStoreCollection, IKeywordHybridSearchable + where TKey : notnull + where TRecord : class +#pragma warning restore CA1711 // Identifiers should not have incorrect suffix +{ + /// Metadata about vector store record collection. + private readonly VectorStoreCollectionMetadata _collectionMetadata; + + /// The default options for vector search. + private static readonly VectorSearchOptions s_defaultVectorSearchOptions = new(); + + /// The default options for hybrid vector search. + private static readonly HybridSearchOptions s_defaultKeywordVectorizedHybridSearchOptions = new(); + + /// that is used to interact with Weaviate API. + private readonly HttpClient _httpClient; + + /// The model for this collection. + private readonly CollectionModel _model; + + /// The mapper to use when mapping between the consumer data model and the Weaviate record. + private readonly WeaviateMapper _mapper; + + /// Weaviate endpoint. + private readonly Uri _endpoint; + + /// Weaviate API key. + private readonly string? _apiKey; + + /// + public override string Name { get; } + + /// Whether the vectors in the store are named and multiple vectors are supported, or whether there is just a single unnamed vector in Weaviate collection. + private readonly bool _hasNamedVectors; + + /// + /// Initializes a new instance of the class. + /// + /// + /// that is used to interact with Weaviate API. + /// should point to remote or local cluster and API key can be configured via . + /// It's also possible to provide these parameters via . + /// + /// The name of the collection that this will access. + /// Optional configuration options for this class. + /// The collection name must start with a capital letter and contain only ASCII letters and digits. + [RequiresUnreferencedCode("The Weaviate provider is currently incompatible with trimming.")] + [RequiresDynamicCode("The Weaviate provider is currently incompatible with NativeAOT.")] + public WeaviateCollection( + HttpClient httpClient, + string name, + WeaviateCollectionOptions? options = default) + : this( + httpClient, + name, + static options => typeof(TRecord) == typeof(Dictionary) + ? throw new NotSupportedException(VectorDataStrings.NonDynamicCollectionWithDictionaryNotSupported(typeof(WeaviateDynamicCollection))) + : new WeaviateModelBuilder(options.HasNamedVectors) + .Build(typeof(TRecord), typeof(TKey), options.Definition, options.EmbeddingGenerator, WeaviateConstants.s_jsonSerializerOptions), + options) + { + } + + internal WeaviateCollection(HttpClient httpClient, string name, Func modelFactory, WeaviateCollectionOptions? options) + { + // Verify. + Throw.IfNull(httpClient); + VerifyCollectionName(name); + + if (typeof(TKey) != typeof(Guid) && typeof(TKey) != typeof(object)) + { + throw new NotSupportedException($"Only {nameof(Guid)} key is supported."); + } + + var endpoint = (options?.Endpoint ?? httpClient.BaseAddress) ?? throw new ArgumentException($"Weaviate endpoint should be provided via HttpClient.BaseAddress property or {nameof(WeaviateCollectionOptions)} options parameter."); + + options ??= WeaviateCollectionOptions.Default; + + // Assign. + _httpClient = httpClient; + _endpoint = endpoint; + Name = name; + _model = modelFactory(options); + _apiKey = options.ApiKey; + _hasNamedVectors = options.HasNamedVectors; + + // Assign mapper. + _mapper = new WeaviateMapper(Name, options.HasNamedVectors, _model, WeaviateConstants.s_jsonSerializerOptions); + + _collectionMetadata = new() + { + VectorStoreSystemName = WeaviateConstants.VectorStoreSystemName, + CollectionName = name + }; + } + + /// + public override async Task CollectionExistsAsync(CancellationToken cancellationToken = default) + { + using var request = new WeaviateGetCollectionSchemaRequest(Name).Build(); + + var response = await + ExecuteRequestWithNotFoundHandlingAsync(request, cancellationToken) + .ConfigureAwait(false); + + return response != null; + } + + /// + public override async Task EnsureCollectionExistsAsync(CancellationToken cancellationToken = default) + { + // Don't even try to create if the collection already exists. + if (await CollectionExistsAsync(cancellationToken).ConfigureAwait(false)) + { + return; + } + + var schema = WeaviateCollectionCreateMapping.MapToSchema( + Name, + _hasNamedVectors, + _model); + + using var request = new WeaviateCreateCollectionSchemaRequest(schema).Build(); + + try + { + await ExecuteRequestAsync(request, cancellationToken: cancellationToken).ConfigureAwait(false); + } + catch (VectorStoreException) + { + // Since weaviate error info is ambiguous, we can check here if the index already exists. + // If it does, we can ignore the error. +#pragma warning disable CA1031 // Do not catch general exception types + try + { + if (await CollectionExistsAsync(cancellationToken).ConfigureAwait(false)) + { + return; + } + } + catch + { + } +#pragma warning restore CA1031 // Do not catch general exception types + + throw; + } + } + + /// + public override async Task EnsureCollectionDeletedAsync(CancellationToken cancellationToken = default) + { + using var request = new WeaviateDeleteCollectionSchemaRequest(Name).Build(); + + await ExecuteRequestAsync(request, cancellationToken: cancellationToken).ConfigureAwait(false); + } + + /// + public override async Task DeleteAsync(TKey key, CancellationToken cancellationToken = default) + { + var guid = key switch + { + Guid g => g, + object o => (Guid)o, + _ => throw new UnreachableException("Guid key should have been validated during model building") + }; + + using var request = new WeaviateDeleteObjectRequest(Name, guid).Build(); + + await ExecuteRequestAsync(request, cancellationToken: cancellationToken).ConfigureAwait(false); + } + + /// + public override async Task DeleteAsync(IEnumerable keys, CancellationToken cancellationToken = default) + { + const string ContainsAnyOperator = "ContainsAny"; + + Throw.IfNull(keys); + + var stringKeys = keys.Select(key => key.ToString()).ToList(); + + if (stringKeys.Count == 0) + { + return; + } + + var match = new WeaviateQueryMatch + { + CollectionName = Name, + WhereClause = new WeaviateQueryMatchWhereClause + { + Operator = ContainsAnyOperator, + Path = [WeaviateConstants.ReservedKeyPropertyName], + Values = stringKeys! + } + }; + + using var request = new WeaviateDeleteObjectBatchRequest(match).Build(); + await ExecuteRequestAsync(request, cancellationToken: cancellationToken).ConfigureAwait(false); + } + + /// + public override async Task GetAsync(TKey key, RecordRetrievalOptions? options = null, CancellationToken cancellationToken = default) + { + var guid = key as Guid? ?? throw new InvalidCastException("Only Guid keys are supported"); + var includeVectors = options?.IncludeVectors is true; + if (includeVectors && _model.EmbeddingGenerationRequired) + { + throw new NotSupportedException(VectorDataStrings.IncludeVectorsNotSupportedWithEmbeddingGeneration); + } + + using var request = new WeaviateGetCollectionObjectRequest(Name, guid, includeVectors).Build(); + + var jsonObject = await ExecuteRequestWithNotFoundHandlingAsync(request, cancellationToken).ConfigureAwait(false); + + if (jsonObject is null) + { + return default; + } + + return _mapper.MapFromStorageToDataModel(jsonObject!, includeVectors); + } + + /// + public override Task UpsertAsync(TRecord record, CancellationToken cancellationToken = default) + => UpsertAsync([record], cancellationToken); + + /// + public override async Task UpsertAsync(IEnumerable records, CancellationToken cancellationToken = default) + { + Throw.IfNull(records); + + IReadOnlyList? recordsList = null; + + // If an embedding generator is defined, invoke it once per property for all records. + IReadOnlyList?[]? generatedEmbeddings = null; + + var vectorPropertyCount = _model.VectorProperties.Count; + for (var i = 0; i < vectorPropertyCount; i++) + { + var vectorProperty = _model.VectorProperties[i]; + + if (WeaviateModelBuilder.IsVectorPropertyTypeValidCore(vectorProperty.Type, out _)) + { + continue; + } + + // We have a vector property whose type isn't natively supported - we need to generate embeddings. + Debug.Assert(vectorProperty.EmbeddingGenerator is not null); + + // We have a property with embedding generation; materialize the records' enumerable if needed, to + // prevent multiple enumeration. + if (recordsList is null) + { + recordsList = records is IReadOnlyList r ? r : records.ToList(); + + if (recordsList.Count == 0) + { + return; + } + + records = recordsList; + } + + // TODO: Ideally we'd group together vector properties using the same generator (and with the same input and output properties), + // and generate embeddings for them in a single batch. That's some more complexity though. + generatedEmbeddings ??= new IReadOnlyList?[vectorPropertyCount]; + generatedEmbeddings[i] = await vectorProperty.GenerateEmbeddingsAsync(records.Select(r => vectorProperty.GetValueAsObject(r)), cancellationToken).ConfigureAwait(false); + } + + var keyProperty = _model.KeyProperty; + var jsonObjects = new List(); + var recordIndex = 0; + foreach (var record in records) + { + if (keyProperty.IsAutoGenerated && keyProperty.GetValue(record) == Guid.Empty) + { + keyProperty.SetValue(record, Guid.NewGuid()); + } + + jsonObjects.Add(_mapper.MapFromDataToStorageModel(record, recordIndex++, generatedEmbeddings)); + } + + if (jsonObjects.Count == 0) + { + return; + } + + using var request = new WeaviateUpsertCollectionObjectBatchRequest(jsonObjects).Build(); + + await ExecuteRequestAsync>(request, cancellationToken).ConfigureAwait(false); + } + + #region Search + + /// + public override async IAsyncEnumerable> SearchAsync( + TInput searchValue, + int top, + VectorSearchOptions? options = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + Throw.IfNull(searchValue); + Throw.IfLessThan(top, 1); + + options ??= s_defaultVectorSearchOptions; + if (options.IncludeVectors && _model.EmbeddingGenerationRequired) + { + throw new NotSupportedException(VectorDataStrings.IncludeVectorsNotSupportedWithEmbeddingGeneration); + } + + var vectorProperty = _model.GetVectorPropertyOrSingle(options); + var vector = await GetSearchVectorAsync(searchValue, vectorProperty, cancellationToken).ConfigureAwait(false); + + var query = WeaviateQueryBuilder.BuildSearchQuery( + vector, + Name, + vectorProperty.StorageName, + WeaviateConstants.s_jsonSerializerOptions, + top, + options, + _model, + _hasNamedVectors); + + await foreach (var record in ExecuteQueryAsync(query, options.IncludeVectors, WeaviateConstants.ScorePropertyName, operationName: "VectorSearch", cancellationToken).ConfigureAwait(false)) + { + yield return record; + } + } + + private static async ValueTask> GetSearchVectorAsync(TInput searchValue, VectorPropertyModel vectorProperty, CancellationToken cancellationToken) + where TInput : notnull + => searchValue switch + { + ReadOnlyMemory r => r, + float[] f => new ReadOnlyMemory(f), + Embedding e => e.Vector, + _ when vectorProperty.EmbeddingGenerationDispatcher is not null + => ((Embedding)await vectorProperty.GenerateEmbeddingAsync(searchValue, cancellationToken).ConfigureAwait(false)).Vector, + + _ => vectorProperty.EmbeddingGenerator is null + ? throw new NotSupportedException(VectorDataStrings.InvalidSearchInputAndNoEmbeddingGeneratorWasConfigured(searchValue.GetType(), WeaviateModelBuilder.SupportedVectorTypes)) + : throw new InvalidOperationException(VectorDataStrings.IncompatibleEmbeddingGeneratorWasConfiguredForInputType(typeof(TInput), vectorProperty.EmbeddingGenerator.GetType())) + }; + + #endregion Search + + /// + public override IAsyncEnumerable GetAsync(Expression> filter, int top, + FilteredRecordRetrievalOptions? options = null, CancellationToken cancellationToken = default) + { + Throw.IfNull(filter); + Throw.IfLessThan(top, 1); + + options ??= new(); + + var query = WeaviateQueryBuilder.BuildQuery( + filter, + top, + options, + Name, + _model, + _hasNamedVectors); + + return ExecuteQueryAsync(query, options.IncludeVectors, WeaviateConstants.ScorePropertyName, "GetAsync", cancellationToken) + .Select(result => result.Record); + } + + /// + public async IAsyncEnumerable> HybridSearchAsync( + TInput searchValue, + ICollection keywords, + int top, + HybridSearchOptions? options = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + where TInput : notnull + { + const string OperationName = "HybridSearch"; + + Throw.IfLessThan(top, 1); + + options ??= s_defaultKeywordVectorizedHybridSearchOptions; + var vectorProperty = _model.GetVectorPropertyOrSingle(new() { VectorProperty = options.VectorProperty }); + var vector = await GetSearchVectorAsync(searchValue, vectorProperty, cancellationToken).ConfigureAwait(false); + var textDataProperty = _model.GetFullTextDataPropertyOrSingle(options.AdditionalProperty); + + var query = WeaviateQueryBuilder.BuildHybridSearchQuery( + vector, + top, + string.Join(" ", keywords), + Name, + _model, + vectorProperty, + textDataProperty, + WeaviateConstants.s_jsonSerializerOptions, + options, + _hasNamedVectors); + + await foreach (var record in ExecuteQueryAsync(query, options.IncludeVectors, WeaviateConstants.HybridScorePropertyName, OperationName, cancellationToken).ConfigureAwait(false)) + { + // Note that unlike regular vector search, Weaviate hybrid search does not support 'certainty' (filtering via score threshold). + // So we filter client-side here instead. + if (options.ScoreThreshold.HasValue && record.Score < options.ScoreThreshold.Value) + { + continue; + } + + yield return record; + } + } + + /// + public override object? GetService(Type serviceType, object? serviceKey = null) + { + Throw.IfNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreCollectionMetadata) ? _collectionMetadata : + serviceType == typeof(HttpClient) ? _httpClient : + serviceType.IsInstanceOfType(this) ? this : + null; + } + + #region private + + private async IAsyncEnumerable> ExecuteQueryAsync(string query, bool includeVectors, string scorePropertyName, string operationName, [EnumeratorCancellation] CancellationToken cancellationToken) + { + using var request = new WeaviateVectorSearchRequest(query).Build(); + + var (responseModel, content) = await ExecuteRequestWithResponseContentAsync(request, cancellationToken).ConfigureAwait(false); + + var collectionResults = responseModel?.Data?.GetOperation?[Name]; + + if (collectionResults is null) + { + throw new VectorStoreException($"Error occurred during vector search. Response: {content}") + { + VectorStoreSystemName = WeaviateConstants.VectorStoreSystemName, + VectorStoreName = _collectionMetadata.VectorStoreName, + CollectionName = Name, + OperationName = operationName + }; + } + + foreach (var result in collectionResults) + { + if (result is not null) + { + var (storageModel, score) = WeaviateCollectionSearchMapping.MapSearchResult(result, scorePropertyName, _hasNamedVectors); + + var record = _mapper.MapFromStorageToDataModel(storageModel, includeVectors); + + yield return new VectorSearchResult(record, score); + } + } + } + + private Task ExecuteRequestAsync( + HttpRequestMessage request, + bool ensureSuccessStatusCode = true, + CancellationToken cancellationToken = default) + { + request.RequestUri = new Uri(_endpoint, request.RequestUri!); + + if (!string.IsNullOrWhiteSpace(_apiKey)) + { + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _apiKey); + } + + return VectorStoreErrorHandler.RunOperationAsync( + _collectionMetadata, + $"{request.Method} {request.RequestUri}", + async () => + { + var response = await _httpClient + .SendAsync(request, HttpCompletionOption.ResponseContentRead, cancellationToken) + .ConfigureAwait(false); + + if (ensureSuccessStatusCode) + { + response.EnsureSuccessStatusCode(); + } + + return response; + }); + } + + private async Task<(TResponse?, string)> ExecuteRequestWithResponseContentAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + var response = await ExecuteRequestAsync(request, ensureSuccessStatusCode: true, cancellationToken: cancellationToken).ConfigureAwait(false); + + var responseContent = await response.Content.ReadAsStringAsync( +#if NET + cancellationToken +#endif + ).ConfigureAwait(false); + + var responseModel = VectorStoreErrorHandler.RunOperation( + _collectionMetadata, + $"{request.Method} {request.RequestUri}", + () => JsonSerializer.Deserialize(responseContent, WeaviateConstants.s_jsonSerializerOptions)); + + return (responseModel, responseContent); + } + + private async Task ExecuteRequestAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + var (model, _) = await ExecuteRequestWithResponseContentAsync(request, cancellationToken).ConfigureAwait(false); + + return model; + } + + private async Task ExecuteRequestWithNotFoundHandlingAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + var response = await ExecuteRequestAsync(request, ensureSuccessStatusCode: false, cancellationToken: cancellationToken).ConfigureAwait(false); + + if (response.StatusCode == HttpStatusCode.NotFound) + { + return default; + } + + response.EnsureSuccessStatusCode(); + + var responseContent = await response.Content.ReadAsStringAsync( +#if NET + cancellationToken +#endif + ).ConfigureAwait(false); + + var responseModel = VectorStoreErrorHandler.RunOperation( + _collectionMetadata, + $"{request.Method} {request.RequestUri}", + () => JsonSerializer.Deserialize(responseContent, WeaviateConstants.s_jsonSerializerOptions)); + + return responseModel; + } + + private static void VerifyCollectionName(string collectionName) + { + Throw.IfNullOrWhitespace(collectionName); + + // Based on https://weaviate.io/developers/weaviate/starter-guides/managing-collections#collection--property-names + char first = collectionName[0]; + if (first is not (>= 'A' and <= 'Z')) + { + throw new ArgumentException("Collection name must start with an uppercase ASCII letter.", nameof(collectionName)); + } + + foreach (char character in collectionName) + { + if (character is not (>= 'a' and <= 'z' or >= 'A' and <= 'Z' or >= '0' and <= '9' or '_')) + { + throw new ArgumentException("Collection name must contain only ASCII letters and digits or underscores. The first character must be an upper case letter.", nameof(collectionName)); + } + } + } + #endregion +} diff --git a/MEVD/src/Weaviate/WeaviateCollectionCreateMapping.cs b/MEVD/src/Weaviate/WeaviateCollectionCreateMapping.cs new file mode 100644 index 0000000..e6038d4 --- /dev/null +++ b/MEVD/src/Weaviate/WeaviateCollectionCreateMapping.cs @@ -0,0 +1,173 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; + +namespace CommunityToolkit.VectorData.Weaviate; + +/// +/// Class to construct Weaviate collection schema with configuration for data and vector properties. +/// More information here: . +/// +internal static class WeaviateCollectionCreateMapping +{ + /// + /// Maps record type properties to Weaviate collection schema for collection creation. + /// + /// The name of the vector store collection. + /// Gets a value indicating whether the vectors in the store are named and multiple vectors are supported, or whether there is just a single unnamed vector in Weaviate collection. + /// The model. + /// Weaviate collection schema. + public static WeaviateCollectionSchema MapToSchema(string collectionName, bool hasNamedVectors, CollectionModel model) + { + var schema = new WeaviateCollectionSchema(collectionName); + + // Handle data properties. + foreach (var property in model.DataProperties) + { + schema.Properties.Add(new WeaviateCollectionSchemaProperty + { + Name = property.StorageName, + DataType = [MapType(property.Type)], + IndexFilterable = property.IsIndexed, + IndexSearchable = property.IsFullTextIndexed + }); + } + + // Handle vector properties. + if (hasNamedVectors) + { + foreach (var property in model.VectorProperties) + { + schema.VectorConfigurations.Add(property.StorageName, new WeaviateCollectionSchemaVectorConfig + { + VectorIndexType = MapIndexKind(property.IndexKind, property.StorageName), + VectorIndexConfig = new WeaviateCollectionSchemaVectorIndexConfig + { + Distance = MapDistanceFunction(property.DistanceFunction, property.StorageName) + } + }); + } + } + else + { + var vectorProperty = model.VectorProperty; + schema.VectorIndexType = MapIndexKind(vectorProperty.IndexKind, vectorProperty.StorageName); + schema.VectorIndexConfig = new WeaviateCollectionSchemaVectorIndexConfig + { + Distance = MapDistanceFunction(vectorProperty.DistanceFunction, vectorProperty.StorageName) + }; + } + + return schema; + } + + #region private + + /// + /// Maps record vector property index kind to Weaviate index kind. + /// More information here: . + /// + private static string MapIndexKind(string? indexKind, string vectorPropertyName) + { + const string Hnsw = "hnsw"; + const string Flat = "flat"; + const string Dynamic = "dynamic"; + + // If index kind is not provided, use default one. + if (string.IsNullOrWhiteSpace(indexKind)) + { + return Hnsw; + } + + return indexKind switch + { + IndexKind.Hnsw => Hnsw, + IndexKind.Flat => Flat, + IndexKind.Dynamic => Dynamic, + _ => throw new InvalidOperationException( + $"Index kind '{indexKind}' on {nameof(VectorStoreVectorProperty)} '{vectorPropertyName}' is not supported by the Weaviate VectorStore. " + + $"Supported index kinds: {string.Join(", ", + IndexKind.Hnsw, + IndexKind.Flat, + IndexKind.Dynamic)}") + }; + } + + /// + /// Maps record vector property distance function to Weaviate distance function. + /// More information here: . + /// + private static string MapDistanceFunction(string? distanceFunction, string vectorPropertyName) + { + const string Cosine = "cosine"; + const string Dot = "dot"; + const string EuclideanSquared = "l2-squared"; + const string Hamming = "hamming"; + const string Manhattan = "manhattan"; + + // If distance function is not provided, use default one. + if (string.IsNullOrWhiteSpace(distanceFunction)) + { + return Cosine; + } + + return distanceFunction switch + { + DistanceFunction.CosineDistance => Cosine, + DistanceFunction.NegativeDotProductSimilarity => Dot, + DistanceFunction.EuclideanSquaredDistance => EuclideanSquared, + DistanceFunction.HammingDistance => Hamming, + DistanceFunction.ManhattanDistance => Manhattan, + _ => throw new NotSupportedException( + $"Distance function '{distanceFunction}' on {nameof(VectorStoreVectorProperty)} '{vectorPropertyName}' is not supported by the Weaviate VectorStore. " + + $"Supported distance functions: {string.Join(", ", + DistanceFunction.CosineDistance, + DistanceFunction.NegativeDotProductSimilarity, + DistanceFunction.EuclideanSquaredDistance, + DistanceFunction.HammingDistance, + DistanceFunction.ManhattanDistance)}") + }; + } + + /// + /// Maps record property type to Weaviate data type taking into account if the type is a collection or single value. + /// + private static string MapType(Type type) + { + return type switch + { + var t when TryMapType(type, out var mappedType) => mappedType, + var t when type.IsArray && TryMapType(type.GetElementType()!, out var mappedType) => mappedType + "[]", + var t when type.IsGenericType && type.GetGenericTypeDefinition() == typeof(List<>) + && TryMapType(type.GenericTypeArguments[0], out var mappedType) => mappedType + "[]", + _ => throw new NotSupportedException($"Type '{type.Name}' is not supported by Weaviate.") + }; + + bool TryMapType(Type type, [NotNullWhen(true)] out string? mappedType) + { + mappedType = (Nullable.GetUnderlyingType(type) ?? type) switch + { + Type t when t == typeof(string) => "text", + Type t when t == typeof(int) || t == typeof(long) || t == typeof(short) || t == typeof(byte) => "int", + Type t when t == typeof(float) || t == typeof(double) || t == typeof(decimal) => "number", + Type t when t == typeof(DateTime) || t == typeof(DateTimeOffset) => "date", +#if NET + Type t when t == typeof(DateOnly) => "date", +#endif + Type t when t == typeof(Guid) => "uuid", + Type t when t == typeof(bool) => "boolean", + + _ => null + }; + + return mappedType is not null; + } + } + + #endregion +} diff --git a/MEVD/src/Weaviate/WeaviateCollectionOptions.cs b/MEVD/src/Weaviate/WeaviateCollectionOptions.cs new file mode 100644 index 0000000..259606f --- /dev/null +++ b/MEVD/src/Weaviate/WeaviateCollectionOptions.cs @@ -0,0 +1,49 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using Microsoft.Extensions.VectorData; + +namespace CommunityToolkit.VectorData.Weaviate; + +/// +/// Options when creating a . +/// +public sealed class WeaviateCollectionOptions : VectorStoreCollectionOptions +{ + internal static readonly WeaviateCollectionOptions Default = new(); + + /// + /// Initializes a new instance of the class. + /// + public WeaviateCollectionOptions() + { + } + + internal WeaviateCollectionOptions(WeaviateCollectionOptions? source) : base(source) + { + Endpoint = source?.Endpoint; + ApiKey = source?.ApiKey; + HasNamedVectors = source?.HasNamedVectors ?? Default.HasNamedVectors; + } + + /// + /// Weaviate endpoint for remote or local cluster. + /// + public Uri? Endpoint { get; set; } + + /// + /// Weaviate API key. + /// + /// + /// This parameter is optional because authentication may be disabled in local clusters for testing purposes. + /// + public string? ApiKey { get; set; } + + /// + /// Gets or sets a value indicating whether the vectors in the store are named and multiple vectors are supported, or whether there is just a single unnamed vector in Weaviate collection. + /// Defaults to multiple named vectors. + /// . + /// + public bool HasNamedVectors { get; set; } = true; +} diff --git a/MEVD/src/Weaviate/WeaviateCollectionSearchMapping.cs b/MEVD/src/Weaviate/WeaviateCollectionSearchMapping.cs new file mode 100644 index 0000000..abe9a9c --- /dev/null +++ b/MEVD/src/Weaviate/WeaviateCollectionSearchMapping.cs @@ -0,0 +1,48 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text.Json; +using System.Text.Json.Nodes; + +namespace CommunityToolkit.VectorData.Weaviate; + +/// +/// Contains methods to perform Weaviate vector search data mapping. +/// +internal static class WeaviateCollectionSearchMapping +{ + /// + /// Maps vector search result to the format, which is processable by . + /// + public static (JsonObject StorageModel, double? Score) MapSearchResult( + JsonNode result, + string scorePropertyName, + bool hasNamedVectors) + { + var additionalProperties = result[WeaviateConstants.AdditionalPropertiesPropertyName]; + + var scoreProperty = additionalProperties?[scorePropertyName]; + double? score = scoreProperty?.GetValueKind() switch + { + JsonValueKind.Number => scoreProperty.GetValue(), + JsonValueKind.String => double.Parse(scoreProperty.GetValue()), + _ => null + }; + + var vectorPropertyName = hasNamedVectors ? + WeaviateConstants.ReservedVectorPropertyName : + WeaviateConstants.ReservedSingleVectorPropertyName; + + var id = additionalProperties?[WeaviateConstants.ReservedKeyPropertyName]; + var vectors = additionalProperties?[vectorPropertyName]; + + var storageModel = new JsonObject + { + { WeaviateConstants.ReservedKeyPropertyName, id?.DeepClone() }, + { WeaviateConstants.ReservedDataPropertyName, result?.DeepClone() }, + { vectorPropertyName, vectors?.DeepClone() }, + }; + + return (storageModel, score); + } +} diff --git a/MEVD/src/Weaviate/WeaviateConstants.cs b/MEVD/src/Weaviate/WeaviateConstants.cs new file mode 100644 index 0000000..1c46c4b --- /dev/null +++ b/MEVD/src/Weaviate/WeaviateConstants.cs @@ -0,0 +1,56 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text.Json; + +namespace CommunityToolkit.VectorData.Weaviate; + +internal sealed class WeaviateConstants +{ + /// The name of this vector store for telemetry purposes. + public const string VectorStoreSystemName = "weaviate"; + + /// Reserved key property name in Weaviate. + internal const string ReservedKeyPropertyName = "id"; + + /// Reserved data property name in Weaviate. + internal const string ReservedDataPropertyName = "properties"; + + /// Reserved vector property name in Weaviate. + internal const string ReservedVectorPropertyName = "vectors"; + + /// Reserved single vector property name in Weaviate. + internal const string ReservedSingleVectorPropertyName = "vector"; + + /// Collection property name in Weaviate. + internal const string CollectionPropertyName = "class"; + + /// Score property name in Weaviate. + internal const string ScorePropertyName = "distance"; + + /// Score property name for hybrid search in Weaviate. + internal const string HybridScorePropertyName = "score"; + + /// Additional properties property name in Weaviate. + internal const string AdditionalPropertiesPropertyName = "_additional"; + + /// Default vectorizer for vector properties in Weaviate. + internal const string DefaultVectorizer = "none"; + + /// Default JSON serializer options. + internal static readonly JsonSerializerOptions s_jsonSerializerOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + Converters = + { + new WeaviateDateTimeOffsetConverter(), + new WeaviateNullableDateTimeOffsetConverter(), + new WeaviateDateTimeConverter(), + new WeaviateNullableDateTimeConverter(), +#if NET + new WeaviateDateOnlyConverter(), + new WeaviateNullableDateOnlyConverter(), +#endif + } + }; +} diff --git a/MEVD/src/Weaviate/WeaviateDynamicCollection.cs b/MEVD/src/Weaviate/WeaviateDynamicCollection.cs new file mode 100644 index 0000000..345d1d7 --- /dev/null +++ b/MEVD/src/Weaviate/WeaviateDynamicCollection.cs @@ -0,0 +1,43 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Net.Http; + +namespace CommunityToolkit.VectorData.Weaviate; + +/// +/// Represents a collection of vector store records in a Weaviate database, mapped to a dynamic Dictionary<string, object?>. +/// +#pragma warning disable CA1711 // Identifiers should not have incorrect suffix +public sealed class WeaviateDynamicCollection : WeaviateCollection> +#pragma warning restore CA1711 // Identifiers should not have incorrect suffix +{ + /// + /// Initializes a new instance of the class. + /// + /// + /// that is used to interact with Weaviate API. + /// should point to remote or local cluster and API key can be configured via . + /// It's also possible to provide these parameters via . + /// + /// The name of the collection. + /// Optional configuration options for this class. + // TODO: The provider uses unsafe JSON serialization in many places, #11963 + [RequiresUnreferencedCode("The Weaviate provider is currently incompatible with trimming.")] + [RequiresDynamicCode("The Weaviate provider is currently incompatible with NativeAOT.")] + public WeaviateDynamicCollection(HttpClient httpClient, string name, WeaviateCollectionOptions options) + : base( + httpClient, + name, + static options => new WeaviateModelBuilder(options.HasNamedVectors) + .BuildDynamic( + options.Definition ?? throw new ArgumentException("Definition is required for dynamic collections"), + options.EmbeddingGenerator, + WeaviateConstants.s_jsonSerializerOptions), + options) + { + } +} diff --git a/MEVD/src/Weaviate/WeaviateFilterTranslator.cs b/MEVD/src/Weaviate/WeaviateFilterTranslator.cs new file mode 100644 index 0000000..464ef48 --- /dev/null +++ b/MEVD/src/Weaviate/WeaviateFilterTranslator.cs @@ -0,0 +1,305 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections; +using System.Diagnostics; +using System.Linq; +using System.Linq.Expressions; +using System.Text; +using System.Text.Json; +using Microsoft.Extensions.VectorData.ProviderServices; +using Microsoft.Extensions.VectorData.ProviderServices.Filter; + +namespace CommunityToolkit.VectorData.Weaviate; + +// https://weaviate.io/developers/weaviate/api/graphql/filters#filter-structure +internal class WeaviateFilterTranslator : FilterTranslatorBase +{ + private readonly StringBuilder _filter = new(); + + internal string? Translate(LambdaExpression lambdaExpression, CollectionModel model) + { + // Weaviate doesn't seem to have a native way of expressing "always true" filters; since this scenario is important for fetching + // all records (via GetAsync with filter), we special-case and support it here. Note that false isn't supported (useless), + // nor is 'x && true'. + if (lambdaExpression.Body is ConstantExpression { Value: true }) + { + return null; + } + + var preprocessedExpression = PreprocessFilter(lambdaExpression, model, new FilterPreprocessingOptions()); + + Translate(preprocessedExpression); + return _filter.ToString(); + } + + private void Translate(Expression? node) + { + switch (node) + { + case BinaryExpression + { + NodeType: ExpressionType.Equal or ExpressionType.NotEqual + or ExpressionType.GreaterThan or ExpressionType.GreaterThanOrEqual + or ExpressionType.LessThan or ExpressionType.LessThanOrEqual + } binary: + TranslateEqualityComparison(binary); + return; + + case BinaryExpression { NodeType: ExpressionType.AndAlso } andAlso: + _filter.Append("{ operator: And, operands: ["); + Translate(andAlso.Left); + _filter.Append(", "); + Translate(andAlso.Right); + _filter.Append("] }"); + return; + + case BinaryExpression { NodeType: ExpressionType.OrElse } orElse: + _filter.Append("{ operator: Or, operands: ["); + Translate(orElse.Left); + _filter.Append(", "); + Translate(orElse.Right); + _filter.Append("] }"); + return; + + case UnaryExpression { NodeType: ExpressionType.Not } not: + { + switch (not.Operand) + { + // Special handling for !(a == b) and !(a != b), transforming to a != b and a == b respectively. + case BinaryExpression { NodeType: ExpressionType.Equal or ExpressionType.NotEqual } binary: + TranslateEqualityComparison( + Expression.MakeBinary( + binary.NodeType is ExpressionType.Equal ? ExpressionType.NotEqual : ExpressionType.Equal, + binary.Left, + binary.Right)); + return; + + // Not over bool field (r => !r.Bool) + case var negated when negated.Type == typeof(bool) && TryBindProperty(negated, out var property): + GenerateEqualityComparison(property.StorageName, false, ExpressionType.Equal); + return; + + default: + throw new NotSupportedException("Weaviate does not support the NOT operator (see https://github.com/weaviate/weaviate/issues/3683)"); + } + } + + // Handle converting non-nullable to nullable; such nodes are found in e.g. r => r.Int == nullableInt + case UnaryExpression { NodeType: ExpressionType.Convert } convert when Nullable.GetUnderlyingType(convert.Type) == convert.Operand.Type: + Translate(convert.Operand); + return; + + // Special handling for bool constant as the filter expression (r => r.Bool) + case Expression when node.Type == typeof(bool) && TryBindProperty(node, out var property): + GenerateEqualityComparison(property.StorageName, true, ExpressionType.Equal); + return; + + case MethodCallExpression methodCall: + TranslateMethodCall(methodCall); + return; + + default: + throw new NotSupportedException("The following NodeType is unsupported: " + node?.NodeType); + } + } + + private void TranslateEqualityComparison(BinaryExpression binary) + { + if (TryBindProperty(binary.Left, out var property) && binary.Right is ConstantExpression { Value: var rightConstant }) + { + GenerateEqualityComparison(property.StorageName, rightConstant, binary.NodeType); + return; + } + + if (TryBindProperty(binary.Right, out property) && binary.Left is ConstantExpression { Value: var leftConstant }) + { + GenerateEqualityComparison(property.StorageName, leftConstant, binary.NodeType); + return; + } + + throw new NotSupportedException("Invalid equality/comparison"); + } + + private void GenerateEqualityComparison(string propertyStorageName, object? value, ExpressionType nodeType) + { + // { path: ["intPropName"], operator: Equal, ValueInt: 8 } + _filter + .Append("{ path: [\"") + .Append(JsonEncodedText.Encode(propertyStorageName)) + .Append("\"], operator: "); + + // Special handling for null comparisons + if (value is null) + { + if (nodeType is ExpressionType.Equal or ExpressionType.NotEqual) + { + _filter + .Append("IsNull, valueBoolean: ") + .Append(nodeType is ExpressionType.Equal ? "true" : "false") + .Append(" }"); + return; + } + + throw new NotSupportedException("null value supported only with equality/inequality checks"); + } + + // Operator + _filter.Append(nodeType switch + { + ExpressionType.Equal => "Equal", + ExpressionType.NotEqual => "NotEqual", + + ExpressionType.GreaterThan => "GreaterThan", + ExpressionType.GreaterThanOrEqual => "GreaterThanEqual", + ExpressionType.LessThan => "LessThan", + ExpressionType.LessThanOrEqual => "LessThanEqual", + + _ => throw new UnreachableException() + }); + + _filter.Append(", "); + + // FieldType + var type = value.GetType(); + if (Nullable.GetUnderlyingType(type) is Type underlying) + { + type = underlying; + } + + _filter.Append(value.GetType() switch + { + Type t when t == typeof(int) || t == typeof(long) || t == typeof(short) || t == typeof(byte) => "valueInt", + Type t when t == typeof(bool) => "valueBoolean", + Type t when t == typeof(string) || t == typeof(Guid) => "valueText", + Type t when t == typeof(float) || t == typeof(double) || t == typeof(decimal) => "valueNumber", + Type t when t == typeof(DateTime) => "valueDate", + Type t when t == typeof(DateTimeOffset) => "valueDate", +#if NET + Type t when t == typeof(DateOnly) => "valueDate", +#endif + + _ => throw new NotSupportedException($"Unsupported value type {type.FullName} in filter.") + }); + + _filter.Append(": "); + + // Value — use Weaviate's JSON serializer options for proper date/time formatting + _filter.Append(JsonSerializer.Serialize(value, value.GetType(), WeaviateConstants.s_jsonSerializerOptions)); + + _filter.Append('}'); + } + + private void TranslateMethodCall(MethodCallExpression methodCall) + { + switch (methodCall) + { + // Enumerable.Contains(), List.Contains(), MemoryExtensions.Contains() + case var _ when TryMatchContains(methodCall, out var source, out var item): + TranslateContains(source, item); + return; + + // Enumerable.Any() with a Contains predicate (r => r.Strings.Any(s => array.Contains(s))) + case { Method.Name: nameof(Enumerable.Any), Arguments: [var anySource, LambdaExpression lambda] } any + when any.Method.DeclaringType == typeof(Enumerable): + TranslateAny(anySource, lambda); + return; + + default: + throw new NotSupportedException($"Unsupported method call: {methodCall.Method.DeclaringType?.Name}.{methodCall.Method.Name}"); + } + } + + private void TranslateContains(Expression source, Expression item) + { + // Contains over array + // { path: ["stringArrayPropName"], operator: ContainsAny, valueText: ["foo"] } + if (TryBindProperty(source, out var property) && item is ConstantExpression { Value: string stringConstant }) + { + _filter + .Append("{ path: [\"") + .Append(JsonEncodedText.Encode(property.StorageName)) + .Append("\"], operator: ContainsAny, valueText: [") + .Append(JsonEncodedText.Encode(stringConstant)) + .Append("]}"); + return; + } + + throw new NotSupportedException("Contains supported only over tag field"); + } + + /// + /// Translates an Any() call with a Contains predicate, e.g. r.Strings.Any(s => array.Contains(s)). + /// This checks whether any element in the array field is contained in the given values. + /// + private void TranslateAny(Expression source, LambdaExpression lambda) + { + // We only support the pattern: r.ArrayField.Any(x => values.Contains(x)) + // Translates to: { path: ["Field"], operator: ContainsAny, valueText: ["value1", "value2"] } + if (!TryBindProperty(source, out var property) + || lambda.Body is not MethodCallExpression containsCall + || !TryMatchContains(containsCall, out var valuesExpression, out var itemExpression)) + { + throw new NotSupportedException("Unsupported method call: Enumerable.Any"); + } + + // Verify that the item is the lambda parameter + if (itemExpression != lambda.Parameters[0]) + { + throw new NotSupportedException("Unsupported method call: Enumerable.Any"); + } + + // Extract the values + IEnumerable values = valuesExpression switch + { + NewArrayExpression newArray => ExtractArrayValues(newArray), + ConstantExpression { Value: IEnumerable enumerable and not string } => enumerable, + _ => throw new NotSupportedException("Unsupported method call: Enumerable.Any") + }; + + // Generate: { path: ["Field"], operator: ContainsAny, valueText: ["value1", "value2"] } + _filter + .Append("{ path: [\"") + .Append(JsonEncodedText.Encode(property.StorageName)) + .Append("\"], operator: ContainsAny, valueText: ["); + + var isFirst = true; + foreach (var element in values) + { + if (element is not string stringElement) + { + throw new NotSupportedException("Any with Contains over non-string arrays is not supported"); + } + + if (isFirst) + { + isFirst = false; + } + else + { + _filter.Append(", "); + } + + _filter.Append(JsonSerializer.Serialize(stringElement)); + } + + _filter.Append("]}"); + + static object?[] ExtractArrayValues(NewArrayExpression newArray) + { + var result = new object?[newArray.Expressions.Count]; + for (var i = 0; i < newArray.Expressions.Count; i++) + { + if (newArray.Expressions[i] is not ConstantExpression { Value: var elementValue }) + { + throw new NotSupportedException("Invalid element in array"); + } + + result[i] = elementValue; + } + + return result; + } + } +} diff --git a/MEVD/src/Weaviate/WeaviateMapper.cs b/MEVD/src/Weaviate/WeaviateMapper.cs new file mode 100644 index 0000000..102b8c2 --- /dev/null +++ b/MEVD/src/Weaviate/WeaviateMapper.cs @@ -0,0 +1,204 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Nodes; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData.ProviderServices; +using Microsoft.Shared.Diagnostics; + +namespace CommunityToolkit.VectorData.Weaviate; + +internal sealed class WeaviateMapper + where TRecord : class +{ + private readonly string _collectionName; + private readonly bool _hasNamedVectors; + private readonly CollectionModel _model; + private readonly JsonSerializerOptions _jsonSerializerOptions; + + private readonly string _vectorPropertyName; + + public WeaviateMapper( + string collectionName, + bool hasNamedVectors, + CollectionModel model, + JsonSerializerOptions jsonSerializerOptions) + { + _collectionName = collectionName; + _hasNamedVectors = hasNamedVectors; + _model = model; + _jsonSerializerOptions = jsonSerializerOptions; + + _vectorPropertyName = hasNamedVectors ? + WeaviateConstants.ReservedVectorPropertyName : + WeaviateConstants.ReservedSingleVectorPropertyName; + } + + public JsonObject MapFromDataToStorageModel(TRecord dataModel, int recordIndex, IReadOnlyList?[]? generatedEmbeddings) + { + var keyNode = _model.KeyProperty.GetValueAsObject(dataModel) switch + { + Guid g => JsonValue.Create(g), + + null => throw new InvalidOperationException("Key property must not be nulL"), + _ => throw new InvalidOperationException("Key property must be a Guid") + }; + + // Populate data properties. + var dataNode = new JsonObject(); + foreach (var property in _model.DataProperties) + { + if (property.GetValueAsObject(dataModel) is object value) + { + // TODO: NativeAOT support, #11963 + dataNode[property.StorageName] = JsonSerializer.SerializeToNode(value, property.Type, _jsonSerializerOptions); + } + } + + // Populate vector properties. + JsonNode? vectorNode = null; + + if (_hasNamedVectors) + { + vectorNode = new JsonObject(); + + for (var i = 0; i < _model.VectorProperties.Count; i++) + { + var property = _model.VectorProperties[i]; + + var vector = generatedEmbeddings?[i] is IReadOnlyList ge + ? ge[recordIndex] + : property.GetValueAsObject(dataModel); + + vectorNode[property.StorageName] = vector switch + { + ReadOnlyMemory e => BuildJsonArray(e), + Embedding e => BuildJsonArray(e.Vector), + float[] a => BuildJsonArray(a), + + null => null, + + _ => throw new UnreachableException() + }; + } + } + else + { + var vector = generatedEmbeddings?[0] is IReadOnlyList ge + ? ge[recordIndex] + : _model.VectorProperty.GetValueAsObject(dataModel); + + vectorNode = vector switch + { + ReadOnlyMemory e => BuildJsonArray(e), + Embedding e => BuildJsonArray(e.Vector), + float[] a => BuildJsonArray(a), + + null => null, + + _ => throw new UnreachableException() + }; + } + + return new JsonObject + { + { WeaviateConstants.CollectionPropertyName, JsonValue.Create(_collectionName) }, + { WeaviateConstants.ReservedKeyPropertyName, keyNode }, + { WeaviateConstants.ReservedDataPropertyName, dataNode }, + { _vectorPropertyName, vectorNode }, + }; + + static JsonArray BuildJsonArray(ReadOnlyMemory memory) + { + var jsonArray = new JsonArray(); + + foreach (var item in memory.Span) + { + jsonArray.Add(JsonValue.Create(item)); + } + + return jsonArray; + } + } + + public TRecord MapFromStorageToDataModel(JsonObject storageModel, bool includeVectors) + { + Throw.IfNull(storageModel); + + var record = _model.CreateRecord()!; + + if (storageModel[WeaviateConstants.ReservedKeyPropertyName]?.GetValue() is not Guid key) + { + throw new InvalidOperationException("No key property was found in the record retrieved from storage."); + } + + _model.KeyProperty.SetValueAsObject(record, key); + + // Populate data properties. + if (storageModel[WeaviateConstants.ReservedDataPropertyName] is JsonObject dataPropertiesJson) + { + foreach (var property in _model.DataProperties) + { + if (dataPropertiesJson.TryGetPropertyValue(property.StorageName, out var dataValue)) + { + // TODO: NativeAOT support, #11963 + property.SetValueAsObject(record, dataValue?.Deserialize(property.Type, _jsonSerializerOptions)); + } + } + } + + // Populate vector properties. + if (includeVectors) + { + if (_hasNamedVectors && storageModel[_vectorPropertyName] is JsonObject vectorPropertiesJson) + { + foreach (var property in _model.VectorProperties) + { + if (vectorPropertiesJson.TryGetPropertyValue(property.StorageName, out var node)) + { + PopulateVectorProperty(record, node, property); + } + } + } + else + { + if (_model.VectorProperties is [var property] + && storageModel.TryGetPropertyValue(_vectorPropertyName, out var node)) + { + PopulateVectorProperty(record, node, property); + } + } + } + + return record; + + static void PopulateVectorProperty(TRecord record, object? value, VectorPropertyModel property) + { + switch (value) + { + case null: + property.SetValueAsObject(record, null); + return; + + case JsonArray jsonArray: + property.SetValueAsObject(record, (Nullable.GetUnderlyingType(property.Type) ?? property.Type) switch + { + var t when t == typeof(ReadOnlyMemory) => new ReadOnlyMemory(jsonArray.GetValues().ToArray()), + var t when t == typeof(float[]) => jsonArray.GetValues().ToArray(), + var t when t == typeof(Embedding) => new Embedding(jsonArray.GetValues().ToArray()), + + _ => throw new UnreachableException() + }); + return; + + default: + throw new InvalidOperationException("Non-array JSON node received for vector property"); + } + } + } +} diff --git a/MEVD/src/Weaviate/WeaviateModelBuilder.cs b/MEVD/src/Weaviate/WeaviateModelBuilder.cs new file mode 100644 index 0000000..4e586b0 --- /dev/null +++ b/MEVD/src/Weaviate/WeaviateModelBuilder.cs @@ -0,0 +1,126 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; + +namespace CommunityToolkit.VectorData.Weaviate; + +internal class WeaviateModelBuilder(bool hasNamedVectors) : CollectionJsonModelBuilder(GetModelBuildingOptions(hasNamedVectors)) +{ + internal const string SupportedVectorTypes = "ReadOnlyMemory, Embedding, float[]"; + + private static CollectionModelBuildingOptions GetModelBuildingOptions(bool hasNamedVectors) + { + return new() + { + RequiresAtLeastOneVector = !hasNamedVectors, + SupportsMultipleVectors = hasNamedVectors + }; + } + + protected override void ValidateKeyProperty(KeyPropertyModel keyProperty) + { + base.ValidateKeyProperty(keyProperty); + + if (keyProperty.Type != typeof(Guid)) + { + throw new NotSupportedException( + $"Property '{keyProperty.ModelName}' has unsupported type '{keyProperty.Type.Name}'. Key properties must be of type Guid."); + } + } + + protected override bool IsDataPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes) + { + supportedTypes = "string, bool, int, long, short, byte, float, double, decimal, DateTime, DateTimeOffset," +#if NET + + " DateOnly," +#endif + + " Guid, or arrays/lists of these types"; + + return IsValid(type) + || (type.IsArray && IsValid(type.GetElementType()!)) + || (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(List<>) && IsValid(type.GenericTypeArguments[0])); + + static bool IsValid(Type type) + { + if (Nullable.GetUnderlyingType(type) is Type underlyingType) + { + type = underlyingType; + } + + return type == typeof(string) + || type == typeof(bool) + || type == typeof(int) + || type == typeof(long) + || type == typeof(short) + || type == typeof(byte) + || type == typeof(float) + || type == typeof(double) + || type == typeof(decimal) + || type == typeof(DateTime) + || type == typeof(DateTimeOffset) +#if NET + || type == typeof(DateOnly) +#endif + || type == typeof(Guid); + } + } + + protected override bool IsVectorPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes) + => IsVectorPropertyTypeValidCore(type, out supportedTypes); + + internal static bool IsVectorPropertyTypeValidCore(Type type, [NotNullWhen(false)] out string? supportedTypes) + { + supportedTypes = SupportedVectorTypes; + + return type == typeof(ReadOnlyMemory) + || type == typeof(ReadOnlyMemory?) + || type == typeof(Embedding) + || type == typeof(float[]); + } + + /// + protected override void ValidateProperty(PropertyModel propertyModel, VectorStoreCollectionDefinition? definition) + { + base.ValidateProperty(propertyModel, definition); + + // GraphQL identifiers cannot be escaped; storage names are validated during model building. + // See https://spec.graphql.org/October2021/#sec-Names + if (!IsValidIdentifier(propertyModel.StorageName)) + { + throw new InvalidOperationException( + $"Property '{propertyModel.ModelName}' has storage name '{propertyModel.StorageName}' which is not a valid GraphQL identifier. " + + "GraphQL identifiers must start with a letter or underscore, and contain only letters, digits, and underscores."); + } + } + + private static bool IsValidIdentifier(string name) + { + if (string.IsNullOrEmpty(name)) + { + return false; + } + + var first = name[0]; + if (!char.IsLetter(first) && first != '_') + { + return false; + } + + for (var i = 1; i < name.Length; i++) + { + var c = name[i]; + if (!char.IsLetterOrDigit(c) && c != '_') + { + return false; + } + } + + return true; + } +} diff --git a/MEVD/src/Weaviate/WeaviateQueryBuilder.cs b/MEVD/src/Weaviate/WeaviateQueryBuilder.cs new file mode 100644 index 0000000..2241e65 --- /dev/null +++ b/MEVD/src/Weaviate/WeaviateQueryBuilder.cs @@ -0,0 +1,189 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Linq; +using System.Linq.Expressions; +using System.Text.Json; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; + +namespace CommunityToolkit.VectorData.Weaviate; + +/// +/// Contains methods to build Weaviate queries. +/// +internal static class WeaviateQueryBuilder +{ + /// + /// Builds Weaviate search query. + /// More information here: . + /// + public static string BuildSearchQuery( + TVector vector, + string collectionName, + string vectorPropertyName, + JsonSerializerOptions jsonSerializerOptions, + int top, + VectorSearchOptions searchOptions, + CollectionModel model, + bool hasNamedVectors) + { + var vectorsQuery = GetVectorsPropertyQuery(searchOptions.IncludeVectors, hasNamedVectors, model); + + var filter = searchOptions.Filter is not null + ? new WeaviateFilterTranslator().Translate(searchOptions.Filter, model) + : null; + + var vectorArray = JsonSerializer.Serialize(vector, jsonSerializerOptions); + + // Weaviate nearVector supports distance parameter for thresholding. + // Distance works for all distance functions (lower values = more similar). + var distanceFilter = searchOptions.ScoreThreshold.HasValue + ? $"distance: {searchOptions.ScoreThreshold.Value}" + : string.Empty; + + return $$""" + { + Get { + {{collectionName}} ( + limit: {{top}} + offset: {{searchOptions.Skip}} + {{(filter is null ? "" : "where: " + filter)}} + nearVector: { + {{GetTargetVectorsQuery(hasNamedVectors, vectorPropertyName)}} + vector: {{vectorArray}} + {{distanceFilter}} + } + ) { + {{string.Join(" ", model.DataProperties.Select(p => p.StorageName))}} + {{WeaviateConstants.AdditionalPropertiesPropertyName}} { + {{WeaviateConstants.ReservedKeyPropertyName}} + {{WeaviateConstants.ScorePropertyName}} + {{vectorsQuery}} + } + } + } + } + """; + } + + /// + /// Builds Weaviate search query. + /// More information here: . + /// + public static string BuildQuery( + Expression> filter, + int top, + FilteredRecordRetrievalOptions queryOptions, + string collectionName, + CollectionModel model, + bool hasNamedVectors) + { + var vectorsQuery = GetVectorsPropertyQuery(queryOptions.IncludeVectors, hasNamedVectors, model); + + var orderBy = queryOptions.OrderBy?.Invoke(new()).Values; + var sortPaths = orderBy is not { Count: > 0 } ? "" : string.Join(",", orderBy.Select(sortInfo => + { + string sortPath = model.GetDataOrKeyProperty(sortInfo.PropertySelector).StorageName; + + return $$"""{ path: ["{{JsonEncodedText.Encode(sortPath)}}"], order: {{(sortInfo.Ascending ? "asc" : "desc")}} }"""; + })); + + var translatedFilter = new WeaviateFilterTranslator().Translate(filter, model); + + return $$""" + { + Get { + {{collectionName}} ( + limit: {{top}} + offset: {{queryOptions.Skip}} + {{(translatedFilter is null ? "" : "where: " + translatedFilter)}} + sort: [ {{sortPaths}} ] + ) { + {{string.Join(" ", model.DataProperties.Select(p => p.StorageName))}} + {{WeaviateConstants.AdditionalPropertiesPropertyName}} { + {{WeaviateConstants.ReservedKeyPropertyName}} + {{WeaviateConstants.ScorePropertyName}} + {{vectorsQuery}} + } + } + } + } + """; + } + + /// + /// Builds Weaviate hybrid search query. + /// More information here: . + /// + public static string BuildHybridSearchQuery( + TVector vector, + int top, + string keywords, + string collectionName, + CollectionModel model, + VectorPropertyModel vectorProperty, + DataPropertyModel textProperty, + JsonSerializerOptions jsonSerializerOptions, + HybridSearchOptions searchOptions, + bool hasNamedVectors) + { + // https://docs.weaviate.io/weaviate/api/graphql/search-operators#hybrid + var vectorsQuery = GetVectorsPropertyQuery(searchOptions.IncludeVectors, hasNamedVectors, model); + + var filter = searchOptions.Filter is not null + ? new WeaviateFilterTranslator().Translate(searchOptions.Filter, model) + : null; + + var vectorArray = JsonSerializer.Serialize(vector, jsonSerializerOptions); + var sanitizedKeywords = keywords.Replace("\\", "\\\\").Replace("\"", "\\\""); + + return $$""" + { + Get { + {{collectionName}} ( + limit: {{top}} + offset: {{searchOptions.Skip}} + {{(filter is null ? "" : "where: " + filter)}} + hybrid: { + query: "{{sanitizedKeywords}}" + properties: ["{{textProperty.StorageName}}"] + {{GetTargetVectorsQuery(hasNamedVectors, vectorProperty.StorageName)}} + vector: {{vectorArray}} + fusionType: rankedFusion + } + ) { + {{string.Join(" ", model.DataProperties.Select(p => p.StorageName))}} + {{WeaviateConstants.AdditionalPropertiesPropertyName}} { + {{WeaviateConstants.ReservedKeyPropertyName}} + {{WeaviateConstants.HybridScorePropertyName}} + {{vectorsQuery}} + } + } + } + } + """; + } + + #region private + + private static string GetTargetVectorsQuery(bool hasNamedVectors, string vectorPropertyName) + { + return hasNamedVectors ? $"targetVectors: [\"{vectorPropertyName}\"]" : string.Empty; + } + + private static string GetVectorsPropertyQuery( + bool includeVectors, + bool hasNamedVectors, + CollectionModel model) + { + return includeVectors + ? hasNamedVectors + ? $"vectors {{ {string.Join(" ", model.VectorProperties.Select(p => p.StorageName))} }}" + : WeaviateConstants.ReservedSingleVectorPropertyName + : string.Empty; + } + + #endregion +} diff --git a/MEVD/src/Weaviate/WeaviateServiceCollectionExtensions.cs b/MEVD/src/Weaviate/WeaviateServiceCollectionExtensions.cs new file mode 100644 index 0000000..2c6f0b3 --- /dev/null +++ b/MEVD/src/Weaviate/WeaviateServiceCollectionExtensions.cs @@ -0,0 +1,264 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Diagnostics.CodeAnalysis; +using System.Net.Http; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData; +using CommunityToolkit.VectorData.Weaviate; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.DependencyInjection; + +/// +/// Extension methods to register and instances on an +/// +public static class WeaviateServiceCollectionExtensions +{ + private const string DynamicCodeMessage = "This method is incompatible with NativeAOT, consult the documentation for adding collections in a way that's compatible with NativeAOT."; + private const string UnreferencedCodeMessage = "This method is incompatible with trimming, consult the documentation for adding collections in a way that's compatible with NativeAOT."; + + /// + /// Registers a as + /// with returned by + /// or retrieved from the dependency injection container if was not provided. + /// + /// + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddWeaviateVectorStore( + this IServiceCollection services, + Func? clientProvider = default, + Func? optionsProvider = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + => AddKeyedWeaviateVectorStore(services, serviceKey: null, clientProvider, optionsProvider, lifetime); + + /// + /// Registers a keyed as + /// with returned by + /// or retrieved from the dependency injection container if was not provided. + /// + /// The to register the on. + /// The key with which to associate the vector store. + /// The provider. + /// Options provider to further configure the . + /// The service lifetime for the store. Defaults to . + /// Service collection. + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddKeyedWeaviateVectorStore( + this IServiceCollection services, + object? serviceKey, + Func? clientProvider = default, + Func? optionsProvider = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + { + Throw.IfNull(services); + + services.Add(new ServiceDescriptor(typeof(WeaviateVectorStore), serviceKey, (sp, _) => + { + var client = GetHttpClient(clientProvider?.Invoke(sp), sp); + var options = GetStoreOptions(sp, optionsProvider ?? (static s => s.GetService()!)); + + return new WeaviateVectorStore(client, options); + }, lifetime)); + + services.Add(new ServiceDescriptor(typeof(VectorStore), serviceKey, + static (sp, key) => sp.GetRequiredKeyedService(key), lifetime)); + + return services; + } + + /// + /// Registers a as + /// with created with , . + /// + /// + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddWeaviateVectorStore( + this IServiceCollection services, + Uri endpoint, + string? apiKey, + WeaviateVectorStoreOptions? options = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + => AddKeyedWeaviateVectorStore(services, serviceKey: null, endpoint, apiKey, options, lifetime); + + /// + /// Registers a keyed as + /// with created with , . + /// + /// The to register the on. + /// The key with which to associate the vector store. + /// The endpoint to connect to. + /// The API key to use. + /// Options to further configure the . + /// The service lifetime for the store. Defaults to . + /// Service collection. + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddKeyedWeaviateVectorStore( + this IServiceCollection services, + object? serviceKey, + Uri endpoint, + string? apiKey, + WeaviateVectorStoreOptions? options = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + { + Throw.IfNull(endpoint); + + WeaviateVectorStoreOptions copy = new(options) + { + Endpoint = endpoint, + ApiKey = apiKey + }; + + return AddKeyedWeaviateVectorStore(services, serviceKey, sp => GetHttpClient(null, sp), sp => copy, lifetime); + } + + /// + /// Registers a as + /// with returned by + /// or retrieved from the dependency injection container if was not provided. + /// + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddWeaviateCollection( + this IServiceCollection services, + string name, + Func? clientProvider = default, + Func? optionsProvider = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + where TRecord : class + => AddKeyedWeaviateCollection(services, serviceKey: null, name, clientProvider, optionsProvider, lifetime); + + /// + /// Registers a keyed as + /// with returned by + /// or retrieved from the dependency injection container if was not provided. + /// + /// The to register the on. + /// The key with which to associate the collection. + /// The name of the collection. + /// The provider. + /// Options provider to further configure the . + /// The service lifetime for the store. Defaults to . + /// Service collection. + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddKeyedWeaviateCollection( + this IServiceCollection services, + object? serviceKey, + string name, + Func? clientProvider = default, + Func? optionsProvider = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + where TRecord : class + { + Throw.IfNull(services); + Throw.IfNullOrWhitespace(name); + + services.Add(new ServiceDescriptor(typeof(WeaviateCollection), serviceKey, (sp, _) => + { + var client = GetHttpClient(clientProvider?.Invoke(sp), sp); + var options = GetCollectionOptions(sp, optionsProvider ?? (static s => s.GetService()!)); + + return new WeaviateCollection(client, name, options); + }, lifetime)); + + services.Add(new ServiceDescriptor(typeof(VectorStoreCollection), serviceKey, + static (sp, key) => sp.GetRequiredKeyedService>(key), lifetime)); + + services.Add(new ServiceDescriptor(typeof(IVectorSearchable), serviceKey, + static (sp, key) => sp.GetRequiredKeyedService>(key), lifetime)); + + services.Add(new ServiceDescriptor(typeof(IKeywordHybridSearchable), serviceKey, + static (sp, key) => sp.GetRequiredKeyedService>(key), lifetime)); + + return services; + } + + /// + /// Registers a as + /// with created with , . + /// + /// + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddWeaviateCollection( + this IServiceCollection services, + string name, + Uri endpoint, + string? apiKey, + WeaviateCollectionOptions? options = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + where TRecord : class + => AddKeyedWeaviateCollection(services, serviceKey: null, name, endpoint, apiKey, options, lifetime); + + /// + /// Registers a keyed as + /// with created with , . + /// + /// The to register the on. + /// The key with which to associate the collection. + /// The name of the collection. + /// The endpoint to connect to. + /// The API key to use. + /// Options to further configure the . + /// The service lifetime for the store. Defaults to . + /// Service collection. + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddKeyedWeaviateCollection( + this IServiceCollection services, + object? serviceKey, + string name, + Uri endpoint, + string? apiKey, + WeaviateCollectionOptions? options = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + where TRecord : class + { + Throw.IfNull(endpoint); + + WeaviateCollectionOptions copy = new(options) + { + Endpoint = endpoint, + ApiKey = apiKey + }; + + return AddKeyedWeaviateCollection(services, serviceKey, name, sp => GetHttpClient(null, sp), sp => copy, lifetime); + } + + private static WeaviateVectorStoreOptions? GetStoreOptions(IServiceProvider sp, Func? optionsProvider) + { + var options = optionsProvider?.Invoke(sp); + if (options?.EmbeddingGenerator is not null) + { + return options; // The user has provided everything, there is nothing to change. + } + + var embeddingGenerator = sp.GetService(); + return embeddingGenerator is null + ? options // There is nothing to change. + : new(options) { EmbeddingGenerator = embeddingGenerator }; // Create a brand new copy in order to avoid modifying the original options. + } + + private static WeaviateCollectionOptions? GetCollectionOptions(IServiceProvider sp, Func? optionsProvider) + { + var options = optionsProvider?.Invoke(sp); + if (options?.EmbeddingGenerator is not null) + { + return options; // The user has provided everything, there is nothing to change. + } + + var embeddingGenerator = sp.GetService(); + return embeddingGenerator is null + ? options // There is nothing to change. + : new(options) { EmbeddingGenerator = embeddingGenerator }; // Create a brand new copy in order to avoid modifying the original options. + } + + private static HttpClient GetHttpClient(HttpClient? httpClient, IServiceProvider sp) + => httpClient ?? sp.GetService() ?? new HttpClient(); +} diff --git a/MEVD/src/Weaviate/WeaviateVectorStore.cs b/MEVD/src/Weaviate/WeaviateVectorStore.cs new file mode 100644 index 0000000..f03ae0c --- /dev/null +++ b/MEVD/src/Weaviate/WeaviateVectorStore.cs @@ -0,0 +1,187 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Net.Http; +using System.Runtime.CompilerServices; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; +using Microsoft.Shared.Diagnostics; + +namespace CommunityToolkit.VectorData.Weaviate; + +/// +/// Class for accessing the list of collections in a Weaviate vector store. +/// +/// +/// This class can be used with collections of any schema type, but requires you to provide schema information when getting a collection. +/// +public sealed class WeaviateVectorStore : VectorStore +{ + /// Metadata about vector store. + private readonly VectorStoreMetadata _metadata; + + /// that is used to interact with Weaviate API. + private readonly HttpClient _httpClient; + + /// A general purpose definition that can be used to construct a collection when needing to proxy schema agnostic operations. + private static readonly VectorStoreCollectionDefinition s_generalPurposeDefinition = new() { Properties = [new VectorStoreKeyProperty("Key", typeof(Guid)), new VectorStoreVectorProperty("Vector", typeof(ReadOnlyMemory), 1)] }; + + /// Weaviate endpoint for remote or local cluster. + private readonly Uri? _endpoint; + + /// + /// Weaviate API key. + /// + private readonly string? _apiKey; + + /// Whether the vectors in the store are named and multiple vectors are supported, or whether there is just a single unnamed vector in Weaviate collection. + private readonly bool _hasNamedVectors; + + private readonly IEmbeddingGenerator? _embeddingGenerator; + + /// + /// Initializes a new instance of the class. + /// + /// + /// that is used to interact with Weaviate API. + /// should point to remote or local cluster and API key can be configured via . + /// It's also possible to provide these parameters via . + /// + /// Optional configuration options for this class. + [RequiresUnreferencedCode("The Weaviate provider is currently incompatible with trimming.")] + [RequiresDynamicCode("The Weaviate provider is currently incompatible with NativeAOT.")] + public WeaviateVectorStore(HttpClient httpClient, WeaviateVectorStoreOptions? options = null) + { + Throw.IfNull(httpClient); + + _httpClient = httpClient; + + options ??= WeaviateVectorStoreOptions.Default; + _endpoint = options.Endpoint; + _apiKey = options.ApiKey; + _hasNamedVectors = options.HasNamedVectors; + _embeddingGenerator = options.EmbeddingGenerator; + + _metadata = new() + { + VectorStoreSystemName = WeaviateConstants.VectorStoreSystemName + }; + } + +#pragma warning disable IDE0090 // Use 'new(...)' + /// + /// The collection name must start with a capital letter and contain only ASCII letters and digits. + [RequiresUnreferencedCode("The Weaviate provider is currently incompatible with trimming.")] + [RequiresDynamicCode("The Weaviate provider is currently incompatible with NativeAOT.")] +#if NET + public override WeaviateCollection GetCollection(string name, VectorStoreCollectionDefinition? definition = null) +#else + public override VectorStoreCollection GetCollection(string name, VectorStoreCollectionDefinition? definition = null) +#endif + => typeof(TRecord) == typeof(Dictionary) + ? throw new ArgumentException(VectorDataStrings.GetCollectionWithDictionaryNotSupported) + : new WeaviateCollection( + _httpClient, + name, + new() + { + Definition = definition, + Endpoint = _endpoint, + ApiKey = _apiKey, + HasNamedVectors = _hasNamedVectors, + EmbeddingGenerator = _embeddingGenerator + }); + + /// + // TODO: The provider uses unsafe JSON serialization in many places, #11963 + [RequiresUnreferencedCode("The Weaviate provider is currently incompatible with trimming.")] + [RequiresDynamicCode("The Weaviate provider is currently incompatible with NativeAOT.")] +#if NET + public override WeaviateDynamicCollection GetDynamicCollection(string name, VectorStoreCollectionDefinition definition) +#else + public override VectorStoreCollection> GetDynamicCollection(string name, VectorStoreCollectionDefinition definition) +#endif + => new WeaviateDynamicCollection( + _httpClient, + name, + new() + { + Definition = definition, + Endpoint = _endpoint, + ApiKey = _apiKey, + HasNamedVectors = _hasNamedVectors, + EmbeddingGenerator = _embeddingGenerator + }); +#pragma warning restore IDE0090 + + /// + public override async IAsyncEnumerable ListCollectionNamesAsync([EnumeratorCancellation] CancellationToken cancellationToken = default) + { + const string OperationName = "ListCollectionNames"; + + using var request = new WeaviateGetCollectionsRequest().Build(); + + var httpResponseContent = await VectorStoreErrorHandler.RunOperationAsync( + _metadata, + OperationName, + async () => + { + var httpResponse = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseContentRead, cancellationToken).ConfigureAwait(false); + + httpResponse.EnsureSuccessStatusCode(); + + return await httpResponse.Content.ReadAsStringAsync( +#if NET + cancellationToken +#endif + ).ConfigureAwait(false); + }).ConfigureAwait(false); + + var collectionsResponse = VectorStoreErrorHandler.RunOperation( + _metadata, + OperationName, + () => JsonSerializer.Deserialize(httpResponseContent)); + + if (collectionsResponse?.Collections is not null) + { + foreach (var collection in collectionsResponse.Collections) + { + yield return collection.CollectionName; + } + } + } + + /// + public override Task CollectionExistsAsync(string name, CancellationToken cancellationToken = default) + { + var collection = GetDynamicCollection(name, s_generalPurposeDefinition); + return collection.CollectionExistsAsync(cancellationToken); + } + + /// + public override Task EnsureCollectionDeletedAsync(string name, CancellationToken cancellationToken = default) + { + var collection = GetDynamicCollection(name, s_generalPurposeDefinition); + return collection.EnsureCollectionDeletedAsync(cancellationToken); + } + + /// + public override object? GetService(Type serviceType, object? serviceKey = null) + { + Throw.IfNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreMetadata) ? _metadata : + serviceType == typeof(HttpClient) ? _httpClient : + serviceType.IsInstanceOfType(this) ? this : + null; + } +} diff --git a/MEVD/src/Weaviate/WeaviateVectorStoreOptions.cs b/MEVD/src/Weaviate/WeaviateVectorStoreOptions.cs new file mode 100644 index 0000000..cc2b9d9 --- /dev/null +++ b/MEVD/src/Weaviate/WeaviateVectorStoreOptions.cs @@ -0,0 +1,55 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using Microsoft.Extensions.AI; + +namespace CommunityToolkit.VectorData.Weaviate; + +/// +/// Options when creating a . +/// +public sealed class WeaviateVectorStoreOptions +{ + internal static readonly WeaviateVectorStoreOptions Default = new(); + + /// + /// Initializes a new instance of the class. + /// + public WeaviateVectorStoreOptions() + { + } + + internal WeaviateVectorStoreOptions(WeaviateVectorStoreOptions? source) + { + Endpoint = source?.Endpoint; + ApiKey = source?.ApiKey; + HasNamedVectors = source?.HasNamedVectors ?? Default.HasNamedVectors; + EmbeddingGenerator = source?.EmbeddingGenerator; + } + + /// + /// Weaviate endpoint for remote or local cluster. + /// + public Uri? Endpoint { get; set; } + + /// + /// Weaviate API key. + /// + /// + /// This parameter is optional because authentication may be disabled in local clusters for testing purposes. + /// + public string? ApiKey { get; set; } + + /// + /// Gets or sets a value indicating whether the vectors in the store are named and multiple vectors are supported, or whether there is just a single unnamed vector in Weaviate collection. + /// Defaults to multiple named vectors. + /// . + /// + public bool HasNamedVectors { get; set; } = true; + + /// + /// Gets or sets the default embedding generator to use when generating vectors embeddings with this vector store. + /// + public IEmbeddingGenerator? EmbeddingGenerator { get; set; } +} diff --git a/MEVD/test/AzureAISearch.ConformanceTests/AzureAISearch.ConformanceTests.csproj b/MEVD/test/AzureAISearch.ConformanceTests/AzureAISearch.ConformanceTests.csproj new file mode 100644 index 0000000..c449cbf --- /dev/null +++ b/MEVD/test/AzureAISearch.ConformanceTests/AzureAISearch.ConformanceTests.csproj @@ -0,0 +1,43 @@ + + + + net10.0;$(NetFrameworkTfm) + enable + enable + true + false + AzureAI.ConformanceTests + b7762d10-e29b-4bb1-8b74-b6d69a667dd4 + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + + + + + + + + + + + + Always + + + Always + + + + diff --git a/MEVD/test/AzureAISearch.ConformanceTests/AzureAISearchAllSupportedTypesTests.cs b/MEVD/test/AzureAISearch.ConformanceTests/AzureAISearchAllSupportedTypesTests.cs new file mode 100644 index 0000000..24eb44f --- /dev/null +++ b/MEVD/test/AzureAISearch.ConformanceTests/AzureAISearchAllSupportedTypesTests.cs @@ -0,0 +1,106 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using AzureAISearch.ConformanceTests.Support; +using Microsoft.Extensions.VectorData; +using Xunit; + +namespace AzureAISearch.ConformanceTests; + +public class AzureAISearchAllSupportedTypesTests(AzureAISearchFixture fixture) : IClassFixture +{ + [Fact] + public async Task AllTypesBatchGetAsync() + { + var collection = fixture.TestStore.DefaultVectorStore.GetCollection("all-types", AzureAISearchAllTypes.GetRecordDefinition()); + await collection.EnsureCollectionExistsAsync(); + + List records = + [ + new() + { + Id = "all-types-1", + BoolProperty = true, + NullableBoolProperty = false, + StringProperty = "string prop 1", + NullableStringProperty = "nullable prop 1", + IntProperty = 1, + NullableIntProperty = 10, + LongProperty = 100L, + NullableLongProperty = 1000L, + FloatProperty = 10.5f, + NullableFloatProperty = 100.5f, + DoubleProperty = 23.75d, + NullableDoubleProperty = 233.75d, + DateTimeOffsetProperty = DateTimeOffset.UtcNow, + NullableDateTimeOffsetProperty = DateTimeOffset.UtcNow, + StringArray = ["one", "two"], + StringList = ["eleven", "twelve"], + BoolArray = [true, false], + BoolList = [true, false], + IntArray = [1, 2], + IntList = [11, 12], + LongArray = [100L, 200L], + LongList = [1100L, 1200L], + FloatArray = [1.5f, 2.5f], + FloatList = [11.5f, 12.5f], + DoubleArray = [1.5d, 2.5d], + DoubleList = [11.5d, 12.5d], + DateTimeOffsetArray = [DateTimeOffset.UtcNow, DateTimeOffset.UtcNow], + DateTimeOffsetList = [DateTimeOffset.UtcNow, DateTimeOffset.UtcNow], + Embedding = new ReadOnlyMemory([1.5f, 2.5f, 3.5f, 4.5f, 5.5f, 6.5f, 7.5f, 8.5f]) + }, + new() + { + Id = "all-types-2", + BoolProperty = false, + NullableBoolProperty = null, + StringProperty = "string prop 2", + NullableStringProperty = null, + IntProperty = 2, + NullableIntProperty = null, + LongProperty = 200L, + NullableLongProperty = null, + FloatProperty = 20.5f, + NullableFloatProperty = null, + DoubleProperty = 43.75, + NullableDoubleProperty = null, + Embedding = ReadOnlyMemory.Empty, + // From https://learn.microsoft.com/en-us/rest/api/searchservice/supported-data-types: + // "All of the above types are nullable, except for collections of primitive and complex types, for example, Collection(Edm.String)" + // So for collections, we can't use nulls. + StringArray = [], + StringList = [], + BoolArray = [], + BoolList = [], + IntArray = [], + IntList = [], + LongArray = [], + LongList = [], + FloatArray = [], + FloatList = [], + DoubleArray = [], + DoubleList = [], + DateTimeOffsetArray = [], + DateTimeOffsetList = [], + } + ]; + + try + { + await collection.UpsertAsync(records); + + var allTypes = await collection.GetAsync(records.Select(r => r.Id), new RecordRetrievalOptions { IncludeVectors = true }).ToListAsync(); + + var allTypes1 = allTypes.Single(x => x.Id == records[0].Id); + var allTypes2 = allTypes.Single(x => x.Id == records[1].Id); + + records[0].AssertEqual(allTypes1); + records[1].AssertEqual(allTypes2); + } + finally + { + await collection.EnsureCollectionDeletedAsync(); + } + } +} diff --git a/MEVD/test/AzureAISearch.ConformanceTests/AzureAISearchCollectionManagementTests.cs b/MEVD/test/AzureAISearch.ConformanceTests/AzureAISearchCollectionManagementTests.cs new file mode 100644 index 0000000..4101cac --- /dev/null +++ b/MEVD/test/AzureAISearch.ConformanceTests/AzureAISearchCollectionManagementTests.cs @@ -0,0 +1,11 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using AzureAISearch.ConformanceTests.Support; +using VectorData.ConformanceTests; +using Xunit; + +namespace AzureAISearch.ConformanceTests; + +public class AzureAISearchCollectionManagementTests(AzureAISearchFixture fixture) + : CollectionManagementTests(fixture), IClassFixture; diff --git a/MEVD/test/AzureAISearch.ConformanceTests/AzureAISearchDependencyInjectionTests.cs b/MEVD/test/AzureAISearch.ConformanceTests/AzureAISearchDependencyInjectionTests.cs new file mode 100644 index 0000000..ba302c7 --- /dev/null +++ b/MEVD/test/AzureAISearch.ConformanceTests/AzureAISearchDependencyInjectionTests.cs @@ -0,0 +1,119 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Azure; +using Azure.Search.Documents.Indexes; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using CommunityToolkit.VectorData.AzureAISearch; +using VectorData.ConformanceTests; +using Xunit; + +namespace AzureAISearch.ConformanceTests; + +public class AzureAISearchDependencyInjectionTests + : DependencyInjectionTests.Record>, string, DependencyInjectionTests.Record> +{ + private static readonly Uri s_endpoint = new("https://localhost"); + private static readonly AzureKeyCredential s_keyCredential = new("fakeKey"); + + protected override void PopulateConfiguration(ConfigurationManager configuration, object? serviceKey = null) + => configuration.AddInMemoryCollection( + [ + new(CreateConfigKey("AzureAI", serviceKey, "Endpoint"), "https://localhost"), + new(CreateConfigKey("AzureAI", serviceKey, "Key"), "fakeKey"), + ]); + + private static Uri EndpointProvider(IServiceProvider sp, object? serviceKey = null) + => new(sp.GetRequiredService().GetRequiredSection(CreateConfigKey("AzureAI", serviceKey, "Endpoint")).Value!); + + private static AzureKeyCredential KeyProvider(IServiceProvider sp, object? serviceKey = null) + => new(sp.GetRequiredService().GetRequiredSection(CreateConfigKey("AzureAI", serviceKey, "Key")).Value!); + + public override IEnumerable> CollectionDelegates + { + get + { + yield return (services, serviceKey, name, lifetime) => serviceKey is null + ? services + .AddAzureAISearchCollection(name, + sp => new SearchIndexClient(EndpointProvider(sp), KeyProvider(sp)), lifetime: lifetime) + : services + .AddKeyedAzureAISearchCollection(serviceKey, name, + sp => new SearchIndexClient(EndpointProvider(sp, serviceKey), KeyProvider(sp, serviceKey)), lifetime: lifetime); + + yield return (services, serviceKey, name, lifetime) => serviceKey is null + ? services + .AddSingleton(sp => new SearchIndexClient(s_endpoint, s_keyCredential)) + .AddAzureAISearchCollection(name, lifetime: lifetime) + : services + .AddSingleton(sp => new SearchIndexClient(s_endpoint, s_keyCredential)) + .AddKeyedAzureAISearchCollection(serviceKey, name, lifetime: lifetime); + + yield return (services, serviceKey, name, lifetime) => serviceKey is null + ? services.AddAzureAISearchCollection( + name, s_endpoint, s_keyCredential, lifetime: lifetime) + : services.AddKeyedAzureAISearchCollection( + serviceKey, name, s_endpoint, s_keyCredential, lifetime: lifetime); + } + } + + public override IEnumerable> StoreDelegates + { + get + { + yield return (services, serviceKey, lifetime) => serviceKey is null + ? services + .AddAzureAISearchVectorStore(sp => new SearchIndexClient(EndpointProvider(sp), KeyProvider(sp)), lifetime: lifetime) + : services + .AddKeyedAzureAISearchVectorStore(serviceKey, sp => new SearchIndexClient(EndpointProvider(sp, serviceKey), KeyProvider(sp, serviceKey)), lifetime: lifetime); + + yield return (services, serviceKey, lifetime) => serviceKey is null + ? services.AddAzureAISearchVectorStore( + s_endpoint, s_keyCredential, lifetime: lifetime) + : services.AddKeyedAzureAISearchVectorStore( + serviceKey, s_endpoint, s_keyCredential, lifetime: lifetime); + + yield return (services, serviceKey, lifetime) => serviceKey is null + ? services + .AddSingleton(sp => new SearchIndexClient(s_endpoint, s_keyCredential)) + .AddAzureAISearchVectorStore(lifetime: lifetime) + : services + .AddSingleton(sp => new SearchIndexClient(s_endpoint, s_keyCredential)) + .AddKeyedAzureAISearchVectorStore(serviceKey, lifetime: lifetime); + } + } + + [Fact] + public void EndpointCantBeNull() + { + IServiceCollection services = new ServiceCollection(); + + Assert.Throws(() => services.AddAzureAISearchCollection( + name: "notNull", endpoint: null!, s_keyCredential)); + Assert.Throws(() => services.AddKeyedAzureAISearchCollection( + serviceKey: "notNull", name: "notNull", endpoint: null!, s_keyCredential)); + } + + [Fact] + public void KeyCredentialCantBeNull() + { + IServiceCollection services = new ServiceCollection(); + + Assert.Throws(() => services.AddAzureAISearchCollection( + name: "notNull", s_endpoint, keyCredential: null!)); + Assert.Throws(() => services.AddKeyedAzureAISearchCollection( + serviceKey: "notNull", name: "notNull", s_endpoint, keyCredential: null!)); + } + + [Fact] + public void TokenCredentialCantBeNull() + { + IServiceCollection services = new ServiceCollection(); + + Assert.Throws(() => services.AddAzureAISearchCollection( + name: "notNull", s_endpoint, tokenCredential: null!)); + Assert.Throws(() => services.AddKeyedAzureAISearchCollection( + serviceKey: "notNull", name: "notNull", s_endpoint, tokenCredential: null!)); + } +} diff --git a/MEVD/test/AzureAISearch.ConformanceTests/AzureAISearchDistanceFunctionTests.cs b/MEVD/test/AzureAISearch.ConformanceTests/AzureAISearchDistanceFunctionTests.cs new file mode 100644 index 0000000..0a6b809 --- /dev/null +++ b/MEVD/test/AzureAISearch.ConformanceTests/AzureAISearchDistanceFunctionTests.cs @@ -0,0 +1,27 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using AzureAISearch.ConformanceTests.Support; +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace AzureAISearch.ConformanceTests; + +public class AzureAISearchDistanceFunctionTests(AzureAISearchDistanceFunctionTests.Fixture fixture) + : DistanceFunctionTests(fixture), IClassFixture +{ + public override Task CosineDistance() => Assert.ThrowsAsync(base.CosineDistance); + public override Task NegativeDotProductSimilarity() => Assert.ThrowsAsync(base.NegativeDotProductSimilarity); + public override Task EuclideanSquaredDistance() => Assert.ThrowsAsync(base.EuclideanSquaredDistance); + public override Task HammingDistance() => Assert.ThrowsAsync(base.HammingDistance); + public override Task ManhattanDistance() => Assert.ThrowsAsync(base.ManhattanDistance); + + public new class Fixture() : DistanceFunctionTests.Fixture + { + public override TestStore TestStore => AzureAISearchTestStore.Instance; + + // AzureAISearch does not return the expected standard mathematical result for each distance function + public override bool AssertScores => false; + } +} diff --git a/MEVD/test/AzureAISearch.ConformanceTests/AzureAISearchEmbeddingGenerationTests.cs b/MEVD/test/AzureAISearch.ConformanceTests/AzureAISearchEmbeddingGenerationTests.cs new file mode 100644 index 0000000..e09f121 --- /dev/null +++ b/MEVD/test/AzureAISearch.ConformanceTests/AzureAISearchEmbeddingGenerationTests.cs @@ -0,0 +1,64 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using AzureAISearch.ConformanceTests.Support; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.VectorData; +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace AzureAISearch.ConformanceTests; + +public class AzureAISearchEmbeddingGenerationTests(AzureAISearchEmbeddingGenerationTests.StringVectorFixture stringVectorFixture, AzureAISearchEmbeddingGenerationTests.RomOfFloatVectorFixture romOfFloatVectorFixture) + : EmbeddingGenerationTests(stringVectorFixture, romOfFloatVectorFixture), IClassFixture, IClassFixture +{ + // SearchAsync without a generator delegates to the service for AzureAISearch + public override Task SearchAsync_string_without_generator_throws() + => Task.CompletedTask; + + public new class StringVectorFixture : EmbeddingGenerationTests.StringVectorFixture + { + public override TestStore TestStore => AzureAISearchTestStore.Instance; + + public override VectorStore CreateVectorStore(IEmbeddingGenerator? embeddingGenerator) + => AzureAISearchTestStore.Instance.GetVectorStore(new() { EmbeddingGenerator = embeddingGenerator }); + + public override Func[] DependencyInjectionStoreRegistrationDelegates => + [ + services => services + .AddSingleton(AzureAISearchTestStore.Instance.Client) + .AddAzureAISearchVectorStore() + ]; + + public override Func[] DependencyInjectionCollectionRegistrationDelegates => + [ + services => services + .AddSingleton(AzureAISearchTestStore.Instance.Client) + .AddAzureAISearchCollection(this.CollectionName) + ]; + } + + public new class RomOfFloatVectorFixture : EmbeddingGenerationTests.RomOfFloatVectorFixture + { + public override TestStore TestStore => AzureAISearchTestStore.Instance; + + public override VectorStore CreateVectorStore(IEmbeddingGenerator? embeddingGenerator) + => AzureAISearchTestStore.Instance.GetVectorStore(new() { EmbeddingGenerator = embeddingGenerator }); + + public override Func[] DependencyInjectionStoreRegistrationDelegates => + [ + services => services + .AddSingleton(AzureAISearchTestStore.Instance.Client) + .AddAzureAISearchVectorStore() + ]; + + public override Func[] DependencyInjectionCollectionRegistrationDelegates => + [ + services => services + .AddSingleton(AzureAISearchTestStore.Instance.Client) + .AddAzureAISearchCollection(this.CollectionName) + ]; + } +} diff --git a/MEVD/test/AzureAISearch.ConformanceTests/AzureAISearchFilterTests.cs b/MEVD/test/AzureAISearch.ConformanceTests/AzureAISearchFilterTests.cs new file mode 100644 index 0000000..1932450 --- /dev/null +++ b/MEVD/test/AzureAISearch.ConformanceTests/AzureAISearchFilterTests.cs @@ -0,0 +1,22 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using AzureAISearch.ConformanceTests.Support; +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace AzureAISearch.ConformanceTests; + +public class AzureAISearchFilterTests(AzureAISearchFilterTests.Fixture fixture) + : FilterTests(fixture), IClassFixture +{ + // Azure AI Search only supports search.in() over strings + public override Task Contains_over_inline_int_array() + => Assert.ThrowsAsync(() => base.Contains_over_inline_int_array()); + + public new class Fixture : FilterTests.Fixture + { + public override TestStore TestStore => AzureAISearchTestStore.Instance; + } +} diff --git a/MEVD/test/AzureAISearch.ConformanceTests/AzureAISearchHybridSearchTests.cs b/MEVD/test/AzureAISearch.ConformanceTests/AzureAISearchHybridSearchTests.cs new file mode 100644 index 0000000..61b23b7 --- /dev/null +++ b/MEVD/test/AzureAISearch.ConformanceTests/AzureAISearchHybridSearchTests.cs @@ -0,0 +1,27 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using AzureAISearch.ConformanceTests.Support; +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace AzureAISearch.ConformanceTests; + +public class AzureAISearchHybridSearchTests( + AzureAISearchHybridSearchTests.VectorAndStringFixture vectorAndStringFixture, + AzureAISearchHybridSearchTests.MultiTextFixture multiTextFixture) + : HybridSearchTests(vectorAndStringFixture, multiTextFixture), + IClassFixture, + IClassFixture +{ + public new class VectorAndStringFixture : HybridSearchTests.VectorAndStringFixture + { + public override TestStore TestStore => AzureAISearchTestStore.Instance; + } + + public new class MultiTextFixture : HybridSearchTests.MultiTextFixture + { + public override TestStore TestStore => AzureAISearchTestStore.Instance; + } +} diff --git a/MEVD/test/AzureAISearch.ConformanceTests/AzureAISearchIndexKindTests.cs b/MEVD/test/AzureAISearch.ConformanceTests/AzureAISearchIndexKindTests.cs new file mode 100644 index 0000000..42ae43d --- /dev/null +++ b/MEVD/test/AzureAISearch.ConformanceTests/AzureAISearchIndexKindTests.cs @@ -0,0 +1,23 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using AzureAISearch.ConformanceTests.Support; +using Microsoft.Extensions.VectorData; +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace AzureAISearch.ConformanceTests; + +public class AzureAISearchIndexKindTests(AzureAISearchIndexKindTests.Fixture fixture) + : IndexKindTests(fixture), IClassFixture +{ + [Fact] + public virtual Task Hnsw() + => this.Test(IndexKind.Hnsw); + + public new class Fixture() : IndexKindTests.Fixture + { + public override TestStore TestStore => AzureAISearchTestStore.Instance; + } +} diff --git a/MEVD/test/AzureAISearch.ConformanceTests/AzureAISearchTestSuiteImplementationTests.cs b/MEVD/test/AzureAISearch.ConformanceTests/AzureAISearchTestSuiteImplementationTests.cs new file mode 100644 index 0000000..d37e7d3 --- /dev/null +++ b/MEVD/test/AzureAISearch.ConformanceTests/AzureAISearchTestSuiteImplementationTests.cs @@ -0,0 +1,8 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using VectorData.ConformanceTests; + +namespace AzureAISearch.ConformanceTests; + +public class AzureAISearchTestSuiteImplementationTests : TestSuiteImplementationTests; diff --git a/MEVD/test/AzureAISearch.ConformanceTests/ModelTests/AzureAISearchBasicModelTests.cs b/MEVD/test/AzureAISearch.ConformanceTests/ModelTests/AzureAISearchBasicModelTests.cs new file mode 100644 index 0000000..5937a6e --- /dev/null +++ b/MEVD/test/AzureAISearch.ConformanceTests/ModelTests/AzureAISearchBasicModelTests.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using AzureAISearch.ConformanceTests.Support; +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace AzureAISearch.ConformanceTests.ModelTests; + +public class AzureAISearchBasicModelTests(AzureAISearchBasicModelTests.Fixture fixture) + : BasicModelTests(fixture), IClassFixture +{ + public new class Fixture : BasicModelTests.Fixture + { + public override TestStore TestStore => AzureAISearchTestStore.Instance; + } +} diff --git a/MEVD/test/AzureAISearch.ConformanceTests/ModelTests/AzureAISearchDynamicModelTests.cs b/MEVD/test/AzureAISearch.ConformanceTests/ModelTests/AzureAISearchDynamicModelTests.cs new file mode 100644 index 0000000..990f9cd --- /dev/null +++ b/MEVD/test/AzureAISearch.ConformanceTests/ModelTests/AzureAISearchDynamicModelTests.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using AzureAISearch.ConformanceTests.Support; +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace AzureAISearch.ConformanceTests.ModelTests; + +public class AzureAISearchDynamicModelTests(AzureAISearchDynamicModelTests.Fixture fixture) + : DynamicModelTests(fixture), IClassFixture +{ + public new class Fixture : DynamicModelTests.Fixture + { + public override TestStore TestStore => AzureAISearchTestStore.Instance; + } +} diff --git a/MEVD/test/AzureAISearch.ConformanceTests/ModelTests/AzureAISearchMultiVectorModelTests.cs b/MEVD/test/AzureAISearch.ConformanceTests/ModelTests/AzureAISearchMultiVectorModelTests.cs new file mode 100644 index 0000000..c8d3b1e --- /dev/null +++ b/MEVD/test/AzureAISearch.ConformanceTests/ModelTests/AzureAISearchMultiVectorModelTests.cs @@ -0,0 +1,20 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using AzureAISearch.ConformanceTests.Support; +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace AzureAISearch.ConformanceTests.ModelTests; + +public class AzureAISearchMultiVectorModelTests(AzureAISearchMultiVectorModelTests.Fixture fixture) + : MultiVectorModelTests(fixture), IClassFixture +{ + public new class Fixture : MultiVectorModelTests.Fixture + { + public override string CollectionName => "multi-vector-" + AzureAISearchTestEnvironment.TestIndexPostfix; + + public override TestStore TestStore => AzureAISearchTestStore.Instance; + } +} diff --git a/MEVD/test/AzureAISearch.ConformanceTests/ModelTests/AzureAISearchNoDataModelTests.cs b/MEVD/test/AzureAISearch.ConformanceTests/ModelTests/AzureAISearchNoDataModelTests.cs new file mode 100644 index 0000000..7128631 --- /dev/null +++ b/MEVD/test/AzureAISearch.ConformanceTests/ModelTests/AzureAISearchNoDataModelTests.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using AzureAISearch.ConformanceTests.Support; +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace AzureAISearch.ConformanceTests.ModelTests; + +public class AzureAISearchNoDataModelTests(AzureAISearchNoDataModelTests.Fixture fixture) + : NoDataModelTests(fixture), IClassFixture +{ + public new class Fixture : NoDataModelTests.Fixture + { + public override TestStore TestStore => AzureAISearchTestStore.Instance; + } +} diff --git a/MEVD/test/AzureAISearch.ConformanceTests/ModelTests/AzureAISearchNoVectorModelTests.cs b/MEVD/test/AzureAISearch.ConformanceTests/ModelTests/AzureAISearchNoVectorModelTests.cs new file mode 100644 index 0000000..ab7cd6f --- /dev/null +++ b/MEVD/test/AzureAISearch.ConformanceTests/ModelTests/AzureAISearchNoVectorModelTests.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using AzureAISearch.ConformanceTests.Support; +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace AzureAISearch.ConformanceTests.ModelTests; + +public class AzureAISearchNoVectorModelTests(AzureAISearchNoVectorModelTests.Fixture fixture) + : NoVectorModelTests(fixture), IClassFixture +{ + public new class Fixture : NoVectorModelTests.Fixture + { + public override TestStore TestStore => AzureAISearchTestStore.Instance; + } +} diff --git a/MEVD/test/AzureAISearch.ConformanceTests/Properties/AssemblyAttributes.cs b/MEVD/test/AzureAISearch.ConformanceTests/Properties/AssemblyAttributes.cs new file mode 100644 index 0000000..6571e2d --- /dev/null +++ b/MEVD/test/AzureAISearch.ConformanceTests/Properties/AssemblyAttributes.cs @@ -0,0 +1,6 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Xunit; + +[assembly: CollectionBehavior(DisableTestParallelization = true)] diff --git a/MEVD/test/AzureAISearch.ConformanceTests/Support/AzureAISearchAllTypes.cs b/MEVD/test/AzureAISearch.ConformanceTests/Support/AzureAISearchAllTypes.cs new file mode 100644 index 0000000..5436c46 --- /dev/null +++ b/MEVD/test/AzureAISearch.ConformanceTests/Support/AzureAISearchAllTypes.cs @@ -0,0 +1,169 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.VectorData; +using Xunit; + +namespace AzureAISearch.ConformanceTests.Support; + +#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable. + +public class AzureAISearchAllTypes +{ + [VectorStoreKey] + public string Id { get; set; } + + [VectorStoreData] + public bool BoolProperty { get; set; } + [VectorStoreData] + public bool? NullableBoolProperty { get; set; } + [VectorStoreData] + public string StringProperty { get; set; } + [VectorStoreData] + public string? NullableStringProperty { get; set; } + [VectorStoreData] + public int IntProperty { get; set; } + [VectorStoreData] + public int? NullableIntProperty { get; set; } + [VectorStoreData] + public long LongProperty { get; set; } + [VectorStoreData] + public long? NullableLongProperty { get; set; } + [VectorStoreData] + public float FloatProperty { get; set; } + [VectorStoreData] + public float? NullableFloatProperty { get; set; } + [VectorStoreData] + public double DoubleProperty { get; set; } + [VectorStoreData] + public double? NullableDoubleProperty { get; set; } + [VectorStoreData] + public DateTimeOffset DateTimeOffsetProperty { get; set; } + [VectorStoreData] + public DateTimeOffset? NullableDateTimeOffsetProperty { get; set; } + + [VectorStoreData] + public string[] StringArray { get; set; } + [VectorStoreData] + public List StringList { get; set; } + [VectorStoreData] + public bool[] BoolArray { get; set; } + [VectorStoreData] + public List BoolList { get; set; } + [VectorStoreData] + public int[] IntArray { get; set; } + [VectorStoreData] + public List IntList { get; set; } + [VectorStoreData] + public long[] LongArray { get; set; } + [VectorStoreData] + public List LongList { get; set; } + [VectorStoreData] + public float[] FloatArray { get; set; } + [VectorStoreData] + public List FloatList { get; set; } + [VectorStoreData] + public double[] DoubleArray { get; set; } + [VectorStoreData] + public List DoubleList { get; set; } + [VectorStoreData] + public DateTimeOffset[] DateTimeOffsetArray { get; set; } + [VectorStoreData] + public List DateTimeOffsetList { get; set; } + + [VectorStoreVector(dimensions: 8, DistanceFunction = DistanceFunction.DotProductSimilarity)] + public ReadOnlyMemory? Embedding { get; set; } + + internal void AssertEqual(AzureAISearchAllTypes other) + { + Assert.Equal(this.Id, other.Id); + Assert.Equal(this.BoolProperty, other.BoolProperty); + Assert.Equal(this.NullableBoolProperty, other.NullableBoolProperty); + Assert.Equal(this.StringProperty, other.StringProperty); + Assert.Equal(this.NullableStringProperty, other.NullableStringProperty); + Assert.Equal(this.IntProperty, other.IntProperty); + Assert.Equal(this.NullableIntProperty, other.NullableIntProperty); + Assert.Equal(this.LongProperty, other.LongProperty); + Assert.Equal(this.NullableLongProperty, other.NullableLongProperty); + Assert.Equal(this.FloatProperty, other.FloatProperty); + Assert.Equal(this.NullableFloatProperty, other.NullableFloatProperty); + Assert.Equal(this.DoubleProperty, other.DoubleProperty); + Assert.Equal(this.NullableDoubleProperty, other.NullableDoubleProperty); + AssertEqual(this.DateTimeOffsetProperty, other.DateTimeOffsetProperty); + Assert.Equal(this.NullableDateTimeOffsetProperty.HasValue, other.NullableDateTimeOffsetProperty.HasValue); + if (this.NullableDateTimeOffsetProperty.HasValue && other.NullableDateTimeOffsetProperty.HasValue) + { + AssertEqual(this.NullableDateTimeOffsetProperty.Value, other.NullableDateTimeOffsetProperty.Value); + } + + Assert.Equal(this.StringArray, other.StringArray); + Assert.Equal(this.StringList, other.StringList); + Assert.Equal(this.BoolArray, other.BoolArray); + Assert.Equal(this.BoolList, other.BoolList); + Assert.Equal(this.IntArray, other.IntArray); + Assert.Equal(this.IntList, other.IntList); + Assert.Equal(this.LongArray, other.LongArray); + Assert.Equal(this.LongList, other.LongList); + Assert.Equal(this.FloatArray, other.FloatArray); + Assert.Equal(this.FloatList, other.FloatList); + Assert.Equal(this.DoubleArray, other.DoubleArray); + Assert.Equal(this.DoubleList, other.DoubleList); + Assert.Equal(this.DateTimeOffsetArray.Length, other.DateTimeOffsetArray.Length); + for (int i = 0; i < this.DateTimeOffsetArray.Length; i++) + { + AssertEqual(this.DateTimeOffsetArray[i], other.DateTimeOffsetArray[i]); + } + Assert.Equal(this.DateTimeOffsetList.Count, other.DateTimeOffsetList.Count); + for (int i = 0; i < this.DateTimeOffsetList.Count; i++) + { + AssertEqual(this.DateTimeOffsetList[i], other.DateTimeOffsetList[i]); + } + + Assert.Equal(this.Embedding!.Value.ToArray(), other.Embedding!.Value.ToArray()); + + static void AssertEqual(DateTimeOffset expected, DateTimeOffset actual) + { + Assert.Equal(expected, actual, TimeSpan.FromSeconds(0.01)); + } + } + + internal static VectorStoreCollectionDefinition GetRecordDefinition() + => new() + { + Properties = + [ + new VectorStoreKeyProperty(nameof(AzureAISearchAllTypes.Id), typeof(string)), + new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.BoolProperty), typeof(bool)), + new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.NullableBoolProperty), typeof(bool?)), + new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.StringProperty), typeof(string)), + new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.NullableStringProperty), typeof(string)), + new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.IntProperty), typeof(int)), + new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.NullableIntProperty), typeof(int?)), + new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.LongProperty), typeof(long)), + new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.NullableLongProperty), typeof(long?)), + new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.FloatProperty), typeof(float)), + new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.NullableFloatProperty), typeof(float?)), + new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.DoubleProperty), typeof(double)), + new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.NullableDoubleProperty), typeof(double?)), + new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.DateTimeOffsetProperty), typeof(DateTimeOffset)), + new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.NullableDateTimeOffsetProperty), typeof(DateTimeOffset?)), + + new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.StringArray), typeof(string[])), + new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.StringList), typeof(List)), + new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.BoolArray), typeof(bool[])), + new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.BoolList), typeof(List)), + new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.IntArray), typeof(int[])), + new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.IntList), typeof(List)), + new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.LongArray), typeof(long[])), + new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.LongList), typeof(List)), + new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.FloatArray), typeof(float[])), + new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.FloatList), typeof(List)), + new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.DoubleArray), typeof(double[])), + new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.DoubleList), typeof(List)), + new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.DateTimeOffsetArray), typeof(DateTimeOffset[])), + new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.DateTimeOffsetList), typeof(List)), + + new VectorStoreVectorProperty(nameof(AzureAISearchAllTypes.Embedding), typeof(ReadOnlyMemory?), 8) { DistanceFunction = Microsoft.Extensions.VectorData.DistanceFunction.DotProductSimilarity } + ] + }; +} diff --git a/MEVD/test/AzureAISearch.ConformanceTests/Support/AzureAISearchFixture.cs b/MEVD/test/AzureAISearch.ConformanceTests/Support/AzureAISearchFixture.cs new file mode 100644 index 0000000..5b897e9 --- /dev/null +++ b/MEVD/test/AzureAISearch.ConformanceTests/Support/AzureAISearchFixture.cs @@ -0,0 +1,11 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using VectorData.ConformanceTests.Support; + +namespace AzureAISearch.ConformanceTests.Support; + +public class AzureAISearchFixture : VectorStoreFixture +{ + public override TestStore TestStore => AzureAISearchTestStore.Instance; +} diff --git a/MEVD/test/AzureAISearch.ConformanceTests/Support/AzureAISearchTestEnvironment.cs b/MEVD/test/AzureAISearch.ConformanceTests/Support/AzureAISearchTestEnvironment.cs new file mode 100644 index 0000000..fad914f --- /dev/null +++ b/MEVD/test/AzureAISearch.ConformanceTests/Support/AzureAISearchTestEnvironment.cs @@ -0,0 +1,34 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text.RegularExpressions; +using Microsoft.Extensions.Configuration; + +namespace AzureAISearch.ConformanceTests.Support; + +#pragma warning disable CA1810 // Initialize all static fields when those fields are declared + +internal static class AzureAISearchTestEnvironment +{ +#pragma warning disable CA1308 // Normalize strings to uppercase + public static readonly string TestIndexPostfix = '-' + new Regex("[^a-zA-Z0-9]").Replace(Environment.MachineName.ToLowerInvariant(), ""); +#pragma warning restore CA1308 // Normalize strings to uppercase + + public static readonly string? ServiceUrl, ApiKey; + + public static bool IsConnectionInfoDefined => ServiceUrl is not null; + + static AzureAISearchTestEnvironment() + { + var configuration = new ConfigurationBuilder() + .AddJsonFile(path: "testsettings.json", optional: true) + .AddJsonFile(path: "testsettings.development.json", optional: true) + .AddEnvironmentVariables() + .AddUserSecrets() + .Build(); + + var azureAISearchSection = configuration.GetSection("AzureAISearch"); + ServiceUrl = azureAISearchSection?["ServiceUrl"]; + ApiKey = azureAISearchSection?["ApiKey"]; + } +} diff --git a/MEVD/test/AzureAISearch.ConformanceTests/Support/AzureAISearchTestStore.cs b/MEVD/test/AzureAISearch.ConformanceTests/Support/AzureAISearchTestStore.cs new file mode 100644 index 0000000..4ac1a73 --- /dev/null +++ b/MEVD/test/AzureAISearch.ConformanceTests/Support/AzureAISearchTestStore.cs @@ -0,0 +1,109 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Linq.Expressions; +using Azure; +using Azure.Identity; +using Azure.Search.Documents.Indexes; +using Humanizer; +using Microsoft.Extensions.VectorData; +using CommunityToolkit.VectorData.AzureAISearch; +using VectorData.ConformanceTests.Support; + +namespace AzureAISearch.ConformanceTests.Support; + +internal sealed class AzureAISearchTestStore : TestStore +{ + public static AzureAISearchTestStore Instance { get; } = new(); + + private SearchIndexClient? _client; + + // Track all collection names created through this TestStore, so we can clean them up in StopAsync. + // This is needed because Azure AI Search free/basic tiers have a low index limit (e.g. 3), + // so we must delete indexes between test classes. + private readonly HashSet _trackedCollectionNames = new(); + + public SearchIndexClient Client + => this._client ?? throw new InvalidOperationException("Call InitializeAsync() first"); + + public AzureAISearchVectorStore GetVectorStore(AzureAISearchVectorStoreOptions options) + => new(this.Client, options); + + private AzureAISearchTestStore() + { + } + + public override VectorStoreCollection CreateCollection( + string name, + VectorStoreCollectionDefinition definition) + { + _trackedCollectionNames.Add(name); + return base.CreateCollection(name, definition); + } + + public override VectorStoreCollection> CreateDynamicCollection( + string name, + VectorStoreCollectionDefinition definition) + { + _trackedCollectionNames.Add(name); + return base.CreateDynamicCollection(name, definition); + } + + protected override Task StartAsync() + { + (string? serviceUrl, string? apiKey) = (AzureAISearchTestEnvironment.ServiceUrl, AzureAISearchTestEnvironment.ApiKey); + + if (string.IsNullOrWhiteSpace(serviceUrl)) + { + throw new InvalidOperationException("Service URL is not configured, set AzureAISearch:ServiceUrl (and AzureAISearch:ApiKey if you want)"); + } + + this._client = string.IsNullOrWhiteSpace(apiKey) + ? new SearchIndexClient(new Uri(serviceUrl), new DefaultAzureCredential()) + : new SearchIndexClient(new Uri(serviceUrl), new AzureKeyCredential(apiKey!)); + + this.DefaultVectorStore = new AzureAISearchVectorStore(this._client); + + return Task.CompletedTask; + } + + protected override async Task StopAsync() + { + // Delete all tracked indexes to stay within the Azure AI Search index limit. + // With parallelism disabled, StopAsync is called between each test class (refcount goes 1→0). + foreach (var name in _trackedCollectionNames) + { + try + { + await Client.DeleteIndexAsync(name); + } + catch + { + // Best-effort cleanup; ignore failures (e.g. index already deleted by the test) + } + } + + _trackedCollectionNames.Clear(); + } + + // Azure AI search only supports lowercase letters, digits or dashes. + // Also, add a suffix containing machine name to allow multiple developers to work against the same cloud instance. + public override string AdjustCollectionName(string baseName) + => baseName.Kebaberize() + AzureAISearchTestEnvironment.TestIndexPostfix; + + public override async Task WaitForDataAsync( + VectorStoreCollection collection, + int recordCount, + Expression>? filter = null, + Expression>? vectorProperty = null, + int? vectorSize = null, + object? dummyVector = null) + { + await base.WaitForDataAsync(collection, recordCount, filter, vectorProperty, vectorSize, dummyVector); + + // There seems to be some asynchronicity/race condition specific to Azure AI Search which isn't taken care + // of by the generic retry loop in the base implementation. + // TODO: Investigate this and remove + await Task.Delay(TimeSpan.FromMilliseconds(1000)); + } +} diff --git a/MEVD/test/AzureAISearch.ConformanceTests/TypeTests/AzureAISearchDataTypeTests.cs b/MEVD/test/AzureAISearch.ConformanceTests/TypeTests/AzureAISearchDataTypeTests.cs new file mode 100644 index 0000000..6f75247 --- /dev/null +++ b/MEVD/test/AzureAISearch.ConformanceTests/TypeTests/AzureAISearchDataTypeTests.cs @@ -0,0 +1,78 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using AzureAISearch.ConformanceTests.Support; +using Microsoft.Extensions.VectorData; +using VectorData.ConformanceTests.Support; +using VectorData.ConformanceTests.TypeTests; +using Xunit; + +namespace AzureAISearch.ConformanceTests.TypeTests; + +public class AzureAISearchDataTypeTests(AzureAISearchDataTypeTests.Fixture fixture) + : DataTypeTests(fixture), + IClassFixture +{ + [Fact(Skip = "Issues around empty collection initialization")] + public override Task String_array() => Task.CompletedTask; + + protected override object? GenerateEmptyProperty(VectorStoreProperty property) + => property.Type switch + { + null => throw new InvalidOperationException($"Property '{property.Name}' has no type defined."), + + // In Azure AI Search, array fields must be non-null (at least for now) + var t when t.IsArray => Array.CreateInstance(t.GetElementType()!, 0), + + _ => base.GenerateEmptyProperty(property) + }; + + public new class Fixture : DataTypeTests.Fixture + { + public override TestStore TestStore => AzureAISearchTestStore.Instance; + + public override IList GetDataProperties() + => base.GetDataProperties().Where(p => + p.Type != typeof(byte) + && p.Type != typeof(short) + && p.Type != typeof(decimal) + && p.Type != typeof(Guid) +#if NET + && p.Type != typeof(TimeOnly) +#endif + ).ToList(); + + public override Type[] UnsupportedDefaultTypes { get; } = + [ + typeof(byte), + typeof(short), + typeof(decimal), + typeof(Guid), +#if NET + typeof(TimeOnly) +#endif + ]; + + public class AzureAISearchRecord : RecordBase + { + public int Int { get; set; } + public long Long { get; set; } + public float Float { get; set; } + public double Double { get; set; } + + public string? String { get; set; } + public bool Bool { get; set; } + + public DateTime DateTime { get; set; } + public DateTimeOffset DateTimeOffset { get; set; } + +#if NET + public DateOnly DateOnly { get; set; } +#endif + + public string[] StringArray { get; set; } = null!; + + public int? NullableInt { get; set; } + } + } +} diff --git a/MEVD/test/AzureAISearch.ConformanceTests/TypeTests/AzureAISearchEmbeddingTypeTests.cs b/MEVD/test/AzureAISearch.ConformanceTests/TypeTests/AzureAISearchEmbeddingTypeTests.cs new file mode 100644 index 0000000..00bbace --- /dev/null +++ b/MEVD/test/AzureAISearch.ConformanceTests/TypeTests/AzureAISearchEmbeddingTypeTests.cs @@ -0,0 +1,20 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using AzureAISearch.ConformanceTests.Support; +using VectorData.ConformanceTests.Support; +using VectorData.ConformanceTests.TypeTests; +using Xunit; + +#pragma warning disable CA2000 // Dispose objects before losing scope + +namespace AzureAISearch.ConformanceTests.TypeTests; + +public class AzureAISearchEmbeddingTypeTests(AzureAISearchEmbeddingTypeTests.Fixture fixture) + : EmbeddingTypeTests(fixture), IClassFixture +{ + public new class Fixture : EmbeddingTypeTests.Fixture + { + public override TestStore TestStore => AzureAISearchTestStore.Instance; + } +} diff --git a/MEVD/test/AzureAISearch.ConformanceTests/TypeTests/AzureAISearchKeyTypeTests.cs b/MEVD/test/AzureAISearch.ConformanceTests/TypeTests/AzureAISearchKeyTypeTests.cs new file mode 100644 index 0000000..d564425 --- /dev/null +++ b/MEVD/test/AzureAISearch.ConformanceTests/TypeTests/AzureAISearchKeyTypeTests.cs @@ -0,0 +1,21 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using AzureAISearch.ConformanceTests.Support; +using VectorData.ConformanceTests.Support; +using VectorData.ConformanceTests.TypeTests; +using Xunit; + +namespace AzureAISearch.ConformanceTests.TypeTests; + +public class AzureAISearchKeyTypeTests(AzureAISearchKeyTypeTests.Fixture fixture) + : KeyTypeTests(fixture), IClassFixture +{ + [Fact] + public virtual Task String() => this.Test("foo", "bar"); + + public new class Fixture : KeyTypeTests.Fixture + { + public override TestStore TestStore => AzureAISearchTestStore.Instance; + } +} diff --git a/MEVD/test/AzureAISearch.UnitTests/.editorconfig b/MEVD/test/AzureAISearch.UnitTests/.editorconfig new file mode 100644 index 0000000..394eef6 --- /dev/null +++ b/MEVD/test/AzureAISearch.UnitTests/.editorconfig @@ -0,0 +1,6 @@ +# Suppressing errors for Test projects under dotnet folder +[*.cs] +dotnet_diagnostic.CA2007.severity = none # Do not directly await a Task +dotnet_diagnostic.VSTHRD111.severity = none # Use .ConfigureAwait(bool) is hidden by default, set to none to prevent IDE from changing on autosave +dotnet_diagnostic.CS1591.severity = none # Missing XML comment for publicly visible type or member +dotnet_diagnostic.IDE1006.severity = warning # Naming rule violations diff --git a/MEVD/test/AzureAISearch.UnitTests/AzureAISearch.UnitTests.csproj b/MEVD/test/AzureAISearch.UnitTests/AzureAISearch.UnitTests.csproj new file mode 100644 index 0000000..85b2931 --- /dev/null +++ b/MEVD/test/AzureAISearch.UnitTests/AzureAISearch.UnitTests.csproj @@ -0,0 +1,38 @@ + + + + CommunityToolkit.VectorData.AzureAISearch.UnitTests + CommunityToolkit.VectorData.AzureAISearch.UnitTests + net10.0 + true + enable + disable + false + $(NoWarn);SKEXP0001 + $(NoWarn);MEVD9001 + + + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + + + + + diff --git a/MEVD/test/AzureAISearch.UnitTests/AzureAISearchCollectionCreateMappingTests.cs b/MEVD/test/AzureAISearch.UnitTests/AzureAISearchCollectionCreateMappingTests.cs new file mode 100644 index 0000000..4479549 --- /dev/null +++ b/MEVD/test/AzureAISearch.UnitTests/AzureAISearchCollectionCreateMappingTests.cs @@ -0,0 +1,223 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using Azure.Search.Documents.Indexes.Models; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; +using CommunityToolkit.VectorData.AzureAISearch; +using Xunit; + +namespace SemanticKernel.Connectors.AzureAISearch.UnitTests; + +/// +/// Contains tests for the class. +/// +public class AzureAISearchCollectionCreateMappingTests +{ + [Fact] + public void MapKeyFieldCreatesSearchableField() + { + // Arrange + var keyProperty = new KeyPropertyModel("testkey", typeof(string)) { StorageName = "test_key" }; + + // Act + var result = AzureAISearchCollectionCreateMapping.MapKeyField(keyProperty); + + // Assert + Assert.NotNull(result); + Assert.Equal("test_key", result.Name); + Assert.True(result.IsKey); + Assert.True(result.IsFilterable); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void MapFilterableStringDataFieldCreatesSimpleField(bool isFilterable) + { + // Arrange + var dataProperty = new DataPropertyModel("testdata", typeof(string)) + { + IsIndexed = isFilterable, + StorageName = "test_data" + }; + + // Act + var result = AzureAISearchCollectionCreateMapping.MapDataField(dataProperty); + + // Assert + Assert.NotNull(result); + Assert.IsType(result); + Assert.Equal("test_data", result.Name); + Assert.False(result.IsKey); + Assert.Equal(isFilterable, result.IsFilterable); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void MapFullTextSearchableStringDataFieldCreatesSearchableField(bool isFilterable) + { + // Arrange + var dataProperty = new DataPropertyModel("testdata", typeof(string)) + { + IsIndexed = isFilterable, + IsFullTextIndexed = true, + StorageName = "test_data" + }; + + // Act + var result = AzureAISearchCollectionCreateMapping.MapDataField(dataProperty); + + // Assert + Assert.NotNull(result); + Assert.IsType(result); + Assert.Equal("test_data", result.Name); + Assert.False(result.IsKey); + Assert.Equal(isFilterable, result.IsFilterable); + } + + [Fact] + public void MapFullTextSearchableStringDataFieldThrowsForInvalidType() + { + // Arrange + var dataProperty = new DataPropertyModel("testdata", typeof(int)) + { + IsFullTextIndexed = true, + StorageName = "test_data" + }; + + // Act & Assert + Assert.Throws(() => AzureAISearchCollectionCreateMapping.MapDataField(dataProperty)); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void MapDataFieldCreatesSimpleField(bool isFilterable) + { + // Arrange + var dataProperty = new DataPropertyModel("testdata", typeof(int)) + { + IsIndexed = isFilterable, + StorageName = "test_data" + }; + + // Act + var result = AzureAISearchCollectionCreateMapping.MapDataField(dataProperty); + + // Assert + Assert.NotNull(result); + Assert.IsType(result); + Assert.Equal("test_data", result.Name); + Assert.Equal(SearchFieldDataType.Int32, result.Type); + Assert.False(result.IsKey); + Assert.Equal(isFilterable, result.IsFilterable); + } + + [Fact] + public void MapVectorFieldCreatesVectorSearchField() + { + // Arrange + var vectorProperty = new VectorPropertyModel("testvector", typeof(ReadOnlyMemory)) + { + Dimensions = 10, + IndexKind = IndexKind.Flat, + DistanceFunction = DistanceFunction.DotProductSimilarity, + StorageName = "test_vector" + }; + + // Act + var (vectorSearchField, algorithmConfiguration, vectorSearchProfile) = AzureAISearchCollectionCreateMapping.MapVectorField(vectorProperty); + + // Assert + Assert.NotNull(vectorSearchField); + Assert.NotNull(algorithmConfiguration); + Assert.NotNull(vectorSearchProfile); + Assert.Equal("test_vector", vectorSearchField.Name); + Assert.Equal(vectorProperty.Dimensions, vectorSearchField.VectorSearchDimensions); + + Assert.Equal("test_vectorAlgoConfig", algorithmConfiguration.Name); + Assert.IsType(algorithmConfiguration); + var flatConfig = algorithmConfiguration as ExhaustiveKnnAlgorithmConfiguration; + Assert.Equal(VectorSearchAlgorithmMetric.DotProduct, flatConfig!.Parameters.Metric); + + Assert.Equal("test_vectorProfile", vectorSearchProfile.Name); + Assert.Equal("test_vectorAlgoConfig", vectorSearchProfile.AlgorithmConfigurationName); + } + + [Theory] + [InlineData(IndexKind.Hnsw, typeof(HnswAlgorithmConfiguration))] + [InlineData(IndexKind.Flat, typeof(ExhaustiveKnnAlgorithmConfiguration))] + public void MapVectorFieldCreatesExpectedAlgoConfigTypes(string indexKind, Type algoConfigType) + { + // Arrange + var vectorProperty = new VectorPropertyModel("testvector", typeof(ReadOnlyMemory)) + { + Dimensions = 10, + IndexKind = indexKind, + DistanceFunction = DistanceFunction.DotProductSimilarity, + StorageName = "test_vector" + }; + + // Act + var (vectorSearchField, algorithmConfiguration, vectorSearchProfile) = AzureAISearchCollectionCreateMapping.MapVectorField(vectorProperty); + + // Assert + Assert.Equal("test_vectorAlgoConfig", algorithmConfiguration.Name); + Assert.Equal(algoConfigType, algorithmConfiguration.GetType()); + } + + [Fact] + public void MapVectorFieldDefaultsToHsnwAndCosine() + { + // Arrange + var vectorProperty = new VectorPropertyModel("testvector", typeof(ReadOnlyMemory)) { Dimensions = 10 }; + + // Act + var (vectorSearchField, algorithmConfiguration, vectorSearchProfile) = AzureAISearchCollectionCreateMapping.MapVectorField(vectorProperty); + + // Assert + Assert.IsType(algorithmConfiguration); + var hnswConfig = algorithmConfiguration as HnswAlgorithmConfiguration; + Assert.Equal(VectorSearchAlgorithmMetric.Cosine, hnswConfig!.Parameters.Metric); + } + + [Fact] + public void MapVectorFieldThrowsForUnsupportedDistanceFunction() + { + // Arrange + var vectorProperty = new VectorPropertyModel("testvector", typeof(ReadOnlyMemory)) + { + Dimensions = 10, + DistanceFunction = DistanceFunction.ManhattanDistance, + }; + + // Act & Assert + Assert.Throws(() => AzureAISearchCollectionCreateMapping.MapVectorField(vectorProperty)); + } + + [Theory] + [MemberData(nameof(DataTypeMappingOptions))] + public void GetSDKFieldDataTypeMapsTypesCorrectly(Type propertyType, SearchFieldDataType searchFieldDataType) + { + // Act & Assert + Assert.Equal(searchFieldDataType, AzureAISearchCollectionCreateMapping.GetSDKFieldDataType(propertyType)); + } + + public static IEnumerable DataTypeMappingOptions() + { + yield return new object[] { typeof(string), SearchFieldDataType.String }; + yield return new object[] { typeof(bool), SearchFieldDataType.Boolean }; + yield return new object[] { typeof(int), SearchFieldDataType.Int32 }; + yield return new object[] { typeof(long), SearchFieldDataType.Int64 }; + yield return new object[] { typeof(float), SearchFieldDataType.Double }; + yield return new object[] { typeof(double), SearchFieldDataType.Double }; + yield return new object[] { typeof(DateTimeOffset), SearchFieldDataType.DateTimeOffset }; + + yield return new object[] { typeof(string[]), SearchFieldDataType.Collection(SearchFieldDataType.String) }; + yield return new object[] { typeof(List), SearchFieldDataType.Collection(SearchFieldDataType.String) }; + } +} diff --git a/MEVD/test/AzureAISearch.UnitTests/AzureAISearchCollectionTests.cs b/MEVD/test/AzureAISearch.UnitTests/AzureAISearchCollectionTests.cs new file mode 100644 index 0000000..5994f3e --- /dev/null +++ b/MEVD/test/AzureAISearch.UnitTests/AzureAISearchCollectionTests.cs @@ -0,0 +1,601 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; +using Azure; +using Azure.Search.Documents; +using Azure.Search.Documents.Indexes; +using Azure.Search.Documents.Indexes.Models; +using Azure.Search.Documents.Models; +using Microsoft.Extensions.VectorData; +using CommunityToolkit.VectorData.AzureAISearch; +using Moq; +using Xunit; + +namespace SemanticKernel.Connectors.AzureAISearch.UnitTests; + +/// +/// Contains tests for the class. +/// +public class AzureAISearchCollectionTests +{ + private const string TestCollectionName = "testcollection"; + private const string TestRecordKey1 = "testid1"; + private const string TestRecordKey2 = "testid2"; + + private readonly Mock _searchIndexClientMock; + private readonly Mock _searchClientMock; + + private readonly CancellationToken _testCancellationToken = new(false); + + public AzureAISearchCollectionTests() + { + this._searchClientMock = new Mock(MockBehavior.Strict); + this._searchIndexClientMock = new Mock(MockBehavior.Strict); + this._searchIndexClientMock.Setup(x => x.GetSearchClient(TestCollectionName)).Returns(this._searchClientMock.Object); + this._searchIndexClientMock.Setup(x => x.ServiceName).Returns("TestService"); + } + + [Theory] + [InlineData(TestCollectionName, true)] + [InlineData("nonexistentcollection", false)] + public async Task CollectionExistsReturnsCollectionStateAsync(string collectionName, bool expectedExists) + { + this._searchIndexClientMock.Setup(x => x.GetSearchClient(collectionName)).Returns(this._searchClientMock.Object); + + // Arrange. + if (expectedExists) + { + this._searchIndexClientMock + .Setup(x => x.GetIndexAsync(collectionName, this._testCancellationToken)) + .Returns(Task.FromResult?>(null)); + } + else + { + this._searchIndexClientMock + .Setup(x => x.GetIndexAsync(collectionName, this._testCancellationToken)) + .ThrowsAsync(new RequestFailedException(404, "Index not found")); + } + + using var sut = new AzureAISearchCollection(this._searchIndexClientMock.Object, collectionName); + + // Act. + var actual = await sut.CollectionExistsAsync(this._testCancellationToken); + + // Assert. + Assert.Equal(expectedExists, actual); + } + + [Theory] + [InlineData(true, true)] + [InlineData(true, false)] + [InlineData(false, true)] + [InlineData(false, false)] + public async Task EnsureCollectionExistsInvokesSDKAsync(bool useDefinition, bool expectedExists) + { + // Arrange. + if (expectedExists) + { + this._searchIndexClientMock + .Setup(x => x.GetIndexAsync(TestCollectionName, this._testCancellationToken)) + .Returns(Task.FromResult?>(null)); + } + else + { + this._searchIndexClientMock + .Setup(x => x.GetIndexAsync(TestCollectionName, this._testCancellationToken)) + .ThrowsAsync(new RequestFailedException(404, "Index not found")); + } + + this._searchIndexClientMock + .Setup(x => x.CreateIndexAsync(It.IsAny(), this._testCancellationToken)) + .ReturnsAsync(Response.FromValue(new SearchIndex(TestCollectionName), Mock.Of())); + + using var sut = this.CreateRecordCollection(useDefinition); + + // Act. + await sut.EnsureCollectionExistsAsync(); + + // Assert. + if (expectedExists) + { + this._searchIndexClientMock + .Verify( + x => x.CreateIndexAsync( + It.IsAny(), + this._testCancellationToken), + Times.Never); + } + else + { + this._searchIndexClientMock + .Verify( + x => x.CreateIndexAsync( + It.Is(si => si.Fields.Count == 5 && si.Name == TestCollectionName && si.VectorSearch.Profiles.Count == 2 && si.VectorSearch.Algorithms.Count == 2), + this._testCancellationToken), + Times.Once); + } + } + + [Fact] + public async Task CanDeleteCollectionAsync() + { + // Arrange. + this._searchIndexClientMock + .Setup(x => x.DeleteIndexAsync(TestCollectionName, this._testCancellationToken)) + .Returns(Task.FromResult(null)); + + using var sut = this.CreateRecordCollection(false); + + // Act. + await sut.EnsureCollectionDeletedAsync(this._testCancellationToken); + + // Assert. + this._searchIndexClientMock.Verify(x => x.DeleteIndexAsync(TestCollectionName, this._testCancellationToken), Times.Once); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task CanGetRecordWithVectorsAsync(bool useDefinition) + { + // Arrange. + this._searchClientMock.Setup( + x => x.GetDocumentAsync( + TestRecordKey1, + It.Is(x => !x.SelectedFields.Any()), + this._testCancellationToken)) + .ReturnsAsync(Response.FromValue(CreateJsonObjectModel(TestRecordKey1, true), Mock.Of())); + + using var sut = this.CreateRecordCollection(useDefinition); + + // Act. + var actual = await sut.GetAsync( + TestRecordKey1, + new() { IncludeVectors = true }, + this._testCancellationToken); + + // Assert. + Assert.NotNull(actual); + Assert.Equal(TestRecordKey1, actual.Key); + Assert.Equal("data 1", actual.Data1); + Assert.Equal("data 2", actual.Data2); + Assert.Equal(new float[] { 1, 2, 3, 4 }, actual.Vector1!.Value.ToArray()); + Assert.Equal(new float[] { 1, 2, 3, 4 }, actual.Vector2!.Value.ToArray()); + } + + [Theory] + [InlineData(true, true)] + [InlineData(true, false)] + [InlineData(false, true)] + [InlineData(false, false)] + public async Task CanGetRecordWithoutVectorsAsync(bool useDefinition, bool useCustomJsonSerializerOptions) + { + // Arrange. + var storageObject = JsonSerializer.SerializeToNode(CreateModel(TestRecordKey1, false))!.AsObject(); + + string[] expectedSelectFields = useCustomJsonSerializerOptions ? ["key", "storage_data1", "data2"] : ["Key", "storage_data1", "Data2"]; + this._searchClientMock.Setup( + x => x.GetDocumentAsync( + TestRecordKey1, + It.IsAny(), + this._testCancellationToken)) + .ReturnsAsync(Response.FromValue(CreateJsonObjectModel(TestRecordKey1, true, useCustomJsonSerializerOptions), Mock.Of())); + + using var sut = this.CreateRecordCollection(useDefinition, useCustomJsonSerializerOptions); + + // Act. + var actual = await sut.GetAsync( + TestRecordKey1, + new() { IncludeVectors = false }, + this._testCancellationToken); + + // Assert. + Assert.NotNull(actual); + Assert.Equal(TestRecordKey1, actual.Key); + Assert.Equal("data 1", actual.Data1); + Assert.Equal("data 2", actual.Data2); + + this._searchClientMock.Verify( + x => x.GetDocumentAsync( + TestRecordKey1, + It.Is(x => x.SelectedFields.SequenceEqual(expectedSelectFields)), + this._testCancellationToken), + Times.Once); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task CanGetManyRecordsWithVectorsAsync(bool useDefinition) + { + // Arrange. + this._searchClientMock.Setup( + x => x.GetDocumentAsync( + It.IsAny(), + It.IsAny(), + this._testCancellationToken)) + .ReturnsAsync((string id, GetDocumentOptions options, CancellationToken cancellationToken) => + { + return Response.FromValue(CreateJsonObjectModel(id, true), Mock.Of()); + }); + + using var sut = this.CreateRecordCollection(useDefinition); + + // Act. + var actual = await sut.GetAsync( + [TestRecordKey1, TestRecordKey2], + new() { IncludeVectors = true }, + this._testCancellationToken).ToListAsync(); + + // Assert. + Assert.NotNull(actual); + Assert.Equal(2, actual.Count); + Assert.Equal(TestRecordKey1, actual[0].Key); + Assert.Equal(TestRecordKey2, actual[1].Key); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task CanDeleteRecordAsync(bool useDefinition) + { + // Arrange. +#pragma warning disable Moq1002 // Moq: No matching constructor + var indexDocumentsResultMock = new Mock(MockBehavior.Strict, new List()); +#pragma warning restore Moq1002 // Moq: No matching constructor + + this._searchClientMock.Setup( + x => x.DeleteDocumentsAsync( + It.IsAny(), + It.IsAny>(), + It.IsAny(), + this._testCancellationToken)) + .ReturnsAsync(Response.FromValue(indexDocumentsResultMock.Object, Mock.Of())); + + using var sut = this.CreateRecordCollection(useDefinition); + + // Act. + await sut.DeleteAsync( + TestRecordKey1, + cancellationToken: this._testCancellationToken); + + // Assert. + this._searchClientMock.Verify( + x => x.DeleteDocumentsAsync( + "Key", + It.Is>(x => x.Count() == 1 && x.Contains(TestRecordKey1)), + It.IsAny(), + this._testCancellationToken), + Times.Once); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task CanDeleteManyRecordsWithVectorsAsync(bool useDefinition) + { + // Arrange. +#pragma warning disable Moq1002 // Moq: No matching constructor + var indexDocumentsResultMock = new Mock(MockBehavior.Strict, new List()); +#pragma warning restore Moq1002 // Moq: No matching constructor + + this._searchClientMock.Setup( + x => x.DeleteDocumentsAsync( + It.IsAny(), + It.IsAny>(), + It.IsAny(), + this._testCancellationToken)) + .ReturnsAsync(Response.FromValue(indexDocumentsResultMock.Object, Mock.Of())); + + using var sut = this.CreateRecordCollection(useDefinition); + + // Act. + await sut.DeleteAsync( + [TestRecordKey1, TestRecordKey2], + cancellationToken: this._testCancellationToken); + + // Assert. + this._searchClientMock.Verify( + x => x.DeleteDocumentsAsync( + "Key", + It.Is>(x => x.Count() == 2 && x.Contains(TestRecordKey1) && x.Contains(TestRecordKey2)), + It.IsAny(), + this._testCancellationToken), + Times.Once); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task CanUpsertRecordAsync(bool useDefinition) + { + // Arrange upload result object. +#pragma warning disable Moq1002 // Moq: No matching constructor + var indexingResult = new Mock(MockBehavior.Strict, TestRecordKey1, true, 200); + var indexingResults = new List + { + indexingResult.Object + }; + var indexDocumentsResultMock = new Mock(MockBehavior.Strict, indexingResults); +#pragma warning restore Moq1002 // Moq: No matching constructor + + // Arrange upload. + this._searchClientMock.Setup( + x => x.UploadDocumentsAsync( + It.IsAny>(), + It.IsAny(), + this._testCancellationToken)) + .ReturnsAsync(Response.FromValue(indexDocumentsResultMock.Object, Mock.Of())); + + // Arrange sut. + using var sut = this.CreateRecordCollection(useDefinition); + + var model = CreateModel(TestRecordKey1, true); + + // Act. + await sut.UpsertAsync( + model, + cancellationToken: this._testCancellationToken); + + // Assert. + this._searchClientMock.Verify( + x => x.UploadDocumentsAsync( + It.Is>(x => x.Count() == 1 && x.First()["Key"]!.ToString() == TestRecordKey1), + It.Is(x => x.ThrowOnAnyError == true), + this._testCancellationToken), + Times.Once); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task CanUpsertManyRecordsAsync(bool useDefinition) + { + // Arrange upload result object. +#pragma warning disable Moq1002 // Moq: No matching constructor + var indexingResult1 = new Mock(MockBehavior.Strict, TestRecordKey1, true, 200); + var indexingResult2 = new Mock(MockBehavior.Strict, TestRecordKey2, true, 200); + + var indexingResults = new List + { + indexingResult1.Object, + indexingResult2.Object + }; + var indexDocumentsResultMock = new Mock(MockBehavior.Strict, indexingResults); +#pragma warning restore Moq1002 // Moq: No matching constructor + + // Arrange upload. + this._searchClientMock.Setup( + x => x.UploadDocumentsAsync( + It.IsAny>(), + It.IsAny(), + this._testCancellationToken)) + .ReturnsAsync(Response.FromValue(indexDocumentsResultMock.Object, Mock.Of())); + + // Arrange sut. + using var sut = this.CreateRecordCollection(useDefinition); + + var model1 = CreateModel(TestRecordKey1, true); + var model2 = CreateModel(TestRecordKey2, true); + + // Act. + await sut.UpsertAsync( + [model1, model2], + cancellationToken: this._testCancellationToken); + + // Assert. + this._searchClientMock.Verify( + x => x.UploadDocumentsAsync( + It.Is>(x => x.Count() == 2 && x.First()["Key"]!.ToString() == TestRecordKey1 && x.ElementAt(1)["Key"]!.ToString() == TestRecordKey2), + It.Is(x => x.ThrowOnAnyError == true), + this._testCancellationToken), + Times.Once); + } + + /// + /// Tests that the collection can be created even if the definition and the type do not match. + /// In this case, the expectation is that a custom mapper will be provided to map between the + /// schema as defined by the definition and the different data model. + /// + [Fact] + public void CanCreateCollectionWithMismatchedDefinitionAndType() + { + // Arrange. + var definition = new VectorStoreCollectionDefinition() + { + Properties = + [ + new VectorStoreKeyProperty("Key", typeof(string)), + new VectorStoreDataProperty("Data1", typeof(string)), + new VectorStoreVectorProperty("Vector1", typeof(ReadOnlyMemory), 4), + ] + }; + + // Act. + using var sut = new AzureAISearchCollection( + this._searchIndexClientMock.Object, + TestCollectionName, + new() { Definition = definition }); + } + + [Fact] + public async Task CanSearchWithVectorAndFilterAsync() + { + // Arrange. +#pragma warning disable Moq1002 // Could not find a matching constructor for arguments: SearchResults has an internal parameterless constructor. + var searchResultsMock = Mock.Of>(); +#pragma warning restore Moq1002 // Could not find a matching constructor for arguments: SearchResults has an internal parameterless constructor. + this._searchClientMock + .Setup(x => x.SearchAsync(null, It.IsAny(), It.IsAny())) + .ReturnsAsync(Response.FromValue(searchResultsMock, Mock.Of())); + + using var sut = new AzureAISearchCollection( + this._searchIndexClientMock.Object, + TestCollectionName); + + // Act. + var searchResults = await sut.SearchAsync( + new ReadOnlyMemory(new float[4]), + top: 5, + new() + { + Skip = 3, + Filter = r => r.Data1 == "Data1FilterValue", + VectorProperty = record => record.Vector1 + }, + this._testCancellationToken).ToListAsync(); + + // Assert. + this._searchClientMock.Verify( + x => x.SearchAsync( + null, + It.Is(x => + x.Filter == "(storage_data1 eq 'Data1FilterValue')" && + x.Size == 5 && + x.Skip == 3 && + x.VectorSearch.Queries.First().GetType() == typeof(VectorizedQuery) && + x.VectorSearch.Queries.First().Fields.First() == "storage_vector1"), + It.IsAny()), + Times.Once); + } + + [Fact] + public async Task CanSearchWithTextAndFilterAsync() + { + // Arrange. +#pragma warning disable Moq1002 // Could not find a matching constructor for arguments: SearchResults has an internal parameterless constructor. + var searchResultsMock = Mock.Of>(); +#pragma warning restore Moq1002 // Could not find a matching constructor for arguments: SearchResults has an internal parameterless constructor. + this._searchClientMock + .Setup(x => x.SearchAsync(null, It.IsAny(), It.IsAny())) + .ReturnsAsync(Response.FromValue(searchResultsMock, Mock.Of())); + + using var sut = new AzureAISearchCollection( + this._searchIndexClientMock.Object, + TestCollectionName); + + // Act. + var searchResults = await sut.SearchAsync( + "search string", + top: 5, + new() + { + Skip = 3, + Filter = r => r.Data1 == "Data1FilterValue", + VectorProperty = record => record.Vector1 + }, + this._testCancellationToken).ToListAsync(); + + // Assert. + this._searchClientMock.Verify( + x => x.SearchAsync( + null, + It.Is(x => + x.Filter == "(storage_data1 eq 'Data1FilterValue')" && + x.Size == 5 && + x.Skip == 3 && + x.VectorSearch.Queries.First().GetType() == typeof(VectorizableTextQuery) && + x.VectorSearch.Queries.First().Fields.First() == "storage_vector1" && + ((VectorizableTextQuery)x.VectorSearch.Queries.First()).Text == "search string"), + It.IsAny()), + Times.Once); + } + + private AzureAISearchCollection CreateRecordCollection(bool useDefinition, bool useCustomJsonSerializerOptions = false) + { + return new AzureAISearchCollection( + this._searchIndexClientMock.Object, + TestCollectionName, + new() + { + Definition = useDefinition ? this._multiPropsDefinition : null, + JsonSerializerOptions = useCustomJsonSerializerOptions ? this._customJsonSerializerOptions : null + }); + } + + private static MultiPropsModel CreateModel(string key, bool withVectors) + { + return new MultiPropsModel + { + Key = key, + Data1 = "data 1", + Data2 = "data 2", + Vector1 = withVectors ? new float[] { 1, 2, 3, 4 } : null, + Vector2 = withVectors ? new float[] { 1, 2, 3, 4 } : null, + NotAnnotated = null, + }; + } + + private static JsonObject CreateJsonObjectModel(string key, bool withVectors, bool useCustomJsonSerializerOptions = false) + { + if (useCustomJsonSerializerOptions) + { + return new JsonObject + { + ["key"] = key, + ["storage_data1"] = "data 1", + ["data2"] = "data 2", + ["storage_vector1"] = withVectors ? new JsonArray { 1, 2, 3, 4 } : null, + ["vector2"] = withVectors ? new JsonArray { 1, 2, 3, 4 } : null, + ["notAnnotated"] = null, + }; + } + + return new JsonObject + { + ["Key"] = key, + ["storage_data1"] = "data 1", + ["Data2"] = "data 2", + ["storage_vector1"] = withVectors ? new JsonArray { 1, 2, 3, 4 } : null, + ["Vector2"] = withVectors ? new JsonArray { 1, 2, 3, 4 } : null, + ["NotAnnotated"] = null, + }; + } + + private readonly JsonSerializerOptions _customJsonSerializerOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase + }; + + private readonly VectorStoreCollectionDefinition _multiPropsDefinition = new() + { + Properties = + [ + new VectorStoreKeyProperty("Key", typeof(string)), + new VectorStoreDataProperty("Data1", typeof(string)), + new VectorStoreDataProperty("Data2", typeof(string)), + new VectorStoreVectorProperty("Vector1", typeof(ReadOnlyMemory), 4), + new VectorStoreVectorProperty("Vector2", typeof(ReadOnlyMemory), 4) + ] + }; + + public sealed class MultiPropsModel + { + [VectorStoreKey] + public string Key { get; set; } = string.Empty; + + [JsonPropertyName("storage_data1")] + [VectorStoreData(IsIndexed = true)] + public string Data1 { get; set; } = string.Empty; + + [VectorStoreData] + public string Data2 { get; set; } = string.Empty; + + [JsonPropertyName("storage_vector1")] + [VectorStoreVector(dimensions: 4)] + public ReadOnlyMemory? Vector1 { get; set; } + + [VectorStoreVector(dimensions: 4)] + public ReadOnlyMemory? Vector2 { get; set; } + + public string? NotAnnotated { get; set; } + } +} diff --git a/MEVD/test/AzureAISearch.UnitTests/AzureAISearchDynamicMapperTests.cs b/MEVD/test/AzureAISearch.UnitTests/AzureAISearchDynamicMapperTests.cs new file mode 100644 index 0000000..e01983e --- /dev/null +++ b/MEVD/test/AzureAISearch.UnitTests/AzureAISearchDynamicMapperTests.cs @@ -0,0 +1,285 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json.Nodes; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; +using CommunityToolkit.VectorData.AzureAISearch; +using Xunit; + +namespace SemanticKernel.Connectors.AzureAISearch.UnitTests; + +/// +/// Tests for the class. +/// +public class AzureAISearchDynamicMapperTests +{ + private static readonly CollectionModel s_model = BuildModel( + [ + new VectorStoreKeyProperty("Key", typeof(string)), + new VectorStoreDataProperty("StringDataProp", typeof(string)), + new VectorStoreDataProperty("IntDataProp", typeof(int)), + new VectorStoreDataProperty("NullableIntDataProp", typeof(int?)), + new VectorStoreDataProperty("LongDataProp", typeof(long)), + new VectorStoreDataProperty("NullableLongDataProp", typeof(long?)), + new VectorStoreDataProperty("FloatDataProp", typeof(float)), + new VectorStoreDataProperty("NullableFloatDataProp", typeof(float?)), + new VectorStoreDataProperty("DoubleDataProp", typeof(double)), + new VectorStoreDataProperty("NullableDoubleDataProp", typeof(double?)), + new VectorStoreDataProperty("BoolDataProp", typeof(bool)), + new VectorStoreDataProperty("NullableBoolDataProp", typeof(bool?)), + new VectorStoreDataProperty("DateTimeOffsetDataProp", typeof(DateTimeOffset)), + new VectorStoreDataProperty("NullableDateTimeOffsetDataProp", typeof(DateTimeOffset?)), + new VectorStoreDataProperty("TagListDataProp", typeof(string[])), + new VectorStoreVectorProperty("FloatVector", typeof(ReadOnlyMemory), 10), + new VectorStoreVectorProperty("NullableFloatVector", typeof(ReadOnlyMemory?), 10), + ]); + + private static readonly float[] s_vector1 = [1.0f, 2.0f, 3.0f]; + private static readonly float[] s_vector2 = [4.0f, 5.0f, 6.0f]; + private static readonly string[] s_taglist = ["tag1", "tag2"]; + + [Fact] + public void MapFromDataToStorageModelMapsAllSupportedTypes() + { + // Arrange + var sut = new AzureAISearchDynamicMapper(s_model, null); + var dataModel = new Dictionary + { + ["Key"] = "key", + + ["StringDataProp"] = "string", + ["IntDataProp"] = 1, + ["NullableIntDataProp"] = 2, + ["LongDataProp"] = 3L, + ["NullableLongDataProp"] = 4L, + ["FloatDataProp"] = 5.0f, + ["NullableFloatDataProp"] = 6.0f, + ["DoubleDataProp"] = 7.0, + ["NullableDoubleDataProp"] = 8.0, + ["BoolDataProp"] = true, + ["NullableBoolDataProp"] = false, + ["DateTimeOffsetDataProp"] = new DateTimeOffset(2021, 1, 1, 0, 0, 0, TimeSpan.Zero), + ["NullableDateTimeOffsetDataProp"] = new DateTimeOffset(2021, 1, 1, 0, 0, 0, TimeSpan.Zero), + ["TagListDataProp"] = s_taglist, + + ["FloatVector"] = new ReadOnlyMemory(s_vector1), + ["NullableFloatVector"] = new ReadOnlyMemory(s_vector2) + }; + + // Act + var storageModel = sut.MapFromDataToStorageModel(dataModel, 0, null); + + // Assert + Assert.Equal("key", (string?)storageModel["Key"]); + Assert.Equal("string", (string?)storageModel["StringDataProp"]); + Assert.Equal(1, (int?)storageModel["IntDataProp"]); + Assert.Equal(2, (int?)storageModel["NullableIntDataProp"]); + Assert.Equal(3L, (long?)storageModel["LongDataProp"]); + Assert.Equal(4L, (long?)storageModel["NullableLongDataProp"]); + Assert.Equal(5.0f, (float?)storageModel["FloatDataProp"]); + Assert.Equal(6.0f, (float?)storageModel["NullableFloatDataProp"]); + Assert.Equal(7.0, (double?)storageModel["DoubleDataProp"]); + Assert.Equal(8.0, (double?)storageModel["NullableDoubleDataProp"]); + Assert.Equal(true, (bool?)storageModel["BoolDataProp"]); + Assert.Equal(false, (bool?)storageModel["NullableBoolDataProp"]); + Assert.Equal(new DateTimeOffset(2021, 1, 1, 0, 0, 0, TimeSpan.Zero), (DateTimeOffset?)storageModel["DateTimeOffsetDataProp"]); + Assert.Equal(new DateTimeOffset(2021, 1, 1, 0, 0, 0, TimeSpan.Zero), (DateTimeOffset?)storageModel["NullableDateTimeOffsetDataProp"]); + Assert.Equal(s_taglist, storageModel["TagListDataProp"]!.AsArray().Select(x => (string)x!).ToArray()); + Assert.Equal(s_vector1, storageModel["FloatVector"]!.AsArray().Select(x => (float)x!).ToArray()); + Assert.Equal(s_vector2, storageModel["NullableFloatVector"]!.AsArray().Select(x => (float)x!).ToArray()); + } + + [Fact] + public void MapFromDataToStorageModelMapsNullValues() + { + // Arrange + var model = BuildModel( + [ + new VectorStoreKeyProperty("Key", typeof(string)), + new VectorStoreDataProperty("StringDataProp", typeof(string)), + new VectorStoreDataProperty("NullableIntDataProp", typeof(int?)), + new VectorStoreVectorProperty("NullableFloatVector", typeof(ReadOnlyMemory?), 10), + ]); + + var dataModel = new Dictionary + { + ["Key"] = "key", + ["StringDataProp"] = null, + ["NullableIntDataProp"] = null, + ["NullableFloatVector"] = null + }; + + var sut = new AzureAISearchDynamicMapper(model, null); + + // Act + var storageModel = sut.MapFromDataToStorageModel(dataModel, 0, null); + + // Assert + Assert.Null(storageModel["StringDataProp"]); + Assert.Null(storageModel["NullableIntDataProp"]); + Assert.Null(storageModel["NullableFloatVector"]); + } + + [Fact] + public void MapFromStorageToDataModelMapsAllSupportedTypes() + { + // Arrange + var sut = new AzureAISearchDynamicMapper(s_model, null); + var storageModel = new JsonObject + { + ["Key"] = "key", + ["StringDataProp"] = "string", + ["IntDataProp"] = 1, + ["NullableIntDataProp"] = 2, + ["LongDataProp"] = 3L, + ["NullableLongDataProp"] = 4L, + ["FloatDataProp"] = 5.0f, + ["NullableFloatDataProp"] = 6.0f, + ["DoubleDataProp"] = 7.0, + ["NullableDoubleDataProp"] = 8.0, + ["BoolDataProp"] = true, + ["NullableBoolDataProp"] = false, + ["DateTimeOffsetDataProp"] = new DateTimeOffset(2021, 1, 1, 0, 0, 0, TimeSpan.Zero), + ["NullableDateTimeOffsetDataProp"] = new DateTimeOffset(2021, 1, 1, 0, 0, 0, TimeSpan.Zero), + ["TagListDataProp"] = new JsonArray { "tag1", "tag2" }, + ["FloatVector"] = new JsonArray { 1.0f, 2.0f, 3.0f }, + ["NullableFloatVector"] = new JsonArray { 4.0f, 5.0f, 6.0f } + }; + + // Act + var dataModel = sut.MapFromStorageToDataModel(storageModel, includeVectors: true); + + // Assert + Assert.Equal("key", dataModel["Key"]); + Assert.Equal("string", dataModel["StringDataProp"]); + Assert.Equal(1, dataModel["IntDataProp"]); + Assert.Equal(2, dataModel["NullableIntDataProp"]); + Assert.Equal(3L, dataModel["LongDataProp"]); + Assert.Equal(4L, dataModel["NullableLongDataProp"]); + Assert.Equal(5.0f, dataModel["FloatDataProp"]); + Assert.Equal(6.0f, dataModel["NullableFloatDataProp"]); + Assert.Equal(7.0, dataModel["DoubleDataProp"]); + Assert.Equal(8.0, dataModel["NullableDoubleDataProp"]); + Assert.Equal(true, dataModel["BoolDataProp"]); + Assert.Equal(false, dataModel["NullableBoolDataProp"]); + Assert.Equal(new DateTimeOffset(2021, 1, 1, 0, 0, 0, TimeSpan.Zero), dataModel["DateTimeOffsetDataProp"]); + Assert.Equal(new DateTimeOffset(2021, 1, 1, 0, 0, 0, TimeSpan.Zero), dataModel["NullableDateTimeOffsetDataProp"]); + Assert.Equal(s_taglist, dataModel["TagListDataProp"]); + Assert.Equal(s_vector1, ((ReadOnlyMemory)dataModel["FloatVector"]!).ToArray()); + Assert.Equal(s_vector2, ((ReadOnlyMemory)dataModel["NullableFloatVector"]!)!.ToArray()); + } + + [Fact] + public void MapFromStorageToDataModelMapsNullValues() + { + // Arrange + var model = BuildModel( + [ + new VectorStoreKeyProperty("Key", typeof(string)), + new VectorStoreDataProperty("StringDataProp", typeof(string)), + new VectorStoreDataProperty("NullableIntDataProp", typeof(int?)), + new VectorStoreVectorProperty("NullableFloatVector", typeof(ReadOnlyMemory?), 10), + ]); + + var storageModel = new JsonObject + { + ["Key"] = "key", + ["StringDataProp"] = null, + ["NullableIntDataProp"] = null, + ["NullableFloatVector"] = null + }; + + var sut = new AzureAISearchDynamicMapper(model, null); + + // Act + var dataModel = sut.MapFromStorageToDataModel(storageModel, includeVectors: true); + + // Assert + Assert.Equal("key", dataModel["Key"]); + Assert.Null(dataModel["StringDataProp"]); + Assert.Null(dataModel["NullableIntDataProp"]); + Assert.Null(dataModel["NullableFloatVector"]); + } + + [Fact] + public void MapFromStorageToDataModelThrowsForMissingKey() + { + // Arrange + var model = BuildModel( + [ + new VectorStoreKeyProperty("Key", typeof(string)), + new VectorStoreDataProperty("StringDataProp", typeof(string)), + new VectorStoreDataProperty("NullableIntDataProp", typeof(int?)), + new VectorStoreVectorProperty("NullableFloatVector", typeof(ReadOnlyMemory?), 10), + ]); + + var sut = new AzureAISearchDynamicMapper(model, null); + var storageModel = new JsonObject(); + + // Act + var exception = Assert.Throws(() => sut.MapFromStorageToDataModel(storageModel, includeVectors: true)); + + // Assert + Assert.Equal("The key property 'Key' is missing from the record retrieved from storage.", exception.Message); + } + + [Fact] + public void MapFromDataToStorageModelSkipsMissingProperties() + { + // Arrange + var model = BuildModel( + [ + new VectorStoreKeyProperty("Key", typeof(string)), + new VectorStoreDataProperty("StringDataProp", typeof(string)), + new VectorStoreVectorProperty("FloatVector", typeof(ReadOnlyMemory), 10), + ]); + + var dataModel = new Dictionary { ["Key"] = "key" }; + var sut = new AzureAISearchDynamicMapper(model, null); + + // Act + var storageModel = sut.MapFromDataToStorageModel(dataModel, 0, null); + + // Assert + Assert.Equal("key", (string?)storageModel["Key"]); + Assert.False(storageModel.ContainsKey("StringDataProp")); + Assert.False(storageModel.ContainsKey("FloatVector")); + } + + [Fact] + public void MapFromStorageToDataModelSkipsMissingProperties() + { + // Arrange + var model = BuildModel( + [ + new VectorStoreKeyProperty("Key", typeof(string)), + new VectorStoreDataProperty("StringDataProp", typeof(string)), + new VectorStoreVectorProperty("FloatVector", typeof(ReadOnlyMemory), 10), + ]); + + var storageModel = new JsonObject + { + ["Key"] = "key" + }; + + var sut = new AzureAISearchDynamicMapper(model, null); + + // Act + var dataModel = sut.MapFromStorageToDataModel(storageModel, includeVectors: true); + + // Assert + Assert.Equal("key", dataModel["Key"]); + Assert.False(dataModel.ContainsKey("StringDataProp")); + Assert.False(dataModel.ContainsKey("FloatVector")); + } + + private static CollectionModel BuildModel(List properties) + => new AzureAISearchDynamicModelBuilder() + .BuildDynamic( + new() { Properties = properties }, + defaultEmbeddingGenerator: null); +} diff --git a/MEVD/test/AzureAISearch.UnitTests/AzureAISearchVectorStoreTests.cs b/MEVD/test/AzureAISearch.UnitTests/AzureAISearchVectorStoreTests.cs new file mode 100644 index 0000000..686ec22 --- /dev/null +++ b/MEVD/test/AzureAISearch.UnitTests/AzureAISearchVectorStoreTests.cs @@ -0,0 +1,105 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Azure; +using Azure.Search.Documents; +using Azure.Search.Documents.Indexes; +using Microsoft.Extensions.VectorData; +using CommunityToolkit.VectorData.AzureAISearch; +using Moq; +using Xunit; + +namespace SemanticKernel.Connectors.AzureAISearch.UnitTests; + +/// +/// Contains tests for the class. +/// +public class AzureAISearchVectorStoreTests +{ + private const string TestCollectionName = "testcollection"; + + private readonly Mock _searchIndexClientMock; + private readonly Mock _searchClientMock; + + private readonly CancellationToken _testCancellationToken = new(false); + + public AzureAISearchVectorStoreTests() + { + this._searchClientMock = new Mock(MockBehavior.Strict); + this._searchIndexClientMock = new Mock(MockBehavior.Strict); + this._searchIndexClientMock.Setup(x => x.GetSearchClient(TestCollectionName)).Returns(this._searchClientMock.Object); + this._searchIndexClientMock.Setup(x => x.ServiceName).Returns("TestService"); + } + + [Fact] + public void GetCollectionReturnsCollection() + { + // Arrange. + using var sut = new AzureAISearchVectorStore(this._searchIndexClientMock.Object); + + // Act. + var actual = sut.GetCollection(TestCollectionName); + + // Assert. + Assert.NotNull(actual); + Assert.IsType>(actual); + } + + [Fact] + public void GetCollectionThrowsForInvalidKeyType() + { + // Arrange. + using var sut = new AzureAISearchVectorStore(this._searchIndexClientMock.Object); + + // Act & Assert. + Assert.Throws(() => sut.GetCollection(TestCollectionName)); + } + + [Fact] + public async Task ListCollectionNamesCallsSDKAsync() + { + // Arrange async enumerator mock. + var iterationCounter = 0; + var asyncEnumeratorMock = new Mock>(MockBehavior.Strict); + asyncEnumeratorMock.Setup(x => x.MoveNextAsync()).Returns(() => ValueTask.FromResult(iterationCounter++ <= 4)); + asyncEnumeratorMock.Setup(x => x.Current).Returns(() => $"testcollection{iterationCounter}"); + + // Arrange pageable mock. + var pageableMock = new Mock>(MockBehavior.Strict); + pageableMock.Setup(x => x.GetAsyncEnumerator(this._testCancellationToken)).Returns(asyncEnumeratorMock.Object); + + // Arrange search index client mock and sut. + this._searchIndexClientMock + .Setup(x => x.GetIndexNamesAsync(this._testCancellationToken)) + .Returns(pageableMock.Object); + using var sut = new AzureAISearchVectorStore(this._searchIndexClientMock.Object); + + // Act. + var actual = sut.ListCollectionNamesAsync(this._testCancellationToken); + + // Assert. + Assert.NotNull(actual); + var actualList = await actual.ToListAsync(); + Assert.Equal(5, actualList.Count); + Assert.All(actualList, (value, index) => Assert.Equal($"testcollection{index + 1}", value)); + } + + public sealed class SinglePropsModel + { + [VectorStoreKey] + public string Key { get; set; } = string.Empty; + + [VectorStoreData] + public string Data { get; set; } = string.Empty; + + [VectorStoreVector(dimensions: 4)] + public ReadOnlyMemory? Vector { get; set; } + + public string? NotAnnotated { get; set; } + } +} diff --git a/MEVD/test/CosmosMongoDB.ConformanceTests/CosmosMongoBsonMappingTests.cs b/MEVD/test/CosmosMongoDB.ConformanceTests/CosmosMongoBsonMappingTests.cs new file mode 100644 index 0000000..cec9022 --- /dev/null +++ b/MEVD/test/CosmosMongoDB.ConformanceTests/CosmosMongoBsonMappingTests.cs @@ -0,0 +1,144 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using CosmosMongoDB.ConformanceTests.Support; +using Microsoft.Extensions.VectorData; +using CommunityToolkit.VectorData.CosmosMongoDB; +using MongoDB.Bson.Serialization.Attributes; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace CosmosMongoDB.ConformanceTests; + +public sealed class CosmosMongoBsonMappingTests(CosmosMongoBsonMappingTests.Fixture fixture) + : IClassFixture +{ + [Fact] + public async Task Upsert_with_bson_model_works() + { + var store = (CosmosMongoTestStore)fixture.TestStore; + var collectionName = fixture.TestStore.AdjustCollectionName("BsonModel"); + + var definition = new VectorStoreCollectionDefinition + { + Properties = + [ + new VectorStoreKeyProperty(nameof(BsonTestModel.Id), typeof(string)), + new VectorStoreDataProperty(nameof(BsonTestModel.HotelName), typeof(string)) + ] + }; + + var model = new BsonTestModel { Id = "key", HotelName = "Test Name" }; + + using var collection = new CosmosMongoCollection( + store.Database, + collectionName, + new() { Definition = definition }); + + await collection.EnsureCollectionExistsAsync(); + + try + { + await collection.UpsertAsync(model); + var getResult = await collection.GetAsync(model.Id!); + + Assert.NotNull(getResult); + Assert.Equal("key", getResult!.Id); + Assert.Equal("Test Name", getResult.HotelName); + } + finally + { + await collection.EnsureCollectionDeletedAsync(); + } + } + + [Fact] + public async Task Upsert_with_bson_vector_store_model_works() + { + var store = (CosmosMongoTestStore)fixture.TestStore; + var collectionName = fixture.TestStore.AdjustCollectionName("BsonVectorStoreModel"); + + var model = new BsonVectorStoreTestModel { HotelId = "key", HotelName = "Test Name" }; + + using var collection = new CosmosMongoCollection(store.Database, collectionName); + + await collection.EnsureCollectionExistsAsync(); + + try + { + await collection.UpsertAsync(model); + var getResult = await collection.GetAsync(model.HotelId!); + + Assert.NotNull(getResult); + Assert.Equal("key", getResult!.HotelId); + Assert.Equal("Test Name", getResult.HotelName); + } + finally + { + await collection.EnsureCollectionDeletedAsync(); + } + } + + [Fact] + public async Task Upsert_with_bson_vector_store_with_name_model_works() + { + var store = (CosmosMongoTestStore)fixture.TestStore; + var collectionName = fixture.TestStore.AdjustCollectionName("BsonVectorStoreWithNameModel"); + + var model = new BsonVectorStoreWithNameTestModel { Id = "key", HotelName = "Test Name" }; + + using var collection = new CosmosMongoCollection(store.Database, collectionName); + + await collection.EnsureCollectionExistsAsync(); + + try + { + await collection.UpsertAsync(model); + var getResult = await collection.GetAsync(model.Id!); + + Assert.NotNull(getResult); + Assert.Equal("key", getResult!.Id); + Assert.Equal("Test Name", getResult.HotelName); + } + finally + { + await collection.EnsureCollectionDeletedAsync(); + } + } + + public sealed class Fixture : VectorStoreFixture + { + public override TestStore TestStore => CosmosMongoTestStore.Instance; + } + + private sealed class BsonTestModel + { + [BsonId] + public string? Id { get; set; } + + [BsonElement("hotel_name")] + public string? HotelName { get; set; } + } + + private sealed class BsonVectorStoreTestModel + { + [BsonId] + [VectorStoreKey] + public string? HotelId { get; set; } + + [BsonElement("hotel_name")] + [VectorStoreData] + public string? HotelName { get; set; } + } + + private sealed class BsonVectorStoreWithNameTestModel + { + [BsonId] + [VectorStoreKey] + public string? Id { get; set; } + + [BsonElement("bson_hotel_name")] + [VectorStoreData(StorageName = "storage_hotel_name")] + public string? HotelName { get; set; } + } +} diff --git a/MEVD/test/CosmosMongoDB.ConformanceTests/CosmosMongoCollectionManagementTests.cs b/MEVD/test/CosmosMongoDB.ConformanceTests/CosmosMongoCollectionManagementTests.cs new file mode 100644 index 0000000..c7955e9 --- /dev/null +++ b/MEVD/test/CosmosMongoDB.ConformanceTests/CosmosMongoCollectionManagementTests.cs @@ -0,0 +1,13 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using CosmosMongoDB.ConformanceTests.Support; +using VectorData.ConformanceTests; +using Xunit; + +namespace CosmosMongoDB.ConformanceTests; + +public class CosmosMongoCollectionManagementTests(CosmosMongoFixture fixture) + : CollectionManagementTests(fixture), IClassFixture +{ +} diff --git a/MEVD/test/CosmosMongoDB.ConformanceTests/CosmosMongoDB.ConformanceTests.csproj b/MEVD/test/CosmosMongoDB.ConformanceTests/CosmosMongoDB.ConformanceTests.csproj new file mode 100644 index 0000000..2771850 --- /dev/null +++ b/MEVD/test/CosmosMongoDB.ConformanceTests/CosmosMongoDB.ConformanceTests.csproj @@ -0,0 +1,40 @@ + + + + net10.0;$(NetFrameworkTfm) + enable + enable + true + false + CosmosMongoDB.ConformanceTests + b7762d10-e29b-4bb1-8b74-b6d69a667dd4 + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + + + + + + + + + Always + + + Always + + + + diff --git a/MEVD/test/CosmosMongoDB.ConformanceTests/CosmosMongoDependencyInjectionTests.cs b/MEVD/test/CosmosMongoDB.ConformanceTests/CosmosMongoDependencyInjectionTests.cs new file mode 100644 index 0000000..1b2317d --- /dev/null +++ b/MEVD/test/CosmosMongoDB.ConformanceTests/CosmosMongoDependencyInjectionTests.cs @@ -0,0 +1,143 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using CommunityToolkit.VectorData.CosmosMongoDB; +using MongoDB.Driver; +using VectorData.ConformanceTests; +using Xunit; + +namespace CosmosMongoDB.ConformanceTests; + +public class CosmosMongoDependencyInjectionTests + : DependencyInjectionTests.Record>, string, DependencyInjectionTests.Record> +{ + protected const string ConnectionString = "mongodb://localhost:27017"; + protected const string DatabaseName = "dbName"; + + protected override void PopulateConfiguration(ConfigurationManager configuration, object? serviceKey = null) + => configuration.AddInMemoryCollection( + [ + new(CreateConfigKey("CosmosMongo", serviceKey, "ConnectionString"), ConnectionString), + new(CreateConfigKey("CosmosMongo", serviceKey, "DatabaseName"), DatabaseName), + ]); + + private static string ConnectionStringProvider(IServiceProvider sp) + => sp.GetRequiredService().GetRequiredSection("CosmosMongo:ConnectionString").Value!; + + private static string ConnectionStringProvider(IServiceProvider sp, object serviceKey) + => sp.GetRequiredService().GetRequiredSection(CreateConfigKey("CosmosMongo", serviceKey, "ConnectionString")).Value!; + + private static string DatabaseNameProvider(IServiceProvider sp) + => sp.GetRequiredService().GetRequiredSection("CosmosMongo:DatabaseName").Value!; + + private static string DatabaseNameProvider(IServiceProvider sp, object serviceKey) + => sp.GetRequiredService().GetRequiredSection(CreateConfigKey("CosmosMongo", serviceKey, "DatabaseName")).Value!; + + public override IEnumerable> CollectionDelegates + { + get + { + yield return (services, serviceKey, name, lifetime) => serviceKey is null + ? services + .AddSingleton(sp => new MongoClient(MongoClientSettings.FromConnectionString(ConnectionString))) + .AddSingleton(sp => sp.GetRequiredService().GetDatabase(DatabaseName)) + .AddCosmosMongoCollection(name, lifetime: lifetime) + : services + .AddSingleton(sp => new MongoClient(MongoClientSettings.FromConnectionString(ConnectionString))) + .AddSingleton(sp => sp.GetRequiredService().GetDatabase(DatabaseName)) + .AddKeyedCosmosMongoCollection(serviceKey, name, lifetime: lifetime); + + yield return (services, serviceKey, name, lifetime) => serviceKey is null + ? services.AddCosmosMongoCollection( + name, ConnectionString, DatabaseName, lifetime: lifetime) + : services.AddKeyedCosmosMongoCollection( + serviceKey, name, ConnectionString, DatabaseName, lifetime: lifetime); + + yield return (services, serviceKey, name, lifetime) => serviceKey is null + ? services.AddCosmosMongoCollection( + name, ConnectionStringProvider, DatabaseNameProvider, lifetime: lifetime) + : services.AddKeyedCosmosMongoCollection( + serviceKey, name, sp => ConnectionStringProvider(sp, serviceKey), sp => DatabaseNameProvider(sp, serviceKey), lifetime: lifetime); + } + } + + public override IEnumerable> StoreDelegates + { + get + { + yield return (services, serviceKey, lifetime) => serviceKey is null + ? services.AddCosmosMongoVectorStore( + ConnectionString, DatabaseName, lifetime: lifetime) + : services.AddKeyedCosmosMongoVectorStore( + serviceKey, ConnectionString, DatabaseName, lifetime: lifetime); + + yield return (services, serviceKey, lifetime) => serviceKey is null + ? services + .AddSingleton(sp => new MongoClient(MongoClientSettings.FromConnectionString(ConnectionString))) + .AddSingleton(sp => sp.GetRequiredService().GetDatabase(DatabaseName)) + .AddCosmosMongoVectorStore(lifetime: lifetime) + : services + .AddSingleton(sp => new MongoClient(MongoClientSettings.FromConnectionString(ConnectionString))) + .AddSingleton(sp => sp.GetRequiredService().GetDatabase(DatabaseName)) + .AddKeyedCosmosMongoVectorStore(serviceKey, lifetime: lifetime); + } + } + + [Fact] + public void ConnectionStringProviderCantBeNull() + { + IServiceCollection services = new ServiceCollection(); + + Assert.Throws(() => services.AddCosmosMongoCollection( + name: "notNull", connectionStringProvider: null!, databaseNameProvider: DatabaseNameProvider)); + Assert.Throws(() => services.AddKeyedCosmosMongoCollection( + serviceKey: "notNull", name: "notNull", connectionStringProvider: null!, databaseNameProvider: DatabaseNameProvider)); + } + + [Fact] + public void DatabaseNameProviderCantBeNull() + { + IServiceCollection services = new ServiceCollection(); + + Assert.Throws(() => services.AddCosmosMongoCollection( + name: "notNull", connectionStringProvider: ConnectionStringProvider, databaseNameProvider: null!)); + Assert.Throws(() => services.AddKeyedCosmosMongoCollection( + serviceKey: "notNull", name: "notNull", connectionStringProvider: ConnectionStringProvider, databaseNameProvider: null!)); + } + + [Fact] + public void ConnectionStringCantBeNullOrEmpty() + { + IServiceCollection services = new ServiceCollection(); + + Assert.Throws(() => services.AddCosmosMongoVectorStore(connectionString: null!, DatabaseName)); + Assert.Throws(() => services.AddKeyedCosmosMongoVectorStore(serviceKey: "notNull", connectionString: null!, DatabaseName)); + Assert.Throws(() => services.AddCosmosMongoCollection( + name: "notNull", connectionString: null!, DatabaseName)); + Assert.Throws(() => services.AddCosmosMongoCollection( + name: "notNull", connectionString: "", DatabaseName)); + Assert.Throws(() => services.AddKeyedCosmosMongoCollection( + serviceKey: "notNull", name: "notNull", connectionString: null!, DatabaseName)); + Assert.Throws(() => services.AddKeyedCosmosMongoCollection( + serviceKey: "notNull", name: "notNull", connectionString: "", DatabaseName)); + } + + [Fact] + public void DatabaseNameCantBeNullOrEmpty() + { + IServiceCollection services = new ServiceCollection(); + + Assert.Throws(() => services.AddCosmosMongoVectorStore(ConnectionString, databaseName: null!)); + Assert.Throws(() => services.AddKeyedCosmosMongoVectorStore(serviceKey: "notNull", ConnectionString, databaseName: null!)); + Assert.Throws(() => services.AddCosmosMongoCollection( + name: "notNull", ConnectionString, databaseName: null!)); + Assert.Throws(() => services.AddCosmosMongoCollection( + name: "notNull", ConnectionString, databaseName: "")); + Assert.Throws(() => services.AddKeyedCosmosMongoCollection( + serviceKey: "notNull", name: "notNull", ConnectionString, databaseName: null!)); + Assert.Throws(() => services.AddKeyedCosmosMongoCollection( + serviceKey: "notNull", name: "notNull", ConnectionString, databaseName: "")); + } +} diff --git a/MEVD/test/CosmosMongoDB.ConformanceTests/CosmosMongoDistanceFunctionTests.cs b/MEVD/test/CosmosMongoDB.ConformanceTests/CosmosMongoDistanceFunctionTests.cs new file mode 100644 index 0000000..f728cb8 --- /dev/null +++ b/MEVD/test/CosmosMongoDB.ConformanceTests/CosmosMongoDistanceFunctionTests.cs @@ -0,0 +1,32 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using CosmosMongoDB.ConformanceTests.Support; +using Microsoft.Extensions.VectorData; +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace CosmosMongoDB.ConformanceTests; + +public class CosmosMongoDistanceFunctionTests(CosmosMongoDistanceFunctionTests.Fixture fixture) + : DistanceFunctionTests(fixture), IClassFixture +{ + public override Task CosineSimilarity() => Assert.ThrowsAsync(base.CosineSimilarity); + public override Task EuclideanSquaredDistance() => Assert.ThrowsAsync(base.EuclideanSquaredDistance); + public override Task NegativeDotProductSimilarity() => Assert.ThrowsAsync(base.NegativeDotProductSimilarity); + public override Task HammingDistance() => Assert.ThrowsAsync(base.HammingDistance); + public override Task ManhattanDistance() => Assert.ThrowsAsync(base.ManhattanDistance); + + // CosmosMongo EuclideanDistance doesn't correctly filter by score threshold (returns all results). + // This is a pre-existing bug also present in the Semantic Kernel repo. + protected override Task TestScoreThreshold(VectorStoreCollection.SearchRecord> collection) + => Task.CompletedTask; + + public new class Fixture() : DistanceFunctionTests.Fixture + { + public override TestStore TestStore => CosmosMongoTestStore.Instance; + + public override bool AssertScores { get; } = false; + } +} diff --git a/MEVD/test/CosmosMongoDB.ConformanceTests/CosmosMongoEmbeddingGenerationTests.cs b/MEVD/test/CosmosMongoDB.ConformanceTests/CosmosMongoEmbeddingGenerationTests.cs new file mode 100644 index 0000000..88d7714 --- /dev/null +++ b/MEVD/test/CosmosMongoDB.ConformanceTests/CosmosMongoEmbeddingGenerationTests.cs @@ -0,0 +1,60 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using CosmosMongoDB.ConformanceTests.Support; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.VectorData; +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace CosmosMongoDB.ConformanceTests; + +public class CosmosMongoEmbeddingGenerationTests(CosmosMongoEmbeddingGenerationTests.StringVectorFixture stringVectorFixture, CosmosMongoEmbeddingGenerationTests.RomOfFloatVectorFixture romOfFloatVectorFixture) + : EmbeddingGenerationTests(stringVectorFixture, romOfFloatVectorFixture), IClassFixture, IClassFixture +{ + public new class StringVectorFixture : EmbeddingGenerationTests.StringVectorFixture + { + public override TestStore TestStore => CosmosMongoTestStore.Instance; + + public override VectorStore CreateVectorStore(IEmbeddingGenerator? embeddingGenerator) + => CosmosMongoTestStore.Instance.GetVectorStore(new() { EmbeddingGenerator = embeddingGenerator }); + + public override Func[] DependencyInjectionStoreRegistrationDelegates => + [ + services => services + .AddSingleton(CosmosMongoTestStore.Instance.Database) + .AddCosmosMongoVectorStore() + ]; + + public override Func[] DependencyInjectionCollectionRegistrationDelegates => + [ + services => services + .AddSingleton(CosmosMongoTestStore.Instance.Database) + .AddCosmosMongoCollection(this.CollectionName) + ]; + } + + public new class RomOfFloatVectorFixture : EmbeddingGenerationTests.RomOfFloatVectorFixture + { + public override TestStore TestStore => CosmosMongoTestStore.Instance; + + public override VectorStore CreateVectorStore(IEmbeddingGenerator? embeddingGenerator) + => CosmosMongoTestStore.Instance.GetVectorStore(new() { EmbeddingGenerator = embeddingGenerator }); + + public override Func[] DependencyInjectionStoreRegistrationDelegates => + [ + services => services + .AddSingleton(CosmosMongoTestStore.Instance.Database) + .AddCosmosMongoVectorStore() + ]; + + public override Func[] DependencyInjectionCollectionRegistrationDelegates => + [ + services => services + .AddSingleton(CosmosMongoTestStore.Instance.Database) + .AddCosmosMongoCollection(this.CollectionName) + ]; + } +} diff --git a/MEVD/test/CosmosMongoDB.ConformanceTests/CosmosMongoFilterTests.cs b/MEVD/test/CosmosMongoDB.ConformanceTests/CosmosMongoFilterTests.cs new file mode 100644 index 0000000..31e841e --- /dev/null +++ b/MEVD/test/CosmosMongoDB.ConformanceTests/CosmosMongoFilterTests.cs @@ -0,0 +1,94 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using CosmosMongoDB.ConformanceTests.Support; +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace CosmosMongoDB.ConformanceTests; + +public class CosmosMongoFilterTests(CosmosMongoFilterTests.Fixture fixture) + : FilterTests(fixture), IClassFixture +{ + // Specialized MongoDB syntax for NOT over Contains ($nin) + [Fact] + public virtual Task Not_over_Contains() + => this.TestFilterAsync( + r => !new[] { 8, 10 }.Contains(r.Int), + r => !new[] { 8, 10 }.Contains((int)r["Int"]!)); + + #region Null checking + + // MongoDB currently doesn't support null checking ({ "Foo" : null }) in vector search pre-filters + public override Task Equal_with_null_reference_type() + => Assert.ThrowsAsync(() => base.Equal_with_null_reference_type()); + + public override Task Equal_with_null_captured() + => Assert.ThrowsAsync(() => base.Equal_with_null_captured()); + + public override Task NotEqual_with_null_reference_type() + => Assert.ThrowsAsync(() => base.NotEqual_with_null_reference_type()); + + public override Task NotEqual_with_null_captured() + => Assert.ThrowsAsync(() => base.NotEqual_with_null_captured()); + + public override Task Equal_int_property_with_null_nullable_int() + => Assert.ThrowsAsync(() => base.Equal_int_property_with_null_nullable_int()); + + #endregion + + #region Not + + // MongoDB currently doesn't support NOT in vector search pre-filters + // (https://www.mongodb.com/docs/atlas/atlas-vector-search/vector-search-stage/#atlas-vector-search-pre-filter) + public override Task Not_over_And() + => Assert.ThrowsAsync(() => base.Not_over_And()); + + public override Task Not_over_Or() + => Assert.ThrowsAsync(() => base.Not_over_Or()); + + #endregion + + public override Task Contains_over_field_string_array() + => Assert.ThrowsAsync(() => base.Contains_over_field_string_array()); + + public override Task Contains_over_field_string_List() + => Assert.ThrowsAsync(() => base.Contains_over_field_string_List()); + + #region Enumerable.Any / Contains + + // CosmosMongo filter translator doesn't support Enumerable.Any or Enumerable.Contains/MemoryExtensions.Contains + public override Task Any_with_Contains_over_inline_string_array() + => Assert.ThrowsAsync(() => base.Any_with_Contains_over_inline_string_array()); + + public override Task Any_with_Contains_over_captured_string_array() + => Assert.ThrowsAsync(() => base.Any_with_Contains_over_captured_string_array()); + + public override Task Any_with_Contains_over_captured_string_list() + => Assert.ThrowsAsync(() => base.Any_with_Contains_over_captured_string_list()); + + public override Task Any_over_List_with_Contains_over_captured_string_array() + => Assert.ThrowsAsync(() => base.Any_over_List_with_Contains_over_captured_string_array()); + + public override Task Contains_with_Enumerable_Contains() + => Assert.ThrowsAsync(() => base.Contains_with_Enumerable_Contains()); + +#if NET + public override Task Contains_with_MemoryExtensions_Contains() + => Assert.ThrowsAsync(() => base.Contains_with_MemoryExtensions_Contains()); + + public override Task Contains_with_MemoryExtensions_Contains_with_null_comparer() + => Assert.ThrowsAsync(() => base.Contains_with_MemoryExtensions_Contains_with_null_comparer()); +#endif + + #endregion + + public new class Fixture : FilterTests.Fixture + { + public override TestStore TestStore => CosmosMongoTestStore.Instance; + + protected override string IndexKind => Microsoft.Extensions.VectorData.IndexKind.IvfFlat; + protected override string DistanceFunction => Microsoft.Extensions.VectorData.DistanceFunction.CosineDistance; + } +} diff --git a/MEVD/test/CosmosMongoDB.ConformanceTests/CosmosMongoIndexKindTests.cs b/MEVD/test/CosmosMongoDB.ConformanceTests/CosmosMongoIndexKindTests.cs new file mode 100644 index 0000000..10b6904 --- /dev/null +++ b/MEVD/test/CosmosMongoDB.ConformanceTests/CosmosMongoIndexKindTests.cs @@ -0,0 +1,31 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using CosmosMongoDB.ConformanceTests.Support; +using Microsoft.Extensions.VectorData; +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace CosmosMongoDB.ConformanceTests; + +public class CosmosMongoIndexKindTests(CosmosMongoIndexKindTests.Fixture fixture) + : IndexKindTests(fixture), IClassFixture +{ + // Note: Cosmos Mongo support HNSW, but only in a specific tier. + // [Fact] + // public virtual Task Hnsw() + // => this.Test(IndexKind.Hnsw); + + [Fact] + public virtual Task IvfFlat() + => this.Test(IndexKind.IvfFlat); + + // Cosmos Mongo does not support index-less searching + public override Task Flat() => Assert.ThrowsAsync(base.Flat); + + public new class Fixture() : IndexKindTests.Fixture + { + public override TestStore TestStore => CosmosMongoTestStore.Instance; + } +} diff --git a/MEVD/test/CosmosMongoDB.ConformanceTests/CosmosMongoTestSuiteImplementationTests.cs b/MEVD/test/CosmosMongoDB.ConformanceTests/CosmosMongoTestSuiteImplementationTests.cs new file mode 100644 index 0000000..ab421b4 --- /dev/null +++ b/MEVD/test/CosmosMongoDB.ConformanceTests/CosmosMongoTestSuiteImplementationTests.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.ModelTests; + +namespace CosmosMongoDB.ConformanceTests; + +public class CosmosMongoTestSuiteImplementationTests : TestSuiteImplementationTests +{ + protected override ICollection IgnoredTestBases { get; } = + [ + typeof(DynamicModelTests<>), + + // Hybrid search not supported + typeof(HybridSearchTests<>), + ]; +} diff --git a/MEVD/test/CosmosMongoDB.ConformanceTests/ModelTests/CosmosMongoBasicModelTests.cs b/MEVD/test/CosmosMongoDB.ConformanceTests/ModelTests/CosmosMongoBasicModelTests.cs new file mode 100644 index 0000000..9ae768c --- /dev/null +++ b/MEVD/test/CosmosMongoDB.ConformanceTests/ModelTests/CosmosMongoBasicModelTests.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using CosmosMongoDB.ConformanceTests.Support; +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace CosmosMongoDB.ConformanceTests.ModelTests; + +public class CosmosMongoBasicModelTests(CosmosMongoBasicModelTests.Fixture fixture) + : BasicModelTests(fixture), IClassFixture +{ + public new class Fixture : BasicModelTests.Fixture + { + public override TestStore TestStore => CosmosMongoTestStore.Instance; + } +} diff --git a/MEVD/test/CosmosMongoDB.ConformanceTests/ModelTests/CosmosMongoMultiVectorModelTests.cs b/MEVD/test/CosmosMongoDB.ConformanceTests/ModelTests/CosmosMongoMultiVectorModelTests.cs new file mode 100644 index 0000000..f133234 --- /dev/null +++ b/MEVD/test/CosmosMongoDB.ConformanceTests/ModelTests/CosmosMongoMultiVectorModelTests.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using CosmosMongoDB.ConformanceTests.Support; +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace CosmosMongoDB.ConformanceTests.ModelTests; + +public class CosmosMongoMultiVectorModelTests(CosmosMongoMultiVectorModelTests.Fixture fixture) + : MultiVectorModelTests(fixture), IClassFixture +{ + public new class Fixture : MultiVectorModelTests.Fixture + { + public override TestStore TestStore => CosmosMongoTestStore.Instance; + } +} diff --git a/MEVD/test/CosmosMongoDB.ConformanceTests/ModelTests/CosmosMongoNoDataModelTests.cs b/MEVD/test/CosmosMongoDB.ConformanceTests/ModelTests/CosmosMongoNoDataModelTests.cs new file mode 100644 index 0000000..f3923e5 --- /dev/null +++ b/MEVD/test/CosmosMongoDB.ConformanceTests/ModelTests/CosmosMongoNoDataModelTests.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using CosmosMongoDB.ConformanceTests.Support; +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace CosmosMongoDB.ConformanceTests.ModelTests; + +public class CosmosMongoNoDataModelTests(CosmosMongoNoDataModelTests.Fixture fixture) + : NoDataModelTests(fixture), IClassFixture +{ + public new class Fixture : NoDataModelTests.Fixture + { + public override TestStore TestStore => CosmosMongoTestStore.Instance; + } +} diff --git a/MEVD/test/CosmosMongoDB.ConformanceTests/ModelTests/CosmosMongoNoVectorModelTests.cs b/MEVD/test/CosmosMongoDB.ConformanceTests/ModelTests/CosmosMongoNoVectorModelTests.cs new file mode 100644 index 0000000..edd73c8 --- /dev/null +++ b/MEVD/test/CosmosMongoDB.ConformanceTests/ModelTests/CosmosMongoNoVectorModelTests.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using CosmosMongoDB.ConformanceTests.Support; +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace CosmosMongoDB.ConformanceTests.ModelTests; + +public class CosmosMongoNoVectorModelTests(CosmosMongoNoVectorModelTests.Fixture fixture) + : NoVectorModelTests(fixture), IClassFixture +{ + public new class Fixture : NoVectorModelTests.Fixture + { + public override TestStore TestStore => CosmosMongoTestStore.Instance; + } +} diff --git a/MEVD/test/CosmosMongoDB.ConformanceTests/Properties/AssemblyAttributes.cs b/MEVD/test/CosmosMongoDB.ConformanceTests/Properties/AssemblyAttributes.cs new file mode 100644 index 0000000..6571e2d --- /dev/null +++ b/MEVD/test/CosmosMongoDB.ConformanceTests/Properties/AssemblyAttributes.cs @@ -0,0 +1,6 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Xunit; + +[assembly: CollectionBehavior(DisableTestParallelization = true)] diff --git a/MEVD/test/CosmosMongoDB.ConformanceTests/Support/CosmosMongoFixture.cs b/MEVD/test/CosmosMongoDB.ConformanceTests/Support/CosmosMongoFixture.cs new file mode 100644 index 0000000..ed82665 --- /dev/null +++ b/MEVD/test/CosmosMongoDB.ConformanceTests/Support/CosmosMongoFixture.cs @@ -0,0 +1,11 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using VectorData.ConformanceTests.Support; + +namespace CosmosMongoDB.ConformanceTests.Support; + +public class CosmosMongoFixture : VectorStoreFixture +{ + public override TestStore TestStore => CosmosMongoTestStore.Instance; +} diff --git a/MEVD/test/CosmosMongoDB.ConformanceTests/Support/CosmosMongoTestEnvironment.cs b/MEVD/test/CosmosMongoDB.ConformanceTests/Support/CosmosMongoTestEnvironment.cs new file mode 100644 index 0000000..5e7ab5a --- /dev/null +++ b/MEVD/test/CosmosMongoDB.ConformanceTests/Support/CosmosMongoTestEnvironment.cs @@ -0,0 +1,27 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.Configuration; + +namespace CosmosMongoDB.ConformanceTests.Support; + +#pragma warning disable CA1810 // Initialize all static fields when those fields are declared + +public static class CosmosMongoTestEnvironment +{ + public static readonly string? ConnectionString; + + public static bool IsConnectionStringDefined => ConnectionString is not null; + + static CosmosMongoTestEnvironment() + { + var configuration = new ConfigurationBuilder() + .AddJsonFile(path: "testsettings.json", optional: true) + .AddJsonFile(path: "testsettings.development.json", optional: true) + .AddEnvironmentVariables() + .AddUserSecrets() + .Build(); + + ConnectionString = configuration["CosmosMongo:ConnectionString"]; + } +} diff --git a/MEVD/test/CosmosMongoDB.ConformanceTests/Support/CosmosMongoTestStore.cs b/MEVD/test/CosmosMongoDB.ConformanceTests/Support/CosmosMongoTestStore.cs new file mode 100644 index 0000000..89224e7 --- /dev/null +++ b/MEVD/test/CosmosMongoDB.ConformanceTests/Support/CosmosMongoTestStore.cs @@ -0,0 +1,46 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using CommunityToolkit.VectorData.CosmosMongoDB; +using MongoDB.Driver; +using VectorData.ConformanceTests.Support; + +namespace CosmosMongoDB.ConformanceTests.Support; + +#pragma warning disable CA1001 +public sealed class CosmosMongoTestStore : TestStore +#pragma warning restore CA1001 +{ + public static CosmosMongoTestStore Instance { get; } = new(); + + private MongoClient? _client; + private IMongoDatabase? _database; + + public MongoClient Client => this._client ?? throw new InvalidOperationException("Not initialized"); + public IMongoDatabase Database => this._database ?? throw new InvalidOperationException("Not initialized"); + + public override string DefaultIndexKind => Microsoft.Extensions.VectorData.IndexKind.IvfFlat; + + public override string DefaultDistanceFunction => Microsoft.Extensions.VectorData.DistanceFunction.CosineDistance; + + public CosmosMongoVectorStore GetVectorStore(CosmosMongoVectorStoreOptions options) + => new(this.Database, options); + + private CosmosMongoTestStore() + { + } + + protected override Task StartAsync() + { + if (string.IsNullOrWhiteSpace(CosmosMongoTestEnvironment.ConnectionString)) + { + throw new InvalidOperationException("Connection string is not configured, set the CosmosMongo:ConnectionString environment variable"); + } + + this._client = new MongoClient(CosmosMongoTestEnvironment.ConnectionString); + this._database = this._client.GetDatabase("VectorSearchTests"); + this.DefaultVectorStore = new CosmosMongoVectorStore(this._database); + + return Task.CompletedTask; + } +} diff --git a/MEVD/test/CosmosMongoDB.ConformanceTests/TypeTests/CosmosMongoDataTypeTests.cs b/MEVD/test/CosmosMongoDB.ConformanceTests/TypeTests/CosmosMongoDataTypeTests.cs new file mode 100644 index 0000000..f526284 --- /dev/null +++ b/MEVD/test/CosmosMongoDB.ConformanceTests/TypeTests/CosmosMongoDataTypeTests.cs @@ -0,0 +1,55 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using CosmosMongoDB.ConformanceTests.Support; +using VectorData.ConformanceTests.Support; +using VectorData.ConformanceTests.TypeTests; +using Xunit; + +namespace CosmosMongoDB.ConformanceTests.TypeTests; + +public class CosmosMongoDataTypeTests(CosmosMongoDataTypeTests.Fixture fixture) + : DataTypeTests.DefaultRecord>(fixture), IClassFixture +{ + public override Task Decimal() + => this.Test( + "Decimal", 8.5m, 9.5m, + isFilterable: false); // TODO: Filtering doesn't fail but the data doesn't seem to appear... + + public override Task DateTime() + => this.Test( + "DateTime", + new DateTime(2020, 1, 1, 12, 30, 45, DateTimeKind.Utc), + new DateTime(2021, 2, 3, 13, 40, 55, DateTimeKind.Utc), + instantiationExpression: () => new DateTime(2020, 1, 1, 12, 30, 45, DateTimeKind.Utc)); + + // MongoDB stores DateTimeOffset as UTC BsonDateTime; the offset is lost on round-trip. + // Filtering is not supported because the default MongoDB serializer stores DateTimeOffset as a BsonDocument + // (with DateTime/Ticks/Offset fields), but BsonValueFactory.Create produces a simple BsonDateTime for filter + // comparisons — the representations don't match. This is a pre-existing bug also present in SK. + public override Task DateTimeOffset() + => this.Test( + "DateTimeOffset", + new DateTimeOffset(2020, 1, 1, 12, 30, 45, TimeSpan.Zero), + new DateTimeOffset(2021, 2, 3, 13, 40, 55, TimeSpan.Zero), + instantiationExpression: () => new DateTimeOffset(2020, 1, 1, 12, 30, 45, TimeSpan.Zero), + isFilterable: false); + + public new class Fixture : DataTypeTests.DefaultRecord>.Fixture + { + public override TestStore TestStore => CosmosMongoTestStore.Instance; + + // MongoDB does not support null checks in vector search pre-filters + public override bool IsNullFilteringSupported => false; + + public override Type[] UnsupportedDefaultTypes { get; } = + [ + typeof(byte), + typeof(short), + typeof(Guid), +#if NET + typeof(TimeOnly) +#endif + ]; + } +} diff --git a/MEVD/test/CosmosMongoDB.ConformanceTests/TypeTests/CosmosMongoEmbeddingTypeTests.cs b/MEVD/test/CosmosMongoDB.ConformanceTests/TypeTests/CosmosMongoEmbeddingTypeTests.cs new file mode 100644 index 0000000..2bf7550 --- /dev/null +++ b/MEVD/test/CosmosMongoDB.ConformanceTests/TypeTests/CosmosMongoEmbeddingTypeTests.cs @@ -0,0 +1,20 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using CosmosMongoDB.ConformanceTests.Support; +using VectorData.ConformanceTests.Support; +using VectorData.ConformanceTests.TypeTests; +using Xunit; + +#pragma warning disable CA2000 // Dispose objects before losing scope + +namespace CosmosMongoDB.ConformanceTests.TypeTests; + +public class CosmosMongoEmbeddingTypeTests(CosmosMongoEmbeddingTypeTests.Fixture fixture) + : EmbeddingTypeTests(fixture), IClassFixture +{ + public new class Fixture : EmbeddingTypeTests.Fixture + { + public override TestStore TestStore => CosmosMongoTestStore.Instance; + } +} diff --git a/MEVD/test/CosmosMongoDB.ConformanceTests/TypeTests/CosmosMongoKeyTypeTests.cs b/MEVD/test/CosmosMongoDB.ConformanceTests/TypeTests/CosmosMongoKeyTypeTests.cs new file mode 100644 index 0000000..d7c22ff --- /dev/null +++ b/MEVD/test/CosmosMongoDB.ConformanceTests/TypeTests/CosmosMongoKeyTypeTests.cs @@ -0,0 +1,31 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using CosmosMongoDB.ConformanceTests.Support; +using MongoDB.Bson; +using VectorData.ConformanceTests.Support; +using VectorData.ConformanceTests.TypeTests; +using Xunit; + +namespace CosmosMongoDB.ConformanceTests.TypeTests; + +public class CosmosMongoKeyTypeTests(CosmosMongoKeyTypeTests.Fixture fixture) + : KeyTypeTests(fixture), IClassFixture +{ + [Fact] + public virtual Task ObjectId() => this.Test(new("652f8c3e8f9b2c1a4d3e6a7b"), supportsAutoGeneration: true); + + [Fact] + public virtual Task String() => this.Test("foo", "bar"); + + [Fact] + public virtual Task Int() => this.Test(8); + + [Fact] + public virtual Task Long() => this.Test(8L); + + public new class Fixture : KeyTypeTests.Fixture + { + public override TestStore TestStore => CosmosMongoTestStore.Instance; + } +} diff --git a/MEVD/test/CosmosMongoDB.UnitTests/.editorconfig b/MEVD/test/CosmosMongoDB.UnitTests/.editorconfig new file mode 100644 index 0000000..394eef6 --- /dev/null +++ b/MEVD/test/CosmosMongoDB.UnitTests/.editorconfig @@ -0,0 +1,6 @@ +# Suppressing errors for Test projects under dotnet folder +[*.cs] +dotnet_diagnostic.CA2007.severity = none # Do not directly await a Task +dotnet_diagnostic.VSTHRD111.severity = none # Use .ConfigureAwait(bool) is hidden by default, set to none to prevent IDE from changing on autosave +dotnet_diagnostic.CS1591.severity = none # Missing XML comment for publicly visible type or member +dotnet_diagnostic.IDE1006.severity = warning # Naming rule violations diff --git a/MEVD/test/CosmosMongoDB.UnitTests/CosmosMongoCollectionTests.cs b/MEVD/test/CosmosMongoDB.UnitTests/CosmosMongoCollectionTests.cs new file mode 100644 index 0000000..c0c832f --- /dev/null +++ b/MEVD/test/CosmosMongoDB.UnitTests/CosmosMongoCollectionTests.cs @@ -0,0 +1,714 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Linq.Expressions; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.VectorData; +using CommunityToolkit.VectorData.CosmosMongoDB; +using MongoDB.Bson; +using MongoDB.Bson.Serialization; +using MongoDB.Bson.Serialization.Attributes; +using MongoDB.Driver; +using Moq; +using Xunit; +using MEVD = Microsoft.Extensions.VectorData; + +namespace SemanticKernel.Connectors.CosmosMongoDB.UnitTests; + +/// +/// Unit tests for class. +/// +public sealed class CosmosMongoCollectionTests +{ + private readonly Mock _mockMongoDatabase = new(); + private readonly Mock> _mockMongoCollection = new(); + + public CosmosMongoCollectionTests() + { + this._mockMongoDatabase + .Setup(l => l.GetCollection(It.IsAny(), It.IsAny())) + .Returns(this._mockMongoCollection.Object); + } + + [Fact] + public void ConstructorForModelWithoutKeyThrowsException() + { + // Act & Assert + var exception = Assert.Throws(() => new CosmosMongoCollection(this._mockMongoDatabase.Object, "collection")); + Assert.Contains("No key property found", exception.Message); + } + + [Fact] + public void ConstructorWithDeclarativeModelInitializesCollection() + { + // Act & Assert + using var collection = new CosmosMongoCollection( + this._mockMongoDatabase.Object, + "collection"); + + Assert.NotNull(collection); + } + + [Fact] + public void ConstructorWithImperativeModelInitializesCollection() + { + // Arrange + var definition = new VectorStoreCollectionDefinition + { + Properties = [new VectorStoreKeyProperty("Id", typeof(string))] + }; + + // Act + using var collection = new CosmosMongoCollection( + this._mockMongoDatabase.Object, + "collection", + new() { Definition = definition }); + + // Assert + Assert.NotNull(collection); + } + + [Theory] + [MemberData(nameof(CollectionExistsData))] + public async Task CollectionExistsReturnsValidResultAsync(List collections, string collectionName, bool expectedResult) + { + // Arrange + var mockCursor = new Mock>(); + + mockCursor + .Setup(l => l.MoveNextAsync(It.IsAny())) + .ReturnsAsync(true); + + mockCursor + .Setup(l => l.Current) + .Returns(collections); + + this._mockMongoDatabase + .Setup(l => l.ListCollectionNamesAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(mockCursor.Object); + + using var sut = new CosmosMongoCollection( + this._mockMongoDatabase.Object, + collectionName); + + // Act + var actualResult = await sut.CollectionExistsAsync(); + + // Assert + Assert.Equal(expectedResult, actualResult); + } + + [Fact] + public async Task EnsureCollectionExistsInvokesValidMethodsAsync() + { + // Arrange + const string CollectionName = "collection"; + + var mockCursor = new Mock>(); + mockCursor + .Setup(l => l.MoveNextAsync(It.IsAny())) + .ReturnsAsync(true); + + mockCursor + .Setup(l => l.Current) + .Returns([]); + + this._mockMongoDatabase + .Setup(l => l.ListCollectionNamesAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(mockCursor.Object); + + var mockIndexCursor = new Mock>(); + mockIndexCursor + .SetupSequence(l => l.MoveNext(It.IsAny())) + .Returns(true) + .Returns(false); + + mockIndexCursor + .Setup(l => l.Current) + .Returns([]); + + var mockMongoIndexManager = new Mock>(); + + mockMongoIndexManager + .Setup(l => l.ListAsync(It.IsAny())) + .ReturnsAsync(mockIndexCursor.Object); + + this._mockMongoCollection + .Setup(l => l.Indexes) + .Returns(mockMongoIndexManager.Object); + + using var sut = new CosmosMongoCollection( + this._mockMongoDatabase.Object, + CollectionName); + + // Act + await sut.EnsureCollectionExistsAsync(); + + // Assert + this._mockMongoDatabase.Verify(l => l.CreateCollectionAsync( + CollectionName, + It.IsAny(), + It.IsAny()), Times.Exactly(1)); + + this._mockMongoDatabase.Verify(l => l.ListCollectionNamesAsync( + It.IsAny(), + It.IsAny()), Times.Never); + } + + [Fact] + public async Task DeleteInvokesValidMethodsAsync() + { + // Arrange + const string RecordKey = "key"; + + using var sut = new CosmosMongoCollection( + this._mockMongoDatabase.Object, + "collection"); + + var serializerRegistry = BsonSerializer.SerializerRegistry; + var documentSerializer = serializerRegistry.GetSerializer(); + var expectedDefinition = Builders.Filter.Eq(document => document["_id"], RecordKey); + + // Act + await sut.DeleteAsync(RecordKey); + + // Assert + this._mockMongoCollection.Verify(l => l.DeleteOneAsync( + It.Is>(definition => + CompareFilterDefinitions(definition, expectedDefinition, documentSerializer, serializerRegistry)), + It.IsAny()), Times.Once()); + } + + [Fact] + public async Task DeleteBatchInvokesValidMethodsAsync() + { + // Arrange + List recordKeys = ["key1", "key2"]; + + using var sut = new CosmosMongoCollection( + this._mockMongoDatabase.Object, + "collection"); + + var serializerRegistry = BsonSerializer.SerializerRegistry; + var documentSerializer = serializerRegistry.GetSerializer(); + var expectedDefinition = Builders.Filter.In(document => document["_id"].AsString, recordKeys); + + // Act + await sut.DeleteAsync(recordKeys); + + // Assert + this._mockMongoCollection.Verify(l => l.DeleteManyAsync( + It.Is>(definition => + CompareFilterDefinitions(definition, expectedDefinition, documentSerializer, serializerRegistry)), + It.IsAny()), Times.Once()); + } + + [Fact] + public async Task DeleteCollectionInvokesValidMethodsAsync() + { + // Arrange + const string CollectionName = "collection"; + + using var sut = new CosmosMongoCollection( + this._mockMongoDatabase.Object, + CollectionName); + + // Act + await sut.EnsureCollectionDeletedAsync(); + + // Assert + this._mockMongoDatabase.Verify(l => l.DropCollectionAsync( + It.Is(name => name == CollectionName), + It.IsAny()), Times.Once()); + } + + [Fact] + public async Task GetReturnsValidRecordAsync() + { + // Arrange + const string RecordKey = "key"; + + var document = new BsonDocument { ["_id"] = RecordKey, ["HotelName"] = "Test Name" }; + + var mockCursor = new Mock>(); + mockCursor + .Setup(l => l.MoveNextAsync(It.IsAny())) + .ReturnsAsync(true); + + mockCursor + .Setup(l => l.Current) + .Returns([document]); + + this._mockMongoCollection + .Setup(l => l.FindAsync( + It.IsAny>(), + It.IsAny>(), + It.IsAny())) + .ReturnsAsync(mockCursor.Object); + + using var sut = new CosmosMongoCollection( + this._mockMongoDatabase.Object, + "collection"); + + // Act + var result = await sut.GetAsync(RecordKey); + + // Assert + Assert.NotNull(result); + Assert.Equal(RecordKey, result.HotelId); + Assert.Equal("Test Name", result.HotelName); + } + + [Fact] + public async Task GetBatchReturnsValidRecordAsync() + { + // Arrange + var document1 = new BsonDocument { ["_id"] = "key1", ["HotelName"] = "Test Name 1" }; + var document2 = new BsonDocument { ["_id"] = "key2", ["HotelName"] = "Test Name 2" }; + var document3 = new BsonDocument { ["_id"] = "key3", ["HotelName"] = "Test Name 3" }; + + var mockCursor = new Mock>(); + mockCursor + .SetupSequence(l => l.MoveNextAsync(It.IsAny())) + .ReturnsAsync(true) + .ReturnsAsync(false); + + mockCursor + .Setup(l => l.Current) + .Returns([document1, document2, document3]); + + this._mockMongoCollection + .Setup(l => l.FindAsync( + It.IsAny>(), + It.IsAny>(), + It.IsAny())) + .ReturnsAsync(mockCursor.Object); + + using var sut = new CosmosMongoCollection( + this._mockMongoDatabase.Object, + "collection"); + + // Act + var results = await sut.GetAsync(["key1", "key2", "key3"]).ToListAsync(); + + // Assert + Assert.NotNull(results[0]); + Assert.Equal("key1", results[0].HotelId); + Assert.Equal("Test Name 1", results[0].HotelName); + + Assert.NotNull(results[1]); + Assert.Equal("key2", results[1].HotelId); + Assert.Equal("Test Name 2", results[1].HotelName); + + Assert.NotNull(results[2]); + Assert.Equal("key3", results[2].HotelId); + Assert.Equal("Test Name 3", results[2].HotelName); + } + + [Fact] + public async Task CanUpsertRecordAsync() + { + // Arrange + var hotel = new CosmosMongoHotelModel("key") { HotelName = "Test Name" }; + + var serializerRegistry = BsonSerializer.SerializerRegistry; + var documentSerializer = serializerRegistry.GetSerializer(); + var expectedDefinition = Builders.Filter.Eq(document => document["_id"], "key"); + + using var sut = new CosmosMongoCollection( + this._mockMongoDatabase.Object, + "collection"); + + // Act + await sut.UpsertAsync(hotel); + + // Assert + this._mockMongoCollection.Verify(l => l.ReplaceOneAsync( + It.Is>(definition => + CompareFilterDefinitions(definition, expectedDefinition, documentSerializer, serializerRegistry)), + It.Is(document => + document["_id"] == "key" && + document["HotelName"] == "Test Name"), + It.IsAny(), + It.IsAny()), Times.Once()); + } + + [Fact] + public async Task UpsertBatchReturnsRecordKeysAsync() + { + // Arrange + var hotel1 = new CosmosMongoHotelModel("key1") { HotelName = "Test Name 1" }; + var hotel2 = new CosmosMongoHotelModel("key2") { HotelName = "Test Name 2" }; + var hotel3 = new CosmosMongoHotelModel("key3") { HotelName = "Test Name 3" }; + + using var sut = new CosmosMongoCollection( + this._mockMongoDatabase.Object, + "collection"); + + // Act + await sut.UpsertAsync([hotel1, hotel2, hotel3]); + } + + [Fact] + public async Task UpsertWithModelWorksCorrectlyAsync() + { + var definition = new VectorStoreCollectionDefinition + { + Properties = + [ + new VectorStoreKeyProperty("Id", typeof(string)), + new VectorStoreDataProperty("HotelName", typeof(string)) + ] + }; + + await this.TestUpsertWithModelAsync( + dataModel: new TestModel { Id = "key", HotelName = "Test Name" }, + expectedPropertyName: "HotelName", + definition: definition); + } + + [Fact] + public async Task UpsertWithVectorStoreModelWorksCorrectlyAsync() + { + await this.TestUpsertWithModelAsync( + dataModel: new VectorStoreTestModel { Id = "key", HotelName = "Test Name" }, + expectedPropertyName: "HotelName"); + } + + [Fact] + public async Task UpsertWithBsonModelWorksCorrectlyAsync() + { + var definition = new VectorStoreCollectionDefinition + { + Properties = + [ + new VectorStoreKeyProperty("Id", typeof(string)), + new VectorStoreDataProperty("HotelName", typeof(string)) + ] + }; + + await this.TestUpsertWithModelAsync( + dataModel: new BsonTestModel { Id = "key", HotelName = "Test Name" }, + expectedPropertyName: "hotel_name", + definition: definition); + } + + [Fact] + public async Task UpsertWithBsonVectorStoreModelWorksCorrectlyAsync() + { + await this.TestUpsertWithModelAsync( + dataModel: new BsonVectorStoreTestModel { Id = "key", HotelName = "Test Name" }, + expectedPropertyName: "hotel_name"); + } + + [Fact] + public async Task UpsertWithBsonVectorStoreWithNameModelWorksCorrectlyAsync() + { + await this.TestUpsertWithModelAsync( + dataModel: new BsonVectorStoreWithNameTestModel { Id = "key", HotelName = "Test Name" }, + expectedPropertyName: "bson_hotel_name"); + } + + [Theory] + [MemberData(nameof(SearchVectorTypeData))] + public async Task SearchThrowsExceptionWithInvalidVectorTypeAsync(object vector, bool exceptionExpected) + { + // Arrange + this.MockCollectionForSearch(); + + using var sut = new CosmosMongoCollection( + this._mockMongoDatabase.Object, + "collection"); + + // Act & Assert + if (exceptionExpected) + { + await Assert.ThrowsAsync(async () => await sut.SearchAsync(vector, top: 3).ToListAsync()); + } + else + { + Assert.NotNull(await sut.SearchAsync(vector, top: 3).FirstOrDefaultAsync()); + } + } + + [Theory] + [InlineData("TestEmbedding1", "TestEmbedding1", 3, 3)] + [InlineData("TestEmbedding2", "test_embedding_2", 4, 4)] + public async Task SearchUsesValidQueryAsync( + string? vectorPropertyName, + string expectedVectorPropertyName, + int actualTop, + int expectedTop) + { + // Arrange + var vector = new ReadOnlyMemory([1f, 2f, 3f]); + + var expectedSearch = new BsonDocument + { + { "$search", + new BsonDocument + { + { "cosmosSearch", + new BsonDocument + { + { "vector", BsonArray.Create(vector.ToArray()) }, + { "path", expectedVectorPropertyName }, + { "k", expectedTop }, + } + }, + { "returnStoredSource", true } + } + } + }; + + var expectedProjection = new BsonDocument + { + { "$project", + new BsonDocument + { + { "similarityScore", new BsonDocument { { "$meta", "searchScore" } } }, + { "document", "$$ROOT" } + } + } + }; + + this.MockCollectionForSearch(); + + using var sut = new CosmosMongoCollection( + this._mockMongoDatabase.Object, + "collection"); + + Expression>? vectorSelector = vectorPropertyName switch + { + "TestEmbedding1" => record => record.TestEmbedding1, + "TestEmbedding2" => record => record.TestEmbedding2, + _ => null + }; + + // Act + var actual = await sut.SearchAsync(vector, top: actualTop, new() + { + VectorProperty = vectorSelector, + }).FirstOrDefaultAsync(); + + // Assert + Assert.NotNull(actual); + + this._mockMongoCollection.Verify(l => l.AggregateAsync( + It.Is>(pipeline => + this.ComparePipeline(pipeline, expectedSearch, expectedProjection)), + It.IsAny(), + It.IsAny()), Times.Once()); + } + + [Fact] + public async Task SearchThrowsExceptionWithNonExistentVectorPropertyNameAsync() + { + // Arrange + this.MockCollectionForSearch(); + + using var sut = new CosmosMongoCollection( + this._mockMongoDatabase.Object, + "collection"); + + var options = new MEVD.VectorSearchOptions { VectorProperty = r => "non-existent-property" }; + + // Act & Assert + await Assert.ThrowsAsync(async () => await sut.SearchAsync(new ReadOnlyMemory([1f, 2f, 3f]), top: 3, options).FirstOrDefaultAsync()); + } + + [Fact] + public async Task SearchReturnsRecordWithScoreAsync() + { + // Arrange + this.MockCollectionForSearch(); + + using var sut = new CosmosMongoCollection( + this._mockMongoDatabase.Object, + "collection"); + + // Act + var result = await sut.SearchAsync(new ReadOnlyMemory([1f, 2f, 3f]), top: 3).FirstOrDefaultAsync(); + + // Assert + Assert.NotNull(result); + Assert.Equal("key", result.Record.HotelId); + Assert.Equal("Test Name", result.Record.HotelName); + Assert.Equal(0.99f, result.Score); + } + + public static TheoryData, string, bool> CollectionExistsData => new() + { + { ["collection-2"], "collection-2", true }, + { [], "non-existent-collection", false } + }; + + public static TheoryData, int> EnsureCollectionExistsData => new() + { + { ["collection"], 0 }, + { [], 1 } + }; + + public static TheoryData SearchVectorTypeData => new() + { + { new ReadOnlyMemory([1f, 2f, 3f]), false }, + { new ReadOnlyMemory?(new([1f, 2f, 3f])), false }, + { new List([1f, 2f, 3f]), true }, + }; + + #region private + + private bool ComparePipeline( + PipelineDefinition actualPipeline, + BsonDocument expectedSearch, + BsonDocument expectedProjection) + { + var serializerRegistry = BsonSerializer.SerializerRegistry; + var documentSerializer = serializerRegistry.GetSerializer(); + + var documents = actualPipeline.Render(new RenderArgs(documentSerializer, serializerRegistry)).Documents; + + return + documents[0].ToJson() == expectedSearch.ToJson() && + documents[1].ToJson() == expectedProjection.ToJson(); + } + + private void MockCollectionForSearch() + { + var document = new BsonDocument { ["_id"] = "key", ["HotelName"] = "Test Name" }; + var searchResult = new BsonDocument { ["document"] = document, ["similarityScore"] = 0.99f }; + + var mockCursor = new Mock>(); + mockCursor + .Setup(l => l.MoveNextAsync(It.IsAny())) + .ReturnsAsync(true); + + mockCursor + .Setup(l => l.Current) + .Returns([searchResult]); + + this._mockMongoCollection + .Setup(l => l.AggregateAsync( + It.IsAny>(), + It.IsAny(), + It.IsAny())) + .ReturnsAsync(mockCursor.Object); + } + + private async Task TestUpsertWithModelAsync( + TDataModel dataModel, + string expectedPropertyName, + VectorStoreCollectionDefinition? definition = null) + where TDataModel : class + { + // Arrange + var serializerRegistry = BsonSerializer.SerializerRegistry; + var documentSerializer = serializerRegistry.GetSerializer(); + var expectedDefinition = Builders.Filter.Eq(document => document["_id"], "key"); + + CosmosMongoCollectionOptions? options = definition != null ? + new() { Definition = definition } : + null; + + using var sut = new CosmosMongoCollection( + this._mockMongoDatabase.Object, + "collection", + options); + + // Act + await sut.UpsertAsync(dataModel); + + // Assert + this._mockMongoCollection.Verify(l => l.ReplaceOneAsync( + It.Is>(definition => + CompareFilterDefinitions(definition, expectedDefinition, documentSerializer, serializerRegistry)), + It.Is(document => + document["_id"] == "key" && + document.Contains(expectedPropertyName) && + document[expectedPropertyName] == "Test Name"), + It.IsAny(), + It.IsAny()), Times.Once()); + } + + private static bool CompareFilterDefinitions( + FilterDefinition actual, + FilterDefinition expected, + IBsonSerializer documentSerializer, + IBsonSerializerRegistry serializerRegistry) + { + return actual.Render(new RenderArgs(documentSerializer, serializerRegistry)) == + expected.Render(new RenderArgs(documentSerializer, serializerRegistry)); + } + +#pragma warning disable CA1812 + private sealed class TestModel + { + public string? Id { get; set; } + + public string? HotelName { get; set; } + } + + private sealed class VectorStoreTestModel + { + [VectorStoreKey] + public string? Id { get; set; } + + [VectorStoreData(StorageName = "hotel_name")] + public string? HotelName { get; set; } + } + + private sealed class BsonTestModel + { + [BsonId] + public string? Id { get; set; } + + [BsonElement("hotel_name")] + public string? HotelName { get; set; } + } + + private sealed class BsonVectorStoreTestModel + { + [BsonId] + [VectorStoreKey] + public string? Id { get; set; } + + [BsonElement("hotel_name")] + [VectorStoreData] + public string? HotelName { get; set; } + } + + private sealed class BsonVectorStoreWithNameTestModel + { + [BsonId] + [VectorStoreKey] + public string? Id { get; set; } + + [BsonElement("bson_hotel_name")] + [VectorStoreData(StorageName = "storage_hotel_name")] + public string? HotelName { get; set; } + } + + private sealed class VectorSearchModel + { + [BsonId] + [VectorStoreKey] + public string? Id { get; set; } + + [VectorStoreData] + public string? HotelName { get; set; } + + [VectorStoreVector(dimensions: 4, DistanceFunction = DistanceFunction.CosineDistance, IndexKind = IndexKind.IvfFlat, StorageName = "test_embedding_1")] + public ReadOnlyMemory TestEmbedding1 { get; set; } + + [BsonElement("test_embedding_2")] + [VectorStoreVector(dimensions: 4, DistanceFunction = DistanceFunction.CosineDistance, IndexKind = IndexKind.IvfFlat)] + public ReadOnlyMemory TestEmbedding2 { get; set; } + } +#pragma warning restore CA1812 + + #endregion +} diff --git a/MEVD/test/CosmosMongoDB.UnitTests/CosmosMongoDB.UnitTests.csproj b/MEVD/test/CosmosMongoDB.UnitTests/CosmosMongoDB.UnitTests.csproj new file mode 100644 index 0000000..ac6a96f --- /dev/null +++ b/MEVD/test/CosmosMongoDB.UnitTests/CosmosMongoDB.UnitTests.csproj @@ -0,0 +1,33 @@ + + + + CommunityToolkit.VectorData.CosmosMongoDB.UnitTests + CommunityToolkit.VectorData.CosmosMongoDB.UnitTests + net10.0 + true + enable + disable + false + $(NoWarn);SKEXP0001 + + + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + diff --git a/MEVD/test/CosmosMongoDB.UnitTests/CosmosMongoHotelModel.cs b/MEVD/test/CosmosMongoDB.UnitTests/CosmosMongoHotelModel.cs new file mode 100644 index 0000000..1a82f74 --- /dev/null +++ b/MEVD/test/CosmosMongoDB.UnitTests/CosmosMongoHotelModel.cs @@ -0,0 +1,45 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using Microsoft.Extensions.VectorData; +using MongoDB.Bson.Serialization.Attributes; + +namespace SemanticKernel.Connectors.CosmosMongoDB.UnitTests; + +public class CosmosMongoHotelModel(string hotelId) +{ + /// The key of the record. + [VectorStoreKey] + public string HotelId { get; init; } = hotelId; + + /// A string metadata field. + [VectorStoreData(IsIndexed = true)] + public string? HotelName { get; set; } + + /// An int metadata field. + [VectorStoreData] + public int HotelCode { get; set; } + + /// A float metadata field. + [VectorStoreData] + public float? HotelRating { get; set; } + + /// A bool metadata field. + [BsonElement("parking_is_included")] + [VectorStoreData] + public bool ParkingIncluded { get; set; } + + /// An array metadata field. + [VectorStoreData] + public List Tags { get; set; } = []; + + /// A data field. + [VectorStoreData] + public string? Description { get; set; } + + /// A vector field. + [VectorStoreVector(dimensions: 4, DistanceFunction = DistanceFunction.CosineDistance, IndexKind = IndexKind.IvfFlat)] + public ReadOnlyMemory? DescriptionEmbedding { get; set; } +} diff --git a/MEVD/test/CosmosMongoDB.UnitTests/CosmosMongoVectorStoreTests.cs b/MEVD/test/CosmosMongoDB.UnitTests/CosmosMongoVectorStoreTests.cs new file mode 100644 index 0000000..f35a9c0 --- /dev/null +++ b/MEVD/test/CosmosMongoDB.UnitTests/CosmosMongoVectorStoreTests.cs @@ -0,0 +1,74 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using CommunityToolkit.VectorData.CosmosMongoDB; +using MongoDB.Driver; +using Moq; +using Xunit; + +namespace SemanticKernel.Connectors.CosmosMongoDB.UnitTests; + +/// +/// Unit tests for class. +/// +public sealed class CosmosMongoVectorStoreTests +{ + private readonly Mock _mockMongoDatabase = new(); + + [Fact] + public void GetCollectionWithNotSupportedKeyThrowsException() + { + // Arrange + using var sut = new CosmosMongoVectorStore(this._mockMongoDatabase.Object); + + // Act & Assert + Assert.Throws(() => sut.GetCollection("collection")); + } + + [Fact] + public void GetCollectionWithoutFactoryReturnsDefaultCollection() + { + // Arrange + using var sut = new CosmosMongoVectorStore(this._mockMongoDatabase.Object); + + // Act + var collection = sut.GetCollection("collection"); + + // Assert + Assert.NotNull(collection); + } + + [Fact] + public async Task ListCollectionNamesReturnsCollectionNamesAsync() + { + // Arrange + var expectedCollectionNames = new List { "collection-1", "collection-2", "collection-3" }; + + var mockCursor = new Mock>(); + mockCursor + .SetupSequence(l => l.MoveNextAsync(It.IsAny())) + .ReturnsAsync(true) + .ReturnsAsync(false); + + mockCursor + .Setup(l => l.Current) + .Returns(expectedCollectionNames); + + this._mockMongoDatabase + .Setup(l => l.ListCollectionNamesAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(mockCursor.Object); + + using var sut = new CosmosMongoVectorStore(this._mockMongoDatabase.Object); + + // Act + var actualCollectionNames = await sut.ListCollectionNamesAsync().ToListAsync(); + + // Assert + Assert.Equal(expectedCollectionNames, actualCollectionNames); + } +} diff --git a/MEVD/test/Directory.Build.props b/MEVD/test/Directory.Build.props new file mode 100644 index 0000000..7abea26 --- /dev/null +++ b/MEVD/test/Directory.Build.props @@ -0,0 +1,30 @@ + + + + + + + net472 + + $(NoWarn);MEVD9000,MEVD9001 + $(NoWarn);CA1515 + $(NoWarn);CA1707 + $(NoWarn);CA1716 + $(NoWarn);CA1720 + $(NoWarn);CA1721 + $(NoWarn);CA1819 + $(NoWarn);CS1819 + $(NoWarn);CA1861 + $(NoWarn);CA1863 + $(NoWarn);CA2007;VSTHRD111 + $(NoWarn);CS1591 + $(NoWarn);IDE1006 + + + $(NoWarn);IDE0340 + + + true + + + diff --git a/MEVD/test/InMemory.ConformanceTests/InMemory.ConformanceTests.csproj b/MEVD/test/InMemory.ConformanceTests/InMemory.ConformanceTests.csproj new file mode 100644 index 0000000..44e0c31 --- /dev/null +++ b/MEVD/test/InMemory.ConformanceTests/InMemory.ConformanceTests.csproj @@ -0,0 +1,27 @@ + + + + net10.0;$(NetFrameworkTfm) + enable + enable + true + false + InMemory.ConformanceTests + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + + + + diff --git a/MEVD/test/InMemory.ConformanceTests/InMemoryCollectionManagementTests.cs b/MEVD/test/InMemory.ConformanceTests/InMemoryCollectionManagementTests.cs new file mode 100644 index 0000000..5da8dc0 --- /dev/null +++ b/MEVD/test/InMemory.ConformanceTests/InMemoryCollectionManagementTests.cs @@ -0,0 +1,13 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using InMemory.ConformanceTests.Support; +using VectorData.ConformanceTests; +using Xunit; + +namespace InMemory.ConformanceTests; + +public class InMemoryCollectionManagementTests(InMemoryFixture fixture) + : CollectionManagementTests(fixture), IClassFixture +{ +} diff --git a/MEVD/test/InMemory.ConformanceTests/InMemoryDistanceFunctionTests.cs b/MEVD/test/InMemory.ConformanceTests/InMemoryDistanceFunctionTests.cs new file mode 100644 index 0000000..539da32 --- /dev/null +++ b/MEVD/test/InMemory.ConformanceTests/InMemoryDistanceFunctionTests.cs @@ -0,0 +1,23 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using InMemory.ConformanceTests.Support; +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace InMemory.ConformanceTests; + +public class InMemoryDistanceFunctionTests(InMemoryDistanceFunctionTests.Fixture fixture) + : DistanceFunctionTests(fixture), IClassFixture +{ + public override Task EuclideanSquaredDistance() => Assert.ThrowsAsync(base.EuclideanSquaredDistance); + public override Task NegativeDotProductSimilarity() => Assert.ThrowsAsync(base.NegativeDotProductSimilarity); + public override Task HammingDistance() => Assert.ThrowsAsync(base.HammingDistance); + public override Task ManhattanDistance() => Assert.ThrowsAsync(base.ManhattanDistance); + + public new class Fixture() : DistanceFunctionTests.Fixture + { + public override TestStore TestStore => InMemoryTestStore.Instance; + } +} diff --git a/MEVD/test/InMemory.ConformanceTests/InMemoryEmbeddingGenerationTests.cs b/MEVD/test/InMemory.ConformanceTests/InMemoryEmbeddingGenerationTests.cs new file mode 100644 index 0000000..6f1cb20 --- /dev/null +++ b/MEVD/test/InMemory.ConformanceTests/InMemoryEmbeddingGenerationTests.cs @@ -0,0 +1,77 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using InMemory.ConformanceTests.Support; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.VectorData; +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace InMemory.ConformanceTests; + +public class InMemoryEmbeddingGenerationTests(InMemoryEmbeddingGenerationTests.StringVectorFixture stringVectorFixture, InMemoryEmbeddingGenerationTests.RomOfFloatVectorFixture romOfFloatVectorFixture) + : EmbeddingGenerationTests(stringVectorFixture, romOfFloatVectorFixture), IClassFixture, IClassFixture +{ + // InMemory doesn't allowing accessing the same collection via different .NET types (it's unique in this). + // The following dynamic tests attempt to access the fixture collection - which is created with Record - via + // Dictionary. + public override Task SearchAsync_with_property_generator_dynamic() => Task.CompletedTask; + public override Task UpsertAsync_dynamic() => Task.CompletedTask; + public override Task UpsertAsync_batch_dynamic() => Task.CompletedTask; + + // The same applies to the custom type test: + public override Task SearchAsync_with_custom_input_type() => Task.CompletedTask; + + // The test relies on creating a new InMemoryVectorStore configured with a store-default generator, but with InMemory that store + // doesn't share the seeded data with the fixture store (since each InMemoryVectorStore has its own private data). + // Test coverage is already largely sufficient via the property and collection tests. + public override Task SearchAsync_with_store_generator() => Task.CompletedTask; + + public new class StringVectorFixture : EmbeddingGenerationTests.StringVectorFixture + { + public override TestStore TestStore => InMemoryTestStore.Instance; + + // Note that with InMemory specifically, we can't create a vector store with an embedding generator, since it wouldn't share the seeded data with the fixture store. + public override VectorStore CreateVectorStore(IEmbeddingGenerator? embeddingGenerator) + => InMemoryTestStore.Instance.DefaultVectorStore; + + public override Func[] DependencyInjectionStoreRegistrationDelegates => + [ + // The InMemory DI methods register a new vector store instance, which doesn't share the collection seeded by the + // fixture and the test fails. + // services => services.AddInMemoryVectorStore() + ]; + + public override Func[] DependencyInjectionCollectionRegistrationDelegates => + [ + // The InMemory DI methods register a new vector store instance, which doesn't share the collection seeded by the + // fixture and the test fails. + // services => services.AddInMemoryVectorStoreRecordCollection(this.CollectionName) + ]; + } + + public new class RomOfFloatVectorFixture : EmbeddingGenerationTests.RomOfFloatVectorFixture + { + public override TestStore TestStore => InMemoryTestStore.Instance; + + // Note that with InMemory specifically, we can't create a vector store with an embedding generator, since it wouldn't share the seeded data with the fixture store. + public override VectorStore CreateVectorStore(IEmbeddingGenerator? embeddingGenerator) + => InMemoryTestStore.Instance.DefaultVectorStore; + + public override Func[] DependencyInjectionStoreRegistrationDelegates => + [ + // The InMemory DI methods register a new vector store instance, which doesn't share the collection seeded by the + // fixture and the test fails. + // services => services.AddInMemoryVectorStore() + ]; + + public override Func[] DependencyInjectionCollectionRegistrationDelegates => + [ + // The InMemory DI methods register a new vector store instance, which doesn't share the collection seeded by the + // fixture and the test fails. + // services => services.AddInMemoryVectorStoreRecordCollection(this.CollectionName) + ]; + } +} diff --git a/MEVD/test/InMemory.ConformanceTests/InMemoryFilterTests.cs b/MEVD/test/InMemory.ConformanceTests/InMemoryFilterTests.cs new file mode 100644 index 0000000..38f3aeb --- /dev/null +++ b/MEVD/test/InMemory.ConformanceTests/InMemoryFilterTests.cs @@ -0,0 +1,26 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using InMemory.ConformanceTests.Support; +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace InMemory.ConformanceTests; + +public class InMemoryFilterTests(InMemoryFilterTests.Fixture fixture) + : FilterTests(fixture), IClassFixture +{ + public new class Fixture : FilterTests.Fixture + { + public override TestStore TestStore => InMemoryTestStore.Instance; + + // BaseFilterTests attempts to create two InMemoryVectorStoreRecordCollection with different .NET types: + // 1. One for strongly-typed mapping (TRecord=FilterRecord) + // 2. One for dynamic mapping (TRecord=Dictionary) + // Unfortunately, InMemoryVectorStore does not allow mapping the same collection name to different types; + // at the same time, it simply evaluates all filtering via .NET AsQueryable(), so actual test coverage + // isn't very important here. So we disable the dynamic tests. + public override bool TestDynamic => false; + } +} diff --git a/MEVD/test/InMemory.ConformanceTests/InMemoryIndexKindTests.cs b/MEVD/test/InMemory.ConformanceTests/InMemoryIndexKindTests.cs new file mode 100644 index 0000000..456798d --- /dev/null +++ b/MEVD/test/InMemory.ConformanceTests/InMemoryIndexKindTests.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using InMemory.ConformanceTests.Support; +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace InMemory.ConformanceTests; + +public class InMemoryIndexKindTests(InMemoryIndexKindTests.Fixture fixture) + : IndexKindTests(fixture), IClassFixture +{ + public new class Fixture() : IndexKindTests.Fixture + { + public override TestStore TestStore => InMemoryTestStore.Instance; + } +} diff --git a/MEVD/test/InMemory.ConformanceTests/InMemoryTestSuiteImplementationTests.cs b/MEVD/test/InMemory.ConformanceTests/InMemoryTestSuiteImplementationTests.cs new file mode 100644 index 0000000..79dcdd9 --- /dev/null +++ b/MEVD/test/InMemory.ConformanceTests/InMemoryTestSuiteImplementationTests.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using VectorData.ConformanceTests; + +namespace InMemory.ConformanceTests; + +public class InMemoryTestSuiteImplementationTests : TestSuiteImplementationTests +{ + protected override ICollection IgnoredTestBases { get; } = + [ + typeof(DependencyInjectionTests<,,,>), + typeof(DependencyInjectionTests<>), + + // Hybrid search not supported + typeof(HybridSearchTests<>) + ]; +} diff --git a/MEVD/test/InMemory.ConformanceTests/ModelTests/InMemoryBasicModelTests.cs b/MEVD/test/InMemory.ConformanceTests/ModelTests/InMemoryBasicModelTests.cs new file mode 100644 index 0000000..625134d --- /dev/null +++ b/MEVD/test/InMemory.ConformanceTests/ModelTests/InMemoryBasicModelTests.cs @@ -0,0 +1,144 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using InMemory.ConformanceTests.Support; +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace InMemory.ConformanceTests.ModelTests; + +public class InMemoryBasicModelTests(InMemoryBasicModelTests.Fixture fixture) + : BasicModelTests(fixture), IClassFixture +{ + public override async Task GetAsync_single_record(bool includeVectors) + { + if (includeVectors) + { + await base.GetAsync_single_record(includeVectors); + return; + } + + // InMemory always returns the vectors (IncludeVectors = false isn't respected) + var expectedRecord = fixture.TestData[0]; + var received = await fixture.Collection.GetAsync(expectedRecord.Key, new() { IncludeVectors = false }); + + expectedRecord.AssertEqual(received, includeVectors: true, fixture.TestStore.VectorsComparable); + } + + public override async Task GetAsync_multiple_records(bool includeVectors) + { + if (includeVectors) + { + await base.GetAsync_multiple_records(includeVectors); + return; + } + + // InMemory always returns the vectors (IncludeVectors = false isn't respected) + var expectedRecords = fixture.TestData.Take(2); // the last two records can get deleted by other tests + var ids = expectedRecords.Select(record => record.Key); + + var received = await fixture.Collection.GetAsync(ids, new() { IncludeVectors = false }).ToArrayAsync(); + + foreach (var record in expectedRecords) + { + record.AssertEqual( + received.Single(r => r.Key.Equals(record.Key)), + includeVectors: true, + fixture.TestStore.VectorsComparable); + } + } + + public override async Task GetAsync_with_filter(bool includeVectors) + { + if (includeVectors) + { + await base.GetAsync_with_filter(includeVectors); + return; + } + + // InMemory always returns the vectors (IncludeVectors = false isn't respected) + var expectedRecord = fixture.TestData[0]; + + var results = await this.Collection.GetAsync( + r => r.Number == 1, + top: 2, + new() { IncludeVectors = includeVectors }) + .ToListAsync(); + + var receivedRecord = Assert.Single(results); + expectedRecord.AssertEqual(receivedRecord, includeVectors: true, fixture.TestStore.VectorsComparable); + } + + public override async Task SearchAsync(bool includeVectors) + { + if (includeVectors) + { + await base.SearchAsync(includeVectors); + return; + } + + // InMemory always returns the vectors (IncludeVectors = false isn't respected) + var expectedRecord = fixture.TestData[0]; + + var result = await Collection + .SearchAsync( + expectedRecord.Vector, + top: 1, + new() { IncludeVectors = includeVectors }) + .SingleAsync(); + + expectedRecord.AssertEqual(result.Record, includeVectors: true, fixture.TestStore.VectorsComparable); + } + + public override async Task SearchAsync_with_Filter() + { + // InMemory always returns the vectors (IncludeVectors = false isn't respected) + var result = await Collection + .SearchAsync( + fixture.TestData[0].Vector, + top: 1, + new() { Filter = r => r.Number == 2 }) + .SingleAsync(); + + fixture.TestData[1].AssertEqual(result.Record, includeVectors: true, fixture.TestStore.VectorsComparable); + } + + public override async Task SearchAsync_with_Skip() + { + // InMemory always returns the vectors (IncludeVectors = false isn't respected) + var result = await Collection + .SearchAsync( + fixture.TestData[0].Vector, + top: 1, + new() { Skip = 1 }) + .SingleAsync(); + + fixture.TestData[1].AssertEqual(result.Record, includeVectors: true, fixture.TestStore.VectorsComparable); + } + + public override async Task GetAsync_multiple_records_with_missing_keys_returns_only_existing() + { + // InMemory always returns the vectors (IncludeVectors = false isn't respected) + var expectedRecords = fixture.TestData.Take(2).ToArray(); + var missingKey = fixture.GenerateNextKey(); + var ids = expectedRecords.Select(record => record.Key).Append(missingKey).ToArray(); + + var received = await Collection.GetAsync(ids).ToListAsync(); + + Assert.Equal(2, received.Count); + + foreach (var record in expectedRecords) + { + record.AssertEqual( + received.Single(r => r.Key.Equals(record.Key)), + includeVectors: true, + fixture.TestStore.VectorsComparable); + } + } + + public new class Fixture : BasicModelTests.Fixture + { + public override TestStore TestStore => InMemoryTestStore.Instance; + } +} diff --git a/MEVD/test/InMemory.ConformanceTests/ModelTests/InMemoryDynamicModelTests.cs b/MEVD/test/InMemory.ConformanceTests/ModelTests/InMemoryDynamicModelTests.cs new file mode 100644 index 0000000..5f7f244 --- /dev/null +++ b/MEVD/test/InMemory.ConformanceTests/ModelTests/InMemoryDynamicModelTests.cs @@ -0,0 +1,127 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using InMemory.ConformanceTests.Support; +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace InMemory.ConformanceTests.ModelTests; + +public class InMemoryDynamicModelTests(InMemoryDynamicModelTests.Fixture fixture) + : DynamicModelTests(fixture), IClassFixture +{ + public override async Task GetAsync_single_record(bool includeVectors) + { + if (includeVectors) + { + await base.GetAsync_single_record(includeVectors); + return; + } + + // InMemory always returns the vectors (IncludeVectors = false isn't respected) + var expectedRecord = fixture.TestData[0]; + var received = await fixture.Collection.GetAsync( + (int)expectedRecord[KeyPropertyName]!, + new() { IncludeVectors = false }); + + AssertEquivalent(expectedRecord, received, includeVectors: true, fixture.TestStore.VectorsComparable); + } + + public override async Task GetAsync_multiple_records(bool includeVectors) + { + if (includeVectors) + { + await base.GetAsync_multiple_records(includeVectors); + return; + } + + // InMemory always returns the vectors (IncludeVectors = false isn't respected) + var expectedRecords = fixture.TestData.Take(2); + var ids = expectedRecords.Select(record => record[KeyPropertyName]!); + + var received = await fixture.Collection.GetAsync(ids, new() { IncludeVectors = false }).ToArrayAsync(); + + foreach (var record in expectedRecords) + { + AssertEquivalent( + record, + received.Single(r => r[KeyPropertyName]!.Equals(record[KeyPropertyName])), + includeVectors: true, + fixture.TestStore.VectorsComparable); + } + } + + public override async Task GetAsync_with_filter(bool includeVectors) + { + if (includeVectors) + { + await base.GetAsync_with_filter(includeVectors); + return; + } + + // InMemory always returns the vectors (IncludeVectors = false isn't respected) + var expectedRecord = fixture.TestData[0]; + + var results = await fixture.Collection.GetAsync( + r => (int)r[IntegerPropertyName]! == 1, + top: 2, + new() { IncludeVectors = includeVectors }) + .ToListAsync(); + + var receivedRecord = Assert.Single(results); + AssertEquivalent(expectedRecord, receivedRecord, includeVectors: true, fixture.TestStore.VectorsComparable); + } + + public override async Task SearchAsync(bool includeVectors) + { + if (includeVectors) + { + await base.SearchAsync(includeVectors); + return; + } + + // InMemory always returns the vectors (IncludeVectors = false isn't respected) + var expectedRecord = fixture.TestData[0]; + + var result = await Collection + .SearchAsync( + expectedRecord[VectorPropertyName]!, + top: 1, + new() { IncludeVectors = includeVectors }) + .SingleAsync(); + + AssertEquivalent(expectedRecord, result.Record, includeVectors: true, fixture.TestStore.VectorsComparable); + } + + public override async Task SearchAsync_with_Filter() + { + // InMemory always returns the vectors (IncludeVectors = false isn't respected) + var result = await Collection + .SearchAsync( + fixture.TestData[0][VectorPropertyName]!, + top: 1, + new() { Filter = r => (int)r[IntegerPropertyName]! == 2 }) + .SingleAsync(); + + AssertEquivalent(fixture.TestData[1], result.Record, includeVectors: true, fixture.TestStore.VectorsComparable); + } + + public override async Task SearchAsync_with_Skip() + { + // InMemory always returns the vectors (IncludeVectors = false isn't respected) + var result = await Collection + .SearchAsync( + fixture.TestData[0][VectorPropertyName]!, + top: 1, + new() { Skip = 1 }) + .SingleAsync(); + + AssertEquivalent(fixture.TestData[1], result.Record, includeVectors: true, fixture.TestStore.VectorsComparable); + } + + public new class Fixture : DynamicModelTests.Fixture + { + public override TestStore TestStore => InMemoryTestStore.Instance; + } +} diff --git a/MEVD/test/InMemory.ConformanceTests/ModelTests/InMemoryMultiVectorModelTests.cs b/MEVD/test/InMemory.ConformanceTests/ModelTests/InMemoryMultiVectorModelTests.cs new file mode 100644 index 0000000..2d6b20b --- /dev/null +++ b/MEVD/test/InMemory.ConformanceTests/ModelTests/InMemoryMultiVectorModelTests.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using InMemory.ConformanceTests.Support; +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace InMemory.ConformanceTests.ModelTests; + +public class InMemoryMultiVectorModelTests(InMemoryMultiVectorModelTests.Fixture fixture) + : MultiVectorModelTests(fixture), IClassFixture +{ + public new class Fixture : MultiVectorModelTests.Fixture + { + public override TestStore TestStore => InMemoryTestStore.Instance; + } +} diff --git a/MEVD/test/InMemory.ConformanceTests/ModelTests/InMemoryNoDataModelTests.cs b/MEVD/test/InMemory.ConformanceTests/ModelTests/InMemoryNoDataModelTests.cs new file mode 100644 index 0000000..2ada41f --- /dev/null +++ b/MEVD/test/InMemory.ConformanceTests/ModelTests/InMemoryNoDataModelTests.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using InMemory.ConformanceTests.Support; +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace InMemory.ConformanceTests.ModelTests; + +public class InMemoryNoDataModelTests(InMemoryNoDataModelTests.Fixture fixture) + : NoDataModelTests(fixture), IClassFixture +{ + public new class Fixture : NoDataModelTests.Fixture + { + public override TestStore TestStore => InMemoryTestStore.Instance; + } +} diff --git a/MEVD/test/InMemory.ConformanceTests/ModelTests/InMemoryNoVectorModelTests.cs b/MEVD/test/InMemory.ConformanceTests/ModelTests/InMemoryNoVectorModelTests.cs new file mode 100644 index 0000000..dd4e1f7 --- /dev/null +++ b/MEVD/test/InMemory.ConformanceTests/ModelTests/InMemoryNoVectorModelTests.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using InMemory.ConformanceTests.Support; +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace InMemory.ConformanceTests.ModelTests; + +public class InMemoryNoVectorModelTests(InMemoryNoVectorModelTests.Fixture fixture) + : NoVectorModelTests(fixture), IClassFixture +{ + public new class Fixture : NoVectorModelTests.Fixture + { + public override TestStore TestStore => InMemoryTestStore.Instance; + } +} diff --git a/MEVD/test/InMemory.ConformanceTests/Support/InMemoryFixture.cs b/MEVD/test/InMemory.ConformanceTests/Support/InMemoryFixture.cs new file mode 100644 index 0000000..2d0e953 --- /dev/null +++ b/MEVD/test/InMemory.ConformanceTests/Support/InMemoryFixture.cs @@ -0,0 +1,11 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using VectorData.ConformanceTests.Support; + +namespace InMemory.ConformanceTests.Support; + +public class InMemoryFixture : VectorStoreFixture +{ + public override TestStore TestStore => InMemoryTestStore.Instance; +} diff --git a/MEVD/test/InMemory.ConformanceTests/Support/InMemoryTestStore.cs b/MEVD/test/InMemory.ConformanceTests/Support/InMemoryTestStore.cs new file mode 100644 index 0000000..a109817 --- /dev/null +++ b/MEVD/test/InMemory.ConformanceTests/Support/InMemoryTestStore.cs @@ -0,0 +1,26 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using CommunityToolkit.VectorData.InMemory; +using VectorData.ConformanceTests.Support; + +namespace InMemory.ConformanceTests.Support; + +internal sealed class InMemoryTestStore : TestStore +{ + public static InMemoryTestStore Instance { get; } = new(); + + public InMemoryVectorStore GetVectorStore(InMemoryVectorStoreOptions options) + => new(new() { EmbeddingGenerator = options.EmbeddingGenerator }); + + private InMemoryTestStore() + { + } + + protected override Task StartAsync() + { + this.DefaultVectorStore = new InMemoryVectorStore(); + + return Task.CompletedTask; + } +} diff --git a/MEVD/test/InMemory.ConformanceTests/TypeTests/InMemoryDataTypeTests.cs b/MEVD/test/InMemory.ConformanceTests/TypeTests/InMemoryDataTypeTests.cs new file mode 100644 index 0000000..524db94 --- /dev/null +++ b/MEVD/test/InMemory.ConformanceTests/TypeTests/InMemoryDataTypeTests.cs @@ -0,0 +1,20 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using InMemory.ConformanceTests.Support; +using VectorData.ConformanceTests.Support; +using VectorData.ConformanceTests.TypeTests; +using Xunit; + +namespace InMemory.ConformanceTests.TypeTests; + +public class InMemoryDataTypeTests(InMemoryDataTypeTests.Fixture fixture) + : DataTypeTests.DefaultRecord>(fixture), IClassFixture +{ + public new class Fixture : DataTypeTests.DefaultRecord>.Fixture + { + public override TestStore TestStore => InMemoryTestStore.Instance; + + public override Type[] UnsupportedDefaultTypes { get; } = []; + } +} diff --git a/MEVD/test/InMemory.ConformanceTests/TypeTests/InMemoryEmbeddingTypeTests.cs b/MEVD/test/InMemory.ConformanceTests/TypeTests/InMemoryEmbeddingTypeTests.cs new file mode 100644 index 0000000..decf02b --- /dev/null +++ b/MEVD/test/InMemory.ConformanceTests/TypeTests/InMemoryEmbeddingTypeTests.cs @@ -0,0 +1,23 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using InMemory.ConformanceTests.Support; +using VectorData.ConformanceTests.Support; +using VectorData.ConformanceTests.TypeTests; +using Xunit; + +#pragma warning disable CA2000 // Dispose objects before losing scope + +namespace InMemory.ConformanceTests.TypeTests; + +public class InMemoryEmbeddingTypeTests(InMemoryEmbeddingTypeTests.Fixture fixture) + : EmbeddingTypeTests(fixture), IClassFixture +{ + public new class Fixture : EmbeddingTypeTests.Fixture + { + public override TestStore TestStore => InMemoryTestStore.Instance; + + public override bool RecreateCollection => true; + public override bool AssertNoVectorsLoadedWithEmbeddingGeneration => false; + } +} diff --git a/MEVD/test/InMemory.ConformanceTests/TypeTests/InMemoryKeyTypeTests.cs b/MEVD/test/InMemory.ConformanceTests/TypeTests/InMemoryKeyTypeTests.cs new file mode 100644 index 0000000..334dd69 --- /dev/null +++ b/MEVD/test/InMemory.ConformanceTests/TypeTests/InMemoryKeyTypeTests.cs @@ -0,0 +1,39 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using InMemory.ConformanceTests.Support; +using VectorData.ConformanceTests.Support; +using VectorData.ConformanceTests.TypeTests; +using Xunit; + +namespace InMemory.ConformanceTests.TypeTests; + +public class InMemoryKeyTypeTests(InMemoryKeyTypeTests.Fixture fixture) + : KeyTypeTests(fixture), IClassFixture +{ + // The InMemory provider supports all .NET types as keys; below are just a few basic tests. + + [Fact] + public virtual Task Int() => this.Test(8, 9); + + [Fact] + public virtual Task Long() => this.Test(8L, 9L); + + [Fact] + public virtual Task String() => this.Test("foo", "bar"); + + protected override async Task Test(TKey key1, TKey key2, bool supportsAutoGeneration = false) + { + await base.Test(key1, key2, supportsAutoGeneration); + + // For InMemory, delete the collection, otherwise the next test that runs will fail because the collection + // already exists but with the previous key type. + using var collection = fixture.CreateCollection(supportsAutoGeneration); + await collection.EnsureCollectionDeletedAsync(); + } + + public new class Fixture : KeyTypeTests.Fixture + { + public override TestStore TestStore => InMemoryTestStore.Instance; + } +} diff --git a/MEVD/test/InMemory.UnitTests/.editorconfig b/MEVD/test/InMemory.UnitTests/.editorconfig new file mode 100644 index 0000000..394eef6 --- /dev/null +++ b/MEVD/test/InMemory.UnitTests/.editorconfig @@ -0,0 +1,6 @@ +# Suppressing errors for Test projects under dotnet folder +[*.cs] +dotnet_diagnostic.CA2007.severity = none # Do not directly await a Task +dotnet_diagnostic.VSTHRD111.severity = none # Use .ConfigureAwait(bool) is hidden by default, set to none to prevent IDE from changing on autosave +dotnet_diagnostic.CS1591.severity = none # Missing XML comment for publicly visible type or member +dotnet_diagnostic.IDE1006.severity = warning # Naming rule violations diff --git a/MEVD/test/InMemory.UnitTests/InMemory.UnitTests.csproj b/MEVD/test/InMemory.UnitTests/InMemory.UnitTests.csproj new file mode 100644 index 0000000..5948eca --- /dev/null +++ b/MEVD/test/InMemory.UnitTests/InMemory.UnitTests.csproj @@ -0,0 +1,39 @@ + + + + CommunityToolkit.VectorData.InMemory.UnitTests + CommunityToolkit.VectorData.InMemory.UnitTests + net10.0 + true + enable + disable + false + $(NoWarn);CA2007,CA1806,CA1869,CA1861,IDE0300,VSTHRD111,SKEXP0001 + + + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + + + + + diff --git a/MEVD/test/InMemory.UnitTests/InMemoryServiceCollectionExtensionsTests.cs b/MEVD/test/InMemory.UnitTests/InMemoryServiceCollectionExtensionsTests.cs new file mode 100644 index 0000000..d494669 --- /dev/null +++ b/MEVD/test/InMemory.UnitTests/InMemoryServiceCollectionExtensionsTests.cs @@ -0,0 +1,71 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.VectorData; +using Microsoft.SemanticKernel; +using CommunityToolkit.VectorData.InMemory; +using Xunit; + +namespace SemanticKernel.Connectors.InMemory.UnitTests; + +/// +/// Contains tests for the class. +/// +public class InMemoryServiceCollectionExtensionsTests +{ + private readonly IServiceCollection _serviceCollection; + + public InMemoryServiceCollectionExtensionsTests() + { + this._serviceCollection = new ServiceCollection(); + } + + [Fact] + public void AddVectorStoreRegistersClass() + { + // Act. + this._serviceCollection.AddInMemoryVectorStore(); + + // Assert. + var serviceProvider = this._serviceCollection.BuildServiceProvider(); + var vectorStore = serviceProvider.GetRequiredService(); + Assert.NotNull(vectorStore); + Assert.IsType(vectorStore); + } + + [Fact] + public void AddVectorStoreRecordCollectionRegistersClass() + { + // Act. + this._serviceCollection.AddInMemoryVectorStoreRecordCollection("testcollection"); + + // Assert. + this.AssertVectorStoreRecordCollectionCreated(); + } + + private void AssertVectorStoreRecordCollectionCreated() + { + var serviceProvider = this._serviceCollection.BuildServiceProvider(); + + var collection = serviceProvider.GetRequiredService>(); + Assert.NotNull(collection); + Assert.IsType>(collection); + + var vectorizedSearch = serviceProvider.GetRequiredService>(); + Assert.NotNull(vectorizedSearch); + Assert.IsType>(vectorizedSearch); + } + +#pragma warning disable CA1812 // Avoid uninstantiated internal classes + private sealed class TestRecord +#pragma warning restore CA1812 // Avoid uninstantiated internal classes + { + [VectorStoreKey] + public string Id { get; set; } = string.Empty; + + [VectorStoreVector(dimensions: 4)] + public ReadOnlyMemory Vector { get; set; } + } +} diff --git a/MEVD/test/InMemory.UnitTests/InMemoryVectorStoreExtensionsTests.cs b/MEVD/test/InMemory.UnitTests/InMemoryVectorStoreExtensionsTests.cs new file mode 100644 index 0000000..15d6543 --- /dev/null +++ b/MEVD/test/InMemory.UnitTests/InMemoryVectorStoreExtensionsTests.cs @@ -0,0 +1,167 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData; +using CommunityToolkit.VectorData.InMemory; +using Xunit; + +namespace SemanticKernel.Connectors.InMemory.UnitTests; + +public class InMemoryVectorStoreExtensionsTests +{ + [Fact] + public async Task SerializeAndDeserializeCollectionRoundtripWorks() + { + // Arrange + using var vectorStore = new InMemoryVectorStore(); + var collectionName = "test-collection"; + var collection = vectorStore.GetCollection(collectionName); + + var record1 = new TestRecord + { + Key = Guid.NewGuid(), + Text = "First record", + Embedding = new ReadOnlyMemory(new float[] { 0.1f, 0.2f, 0.3f }) + }; + var record2 = new TestRecord + { + Key = Guid.NewGuid(), + Text = "Second record", + Embedding = new ReadOnlyMemory(new float[] { 0.4f, 0.5f, 0.6f }) + }; + + await collection.EnsureCollectionExistsAsync(); + await collection.UpsertAsync(new[] { record1, record2 }); + + // Act + using var memStream = new MemoryStream(); + await vectorStore.SerializeCollectionAsJsonAsync(collectionName, memStream); + memStream.Position = 0; + + // Simulate loading into a new store + using var newVectorStore = new InMemoryVectorStore(); + var deserializedCollection = await newVectorStore.DeserializeCollectionFromJsonAsync(memStream); + + // Assert + Assert.NotNull(deserializedCollection); + var loadedRecord1 = await deserializedCollection.GetAsync(record1.Key); + var loadedRecord2 = await deserializedCollection.GetAsync(record2.Key); + + Assert.NotNull(loadedRecord1); + Assert.NotNull(loadedRecord2); + Assert.Equal(record1.Text, loadedRecord1.Text); + Assert.Equal(record2.Text, loadedRecord2.Text); + Assert.Equal(record1.Embedding, loadedRecord1.Embedding); + Assert.Equal(record2.Embedding, loadedRecord2.Embedding); + } + + [Fact] + public async Task SerializeAndDeserializeCollectionRoundtripWithBuiltInEmbeddingGenerationWorks() + { + // Arrange + using var vectorStore = new InMemoryVectorStore(new() { EmbeddingGenerator = new FakeEmbeddingGenerator() }); + var collectionName = "test-collection"; + var collection = vectorStore.GetCollection(collectionName); + + var record1 = new TestRecordAutoEmbed + { + Key = Guid.NewGuid(), + Text = "First record", + }; + var record2 = new TestRecordAutoEmbed + { + Key = Guid.NewGuid(), + Text = "Second record", + }; + + await collection.EnsureCollectionExistsAsync(); + await collection.UpsertAsync(new[] { record1, record2 }); + + // Act + using var memStream = new MemoryStream(); + await vectorStore.SerializeCollectionAsJsonAsync(collectionName, memStream); + memStream.Position = 0; + + // Simulate loading into a new store + using var newVectorStore = new InMemoryVectorStore(new() { EmbeddingGenerator = new FakeEmbeddingGenerator() }); + var deserializedCollection = await newVectorStore.DeserializeCollectionFromJsonAsync(memStream); + + // Assert + Assert.NotNull(deserializedCollection); + var loadedRecord1 = await deserializedCollection.GetAsync(record1.Key); + var loadedRecord2 = await deserializedCollection.GetAsync(record2.Key); + + Assert.NotNull(loadedRecord1); + Assert.NotNull(loadedRecord2); + Assert.Equal(record1.Text, loadedRecord1.Text); + Assert.Equal(record2.Text, loadedRecord2.Text); + } + + [Fact] + public async Task DeserializeCollectionFromJsonAsyncThrowsOnInvalidJson() + { + using var vectorStore = new InMemoryVectorStore(); + using var memStream = new MemoryStream(System.Text.Encoding.UTF8.GetBytes("{ invalid json }")); + + await Assert.ThrowsAsync(async () => + { + await vectorStore.DeserializeCollectionFromJsonAsync(memStream); + }); + } + + private sealed class TestRecord + { + [VectorStoreKey] + public Guid Key { get; init; } + + [VectorStoreData] + public string Text { get; init; } = string.Empty; + + [VectorStoreVector(dimensions: 3)] + public ReadOnlyMemory Embedding { get; init; } + } + + private sealed class TestRecordAutoEmbed + { + [VectorStoreKey] + public Guid Key { get; init; } + + [VectorStoreData] + public string Text { get; init; } = string.Empty; + + [VectorStoreVector(dimensions: 3)] + public string Embedding => this.Text; + } + + private sealed class FakeEmbeddingGenerator() : IEmbeddingGenerator> + { + public Task>> GenerateAsync( + IEnumerable values, + EmbeddingGenerationOptions? options = null, + CancellationToken cancellationToken = default) + { + var results = new GeneratedEmbeddings>(); + + foreach (var value in values) + { + results.Add(new Embedding(new float[] { 0.1f, 0.2f, 0.3f })); + } + + return Task.FromResult(results); + } + + public object? GetService(Type serviceType, object? serviceKey = null) + => null; + + public void Dispose() + { + } + } +} diff --git a/MEVD/test/InMemory.UnitTests/InMemoryVectorStoreTests.cs b/MEVD/test/InMemory.UnitTests/InMemoryVectorStoreTests.cs new file mode 100644 index 0000000..340f583 --- /dev/null +++ b/MEVD/test/InMemory.UnitTests/InMemoryVectorStoreTests.cs @@ -0,0 +1,84 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.VectorData; +using CommunityToolkit.VectorData.InMemory; +using Xunit; + +namespace SemanticKernel.Connectors.InMemory.UnitTests; + +/// +/// Contains tests for the class. +/// +public class InMemoryVectorStoreTests +{ + private const string TestCollectionName = "testcollection"; + + [Fact] + public void GetCollectionReturnsCollection() + { + // Arrange. + using var sut = new InMemoryVectorStore(); + + // Act. + var actual = sut.GetCollection>(TestCollectionName); + + // Assert. + Assert.NotNull(actual); + Assert.IsType>>(actual); + } + + [Fact] + public void GetCollectionReturnsCollectionWithNonStringKey() + { + // Arrange. + using var sut = new InMemoryVectorStore(); + + // Act. + var actual = sut.GetCollection>(TestCollectionName); + + // Assert. + Assert.NotNull(actual); + Assert.IsType>>(actual); + } + + [Fact] + public async Task GetCollectionDoesNotAllowADifferentDataTypeThanPreviouslyUsedAsync() + { + // Arrange. + using var sut = new InMemoryVectorStore(); + var stringKeyCollection = sut.GetCollection>(TestCollectionName); + await stringKeyCollection.EnsureCollectionExistsAsync(); + + // Act and assert. + var exception = Assert.Throws(() => sut.GetCollection(TestCollectionName)); + Assert.Equal($"Collection '{TestCollectionName}' already exists and with data type 'SinglePropsModel`1' so cannot be re-created with data type 'SecondModel'.", exception.Message); + } + +#pragma warning disable CA1812 // Classes are used as generic arguments + private sealed class SinglePropsModel + { + [VectorStoreKey] + public required TKey Key { get; set; } + + [VectorStoreData] + public string Data { get; set; } = string.Empty; + + [VectorStoreVector(dimensions: 4)] + public ReadOnlyMemory? Vector { get; set; } + + public string? NotAnnotated { get; set; } + } + + private sealed class SecondModel + { + [VectorStoreKey] + public required int Key { get; set; } + + [VectorStoreData] + public string Data { get; set; } = string.Empty; + } +#pragma warning restore CA1812 +} diff --git a/MEVD/test/PgVector.ConformanceTests/ModelTests/PostgresBasicModelTests.cs b/MEVD/test/PgVector.ConformanceTests/ModelTests/PostgresBasicModelTests.cs new file mode 100644 index 0000000..6833624 --- /dev/null +++ b/MEVD/test/PgVector.ConformanceTests/ModelTests/PostgresBasicModelTests.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using PgVector.ConformanceTests.Support; +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace PgVector.ConformanceTests.ModelTests; + +public class PostgresBasicModelTests(PostgresBasicModelTests.Fixture fixture) + : BasicModelTests(fixture), IClassFixture +{ + public new class Fixture : BasicModelTests.Fixture + { + public override TestStore TestStore => PostgresTestStore.Instance; + } +} diff --git a/MEVD/test/PgVector.ConformanceTests/ModelTests/PostgresDynamicModelTests.cs b/MEVD/test/PgVector.ConformanceTests/ModelTests/PostgresDynamicModelTests.cs new file mode 100644 index 0000000..c90d9ac --- /dev/null +++ b/MEVD/test/PgVector.ConformanceTests/ModelTests/PostgresDynamicModelTests.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using PgVector.ConformanceTests.Support; +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace PgVector.ConformanceTests.ModelTests; + +public class PostgresDynamicModelTests(PostgresDynamicModelTests.Fixture fixture) + : DynamicModelTests(fixture), IClassFixture +{ + public new class Fixture : DynamicModelTests.Fixture + { + public override TestStore TestStore => PostgresTestStore.Instance; + } +} diff --git a/MEVD/test/PgVector.ConformanceTests/ModelTests/PostgresMultiVectorModelTests.cs b/MEVD/test/PgVector.ConformanceTests/ModelTests/PostgresMultiVectorModelTests.cs new file mode 100644 index 0000000..d2b732f --- /dev/null +++ b/MEVD/test/PgVector.ConformanceTests/ModelTests/PostgresMultiVectorModelTests.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using PgVector.ConformanceTests.Support; +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace PgVector.ConformanceTests.ModelTests; + +public class PostgresMultiVectorModelTests(PostgresMultiVectorModelTests.Fixture fixture) + : MultiVectorModelTests(fixture), IClassFixture +{ + public new class Fixture : MultiVectorModelTests.Fixture + { + public override TestStore TestStore => PostgresTestStore.Instance; + } +} diff --git a/MEVD/test/PgVector.ConformanceTests/ModelTests/PostgresNoDataModelTests.cs b/MEVD/test/PgVector.ConformanceTests/ModelTests/PostgresNoDataModelTests.cs new file mode 100644 index 0000000..88b4e84 --- /dev/null +++ b/MEVD/test/PgVector.ConformanceTests/ModelTests/PostgresNoDataModelTests.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using PgVector.ConformanceTests.Support; +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace PgVector.ConformanceTests.ModelTests; + +public class PostgresNoDataModelTests(PostgresNoDataModelTests.Fixture fixture) + : NoDataModelTests(fixture), IClassFixture +{ + public new class Fixture : NoDataModelTests.Fixture + { + public override TestStore TestStore => PostgresTestStore.Instance; + } +} diff --git a/MEVD/test/PgVector.ConformanceTests/ModelTests/PostgresNoVectorModelTests.cs b/MEVD/test/PgVector.ConformanceTests/ModelTests/PostgresNoVectorModelTests.cs new file mode 100644 index 0000000..07b9230 --- /dev/null +++ b/MEVD/test/PgVector.ConformanceTests/ModelTests/PostgresNoVectorModelTests.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using PgVector.ConformanceTests.Support; +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace PgVector.ConformanceTests.ModelTests; + +public class PostgresNoVectorModelTests(PostgresNoVectorModelTests.Fixture fixture) + : NoVectorModelTests(fixture), IClassFixture +{ + public new class Fixture : NoVectorModelTests.Fixture + { + public override TestStore TestStore => PostgresTestStore.Instance; + } +} diff --git a/MEVD/test/PgVector.ConformanceTests/PgVector.ConformanceTests.csproj b/MEVD/test/PgVector.ConformanceTests/PgVector.ConformanceTests.csproj new file mode 100644 index 0000000..00515ea --- /dev/null +++ b/MEVD/test/PgVector.ConformanceTests/PgVector.ConformanceTests.csproj @@ -0,0 +1,32 @@ + + + + net10.0;$(NetFrameworkTfm) + enable + enable + true + false + PgVector.ConformanceTests + b7762d10-e29b-4bb1-8b74-b6d69a667dd4 + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + + + + + + + + diff --git a/MEVD/test/PgVector.ConformanceTests/PgVectorTestSuiteImplementationTests.cs b/MEVD/test/PgVector.ConformanceTests/PgVectorTestSuiteImplementationTests.cs new file mode 100644 index 0000000..ea96d34 --- /dev/null +++ b/MEVD/test/PgVector.ConformanceTests/PgVectorTestSuiteImplementationTests.cs @@ -0,0 +1,15 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using VectorData.ConformanceTests; + +namespace PgVector.ConformanceTests; + +public class PostgresTestSuiteImplementationTests : TestSuiteImplementationTests +{ + protected override ICollection IgnoredTestBases { get; } = + [ + // Hybrid search not supported + typeof(HybridSearchTests<>) + ]; +} diff --git a/MEVD/test/PgVector.ConformanceTests/PostgresCollectionManagementTests.cs b/MEVD/test/PgVector.ConformanceTests/PostgresCollectionManagementTests.cs new file mode 100644 index 0000000..392f9d8 --- /dev/null +++ b/MEVD/test/PgVector.ConformanceTests/PostgresCollectionManagementTests.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using PgVector.ConformanceTests.Support; +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace PgVector.ConformanceTests; + +public class PostgresCollectionManagementTests(PostgresCollectionManagementTests.Fixture fixture) + : CollectionManagementTests(fixture), IClassFixture +{ + public class Fixture : VectorStoreFixture + { + public override TestStore TestStore => PostgresTestStore.Instance; + } +} diff --git a/MEVD/test/PgVector.ConformanceTests/PostgresDependencyInjectionTests.cs b/MEVD/test/PgVector.ConformanceTests/PostgresDependencyInjectionTests.cs new file mode 100644 index 0000000..2b1772e --- /dev/null +++ b/MEVD/test/PgVector.ConformanceTests/PostgresDependencyInjectionTests.cs @@ -0,0 +1,90 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using CommunityToolkit.VectorData.PgVector; +using VectorData.ConformanceTests; +using Xunit; + +namespace PgVector.ConformanceTests; + +public class PostgresDependencyInjectionTests + : DependencyInjectionTests.Record>, string, DependencyInjectionTests.Record> +{ + protected const string ConnectionString = "Host=localhost;Database=test;"; + + protected override void PopulateConfiguration(ConfigurationManager configuration, object? serviceKey = null) + => configuration.AddInMemoryCollection( + [ + new(CreateConfigKey("Postgres", serviceKey, "ConnectionString"), ConnectionString), + ]); + + private static string ConnectionStringProvider(IServiceProvider sp) + => sp.GetRequiredService().GetRequiredSection("Postgres:ConnectionString").Value!; + + private static string ConnectionStringProvider(IServiceProvider sp, object serviceKey) + => sp.GetRequiredService().GetRequiredSection(CreateConfigKey("Postgres", serviceKey, "ConnectionString")).Value!; + + public override IEnumerable> CollectionDelegates + { + get + { + yield return (services, serviceKey, name, lifetime) => serviceKey is null + ? services.AddPostgresCollection( + name, connectionString: ConnectionString, lifetime: lifetime) + : services.AddKeyedPostgresCollection( + serviceKey, name, connectionString: ConnectionString, lifetime: lifetime); + + yield return (services, serviceKey, name, lifetime) => serviceKey is null + ? services.AddPostgresCollection( + name, ConnectionStringProvider, lifetime: lifetime) + : services.AddKeyedPostgresCollection( + serviceKey, name, sp => ConnectionStringProvider(sp, serviceKey), lifetime: lifetime); + } + } + + public override IEnumerable> StoreDelegates + { + get + { + yield return (services, serviceKey, lifetime) => serviceKey is null + ? services.AddPostgresVectorStore( + ConnectionString, lifetime: lifetime) + : services.AddKeyedPostgresVectorStore( + serviceKey, ConnectionString, lifetime: lifetime); + + yield return (services, serviceKey, lifetime) => serviceKey is null + ? services.AddPostgresVectorStore( + connectionStringProvider: ConnectionStringProvider, lifetime: lifetime) + : services.AddKeyedPostgresVectorStore( + serviceKey, connectionStringProvider: sp => ConnectionStringProvider(sp, serviceKey), lifetime: lifetime); + } + } + + [Fact] + public void ConnectionStringProviderCantBeNull() + { + IServiceCollection services = new ServiceCollection(); + + Assert.Throws(() => services.AddPostgresCollection(name: "notNull", connectionStringProvider: null!)); + Assert.Throws(() => services.AddKeyedPostgresCollection(serviceKey: "notNull", name: "notNull", connectionStringProvider: null!)); + } + + [Fact] + public void ConnectionStringCantBeNullOrEmpty() + { + IServiceCollection services = new ServiceCollection(); + + Assert.Throws(() => services.AddPostgresVectorStore(connectionString: null!)); + Assert.Throws(() => services.AddKeyedPostgresVectorStore(serviceKey: "notNull", connectionString: null!)); + Assert.Throws(() => services.AddPostgresCollection( + name: "notNull", connectionString: null!)); + Assert.Throws(() => services.AddPostgresCollection( + name: "notNull", connectionString: "")); + Assert.Throws(() => services.AddKeyedPostgresCollection( + serviceKey: "notNull", name: "notNull", connectionString: null!)); + Assert.Throws(() => services.AddKeyedPostgresCollection( + serviceKey: "notNull", name: "notNull", connectionString: "")); + } +} diff --git a/MEVD/test/PgVector.ConformanceTests/PostgresDistanceFunctionTests.cs b/MEVD/test/PgVector.ConformanceTests/PostgresDistanceFunctionTests.cs new file mode 100644 index 0000000..9e14cee --- /dev/null +++ b/MEVD/test/PgVector.ConformanceTests/PostgresDistanceFunctionTests.cs @@ -0,0 +1,29 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.VectorData; +using PgVector.ConformanceTests.Support; +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace PgVector.ConformanceTests; + +public class PostgresDistanceFunctionTests(PostgresDistanceFunctionTests.Fixture fixture) + : DistanceFunctionTests(fixture), IClassFixture +{ + public override Task EuclideanSquaredDistance() => Assert.ThrowsAsync(base.EuclideanSquaredDistance); + public override Task NegativeDotProductSimilarity() => Assert.ThrowsAsync(base.NegativeDotProductSimilarity); + + public override async Task HammingDistance() + { + // Hamming distance is supported by pgvector, but only on binary vectors (bit(x)), and the test uses float32 vectors (vector(x)). + var exception = await Assert.ThrowsAsync(base.HammingDistance); + Assert.IsType(exception.InnerException); + } + + public new class Fixture() : DistanceFunctionTests.Fixture + { + public override TestStore TestStore => PostgresTestStore.Instance; + } +} diff --git a/MEVD/test/PgVector.ConformanceTests/PostgresEmbeddingGenerationTests.cs b/MEVD/test/PgVector.ConformanceTests/PostgresEmbeddingGenerationTests.cs new file mode 100644 index 0000000..baa240f --- /dev/null +++ b/MEVD/test/PgVector.ConformanceTests/PostgresEmbeddingGenerationTests.cs @@ -0,0 +1,60 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.VectorData; +using PgVector.ConformanceTests.Support; +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace PgVector.ConformanceTests; + +public class PostgresEmbeddingGenerationTests(PostgresEmbeddingGenerationTests.StringVectorFixture stringVectorFixture, PostgresEmbeddingGenerationTests.RomOfFloatVectorFixture romOfFloatVectorFixture) + : EmbeddingGenerationTests(stringVectorFixture, romOfFloatVectorFixture), IClassFixture, IClassFixture +{ + public new class StringVectorFixture : EmbeddingGenerationTests.StringVectorFixture + { + public override TestStore TestStore => PostgresTestStore.Instance; + + public override VectorStore CreateVectorStore(IEmbeddingGenerator? embeddingGenerator) + => PostgresTestStore.Instance.GetVectorStore(new() { EmbeddingGenerator = embeddingGenerator }); + + public override Func[] DependencyInjectionStoreRegistrationDelegates => + [ + services => services + .AddSingleton(PostgresTestStore.Instance.DataSource) + .AddPostgresVectorStore() + ]; + + public override Func[] DependencyInjectionCollectionRegistrationDelegates => + [ + services => services + .AddSingleton(PostgresTestStore.Instance.DataSource) + .AddPostgresCollection(this.CollectionName) + ]; + } + + public new class RomOfFloatVectorFixture : EmbeddingGenerationTests.RomOfFloatVectorFixture + { + public override TestStore TestStore => PostgresTestStore.Instance; + + public override VectorStore CreateVectorStore(IEmbeddingGenerator? embeddingGenerator) + => PostgresTestStore.Instance.GetVectorStore(new() { EmbeddingGenerator = embeddingGenerator }); + + public override Func[] DependencyInjectionStoreRegistrationDelegates => + [ + services => services + .AddSingleton(PostgresTestStore.Instance.DataSource) + .AddPostgresVectorStore(), + ]; + + public override Func[] DependencyInjectionCollectionRegistrationDelegates => + [ + services => services + .AddSingleton(PostgresTestStore.Instance.DataSource) + .AddPostgresCollection(this.CollectionName), + ]; + } +} diff --git a/MEVD/test/PgVector.ConformanceTests/PostgresFilterTests.cs b/MEVD/test/PgVector.ConformanceTests/PostgresFilterTests.cs new file mode 100644 index 0000000..b23fbb6 --- /dev/null +++ b/MEVD/test/PgVector.ConformanceTests/PostgresFilterTests.cs @@ -0,0 +1,43 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using PgVector.ConformanceTests.Support; +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.Support; +using Xunit; +using Xunit.Sdk; + +namespace PgVector.ConformanceTests; + +#pragma warning disable CS0252 // Possible unintended reference comparison; left hand side needs cast + +public class PostgresFilterTests(PostgresFilterTests.Fixture fixture) + : FilterTests(fixture), IClassFixture +{ + public override async Task Not_over_Or() + { + // Test sends: WHERE (NOT (("Int" = 8) OR ("String" = 'foo'))) + // There's a NULL string in the database, and relational null semantics in conjunction with negation makes the default implementation fail. + await Assert.ThrowsAsync(() => base.Not_over_Or()); + + // Compensate by adding a null check: + await this.TestFilterAsync( + r => r.String != null && !(r.Int == 8 || r.String == "foo"), + r => r["String"] != null && !((int)r["Int"]! == 8 || r["String"] == "foo")); + } + + public override async Task NotEqual_with_string() + { + // As above, null semantics + negation + await Assert.ThrowsAsync(() => base.NotEqual_with_string()); + + await this.TestFilterAsync( + r => r.String != null && r.String != "foo", + r => r["String"] != null && r["String"] != "foo"); + } + + public new class Fixture : FilterTests.Fixture + { + public override TestStore TestStore => PostgresTestStore.Instance; + } +} diff --git a/MEVD/test/PgVector.ConformanceTests/PostgresHybridSearchTests.cs b/MEVD/test/PgVector.ConformanceTests/PostgresHybridSearchTests.cs new file mode 100644 index 0000000..53d1e3f --- /dev/null +++ b/MEVD/test/PgVector.ConformanceTests/PostgresHybridSearchTests.cs @@ -0,0 +1,27 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using PgVector.ConformanceTests.Support; +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace PgVector.ConformanceTests; + +public class PostgresHybridSearchTests( + PostgresHybridSearchTests.VectorAndStringFixture vectorAndStringFixture, + PostgresHybridSearchTests.MultiTextFixture multiTextFixture) + : HybridSearchTests(vectorAndStringFixture, multiTextFixture), + IClassFixture, + IClassFixture +{ + public new class VectorAndStringFixture : HybridSearchTests.VectorAndStringFixture + { + public override TestStore TestStore => PostgresTestStore.Instance; + } + + public new class MultiTextFixture : HybridSearchTests.MultiTextFixture + { + public override TestStore TestStore => PostgresTestStore.Instance; + } +} diff --git a/MEVD/test/PgVector.ConformanceTests/PostgresIndexKindTests.cs b/MEVD/test/PgVector.ConformanceTests/PostgresIndexKindTests.cs new file mode 100644 index 0000000..45a7c59 --- /dev/null +++ b/MEVD/test/PgVector.ConformanceTests/PostgresIndexKindTests.cs @@ -0,0 +1,23 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.VectorData; +using PgVector.ConformanceTests.Support; +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace PgVector.ConformanceTests; + +public class PostgresIndexKindTests(PostgresIndexKindTests.Fixture fixture) + : IndexKindTests(fixture), IClassFixture +{ + [Fact] + public virtual Task Hnsw() + => this.Test(IndexKind.Hnsw); + + public new class Fixture() : IndexKindTests.Fixture + { + public override TestStore TestStore => PostgresTestStore.Instance; + } +} diff --git a/MEVD/test/PgVector.ConformanceTests/README.md b/MEVD/test/PgVector.ConformanceTests/README.md new file mode 100644 index 0000000..3d03323 --- /dev/null +++ b/MEVD/test/PgVector.ConformanceTests/README.md @@ -0,0 +1,64 @@ +# PostgreSQL Vector Store Conformance Tests + +This project contains conformance tests for the PostgreSQL (pgvector) Vector Store implementation. + +## Running the Tests + +By default, the tests will automatically use a testcontainer to spin up a PostgreSQL instance with the pgvector extension. Docker must be running on your machine for this to work. + +### Using an External PostgreSQL Instance + +If you want to run the tests against an external PostgreSQL instance (e.g., a cloud database, local PostgreSQL server, or any other instance), you can provide a connection string through one of the following methods: + +**Note**: The external PostgreSQL instance must have the `pgvector` extension available. The tests will attempt to create the extension if it doesn't exist. + +#### Option 1: Environment Variable + +Set the `Postgres__ConnectionString` environment variable: + +```bash +# Bash/Linux/macOS +export Postgres__ConnectionString="Host=myserver.postgres.database.azure.com;Database=mydb;Username=myuser;Password=mypassword;" + +# PowerShell +$env:Postgres__ConnectionString = "Host=myserver.postgres.database.azure.com;Database=mydb;Username=myuser;Password=mypassword;" +``` + +#### Option 2: Configuration File + +Create a `testsettings.development.json` file in this directory with the following content: + +```json +{ + "Postgres": { + "ConnectionString": "Host=myserver.postgres.database.azure.com;Database=mydb;Username=myuser;Password=mypassword;" + } +} +``` + +This file is git-ignored and safe for local development. + +#### Option 3: User Secrets + +```bash +cd test/VectorData/PgVector.ConformanceTests +dotnet user-secrets set "Postgres:ConnectionString" "Host=myserver.postgres.database.azure.com;Database=mydb;Username=myuser;Password=mypassword;" +``` + +## Benefits of Using an External Instance + +Using an external PostgreSQL instance can be beneficial when: +- You want to avoid the overhead of spinning up Docker containers +- You need to test against a specific PostgreSQL version or configuration +- You want faster test execution (no container startup time) +- You're running tests in an environment where Docker is not available +- You need to test against cloud-hosted PostgreSQL (e.g., Azure Database for PostgreSQL, AWS RDS) + +## Prerequisites for External Instances + +The external PostgreSQL instance must: +- Have the `pgvector` extension installed and available +- Have sufficient permissions for the connecting user to: + - Create the vector extension (if not already created) + - Create tables and indexes + - Insert, update, and delete data diff --git a/MEVD/test/PgVector.ConformanceTests/Support/PostgresTestEnvironment.cs b/MEVD/test/PgVector.ConformanceTests/Support/PostgresTestEnvironment.cs new file mode 100644 index 0000000..90a116b --- /dev/null +++ b/MEVD/test/PgVector.ConformanceTests/Support/PostgresTestEnvironment.cs @@ -0,0 +1,28 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.Configuration; + +namespace PgVector.ConformanceTests.Support; + +#pragma warning disable CA1810 // Initialize all static fields when those fields are declared + +internal static class PostgresTestEnvironment +{ + public static readonly string? ConnectionString; + + public static bool IsConnectionStringDefined => ConnectionString is not null; + + static PostgresTestEnvironment() + { + var configuration = new ConfigurationBuilder() + .AddJsonFile(path: "testsettings.json", optional: true) + .AddJsonFile(path: "testsettings.development.json", optional: true) + .AddEnvironmentVariables() + .AddUserSecrets() + .Build(); + + var postgresSection = configuration.GetSection("Postgres"); + ConnectionString = postgresSection["ConnectionString"]; + } +} diff --git a/MEVD/test/PgVector.ConformanceTests/Support/PostgresTestStore.cs b/MEVD/test/PgVector.ConformanceTests/Support/PostgresTestStore.cs new file mode 100644 index 0000000..491de3d --- /dev/null +++ b/MEVD/test/PgVector.ConformanceTests/Support/PostgresTestStore.cs @@ -0,0 +1,85 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using CommunityToolkit.VectorData.PgVector; +using Npgsql; +using Testcontainers.PostgreSql; +using VectorData.ConformanceTests.Support; + +namespace PgVector.ConformanceTests.Support; + +#pragma warning disable CA1001 // Type owns disposable fields but is not disposable + +internal sealed class PostgresTestStore : TestStore +{ + public static PostgresTestStore Instance { get; } = new(); + + private static readonly PostgreSqlContainer s_container = new PostgreSqlBuilder("pgvector/pgvector:pg18") + .Build(); + + private NpgsqlDataSource? _dataSource; + private string? _connectionString; + private bool _useExternalInstance; + + public NpgsqlDataSource DataSource => this._dataSource ?? throw new InvalidOperationException("Not initialized"); + + public string ConnectionString => this._connectionString ?? throw new InvalidOperationException("Not initialized"); + + public PostgresVectorStore GetVectorStore(PostgresVectorStoreOptions options) + { + // The DataSource is shared with the static instance, we don't want any of the tests to dispose it. + return new(this.DataSource, ownsDataSource: false, options); + } + + private PostgresTestStore() + { + } + + protected override async Task StartAsync() + { + // Determine connection string source + if (PostgresTestEnvironment.IsConnectionStringDefined) + { + this._connectionString = PostgresTestEnvironment.ConnectionString!; + this._useExternalInstance = true; + } + else + { + // Use testcontainer if no external connection string is provided + await s_container.StartAsync(); + + NpgsqlConnectionStringBuilder connectionStringBuilder = new() + { + Host = s_container.Hostname, + Port = s_container.GetMappedPublicPort(5432), + Username = PostgreSqlBuilder.DefaultUsername, + Password = PostgreSqlBuilder.DefaultPassword, + Database = PostgreSqlBuilder.DefaultDatabase + }; + + this._connectionString = connectionStringBuilder.ConnectionString; + this._useExternalInstance = false; + } + + NpgsqlDataSourceBuilder dataSourceBuilder = new(this._connectionString!); + dataSourceBuilder.UseVector(); + this._dataSource = dataSourceBuilder.Build(); + + // It's a shared static instance, we don't want any of the tests to dispose it. + this.DefaultVectorStore = new PostgresVectorStore(this._dataSource, ownsDataSource: false); + } + + protected override async Task StopAsync() + { + if (this._dataSource is not null) + { + await this._dataSource.DisposeAsync(); + } + + // Only stop the container if we started it + if (!this._useExternalInstance) + { + await s_container.StopAsync(); + } + } +} diff --git a/MEVD/test/PgVector.ConformanceTests/TypeTests/PostgresDataTypeTests.cs b/MEVD/test/PgVector.ConformanceTests/TypeTests/PostgresDataTypeTests.cs new file mode 100644 index 0000000..08f8318 --- /dev/null +++ b/MEVD/test/PgVector.ConformanceTests/TypeTests/PostgresDataTypeTests.cs @@ -0,0 +1,46 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using PgVector.ConformanceTests.Support; +using VectorData.ConformanceTests.Support; +using VectorData.ConformanceTests.TypeTests; +using Xunit; + +namespace PgVector.ConformanceTests.TypeTests; + +public class PostgresDataTypeTests(PostgresDataTypeTests.Fixture fixture) + : DataTypeTests.DefaultRecord>(fixture), IClassFixture +{ + // Npgsql maps DateTime to timestamptz by default, and so requires UTC DateTimes (the base test uses Unspecified). + public override Task DateTime() + => Assert.ThrowsAsync(base.DateTime); + + public virtual Task DateTime_utc() + => this.Test( + "DateTime", + System.DateTime.SpecifyKind(new DateTime(2020, 1, 1, 12, 30, 45), DateTimeKind.Utc), + System.DateTime.SpecifyKind(new DateTime(2021, 2, 3, 13, 40, 55), DateTimeKind.Utc), + instantiationExpression: () => System.DateTime.SpecifyKind(new DateTime(2020, 1, 1, 12, 30, 45), DateTimeKind.Utc)); + + // PostgreSQL does not support representing an offset, so only DateTimeOffsets with offset=0 are supported. + public override Task DateTimeOffset() + => Assert.ThrowsAsync(base.DateTimeOffset); + + // PostgreSQL does not support representing an offset, so only DateTimeOffsets with offset=0 are supported. + public virtual Task DateTimeOffset_with_offset_zero() + => this.Test( + "DateTimeOffset", + new DateTimeOffset(2020, 1, 1, 12, 30, 45, TimeSpan.FromHours(0)), + new DateTimeOffset(2021, 2, 3, 13, 40, 55, TimeSpan.FromHours(0)), + instantiationExpression: () => new DateTimeOffset(2020, 1, 1, 12, 30, 45, TimeSpan.FromHours(0))); + + public new class Fixture : DataTypeTests.DefaultRecord>.Fixture + { + public override TestStore TestStore => PostgresTestStore.Instance; + + public override Type[] UnsupportedDefaultTypes { get; } = + [ + typeof(byte), + ]; + } +} diff --git a/MEVD/test/PgVector.ConformanceTests/TypeTests/PostgresEmbeddingTypeTests.cs b/MEVD/test/PgVector.ConformanceTests/TypeTests/PostgresEmbeddingTypeTests.cs new file mode 100644 index 0000000..cf25d43 --- /dev/null +++ b/MEVD/test/PgVector.ConformanceTests/TypeTests/PostgresEmbeddingTypeTests.cs @@ -0,0 +1,67 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData; +using Pgvector; +using PgVector.ConformanceTests.Support; +using VectorData.ConformanceTests.Support; +using VectorData.ConformanceTests.TypeTests; +using Xunit; + +#pragma warning disable CA2000 // Dispose objects before losing scope + +namespace PgVector.ConformanceTests.TypeTests; + +public class PostgresEmbeddingTypeTests(PostgresEmbeddingTypeTests.Fixture fixture) + : EmbeddingTypeTests(fixture), IClassFixture +{ +#if NET + [Fact] + public virtual Task ReadOnlyMemory_of_Half() + => this.Test>( + new ReadOnlyMemory([(byte)1, (byte)2, (byte)3]), + new ReadOnlyMemoryEmbeddingGenerator([(byte)1, (byte)2, (byte)3]), + vectorEqualityAsserter: (e, a) => Assert.Equal(e.Span.ToArray(), a.Span.ToArray())); + + [Fact] + public virtual Task Embedding_of_Half() + => this.Test>( + new Embedding(new ReadOnlyMemory([(byte)1, (byte)2, (byte)3])), + new ReadOnlyMemoryEmbeddingGenerator([(byte)1, (byte)2, (byte)3]), + vectorEqualityAsserter: (e, a) => Assert.Equal(e.Vector.Span.ToArray(), a.Vector.Span.ToArray())); + + [Fact] + public virtual Task Array_of_Half() + => this.Test( + [(byte)1, (byte)2, (byte)3], + new ReadOnlyMemoryEmbeddingGenerator([(byte)1, (byte)2, (byte)3])); +#endif + + [Fact] + public virtual Task BitArray() + => this.Test( + new BitArray([true, false, true]), + new BinaryEmbeddingGenerator(new BitArray([true, false, true])), + distanceFunction: DistanceFunction.HammingDistance); + + [Fact] + public virtual Task BinaryEmbedding() + => this.Test( + new BinaryEmbedding(new([true, false, true])), + new BinaryEmbeddingGenerator(new BitArray([true, false, true])), + distanceFunction: DistanceFunction.HammingDistance, + vectorEqualityAsserter: (e, a) => Assert.Equal(e.Vector, a.Vector)); + + [Fact] + public virtual Task SparseVector() + => this.Test(new SparseVector(new ReadOnlyMemory([1, 2, 3])), embeddingGenerator: null); + + // TODO: Figure out the embedding generation story for sparsevec - need an Embedding wrapper + + public new class Fixture : EmbeddingTypeTests.Fixture + { + public override TestStore TestStore => PostgresTestStore.Instance; + } +} diff --git a/MEVD/test/PgVector.ConformanceTests/TypeTests/PostgresKeyTypeTests.cs b/MEVD/test/PgVector.ConformanceTests/TypeTests/PostgresKeyTypeTests.cs new file mode 100644 index 0000000..578a0a6 --- /dev/null +++ b/MEVD/test/PgVector.ConformanceTests/TypeTests/PostgresKeyTypeTests.cs @@ -0,0 +1,27 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using PgVector.ConformanceTests.Support; +using VectorData.ConformanceTests.Support; +using VectorData.ConformanceTests.TypeTests; +using Xunit; + +namespace PgVector.ConformanceTests.TypeTests; + +public class PostgresKeyTypeTests(PostgresKeyTypeTests.Fixture fixture) + : KeyTypeTests(fixture), IClassFixture +{ + [Fact] + public virtual Task Int() => this.Test(8, supportsAutoGeneration: true); + + [Fact] + public virtual Task Long() => this.Test(8L, supportsAutoGeneration: true); + + [Fact] + public virtual Task String() => this.Test("foo", "bar"); + + public new class Fixture : KeyTypeTests.Fixture + { + public override TestStore TestStore => PostgresTestStore.Instance; + } +} diff --git a/MEVD/test/PgVector.ConformanceTests/testsettings.json b/MEVD/test/PgVector.ConformanceTests/testsettings.json new file mode 100644 index 0000000..d57a302 --- /dev/null +++ b/MEVD/test/PgVector.ConformanceTests/testsettings.json @@ -0,0 +1,8 @@ +{ + // Optional: Provide a connection string to use an external PostgreSQL instance instead of testcontainers + // This is useful for running tests against external PostgreSQL instances or cloud databases + // If not specified, tests will automatically use a testcontainer + "Postgres": { + "ConnectionString": "" + } +} diff --git a/MEVD/test/PgVector.UnitTests/PgVector.UnitTests.csproj b/MEVD/test/PgVector.UnitTests/PgVector.UnitTests.csproj new file mode 100644 index 0000000..ef91455 --- /dev/null +++ b/MEVD/test/PgVector.UnitTests/PgVector.UnitTests.csproj @@ -0,0 +1,34 @@ + + + + CommunityToolkit.VectorData.PgVector.UnitTests + CommunityToolkit.VectorData.PgVector.UnitTests + net10.0 + true + enable + disable + false + $(NoWarn);SKEXP0001,VSTHRD111,CA2007,CS1591 + $(NoWarn);MEVD9001 + + + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + diff --git a/MEVD/test/PgVector.UnitTests/PostgresCollectionTests.cs b/MEVD/test/PgVector.UnitTests/PostgresCollectionTests.cs new file mode 100644 index 0000000..da69527 --- /dev/null +++ b/MEVD/test/PgVector.UnitTests/PostgresCollectionTests.cs @@ -0,0 +1,34 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using Microsoft.Extensions.VectorData; +using CommunityToolkit.VectorData.PgVector; +using Xunit; + +namespace SemanticKernel.Connectors.PgVector.UnitTests; + +public class PostgresCollectionTests +{ + private const string TestCollectionName = "testcollection"; + + [Fact] + public void ThrowsForUnsupportedType() + { + // Arrange + var recordDefinition = new VectorStoreCollectionDefinition + { + Properties = [ + new VectorStoreKeyProperty("HotelId", typeof(ulong)), + new VectorStoreDataProperty("HotelName", typeof(string)) { IsIndexed = true, IsFullTextIndexed = true }, + ] + }; + var options = new PostgresCollectionOptions() + { + Definition = recordDefinition + }; + + // Act & Assert + Assert.Throws(() => new PostgresDynamicCollection("Host=localhost;Database=test;", TestCollectionName, options)); + } +} diff --git a/MEVD/test/PgVector.UnitTests/PostgresFilterTranslatorTests.cs b/MEVD/test/PgVector.UnitTests/PostgresFilterTranslatorTests.cs new file mode 100644 index 0000000..c947838 --- /dev/null +++ b/MEVD/test/PgVector.UnitTests/PostgresFilterTranslatorTests.cs @@ -0,0 +1,101 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Linq.Expressions; +using System.Text; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; +using CommunityToolkit.VectorData.PgVector; +using Xunit; + +namespace SemanticKernel.Connectors.PgVector.UnitTests; + +public sealed class PostgresFilterTranslatorTests +{ + [Fact] + public void DateTime_Utc_FormattedWithZ() + { + // Inline new DateTime(..., DateTimeKind.Utc) in expression tree to exercise TranslateConstant + var sql = TranslateFilter(r => r.Created == new DateTime(2024, 6, 15, 10, 30, 45, DateTimeKind.Utc)); + Assert.Contains("'2024-06-15T10:30:45Z'", sql); + } + + [Fact] + public void DateTime_Unspecified_FormattedWithoutZ() + { + // Inline new DateTime(...) has Kind=Unspecified + var sql = TranslateFilter(r => r.Created == new DateTime(2024, 6, 15, 10, 30, 45)); + Assert.Contains("'2024-06-15T10:30:45'", sql); + Assert.DoesNotContain("Z'", sql); + } + + [Fact] + public void DateTimeOffset_Utc_FormattedWithZ() + { + // Use a ConstantExpression directly to bypass VisitNew issues with TimeSpan.Zero + var sql = TranslateFilterWithConstant( + nameof(TestRecord.CreatedOffset), + new DateTimeOffset(2024, 6, 15, 10, 30, 45, TimeSpan.Zero)); + Assert.Contains("'2024-06-15T10:30:45Z'", sql); + } + + [Fact] + public void DateTimeOffset_NonZeroOffset_Throws() + { + Assert.ThrowsAny(() => + TranslateFilterWithConstant( + nameof(TestRecord.CreatedOffset), + new DateTimeOffset(2024, 6, 15, 10, 30, 45, TimeSpan.FromHours(2)))); + } + + private static string TranslateFilter(Expression> filter) + { + var model = BuildModel(); + var sb = new StringBuilder(); + var translator = new PostgresFilterTranslator(model, filter, startParamIndex: 1, sb); + translator.Translate(appendWhere: false); + return sb.ToString(); + } + + private static string TranslateFilterWithConstant(string propertyName, TValue value) + { + var model = BuildModel(); + var param = Expression.Parameter(typeof(TRecord), "r"); + var prop = Expression.Property(param, propertyName); + var constant = Expression.Constant(value, typeof(TValue)); + var body = Expression.Equal(prop, constant); + var filter = Expression.Lambda>(body, param); + + var sb = new StringBuilder(); + var translator = new PostgresFilterTranslator(model, filter, startParamIndex: 1, sb); + translator.Translate(appendWhere: false); + return sb.ToString(); + } + + private static CollectionModel BuildModel() + { + var definition = new VectorStoreCollectionDefinition + { + Properties = + [ + new VectorStoreKeyProperty("Id", typeof(Guid)), + new VectorStoreDataProperty("Created", typeof(DateTime)), + new VectorStoreDataProperty("CreatedOffset", typeof(DateTimeOffset)), + new VectorStoreVectorProperty("Embedding", typeof(ReadOnlyMemory), 10) + ] + }; + + return new PostgresModelBuilder().BuildDynamic(definition, defaultEmbeddingGenerator: null); + } + +#pragma warning disable CA1812 // internal class that is apparently never instantiated. + private sealed record TestRecord + { + public Guid Id { get; set; } + public DateTime Created { get; set; } + public DateTimeOffset CreatedOffset { get; set; } + public ReadOnlyMemory Embedding { get; set; } + } +#pragma warning restore CA1812 +} diff --git a/MEVD/test/PgVector.UnitTests/PostgresHotel.cs b/MEVD/test/PgVector.UnitTests/PostgresHotel.cs new file mode 100644 index 0000000..5be93b9 --- /dev/null +++ b/MEVD/test/PgVector.UnitTests/PostgresHotel.cs @@ -0,0 +1,52 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using Microsoft.Extensions.VectorData; + +namespace SemanticKernel.Connectors.PgVector.UnitTests; + +#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable. + +/// +/// A test model for the postgres vector store. +/// +public record PostgresHotel() +{ + /// The key of the record. + [VectorStoreKey] + public T HotelId { get; init; } + + /// A string metadata field. + [VectorStoreData()] + public string? HotelName { get; set; } + + /// An int metadata field. + [VectorStoreData()] + public int HotelCode { get; set; } + + /// A float metadata field. + [VectorStoreData()] + public float? HotelRating { get; set; } + + /// A bool metadata field. + [VectorStoreData(StorageName = "parking_is_included")] + public bool ParkingIncluded { get; set; } + + [VectorStoreData] + public List Tags { get; set; } = []; + + /// A data field. + [VectorStoreData] + public string Description { get; set; } + + public DateTime CreatedAt { get; set; } = DateTime.UtcNow; + + public DateTimeOffset UpdatedAt { get; set; } = DateTimeOffset.UtcNow; + + /// A vector field. + [VectorStoreVector(dimensions: 4, DistanceFunction = IndexKind.Hnsw, IndexKind = DistanceFunction.ManhattanDistance)] + public ReadOnlyMemory? DescriptionEmbedding { get; set; } +} +#pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable. diff --git a/MEVD/test/PgVector.UnitTests/PostgresPropertyExtensionsTests.cs b/MEVD/test/PgVector.UnitTests/PostgresPropertyExtensionsTests.cs new file mode 100644 index 0000000..c02569d --- /dev/null +++ b/MEVD/test/PgVector.UnitTests/PostgresPropertyExtensionsTests.cs @@ -0,0 +1,126 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using Microsoft.Extensions.VectorData; +using CommunityToolkit.VectorData.PgVector; +using Xunit; + +namespace SemanticKernel.Connectors.PgVector.UnitTests; + +public sealed class PostgresPropertyExtensionsTests +{ + [Theory] + [InlineData("timestamp")] + [InlineData("TIMESTAMP")] + [InlineData("timestamp without time zone")] + [InlineData("TIMESTAMP WITHOUT TIME ZONE")] + public void WithStoreType_Timestamp_SetsAnnotation(string storeType) + { + var property = new VectorStoreDataProperty("test", typeof(DateTime)); + + var result = property.WithStoreType(storeType); + + Assert.Same(property, result); + Assert.Equal(storeType, property.GetStoreType()); + } + + [Theory] + [InlineData("timestamptz")] + [InlineData("timestamp with time zone")] + [InlineData("integer")] + [InlineData("text")] + [InlineData("")] + public void WithStoreType_UnsupportedType_Throws(string storeType) + { + var property = new VectorStoreDataProperty("test", typeof(DateTime)); + + Assert.Throws(() => property.WithStoreType(storeType)); + } + + [Fact] + public void GetStoreType_NotSet_ReturnsNull() + { + var property = new VectorStoreDataProperty("test", typeof(DateTime)); + + Assert.Null(property.GetStoreType()); + } + + [Fact] + public void WithStoreType_OnlyAllowedOnDateTimeProperties() + { + // Setting the annotation on a non-DateTime property should succeed at the property level + // (validation happens in the model builder), but building the model should throw. + var definition = new VectorStoreCollectionDefinition + { + Properties = + [ + new VectorStoreKeyProperty("id", typeof(Guid)), + new VectorStoreDataProperty("name", typeof(string)).WithStoreType("timestamp"), + new VectorStoreVectorProperty("embedding", typeof(ReadOnlyMemory), 10) + ] + }; + + Assert.Throws(() => + new PostgresModelBuilder().BuildDynamic(definition, defaultEmbeddingGenerator: null)); + } + + [Fact] + public void WithStoreType_AllowedOnDateTimeProperty() + { + var definition = new VectorStoreCollectionDefinition + { + Properties = + [ + new VectorStoreKeyProperty("id", typeof(Guid)), + new VectorStoreDataProperty("created", typeof(DateTime)).WithStoreType("timestamp"), + new VectorStoreVectorProperty("embedding", typeof(ReadOnlyMemory), 10) + ] + }; + + // Should not throw + var model = new PostgresModelBuilder().BuildDynamic(definition, defaultEmbeddingGenerator: null); + Assert.NotNull(model); + } + + [Fact] + public void WithStoreType_AllowedOnKeyProperty() + { + var property = new VectorStoreKeyProperty("id", typeof(DateTime)).WithStoreType("timestamp"); + Assert.Equal("timestamp", property.GetStoreType()); + } + + [Fact] + public void WithStoreType_OnKeyProperty_NonDateTime_Throws() + { + var definition = new VectorStoreCollectionDefinition + { + Properties = + [ + new VectorStoreKeyProperty("id", typeof(Guid)).WithStoreType("timestamp"), + new VectorStoreVectorProperty("embedding", typeof(ReadOnlyMemory), 10) + ] + }; + + Assert.Throws(() => + new PostgresModelBuilder().BuildDynamic(definition, defaultEmbeddingGenerator: null)); + } + + [Fact] + public void WithStoreType_AllowedOnNullableDateTimeProperty() + { + var definition = new VectorStoreCollectionDefinition + { + Properties = + [ + new VectorStoreKeyProperty("id", typeof(Guid)), + new VectorStoreDataProperty("created", typeof(DateTime?)).WithStoreType("timestamp"), + new VectorStoreVectorProperty("embedding", typeof(ReadOnlyMemory), 10) + ] + }; + + // Should not throw + var model = new PostgresModelBuilder().BuildDynamic(definition, defaultEmbeddingGenerator: null); + Assert.NotNull(model); + } +} diff --git a/MEVD/test/PgVector.UnitTests/PostgresPropertyMappingTests.cs b/MEVD/test/PgVector.UnitTests/PostgresPropertyMappingTests.cs new file mode 100644 index 0000000..a828daf --- /dev/null +++ b/MEVD/test/PgVector.UnitTests/PostgresPropertyMappingTests.cs @@ -0,0 +1,198 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; +using CommunityToolkit.VectorData.PgVector; +using NpgsqlTypes; +using Pgvector; +using Xunit; + +namespace SemanticKernel.Connectors.PgVector.UnitTests; + +/// +/// Unit tests for class. +/// +public sealed class PostgresPropertyMappingTests +{ + [Fact] + public void MapVectorForStorageModelWithInvalidVectorTypeThrowsException() + { + // Arrange + var vector = new List { 1f, 2f, 3f }; + + // Act & Assert + Assert.Throws(() => PostgresPropertyMapping.MapVectorForStorageModel(vector)); + } + + [Fact] + public void MapVectorForStorageModelReturnsVector() + { + // Arrange + var vector = new ReadOnlyMemory([1.1f, 2.2f, 3.3f, 4.4f]); + + // Act + var storageModelVector = PostgresPropertyMapping.MapVectorForStorageModel(vector); + + // Assert + var actual = Assert.IsType(storageModelVector); + Assert.True(actual.ToArray().Length > 0); + } + + [Fact] + public void GetPropertyValueReturnsCorrectValuesForLists() + { + // Arrange + var typesAndExpectedValues = new List<(Type, object)> + { + (typeof(List), "INTEGER[]"), + (typeof(List), "REAL[]"), + (typeof(List), "DOUBLE PRECISION[]"), + (typeof(List), "TEXT[]"), + (typeof(List), "BOOLEAN[]"), + (typeof(List), "TIMESTAMPTZ[]"), + (typeof(List), "UUID[]"), + }; + + // Act & Assert + foreach (var (type, expectedValue) in typesAndExpectedValues) + { + var property = new DataPropertyModel("test", type); + var (pgType, _) = PostgresPropertyMapping.GetPostgresTypeName(property); + Assert.Equal(expectedValue, pgType); + } + } + + [Fact] + public void GetPropertyValueReturnsCorrectNullableValue() + { + // Arrange + var typesAndExpectedValues = new List<(Type, object)> + { + (typeof(short), false), + (typeof(short?), true), + (typeof(int?), true), + (typeof(long), false), + (typeof(string), true), + (typeof(bool?), true), + (typeof(DateTime?), true), + (typeof(Guid), false), + }; + + // Act & Assert + foreach (var (type, expectedValue) in typesAndExpectedValues) + { + var property = new DataPropertyModel("test", type); + var (_, isNullable) = PostgresPropertyMapping.GetPostgresTypeName(property); + Assert.Equal(expectedValue, isNullable); + } + } + + [Fact] + public void GetIndexInfoReturnsCorrectValues() + { + // Arrange + List vectorProperties = + [ + new VectorPropertyModel("vector1", typeof(ReadOnlyMemory?)) { IndexKind = IndexKind.Hnsw, Dimensions = 1000 }, + new VectorPropertyModel("vector2", typeof(ReadOnlyMemory?)) { IndexKind = IndexKind.Flat, Dimensions = 3000 }, + new VectorPropertyModel("vector3", typeof(ReadOnlyMemory?)) { IndexKind = IndexKind.Hnsw, Dimensions = 900, DistanceFunction = DistanceFunction.ManhattanDistance }, + new DataPropertyModel("data1", typeof(string)) { IsIndexed = true }, + new DataPropertyModel("data2", typeof(string)) { IsIndexed = false } + ]; + + // Act + var indexInfo = PostgresPropertyMapping.GetIndexInfo(vectorProperties); + + // Assert + Assert.Equal(3, indexInfo.Count); + foreach (var (columnName, indexKind, distanceFunction, isVector, isFullText, fullTextLanguage) in indexInfo) + { + if (columnName == "vector1") + { + Assert.True(isVector); + Assert.False(isFullText); + Assert.Equal(IndexKind.Hnsw, indexKind); + Assert.Equal(DistanceFunction.CosineDistance, distanceFunction); + } + else if (columnName == "vector3") + { + Assert.True(isVector); + Assert.False(isFullText); + Assert.Equal(IndexKind.Hnsw, indexKind); + Assert.Equal(DistanceFunction.ManhattanDistance, distanceFunction); + } + else if (columnName == "data1") + { + Assert.False(isVector); + Assert.False(isFullText); + } + else + { + Assert.Fail("Unexpected column name"); + } + } + } + + [Theory] + [InlineData(IndexKind.Hnsw, 3000)] + public void GetVectorIndexInfoReturnsThrowsForInvalidDimensions(string indexKind, int dimensions) + { + // Arrange + var vectorProperty = new VectorPropertyModel("vector", typeof(ReadOnlyMemory?)) { IndexKind = indexKind, Dimensions = dimensions }; + + // Act & Assert + Assert.Throws(() => PostgresPropertyMapping.GetIndexInfo([vectorProperty])); + } + + [Fact] + public void GetPostgresTypeName_DateTime_WithTimestampAnnotation_ReturnsTimestamp() + { + var dataProperty = new DataPropertyModel("created", typeof(DateTime)); + dataProperty.ProviderAnnotations = new() { ["Postgres:StoreType"] = "timestamp" }; + + var (pgType, _) = PostgresPropertyMapping.GetPostgresTypeName(dataProperty); + Assert.Equal("TIMESTAMP", pgType); + } + + [Fact] + public void GetPostgresTypeName_DateTime_WithoutAnnotation_ReturnsTimestampTz() + { + var dataProperty = new DataPropertyModel("created", typeof(DateTime)); + + var (pgType, _) = PostgresPropertyMapping.GetPostgresTypeName(dataProperty); + Assert.Equal("TIMESTAMPTZ", pgType); + } + + [Fact] + public void GetPostgresTypeName_NullableDateTime_WithTimestampAnnotation_ReturnsTimestamp() + { + var dataProperty = new DataPropertyModel("created", typeof(DateTime?)); + dataProperty.ProviderAnnotations = new() { ["Postgres:StoreType"] = "timestamp" }; + + var (pgType, isNullable) = PostgresPropertyMapping.GetPostgresTypeName(dataProperty); + Assert.Equal("TIMESTAMP", pgType); + Assert.True(isNullable); + } + + [Fact] + public void GetNpgsqlDbType_DateTime_WithTimestampAnnotation_ReturnsTimestamp() + { + var dataProperty = new DataPropertyModel("created", typeof(DateTime)); + dataProperty.ProviderAnnotations = new() { ["Postgres:StoreType"] = "timestamp" }; + + var dbType = PostgresPropertyMapping.GetNpgsqlDbType(dataProperty); + Assert.Equal(NpgsqlDbType.Timestamp, dbType); + } + + [Fact] + public void GetNpgsqlDbType_DateTime_WithoutAnnotation_ReturnsTimestampTz() + { + var dataProperty = new DataPropertyModel("created", typeof(DateTime)); + + var dbType = PostgresPropertyMapping.GetNpgsqlDbType(dataProperty); + Assert.Equal(NpgsqlDbType.TimestampTz, dbType); + } +} diff --git a/MEVD/test/PgVector.UnitTests/PostgresSqlBuilderTests.cs b/MEVD/test/PgVector.UnitTests/PostgresSqlBuilderTests.cs new file mode 100644 index 0000000..e23429c --- /dev/null +++ b/MEVD/test/PgVector.UnitTests/PostgresSqlBuilderTests.cs @@ -0,0 +1,670 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; +using CommunityToolkit.VectorData.PgVector; +using Npgsql; +using Xunit; +using Xunit.Abstractions; + +namespace SemanticKernel.Connectors.PgVector.UnitTests; + +public class PostgresSqlBuilderTests +{ + private readonly ITestOutputHelper _output; + private static readonly float[] s_vector = new float[] { 1.0f, 2.0f, 3.0f }; + + public PostgresSqlBuilderTests(ITestOutputHelper output) + { + this._output = output; + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void TestBuildCreateTableCommand(bool ifNotExists) + { + var recordDefinition = new VectorStoreCollectionDefinition() + { + Properties = [ + new VectorStoreKeyProperty("id", typeof(long)), + new VectorStoreDataProperty("name", typeof(string)), + new VectorStoreDataProperty("code", typeof(int)), + new VectorStoreDataProperty("rating", typeof(float?)), + new VectorStoreDataProperty("description", typeof(string)), + new VectorStoreDataProperty("parking_is_included", typeof(bool)) { StorageName = "free_parking" }, + new VectorStoreDataProperty("tags", typeof(List)), + new VectorStoreVectorProperty("embedding1", typeof(ReadOnlyMemory), 10) + { + Dimensions = 10, + IndexKind = "hnsw", + }, + new VectorStoreVectorProperty("embedding2", typeof(ReadOnlyMemory?), 10) + { + Dimensions = 10, + IndexKind = "hnsw", + } + ] + }; + + var model = new PostgresModelBuilder().BuildDynamic(recordDefinition, defaultEmbeddingGenerator: null); + + var sql = PostgresSqlBuilder.BuildCreateTableSql(schema: null, "testcollection", model, pgVersion: new Version(18, 0), ifNotExists: ifNotExists); + + // Check for expected properties; integration tests will validate the actual SQL. + Assert.Contains("\"testcollection\" (", sql); + Assert.DoesNotContain("\"public\"", sql); + Assert.Contains("\"name\" TEXT", sql); + Assert.Contains("\"code\" INTEGER NOT NULL", sql); + Assert.Contains("\"rating\" REAL", sql); + Assert.Contains("\"description\" TEXT", sql); + Assert.Contains("\"free_parking\" BOOLEAN NOT NULL", sql); + Assert.Contains("\"tags\" TEXT[]", sql); + Assert.Contains("\"description\" TEXT", sql); + Assert.Contains("\"embedding1\" VECTOR(10) NOT NULL", sql); + Assert.Contains("\"embedding2\" VECTOR(10)", sql); + Assert.Contains("PRIMARY KEY (\"id\")", sql); + + if (ifNotExists) + { + Assert.Contains("IF NOT EXISTS", sql); + } + + // Output + this._output.WriteLine(sql); + } + + [Fact] + public void TestBuildCreateTableCommand_WithTimestampStoreType() + { + var recordDefinition = new VectorStoreCollectionDefinition() + { + Properties = [ + new VectorStoreKeyProperty("id", typeof(long)), + new VectorStoreDataProperty("created_utc", typeof(DateTime)), + new VectorStoreDataProperty("created_local", typeof(DateTime)).WithStoreType("timestamp"), + new VectorStoreDataProperty("created_nullable", typeof(DateTime?)).WithStoreType("timestamp without time zone"), + new VectorStoreVectorProperty("embedding", typeof(ReadOnlyMemory), 10) + ] + }; + + var model = new PostgresModelBuilder().BuildDynamic(recordDefinition, defaultEmbeddingGenerator: null); + + var sql = PostgresSqlBuilder.BuildCreateTableSql(schema: null, "testcollection", model, pgVersion: new Version(18, 0)); + + Assert.Contains("\"created_utc\" TIMESTAMPTZ NOT NULL", sql); + Assert.Contains("\"created_local\" TIMESTAMP NOT NULL", sql); + Assert.Contains("\"created_nullable\" TIMESTAMP", sql); + // Make sure it's TIMESTAMP (not TIMESTAMPTZ) for the nullable one + var idx = sql.IndexOf("\"created_nullable\"", StringComparison.Ordinal); + var fragment = sql.Substring(idx, sql.IndexOf('\n', idx) - idx); + Assert.DoesNotContain("TIMESTAMPTZ", fragment); + + this._output.WriteLine(sql); + } + + [Theory] + [InlineData(IndexKind.Hnsw, DistanceFunction.EuclideanDistance, true)] + [InlineData(IndexKind.Hnsw, DistanceFunction.EuclideanDistance, false)] + [InlineData(IndexKind.IvfFlat, DistanceFunction.DotProductSimilarity, true)] + [InlineData(IndexKind.IvfFlat, DistanceFunction.DotProductSimilarity, false)] + [InlineData(IndexKind.Hnsw, DistanceFunction.CosineDistance, true)] + [InlineData(IndexKind.Hnsw, DistanceFunction.CosineDistance, false)] + public void TestBuildCreateIndexCommand(string indexKind, string distanceFunction, bool ifNotExists) + { + var vectorColumn = "embedding1"; + + if (indexKind != IndexKind.Hnsw) + { + Assert.Throws(() => PostgresSqlBuilder.BuildCreateIndexSql(schema: null, "testcollection", vectorColumn, indexKind, distanceFunction, isVector: true, isFullText: false, fullTextLanguage: null, ifNotExists)); + Assert.Throws(() => PostgresSqlBuilder.BuildCreateIndexSql(schema: null, "testcollection", vectorColumn, indexKind, distanceFunction, isVector: true, isFullText: false, fullTextLanguage: null, ifNotExists)); + return; + } + + var sql = PostgresSqlBuilder.BuildCreateIndexSql(schema: null, "1testcollection", vectorColumn, indexKind, distanceFunction, isVector: true, isFullText: false, fullTextLanguage: null, ifNotExists); + + // Check for expected properties; integration tests will validate the actual SQL. + Assert.Contains("CREATE INDEX ", sql); + // Make sure ifNotExists is respected + if (ifNotExists) + { + Assert.Contains("CREATE INDEX IF NOT EXISTS", sql); + } + else + { + Assert.DoesNotContain("CREATE INDEX IF NOT EXISTS", sql); + } + // Make sure the name is escaped, so names starting with a digit are OK. + Assert.Contains($"\"1testcollection_{vectorColumn}_index\"", sql); + + Assert.Contains("ON \"1testcollection\" USING hnsw (\"embedding1\" ", sql); + if (distanceFunction == null) + { + // Check for distance function defaults to cosine distance + Assert.Contains("vector_cosine_ops)", sql); + } + else if (distanceFunction == DistanceFunction.CosineDistance) + { + Assert.Contains("vector_cosine_ops)", sql); + } + else if (distanceFunction == DistanceFunction.EuclideanDistance) + { + Assert.Contains("vector_l2_ops)", sql); + } + else + { + throw new NotImplementedException($"Test case for Distance function {distanceFunction} is not implemented."); + } + // Output + this._output.WriteLine(sql); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void TestBuildCreateNonVectorIndexCommand(bool ifNotExists) + { + var sql = PostgresSqlBuilder.BuildCreateIndexSql("schema", "tableName", "columnName", indexKind: "", distanceFunction: "", isVector: false, isFullText: false, fullTextLanguage: null, ifNotExists); + + var expectedCommandText = ifNotExists + ? "CREATE INDEX IF NOT EXISTS \"tableName_columnName_index\" ON \"schema\".\"tableName\" (\"columnName\")" + : "CREATE INDEX \"tableName_columnName_index\" ON \"schema\".\"tableName\" (\"columnName\")"; + + Assert.Equal(expectedCommandText, sql); + } + + [Theory] + [InlineData(null, "english")] // Default language + [InlineData("spanish", "spanish")] + [InlineData("german", "german")] + public void TestBuildCreateFullTextIndexCommand(string? configuredLanguage, string expectedLanguage) + { + var sql = PostgresSqlBuilder.BuildCreateIndexSql("schema", "tableName", "content", indexKind: "", distanceFunction: "", isVector: false, isFullText: true, fullTextLanguage: configuredLanguage, ifNotExists: true); + + var expectedCommandText = $"CREATE INDEX IF NOT EXISTS \"tableName_content_index\" ON \"schema\".\"tableName\" USING GIN (to_tsvector('{expectedLanguage}', \"content\"))"; + + Assert.Equal(expectedCommandText, sql); + } + + [Fact] + public void TestBuildCreateFullTextIndexCommand_EscapesSingleQuotes() + { + // Verify that single quotes in the language name are properly escaped to prevent SQL injection + var sql = PostgresSqlBuilder.BuildCreateIndexSql("schema", "tableName", "content", indexKind: "", distanceFunction: "", isVector: false, isFullText: true, fullTextLanguage: "test'injection", ifNotExists: true); + + var expectedCommandText = "CREATE INDEX IF NOT EXISTS \"tableName_content_index\" ON \"schema\".\"tableName\" USING GIN (to_tsvector('test''injection', \"content\"))"; + + Assert.Equal(expectedCommandText, sql); + } + + [Fact] + public void TestBuildDropTableCommand() + { + using var command = new NpgsqlCommand(); + PostgresSqlBuilder.BuildDropTableCommand(command, schema: null, "testcollection"); + + // Check for expected properties; integration tests will validate the actual SQL. + Assert.Contains("DROP TABLE IF EXISTS \"testcollection\"", command.CommandText); + Assert.DoesNotContain("\"public\"", command.CommandText); + + // Output + this._output.WriteLine(command.CommandText); + } + + [Fact] + public void TestBuildUpsertCommand() + { + var recordDefinition = new VectorStoreCollectionDefinition() + { + Properties = [ + new VectorStoreKeyProperty("id", typeof(int)), + new VectorStoreDataProperty("name", typeof(string)), + new VectorStoreDataProperty("code", typeof(int)), + new VectorStoreDataProperty("rating", typeof(float?)), + new VectorStoreDataProperty("description", typeof(string)), + new VectorStoreDataProperty("parking_is_included", typeof(bool)) { StorageName = "free_parking" }, + new VectorStoreDataProperty("tags", typeof(List)), + new VectorStoreVectorProperty("embedding1", typeof(ReadOnlyMemory), 10) + { + Dimensions = 10, + IndexKind = "hnsw", + }, + new VectorStoreVectorProperty("embedding2", typeof(ReadOnlyMemory?), 10) + { + Dimensions = 10, + IndexKind = "hnsw", + } + ] + }; + + var model = new PostgresModelBuilder().BuildDynamic(recordDefinition, defaultEmbeddingGenerator: null); + + var record = new Dictionary + { + ["id"] = 123, + ["name"] = "Hotel", + ["code"] = 456, + ["rating"] = 4.5f, + ["description"] = "Hotel description", + ["parking_is_included"] = true, + ["tags"] = new List { "tag1", "tag2" }, + ["embedding1"] = new ReadOnlyMemory(s_vector), + }; + + using var batch = new NpgsqlBatch(); + _ = PostgresSqlBuilder.BuildUpsertCommand(batch, schema: null, "testcollection", model, [record], generatedEmbeddings: null); + + // Check for expected properties; integration tests will validate the actual SQL. + Assert.Single(batch.BatchCommands); + var command = batch.BatchCommands[0]; + Assert.Contains("INSERT INTO \"testcollection\" (", command.CommandText); + Assert.Contains("ON CONFLICT (\"id\")", command.CommandText); + Assert.Contains("DO UPDATE SET", command.CommandText); + Assert.NotNull(command.Parameters); + + foreach (var (column, index) in record.Keys.Select((key, index) => (key, index))) + { + var expectedValue = column is "embedding1" + ? PostgresPropertyMapping.MapVectorForStorageModel((ReadOnlyMemory)record[column]!) + : record[column]; + Assert.Equal(expectedValue, command.Parameters[index].Value); + + // If the key is not the key column, it should be included in the update clause. + if (column is "id") + { + continue; + } + + var storageName = column is "parking_is_included" ? "free_parking" : column; + + Assert.Contains($"\"{storageName}\" = EXCLUDED.\"{storageName}\"", command.CommandText); + } + + // Output + this._output.WriteLine(command.CommandText); + } + + [Fact] + public void TestBuildGetCommand() + { + // Arrange + var recordDefinition = new VectorStoreCollectionDefinition() + { + Properties = [ + new VectorStoreKeyProperty("id", typeof(long)), + new VectorStoreDataProperty("name", typeof(string)), + new VectorStoreDataProperty("code", typeof(int)), + new VectorStoreDataProperty("rating", typeof(float?)), + new VectorStoreDataProperty("description", typeof(string)), + new VectorStoreDataProperty("parking_is_included", typeof(bool)) { StorageName = "free_parking" }, + new VectorStoreDataProperty("tags", typeof(List)), + new VectorStoreVectorProperty("embedding1", typeof(ReadOnlyMemory), 10) + { + IndexKind = "hnsw", + }, + new VectorStoreVectorProperty("embedding2", typeof(ReadOnlyMemory?), 10) + { + IndexKind = "hnsw", + } + ] + }; + + var model = new PostgresModelBuilder().BuildDynamic(recordDefinition, defaultEmbeddingGenerator: null); + + var key = 123; + + // Act + using var command = new NpgsqlCommand(); + PostgresSqlBuilder.BuildGetCommand(command, schema: null, "testcollection", model, key, includeVectors: true); + + // Assert + Assert.Contains("SELECT", command.CommandText); + Assert.Contains("\"free_parking\"", command.CommandText); + Assert.Contains("\"embedding1\"", command.CommandText); + Assert.Contains("FROM \"testcollection\"", command.CommandText); + Assert.Contains("WHERE \"id\" = $1", command.CommandText); + + // Output + this._output.WriteLine(command.CommandText); + } + + [Fact] + public void TestBuildGetBatchCommand() + { + // Arrange + var recordDefinition = new VectorStoreCollectionDefinition() + { + Properties = [ + new VectorStoreKeyProperty("id", typeof(long)), + new VectorStoreDataProperty("name", typeof(string)), + new VectorStoreDataProperty("code", typeof(int)), + new VectorStoreDataProperty("rating", typeof(float?)), + new VectorStoreDataProperty("description", typeof(string)), + new VectorStoreDataProperty("parking_is_included", typeof(bool)) { StorageName = "free_parking" }, + new VectorStoreDataProperty("tags", typeof(List)), + new VectorStoreVectorProperty("embedding1", typeof(ReadOnlyMemory), 10) + { + IndexKind = "hnsw", + }, + new VectorStoreVectorProperty("embedding2", typeof(ReadOnlyMemory?), 10) + { + IndexKind = "hnsw", + } + ] + }; + + var keys = new List { 123, 124 }; + + var model = new PostgresModelBuilder().BuildDynamic(recordDefinition, defaultEmbeddingGenerator: null); + + // Act + using var command = new NpgsqlCommand(); + PostgresSqlBuilder.BuildGetBatchCommand(command, schema: null, "testcollection", model, keys, includeVectors: true); + + // Assert + Assert.Contains("SELECT", command.CommandText); + Assert.Contains("\"code\"", command.CommandText); + Assert.Contains("\"free_parking\"", command.CommandText); + Assert.Contains("FROM \"testcollection\"", command.CommandText); + Assert.Contains("WHERE \"id\" = ANY($1)", command.CommandText); + Assert.NotNull(command.Parameters); + Assert.Single(command.Parameters); + Assert.Equal(keys, command.Parameters[0].Value); + + // Output + this._output.WriteLine(command.CommandText); + } + + [Fact] + public void TestBuildDeleteCommand() + { + // Arrange + var key = 123; + + // Act + using var command = new NpgsqlCommand(); + PostgresSqlBuilder.BuildDeleteCommand(command, schema: null, "testcollection", "id", key); + + // Assert + Assert.Contains("DELETE", command.CommandText); + Assert.Contains("FROM \"testcollection\"", command.CommandText); + Assert.Contains("WHERE \"id\" = $1", command.CommandText); + + // Output + this._output.WriteLine(command.CommandText); + } + + [Fact] + public void TestBuildDeleteBatchCommand() + { + // Arrange + var keys = new List { 123, 124 }; + + // Act + using var command = new NpgsqlCommand(); + var keyProperty = new KeyPropertyModel("id", typeof(long)); + PostgresSqlBuilder.BuildDeleteBatchCommand(command, schema: null, "testcollection", keyProperty, keys); + + // Assert + Assert.Contains("DELETE", command.CommandText); + Assert.Contains("FROM \"testcollection\"", command.CommandText); + Assert.Contains("WHERE \"id\" = ANY($1)", command.CommandText); + Assert.NotNull(command.Parameters); + Assert.Single(command.Parameters); + Assert.Equal(keys, command.Parameters[0].Value); + + // Output + this._output.WriteLine(command.CommandText); + } + + #region Schema-specified tests + + [Theory] + [InlineData(null, "public")] + [InlineData("myschema", "myschema")] + public void TestBuildDoesTableExistCommand(string? schema, string expectedSchema) + { + using var command = new NpgsqlCommand(); + PostgresSqlBuilder.BuildDoesTableExistCommand(command, schema, "testcollection"); + + Assert.Contains("table_schema = $1", command.CommandText); + Assert.Contains("table_name = $2", command.CommandText); + Assert.Equal(2, command.Parameters.Count); + Assert.Equal(expectedSchema, command.Parameters[0].Value); + Assert.Equal("testcollection", command.Parameters[1].Value); + + this._output.WriteLine(command.CommandText); + } + + [Theory] + [InlineData(null, "public")] + [InlineData("myschema", "myschema")] + public void TestBuildGetTablesCommand(string? schema, string expectedSchema) + { + using var command = new NpgsqlCommand(); + PostgresSqlBuilder.BuildGetTablesCommand(command, schema); + + Assert.Contains("table_schema = $1", command.CommandText); + Assert.Single(command.Parameters); + Assert.Equal(expectedSchema, command.Parameters[0].Value); + + this._output.WriteLine(command.CommandText); + } + + [Fact] + public void TestBuildCreateTableCommand_WithSchema() + { + var recordDefinition = new VectorStoreCollectionDefinition() + { + Properties = [ + new VectorStoreKeyProperty("id", typeof(long)), + new VectorStoreDataProperty("name", typeof(string)), + new VectorStoreVectorProperty("embedding1", typeof(ReadOnlyMemory), 10) { IndexKind = "hnsw" } + ] + }; + + var model = new PostgresModelBuilder().BuildDynamic(recordDefinition, defaultEmbeddingGenerator: null); + + var sql = PostgresSqlBuilder.BuildCreateTableSql(schema: "myschema", "testcollection", model, pgVersion: new Version(18, 0)); + + Assert.Contains("\"myschema\".\"testcollection\"", sql); + + this._output.WriteLine(sql); + } + + [Fact] + public void TestBuildCreateIndexCommand_WithSchema() + { + var sql = PostgresSqlBuilder.BuildCreateIndexSql("myschema", "testcollection", "embedding1", IndexKind.Hnsw, DistanceFunction.CosineDistance, isVector: true, isFullText: false, fullTextLanguage: null, ifNotExists: true); + + Assert.Contains("ON \"myschema\".\"testcollection\"", sql); + + this._output.WriteLine(sql); + } + + [Fact] + public void TestBuildDropTableCommand_WithSchema() + { + using var command = new NpgsqlCommand(); + PostgresSqlBuilder.BuildDropTableCommand(command, schema: "myschema", "testcollection"); + + Assert.Contains("DROP TABLE IF EXISTS \"myschema\".\"testcollection\"", command.CommandText); + + this._output.WriteLine(command.CommandText); + } + + [Fact] + public void TestBuildUpsertCommand_WithSchema() + { + var recordDefinition = new VectorStoreCollectionDefinition() + { + Properties = [ + new VectorStoreKeyProperty("id", typeof(int)), + new VectorStoreDataProperty("name", typeof(string)), + new VectorStoreVectorProperty("embedding1", typeof(ReadOnlyMemory), 10) { IndexKind = "hnsw" } + ] + }; + + var model = new PostgresModelBuilder().BuildDynamic(recordDefinition, defaultEmbeddingGenerator: null); + + var record = new Dictionary + { + ["id"] = 1, + ["name"] = "Test", + ["embedding1"] = new ReadOnlyMemory(s_vector), + }; + + using var batch = new NpgsqlBatch(); + _ = PostgresSqlBuilder.BuildUpsertCommand(batch, schema: "myschema", "testcollection", model, [record], generatedEmbeddings: null); + + var command = batch.BatchCommands[0]; + Assert.Contains("INSERT INTO \"myschema\".\"testcollection\"", command.CommandText); + + this._output.WriteLine(command.CommandText); + } + + [Fact] + public void TestBuildGetCommand_WithSchema() + { + var recordDefinition = new VectorStoreCollectionDefinition() + { + Properties = [ + new VectorStoreKeyProperty("id", typeof(long)), + new VectorStoreDataProperty("name", typeof(string)), + new VectorStoreVectorProperty("embedding1", typeof(ReadOnlyMemory), 10) { IndexKind = "hnsw" } + ] + }; + + var model = new PostgresModelBuilder().BuildDynamic(recordDefinition, defaultEmbeddingGenerator: null); + + using var command = new NpgsqlCommand(); + PostgresSqlBuilder.BuildGetCommand(command, schema: "myschema", "testcollection", model, 123, includeVectors: true); + + Assert.Contains("FROM \"myschema\".\"testcollection\"", command.CommandText); + + this._output.WriteLine(command.CommandText); + } + + [Fact] + public void TestBuildGetBatchCommand_WithSchema() + { + var recordDefinition = new VectorStoreCollectionDefinition() + { + Properties = [ + new VectorStoreKeyProperty("id", typeof(long)), + new VectorStoreDataProperty("name", typeof(string)), + new VectorStoreVectorProperty("embedding1", typeof(ReadOnlyMemory), 10) { IndexKind = "hnsw" } + ] + }; + + var model = new PostgresModelBuilder().BuildDynamic(recordDefinition, defaultEmbeddingGenerator: null); + + using var command = new NpgsqlCommand(); + PostgresSqlBuilder.BuildGetBatchCommand(command, schema: "myschema", "testcollection", model, new List { 1, 2 }, includeVectors: true); + + Assert.Contains("FROM \"myschema\".\"testcollection\"", command.CommandText); + + this._output.WriteLine(command.CommandText); + } + + [Fact] + public void TestBuildDeleteCommand_WithSchema() + { + using var command = new NpgsqlCommand(); + PostgresSqlBuilder.BuildDeleteCommand(command, schema: "myschema", "testcollection", "id", 123); + + Assert.Contains("FROM \"myschema\".\"testcollection\"", command.CommandText); + + this._output.WriteLine(command.CommandText); + } + + [Fact] + public void TestBuildDeleteBatchCommand_WithSchema() + { + using var command = new NpgsqlCommand(); + var keyProperty = new KeyPropertyModel("id", typeof(long)); + PostgresSqlBuilder.BuildDeleteBatchCommand(command, schema: "myschema", "testcollection", keyProperty, new List { 1, 2 }); + + Assert.Contains("FROM \"myschema\".\"testcollection\"", command.CommandText); + + this._output.WriteLine(command.CommandText); + } + + #endregion + + #region NRT (Nullable Reference Type) detection + +#if NET // NRT detection via NullabilityInfoContext is only available on .NET 6+ + [Fact] + public void TestBuildCreateTableCommand_WithNrtAnnotations() + { + var model = new PostgresModelBuilder().Build( + typeof(NrtTestRecord), + typeof(long), + definition: null, + defaultEmbeddingGenerator: null); + + var sql = PostgresSqlBuilder.BuildCreateTableSql(schema: null, "testcollection", model, pgVersion: new Version(18, 0)); + + // Non-nullable reference types should be NOT NULL + Assert.Contains("\"NonNullableString\" TEXT NOT NULL", sql); + Assert.Contains("\"NonNullableByteArray\" BYTEA NOT NULL", sql); + + // Nullable reference types should not have NOT NULL + Assert.Contains("\"NullableString\" TEXT", sql); + Assert.DoesNotContain("\"NullableString\" TEXT NOT NULL", sql); + Assert.Contains("\"NullableByteArray\" BYTEA", sql); + Assert.DoesNotContain("\"NullableByteArray\" BYTEA NOT NULL", sql); + + // Non-nullable value types should be NOT NULL (unchanged from before) + Assert.Contains("\"NonNullableInt\" INTEGER NOT NULL", sql); + Assert.Contains("\"NonNullableBool\" BOOLEAN NOT NULL", sql); + + // Nullable value types should not have NOT NULL (unchanged from before) + Assert.Contains("\"NullableInt\" INTEGER", sql); + Assert.DoesNotContain("\"NullableInt\" INTEGER NOT NULL", sql); + + this._output.WriteLine(sql); + } + +#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor +#pragma warning disable CA1812 // Class is used via reflection + private sealed class NrtTestRecord + { + [VectorStoreKey] + public long Id { get; set; } + + [VectorStoreData] + public string NonNullableString { get; set; } + + [VectorStoreData] + public string? NullableString { get; set; } + + [VectorStoreData] + public byte[] NonNullableByteArray { get; set; } + + [VectorStoreData] + public byte[]? NullableByteArray { get; set; } + + [VectorStoreData] + public int NonNullableInt { get; set; } + + [VectorStoreData] + public int? NullableInt { get; set; } + + [VectorStoreData] + public bool NonNullableBool { get; set; } + + [VectorStoreVector(dimensions: 10)] + public ReadOnlyMemory Embedding { get; set; } + } +#pragma warning restore CA1812 +#pragma warning restore CS8618 +#endif + + #endregion +} diff --git a/MEVD/test/Pinecone.ConformanceTests/ModelTests/PineconeBasicModelTests.cs b/MEVD/test/Pinecone.ConformanceTests/ModelTests/PineconeBasicModelTests.cs new file mode 100644 index 0000000..676536a --- /dev/null +++ b/MEVD/test/Pinecone.ConformanceTests/ModelTests/PineconeBasicModelTests.cs @@ -0,0 +1,36 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Pinecone.ConformanceTests.Support; +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace Pinecone.ConformanceTests.ModelTests; + +public class PineconeBasicModelTests(PineconeBasicModelTests.Fixture fixture) + : BasicModelTests(fixture), IClassFixture +{ + public override async Task GetAsync_with_filter_and_OrderBy() + { + var exception = await Assert.ThrowsAsync(base.GetAsync_with_filter_and_OrderBy); + Assert.Equal("Pinecone does not support ordering.", exception.Message); + } + + public override async Task GetAsync_with_filter_and_multiple_OrderBys() + { + var exception = await Assert.ThrowsAsync(base.GetAsync_with_filter_and_multiple_OrderBys); + Assert.Equal("Pinecone does not support ordering.", exception.Message); + } + + public override async Task GetAsync_with_filter_and_OrderBy_and_Skip() + { + var exception = await Assert.ThrowsAsync(base.GetAsync_with_filter_and_OrderBy_and_Skip); + Assert.Equal("Pinecone does not support ordering.", exception.Message); + } + + public new class Fixture : BasicModelTests.Fixture + { + public override TestStore TestStore => PineconeTestStore.Instance; + } +} diff --git a/MEVD/test/Pinecone.ConformanceTests/ModelTests/PineconeDynamicModelTests.cs b/MEVD/test/Pinecone.ConformanceTests/ModelTests/PineconeDynamicModelTests.cs new file mode 100644 index 0000000..627ca9f --- /dev/null +++ b/MEVD/test/Pinecone.ConformanceTests/ModelTests/PineconeDynamicModelTests.cs @@ -0,0 +1,36 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Pinecone.ConformanceTests.Support; +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace Pinecone.ConformanceTests.ModelTests; + +public class PineconeDynamicModelTests(PineconeDynamicModelTests.Fixture fixture) + : DynamicModelTests(fixture), IClassFixture +{ + public override async Task GetAsync_with_filter_and_OrderBy() + { + var exception = await Assert.ThrowsAsync(base.GetAsync_with_filter_and_OrderBy); + Assert.Equal("Pinecone does not support ordering.", exception.Message); + } + + public override async Task GetAsync_with_filter_and_multiple_OrderBys() + { + var exception = await Assert.ThrowsAsync(base.GetAsync_with_filter_and_multiple_OrderBys); + Assert.Equal("Pinecone does not support ordering.", exception.Message); + } + + public override async Task GetAsync_with_filter_and_OrderBy_and_Skip() + { + var exception = await Assert.ThrowsAsync(base.GetAsync_with_filter_and_OrderBy_and_Skip); + Assert.Equal("Pinecone does not support ordering.", exception.Message); + } + + public new class Fixture : DynamicModelTests.Fixture + { + public override TestStore TestStore => PineconeTestStore.Instance; + } +} diff --git a/MEVD/test/Pinecone.ConformanceTests/ModelTests/PineconeNoDataModelTests.cs b/MEVD/test/Pinecone.ConformanceTests/ModelTests/PineconeNoDataModelTests.cs new file mode 100644 index 0000000..7c3b9ca --- /dev/null +++ b/MEVD/test/Pinecone.ConformanceTests/ModelTests/PineconeNoDataModelTests.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Pinecone.ConformanceTests.Support; +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace Pinecone.ConformanceTests.ModelTests; + +public class PineconeNoDataModelTests(PineconeNoDataModelTests.Fixture fixture) + : NoDataModelTests(fixture), IClassFixture +{ + public new class Fixture : NoDataModelTests.Fixture + { + public override TestStore TestStore => PineconeTestStore.Instance; + } +} diff --git a/MEVD/test/Pinecone.ConformanceTests/Pinecone.ConformanceTests.csproj b/MEVD/test/Pinecone.ConformanceTests/Pinecone.ConformanceTests.csproj new file mode 100644 index 0000000..87bb744 --- /dev/null +++ b/MEVD/test/Pinecone.ConformanceTests/Pinecone.ConformanceTests.csproj @@ -0,0 +1,43 @@ + + + + + net10.0 + enable + enable + Pinecone.ConformanceTests + + false + true + + $(NoWarn);CA2007,SKEXP0001,VSTHRD111;CS1685 + $(NoWarn);MEVD9001 + b7762d10-e29b-4bb1-8b74-b6d69a667dd4 + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/MEVD/test/Pinecone.ConformanceTests/PineconeAllSupportedTypesTests.cs b/MEVD/test/Pinecone.ConformanceTests/PineconeAllSupportedTypesTests.cs new file mode 100644 index 0000000..ebef39b --- /dev/null +++ b/MEVD/test/Pinecone.ConformanceTests/PineconeAllSupportedTypesTests.cs @@ -0,0 +1,74 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.VectorData; +using Pinecone.ConformanceTests.Support; +using Xunit; + +namespace Pinecone.ConformanceTests; + +public class PineconeAllSupportedTypesTests(PineconeFixture fixture) : IClassFixture +{ + [Fact] + public async Task AllTypesBatchGetAsync() + { + var collection = fixture.TestStore.DefaultVectorStore.GetCollection("all-types", PineconeAllTypes.GetRecordDefinition()); + await collection.EnsureCollectionExistsAsync(); + + List records = + [ + new() + { + Id = "all-types-1", + BoolProperty = true, + NullableBoolProperty = false, + StringProperty = "string prop 1", + NullableStringProperty = "nullable prop 1", + IntProperty = 1, + NullableIntProperty = 10, + LongProperty = 100L, + NullableLongProperty = 1000L, + FloatProperty = 10.5f, + NullableFloatProperty = 100.5f, + DoubleProperty = 23.75d, + NullableDoubleProperty = 233.75d, + StringArray = ["one", "two"], + NullableStringArray = ["five", "six"], + StringList = ["eleven", "twelve"], + NullableStringList = ["fifteen", "sixteen"], + Embedding = new ReadOnlyMemory([1.5f, 2.5f, 3.5f, 4.5f, 5.5f, 6.5f, 7.5f, 8.5f]) + }, + new() + { + Id = "all-types-2", + BoolProperty = false, + NullableBoolProperty = null, + StringProperty = "string prop 2", + NullableStringProperty = null, + IntProperty = 2, + NullableIntProperty = null, + LongProperty = 200L, + NullableLongProperty = null, + FloatProperty = 20.5f, + NullableFloatProperty = null, + DoubleProperty = 43.75, + NullableDoubleProperty = null, + StringArray = [], + NullableStringArray = null, + StringList = [], + NullableStringList = null, + Embedding = new ReadOnlyMemory([10.5f, 20.5f, 30.5f, 40.5f, 50.5f, 60.5f, 70.5f, 80.5f]) + } + ]; + + await collection.UpsertAsync(records); + + var allTypes = await collection.GetAsync(records.Select(r => r.Id), new RecordRetrievalOptions { IncludeVectors = true }).ToListAsync(); + + var allTypes1 = allTypes.Single(x => x.Id == records[0].Id); + var allTypes2 = allTypes.Single(x => x.Id == records[1].Id); + + records[0].AssertEqual(allTypes1); + records[1].AssertEqual(allTypes2); + } +} diff --git a/MEVD/test/Pinecone.ConformanceTests/PineconeCollectionManagementTests.cs b/MEVD/test/Pinecone.ConformanceTests/PineconeCollectionManagementTests.cs new file mode 100644 index 0000000..1df9fc3 --- /dev/null +++ b/MEVD/test/Pinecone.ConformanceTests/PineconeCollectionManagementTests.cs @@ -0,0 +1,11 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Pinecone.ConformanceTests.Support; +using VectorData.ConformanceTests; +using Xunit; + +namespace Pinecone.ConformanceTests; + +public class PineconeCollectionManagementTests(PineconeFixture fixture) + : CollectionManagementTests(fixture), IClassFixture; diff --git a/MEVD/test/Pinecone.ConformanceTests/PineconeDependencyInjectionTests.cs b/MEVD/test/Pinecone.ConformanceTests/PineconeDependencyInjectionTests.cs new file mode 100644 index 0000000..fb19458 --- /dev/null +++ b/MEVD/test/Pinecone.ConformanceTests/PineconeDependencyInjectionTests.cs @@ -0,0 +1,93 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using CommunityToolkit.VectorData.Pinecone; +using VectorData.ConformanceTests; +using Xunit; + +namespace Pinecone.ConformanceTests; + +public class PineconeDependencyInjectionTests + : DependencyInjectionTests.Record>, string, DependencyInjectionTests.Record> +{ + private const string ApiKey = "Fake API Key"; + private static readonly ClientOptions s_clientOptions = new() { MaxRetries = 1 }; + + protected override string CollectionName => "lowercase"; + + protected override void PopulateConfiguration(ConfigurationManager configuration, object? serviceKey = null) + => configuration.AddInMemoryCollection( + [ + new(CreateConfigKey("Pinecone", serviceKey, "ApiKey"), ApiKey), + ]); + + private static string ApiKeyProvider(IServiceProvider sp, object? serviceKey = null) + => sp.GetRequiredService().GetRequiredSection(CreateConfigKey("Pinecone", serviceKey, "ApiKey")).Value!; + + private static ClientOptions ClientOptionsProvider(IServiceProvider sp, object? serviceKey = null) => s_clientOptions; + + public override IEnumerable> CollectionDelegates + { + get + { + yield return (services, serviceKey, name, lifetime) => serviceKey is null + ? services + .AddSingleton(sp => new PineconeClient(ApiKey)) + .AddPineconeCollection(name, lifetime: lifetime) + : services + .AddSingleton(sp => new PineconeClient(ApiKey)) + .AddKeyedPineconeCollection(serviceKey, name, lifetime: lifetime); + + yield return (services, serviceKey, name, lifetime) => serviceKey is null + ? services.AddPineconeCollection( + name, ApiKey, lifetime: lifetime) + : services.AddKeyedPineconeCollection( + serviceKey, name, ApiKey, lifetime: lifetime); + + yield return (services, serviceKey, name, lifetime) => serviceKey is null + ? services.AddPineconeCollection( + name, sp => new PineconeClient(ApiKeyProvider(sp), ClientOptionsProvider(sp)), lifetime: lifetime) + : services.AddKeyedPineconeCollection( + serviceKey, name, sp => new PineconeClient(ApiKeyProvider(sp, serviceKey), ClientOptionsProvider(sp, serviceKey)), lifetime: lifetime); + } + } + + public override IEnumerable> StoreDelegates + { + get + { + yield return (services, serviceKey, lifetime) => serviceKey is null + ? services.AddPineconeVectorStore( + ApiKey, s_clientOptions, lifetime: lifetime) + : services.AddKeyedPineconeVectorStore( + serviceKey, ApiKey, s_clientOptions, lifetime: lifetime); + + yield return (services, serviceKey, lifetime) => serviceKey is null + ? services + .AddSingleton(sp => new PineconeClient(ApiKey)) + .AddPineconeVectorStore(lifetime: lifetime) + : services + .AddSingleton(sp => new PineconeClient(ApiKey)) + .AddKeyedPineconeVectorStore(serviceKey, lifetime: lifetime); + } + } + + [Fact] + public void ApiKeyCantBeNullOrEmpty() + { + IServiceCollection services = new ServiceCollection(); + + Assert.Throws(() => services.AddPineconeVectorStore(apiKey: null!)); + Assert.Throws(() => services.AddKeyedPineconeVectorStore(serviceKey: "notNull", apiKey: null!)); + Assert.Throws(() => services.AddPineconeCollection( + name: "notNull", apiKey: null!)); + Assert.Throws(() => services.AddPineconeCollection( + name: "notNull", apiKey: "")); + Assert.Throws(() => services.AddKeyedPineconeCollection( + serviceKey: "notNull", name: "notNull", apiKey: null!)); + Assert.Throws(() => services.AddKeyedPineconeCollection( + serviceKey: "notNull", name: "notNull", apiKey: "")); + } +} diff --git a/MEVD/test/Pinecone.ConformanceTests/PineconeDistanceFunctionTests.cs b/MEVD/test/Pinecone.ConformanceTests/PineconeDistanceFunctionTests.cs new file mode 100644 index 0000000..226db13 --- /dev/null +++ b/MEVD/test/Pinecone.ConformanceTests/PineconeDistanceFunctionTests.cs @@ -0,0 +1,76 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.VectorData; +using Pinecone.ConformanceTests.Support; +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace Pinecone.ConformanceTests; + +public class PineconeDistanceFunctionTests(PineconeDistanceFunctionTests.Fixture fixture) + : DistanceFunctionTests(fixture), IClassFixture +{ + public override Task CosineDistance() => Assert.ThrowsAsync(base.CosineDistance); + public override Task EuclideanDistance() => Assert.ThrowsAsync(base.EuclideanDistance); + public override Task HammingDistance() => Assert.ThrowsAsync(base.HammingDistance); + public override Task ManhattanDistance() => Assert.ThrowsAsync(base.ManhattanDistance); + public override Task NegativeDotProductSimilarity() => Assert.ThrowsAsync(base.NegativeDotProductSimilarity); + + // Pinecone's score threshold logic always uses "Score >= threshold", which is incorrect + // for distance-based metrics (where lower is better). Skip score threshold validation. + protected override Task TestScoreThreshold(VectorStoreCollection collection) + => Task.CompletedTask; + + protected override async Task Test( + string distanceFunction, + double expectedExactMatchScore, + double expectedOppositeScore, + double expectedOrthogonalScore, + int[] resultOrder) + { + await base.Test( + distanceFunction, + expectedExactMatchScore, + expectedOppositeScore, + expectedOrthogonalScore, + resultOrder); + + // The Pinecone emulator needs some extra time to spawn a new index service + // that uses a different distance function. + await Task.Delay(TimeSpan.FromSeconds(5)); + } + + public new class Fixture() : DistanceFunctionTests.Fixture + { + public override TestStore TestStore => PineconeTestStore.Instance; + + // Use a shorter base name to stay within Pinecone's 45-character index name limit. + protected override string CollectionNameBase => "df-tests"; + + // The Pinecone local emulator doesn't handle rapid delete/recreate of the same index + // with different distance functions. Use a unique collection name per distance function + // so each test gets its own index. + public override VectorStoreCollection CreateCollection(string distanceFunction) + { + var name = TestStore.AdjustCollectionName($"{CollectionName}-{distanceFunction}"); + + VectorStoreCollectionDefinition definition = new() + { + Properties = + [ + new VectorStoreKeyProperty(nameof(SearchRecord.Key), typeof(string)), + new VectorStoreDataProperty(nameof(SearchRecord.Int), typeof(int)), + new VectorStoreVectorProperty(nameof(SearchRecord.Vector), typeof(ReadOnlyMemory), dimensions: 4) + { + DistanceFunction = distanceFunction, + IndexKind = IndexKind ?? DefaultIndexKind + } + ] + }; + + return TestStore.CreateCollection(name, definition); + } + } +} diff --git a/MEVD/test/Pinecone.ConformanceTests/PineconeEmbeddingGenerationTests.cs b/MEVD/test/Pinecone.ConformanceTests/PineconeEmbeddingGenerationTests.cs new file mode 100644 index 0000000..2bb3b7d --- /dev/null +++ b/MEVD/test/Pinecone.ConformanceTests/PineconeEmbeddingGenerationTests.cs @@ -0,0 +1,80 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.VectorData; +using Pinecone.ConformanceTests.Support; +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace Pinecone.ConformanceTests; + +public class PineconeEmbeddingGenerationTests(PineconeEmbeddingGenerationTests.StringVectorFixture stringVectorFixture, PineconeEmbeddingGenerationTests.RomOfFloatVectorFixture romOfFloatVectorFixture) + : EmbeddingGenerationTests(stringVectorFixture, romOfFloatVectorFixture), IClassFixture, IClassFixture +{ + // Overriding since Pinecone requires collection names to only contain ASCII lowercase letters, digits and dashes. + public override async Task SearchAsync_string_without_generator_throws() + { + // The database doesn't support embedding generation, and no client-side generator has been configured at any level, + // so SearchAsync should throw. + var collection = stringVectorFixture.GetCollection(stringVectorFixture.TestStore.DefaultVectorStore, stringVectorFixture.CollectionName + "-without-generator"); + + var exception = await Assert.ThrowsAsync(() => collection.SearchAsync("foo", top: 1).ToListAsync().AsTask()); + + Assert.StartsWith( + "A value of type 'string' was passed to 'SearchAsync', but that isn't a supported vector type by your provider and no embedding generator was configured. The supported vector types are:", + exception.Message); + } + + public new class StringVectorFixture : EmbeddingGenerationTests.StringVectorFixture + { + public override TestStore TestStore => PineconeTestStore.Instance; + + public override VectorStore CreateVectorStore(IEmbeddingGenerator? embeddingGenerator) + => PineconeTestStore.Instance.GetVectorStore(new() { EmbeddingGenerator = embeddingGenerator }); + + public override Func[] DependencyInjectionStoreRegistrationDelegates => + [ + services => services + .AddSingleton(PineconeTestStore.Instance.Client) + .AddPineconeVectorStore(), + services => services + .AddPineconeVectorStore("ForPineconeLocalTheApiKeysAreIgnored", PineconeTestStore.Instance.ClientOptions) + ]; + + public override Func[] DependencyInjectionCollectionRegistrationDelegates => + [ + services => services + .AddSingleton(PineconeTestStore.Instance.Client) + .AddPineconeCollection(this.CollectionName), + services => services + .AddPineconeCollection(this.CollectionName, "ForPineconeLocalTheApiKeysAreIgnored", PineconeTestStore.Instance.ClientOptions), + services => services + .AddPineconeCollection(this.CollectionName, _ => new PineconeClient("ForPineconeLocalTheApiKeysAreIgnored", PineconeTestStore.Instance.ClientOptions)) + ]; + } + + public new class RomOfFloatVectorFixture : EmbeddingGenerationTests.RomOfFloatVectorFixture + { + public override TestStore TestStore => PineconeTestStore.Instance; + + public override VectorStore CreateVectorStore(IEmbeddingGenerator? embeddingGenerator) + => PineconeTestStore.Instance.GetVectorStore(new() { EmbeddingGenerator = embeddingGenerator }); + + public override Func[] DependencyInjectionStoreRegistrationDelegates => + [ + services => services + .AddSingleton(PineconeTestStore.Instance.Client) + .AddPineconeVectorStore() + ]; + + public override Func[] DependencyInjectionCollectionRegistrationDelegates => + [ + services => services + .AddSingleton(PineconeTestStore.Instance.Client) + .AddPineconeCollection(this.CollectionName) + ]; + } +} diff --git a/MEVD/test/Pinecone.ConformanceTests/PineconeFilterTests.cs b/MEVD/test/Pinecone.ConformanceTests/PineconeFilterTests.cs new file mode 100644 index 0000000..9a681d1 --- /dev/null +++ b/MEVD/test/Pinecone.ConformanceTests/PineconeFilterTests.cs @@ -0,0 +1,93 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Pinecone.ConformanceTests.Support; +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace Pinecone.ConformanceTests; + +#pragma warning disable CS8605 // Unboxing a possibly null value. + +public class PineconeFilterTests(PineconeFilterTests.Fixture fixture) + : FilterTests(fixture), IClassFixture +{ + // Specialized Pinecone syntax for NOT over Contains ($nin) + [Fact] + public virtual Task Not_over_Contains() + => this.TestFilterAsync( + r => !new[] { 8, 10 }.Contains(r.Int), + r => !new[] { 8, 10 }.Contains((int)r["Int"])); + + #region Null checking + + // Pinecone currently doesn't support null checking ({ "Foo" : null }) in vector search pre-filters + public override Task Equal_with_null_reference_type() + => Assert.ThrowsAsync(base.Equal_with_null_reference_type); + + public override Task Equal_with_null_captured() + => Assert.ThrowsAsync(base.Equal_with_null_captured); + + public override Task NotEqual_with_null_reference_type() + => Assert.ThrowsAsync(base.NotEqual_with_null_reference_type); + + public override Task NotEqual_with_null_captured() + => Assert.ThrowsAsync(base.NotEqual_with_null_captured); + + public override Task Equal_int_property_with_null_nullable_int() + => Assert.ThrowsAsync(base.Equal_int_property_with_null_nullable_int); + + #endregion + + #region Not + + // Pinecone currently doesn't support NOT in vector search pre-filters + // (https://www.mongodb.com/docs/atlas/atlas-vector-search/vector-search-stage/#atlas-vector-search-pre-filter) + public override Task Not_over_And() + => Assert.ThrowsAsync(base.Not_over_And); + + public override Task Not_over_Or() + => Assert.ThrowsAsync(base.Not_over_Or); + + #endregion + + public override Task Contains_over_field_string_array() + => Assert.ThrowsAsync(base.Contains_over_field_string_array); + + public override Task Contains_over_field_string_List() + => Assert.ThrowsAsync(base.Contains_over_field_string_List); + + // List fields not (currently) supported on SQLite (see #10343) + public override Task Contains_with_Enumerable_Contains() + => Assert.ThrowsAsync(base.Contains_with_Enumerable_Contains); + +#if !NETFRAMEWORK + // List fields not (currently) supported on SQLite (see #10343) + public override Task Contains_with_MemoryExtensions_Contains() + => Assert.ThrowsAsync(base.Contains_with_MemoryExtensions_Contains); +#endif + +#if NET10_0_OR_GREATER + public override Task Contains_with_MemoryExtensions_Contains_with_null_comparer() + => Assert.ThrowsAsync(base.Contains_with_MemoryExtensions_Contains_with_null_comparer); +#endif + + // Any with Contains over array fields not supported on Pinecone + public override Task Any_with_Contains_over_inline_string_array() + => Assert.ThrowsAsync(base.Any_with_Contains_over_inline_string_array); + + public override Task Any_with_Contains_over_captured_string_array() + => Assert.ThrowsAsync(base.Any_with_Contains_over_captured_string_array); + + public override Task Any_with_Contains_over_captured_string_list() + => Assert.ThrowsAsync(base.Any_with_Contains_over_captured_string_list); + + public override Task Any_over_List_with_Contains_over_captured_string_array() + => Assert.ThrowsAsync(base.Any_over_List_with_Contains_over_captured_string_array); + + public new class Fixture : FilterTests.Fixture + { + public override TestStore TestStore => PineconeTestStore.Instance; + } +} diff --git a/MEVD/test/Pinecone.ConformanceTests/PineconeIndexKindTests.cs b/MEVD/test/Pinecone.ConformanceTests/PineconeIndexKindTests.cs new file mode 100644 index 0000000..5ed8129 --- /dev/null +++ b/MEVD/test/Pinecone.ConformanceTests/PineconeIndexKindTests.cs @@ -0,0 +1,34 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Pinecone.ConformanceTests.Support; +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace Pinecone.ConformanceTests; + +public class PineconeIndexKindTests(PineconeIndexKindTests.Fixture fixture) + : IndexKindTests(fixture), IClassFixture +{ + // Pinecone does not support index-less searching + public override Task Flat() => Assert.ThrowsAsync(base.Flat); + + [Fact] + public virtual Task PGA() + => this.Test("PGA"); + + protected override async Task Test(string indexKind) + { + await base.Test(indexKind); + + // The Pinecone emulator needs some extra time to spawn a new index service + // that uses a different distance function. + await Task.Delay(TimeSpan.FromSeconds(5)); + } + + public new class Fixture() : IndexKindTests.Fixture + { + public override TestStore TestStore => PineconeTestStore.Instance; + } +} diff --git a/MEVD/test/Pinecone.ConformanceTests/PineconeTestSuiteImplementationTests.cs b/MEVD/test/Pinecone.ConformanceTests/PineconeTestSuiteImplementationTests.cs new file mode 100644 index 0000000..f0a96ac --- /dev/null +++ b/MEVD/test/Pinecone.ConformanceTests/PineconeTestSuiteImplementationTests.cs @@ -0,0 +1,22 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.ModelTests; + +namespace Pinecone.ConformanceTests; + +public class PineconeTestSuiteImplementationTests : TestSuiteImplementationTests +{ + protected override ICollection IgnoredTestBases { get; } = + [ + // Pinecone does not support multiple vectors + typeof(MultiVectorModelTests<>), + + // Hybrid search not supported + typeof(HybridSearchTests<>), + + // Pinecone requires at least one vector property; records without vectors are not supported. + typeof(NoVectorModelTests<>), + ]; +} diff --git a/MEVD/test/Pinecone.ConformanceTests/Properties/AssemblyAttributes.cs b/MEVD/test/Pinecone.ConformanceTests/Properties/AssemblyAttributes.cs new file mode 100644 index 0000000..6571e2d --- /dev/null +++ b/MEVD/test/Pinecone.ConformanceTests/Properties/AssemblyAttributes.cs @@ -0,0 +1,6 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Xunit; + +[assembly: CollectionBehavior(DisableTestParallelization = true)] diff --git a/MEVD/test/Pinecone.ConformanceTests/Support/PineconeAllTypes.cs b/MEVD/test/Pinecone.ConformanceTests/Support/PineconeAllTypes.cs new file mode 100644 index 0000000..3e3092f --- /dev/null +++ b/MEVD/test/Pinecone.ConformanceTests/Support/PineconeAllTypes.cs @@ -0,0 +1,103 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.VectorData; +using Xunit; + +namespace Pinecone.ConformanceTests.Support; + +#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable. +public record PineconeAllTypes() +{ + [VectorStoreKey] + public string Id { get; set; } + + [VectorStoreData] + public bool BoolProperty { get; set; } + [VectorStoreData] + public bool? NullableBoolProperty { get; set; } + [VectorStoreData] + public string StringProperty { get; set; } + [VectorStoreData] + public string? NullableStringProperty { get; set; } + [VectorStoreData] + public int IntProperty { get; set; } + [VectorStoreData] + public int? NullableIntProperty { get; set; } + [VectorStoreData] + public long LongProperty { get; set; } + [VectorStoreData] + public long? NullableLongProperty { get; set; } + [VectorStoreData] + public float FloatProperty { get; set; } + [VectorStoreData] + public float? NullableFloatProperty { get; set; } + [VectorStoreData] + public double DoubleProperty { get; set; } + [VectorStoreData] + public double? NullableDoubleProperty { get; set; } + +#pragma warning disable CA1819 // Properties should not return arrays + [VectorStoreData] + public string[] StringArray { get; set; } + [VectorStoreData] + public string[]? NullableStringArray { get; set; } +#pragma warning restore CA1819 // Properties should not return arrays + + [VectorStoreData] + public List StringList { get; set; } + [VectorStoreData] + public List? NullableStringList { get; set; } + + [VectorStoreVector(dimensions: 8, DistanceFunction = DistanceFunction.DotProductSimilarity)] + public ReadOnlyMemory? Embedding { get; set; } + + internal void AssertEqual(PineconeAllTypes other) + { + Assert.Equal(this.Id, other.Id); + Assert.Equal(this.BoolProperty, other.BoolProperty); + Assert.Equal(this.NullableBoolProperty, other.NullableBoolProperty); + Assert.Equal(this.StringProperty, other.StringProperty); + Assert.Equal(this.NullableStringProperty, other.NullableStringProperty); + Assert.Equal(this.IntProperty, other.IntProperty); + Assert.Equal(this.NullableIntProperty, other.NullableIntProperty); + Assert.Equal(this.LongProperty, other.LongProperty); + Assert.Equal(this.NullableLongProperty, other.NullableLongProperty); + Assert.Equal(this.FloatProperty, other.FloatProperty); + Assert.Equal(this.NullableFloatProperty, other.NullableFloatProperty); + Assert.Equal(this.DoubleProperty, other.DoubleProperty); + Assert.Equal(this.NullableDoubleProperty, other.NullableDoubleProperty); + Assert.Equal(this.StringArray, other.StringArray); + Assert.Equal(this.NullableStringArray, other.NullableStringArray); + Assert.Equal(this.StringList, other.StringList); + Assert.Equal(this.NullableStringList, other.NullableStringList); + Assert.Equal(this.Embedding!.Value.ToArray(), other.Embedding!.Value.ToArray()); + } + + internal static VectorStoreCollectionDefinition GetRecordDefinition() + => new() + { + Properties = + [ + new VectorStoreKeyProperty(nameof(PineconeAllTypes.Id), typeof(string)), + new VectorStoreDataProperty(nameof(PineconeAllTypes.BoolProperty), typeof(bool)), + new VectorStoreDataProperty(nameof(PineconeAllTypes.NullableBoolProperty), typeof(bool?)), + new VectorStoreDataProperty(nameof(PineconeAllTypes.StringProperty), typeof(string)), + new VectorStoreDataProperty(nameof(PineconeAllTypes.NullableStringProperty), typeof(string)), + new VectorStoreDataProperty(nameof(PineconeAllTypes.IntProperty), typeof(int)), + new VectorStoreDataProperty(nameof(PineconeAllTypes.NullableIntProperty), typeof(int?)), + new VectorStoreDataProperty(nameof(PineconeAllTypes.LongProperty), typeof(long)), + new VectorStoreDataProperty(nameof(PineconeAllTypes.NullableLongProperty), typeof(long?)), + new VectorStoreDataProperty(nameof(PineconeAllTypes.FloatProperty), typeof(float)), + new VectorStoreDataProperty(nameof(PineconeAllTypes.NullableFloatProperty), typeof(float?)), + new VectorStoreDataProperty(nameof(PineconeAllTypes.DoubleProperty), typeof(double)), + new VectorStoreDataProperty(nameof(PineconeAllTypes.NullableDoubleProperty), typeof(double?)), + new VectorStoreDataProperty(nameof(PineconeAllTypes.StringArray), typeof(string[])), + new VectorStoreDataProperty(nameof(PineconeAllTypes.NullableStringArray), typeof(string[])), + new VectorStoreDataProperty(nameof(PineconeAllTypes.StringList), typeof(List)), + new VectorStoreDataProperty(nameof(PineconeAllTypes.NullableStringList), typeof(List)), + new VectorStoreVectorProperty(nameof(PineconeAllTypes.Embedding), typeof(ReadOnlyMemory?), 8) { DistanceFunction = Microsoft.Extensions.VectorData.DistanceFunction.DotProductSimilarity } + ] + }; +} +#pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable. diff --git a/MEVD/test/Pinecone.ConformanceTests/Support/PineconeFixture.cs b/MEVD/test/Pinecone.ConformanceTests/Support/PineconeFixture.cs new file mode 100644 index 0000000..160175a --- /dev/null +++ b/MEVD/test/Pinecone.ConformanceTests/Support/PineconeFixture.cs @@ -0,0 +1,11 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using VectorData.ConformanceTests.Support; + +namespace Pinecone.ConformanceTests.Support; + +public class PineconeFixture : VectorStoreFixture +{ + public override TestStore TestStore => PineconeTestStore.Instance; +} diff --git a/MEVD/test/Pinecone.ConformanceTests/Support/PineconeTestStore.cs b/MEVD/test/Pinecone.ConformanceTests/Support/PineconeTestStore.cs new file mode 100644 index 0000000..d4e771a --- /dev/null +++ b/MEVD/test/Pinecone.ConformanceTests/Support/PineconeTestStore.cs @@ -0,0 +1,178 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#pragma warning disable IDE0005 // Using directive is unnecessary. +#pragma warning restore IDE0005 // Using directive is unnecessary. +using System.Net.Http; +using DotNet.Testcontainers.Builders; +using DotNet.Testcontainers.Containers; +using Humanizer; +using CommunityToolkit.VectorData.Pinecone; +using VectorData.ConformanceTests.Support; + +namespace Pinecone.ConformanceTests.Support; + +#pragma warning disable CA1001 // Type owns disposable fields but is not disposable +#pragma warning disable CA2000 // Dispose objects before losing scope + +internal sealed class PineconeTestStore : TestStore +{ + // Values taken from https://docs.pinecone.io/guides/operations/local-development + // v0.7.0 works with 2.1 client + // v1.0.0 works with 3.0 client + // We use hardcoded version to avoid breaking changes. + private const string Image = "ghcr.io/pinecone-io/pinecone-local:v1.0.0.rc0"; + private const ushort FirstPort = 5080; + private const int IndexServiceCount = 20; + + public static PineconeTestStore Instance { get; } = new(); + + private IContainer? _container; + private PineconeClient? _client; + private ClientOptions? _clientOptions; + + public PineconeClient Client => this._client ?? throw new InvalidOperationException("Not initialized"); + + public ClientOptions ClientOptions => this._clientOptions ?? throw new InvalidOperationException("Not initialized"); + + public PineconeVectorStore GetVectorStore(PineconeVectorStoreOptions options) + => new(this.Client, options); + + // Pinecone does not support distance functions other than PGA which is always enabled. + public override string DefaultIndexKind => ""; + + private PineconeTestStore() + { + } + + protected override async Task StartAsync() + { + this._container = await this.StartContainerAsync(); + + Dictionary containerToHostPort = Enumerable.Range(FirstPort, IndexServiceCount + 1) + .ToDictionary(port => port, port => (int)this._container.GetMappedPublicPort(port)); + + UriBuilder baseAddress = new() + { + Scheme = "http", + Host = this._container.Hostname, + Port = this._container.GetMappedPublicPort(FirstPort) + }; + + this._clientOptions = new() + { + BaseUrl = baseAddress.Uri.ToString(), + MaxRetries = 3, + IsTlsEnabled = false, + // The Pinecone SDK only retries on HTTP status codes (408/429/500+), not on connection failures. + // The Pinecone local container can reset idle connections between test class executions, so we + // use a retry handler that catches HttpRequestException (connection reset) and retries. + HttpClient = new(new RetryHandler()) + { + BaseAddress = baseAddress.Uri + }, + GrpcOptions = new() + { + HttpClient = new(new RedirectHandler(containerToHostPort), disposeHandler: true) + { + BaseAddress = baseAddress.Uri + }, + } + }; + + this._client = new( + apiKey: "ForPineconeLocalTheApiKeysAreIgnored", + clientOptions: this._clientOptions); + + this.DefaultVectorStore = new PineconeVectorStore(this._client); + } + + protected override async Task StopAsync() + { + if (this._container is not null) + { + await this._container.DisposeAsync(); + } + } + + private async Task StartContainerAsync() + { + ContainerBuilder builder = new ContainerBuilder("ghcr.io/pinecone-io/pinecone-local:latest") + .WithImage(Image) + // Pinecone Local will run on port $FirstPort. + .WithPortBinding(FirstPort, assignRandomHostPort: true) + // We are currently using the default Pinecone port (5080), but we can change it to a random port. + // In such case, we are going to need to set the PORT environment variable to the new port. + .WithEnvironment("PORT", FirstPort.ToString()); + + for (int indexService = 1; indexService <= IndexServiceCount; indexService++) + { + // And the index services on the following ports. + builder = builder.WithPortBinding(FirstPort + indexService, assignRandomHostPort: true); + } + + var container = builder.Build(); + + await container.StartAsync(); + + return container; + } + + // Collection name must contain only ASCII lowercase letters, digits and dashes. + // https://docs.pinecone.io/troubleshooting/restrictions-on-index-names + public override string AdjustCollectionName(string baseName) + => baseName.Kebaberize(); + + /// + /// Retries on (connection reset by peer) which the Pinecone SDK + /// doesn't handle — its built-in retry only covers HTTP status codes. + /// + private sealed class RetryHandler() : DelegatingHandler(new SocketsHttpHandler()) + { + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + for (var attempt = 0; ; attempt++) + { + try + { + return await base.SendAsync(request, cancellationToken); + } + catch (HttpRequestException) when (attempt < 3) + { + await Task.Delay(200 * (attempt + 1), cancellationToken); + } + } + } + } + + private sealed class RedirectHandler : DelegatingHandler + { + private readonly Dictionary _containerToHostPort; + + public RedirectHandler(Dictionary portRedirections) + : base(new HttpClientHandler()) + { + this._containerToHostPort = portRedirections; + } + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + // When "host" argument is not provided for PineconeClient.Index, + // it will try to get the host from the Pinecone service. + // In the cloud environment it's fine, but with the local emulator + // it reports the address with the container port, not the host port. + if (request.RequestUri != null && request.RequestUri.IsAbsoluteUri + && request.RequestUri.Host == "localhost" + && this._containerToHostPort.TryGetValue(request.RequestUri.Port, out int hostPort)) + { + UriBuilder builder = new(request.RequestUri) + { + Port = hostPort + }; + request.RequestUri = builder.Uri; + } + + return base.SendAsync(request, cancellationToken); + } + } +} diff --git a/MEVD/test/Pinecone.ConformanceTests/TypeTests/PineconeDataTypeTests.cs b/MEVD/test/Pinecone.ConformanceTests/TypeTests/PineconeDataTypeTests.cs new file mode 100644 index 0000000..65b43de --- /dev/null +++ b/MEVD/test/Pinecone.ConformanceTests/TypeTests/PineconeDataTypeTests.cs @@ -0,0 +1,37 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Pinecone.ConformanceTests.Support; +using VectorData.ConformanceTests.Support; +using VectorData.ConformanceTests.TypeTests; +using Xunit; + +namespace Pinecone.ConformanceTests.TypeTests; + +public class PineconeDataTypeTests(PineconeDataTypeTests.Fixture fixture) + : DataTypeTests.DefaultRecord>(fixture), IClassFixture +{ + public new class Fixture : DataTypeTests.DefaultRecord>.Fixture + { + public override TestStore TestStore => PineconeTestStore.Instance; + + // Pincone does not support null checks in vector search pre-filters + public override bool IsNullFilteringSupported => false; + + public override Type[] UnsupportedDefaultTypes { get; } = + [ + typeof(byte), + typeof(short), + typeof(decimal), + typeof(Guid), + typeof(DateTime), + typeof(DateTimeOffset), + typeof(string[]), // TODO: Error with gRPC status code 3 + +#if NET + typeof(DateOnly), + typeof(TimeOnly) +#endif + ]; + } +} diff --git a/MEVD/test/Pinecone.ConformanceTests/TypeTests/PineconeEmbeddingTypeTests.cs b/MEVD/test/Pinecone.ConformanceTests/TypeTests/PineconeEmbeddingTypeTests.cs new file mode 100644 index 0000000..196fd99 --- /dev/null +++ b/MEVD/test/Pinecone.ConformanceTests/TypeTests/PineconeEmbeddingTypeTests.cs @@ -0,0 +1,20 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Pinecone.ConformanceTests.Support; +using VectorData.ConformanceTests.Support; +using VectorData.ConformanceTests.TypeTests; +using Xunit; + +#pragma warning disable CA2000 // Dispose objects before losing scope + +namespace Pinecone.ConformanceTests.TypeTests; + +public class PineconeEmbeddingTypeTests(PineconeEmbeddingTypeTests.Fixture fixture) + : EmbeddingTypeTests(fixture), IClassFixture +{ + public new class Fixture : EmbeddingTypeTests.Fixture + { + public override TestStore TestStore => PineconeTestStore.Instance; + } +} diff --git a/MEVD/test/Pinecone.ConformanceTests/TypeTests/PineconeKeyTypeTests.cs b/MEVD/test/Pinecone.ConformanceTests/TypeTests/PineconeKeyTypeTests.cs new file mode 100644 index 0000000..bcf195f --- /dev/null +++ b/MEVD/test/Pinecone.ConformanceTests/TypeTests/PineconeKeyTypeTests.cs @@ -0,0 +1,35 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.VectorData; +using Pinecone.ConformanceTests.Support; +using VectorData.ConformanceTests.Support; +using VectorData.ConformanceTests.TypeTests; +using Xunit; + +namespace Pinecone.ConformanceTests.TypeTests; + +public class PineconeKeyTypeTests(PineconeKeyTypeTests.Fixture fixture) + : KeyTypeTests(fixture), IClassFixture +{ + [Fact] + public virtual Task String() => this.Test("foo", "bar"); + + public new class Fixture : KeyTypeTests.Fixture + { + public override TestStore TestStore => PineconeTestStore.Instance; + + // The Pinecone local emulator has eventual consistency, so deleting and recreating + // a collection with the same name but different key types can cause stale data from + // the previous test to bleed through. Use unique collection names per key type. + public override VectorStoreCollection> CreateCollection(bool? withAutoGeneration) + => TestStore.CreateCollection>( + TestStore.AdjustCollectionName($"key-type-{typeof(TKey).Name}"), + CreateRecordDefinition(withAutoGeneration)); + + public override VectorStoreCollection> CreateDynamicCollection(bool withAutoGeneration) + => TestStore.CreateDynamicCollection( + TestStore.AdjustCollectionName($"key-type-{typeof(TKey).Name}"), + CreateRecordDefinition(withAutoGeneration)); + } +} diff --git a/MEVD/test/Pinecone.UnitTests/.editorconfig b/MEVD/test/Pinecone.UnitTests/.editorconfig new file mode 100644 index 0000000..394eef6 --- /dev/null +++ b/MEVD/test/Pinecone.UnitTests/.editorconfig @@ -0,0 +1,6 @@ +# Suppressing errors for Test projects under dotnet folder +[*.cs] +dotnet_diagnostic.CA2007.severity = none # Do not directly await a Task +dotnet_diagnostic.VSTHRD111.severity = none # Use .ConfigureAwait(bool) is hidden by default, set to none to prevent IDE from changing on autosave +dotnet_diagnostic.CS1591.severity = none # Missing XML comment for publicly visible type or member +dotnet_diagnostic.IDE1006.severity = warning # Naming rule violations diff --git a/MEVD/test/Pinecone.UnitTests/Pinecone.UnitTests.csproj b/MEVD/test/Pinecone.UnitTests/Pinecone.UnitTests.csproj new file mode 100644 index 0000000..8472547 --- /dev/null +++ b/MEVD/test/Pinecone.UnitTests/Pinecone.UnitTests.csproj @@ -0,0 +1,39 @@ + + + + CommunityToolkit.VectorData.Pinecone.UnitTests + CommunityToolkit.VectorData.Pinecone.UnitTests + net10.0 + true + enable + disable + false + $(NoWarn);CA2007,CA1806,CA1869,CA1861,IDE0300,VSTHRD111,SKEXP0001,SKEXP0010,SKEXP0050 + + + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + + + + + diff --git a/MEVD/test/Pinecone.UnitTests/PineconeCollectionTests.cs b/MEVD/test/Pinecone.UnitTests/PineconeCollectionTests.cs new file mode 100644 index 0000000..b7ed6e4 --- /dev/null +++ b/MEVD/test/Pinecone.UnitTests/PineconeCollectionTests.cs @@ -0,0 +1,56 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using Microsoft.Extensions.VectorData; +using CommunityToolkit.VectorData.Pinecone; +using Pinecone; +using Xunit; + +namespace SemanticKernel.Connectors.Pinecone.UnitTests; + +/// +/// Contains tests for the class. +/// +public class PineconeCollectionTests +{ + private const string TestCollectionName = "testcollection"; + + /// + /// Tests that the collection can be created even if the definition and the type do not match. + /// In this case, the expectation is that a custom mapper will be provided to map between the + /// schema as defined by the definition and the different data model. + /// + [Fact] + public void CanCreateCollectionWithMismatchedDefinitionAndType() + { + // Arrange. + var definition = new VectorStoreCollectionDefinition() + { + Properties = + [ + new VectorStoreKeyProperty("Key", typeof(string)), + new VectorStoreDataProperty("OriginalNameData", typeof(string)), + new VectorStoreVectorProperty("Vector", typeof(ReadOnlyMemory?), 4), + ] + }; + var pineconeClient = new PineconeClient("fake api key"); + + // Act. + using var sut = new PineconeCollection( + pineconeClient, + TestCollectionName, + new() { Definition = definition }); + } + + public sealed class SinglePropsModel + { + public string Key { get; set; } = string.Empty; + + public string OriginalNameData { get; set; } = string.Empty; + + public string Data { get; set; } = string.Empty; + + public ReadOnlyMemory? Vector { get; set; } + } +} diff --git a/MEVD/test/Qdrant.ConformanceTests/ModelTests/QdrantBasicModelTests.cs b/MEVD/test/Qdrant.ConformanceTests/ModelTests/QdrantBasicModelTests.cs new file mode 100644 index 0000000..54e510e --- /dev/null +++ b/MEVD/test/Qdrant.ConformanceTests/ModelTests/QdrantBasicModelTests.cs @@ -0,0 +1,25 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Qdrant.ConformanceTests.Support; +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace Qdrant.ConformanceTests.ModelTests; + +public class QdrantBasicModelTests(QdrantBasicModelTests.Fixture fixture) + : BasicModelTests(fixture), IClassFixture +{ + public override async Task GetAsync_with_filter_and_multiple_OrderBys() + { + var exception = await Assert.ThrowsAsync(base.GetAsync_with_filter_and_multiple_OrderBys); + + Assert.Equal("Qdrant does not support ordering by more than one property.", exception.Message); + } + + public new class Fixture : BasicModelTests.Fixture + { + public override TestStore TestStore => QdrantTestStore.NamedVectorsInstance; + } +} diff --git a/MEVD/test/Qdrant.ConformanceTests/ModelTests/QdrantDynamicModelTests.cs b/MEVD/test/Qdrant.ConformanceTests/ModelTests/QdrantDynamicModelTests.cs new file mode 100644 index 0000000..d6f0de1 --- /dev/null +++ b/MEVD/test/Qdrant.ConformanceTests/ModelTests/QdrantDynamicModelTests.cs @@ -0,0 +1,41 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Qdrant.ConformanceTests.Support; +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace Qdrant.ConformanceTests.ModelTests; + +public class QdrantDynamicModelTests_NamedVectors(QdrantDynamicModelTests_NamedVectors.Fixture fixture) + : DynamicModelTests(fixture), IClassFixture +{ + public override async Task GetAsync_with_filter_and_multiple_OrderBys() + { + var exception = await Assert.ThrowsAsync(base.GetAsync_with_filter_and_multiple_OrderBys); + + Assert.Equal("Qdrant does not support ordering by more than one property.", exception.Message); + } + + public new class Fixture : DynamicModelTests.Fixture + { + public override TestStore TestStore => QdrantTestStore.NamedVectorsInstance; + } +} + +public class QdrantDynamicModelTests_UnnamedVector(QdrantDynamicModelTests_UnnamedVector.Fixture fixture) + : DynamicModelTests(fixture), IClassFixture +{ + public override async Task GetAsync_with_filter_and_multiple_OrderBys() + { + var exception = await Assert.ThrowsAsync(base.GetAsync_with_filter_and_multiple_OrderBys); + + Assert.Equal("Qdrant does not support ordering by more than one property.", exception.Message); + } + + public new class Fixture : DynamicModelTests.Fixture + { + public override TestStore TestStore => QdrantTestStore.UnnamedVectorInstance; + } +} diff --git a/MEVD/test/Qdrant.ConformanceTests/ModelTests/QdrantMultiVectorModelTests.cs b/MEVD/test/Qdrant.ConformanceTests/ModelTests/QdrantMultiVectorModelTests.cs new file mode 100644 index 0000000..9f6fdbb --- /dev/null +++ b/MEVD/test/Qdrant.ConformanceTests/ModelTests/QdrantMultiVectorModelTests.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Qdrant.ConformanceTests.Support; +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace Qdrant.ConformanceTests.ModelTests; + +public class QdrantMultiVectorModelTests(QdrantMultiVectorModelTests.Fixture fixture) + : MultiVectorModelTests(fixture), IClassFixture +{ + public new class Fixture : MultiVectorModelTests.Fixture + { + public override TestStore TestStore => QdrantTestStore.NamedVectorsInstance; + } +} diff --git a/MEVD/test/Qdrant.ConformanceTests/ModelTests/QdrantNoDataModelTests.cs b/MEVD/test/Qdrant.ConformanceTests/ModelTests/QdrantNoDataModelTests.cs new file mode 100644 index 0000000..f357e63 --- /dev/null +++ b/MEVD/test/Qdrant.ConformanceTests/ModelTests/QdrantNoDataModelTests.cs @@ -0,0 +1,27 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Qdrant.ConformanceTests.Support; +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace Qdrant.ConformanceTests.ModelTests; + +public class QdrantNoDataModelTests_NamedVectors(QdrantNoDataModelTests_NamedVectors.Fixture fixture) + : NoDataModelTests(fixture), IClassFixture +{ + public new class Fixture : NoDataModelTests.Fixture + { + public override TestStore TestStore => QdrantTestStore.NamedVectorsInstance; + } +} + +public class QdrantNoDataModelTests_UnnamedVectors(QdrantNoDataModelTests_UnnamedVectors.Fixture fixture) + : NoDataModelTests(fixture), IClassFixture +{ + public new class Fixture : NoDataModelTests.Fixture + { + public override TestStore TestStore => QdrantTestStore.UnnamedVectorInstance; + } +} diff --git a/MEVD/test/Qdrant.ConformanceTests/ModelTests/QdrantNoVectorModelTests.cs b/MEVD/test/Qdrant.ConformanceTests/ModelTests/QdrantNoVectorModelTests.cs new file mode 100644 index 0000000..d2f4c14 --- /dev/null +++ b/MEVD/test/Qdrant.ConformanceTests/ModelTests/QdrantNoVectorModelTests.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Qdrant.ConformanceTests.Support; +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace Qdrant.ConformanceTests.ModelTests; + +public class QdrantNoVectorModelTests_NamedVectors(QdrantNoVectorModelTests_NamedVectors.Fixture fixture) + : NoVectorModelTests(fixture), IClassFixture +{ + public new class Fixture : NoVectorModelTests.Fixture + { + public override TestStore TestStore => QdrantTestStore.NamedVectorsInstance; + } +} diff --git a/MEVD/test/Qdrant.ConformanceTests/Qdrant.ConformanceTests.csproj b/MEVD/test/Qdrant.ConformanceTests/Qdrant.ConformanceTests.csproj new file mode 100644 index 0000000..9057bbc --- /dev/null +++ b/MEVD/test/Qdrant.ConformanceTests/Qdrant.ConformanceTests.csproj @@ -0,0 +1,28 @@ + + + + net10.0;$(NetFrameworkTfm) + enable + enable + true + false + Qdrant.ConformanceTests + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + + + + + diff --git a/MEVD/test/Qdrant.ConformanceTests/QdrantCollectionManagementTests.cs b/MEVD/test/Qdrant.ConformanceTests/QdrantCollectionManagementTests.cs new file mode 100644 index 0000000..07b83d2 --- /dev/null +++ b/MEVD/test/Qdrant.ConformanceTests/QdrantCollectionManagementTests.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Qdrant.ConformanceTests.Support; +using VectorData.ConformanceTests; +using Xunit; + +namespace Qdrant.ConformanceTests; + +public class QdrantCollectionManagementTests_NamedVectors(QdrantNamedVectorsFixture fixture) + : CollectionManagementTests(fixture), IClassFixture +{ +} + +public class QdrantCollectionManagementTests_UnnamedVector(QdrantUnnamedVectorFixture fixture) + : CollectionManagementTests(fixture), IClassFixture +{ +} diff --git a/MEVD/test/Qdrant.ConformanceTests/QdrantDependencyInjectionTests.cs b/MEVD/test/Qdrant.ConformanceTests/QdrantDependencyInjectionTests.cs new file mode 100644 index 0000000..a134b4c --- /dev/null +++ b/MEVD/test/Qdrant.ConformanceTests/QdrantDependencyInjectionTests.cs @@ -0,0 +1,99 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using CommunityToolkit.VectorData.Qdrant; +using Qdrant.Client; +using VectorData.ConformanceTests; +using Xunit; + +namespace Qdrant.ConformanceTests; + +public class QdrantDependencyInjectionTests + : DependencyInjectionTests.Record>, ulong, DependencyInjectionTests.Record> +{ + private const string Host = "localhost"; + private const int Port = 8080; + private const string ApiKey = "fakeKey"; + + protected override void PopulateConfiguration(ConfigurationManager configuration, object? serviceKey = null) + => configuration.AddInMemoryCollection( + [ + new(CreateConfigKey("Qdrant", serviceKey, "Host"), Host), + new(CreateConfigKey("Qdrant", serviceKey, "Port"), Port.ToString()), + new(CreateConfigKey("Qdrant", serviceKey, "ApiKey"), ApiKey), + ]); + + private static string HostProvider(IServiceProvider sp, object? serviceKey = null) + => sp.GetRequiredService().GetRequiredSection(CreateConfigKey("Qdrant", serviceKey, "Host")).Value!; + + private static int PortProvider(IServiceProvider sp, object? serviceKey = null) + => int.Parse(sp.GetRequiredService().GetRequiredSection(CreateConfigKey("Qdrant", serviceKey, "Port")).Value!); + + private static string ApiKeyProvider(IServiceProvider sp, object? serviceKey = null) + => sp.GetRequiredService().GetRequiredSection(CreateConfigKey("Qdrant", serviceKey, "ApiKey")).Value!; + + public override IEnumerable> CollectionDelegates + { + get + { + yield return (services, serviceKey, name, lifetime) => serviceKey is null + ? services + .AddSingleton(sp => new QdrantClient(Host, Port, apiKey: ApiKey)) + .AddQdrantCollection(name, lifetime: lifetime) + : services + .AddSingleton(sp => new QdrantClient(Host, Port, apiKey: ApiKey)) + .AddKeyedQdrantCollection(serviceKey, name, lifetime: lifetime); + + yield return (services, serviceKey, name, lifetime) => serviceKey is null + ? services.AddQdrantCollection( + name, Host, Port, apiKey: ApiKey, lifetime: lifetime) + : services.AddKeyedQdrantCollection( + serviceKey, name, Host, Port, apiKey: ApiKey, lifetime: lifetime); + + yield return (services, serviceKey, name, lifetime) => serviceKey is null + ? services.AddQdrantCollection( + name, sp => new QdrantClient(HostProvider(sp), PortProvider(sp), apiKey: ApiKeyProvider(sp)), lifetime: lifetime) + : services.AddKeyedQdrantCollection( + serviceKey, name, sp => new QdrantClient(HostProvider(sp, serviceKey), PortProvider(sp, serviceKey), apiKey: ApiKeyProvider(sp, serviceKey)), lifetime: lifetime); + } + } + + public override IEnumerable> StoreDelegates + { + get + { + yield return (services, serviceKey, lifetime) => serviceKey is null + ? services.AddQdrantVectorStore( + Host, Port, apiKey: ApiKey, lifetime: lifetime) + : services.AddKeyedQdrantVectorStore( + serviceKey, Host, Port, apiKey: ApiKey, lifetime: lifetime); + + yield return (services, serviceKey, lifetime) => serviceKey is null + ? services + .AddSingleton(sp => new QdrantClient(Host, Port, apiKey: ApiKey)) + .AddQdrantVectorStore(lifetime: lifetime) + : services + .AddSingleton(sp => new QdrantClient(Host, Port, apiKey: ApiKey)) + .AddKeyedQdrantVectorStore(serviceKey, lifetime: lifetime); + } + } + + [Fact] + public void HostCantBeNullOrEmpty() + { + IServiceCollection services = new ServiceCollection(); + + Assert.Throws(() => services.AddQdrantVectorStore(host: null!)); + Assert.Throws(() => services.AddKeyedQdrantVectorStore(serviceKey: "notNull", host: null!)); + Assert.Throws(() => services.AddQdrantCollection( + name: "notNull", host: null!)); + Assert.Throws(() => services.AddQdrantCollection( + name: "notNull", host: "")); + Assert.Throws(() => services.AddKeyedQdrantCollection( + serviceKey: "notNull", name: "notNull", host: null!)); + Assert.Throws(() => services.AddKeyedQdrantCollection( + serviceKey: "notNull", name: "notNull", host: "")); + } +} diff --git a/MEVD/test/Qdrant.ConformanceTests/QdrantDistanceFunctionTests.cs b/MEVD/test/Qdrant.ConformanceTests/QdrantDistanceFunctionTests.cs new file mode 100644 index 0000000..050430c --- /dev/null +++ b/MEVD/test/Qdrant.ConformanceTests/QdrantDistanceFunctionTests.cs @@ -0,0 +1,23 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Qdrant.ConformanceTests.Support; +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace Qdrant.ConformanceTests; + +public class QdrantDistanceFunctionTests(QdrantDistanceFunctionTests.Fixture fixture) + : DistanceFunctionTests(fixture), IClassFixture +{ + public override Task CosineDistance() => Assert.ThrowsAsync(base.CosineDistance); + public override Task NegativeDotProductSimilarity() => Assert.ThrowsAsync(base.NegativeDotProductSimilarity); + public override Task EuclideanSquaredDistance() => Assert.ThrowsAsync(base.EuclideanSquaredDistance); + public override Task HammingDistance() => Assert.ThrowsAsync(base.HammingDistance); + + public new class Fixture() : DistanceFunctionTests.Fixture + { + public override TestStore TestStore => QdrantTestStore.NamedVectorsInstance; + } +} diff --git a/MEVD/test/Qdrant.ConformanceTests/QdrantEmbeddingGenerationTests.cs b/MEVD/test/Qdrant.ConformanceTests/QdrantEmbeddingGenerationTests.cs new file mode 100644 index 0000000..706a1ac --- /dev/null +++ b/MEVD/test/Qdrant.ConformanceTests/QdrantEmbeddingGenerationTests.cs @@ -0,0 +1,60 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.VectorData; +using Qdrant.ConformanceTests.Support; +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace Qdrant.ConformanceTests; + +public class QdrantEmbeddingGenerationTests(QdrantEmbeddingGenerationTests.StringVectorFixture stringVectorFixture, QdrantEmbeddingGenerationTests.RomOfFloatVectorFixture romOfFloatVectorFixture) + : EmbeddingGenerationTests(stringVectorFixture, romOfFloatVectorFixture), IClassFixture, IClassFixture +{ + public new class StringVectorFixture : EmbeddingGenerationTests.StringVectorFixture + { + public override TestStore TestStore => QdrantTestStore.UnnamedVectorInstance; + + public override VectorStore CreateVectorStore(IEmbeddingGenerator? embeddingGenerator) + => QdrantTestStore.UnnamedVectorInstance.GetVectorStore(new() { EmbeddingGenerator = embeddingGenerator }); + + public override Func[] DependencyInjectionStoreRegistrationDelegates => + [ + services => services + .AddSingleton(QdrantTestStore.UnnamedVectorInstance.Client) + .AddQdrantVectorStore() + ]; + + public override Func[] DependencyInjectionCollectionRegistrationDelegates => + [ + services => services + .AddSingleton(QdrantTestStore.UnnamedVectorInstance.Client) + .AddQdrantCollection(this.CollectionName) + ]; + } + + public new class RomOfFloatVectorFixture : EmbeddingGenerationTests.RomOfFloatVectorFixture + { + public override TestStore TestStore => QdrantTestStore.UnnamedVectorInstance; + + public override VectorStore CreateVectorStore(IEmbeddingGenerator? embeddingGenerator) + => QdrantTestStore.UnnamedVectorInstance.GetVectorStore(new() { EmbeddingGenerator = embeddingGenerator }); + + public override Func[] DependencyInjectionStoreRegistrationDelegates => + [ + services => services + .AddSingleton(QdrantTestStore.UnnamedVectorInstance.Client) + .AddQdrantVectorStore() + ]; + + public override Func[] DependencyInjectionCollectionRegistrationDelegates => + [ + services => services + .AddSingleton(QdrantTestStore.UnnamedVectorInstance.Client) + .AddQdrantCollection(this.CollectionName) + ]; + } +} diff --git a/MEVD/test/Qdrant.ConformanceTests/QdrantFilterTests.cs b/MEVD/test/Qdrant.ConformanceTests/QdrantFilterTests.cs new file mode 100644 index 0000000..a21578f --- /dev/null +++ b/MEVD/test/Qdrant.ConformanceTests/QdrantFilterTests.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Qdrant.ConformanceTests.Support; +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace Qdrant.ConformanceTests; + +public class QdrantFilterTests(QdrantFilterTests.Fixture fixture) + : FilterTests(fixture), IClassFixture +{ + public new class Fixture : FilterTests.Fixture + { + public override TestStore TestStore => QdrantTestStore.NamedVectorsInstance; + } +} diff --git a/MEVD/test/Qdrant.ConformanceTests/QdrantHybridSearchTests.cs b/MEVD/test/Qdrant.ConformanceTests/QdrantHybridSearchTests.cs new file mode 100644 index 0000000..28153e0 --- /dev/null +++ b/MEVD/test/Qdrant.ConformanceTests/QdrantHybridSearchTests.cs @@ -0,0 +1,45 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Qdrant.ConformanceTests.Support; +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace Qdrant.ConformanceTests; + +public class QdrantHybridSearchTests_NamedVectors( + QdrantHybridSearchTests_NamedVectors.VectorAndStringFixture vectorAndStringFixture, + QdrantHybridSearchTests_NamedVectors.MultiTextFixture multiTextFixture) + : HybridSearchTests(vectorAndStringFixture, multiTextFixture), + IClassFixture, + IClassFixture +{ + public new class VectorAndStringFixture : HybridSearchTests.VectorAndStringFixture + { + public override TestStore TestStore => QdrantTestStore.NamedVectorsInstance; + } + + public new class MultiTextFixture : HybridSearchTests.MultiTextFixture + { + public override TestStore TestStore => QdrantTestStore.NamedVectorsInstance; + } +} + +public class QdrantHybridSearchTests_UnnamedVectors( + QdrantHybridSearchTests_UnnamedVectors.VectorAndStringFixture vectorAndStringFixture, + QdrantHybridSearchTests_UnnamedVectors.MultiTextFixture multiTextFixture) + : HybridSearchTests(vectorAndStringFixture, multiTextFixture), + IClassFixture, + IClassFixture +{ + public new class VectorAndStringFixture : HybridSearchTests.VectorAndStringFixture + { + public override TestStore TestStore => QdrantTestStore.UnnamedVectorInstance; + } + + public new class MultiTextFixture : HybridSearchTests.MultiTextFixture + { + public override TestStore TestStore => QdrantTestStore.UnnamedVectorInstance; + } +} diff --git a/MEVD/test/Qdrant.ConformanceTests/QdrantIndexKindTests.cs b/MEVD/test/Qdrant.ConformanceTests/QdrantIndexKindTests.cs new file mode 100644 index 0000000..19db666 --- /dev/null +++ b/MEVD/test/Qdrant.ConformanceTests/QdrantIndexKindTests.cs @@ -0,0 +1,26 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.VectorData; +using Qdrant.ConformanceTests.Support; +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace Qdrant.ConformanceTests; + +public class QdrantIndexKindTests(QdrantIndexKindTests.Fixture fixture) + : IndexKindTests(fixture), IClassFixture +{ + // Qdrant does not support index-less searching + public override Task Flat() => Assert.ThrowsAsync(base.Flat); + + [Fact] + public virtual Task Hnsw() + => this.Test(IndexKind.Hnsw); + + public new class Fixture() : IndexKindTests.Fixture + { + public override TestStore TestStore => QdrantTestStore.NamedVectorsInstance; + } +} diff --git a/MEVD/test/Qdrant.ConformanceTests/QdrantTestSuiteImplementationTests.cs b/MEVD/test/Qdrant.ConformanceTests/QdrantTestSuiteImplementationTests.cs new file mode 100644 index 0000000..2a5ecef --- /dev/null +++ b/MEVD/test/Qdrant.ConformanceTests/QdrantTestSuiteImplementationTests.cs @@ -0,0 +1,8 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using VectorData.ConformanceTests; + +namespace Qdrant.ConformanceTests; + +public class QdrantTestSuiteImplementationTests : TestSuiteImplementationTests; diff --git a/MEVD/test/Qdrant.ConformanceTests/Support/QdrantNamedVectorsFixture.cs b/MEVD/test/Qdrant.ConformanceTests/Support/QdrantNamedVectorsFixture.cs new file mode 100644 index 0000000..94eb21a --- /dev/null +++ b/MEVD/test/Qdrant.ConformanceTests/Support/QdrantNamedVectorsFixture.cs @@ -0,0 +1,11 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using VectorData.ConformanceTests.Support; + +namespace Qdrant.ConformanceTests.Support; + +public class QdrantNamedVectorsFixture : VectorStoreFixture +{ + public override TestStore TestStore => QdrantTestStore.NamedVectorsInstance; +} diff --git a/MEVD/test/Qdrant.ConformanceTests/Support/QdrantTestStore.cs b/MEVD/test/Qdrant.ConformanceTests/Support/QdrantTestStore.cs new file mode 100644 index 0000000..d1592c5 --- /dev/null +++ b/MEVD/test/Qdrant.ConformanceTests/Support/QdrantTestStore.cs @@ -0,0 +1,60 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.VectorData; +using CommunityToolkit.VectorData.Qdrant; +using Qdrant.Client; +using Testcontainers.Qdrant; +using VectorData.ConformanceTests.Support; + +namespace Qdrant.ConformanceTests.Support; + +#pragma warning disable CA1001 // Type owns disposable fields but is not disposable + +internal sealed class QdrantTestStore : TestStore +{ + public static QdrantTestStore NamedVectorsInstance { get; } = new(hasNamedVectors: true); + public static QdrantTestStore UnnamedVectorInstance { get; } = new(hasNamedVectors: false); + + // Qdrant doesn't support the default Flat index kind + public override string DefaultIndexKind => IndexKind.Hnsw; + + private readonly QdrantContainer _container = new QdrantBuilder("qdrant/qdrant:v1.17.0").Build(); + private readonly bool _hasNamedVectors; + private QdrantClient? _client; + + public QdrantClient Client => this._client ?? throw new InvalidOperationException("Not initialized"); + + public QdrantVectorStore GetVectorStore(QdrantVectorStoreOptions options) + => new(this.Client, + ownsClient: false, // The client is shared, it's not owned by the vector store. + new() + { + HasNamedVectors = options.HasNamedVectors, + EmbeddingGenerator = options.EmbeddingGenerator + }); + + private QdrantTestStore(bool hasNamedVectors) => this._hasNamedVectors = hasNamedVectors; + + /// + /// Qdrant normalizes vectors on upsert, so we cannot compare + /// what we upserted and what we retrieve, we can only check + /// that a vector was returned. + /// https://github.com/qdrant/qdrant-client/discussions/727 + /// + public override bool VectorsComparable => false; + + protected override async Task StartAsync() + { + await this._container.StartAsync(); + this._client = new QdrantClient(this._container.Hostname, this._container.GetMappedPublicPort(QdrantBuilder.QdrantGrpcPort)); + // The client is shared, it's not owned by the vector store. + this.DefaultVectorStore = new QdrantVectorStore(this._client, ownsClient: false, new() { HasNamedVectors = this._hasNamedVectors }); + } + + protected override async Task StopAsync() + { + this._client?.Dispose(); + await this._container.StopAsync(); + } +} diff --git a/MEVD/test/Qdrant.ConformanceTests/Support/QdrantUnnamedVectorFixture.cs b/MEVD/test/Qdrant.ConformanceTests/Support/QdrantUnnamedVectorFixture.cs new file mode 100644 index 0000000..58a77c4 --- /dev/null +++ b/MEVD/test/Qdrant.ConformanceTests/Support/QdrantUnnamedVectorFixture.cs @@ -0,0 +1,11 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using VectorData.ConformanceTests.Support; + +namespace Qdrant.ConformanceTests.Support; + +public class QdrantUnnamedVectorFixture : VectorStoreFixture +{ + public override TestStore TestStore => QdrantTestStore.UnnamedVectorInstance; +} diff --git a/MEVD/test/Qdrant.ConformanceTests/TypeTests/QdrantDataTypeTests.cs b/MEVD/test/Qdrant.ConformanceTests/TypeTests/QdrantDataTypeTests.cs new file mode 100644 index 0000000..0f0fe72 --- /dev/null +++ b/MEVD/test/Qdrant.ConformanceTests/TypeTests/QdrantDataTypeTests.cs @@ -0,0 +1,47 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Qdrant.ConformanceTests.Support; +using VectorData.ConformanceTests.Support; +using VectorData.ConformanceTests.TypeTests; +using Xunit; + +namespace Qdrant.ConformanceTests.TypeTests; + +public class QdrantDataTypeTests(QdrantDataTypeTests.Fixture fixture) + : DataTypeTests.DefaultRecord>(fixture), IClassFixture +{ + // Qdrant doesn't seem to support filtering on float/double or string ararys, + // https://qdrant.tech/documentation/concepts/filtering/#match + [Fact] + public override Task Float() + => this.Test("Float", 8.5f, 9.5f, isFilterable: false); + + [Fact] + public override Task Double() + => this.Test("Double", 8.5d, 9.5d, isFilterable: false); + + [Fact] + public override Task String_array() + => this.Test( + "StringArray", + ["foo", "bar"], + ["foo", "baz"], + isFilterable: false); + + public new class Fixture : DataTypeTests.DefaultRecord>.Fixture + { + public override TestStore TestStore => QdrantTestStore.UnnamedVectorInstance; + + public override Type[] UnsupportedDefaultTypes { get; } = + [ + typeof(byte), + typeof(short), + typeof(decimal), + typeof(Guid), +#if NET + typeof(TimeOnly) +#endif + ]; + } +} diff --git a/MEVD/test/Qdrant.ConformanceTests/TypeTests/QdrantEmbeddingTypeTests.cs b/MEVD/test/Qdrant.ConformanceTests/TypeTests/QdrantEmbeddingTypeTests.cs new file mode 100644 index 0000000..fbec4e6 --- /dev/null +++ b/MEVD/test/Qdrant.ConformanceTests/TypeTests/QdrantEmbeddingTypeTests.cs @@ -0,0 +1,20 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Qdrant.ConformanceTests.Support; +using VectorData.ConformanceTests.Support; +using VectorData.ConformanceTests.TypeTests; +using Xunit; + +#pragma warning disable CA2000 // Dispose objects before losing scope + +namespace Qdrant.ConformanceTests.TypeTests; + +public class QdrantEmbeddingTypeTests(QdrantEmbeddingTypeTests.Fixture fixture) + : EmbeddingTypeTests(fixture), IClassFixture +{ + public new class Fixture : EmbeddingTypeTests.Fixture + { + public override TestStore TestStore => QdrantTestStore.NamedVectorsInstance; + } +} diff --git a/MEVD/test/Qdrant.ConformanceTests/TypeTests/QdrantKeyTypeTests.cs b/MEVD/test/Qdrant.ConformanceTests/TypeTests/QdrantKeyTypeTests.cs new file mode 100644 index 0000000..89b60a5 --- /dev/null +++ b/MEVD/test/Qdrant.ConformanceTests/TypeTests/QdrantKeyTypeTests.cs @@ -0,0 +1,21 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Qdrant.ConformanceTests.Support; +using VectorData.ConformanceTests.Support; +using VectorData.ConformanceTests.TypeTests; +using Xunit; + +namespace Qdrant.ConformanceTests.TypeTests; + +public class QdrantKeyTypeTests(QdrantKeyTypeTests.Fixture fixture) + : KeyTypeTests(fixture), IClassFixture +{ + [Fact] + public virtual Task ULong() => this.Test(8UL); + + public new class Fixture : KeyTypeTests.Fixture + { + public override TestStore TestStore => QdrantTestStore.NamedVectorsInstance; + } +} diff --git a/MEVD/test/Qdrant.UnitTests/.editorconfig b/MEVD/test/Qdrant.UnitTests/.editorconfig new file mode 100644 index 0000000..394eef6 --- /dev/null +++ b/MEVD/test/Qdrant.UnitTests/.editorconfig @@ -0,0 +1,6 @@ +# Suppressing errors for Test projects under dotnet folder +[*.cs] +dotnet_diagnostic.CA2007.severity = none # Do not directly await a Task +dotnet_diagnostic.VSTHRD111.severity = none # Use .ConfigureAwait(bool) is hidden by default, set to none to prevent IDE from changing on autosave +dotnet_diagnostic.CS1591.severity = none # Missing XML comment for publicly visible type or member +dotnet_diagnostic.IDE1006.severity = warning # Naming rule violations diff --git a/MEVD/test/Qdrant.UnitTests/Qdrant.UnitTests.csproj b/MEVD/test/Qdrant.UnitTests/Qdrant.UnitTests.csproj new file mode 100644 index 0000000..88d9cdc --- /dev/null +++ b/MEVD/test/Qdrant.UnitTests/Qdrant.UnitTests.csproj @@ -0,0 +1,40 @@ + + + + CommunityToolkit.VectorData.Qdrant.UnitTests + CommunityToolkit.VectorData.Qdrant.UnitTests + net10.0 + true + enable + disable + false + $(NoWarn);CA2007,CA1806,CA1869,CA1861,IDE0300,VSTHRD111,SKEXP0001,SKEXP0010,SKEXP0050 + $(NoWarn);MEVD9001 + + + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + + + + + diff --git a/MEVD/test/Qdrant.UnitTests/QdrantCollectionCreateMappingTests.cs b/MEVD/test/Qdrant.UnitTests/QdrantCollectionCreateMappingTests.cs new file mode 100644 index 0000000..325e86a --- /dev/null +++ b/MEVD/test/Qdrant.UnitTests/QdrantCollectionCreateMappingTests.cs @@ -0,0 +1,85 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; +using Qdrant.Client.Grpc; +using Xunit; + +namespace CommunityToolkit.VectorData.Qdrant.UnitTests; + +/// +/// Contains tests for the class. +/// +public class QdrantCollectionCreateMappingTests +{ + [Fact] + public void MapSingleVectorCreatesVectorParams() + { + // Arrange. + var vectorProperty = new VectorPropertyModel("testvector", typeof(ReadOnlyMemory)) { Dimensions = 4, DistanceFunction = DistanceFunction.DotProductSimilarity }; + + // Act. + var actual = QdrantCollectionCreateMapping.MapSingleVector(vectorProperty); + + // Assert. + Assert.NotNull(actual); + Assert.Equal(Distance.Dot, actual.Distance); + Assert.Equal(4ul, actual.Size); + } + + [Fact] + public void MapSingleVectorDefaultsToCosine() + { + // Arrange. + var vectorProperty = new VectorPropertyModel("testvector", typeof(ReadOnlyMemory)) { Dimensions = 4 }; + + // Act. + var actual = QdrantCollectionCreateMapping.MapSingleVector(vectorProperty); + + // Assert. + Assert.Equal(Distance.Cosine, actual.Distance); + } + + [Fact] + public void MapSingleVectorThrowsForUnsupportedDistanceFunction() + { + // Arrange. + var vectorProperty = new VectorPropertyModel("testvector", typeof(ReadOnlyMemory)) { Dimensions = 4, DistanceFunction = DistanceFunction.CosineDistance }; + + // Act and assert. + Assert.Throws(() => QdrantCollectionCreateMapping.MapSingleVector(vectorProperty)); + } + + [Fact] + public void MapNamedVectorsCreatesVectorParamsMap() + { + // Arrange. + var vectorProperties = new VectorPropertyModel[] + { + new("testvector1", typeof(ReadOnlyMemory)) + { + Dimensions = 10, + DistanceFunction = DistanceFunction.EuclideanDistance, + StorageName = "storage_testvector1" + }, + new("testvector2", typeof(ReadOnlyMemory)) + { + Dimensions = 20, + StorageName = "storage_testvector2" + } + }; + + // Act. + var actual = QdrantCollectionCreateMapping.MapNamedVectors(vectorProperties); + + // Assert. + Assert.NotNull(actual); + Assert.Equal(2, actual.Map.Count); + Assert.Equal(10ul, actual.Map["storage_testvector1"].Size); + Assert.Equal(Distance.Euclid, actual.Map["storage_testvector1"].Distance); + Assert.Equal(20ul, actual.Map["storage_testvector2"].Size); + Assert.Equal(Distance.Cosine, actual.Map["storage_testvector2"].Distance); + } +} diff --git a/MEVD/test/Qdrant.UnitTests/QdrantCollectionSearchMappingTests.cs b/MEVD/test/Qdrant.UnitTests/QdrantCollectionSearchMappingTests.cs new file mode 100644 index 0000000..7fccd2f --- /dev/null +++ b/MEVD/test/Qdrant.UnitTests/QdrantCollectionSearchMappingTests.cs @@ -0,0 +1,65 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using Microsoft.Extensions.VectorData; +using Qdrant.Client.Grpc; +using Xunit; + +namespace CommunityToolkit.VectorData.Qdrant.UnitTests; + +/// +/// Contains tests for the class. +/// +public class QdrantCollectionSearchMappingTests +{ + [Fact] + public void MapScoredPointToVectorSearchResultMapsResults() + { + var responseVector = VectorOutput.Parser.ParseJson("{ \"data\": [1, 2, 3] }"); + + // Arrange. + var scoredPoint = new ScoredPoint + { + Id = 1, + Payload = { ["storage_DataField"] = "data 1" }, + Vectors = new VectorsOutput() { Vector = responseVector }, + Score = 0.5f + }; + + var model = new QdrantModelBuilder(hasNamedVectors: false) + .Build( + typeof(DataModel), + typeof(ulong), + new() + { + Properties = + [ + new VectorStoreKeyProperty("Id", typeof(ulong)), + new VectorStoreDataProperty("DataField", typeof(string)) { StorageName = "storage_DataField" }, + new VectorStoreVectorProperty("Embedding", typeof(ReadOnlyMemory), 10), + ] + }, + defaultEmbeddingGenerator: null); + + var mapper = new QdrantMapper(model, hasNamedVectors: false); + + // Act. + var actual = QdrantCollectionSearchMapping.MapScoredPointToVectorSearchResult(scoredPoint, mapper, true, "Qdrant", "myvectorstore", "mycollection", "query"); + + // Assert. + Assert.Equal(1ul, actual.Record.Id); + Assert.Equal("data 1", actual.Record.DataField); + Assert.Equal(new float[] { 1, 2, 3 }, actual.Record.Embedding.ToArray()); + Assert.Equal(0.5f, actual.Score); + } + + public class DataModel + { + public ulong Id { get; init; } + + public string? DataField { get; init; } + + public ReadOnlyMemory Embedding { get; init; } + } +} diff --git a/MEVD/test/Qdrant.UnitTests/QdrantCollectionTests.cs b/MEVD/test/Qdrant.UnitTests/QdrantCollectionTests.cs new file mode 100644 index 0000000..4e988d0 --- /dev/null +++ b/MEVD/test/Qdrant.UnitTests/QdrantCollectionTests.cs @@ -0,0 +1,782 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.VectorData; +using Moq; +using Qdrant.Client.Grpc; +using Xunit; + +namespace CommunityToolkit.VectorData.Qdrant.UnitTests; + +/// +/// Contains tests for the class. +/// +public class QdrantCollectionTests +{ + private const string TestCollectionName = "testcollection"; + private const ulong UlongTestRecordKey1 = 1; + private const ulong UlongTestRecordKey2 = 2; + private static readonly Guid s_guidTestRecordKey1 = Guid.Parse("11111111-1111-1111-1111-111111111111"); + private static readonly Guid s_guidTestRecordKey2 = Guid.Parse("22222222-2222-2222-2222-222222222222"); + + private readonly Mock _qdrantClientMock; + + private readonly CancellationToken _testCancellationToken = new(false); + + public QdrantCollectionTests() + { + this._qdrantClientMock = new Mock(MockBehavior.Strict); + } + + [Theory] + [InlineData(TestCollectionName, true)] + [InlineData("nonexistentcollection", false)] + public async Task CollectionExistsReturnsCollectionStateAsync(string collectionName, bool expectedExists) + { + // Arrange. + using var sut = new QdrantCollection>(() => this._qdrantClientMock.Object, collectionName); + + this._qdrantClientMock + .Setup(x => x.CollectionExistsAsync( + It.IsAny(), + this._testCancellationToken)) + .ReturnsAsync(expectedExists); + + // Act. + var actual = await sut.CollectionExistsAsync(this._testCancellationToken); + + // Assert. + Assert.Equal(expectedExists, actual); + } + + [Fact] + public async Task CanCreateCollectionAsync() + { + // Arrange. + using var sut = new QdrantCollection>(() => this._qdrantClientMock.Object, TestCollectionName); + + this._qdrantClientMock + .Setup(x => x.CollectionExistsAsync( + It.IsAny(), + this._testCancellationToken)) + .ReturnsAsync(false); + + this._qdrantClientMock + .Setup(x => x.CreateCollectionAsync( + It.IsAny(), + It.IsAny(), + this._testCancellationToken)) + .Returns(Task.CompletedTask); + + this._qdrantClientMock + .Setup(x => x.CreatePayloadIndexAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + this._testCancellationToken)) + .ReturnsAsync(new UpdateResult()); + + // Act. + await sut.EnsureCollectionExistsAsync(this._testCancellationToken); + + // Assert. + this._qdrantClientMock + .Verify( + x => x.CreateCollectionAsync( + TestCollectionName, + It.Is(x => x.Size == 4), + this._testCancellationToken), + Times.Once); + + this._qdrantClientMock + .Verify( + x => x.CreatePayloadIndexAsync( + TestCollectionName, + "OriginalNameData", + PayloadSchemaType.Keyword, + this._testCancellationToken), + Times.Once); + + this._qdrantClientMock + .Verify( + x => x.CreatePayloadIndexAsync( + TestCollectionName, + "OriginalNameData", + PayloadSchemaType.Text, + this._testCancellationToken), + Times.Once); + + this._qdrantClientMock + .Verify( + x => x.CreatePayloadIndexAsync( + TestCollectionName, + "data_storage_name", + PayloadSchemaType.Keyword, + this._testCancellationToken), + Times.Once); + } + + [Fact] + public async Task CanDeleteCollectionAsync() + { + // Arrange. + using var sut = new QdrantCollection>(() => this._qdrantClientMock.Object, TestCollectionName); + + this._qdrantClientMock + .Setup(x => x.DeleteCollectionAsync( + It.IsAny(), + null, + this._testCancellationToken)) + .Returns(Task.CompletedTask); + + // Act. + await sut.EnsureCollectionDeletedAsync(this._testCancellationToken); + + // Assert. + this._qdrantClientMock + .Verify( + x => x.DeleteCollectionAsync( + TestCollectionName, + null, + this._testCancellationToken), + Times.Once); + } + + [Theory] + [MemberData(nameof(TestOptions))] + public async Task CanGetRecordWithVectorsAsync(bool useDefinition, bool hasNamedVectors, TKey testRecordKey) + where TKey : notnull + { + using var sut = this.CreateRecordCollection(useDefinition, hasNamedVectors); + + // Arrange. + var retrievedPoint = CreateRetrievedPoint(hasNamedVectors, testRecordKey); + this.SetupRetrieveMock([retrievedPoint]); + + // Act. + var actual = await sut.GetAsync( + testRecordKey, + new() { IncludeVectors = true }, + this._testCancellationToken); + + // Assert. + this._qdrantClientMock + .Verify( + x => x.RetrieveAsync( + TestCollectionName, + It.Is>(x => x.Count == 1 && (testRecordKey!.GetType() == typeof(ulong) && x[0].Num == (testRecordKey as ulong?) || testRecordKey!.GetType() == typeof(Guid) && x[0].Uuid == (testRecordKey as Guid?).ToString())), + true, + true, + null, + null, + this._testCancellationToken), + Times.Once); + + Assert.NotNull(actual); + Assert.Equal(testRecordKey, actual.Key); + Assert.Equal("data 1", actual.OriginalNameData); + Assert.Equal("data 1", actual.Data); + Assert.Equal(new float[] { 1, 2, 3, 4 }, actual.Vector!.Value.ToArray()); + } + + [Theory] + [MemberData(nameof(TestOptions))] + public async Task CanGetRecordWithoutVectorsAsync(bool useDefinition, bool hasNamedVectors, TKey testRecordKey) + where TKey : notnull + { + // Arrange. + using var sut = this.CreateRecordCollection(useDefinition, hasNamedVectors); + var retrievedPoint = CreateRetrievedPoint(hasNamedVectors, testRecordKey); + this.SetupRetrieveMock([retrievedPoint]); + + // Act. + var actual = await sut.GetAsync( + testRecordKey, + new() { IncludeVectors = false }, + this._testCancellationToken); + + // Assert. + this._qdrantClientMock + .Verify( + x => x.RetrieveAsync( + TestCollectionName, + It.Is>(x => x.Count == 1 && (testRecordKey!.GetType() == typeof(ulong) && x[0].Num == (testRecordKey as ulong?) || testRecordKey!.GetType() == typeof(Guid) && x[0].Uuid == (testRecordKey as Guid?).ToString())), + true, + false, + null, + null, + this._testCancellationToken), + Times.Once); + + Assert.NotNull(actual); + Assert.Equal(testRecordKey, actual.Key); + Assert.Equal("data 1", actual.OriginalNameData); + Assert.Equal("data 1", actual.Data); + Assert.Null(actual.Vector); + } + + [Theory] + [MemberData(nameof(MultiRecordTestOptions))] + public async Task CanGetManyRecordsWithVectorsAsync(bool useDefinition, bool hasNamedVectors, TKey[] testRecordKeys) + where TKey : notnull + { + // Arrange. + using var sut = this.CreateRecordCollection(useDefinition, hasNamedVectors); + var retrievedPoint1 = CreateRetrievedPoint(hasNamedVectors, UlongTestRecordKey1); + var retrievedPoint2 = CreateRetrievedPoint(hasNamedVectors, UlongTestRecordKey2); + this.SetupRetrieveMock(testRecordKeys.Select(x => CreateRetrievedPoint(hasNamedVectors, x)).ToList()); + + // Act. + var actual = await sut.GetAsync( + testRecordKeys, + new() { IncludeVectors = true }, + this._testCancellationToken).ToListAsync(); + + // Assert. + this._qdrantClientMock + .Verify( + x => x.RetrieveAsync( + TestCollectionName, + It.Is>(x => + x.Count == 2 && + (testRecordKeys[0]!.GetType() == typeof(ulong) && x[0].Num == (testRecordKeys[0] as ulong?) || testRecordKeys[0]!.GetType() == typeof(Guid) && x[0].Uuid == (testRecordKeys[0] as Guid?).ToString()) && + (testRecordKeys[1]!.GetType() == typeof(ulong) && x[1].Num == (testRecordKeys[1] as ulong?) || testRecordKeys[1]!.GetType() == typeof(Guid) && x[1].Uuid == (testRecordKeys[1] as Guid?).ToString())), + true, + true, + null, + null, + this._testCancellationToken), + Times.Once); + + Assert.NotNull(actual); + Assert.Equal(2, actual.Count); + Assert.Equal(testRecordKeys[0], actual[0].Key); + Assert.Equal(testRecordKeys[1], actual[1].Key); + } + + [Theory] + [InlineData(true, true)] + [InlineData(true, false)] + [InlineData(false, true)] + [InlineData(false, false)] + public async Task CanDeleteUlongRecordAsync(bool useDefinition, bool hasNamedVectors) + { + // Arrange + using var sut = this.CreateRecordCollection(useDefinition, hasNamedVectors); + this.SetupDeleteMocks(); + + // Act + await sut.DeleteAsync( + UlongTestRecordKey1, + cancellationToken: this._testCancellationToken); + + // Assert + this._qdrantClientMock + .Verify( + x => x.DeleteAsync( + TestCollectionName, + It.Is(x => x == UlongTestRecordKey1), + true, + null, + null, + this._testCancellationToken), + Times.Once); + } + + [Theory] + [InlineData(true, true)] + [InlineData(true, false)] + [InlineData(false, true)] + [InlineData(false, false)] + public async Task CanDeleteGuidRecordAsync(bool useDefinition, bool hasNamedVectors) + { + // Arrange + using var sut = this.CreateRecordCollection(useDefinition, hasNamedVectors); + this.SetupDeleteMocks(); + + // Act + await sut.DeleteAsync( + s_guidTestRecordKey1, + cancellationToken: this._testCancellationToken); + + // Assert + this._qdrantClientMock + .Verify( + x => x.DeleteAsync( + TestCollectionName, + It.Is(x => x == s_guidTestRecordKey1), + true, + null, + null, + this._testCancellationToken), + Times.Once); + } + + [Theory] + [InlineData(true, true)] + [InlineData(true, false)] + [InlineData(false, true)] + [InlineData(false, false)] + public async Task CanDeleteManyUlongRecordsAsync(bool useDefinition, bool hasNamedVectors) + { + // Arrange + using var sut = this.CreateRecordCollection(useDefinition, hasNamedVectors); + this.SetupDeleteMocks(); + + // Act + await sut.DeleteAsync( + [UlongTestRecordKey1, UlongTestRecordKey2], + cancellationToken: this._testCancellationToken); + + // Assert + this._qdrantClientMock + .Verify( + x => x.DeleteAsync( + TestCollectionName, + It.Is>(x => x.Count == 2 && x.Contains(UlongTestRecordKey1) && x.Contains(UlongTestRecordKey2)), + true, + null, + null, + this._testCancellationToken), + Times.Once); + } + + [Theory] + [InlineData(true, true)] + [InlineData(true, false)] + [InlineData(false, true)] + [InlineData(false, false)] + public async Task CanDeleteManyGuidRecordsAsync(bool useDefinition, bool hasNamedVectors) + { + // Arrange + using var sut = this.CreateRecordCollection(useDefinition, hasNamedVectors); + this.SetupDeleteMocks(); + + // Act + await sut.DeleteAsync( + [s_guidTestRecordKey1, s_guidTestRecordKey2], + cancellationToken: this._testCancellationToken); + + // Assert + this._qdrantClientMock + .Verify( + x => x.DeleteAsync( + TestCollectionName, + It.Is>(x => x.Count == 2 && x.Contains(s_guidTestRecordKey1) && x.Contains(s_guidTestRecordKey2)), + true, + null, + null, + this._testCancellationToken), + Times.Once); + } + + [Theory] + [MemberData(nameof(TestOptions))] + public async Task CanUpsertRecordAsync(bool useDefinition, bool hasNamedVectors, TKey testRecordKey) + where TKey : notnull + { + // Arrange + using var sut = this.CreateRecordCollection(useDefinition, hasNamedVectors); + this.SetupUpsertMock(); + + // Act + await sut.UpsertAsync( + CreateModel(testRecordKey, true), + cancellationToken: this._testCancellationToken); + + // Assert + this._qdrantClientMock + .Verify( + x => x.UpsertAsync( + TestCollectionName, + It.Is>(x => x.Count == 1 && (testRecordKey!.GetType() == typeof(ulong) && x[0].Id.Num == (testRecordKey as ulong?) || testRecordKey!.GetType() == typeof(Guid) && x[0].Id.Uuid == (testRecordKey as Guid?).ToString())), + true, + null, + null, + this._testCancellationToken), + Times.Once); + } + + [Theory] + [MemberData(nameof(MultiRecordTestOptions))] + public async Task CanUpsertManyRecordsAsync(bool useDefinition, bool hasNamedVectors, TKey[] testRecordKeys) + where TKey : notnull + { + // Arrange + using var sut = this.CreateRecordCollection(useDefinition, hasNamedVectors); + this.SetupUpsertMock(); + + var models = testRecordKeys.Select(x => CreateModel(x, true)); + + // Act + await sut.UpsertAsync( + models, + cancellationToken: this._testCancellationToken); + + // Assert + this._qdrantClientMock + .Verify( + x => x.UpsertAsync( + TestCollectionName, + It.Is>(x => + x.Count == 2 && + (testRecordKeys[0]!.GetType() == typeof(ulong) && x[0].Id.Num == (testRecordKeys[0] as ulong?) || testRecordKeys[0]!.GetType() == typeof(Guid) && x[0].Id.Uuid == (testRecordKeys[0] as Guid?).ToString()) && + (testRecordKeys[1]!.GetType() == typeof(ulong) && x[1].Id.Num == (testRecordKeys[1] as ulong?) || testRecordKeys[1]!.GetType() == typeof(Guid) && x[1].Id.Uuid == (testRecordKeys[1] as Guid?).ToString())), + true, + null, + null, + this._testCancellationToken), + Times.Once); + } + + /// + /// Tests that the collection can be created even if the definition and the type do not match. + /// In this case, the expectation is that a custom mapper will be provided to map between the + /// schema as defined by the definition and the different data model. + /// + [Fact] + public void CanCreateCollectionWithMismatchedDefinitionAndType() + { + // Arrange. + var definition = new VectorStoreCollectionDefinition() + { + Properties = + [ + new VectorStoreKeyProperty(nameof(SinglePropsModel.Key), typeof(ulong)), + new VectorStoreDataProperty(nameof(SinglePropsModel.OriginalNameData), typeof(string)), + new VectorStoreVectorProperty(nameof(SinglePropsModel.Vector), typeof(ReadOnlyMemory?), 4), + ] + }; + + // Act. + using var sut = new QdrantCollection>( + () => this._qdrantClientMock.Object, + TestCollectionName, + new() { Definition = definition }); + } + + [Theory] + [MemberData(nameof(TestOptions))] + public async Task CanSearchWithVectorAndFilterAsync(bool useDefinition, bool hasNamedVectors, TKey testRecordKey) + where TKey : notnull + { + using var sut = this.CreateRecordCollection(useDefinition, hasNamedVectors); + + // Arrange. + var scoredPoint = CreateScoredPoint(hasNamedVectors, testRecordKey); + this.SetupQueryMock([scoredPoint]); + + // Act. + var results = await sut.SearchAsync( + new ReadOnlyMemory(new[] { 1f, 2f, 3f, 4f }), + top: 5, + new() { IncludeVectors = true, Filter = r => r.Data == "data 1", Skip = 2 }, + this._testCancellationToken).ToListAsync(); + + // Assert. + this._qdrantClientMock + .Verify( + x => x.QueryAsync( + TestCollectionName, + It.Is(x => x!.Nearest.Dense.Data.ToArray().SequenceEqual(new[] { 1f, 2f, 3f, 4f })), + null, + hasNamedVectors ? "vector_storage_name" : null, + It.Is(x => x!.Must.Count == 1 && x.Must.First().Field.Key == "data_storage_name" && x.Must.First().Field.Match.Keyword == "data 1"), + null, + null, + 5, + 2, + null, + It.Is(x => x!.Enable == true), + null, + null, + null, + null, + this._testCancellationToken), + Times.Once); + + Assert.Single(results); + Assert.Equal(testRecordKey, results.First().Record.Key); + Assert.Equal("data 1", results.First().Record.OriginalNameData); + Assert.Equal("data 1", results.First().Record.Data); + Assert.Equal(new float[] { 1, 2, 3, 4 }, results.First().Record.Vector!.Value.ToArray()); + Assert.Equal(0.5f, results.First().Score); + } + + private void SetupRetrieveMock(List retrievedPoints) + { + this._qdrantClientMock + .Setup(x => x.RetrieveAsync( + It.IsAny(), + It.IsAny>(), + It.IsAny(), // With Payload + It.IsAny(), // With Vectors + It.IsAny(), + It.IsAny(), + this._testCancellationToken)) + .ReturnsAsync(retrievedPoints); + } + + private void SetupQueryMock(List scoredPoints) + { + this._qdrantClientMock + .Setup(x => x.QueryAsync( + It.IsAny(), + It.IsAny(), + null, + It.IsAny(), + It.IsAny(), + null, + null, + It.IsAny(), + It.IsAny(), + null, + It.IsAny(), + null, + null, + null, + null, + It.IsAny())) + .ReturnsAsync(scoredPoints); + } + + private void SetupDeleteMocks() + { + this._qdrantClientMock + .Setup(x => x.DeleteAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), // wait + It.IsAny(), + It.IsAny(), + this._testCancellationToken)) + .ReturnsAsync(new UpdateResult()); + + this._qdrantClientMock + .Setup(x => x.DeleteAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), // wait + It.IsAny(), + It.IsAny(), + this._testCancellationToken)) + .ReturnsAsync(new UpdateResult()); + + this._qdrantClientMock + .Setup(x => x.DeleteAsync( + It.IsAny(), + It.IsAny>(), + It.IsAny(), // wait + It.IsAny(), + It.IsAny(), + this._testCancellationToken)) + .ReturnsAsync(new UpdateResult()); + + this._qdrantClientMock + .Setup(x => x.DeleteAsync( + It.IsAny(), + It.IsAny>(), + It.IsAny(), // wait + It.IsAny(), + It.IsAny(), + this._testCancellationToken)) + .ReturnsAsync(new UpdateResult()); + } + + private void SetupUpsertMock() + { + this._qdrantClientMock + .Setup(x => x.UpsertAsync( + It.IsAny(), + It.IsAny>(), + It.IsAny(), // wait + It.IsAny(), + It.IsAny(), + this._testCancellationToken)) + .ReturnsAsync(new UpdateResult()); + } + + private static RetrievedPoint CreateRetrievedPoint(bool hasNamedVectors, TKey recordKey) + { + var responseVector = VectorOutput.Parser.ParseJson("{ \"data\": [1, 2, 3, 4] }"); + + RetrievedPoint point; + if (hasNamedVectors) + { + var namedVectors = new NamedVectorsOutput(); + namedVectors.Vectors.Add("vector_storage_name", responseVector); + point = new RetrievedPoint() + { + Payload = { ["OriginalNameData"] = "data 1", ["data_storage_name"] = "data 1" }, + Vectors = new VectorsOutput { Vectors = namedVectors } + }; + } + else + { + point = new RetrievedPoint() + { + Payload = { ["OriginalNameData"] = "data 1", ["data_storage_name"] = "data 1" }, + Vectors = new VectorsOutput() { Vector = responseVector } + }; + } + + if (recordKey is ulong ulongKey) + { + point.Id = ulongKey; + } + + if (recordKey is Guid guidKey) + { + point.Id = guidKey; + } + + return point; + } + + private static ScoredPoint CreateScoredPoint(bool hasNamedVectors, TKey recordKey) + { + var responseVector = VectorOutput.Parser.ParseJson("{ \"data\": [1, 2, 3, 4] }"); + + ScoredPoint point; + if (hasNamedVectors) + { + var namedVectors = new NamedVectorsOutput(); + namedVectors.Vectors.Add("vector_storage_name", responseVector); + point = new ScoredPoint() + { + Score = 0.5f, + Payload = { ["OriginalNameData"] = "data 1", ["data_storage_name"] = "data 1" }, + Vectors = new VectorsOutput { Vectors = namedVectors } + }; + } + else + { + point = new ScoredPoint() + { + Score = 0.5f, + Payload = { ["OriginalNameData"] = "data 1", ["data_storage_name"] = "data 1" }, + Vectors = new VectorsOutput() { Vector = responseVector } + }; + } + + if (recordKey is ulong ulongKey) + { + point.Id = ulongKey; + } + + if (recordKey is Guid guidKey) + { + point.Id = guidKey; + } + + return point; + } + + private VectorStoreCollection> CreateRecordCollection(bool useDefinition, bool hasNamedVectors) + where T : notnull + { + var store = new QdrantCollection>( + () => this._qdrantClientMock.Object, + TestCollectionName, + new() + { + Definition = useDefinition ? CreateSinglePropsDefinition(typeof(T)) : null, + HasNamedVectors = hasNamedVectors + }) as VectorStoreCollection>; + return store!; + } + + private static SinglePropsModel CreateModel(T key, bool withVectors) + { + return new SinglePropsModel + { + Key = key, + OriginalNameData = "data 1", + Data = "data 1", + Vector = withVectors ? new float[] { 1, 2, 3, 4 } : null, + NotAnnotated = null, + }; + } + + private static VectorStoreCollectionDefinition CreateSinglePropsDefinition(Type keyType) + { + return new() + { + Properties = + [ + new VectorStoreKeyProperty("Key", keyType), + new VectorStoreDataProperty("OriginalNameData", typeof(string)) { IsIndexed = true, IsFullTextIndexed = true }, + new VectorStoreDataProperty("Data", typeof(string)) { IsIndexed = true, StorageName = "data_storage_name" }, + new VectorStoreVectorProperty("Vector", typeof(ReadOnlyMemory), 4) { StorageName = "vector_storage_name" } + ] + }; + } + + public sealed class SinglePropsModel + { + [VectorStoreKey] + public required T Key { get; set; } + + [VectorStoreData(IsIndexed = true, IsFullTextIndexed = true)] + public string OriginalNameData { get; set; } = string.Empty; + + [JsonPropertyName("ignored_data_json_name")] + [VectorStoreData(IsIndexed = true, StorageName = "data_storage_name")] + public string Data { get; set; } = string.Empty; + + [JsonPropertyName("ignored_vector_json_name")] + [VectorStoreVector(dimensions: 4, StorageName = "vector_storage_name")] + public ReadOnlyMemory? Vector { get; set; } + + public string? NotAnnotated { get; set; } + } + + public static IEnumerable TestOptions + => GenerateAllCombinations(new object[][] { + new object[] { true, false }, + new object[] { true, false }, + new object[] { UlongTestRecordKey1, s_guidTestRecordKey1 } + }); + + public static IEnumerable MultiRecordTestOptions + => GenerateAllCombinations(new object[][] { + new object[] { true, false }, + new object[] { true, false }, + new object[] { new ulong[] { UlongTestRecordKey1, UlongTestRecordKey2 }, new Guid[] { s_guidTestRecordKey1, s_guidTestRecordKey2 } } + }); + + private static object[][] GenerateAllCombinations(object[][] input) + { + var counterArray = Enumerable.Range(0, input.Length).Select(x => 0).ToArray(); + + // Add each item from the first option set as a separate row. + object[][] currentCombinations = input[0].Select(x => new object[1] { x }).ToArray(); + + // Loop through each additional option set. + for (int currentOptionSetIndex = 1; currentOptionSetIndex < input.Length; currentOptionSetIndex++) + { + var iterationCombinations = new List(); + var currentOptionSet = input[currentOptionSetIndex]; + + // Loop through each row we have already. + foreach (var currentCombination in currentCombinations) + { + // Add each of the values from the new options set to the current row to generate a new row. + for (var currentColumnRow = 0; currentColumnRow < currentOptionSet.Length; currentColumnRow++) + { + iterationCombinations.Add(currentCombination.Append(currentOptionSet[currentColumnRow]).ToArray()); + } + } + + currentCombinations = iterationCombinations.ToArray(); + } + + return currentCombinations; + } +} diff --git a/MEVD/test/Qdrant.UnitTests/QdrantMapperTests.cs b/MEVD/test/Qdrant.UnitTests/QdrantMapperTests.cs new file mode 100644 index 0000000..234b64e --- /dev/null +++ b/MEVD/test/Qdrant.UnitTests/QdrantMapperTests.cs @@ -0,0 +1,456 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json.Serialization; +using Microsoft.Extensions.VectorData; +using CommunityToolkit.VectorData.Qdrant; +using Qdrant.Client.Grpc; +using Xunit; + +namespace SemanticKernel.Connectors.Qdrant.UnitTests; + +/// +/// Contains tests for the class. +/// +public class QdrantMapperTests +{ + [Theory] + [InlineData(true)] + [InlineData(false)] + public void MapsSinglePropsFromDataToStorageModelWithUlong(bool hasNamedVectors) + { + // Arrange. + var definition = CreateSinglePropsVectorStoreRecordDefinition(typeof(ulong)); + var model = new QdrantModelBuilder(hasNamedVectors) + .Build(typeof(SinglePropsModel), typeof(ulong), definition, defaultEmbeddingGenerator: null); + var sut = new QdrantMapper>(model, hasNamedVectors); + + // Act. + var actual = sut.MapFromDataToStorageModel(CreateSinglePropsModel(5ul), recordIndex: 0, generatedEmbeddings: null); + + // Assert. + Assert.NotNull(actual); + Assert.Equal(5ul, actual.Id.Num); + Assert.Single(actual.Payload); + Assert.Equal("data value", actual.Payload["data"].StringValue); + + if (hasNamedVectors) + { + Assert.Equal(new float[] { 1, 2, 3, 4 }, actual.Vectors.Vectors_.Vectors["vector"].Dense.Data.ToArray()); + } + else + { + Assert.Equal(new float[] { 1, 2, 3, 4 }, actual.Vectors.Vector.Dense.Data.ToArray()); + } + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void MapsSinglePropsFromDataToStorageModelWithGuid(bool hasNamedVectors) + { + // Arrange. + var definition = CreateSinglePropsVectorStoreRecordDefinition(typeof(Guid)); + var model = new QdrantModelBuilder(hasNamedVectors) + .Build(typeof(SinglePropsModel), typeof(Guid), definition, defaultEmbeddingGenerator: null); + var sut = new QdrantMapper>(model, hasNamedVectors); + + // Act. + var actual = sut.MapFromDataToStorageModel(CreateSinglePropsModel(Guid.Parse("11111111-1111-1111-1111-111111111111")), recordIndex: 0, generatedEmbeddings: null); + + // Assert. + Assert.NotNull(actual); + Assert.Equal(Guid.Parse("11111111-1111-1111-1111-111111111111"), Guid.Parse(actual.Id.Uuid)); + Assert.Single(actual.Payload); + Assert.Equal("data value", actual.Payload["data"].StringValue); + } + + [Theory] + [InlineData(true, true)] + [InlineData(true, false)] + [InlineData(false, true)] + [InlineData(false, false)] + public void MapsSinglePropsFromStorageToDataModelWithUlong(bool hasNamedVectors, bool includeVectors) + { + // Arrange. + var definition = CreateSinglePropsVectorStoreRecordDefinition(typeof(ulong)); + var model = new QdrantModelBuilder(hasNamedVectors) + .Build(typeof(SinglePropsModel), typeof(ulong), definition, defaultEmbeddingGenerator: null); + var sut = new QdrantMapper>(model, hasNamedVectors); + + // Act. + var point = CreateSinglePropsPointStruct(5, hasNamedVectors); + var actual = sut.MapFromStorageToDataModel(point.Id, point.Payload, point.Vectors, includeVectors); + + // Assert. + Assert.NotNull(actual); + Assert.Equal(5ul, actual.Key); + Assert.Equal("data value", actual.Data); + + if (includeVectors) + { + Assert.Equal(new float[] { 1, 2, 3, 4 }, actual.Vector!.Value.ToArray()); + } + else + { + Assert.Null(actual.Vector); + } + } + + [Theory] + [InlineData(true, true)] + [InlineData(true, false)] + [InlineData(false, true)] + [InlineData(false, false)] + public void MapsSinglePropsFromStorageToDataModelWithGuid(bool hasNamedVectors, bool includeVectors) + { + // Arrange. + var definition = CreateSinglePropsVectorStoreRecordDefinition(typeof(Guid)); + var model = new QdrantModelBuilder(hasNamedVectors) + .Build(typeof(SinglePropsModel), typeof(Guid), definition, defaultEmbeddingGenerator: null); + var sut = new QdrantMapper>(model, hasNamedVectors); + + // Act. + var point = CreateSinglePropsPointStruct(Guid.Parse("11111111-1111-1111-1111-111111111111"), hasNamedVectors); + var actual = sut.MapFromStorageToDataModel(point.Id, point.Payload, point.Vectors, includeVectors); + + // Assert. + Assert.NotNull(actual); + Assert.Equal(Guid.Parse("11111111-1111-1111-1111-111111111111"), actual.Key); + Assert.Equal("data value", actual.Data); + + if (includeVectors) + { + Assert.Equal(new float[] { 1, 2, 3, 4 }, actual.Vector!.Value.ToArray()); + } + else + { + Assert.Null(actual.Vector); + } + } + + [Fact] + public void MapsMultiPropsFromDataToStorageModelWithUlong() + { + // Arrange. + var definition = CreateMultiPropsVectorStoreRecordDefinition(typeof(ulong)); + var model = new QdrantModelBuilder(hasNamedVectors: true) + .Build(typeof(MultiPropsModel), typeof(ulong), definition, defaultEmbeddingGenerator: null); + + var sut = new QdrantMapper>(model, hasNamedVectors: true); + + // Act. + var actual = sut.MapFromDataToStorageModel(CreateMultiPropsModel(5ul), recordIndex: 0, generatedEmbeddings: null); + + // Assert. + Assert.NotNull(actual); + Assert.Equal(5ul, actual.Id.Num); + Assert.Equal(8, actual.Payload.Count); + Assert.Equal("data 1", actual.Payload["dataString"].StringValue); + Assert.Equal(5, actual.Payload["dataInt"].IntegerValue); + Assert.Equal(5, actual.Payload["dataLong"].IntegerValue); + Assert.Equal(5.5f, actual.Payload["dataFloat"].DoubleValue); + Assert.Equal(5.5d, actual.Payload["dataDouble"].DoubleValue); + Assert.True(actual.Payload["dataBool"].BoolValue); + Assert.Equal("2025-02-10T05:10:15.0000000+01:00", actual.Payload["dataDateTimeOffset"].StringValue); + Assert.Equal(new int[] { 1, 2, 3, 4 }, actual.Payload["dataArrayInt"].ListValue.Values.Select(x => (int)x.IntegerValue).ToArray()); + Assert.Equal(new float[] { 1, 2, 3, 4 }, actual.Vectors.Vectors_.Vectors["vector1"].Dense.Data.ToArray()); + Assert.Equal(new float[] { 5, 6, 7, 8 }, actual.Vectors.Vectors_.Vectors["vector2"].Dense.Data.ToArray()); + } + + [Fact] + public void MapsMultiPropsFromDataToStorageModelWithGuid() + { + // Arrange. + var definition = CreateMultiPropsVectorStoreRecordDefinition(typeof(Guid)); + var model = new QdrantModelBuilder(hasNamedVectors: true) + .Build(typeof(MultiPropsModel), typeof(Guid), definition, defaultEmbeddingGenerator: null); + var sut = new QdrantMapper>(model, hasNamedVectors: true); + + // Act. + var actual = sut.MapFromDataToStorageModel(CreateMultiPropsModel(Guid.Parse("11111111-1111-1111-1111-111111111111")), recordIndex: 0, generatedEmbeddings: null); + + // Assert. + Assert.NotNull(actual); + Assert.Equal(Guid.Parse("11111111-1111-1111-1111-111111111111"), Guid.Parse(actual.Id.Uuid)); + Assert.Equal(8, actual.Payload.Count); + Assert.Equal("data 1", actual.Payload["dataString"].StringValue); + Assert.Equal(5, actual.Payload["dataInt"].IntegerValue); + Assert.Equal(5, actual.Payload["dataLong"].IntegerValue); + Assert.Equal(5.5f, actual.Payload["dataFloat"].DoubleValue); + Assert.Equal(5.5d, actual.Payload["dataDouble"].DoubleValue); + Assert.True(actual.Payload["dataBool"].BoolValue); + Assert.Equal("2025-02-10T05:10:15.0000000+01:00", actual.Payload["dataDateTimeOffset"].StringValue); + Assert.Equal(new int[] { 1, 2, 3, 4 }, actual.Payload["dataArrayInt"].ListValue.Values.Select(x => (int)x.IntegerValue).ToArray()); + Assert.Equal(new float[] { 1, 2, 3, 4 }, actual.Vectors.Vectors_.Vectors["vector1"].Dense.Data.ToArray()); + Assert.Equal(new float[] { 5, 6, 7, 8 }, actual.Vectors.Vectors_.Vectors["vector2"].Dense.Data.ToArray()); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void MapsMultiPropsFromStorageToDataModelWithUlong(bool includeVectors) + { + // Arrange. + var definition = CreateMultiPropsVectorStoreRecordDefinition(typeof(ulong)); + var model = new QdrantModelBuilder(hasNamedVectors: true) + .Build(typeof(MultiPropsModel), typeof(ulong), definition, defaultEmbeddingGenerator: null); + var sut = new QdrantMapper>(model, hasNamedVectors: true); + + // Act. + var point = CreateMultiPropsPointStruct(5); + var actual = sut.MapFromStorageToDataModel(point.Id, point.Payload, point.Vectors, includeVectors); + + // Assert. + Assert.NotNull(actual); + Assert.Equal(5ul, actual.Key); + Assert.Equal("data 1", actual.DataString); + Assert.Equal(5, actual.DataInt); + Assert.Equal(5L, actual.DataLong); + Assert.Equal(5.5f, actual.DataFloat); + Assert.Equal(5.5d, actual.DataDouble); + Assert.True(actual.DataBool); + Assert.Equal(new DateTimeOffset(2025, 2, 10, 5, 10, 15, TimeSpan.FromHours(1)), actual.DataDateTimeOffset); + Assert.Equal(new int[] { 1, 2, 3, 4 }, actual.DataArrayInt); + + if (includeVectors) + { + Assert.Equal(new float[] { 1, 2, 3, 4 }, actual.Vector1!.Value.ToArray()); + Assert.Equal(new float[] { 5, 6, 7, 8 }, actual.Vector2!.Value.ToArray()); + } + else + { + Assert.Null(actual.Vector1); + Assert.Null(actual.Vector2); + } + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void MapsMultiPropsFromStorageToDataModelWithGuid(bool includeVectors) + { + // Arrange. + var definition = CreateMultiPropsVectorStoreRecordDefinition(typeof(Guid)); + var model = new QdrantModelBuilder(hasNamedVectors: true) + .Build(typeof(MultiPropsModel), typeof(Guid), definition, defaultEmbeddingGenerator: null); + var sut = new QdrantMapper>(model, hasNamedVectors: true); + + // Act. + var point = CreateMultiPropsPointStruct(Guid.Parse("11111111-1111-1111-1111-111111111111")); + var actual = sut.MapFromStorageToDataModel(point.Id, point.Payload, point.Vectors, includeVectors); + + // Assert. + Assert.NotNull(actual); + Assert.Equal(Guid.Parse("11111111-1111-1111-1111-111111111111"), actual.Key); + Assert.Equal("data 1", actual.DataString); + Assert.Equal(5, actual.DataInt); + Assert.Equal(5L, actual.DataLong); + Assert.Equal(5.5f, actual.DataFloat); + Assert.Equal(5.5d, actual.DataDouble); + Assert.True(actual.DataBool); + Assert.Equal(new DateTimeOffset(2025, 2, 10, 5, 10, 15, TimeSpan.FromHours(1)), actual.DataDateTimeOffset); + Assert.Equal(new int[] { 1, 2, 3, 4 }, actual.DataArrayInt); + + if (includeVectors) + { + Assert.Equal(new float[] { 1, 2, 3, 4 }, actual.Vector1!.Value.ToArray()); + Assert.Equal(new float[] { 5, 6, 7, 8 }, actual.Vector2!.Value.ToArray()); + } + else + { + Assert.Null(actual.Vector1); + Assert.Null(actual.Vector2); + } + } + + private static SinglePropsModel CreateSinglePropsModel(TKey key) + { + return new SinglePropsModel + { + Key = key, + Data = "data value", + Vector = new float[] { 1, 2, 3, 4 }, + NotAnnotated = "notAnnotated", + }; + } + + private static MultiPropsModel CreateMultiPropsModel(TKey key) + { + return new MultiPropsModel + { + Key = key, + DataString = "data 1", + DataInt = 5, + DataLong = 5L, + DataFloat = 5.5f, + DataDouble = 5.5d, + DataBool = true, + DataDateTimeOffset = new DateTimeOffset(2025, 2, 10, 5, 10, 15, TimeSpan.FromHours(1)), + DataArrayInt = [1, 2, 3, 4], + Vector1 = new float[] { 1, 2, 3, 4 }, + Vector2 = new float[] { 5, 6, 7, 8 }, + NotAnnotated = "notAnnotated", + }; + } + + private static RetrievedPoint CreateSinglePropsPointStruct(ulong id, bool hasNamedVectors) + { + var pointStruct = new RetrievedPoint(); + pointStruct.Id = new PointId() { Num = id }; + AddDataToSinglePropsPointStruct(pointStruct, hasNamedVectors); + return pointStruct; + } + + private static RetrievedPoint CreateSinglePropsPointStruct(Guid id, bool hasNamedVectors) + { + var pointStruct = new RetrievedPoint(); + pointStruct.Id = new PointId() { Uuid = id.ToString() }; + AddDataToSinglePropsPointStruct(pointStruct, hasNamedVectors); + return pointStruct; + } + + private static void AddDataToSinglePropsPointStruct(RetrievedPoint pointStruct, bool hasNamedVectors) + { + var responseVector = VectorOutput.Parser.ParseJson("{ \"data\": [1, 2, 3, 4] }"); + + pointStruct.Payload.Add("data", "data value"); + + if (hasNamedVectors) + { + var namedVectors = new NamedVectorsOutput(); + namedVectors.Vectors.Add("vector", responseVector); + pointStruct.Vectors = new VectorsOutput() { Vectors = namedVectors }; + } + else + { + pointStruct.Vectors = new VectorsOutput() { Vector = responseVector }; + } + } + + private static RetrievedPoint CreateMultiPropsPointStruct(ulong id) + { + var pointStruct = new RetrievedPoint(); + pointStruct.Id = new PointId() { Num = id }; + AddDataToMultiPropsPointStruct(pointStruct); + return pointStruct; + } + + private static RetrievedPoint CreateMultiPropsPointStruct(Guid id) + { + var pointStruct = new RetrievedPoint(); + pointStruct.Id = new PointId() { Uuid = id.ToString() }; + AddDataToMultiPropsPointStruct(pointStruct); + return pointStruct; + } + + private static void AddDataToMultiPropsPointStruct(RetrievedPoint pointStruct) + { + pointStruct.Payload.Add("dataString", "data 1"); + pointStruct.Payload.Add("dataInt", 5); + pointStruct.Payload.Add("dataLong", 5L); + pointStruct.Payload.Add("dataFloat", 5.5f); + pointStruct.Payload.Add("dataDouble", 5.5d); + pointStruct.Payload.Add("dataBool", true); + pointStruct.Payload.Add("dataDateTimeOffset", "2025-02-10T05:10:15.0000000+01:00"); + + var dataIntArray = new ListValue(); + dataIntArray.Values.Add(1); + dataIntArray.Values.Add(2); + dataIntArray.Values.Add(3); + dataIntArray.Values.Add(4); + pointStruct.Payload.Add("dataArrayInt", new Value { ListValue = dataIntArray }); + + var responseVector1 = VectorOutput.Parser.ParseJson("{ \"data\": [1, 2, 3, 4] }"); + var responseVector2 = VectorOutput.Parser.ParseJson("{ \"data\": [5, 6, 7, 8] }"); + + var namedVectors = new NamedVectorsOutput(); + namedVectors.Vectors.Add("vector1", responseVector1); + namedVectors.Vectors.Add("vector2", responseVector2); + pointStruct.Vectors = new VectorsOutput() { Vectors = namedVectors }; + } + + private static VectorStoreCollectionDefinition CreateSinglePropsVectorStoreRecordDefinition(Type keyType) => new() + { + Properties = + [ + new VectorStoreKeyProperty("Key", keyType) { StorageName = "key" }, + new VectorStoreDataProperty("Data", typeof(string)) { StorageName = "data" }, + new VectorStoreVectorProperty("Vector", typeof(ReadOnlyMemory), 10) { StorageName = "vector" }, + ], + }; + + private sealed class SinglePropsModel + { + [VectorStoreKey(StorageName = "key")] + public TKey? Key { get; set; } = default; + + [VectorStoreData(StorageName = "data")] + public string Data { get; set; } = string.Empty; + + [VectorStoreVector(dimensions: 10, StorageName = "vector")] + public ReadOnlyMemory? Vector { get; set; } + + public string NotAnnotated { get; set; } = string.Empty; + } + + private static VectorStoreCollectionDefinition CreateMultiPropsVectorStoreRecordDefinition(Type keyType) => new() + { + Properties = + [ + new VectorStoreKeyProperty("Key", keyType) { StorageName = "key" }, + new VectorStoreDataProperty("DataString", typeof(string)) { StorageName = "dataString" }, + new VectorStoreDataProperty("DataInt", typeof(int)) { StorageName = "dataInt" }, + new VectorStoreDataProperty("DataLong", typeof(long)) { StorageName = "dataLong" }, + new VectorStoreDataProperty("DataFloat", typeof(float)) { StorageName = "dataFloat" }, + new VectorStoreDataProperty("DataDouble", typeof(double)) { StorageName = "dataDouble" }, + new VectorStoreDataProperty("DataBool", typeof(bool)) { StorageName = "dataBool" }, + new VectorStoreDataProperty("DataDateTimeOffset", typeof(DateTimeOffset)) { StorageName = "dataDateTimeOffset" }, + new VectorStoreDataProperty("DataArrayInt", typeof(List)) { StorageName = "dataArrayInt" }, + new VectorStoreVectorProperty("Vector1", typeof(ReadOnlyMemory), 10) { StorageName = "vector1" }, + new VectorStoreVectorProperty("Vector2", typeof(ReadOnlyMemory), 10) { StorageName = "vector2" }, + ], + }; + + private sealed class MultiPropsModel + { + [VectorStoreKey(StorageName = "key")] + public TKey? Key { get; set; } = default; + + [VectorStoreData(StorageName = "dataString")] + public string DataString { get; set; } = string.Empty; + + [JsonPropertyName("data_int_json")] + [VectorStoreData(StorageName = "dataInt")] + public int DataInt { get; set; } = 0; + + [VectorStoreData(StorageName = "dataLong")] + public long DataLong { get; set; } = 0; + + [VectorStoreData(StorageName = "dataFloat")] + public float DataFloat { get; set; } = 0; + + [VectorStoreData(StorageName = "dataDouble")] + public double DataDouble { get; set; } = 0; + + [VectorStoreData(StorageName = "dataBool")] + public bool DataBool { get; set; } = false; + + [VectorStoreData(StorageName = "dataDateTimeOffset")] + public DateTimeOffset DataDateTimeOffset { get; set; } + + [VectorStoreData(StorageName = "dataArrayInt")] + public List? DataArrayInt { get; set; } + + [VectorStoreVector(dimensions: 10, StorageName = "vector1")] + public ReadOnlyMemory? Vector1 { get; set; } + + [VectorStoreVector(dimensions: 10, StorageName = "vector2")] + public ReadOnlyMemory? Vector2 { get; set; } + + public string NotAnnotated { get; set; } = string.Empty; + } +} diff --git a/MEVD/test/Qdrant.UnitTests/QdrantVectorStoreTests.cs b/MEVD/test/Qdrant.UnitTests/QdrantVectorStoreTests.cs new file mode 100644 index 0000000..6c33dca --- /dev/null +++ b/MEVD/test/Qdrant.UnitTests/QdrantVectorStoreTests.cs @@ -0,0 +1,84 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.VectorData; +using Moq; +using Xunit; + +namespace CommunityToolkit.VectorData.Qdrant.UnitTests; + +/// +/// Contains tests for the class. +/// +public class QdrantVectorStoreTests +{ + private const string TestCollectionName = "testcollection"; + + private readonly Mock _qdrantClientMock; + + private readonly CancellationToken _testCancellationToken = new(false); + + public QdrantVectorStoreTests() + { + this._qdrantClientMock = new Mock(MockBehavior.Strict); + } + + [Fact] + public void GetCollectionReturnsCollection() + { + // Arrange. + using var sut = new QdrantVectorStore(this._qdrantClientMock.Object); + + // Act. + var actual = sut.GetCollection>(TestCollectionName); + + // Assert. + Assert.NotNull(actual); + Assert.IsType>>(actual); + } + + [Fact] + public void GetCollectionThrowsForInvalidKeyType() + { + // Arrange. + using var sut = new QdrantVectorStore(this._qdrantClientMock.Object); + + // Act & Assert. + Assert.Throws(() => sut.GetCollection>(TestCollectionName)); + } + + [Fact] + public async Task ListCollectionNamesCallsSDKAsync() + { + // Arrange. + this._qdrantClientMock + .Setup(x => x.ListCollectionsAsync(It.IsAny())) + .ReturnsAsync(new[] { "collection1", "collection2" }); + using var sut = new QdrantVectorStore(this._qdrantClientMock.Object); + + // Act. + var collectionNames = sut.ListCollectionNamesAsync(this._testCancellationToken); + + // Assert. + var collectionNamesList = await collectionNames.ToListAsync(); + Assert.Equal(new[] { "collection1", "collection2" }, collectionNamesList); + } + + public sealed class SinglePropsModel + { + [VectorStoreKey] + public required TKey Key { get; set; } + + [VectorStoreData] + public string Data { get; set; } = string.Empty; + + [VectorStoreVector(dimensions: 4)] + public ReadOnlyMemory? Vector { get; set; } + + public string? NotAnnotated { get; set; } + } +} diff --git a/MEVD/test/Redis.ConformanceTests/FakeDatabase.cs b/MEVD/test/Redis.ConformanceTests/FakeDatabase.cs new file mode 100644 index 0000000..4558df5 --- /dev/null +++ b/MEVD/test/Redis.ConformanceTests/FakeDatabase.cs @@ -0,0 +1,571 @@ +// +// This code was not auto-generated, we just don't want to get any warnings for it. +// +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#pragma warning disable SER001 // VectorSet types are experimental + +using System.Net; +using StackExchange.Redis; + +#nullable enable + +namespace Redis.ConformanceTests.Support; + +internal class FakeDatabase : IDatabase +{ + public FakeDatabase(string settings) + { + } + + public int Database => 123; + public IConnectionMultiplexer Multiplexer => throw new NotImplementedException(); + public IBatch CreateBatch(object? asyncState = null) => throw new NotImplementedException(); + public ITransaction CreateTransaction(object? asyncState = null) => throw new NotImplementedException(); + public RedisValue DebugObject(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task DebugObjectAsync(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public RedisResult Execute(string command, params object[] args) => throw new NotImplementedException(); + public RedisResult Execute(string command, ICollection args, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task ExecuteAsync(string command, params object[] args) => throw new NotImplementedException(); + public Task ExecuteAsync(string command, ICollection? args, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public bool GeoAdd(RedisKey key, double longitude, double latitude, RedisValue member, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public bool GeoAdd(RedisKey key, GeoEntry value, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long GeoAdd(RedisKey key, GeoEntry[] values, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task GeoAddAsync(RedisKey key, double longitude, double latitude, RedisValue member, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task GeoAddAsync(RedisKey key, GeoEntry value, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task GeoAddAsync(RedisKey key, GeoEntry[] values, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public double? GeoDistance(RedisKey key, RedisValue member1, RedisValue member2, GeoUnit unit = GeoUnit.Meters, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task GeoDistanceAsync(RedisKey key, RedisValue member1, RedisValue member2, GeoUnit unit = GeoUnit.Meters, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public string?[] GeoHash(RedisKey key, RedisValue[] members, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public string? GeoHash(RedisKey key, RedisValue member, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task GeoHashAsync(RedisKey key, RedisValue[] members, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task GeoHashAsync(RedisKey key, RedisValue member, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public GeoPosition?[] GeoPosition(RedisKey key, RedisValue[] members, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public GeoPosition? GeoPosition(RedisKey key, RedisValue member, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task GeoPositionAsync(RedisKey key, RedisValue[] members, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task GeoPositionAsync(RedisKey key, RedisValue member, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public GeoRadiusResult[] GeoRadius(RedisKey key, RedisValue member, double radius, GeoUnit unit = GeoUnit.Meters, int count = -1, Order? order = null, GeoRadiusOptions options = GeoRadiusOptions.Default, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public GeoRadiusResult[] GeoRadius(RedisKey key, double longitude, double latitude, double radius, GeoUnit unit = GeoUnit.Meters, int count = -1, Order? order = null, GeoRadiusOptions options = GeoRadiusOptions.Default, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task GeoRadiusAsync(RedisKey key, RedisValue member, double radius, GeoUnit unit = GeoUnit.Meters, int count = -1, Order? order = null, GeoRadiusOptions options = GeoRadiusOptions.Default, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task GeoRadiusAsync(RedisKey key, double longitude, double latitude, double radius, GeoUnit unit = GeoUnit.Meters, int count = -1, Order? order = null, GeoRadiusOptions options = GeoRadiusOptions.Default, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public bool GeoRemove(RedisKey key, RedisValue member, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task GeoRemoveAsync(RedisKey key, RedisValue member, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public GeoRadiusResult[] GeoSearch(RedisKey key, RedisValue member, GeoSearchShape shape, int count = -1, bool demandClosest = true, Order? order = null, GeoRadiusOptions options = GeoRadiusOptions.Default, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public GeoRadiusResult[] GeoSearch(RedisKey key, double longitude, double latitude, GeoSearchShape shape, int count = -1, bool demandClosest = true, Order? order = null, GeoRadiusOptions options = GeoRadiusOptions.Default, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long GeoSearchAndStore(RedisKey sourceKey, RedisKey destinationKey, RedisValue member, GeoSearchShape shape, int count = -1, bool demandClosest = true, Order? order = null, bool storeDistances = false, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long GeoSearchAndStore(RedisKey sourceKey, RedisKey destinationKey, double longitude, double latitude, GeoSearchShape shape, int count = -1, bool demandClosest = true, Order? order = null, bool storeDistances = false, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task GeoSearchAndStoreAsync(RedisKey sourceKey, RedisKey destinationKey, RedisValue member, GeoSearchShape shape, int count = -1, bool demandClosest = true, Order? order = null, bool storeDistances = false, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task GeoSearchAndStoreAsync(RedisKey sourceKey, RedisKey destinationKey, double longitude, double latitude, GeoSearchShape shape, int count = -1, bool demandClosest = true, Order? order = null, bool storeDistances = false, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task GeoSearchAsync(RedisKey key, RedisValue member, GeoSearchShape shape, int count = -1, bool demandClosest = true, Order? order = null, GeoRadiusOptions options = GeoRadiusOptions.Default, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task GeoSearchAsync(RedisKey key, double longitude, double latitude, GeoSearchShape shape, int count = -1, bool demandClosest = true, Order? order = null, GeoRadiusOptions options = GeoRadiusOptions.Default, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long HashDecrement(RedisKey key, RedisValue hashField, long value = 1, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public double HashDecrement(RedisKey key, RedisValue hashField, double value, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task HashDecrementAsync(RedisKey key, RedisValue hashField, long value = 1, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task HashDecrementAsync(RedisKey key, RedisValue hashField, double value, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public bool HashDelete(RedisKey key, RedisValue hashField, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long HashDelete(RedisKey key, RedisValue[] hashFields, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task HashDeleteAsync(RedisKey key, RedisValue hashField, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task HashDeleteAsync(RedisKey key, RedisValue[] hashFields, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public bool HashExists(RedisKey key, RedisValue hashField, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task HashExistsAsync(RedisKey key, RedisValue hashField, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public ExpireResult[] HashFieldExpire(RedisKey key, RedisValue[] hashFields, TimeSpan expiry, ExpireWhen when = ExpireWhen.Always, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public ExpireResult[] HashFieldExpire(RedisKey key, RedisValue[] hashFields, DateTime expiry, ExpireWhen when = ExpireWhen.Always, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task HashFieldExpireAsync(RedisKey key, RedisValue[] hashFields, TimeSpan expiry, ExpireWhen when = ExpireWhen.Always, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task HashFieldExpireAsync(RedisKey key, RedisValue[] hashFields, DateTime expiry, ExpireWhen when = ExpireWhen.Always, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long[] HashFieldGetExpireDateTime(RedisKey key, RedisValue[] hashFields, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task HashFieldGetExpireDateTimeAsync(RedisKey key, RedisValue[] hashFields, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long[] HashFieldGetTimeToLive(RedisKey key, RedisValue[] hashFields, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task HashFieldGetTimeToLiveAsync(RedisKey key, RedisValue[] hashFields, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public PersistResult[] HashFieldPersist(RedisKey key, RedisValue[] hashFields, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task HashFieldPersistAsync(RedisKey key, RedisValue[] hashFields, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public RedisValue HashGet(RedisKey key, RedisValue hashField, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public RedisValue[] HashGet(RedisKey key, RedisValue[] hashFields, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public HashEntry[] HashGetAll(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task HashGetAllAsync(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task HashGetAsync(RedisKey key, RedisValue hashField, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task HashGetAsync(RedisKey key, RedisValue[] hashFields, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Lease? HashGetLease(RedisKey key, RedisValue hashField, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task?> HashGetLeaseAsync(RedisKey key, RedisValue hashField, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long HashIncrement(RedisKey key, RedisValue hashField, long value = 1, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public double HashIncrement(RedisKey key, RedisValue hashField, double value, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task HashIncrementAsync(RedisKey key, RedisValue hashField, long value = 1, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task HashIncrementAsync(RedisKey key, RedisValue hashField, double value, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public RedisValue[] HashKeys(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task HashKeysAsync(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long HashLength(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task HashLengthAsync(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public RedisValue HashRandomField(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task HashRandomFieldAsync(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public RedisValue[] HashRandomFields(RedisKey key, long count, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task HashRandomFieldsAsync(RedisKey key, long count, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public HashEntry[] HashRandomFieldsWithValues(RedisKey key, long count, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task HashRandomFieldsWithValuesAsync(RedisKey key, long count, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public IEnumerable HashScan(RedisKey key, RedisValue pattern, int pageSize, CommandFlags flags) => throw new NotImplementedException(); + public IEnumerable HashScan(RedisKey key, RedisValue pattern = default, int pageSize = 250, long cursor = 0, int pageOffset = 0, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public IAsyncEnumerable HashScanAsync(RedisKey key, RedisValue pattern = default, int pageSize = 250, long cursor = 0, int pageOffset = 0, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public IEnumerable HashScanNoValues(RedisKey key, RedisValue pattern = default, int pageSize = 250, long cursor = 0, int pageOffset = 0, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public IAsyncEnumerable HashScanNoValuesAsync(RedisKey key, RedisValue pattern = default, int pageSize = 250, long cursor = 0, int pageOffset = 0, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public void HashSet(RedisKey key, HashEntry[] hashFields, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public bool HashSet(RedisKey key, RedisValue hashField, RedisValue value, When when = When.Always, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task HashSetAsync(RedisKey key, HashEntry[] hashFields, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task HashSetAsync(RedisKey key, RedisValue hashField, RedisValue value, When when = When.Always, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long HashStringLength(RedisKey key, RedisValue hashField, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task HashStringLengthAsync(RedisKey key, RedisValue hashField, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public RedisValue[] HashValues(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task HashValuesAsync(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public bool HyperLogLogAdd(RedisKey key, RedisValue value, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public bool HyperLogLogAdd(RedisKey key, RedisValue[] values, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task HyperLogLogAddAsync(RedisKey key, RedisValue value, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task HyperLogLogAddAsync(RedisKey key, RedisValue[] values, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long HyperLogLogLength(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long HyperLogLogLength(RedisKey[] keys, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task HyperLogLogLengthAsync(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task HyperLogLogLengthAsync(RedisKey[] keys, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public void HyperLogLogMerge(RedisKey destination, RedisKey first, RedisKey second, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public void HyperLogLogMerge(RedisKey destination, RedisKey[] sourceKeys, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task HyperLogLogMergeAsync(RedisKey destination, RedisKey first, RedisKey second, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task HyperLogLogMergeAsync(RedisKey destination, RedisKey[] sourceKeys, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public EndPoint? IdentifyEndpoint(RedisKey key = default, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task IdentifyEndpointAsync(RedisKey key = default, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public bool IsConnected(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public bool KeyCopy(RedisKey sourceKey, RedisKey destinationKey, int destinationDatabase = -1, bool replace = false, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task KeyCopyAsync(RedisKey sourceKey, RedisKey destinationKey, int destinationDatabase = -1, bool replace = false, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public bool KeyDelete(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long KeyDelete(RedisKey[] keys, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task KeyDeleteAsync(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task KeyDeleteAsync(RedisKey[] keys, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public byte[]? KeyDump(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task KeyDumpAsync(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public string? KeyEncoding(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task KeyEncodingAsync(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); public bool KeyExists(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long KeyExists(RedisKey[] keys, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task KeyExistsAsync(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task KeyExistsAsync(RedisKey[] keys, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public bool KeyExpire(RedisKey key, TimeSpan? expiry, CommandFlags flags) => throw new NotImplementedException(); + public bool KeyExpire(RedisKey key, TimeSpan? expiry, ExpireWhen when = ExpireWhen.Always, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public bool KeyExpire(RedisKey key, DateTime? expiry, CommandFlags flags) => throw new NotImplementedException(); + public bool KeyExpire(RedisKey key, DateTime? expiry, ExpireWhen when = ExpireWhen.Always, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task KeyExpireAsync(RedisKey key, TimeSpan? expiry, CommandFlags flags) => throw new NotImplementedException(); + public Task KeyExpireAsync(RedisKey key, TimeSpan? expiry, ExpireWhen when = ExpireWhen.Always, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task KeyExpireAsync(RedisKey key, DateTime? expiry, CommandFlags flags) => throw new NotImplementedException(); + public Task KeyExpireAsync(RedisKey key, DateTime? expiry, ExpireWhen when = ExpireWhen.Always, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public DateTime? KeyExpireTime(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task KeyExpireTimeAsync(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long? KeyFrequency(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task KeyFrequencyAsync(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public TimeSpan? KeyIdleTime(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task KeyIdleTimeAsync(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public void KeyMigrate(RedisKey key, EndPoint toServer, int toDatabase = 0, int timeoutMilliseconds = 0, MigrateOptions migrateOptions = MigrateOptions.None, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task KeyMigrateAsync(RedisKey key, EndPoint toServer, int toDatabase = 0, int timeoutMilliseconds = 0, MigrateOptions migrateOptions = MigrateOptions.None, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public bool KeyMove(RedisKey key, int database, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task KeyMoveAsync(RedisKey key, int database, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public bool KeyPersist(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task KeyPersistAsync(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public RedisKey KeyRandom(CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task KeyRandomAsync(CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long? KeyRefCount(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task KeyRefCountAsync(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public bool KeyRename(RedisKey key, RedisKey newKey, When when = When.Always, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task KeyRenameAsync(RedisKey key, RedisKey newKey, When when = When.Always, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public void KeyRestore(RedisKey key, byte[] value, TimeSpan? expiry = null, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task KeyRestoreAsync(RedisKey key, byte[] value, TimeSpan? expiry = null, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public TimeSpan? KeyTimeToLive(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task KeyTimeToLiveAsync(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public bool KeyTouch(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long KeyTouch(RedisKey[] keys, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task KeyTouchAsync(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task KeyTouchAsync(RedisKey[] keys, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public RedisType KeyType(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task KeyTypeAsync(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public RedisValue ListGetByIndex(RedisKey key, long index, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task ListGetByIndexAsync(RedisKey key, long index, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long ListInsertAfter(RedisKey key, RedisValue pivot, RedisValue value, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task ListInsertAfterAsync(RedisKey key, RedisValue pivot, RedisValue value, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long ListInsertBefore(RedisKey key, RedisValue pivot, RedisValue value, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task ListInsertBeforeAsync(RedisKey key, RedisValue pivot, RedisValue value, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public RedisValue ListLeftPop(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public RedisValue[] ListLeftPop(RedisKey key, long count, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public ListPopResult ListLeftPop(RedisKey[] keys, long count, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task ListLeftPopAsync(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task ListLeftPopAsync(RedisKey key, long count, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task ListLeftPopAsync(RedisKey[] keys, long count, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long ListLeftPush(RedisKey key, RedisValue value, When when = When.Always, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long ListLeftPush(RedisKey key, RedisValue[] values, When when = When.Always, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long ListLeftPush(RedisKey key, RedisValue[] values, CommandFlags flags) => throw new NotImplementedException(); + public Task ListLeftPushAsync(RedisKey key, RedisValue value, When when = When.Always, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task ListLeftPushAsync(RedisKey key, RedisValue[] values, When when = When.Always, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task ListLeftPushAsync(RedisKey key, RedisValue[] values, CommandFlags flags) => throw new NotImplementedException(); + public long ListLength(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task ListLengthAsync(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public RedisValue ListMove(RedisKey sourceKey, RedisKey destinationKey, ListSide sourceSide, ListSide destinationSide, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task ListMoveAsync(RedisKey sourceKey, RedisKey destinationKey, ListSide sourceSide, ListSide destinationSide, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long ListPosition(RedisKey key, RedisValue element, long rank = 1, long maxLength = 0, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task ListPositionAsync(RedisKey key, RedisValue element, long rank = 1, long maxLength = 0, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long[] ListPositions(RedisKey key, RedisValue element, long count, long rank = 1, long maxLength = 0, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task ListPositionsAsync(RedisKey key, RedisValue element, long count, long rank = 1, long maxLength = 0, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public RedisValue[] ListRange(RedisKey key, long start = 0, long stop = -1, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task ListRangeAsync(RedisKey key, long start = 0, long stop = -1, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long ListRemove(RedisKey key, RedisValue value, long count = 0, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task ListRemoveAsync(RedisKey key, RedisValue value, long count = 0, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public RedisValue ListRightPop(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public RedisValue[] ListRightPop(RedisKey key, long count, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public ListPopResult ListRightPop(RedisKey[] keys, long count, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task ListRightPopAsync(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task ListRightPopAsync(RedisKey key, long count, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task ListRightPopAsync(RedisKey[] keys, long count, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public RedisValue ListRightPopLeftPush(RedisKey source, RedisKey destination, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task ListRightPopLeftPushAsync(RedisKey source, RedisKey destination, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long ListRightPush(RedisKey key, RedisValue value, When when = When.Always, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long ListRightPush(RedisKey key, RedisValue[] values, When when = When.Always, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long ListRightPush(RedisKey key, RedisValue[] values, CommandFlags flags) => throw new NotImplementedException(); + public Task ListRightPushAsync(RedisKey key, RedisValue value, When when = When.Always, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task ListRightPushAsync(RedisKey key, RedisValue[] values, When when = When.Always, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task ListRightPushAsync(RedisKey key, RedisValue[] values, CommandFlags flags) => throw new NotImplementedException(); + public void ListSetByIndex(RedisKey key, long index, RedisValue value, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task ListSetByIndexAsync(RedisKey key, long index, RedisValue value, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public void ListTrim(RedisKey key, long start, long stop, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task ListTrimAsync(RedisKey key, long start, long stop, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public bool LockExtend(RedisKey key, RedisValue value, TimeSpan expiry, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task LockExtendAsync(RedisKey key, RedisValue value, TimeSpan expiry, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public RedisValue LockQuery(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task LockQueryAsync(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public bool LockRelease(RedisKey key, RedisValue value, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task LockReleaseAsync(RedisKey key, RedisValue value, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public bool LockTake(RedisKey key, RedisValue value, TimeSpan expiry, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task LockTakeAsync(RedisKey key, RedisValue value, TimeSpan expiry, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public TimeSpan Ping(CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task PingAsync(CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long Publish(RedisChannel channel, RedisValue message, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task PublishAsync(RedisChannel channel, RedisValue message, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public RedisResult ScriptEvaluate(string script, RedisKey[]? keys = null, RedisValue[]? values = null, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public RedisResult ScriptEvaluate(byte[] hash, RedisKey[]? keys = null, RedisValue[]? values = null, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public RedisResult ScriptEvaluate(LuaScript script, object? parameters = null, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public RedisResult ScriptEvaluate(LoadedLuaScript script, object? parameters = null, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task ScriptEvaluateAsync(string script, RedisKey[]? keys = null, RedisValue[]? values = null, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task ScriptEvaluateAsync(byte[] hash, RedisKey[]? keys = null, RedisValue[]? values = null, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task ScriptEvaluateAsync(LuaScript script, object? parameters = null, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task ScriptEvaluateAsync(LoadedLuaScript script, object? parameters = null, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public RedisResult ScriptEvaluateReadOnly(string script, RedisKey[]? keys = null, RedisValue[]? values = null, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public RedisResult ScriptEvaluateReadOnly(byte[] hash, RedisKey[]? keys = null, RedisValue[]? values = null, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task ScriptEvaluateReadOnlyAsync(string script, RedisKey[]? keys = null, RedisValue[]? values = null, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task ScriptEvaluateReadOnlyAsync(byte[] hash, RedisKey[]? keys = null, RedisValue[]? values = null, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public bool SetAdd(RedisKey key, RedisValue value, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long SetAdd(RedisKey key, RedisValue[] values, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SetAddAsync(RedisKey key, RedisValue value, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SetAddAsync(RedisKey key, RedisValue[] values, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public RedisValue[] SetCombine(SetOperation operation, RedisKey first, RedisKey second, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public RedisValue[] SetCombine(SetOperation operation, RedisKey[] keys, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long SetCombineAndStore(SetOperation operation, RedisKey destination, RedisKey first, RedisKey second, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long SetCombineAndStore(SetOperation operation, RedisKey destination, RedisKey[] keys, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SetCombineAndStoreAsync(SetOperation operation, RedisKey destination, RedisKey first, RedisKey second, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SetCombineAndStoreAsync(SetOperation operation, RedisKey destination, RedisKey[] keys, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SetCombineAsync(SetOperation operation, RedisKey first, RedisKey second, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SetCombineAsync(SetOperation operation, RedisKey[] keys, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public bool SetContains(RedisKey key, RedisValue value, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public bool[] SetContains(RedisKey key, RedisValue[] values, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SetContainsAsync(RedisKey key, RedisValue value, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SetContainsAsync(RedisKey key, RedisValue[] values, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long SetIntersectionLength(RedisKey[] keys, long limit = 0, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SetIntersectionLengthAsync(RedisKey[] keys, long limit = 0, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long SetLength(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SetLengthAsync(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public RedisValue[] SetMembers(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SetMembersAsync(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public bool SetMove(RedisKey source, RedisKey destination, RedisValue value, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SetMoveAsync(RedisKey source, RedisKey destination, RedisValue value, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public RedisValue SetPop(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public RedisValue[] SetPop(RedisKey key, long count, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SetPopAsync(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SetPopAsync(RedisKey key, long count, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public RedisValue SetRandomMember(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SetRandomMemberAsync(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public RedisValue[] SetRandomMembers(RedisKey key, long count, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SetRandomMembersAsync(RedisKey key, long count, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public bool SetRemove(RedisKey key, RedisValue value, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long SetRemove(RedisKey key, RedisValue[] values, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SetRemoveAsync(RedisKey key, RedisValue value, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SetRemoveAsync(RedisKey key, RedisValue[] values, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public IEnumerable SetScan(RedisKey key, RedisValue pattern, int pageSize, CommandFlags flags) => throw new NotImplementedException(); + public IEnumerable SetScan(RedisKey key, RedisValue pattern = default, int pageSize = 250, long cursor = 0, int pageOffset = 0, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public IAsyncEnumerable SetScanAsync(RedisKey key, RedisValue pattern = default, int pageSize = 250, long cursor = 0, int pageOffset = 0, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public RedisValue[] Sort(RedisKey key, long skip = 0, long take = -1, Order order = Order.Ascending, SortType sortType = SortType.Numeric, RedisValue by = default, RedisValue[]? get = null, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long SortAndStore(RedisKey destination, RedisKey key, long skip = 0, long take = -1, Order order = Order.Ascending, SortType sortType = SortType.Numeric, RedisValue by = default, RedisValue[]? get = null, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SortAndStoreAsync(RedisKey destination, RedisKey key, long skip = 0, long take = -1, Order order = Order.Ascending, SortType sortType = SortType.Numeric, RedisValue by = default, RedisValue[]? get = null, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SortAsync(RedisKey key, long skip = 0, long take = -1, Order order = Order.Ascending, SortType sortType = SortType.Numeric, RedisValue by = default, RedisValue[]? get = null, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public bool SortedSetAdd(RedisKey key, RedisValue member, double score, CommandFlags flags) => throw new NotImplementedException(); + public bool SortedSetAdd(RedisKey key, RedisValue member, double score, When when, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public bool SortedSetAdd(RedisKey key, RedisValue member, double score, SortedSetWhen when = SortedSetWhen.Always, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long SortedSetAdd(RedisKey key, SortedSetEntry[] values, CommandFlags flags) => throw new NotImplementedException(); + public long SortedSetAdd(RedisKey key, SortedSetEntry[] values, When when, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long SortedSetAdd(RedisKey key, SortedSetEntry[] values, SortedSetWhen when = SortedSetWhen.Always, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SortedSetAddAsync(RedisKey key, RedisValue member, double score, CommandFlags flags) => throw new NotImplementedException(); + public Task SortedSetAddAsync(RedisKey key, RedisValue member, double score, When when, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SortedSetAddAsync(RedisKey key, RedisValue member, double score, SortedSetWhen when = SortedSetWhen.Always, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SortedSetAddAsync(RedisKey key, SortedSetEntry[] values, CommandFlags flags) => throw new NotImplementedException(); + public Task SortedSetAddAsync(RedisKey key, SortedSetEntry[] values, When when, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SortedSetAddAsync(RedisKey key, SortedSetEntry[] values, SortedSetWhen when = SortedSetWhen.Always, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public RedisValue[] SortedSetCombine(SetOperation operation, RedisKey[] keys, double[]? weights = null, Aggregate aggregate = Aggregate.Sum, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long SortedSetCombineAndStore(SetOperation operation, RedisKey destination, RedisKey first, RedisKey second, Aggregate aggregate = Aggregate.Sum, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long SortedSetCombineAndStore(SetOperation operation, RedisKey destination, RedisKey[] keys, double[]? weights = null, Aggregate aggregate = Aggregate.Sum, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SortedSetCombineAndStoreAsync(SetOperation operation, RedisKey destination, RedisKey first, RedisKey second, Aggregate aggregate = Aggregate.Sum, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SortedSetCombineAndStoreAsync(SetOperation operation, RedisKey destination, RedisKey[] keys, double[]? weights = null, Aggregate aggregate = Aggregate.Sum, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SortedSetCombineAsync(SetOperation operation, RedisKey[] keys, double[]? weights = null, Aggregate aggregate = Aggregate.Sum, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public SortedSetEntry[] SortedSetCombineWithScores(SetOperation operation, RedisKey[] keys, double[]? weights = null, Aggregate aggregate = Aggregate.Sum, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SortedSetCombineWithScoresAsync(SetOperation operation, RedisKey[] keys, double[]? weights = null, Aggregate aggregate = Aggregate.Sum, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public double SortedSetDecrement(RedisKey key, RedisValue member, double value, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SortedSetDecrementAsync(RedisKey key, RedisValue member, double value, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public double SortedSetIncrement(RedisKey key, RedisValue member, double value, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SortedSetIncrementAsync(RedisKey key, RedisValue member, double value, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long SortedSetIntersectionLength(RedisKey[] keys, long limit = 0, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SortedSetIntersectionLengthAsync(RedisKey[] keys, long limit = 0, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long SortedSetLength(RedisKey key, double min = double.NegativeInfinity, double max = double.PositiveInfinity, Exclude exclude = Exclude.None, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SortedSetLengthAsync(RedisKey key, double min = double.NegativeInfinity, double max = double.PositiveInfinity, Exclude exclude = Exclude.None, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long SortedSetLengthByValue(RedisKey key, RedisValue min, RedisValue max, Exclude exclude = Exclude.None, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SortedSetLengthByValueAsync(RedisKey key, RedisValue min, RedisValue max, Exclude exclude = Exclude.None, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public SortedSetEntry? SortedSetPop(RedisKey key, Order order = Order.Ascending, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public SortedSetEntry[] SortedSetPop(RedisKey key, long count, Order order = Order.Ascending, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public SortedSetPopResult SortedSetPop(RedisKey[] keys, long count, Order order = Order.Ascending, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SortedSetPopAsync(RedisKey key, Order order = Order.Ascending, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SortedSetPopAsync(RedisKey key, long count, Order order = Order.Ascending, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SortedSetPopAsync(RedisKey[] keys, long count, Order order = Order.Ascending, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public RedisValue SortedSetRandomMember(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SortedSetRandomMemberAsync(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public RedisValue[] SortedSetRandomMembers(RedisKey key, long count, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SortedSetRandomMembersAsync(RedisKey key, long count, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public SortedSetEntry[] SortedSetRandomMembersWithScores(RedisKey key, long count, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SortedSetRandomMembersWithScoresAsync(RedisKey key, long count, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long SortedSetRangeAndStore(RedisKey sourceKey, RedisKey destinationKey, RedisValue start, RedisValue stop, SortedSetOrder sortedSetOrder = SortedSetOrder.ByRank, Exclude exclude = Exclude.None, Order order = Order.Ascending, long skip = 0, long? take = null, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SortedSetRangeAndStoreAsync(RedisKey sourceKey, RedisKey destinationKey, RedisValue start, RedisValue stop, SortedSetOrder sortedSetOrder = SortedSetOrder.ByRank, Exclude exclude = Exclude.None, Order order = Order.Ascending, long skip = 0, long? take = null, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public RedisValue[] SortedSetRangeByRank(RedisKey key, long start = 0, long stop = -1, Order order = Order.Ascending, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SortedSetRangeByRankAsync(RedisKey key, long start = 0, long stop = -1, Order order = Order.Ascending, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public SortedSetEntry[] SortedSetRangeByRankWithScores(RedisKey key, long start = 0, long stop = -1, Order order = Order.Ascending, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SortedSetRangeByRankWithScoresAsync(RedisKey key, long start = 0, long stop = -1, Order order = Order.Ascending, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public RedisValue[] SortedSetRangeByScore(RedisKey key, double start = double.NegativeInfinity, double stop = double.PositiveInfinity, Exclude exclude = Exclude.None, Order order = Order.Ascending, long skip = 0, long take = -1, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SortedSetRangeByScoreAsync(RedisKey key, double start = double.NegativeInfinity, double stop = double.PositiveInfinity, Exclude exclude = Exclude.None, Order order = Order.Ascending, long skip = 0, long take = -1, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public SortedSetEntry[] SortedSetRangeByScoreWithScores(RedisKey key, double start = double.NegativeInfinity, double stop = double.PositiveInfinity, Exclude exclude = Exclude.None, Order order = Order.Ascending, long skip = 0, long take = -1, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SortedSetRangeByScoreWithScoresAsync(RedisKey key, double start = double.NegativeInfinity, double stop = double.PositiveInfinity, Exclude exclude = Exclude.None, Order order = Order.Ascending, long skip = 0, long take = -1, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public RedisValue[] SortedSetRangeByValue(RedisKey key, RedisValue min, RedisValue max, Exclude exclude, long skip, long take = -1, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public RedisValue[] SortedSetRangeByValue(RedisKey key, RedisValue min = default, RedisValue max = default, Exclude exclude = Exclude.None, Order order = Order.Ascending, long skip = 0, long take = -1, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SortedSetRangeByValueAsync(RedisKey key, RedisValue min, RedisValue max, Exclude exclude, long skip, long take = -1, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SortedSetRangeByValueAsync(RedisKey key, RedisValue min = default, RedisValue max = default, Exclude exclude = Exclude.None, Order order = Order.Ascending, long skip = 0, long take = -1, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long? SortedSetRank(RedisKey key, RedisValue member, Order order = Order.Ascending, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SortedSetRankAsync(RedisKey key, RedisValue member, Order order = Order.Ascending, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public bool SortedSetRemove(RedisKey key, RedisValue member, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long SortedSetRemove(RedisKey key, RedisValue[] members, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SortedSetRemoveAsync(RedisKey key, RedisValue member, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SortedSetRemoveAsync(RedisKey key, RedisValue[] members, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long SortedSetRemoveRangeByRank(RedisKey key, long start, long stop, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SortedSetRemoveRangeByRankAsync(RedisKey key, long start, long stop, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long SortedSetRemoveRangeByScore(RedisKey key, double start, double stop, Exclude exclude = Exclude.None, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SortedSetRemoveRangeByScoreAsync(RedisKey key, double start, double stop, Exclude exclude = Exclude.None, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long SortedSetRemoveRangeByValue(RedisKey key, RedisValue min, RedisValue max, Exclude exclude = Exclude.None, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SortedSetRemoveRangeByValueAsync(RedisKey key, RedisValue min, RedisValue max, Exclude exclude = Exclude.None, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public IEnumerable SortedSetScan(RedisKey key, RedisValue pattern, int pageSize, CommandFlags flags) => throw new NotImplementedException(); + public IEnumerable SortedSetScan(RedisKey key, RedisValue pattern = default, int pageSize = 250, long cursor = 0, int pageOffset = 0, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public IAsyncEnumerable SortedSetScanAsync(RedisKey key, RedisValue pattern = default, int pageSize = 250, long cursor = 0, int pageOffset = 0, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public double? SortedSetScore(RedisKey key, RedisValue member, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SortedSetScoreAsync(RedisKey key, RedisValue member, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public double?[] SortedSetScores(RedisKey key, RedisValue[] members, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SortedSetScoresAsync(RedisKey key, RedisValue[] members, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public bool SortedSetUpdate(RedisKey key, RedisValue member, double score, SortedSetWhen when = SortedSetWhen.Always, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long SortedSetUpdate(RedisKey key, SortedSetEntry[] values, SortedSetWhen when = SortedSetWhen.Always, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SortedSetUpdateAsync(RedisKey key, RedisValue member, double score, SortedSetWhen when = SortedSetWhen.Always, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task SortedSetUpdateAsync(RedisKey key, SortedSetEntry[] values, SortedSetWhen when = SortedSetWhen.Always, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long StreamAcknowledge(RedisKey key, RedisValue groupName, RedisValue messageId, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long StreamAcknowledge(RedisKey key, RedisValue groupName, RedisValue[] messageIds, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task StreamAcknowledgeAsync(RedisKey key, RedisValue groupName, RedisValue messageId, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task StreamAcknowledgeAsync(RedisKey key, RedisValue groupName, RedisValue[] messageIds, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public RedisValue StreamAdd(RedisKey key, RedisValue streamField, RedisValue streamValue, RedisValue? messageId = null, int? maxLength = null, bool useApproximateMaxLength = false, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public RedisValue StreamAdd(RedisKey key, NameValueEntry[] streamPairs, RedisValue? messageId = null, int? maxLength = null, bool useApproximateMaxLength = false, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task StreamAddAsync(RedisKey key, RedisValue streamField, RedisValue streamValue, RedisValue? messageId = null, int? maxLength = null, bool useApproximateMaxLength = false, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task StreamAddAsync(RedisKey key, NameValueEntry[] streamPairs, RedisValue? messageId = null, int? maxLength = null, bool useApproximateMaxLength = false, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public StreamAutoClaimResult StreamAutoClaim(RedisKey key, RedisValue consumerGroup, RedisValue claimingConsumer, long minIdleTimeInMs, RedisValue startAtId, int? count = null, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task StreamAutoClaimAsync(RedisKey key, RedisValue consumerGroup, RedisValue claimingConsumer, long minIdleTimeInMs, RedisValue startAtId, int? count = null, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public StreamAutoClaimIdsOnlyResult StreamAutoClaimIdsOnly(RedisKey key, RedisValue consumerGroup, RedisValue claimingConsumer, long minIdleTimeInMs, RedisValue startAtId, int? count = null, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task StreamAutoClaimIdsOnlyAsync(RedisKey key, RedisValue consumerGroup, RedisValue claimingConsumer, long minIdleTimeInMs, RedisValue startAtId, int? count = null, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public StreamEntry[] StreamClaim(RedisKey key, RedisValue consumerGroup, RedisValue claimingConsumer, long minIdleTimeInMs, RedisValue[] messageIds, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task StreamClaimAsync(RedisKey key, RedisValue consumerGroup, RedisValue claimingConsumer, long minIdleTimeInMs, RedisValue[] messageIds, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public RedisValue[] StreamClaimIdsOnly(RedisKey key, RedisValue consumerGroup, RedisValue claimingConsumer, long minIdleTimeInMs, RedisValue[] messageIds, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task StreamClaimIdsOnlyAsync(RedisKey key, RedisValue consumerGroup, RedisValue claimingConsumer, long minIdleTimeInMs, RedisValue[] messageIds, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public bool StreamConsumerGroupSetPosition(RedisKey key, RedisValue groupName, RedisValue position, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task StreamConsumerGroupSetPositionAsync(RedisKey key, RedisValue groupName, RedisValue position, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public StreamConsumerInfo[] StreamConsumerInfo(RedisKey key, RedisValue groupName, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task StreamConsumerInfoAsync(RedisKey key, RedisValue groupName, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public bool StreamCreateConsumerGroup(RedisKey key, RedisValue groupName, RedisValue? position, CommandFlags flags) => throw new NotImplementedException(); + public bool StreamCreateConsumerGroup(RedisKey key, RedisValue groupName, RedisValue? position = null, bool createStream = true, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task StreamCreateConsumerGroupAsync(RedisKey key, RedisValue groupName, RedisValue? position, CommandFlags flags) => throw new NotImplementedException(); + public Task StreamCreateConsumerGroupAsync(RedisKey key, RedisValue groupName, RedisValue? position = null, bool createStream = true, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long StreamDelete(RedisKey key, RedisValue[] messageIds, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task StreamDeleteAsync(RedisKey key, RedisValue[] messageIds, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long StreamDeleteConsumer(RedisKey key, RedisValue groupName, RedisValue consumerName, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task StreamDeleteConsumerAsync(RedisKey key, RedisValue groupName, RedisValue consumerName, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public bool StreamDeleteConsumerGroup(RedisKey key, RedisValue groupName, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task StreamDeleteConsumerGroupAsync(RedisKey key, RedisValue groupName, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public StreamGroupInfo[] StreamGroupInfo(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task StreamGroupInfoAsync(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public StreamInfo StreamInfo(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task StreamInfoAsync(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long StreamLength(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task StreamLengthAsync(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public StreamPendingInfo StreamPending(RedisKey key, RedisValue groupName, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task StreamPendingAsync(RedisKey key, RedisValue groupName, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public StreamPendingMessageInfo[] StreamPendingMessages(RedisKey key, RedisValue groupName, int count, RedisValue consumerName, RedisValue? minId = null, RedisValue? maxId = null, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task StreamPendingMessagesAsync(RedisKey key, RedisValue groupName, int count, RedisValue consumerName, RedisValue? minId = null, RedisValue? maxId = null, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public StreamEntry[] StreamRange(RedisKey key, RedisValue? minId = null, RedisValue? maxId = null, int? count = null, Order messageOrder = Order.Ascending, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task StreamRangeAsync(RedisKey key, RedisValue? minId = null, RedisValue? maxId = null, int? count = null, Order messageOrder = Order.Ascending, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public StreamEntry[] StreamRead(RedisKey key, RedisValue position, int? count = null, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public RedisStream[] StreamRead(StreamPosition[] streamPositions, int? countPerStream = null, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task StreamReadAsync(RedisKey key, RedisValue position, int? count = null, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task StreamReadAsync(StreamPosition[] streamPositions, int? countPerStream = null, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public StreamEntry[] StreamReadGroup(RedisKey key, RedisValue groupName, RedisValue consumerName, RedisValue? position, int? count, CommandFlags flags) => throw new NotImplementedException(); + public StreamEntry[] StreamReadGroup(RedisKey key, RedisValue groupName, RedisValue consumerName, RedisValue? position = null, int? count = null, bool noAck = false, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public RedisStream[] StreamReadGroup(StreamPosition[] streamPositions, RedisValue groupName, RedisValue consumerName, int? countPerStream, CommandFlags flags) => throw new NotImplementedException(); + public RedisStream[] StreamReadGroup(StreamPosition[] streamPositions, RedisValue groupName, RedisValue consumerName, int? countPerStream = null, bool noAck = false, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task StreamReadGroupAsync(RedisKey key, RedisValue groupName, RedisValue consumerName, RedisValue? position, int? count, CommandFlags flags) => throw new NotImplementedException(); + public Task StreamReadGroupAsync(RedisKey key, RedisValue groupName, RedisValue consumerName, RedisValue? position = null, int? count = null, bool noAck = false, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task StreamReadGroupAsync(StreamPosition[] streamPositions, RedisValue groupName, RedisValue consumerName, int? countPerStream, CommandFlags flags) => throw new NotImplementedException(); + public Task StreamReadGroupAsync(StreamPosition[] streamPositions, RedisValue groupName, RedisValue consumerName, int? countPerStream = null, bool noAck = false, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long StreamTrim(RedisKey key, int maxLength, bool useApproximateMaxLength = false, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task StreamTrimAsync(RedisKey key, int maxLength, bool useApproximateMaxLength = false, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long StringAppend(RedisKey key, RedisValue value, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task StringAppendAsync(RedisKey key, RedisValue value, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long StringBitCount(RedisKey key, long start, long end, CommandFlags flags) => throw new NotImplementedException(); + public long StringBitCount(RedisKey key, long start = 0, long end = -1, StringIndexType indexType = StringIndexType.Byte, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task StringBitCountAsync(RedisKey key, long start, long end, CommandFlags flags) => throw new NotImplementedException(); + public Task StringBitCountAsync(RedisKey key, long start = 0, long end = -1, StringIndexType indexType = StringIndexType.Byte, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long StringBitOperation(Bitwise operation, RedisKey destination, RedisKey first, RedisKey second = default, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long StringBitOperation(Bitwise operation, RedisKey destination, RedisKey[] keys, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task StringBitOperationAsync(Bitwise operation, RedisKey destination, RedisKey first, RedisKey second = default, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task StringBitOperationAsync(Bitwise operation, RedisKey destination, RedisKey[] keys, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long StringBitPosition(RedisKey key, bool bit, long start, long end, CommandFlags flags) => throw new NotImplementedException(); + public long StringBitPosition(RedisKey key, bool bit, long start = 0, long end = -1, StringIndexType indexType = StringIndexType.Byte, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task StringBitPositionAsync(RedisKey key, bool bit, long start, long end, CommandFlags flags) => throw new NotImplementedException(); + public Task StringBitPositionAsync(RedisKey key, bool bit, long start = 0, long end = -1, StringIndexType indexType = StringIndexType.Byte, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long StringDecrement(RedisKey key, long value = 1, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public double StringDecrement(RedisKey key, double value, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task StringDecrementAsync(RedisKey key, long value = 1, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task StringDecrementAsync(RedisKey key, double value, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public RedisValue StringGet(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public RedisValue[] StringGet(RedisKey[] keys, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task StringGetAsync(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task StringGetAsync(RedisKey[] keys, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public bool StringGetBit(RedisKey key, long offset, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task StringGetBitAsync(RedisKey key, long offset, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public RedisValue StringGetDelete(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task StringGetDeleteAsync(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Lease? StringGetLease(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task?> StringGetLeaseAsync(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public RedisValue StringGetRange(RedisKey key, long start, long end, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task StringGetRangeAsync(RedisKey key, long start, long end, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public RedisValue StringGetSet(RedisKey key, RedisValue value, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task StringGetSetAsync(RedisKey key, RedisValue value, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public RedisValue StringGetSetExpiry(RedisKey key, TimeSpan? expiry, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public RedisValue StringGetSetExpiry(RedisKey key, DateTime expiry, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task StringGetSetExpiryAsync(RedisKey key, TimeSpan? expiry, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task StringGetSetExpiryAsync(RedisKey key, DateTime expiry, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public RedisValueWithExpiry StringGetWithExpiry(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task StringGetWithExpiryAsync(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long StringIncrement(RedisKey key, long value = 1, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public double StringIncrement(RedisKey key, double value, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task StringIncrementAsync(RedisKey key, long value = 1, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task StringIncrementAsync(RedisKey key, double value, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long StringLength(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task StringLengthAsync(RedisKey key, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public string? StringLongestCommonSubsequence(RedisKey first, RedisKey second, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task StringLongestCommonSubsequenceAsync(RedisKey first, RedisKey second, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public long StringLongestCommonSubsequenceLength(RedisKey first, RedisKey second, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task StringLongestCommonSubsequenceLengthAsync(RedisKey first, RedisKey second, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public LCSMatchResult StringLongestCommonSubsequenceWithMatches(RedisKey first, RedisKey second, long minLength = 0, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task StringLongestCommonSubsequenceWithMatchesAsync(RedisKey first, RedisKey second, long minLength = 0, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public bool StringSet(RedisKey key, RedisValue value, TimeSpan? expiry, When when) => throw new NotImplementedException(); + public bool StringSet(RedisKey key, RedisValue value, TimeSpan? expiry, When when, CommandFlags flags) => throw new NotImplementedException(); + public bool StringSet(RedisKey key, RedisValue value, TimeSpan? expiry = null, bool keepTtl = false, When when = When.Always, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public bool StringSet(KeyValuePair[] values, When when = When.Always, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public RedisValue StringSetAndGet(RedisKey key, RedisValue value, TimeSpan? expiry, When when, CommandFlags flags) => throw new NotImplementedException(); + public RedisValue StringSetAndGet(RedisKey key, RedisValue value, TimeSpan? expiry = null, bool keepTtl = false, When when = When.Always, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task StringSetAndGetAsync(RedisKey key, RedisValue value, TimeSpan? expiry, When when, CommandFlags flags) => throw new NotImplementedException(); + public Task StringSetAndGetAsync(RedisKey key, RedisValue value, TimeSpan? expiry = null, bool keepTtl = false, When when = When.Always, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task StringSetAsync(RedisKey key, RedisValue value, TimeSpan? expiry, When when) => throw new NotImplementedException(); + public Task StringSetAsync(RedisKey key, RedisValue value, TimeSpan? expiry, When when, CommandFlags flags) => throw new NotImplementedException(); + public Task StringSetAsync(RedisKey key, RedisValue value, TimeSpan? expiry = null, bool keepTtl = false, When when = When.Always, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task StringSetAsync(KeyValuePair[] values, When when = When.Always, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public bool StringSetBit(RedisKey key, long offset, bool bit, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task StringSetBitAsync(RedisKey key, long offset, bool bit, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public RedisValue StringSetRange(RedisKey key, long offset, RedisValue value, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public Task StringSetRangeAsync(RedisKey key, long offset, RedisValue value, CommandFlags flags = CommandFlags.None) => throw new NotImplementedException(); + public bool TryWait(Task task) => throw new NotImplementedException(); + public void Wait(Task task) => throw new NotImplementedException(); + public T Wait(Task task) => throw new NotImplementedException(); + public void WaitAll(params Task[] tasks) => throw new NotImplementedException(); + public RedisValue HashFieldGetAndDelete(RedisKey arg0, RedisValue arg1, CommandFlags arg2) => throw new NotImplementedException(); + public RedisValue[] HashFieldGetAndDelete(RedisKey arg0, RedisValue[] arg1, CommandFlags arg2) => throw new NotImplementedException(); + public RedisValue HashFieldGetAndSetExpiry(RedisKey arg0, RedisValue arg1, DateTime arg2, CommandFlags arg3) => throw new NotImplementedException(); + public RedisValue HashFieldGetAndSetExpiry(RedisKey arg0, RedisValue arg1, TimeSpan? arg2, bool arg3, CommandFlags arg4) => throw new NotImplementedException(); + public RedisValue[] HashFieldGetAndSetExpiry(RedisKey arg0, RedisValue[] arg1, DateTime arg2, CommandFlags arg3) => throw new NotImplementedException(); + public RedisValue[] HashFieldGetAndSetExpiry(RedisKey arg0, RedisValue[] arg1, TimeSpan? arg2, bool arg3, CommandFlags arg4) => throw new NotImplementedException(); + public Lease? HashFieldGetLeaseAndDelete(RedisKey arg0, RedisValue arg1, CommandFlags arg2) => throw new NotImplementedException(); + public Lease? HashFieldGetLeaseAndSetExpiry(RedisKey arg0, RedisValue arg1, DateTime arg2, CommandFlags arg3) => throw new NotImplementedException(); + public Lease? HashFieldGetLeaseAndSetExpiry(RedisKey arg0, RedisValue arg1, TimeSpan? arg2, bool arg3, CommandFlags arg4) => throw new NotImplementedException(); + public RedisValue HashFieldSetAndSetExpiry(RedisKey arg0, HashEntry[] arg1, DateTime arg2, When arg3, CommandFlags arg4) => throw new NotImplementedException(); + public RedisValue HashFieldSetAndSetExpiry(RedisKey arg0, HashEntry[] arg1, TimeSpan? arg2, bool arg3, When arg4, CommandFlags arg5) => throw new NotImplementedException(); + public RedisValue HashFieldSetAndSetExpiry(RedisKey arg0, RedisValue arg1, RedisValue arg2, DateTime arg3, When arg4, CommandFlags arg5) => throw new NotImplementedException(); + public RedisValue HashFieldSetAndSetExpiry(RedisKey arg0, RedisValue arg1, RedisValue arg2, TimeSpan? arg3, bool arg4, When arg5, CommandFlags arg6) => throw new NotImplementedException(); + public StreamTrimResult StreamAcknowledgeAndDelete(RedisKey arg0, RedisValue arg1, StreamTrimMode arg2, RedisValue arg3, CommandFlags arg4) => throw new NotImplementedException(); + public StreamTrimResult[] StreamAcknowledgeAndDelete(RedisKey arg0, RedisValue arg1, StreamTrimMode arg2, RedisValue[] arg3, CommandFlags arg4) => throw new NotImplementedException(); + public RedisValue StreamAdd(RedisKey arg0, NameValueEntry[] arg1, RedisValue? arg2, long? arg3, bool arg4, long? arg5, StreamTrimMode arg6, CommandFlags arg7) => throw new NotImplementedException(); + public RedisValue StreamAdd(RedisKey arg0, RedisValue arg1, RedisValue arg2, RedisValue? arg3, long? arg4, bool arg5, long? arg6, StreamTrimMode arg7, CommandFlags arg8) => throw new NotImplementedException(); + public StreamTrimResult[] StreamDelete(RedisKey arg0, RedisValue[] arg1, StreamTrimMode arg2, CommandFlags arg3) => throw new NotImplementedException(); + public StreamPendingMessageInfo[] StreamPendingMessages(RedisKey arg0, RedisValue arg1, int arg2, RedisValue arg3, RedisValue? arg4, RedisValue? arg5, long? arg6, CommandFlags arg7) => throw new NotImplementedException(); + public long StreamTrim(RedisKey arg0, long arg1, bool arg2, long? arg3, StreamTrimMode arg4, CommandFlags arg5) => throw new NotImplementedException(); + public long StreamTrimByMinId(RedisKey arg0, RedisValue arg1, bool arg2, long? arg3, StreamTrimMode arg4, CommandFlags arg5) => throw new NotImplementedException(); + public bool VectorSetAdd(RedisKey arg0, VectorSetAddRequest arg1, CommandFlags arg2) => throw new NotImplementedException(); + public bool VectorSetContains(RedisKey arg0, RedisValue arg1, CommandFlags arg2) => throw new NotImplementedException(); + public int VectorSetDimension(RedisKey arg0, CommandFlags arg1) => throw new NotImplementedException(); + public Lease VectorSetGetApproximateVector(RedisKey arg0, RedisValue arg1, CommandFlags arg2) => throw new NotImplementedException(); + public string VectorSetGetAttributesJson(RedisKey arg0, RedisValue arg1, CommandFlags arg2) => throw new NotImplementedException(); + public Lease VectorSetGetLinks(RedisKey arg0, RedisValue arg1, CommandFlags arg2) => throw new NotImplementedException(); + public Lease VectorSetGetLinksWithScores(RedisKey arg0, RedisValue arg1, CommandFlags arg2) => throw new NotImplementedException(); + public VectorSetInfo? VectorSetInfo(RedisKey arg0, CommandFlags arg1) => throw new NotImplementedException(); + public long VectorSetLength(RedisKey arg0, CommandFlags arg1) => throw new NotImplementedException(); + public RedisValue VectorSetRandomMember(RedisKey arg0, CommandFlags arg1) => throw new NotImplementedException(); + public RedisValue[] VectorSetRandomMembers(RedisKey arg0, long arg1, CommandFlags arg2) => throw new NotImplementedException(); + public bool VectorSetRemove(RedisKey arg0, RedisValue arg1, CommandFlags arg2) => throw new NotImplementedException(); + public bool VectorSetSetAttributesJson(RedisKey arg0, RedisValue arg1, string arg2, CommandFlags arg3) => throw new NotImplementedException(); + public Lease VectorSetSimilaritySearch(RedisKey arg0, VectorSetSimilaritySearchRequest arg1, CommandFlags arg2) => throw new NotImplementedException(); + public Task HashFieldGetAndDeleteAsync(RedisKey arg0, RedisValue arg1, CommandFlags arg2) => throw new NotImplementedException(); + public Task HashFieldGetAndDeleteAsync(RedisKey arg0, RedisValue[] arg1, CommandFlags arg2) => throw new NotImplementedException(); + public Task HashFieldGetAndSetExpiryAsync(RedisKey arg0, RedisValue arg1, DateTime arg2, CommandFlags arg3) => throw new NotImplementedException(); + public Task HashFieldGetAndSetExpiryAsync(RedisKey arg0, RedisValue arg1, TimeSpan? arg2, bool arg3, CommandFlags arg4) => throw new NotImplementedException(); + public Task HashFieldGetAndSetExpiryAsync(RedisKey arg0, RedisValue[] arg1, DateTime arg2, CommandFlags arg3) => throw new NotImplementedException(); + public Task HashFieldGetAndSetExpiryAsync(RedisKey arg0, RedisValue[] arg1, TimeSpan? arg2, bool arg3, CommandFlags arg4) => throw new NotImplementedException(); + public Task?> HashFieldGetLeaseAndDeleteAsync(RedisKey arg0, RedisValue arg1, CommandFlags arg2) => throw new NotImplementedException(); + public Task?> HashFieldGetLeaseAndSetExpiryAsync(RedisKey arg0, RedisValue arg1, DateTime arg2, CommandFlags arg3) => throw new NotImplementedException(); + public Task?> HashFieldGetLeaseAndSetExpiryAsync(RedisKey arg0, RedisValue arg1, TimeSpan? arg2, bool arg3, CommandFlags arg4) => throw new NotImplementedException(); + public Task HashFieldSetAndSetExpiryAsync(RedisKey arg0, HashEntry[] arg1, DateTime arg2, When arg3, CommandFlags arg4) => throw new NotImplementedException(); + public Task HashFieldSetAndSetExpiryAsync(RedisKey arg0, HashEntry[] arg1, TimeSpan? arg2, bool arg3, When arg4, CommandFlags arg5) => throw new NotImplementedException(); + public Task HashFieldSetAndSetExpiryAsync(RedisKey arg0, RedisValue arg1, RedisValue arg2, DateTime arg3, When arg4, CommandFlags arg5) => throw new NotImplementedException(); + public Task HashFieldSetAndSetExpiryAsync(RedisKey arg0, RedisValue arg1, RedisValue arg2, TimeSpan? arg3, bool arg4, When arg5, CommandFlags arg6) => throw new NotImplementedException(); + public Task StreamAcknowledgeAndDeleteAsync(RedisKey arg0, RedisValue arg1, StreamTrimMode arg2, RedisValue arg3, CommandFlags arg4) => throw new NotImplementedException(); + public Task StreamAcknowledgeAndDeleteAsync(RedisKey arg0, RedisValue arg1, StreamTrimMode arg2, RedisValue[] arg3, CommandFlags arg4) => throw new NotImplementedException(); + public Task StreamAddAsync(RedisKey arg0, NameValueEntry[] arg1, RedisValue? arg2, long? arg3, bool arg4, long? arg5, StreamTrimMode arg6, CommandFlags arg7) => throw new NotImplementedException(); + public Task StreamAddAsync(RedisKey arg0, RedisValue arg1, RedisValue arg2, RedisValue? arg3, long? arg4, bool arg5, long? arg6, StreamTrimMode arg7, CommandFlags arg8) => throw new NotImplementedException(); + public Task StreamDeleteAsync(RedisKey arg0, RedisValue[] arg1, StreamTrimMode arg2, CommandFlags arg3) => throw new NotImplementedException(); + public Task StreamPendingMessagesAsync(RedisKey arg0, RedisValue arg1, int arg2, RedisValue arg3, RedisValue? arg4, RedisValue? arg5, long? arg6, CommandFlags arg7) => throw new NotImplementedException(); + public Task StreamTrimAsync(RedisKey arg0, long arg1, bool arg2, long? arg3, StreamTrimMode arg4, CommandFlags arg5) => throw new NotImplementedException(); + public Task StreamTrimByMinIdAsync(RedisKey arg0, RedisValue arg1, bool arg2, long? arg3, StreamTrimMode arg4, CommandFlags arg5) => throw new NotImplementedException(); + public Task VectorSetAddAsync(RedisKey arg0, VectorSetAddRequest arg1, CommandFlags arg2) => throw new NotImplementedException(); + public Task VectorSetContainsAsync(RedisKey arg0, RedisValue arg1, CommandFlags arg2) => throw new NotImplementedException(); + public Task VectorSetDimensionAsync(RedisKey arg0, CommandFlags arg1) => throw new NotImplementedException(); + public Task?> VectorSetGetApproximateVectorAsync(RedisKey arg0, RedisValue arg1, CommandFlags arg2) => throw new NotImplementedException(); + public Task VectorSetGetAttributesJsonAsync(RedisKey arg0, RedisValue arg1, CommandFlags arg2) => throw new NotImplementedException(); + public Task?> VectorSetGetLinksAsync(RedisKey arg0, RedisValue arg1, CommandFlags arg2) => throw new NotImplementedException(); + public Task?> VectorSetGetLinksWithScoresAsync(RedisKey arg0, RedisValue arg1, CommandFlags arg2) => throw new NotImplementedException(); + public Task VectorSetInfoAsync(RedisKey arg0, CommandFlags arg1) => throw new NotImplementedException(); + public Task VectorSetLengthAsync(RedisKey arg0, CommandFlags arg1) => throw new NotImplementedException(); + public Task VectorSetRandomMemberAsync(RedisKey arg0, CommandFlags arg1) => throw new NotImplementedException(); + public Task VectorSetRandomMembersAsync(RedisKey arg0, long arg1, CommandFlags arg2) => throw new NotImplementedException(); + public Task VectorSetRemoveAsync(RedisKey arg0, RedisValue arg1, CommandFlags arg2) => throw new NotImplementedException(); + public Task VectorSetSetAttributesJsonAsync(RedisKey arg0, RedisValue arg1, string arg2, CommandFlags arg3) => throw new NotImplementedException(); + public Task?> VectorSetSimilaritySearchAsync(RedisKey arg0, VectorSetSimilaritySearchRequest arg1, CommandFlags arg2) => throw new NotImplementedException(); +} diff --git a/MEVD/test/Redis.ConformanceTests/ModelTests/RedisHashSetBasicModelTests.cs b/MEVD/test/Redis.ConformanceTests/ModelTests/RedisHashSetBasicModelTests.cs new file mode 100644 index 0000000..45bc7ef --- /dev/null +++ b/MEVD/test/Redis.ConformanceTests/ModelTests/RedisHashSetBasicModelTests.cs @@ -0,0 +1,25 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Redis.ConformanceTests.Support; +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace Redis.ConformanceTests.ModelTests; + +public class RedisHashSetBasicModelTests(RedisHashSetBasicModelTests.Fixture fixture) + : BasicModelTests(fixture), IClassFixture +{ + public override async Task GetAsync_with_filter_and_multiple_OrderBys() + { + var exception = await Assert.ThrowsAsync(base.GetAsync_with_filter_and_multiple_OrderBys); + + Assert.Equal("Redis does not support ordering by more than one property.", exception.Message); + } + + public new class Fixture : BasicModelTests.Fixture + { + public override TestStore TestStore => RedisTestStore.HashSetInstance; + } +} diff --git a/MEVD/test/Redis.ConformanceTests/ModelTests/RedisHashSetDynamicModelTests.cs b/MEVD/test/Redis.ConformanceTests/ModelTests/RedisHashSetDynamicModelTests.cs new file mode 100644 index 0000000..2aeb1c4 --- /dev/null +++ b/MEVD/test/Redis.ConformanceTests/ModelTests/RedisHashSetDynamicModelTests.cs @@ -0,0 +1,25 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Redis.ConformanceTests.Support; +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace Redis.ConformanceTests.ModelTests; + +public class RedisHashSetDynamicModelTests(RedisHashSetDynamicModelTests.Fixture fixture) + : DynamicModelTests(fixture), IClassFixture +{ + public override async Task GetAsync_with_filter_and_multiple_OrderBys() + { + var exception = await Assert.ThrowsAsync(base.GetAsync_with_filter_and_multiple_OrderBys); + + Assert.Equal("Redis does not support ordering by more than one property.", exception.Message); + } + + public new class Fixture : DynamicModelTests.Fixture + { + public override TestStore TestStore => RedisTestStore.HashSetInstance; + } +} diff --git a/MEVD/test/Redis.ConformanceTests/ModelTests/RedisHashSetMultiVectorModelTests.cs b/MEVD/test/Redis.ConformanceTests/ModelTests/RedisHashSetMultiVectorModelTests.cs new file mode 100644 index 0000000..22dc3fd --- /dev/null +++ b/MEVD/test/Redis.ConformanceTests/ModelTests/RedisHashSetMultiVectorModelTests.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Redis.ConformanceTests.Support; +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace Redis.ConformanceTests.ModelTests; + +public class RedisHashSetMultiVectorModelTests(RedisHashSetMultiVectorModelTests.Fixture fixture) + : MultiVectorModelTests(fixture), IClassFixture +{ + public new class Fixture : MultiVectorModelTests.Fixture + { + public override TestStore TestStore => RedisTestStore.HashSetInstance; + } +} diff --git a/MEVD/test/Redis.ConformanceTests/ModelTests/RedisHashSetNoDataModelTests.cs b/MEVD/test/Redis.ConformanceTests/ModelTests/RedisHashSetNoDataModelTests.cs new file mode 100644 index 0000000..ebd0862 --- /dev/null +++ b/MEVD/test/Redis.ConformanceTests/ModelTests/RedisHashSetNoDataModelTests.cs @@ -0,0 +1,35 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Redis.ConformanceTests.Support; +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace Redis.ConformanceTests.ModelTests; + +public class RedisHashSetNoDataModelTests(RedisHashSetNoDataModelTests.Fixture fixture) + : NoDataModelTests(fixture), IClassFixture +{ + public override async Task GetAsync_single_record(bool includeVectors) + { + var expectedRecord = fixture.TestData[0]; + + var received = await this.Collection.GetAsync(expectedRecord.Key, new() { IncludeVectors = includeVectors }); + + if (includeVectors) + { + expectedRecord.AssertEqual(received, includeVectors, fixture.TestStore.VectorsComparable); + } + else + { + // When vectors aren't included and there's no other data, we get null back. + Assert.Null(received); + } + } + + public new class Fixture : NoDataModelTests.Fixture + { + public override TestStore TestStore => RedisTestStore.HashSetInstance; + } +} diff --git a/MEVD/test/Redis.ConformanceTests/ModelTests/RedisHashSetNoVectorModelTests.cs b/MEVD/test/Redis.ConformanceTests/ModelTests/RedisHashSetNoVectorModelTests.cs new file mode 100644 index 0000000..1f727ab --- /dev/null +++ b/MEVD/test/Redis.ConformanceTests/ModelTests/RedisHashSetNoVectorModelTests.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Redis.ConformanceTests.Support; +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace Redis.ConformanceTests.ModelTests; + +public class RedisHashSetNoVectorModelTests(RedisHashSetNoVectorModelTests.Fixture fixture) + : NoVectorModelTests(fixture), IClassFixture +{ + public new class Fixture : NoVectorModelTests.Fixture + { + public override TestStore TestStore => RedisTestStore.HashSetInstance; + } +} diff --git a/MEVD/test/Redis.ConformanceTests/ModelTests/RedisJsonBasicModelTests.cs b/MEVD/test/Redis.ConformanceTests/ModelTests/RedisJsonBasicModelTests.cs new file mode 100644 index 0000000..58f0de0 --- /dev/null +++ b/MEVD/test/Redis.ConformanceTests/ModelTests/RedisJsonBasicModelTests.cs @@ -0,0 +1,25 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Redis.ConformanceTests.Support; +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace Redis.ConformanceTests.ModelTests; + +public class RedisJsonBasicModelTests(RedisJsonBasicModelTests.Fixture fixture) + : BasicModelTests(fixture), IClassFixture +{ + public override async Task GetAsync_with_filter_and_multiple_OrderBys() + { + var exception = await Assert.ThrowsAsync(base.GetAsync_with_filter_and_multiple_OrderBys); + + Assert.Equal("Redis does not support ordering by more than one property.", exception.Message); + } + + public new class Fixture : BasicModelTests.Fixture + { + public override TestStore TestStore => RedisTestStore.JsonInstance; + } +} diff --git a/MEVD/test/Redis.ConformanceTests/ModelTests/RedisJsonDynamicModelTests.cs b/MEVD/test/Redis.ConformanceTests/ModelTests/RedisJsonDynamicModelTests.cs new file mode 100644 index 0000000..282ad97 --- /dev/null +++ b/MEVD/test/Redis.ConformanceTests/ModelTests/RedisJsonDynamicModelTests.cs @@ -0,0 +1,25 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Redis.ConformanceTests.Support; +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace Redis.ConformanceTests.ModelTests; + +public class RedisJsonDynamicModelTests(RedisJsonDynamicModelTests.Fixture fixture) + : DynamicModelTests(fixture), IClassFixture +{ + public override async Task GetAsync_with_filter_and_multiple_OrderBys() + { + var exception = await Assert.ThrowsAsync(base.GetAsync_with_filter_and_multiple_OrderBys); + + Assert.Equal("Redis does not support ordering by more than one property.", exception.Message); + } + + public new class Fixture : DynamicModelTests.Fixture + { + public override TestStore TestStore => RedisTestStore.JsonInstance; + } +} diff --git a/MEVD/test/Redis.ConformanceTests/ModelTests/RedisJsonMultiVectorModelTests.cs b/MEVD/test/Redis.ConformanceTests/ModelTests/RedisJsonMultiVectorModelTests.cs new file mode 100644 index 0000000..c0d6330 --- /dev/null +++ b/MEVD/test/Redis.ConformanceTests/ModelTests/RedisJsonMultiVectorModelTests.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Redis.ConformanceTests.Support; +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace Redis.ConformanceTests.ModelTests; + +public class RedisJsonMultiVectorModelTests(RedisJsonMultiVectorModelTests.Fixture fixture) + : MultiVectorModelTests(fixture), IClassFixture +{ + public new class Fixture : MultiVectorModelTests.Fixture + { + public override TestStore TestStore => RedisTestStore.JsonInstance; + } +} diff --git a/MEVD/test/Redis.ConformanceTests/ModelTests/RedisJsonNoDataModelTests.cs b/MEVD/test/Redis.ConformanceTests/ModelTests/RedisJsonNoDataModelTests.cs new file mode 100644 index 0000000..17ec4d3 --- /dev/null +++ b/MEVD/test/Redis.ConformanceTests/ModelTests/RedisJsonNoDataModelTests.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Redis.ConformanceTests.Support; +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace Redis.ConformanceTests.ModelTests; + +public class RedisJsonNoDataModelTests(RedisJsonNoDataModelTests.Fixture fixture) + : NoDataModelTests(fixture), IClassFixture +{ + public new class Fixture : NoDataModelTests.Fixture + { + public override TestStore TestStore => RedisTestStore.JsonInstance; + } +} diff --git a/MEVD/test/Redis.ConformanceTests/ModelTests/RedisJsonNoVectorModelTests.cs b/MEVD/test/Redis.ConformanceTests/ModelTests/RedisJsonNoVectorModelTests.cs new file mode 100644 index 0000000..f668c41 --- /dev/null +++ b/MEVD/test/Redis.ConformanceTests/ModelTests/RedisJsonNoVectorModelTests.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Redis.ConformanceTests.Support; +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace Redis.ConformanceTests.ModelTests; + +public class RedisJsonNoVectorModelTests(RedisJsonNoVectorModelTests.Fixture fixture) + : NoVectorModelTests(fixture), IClassFixture +{ + public new class Fixture : NoVectorModelTests.Fixture + { + public override TestStore TestStore => RedisTestStore.JsonInstance; + } +} diff --git a/MEVD/test/Redis.ConformanceTests/Properties/AssemblyAttributes.cs b/MEVD/test/Redis.ConformanceTests/Properties/AssemblyAttributes.cs new file mode 100644 index 0000000..694f409 --- /dev/null +++ b/MEVD/test/Redis.ConformanceTests/Properties/AssemblyAttributes.cs @@ -0,0 +1,7 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Xunit; + +// The Redis tests are flaky when parallelization is enabled +[assembly: CollectionBehavior(DisableTestParallelization = true)] diff --git a/MEVD/test/Redis.ConformanceTests/Redis.ConformanceTests.csproj b/MEVD/test/Redis.ConformanceTests/Redis.ConformanceTests.csproj new file mode 100644 index 0000000..bf0529c --- /dev/null +++ b/MEVD/test/Redis.ConformanceTests/Redis.ConformanceTests.csproj @@ -0,0 +1,27 @@ + + + + net10.0;$(NetFrameworkTfm) + enable + enable + true + false + Redis.ConformanceTests + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + + + + diff --git a/MEVD/test/Redis.ConformanceTests/RedisFilterTests.cs b/MEVD/test/Redis.ConformanceTests/RedisFilterTests.cs new file mode 100644 index 0000000..f04f98b --- /dev/null +++ b/MEVD/test/Redis.ConformanceTests/RedisFilterTests.cs @@ -0,0 +1,180 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.VectorData; +using CommunityToolkit.VectorData.Redis; +using Redis.ConformanceTests.Support; +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.Support; +using Xunit; +using Xunit.Sdk; + +namespace Redis.ConformanceTests; + +public abstract class RedisFilterTests(FilterTests.Fixture fixture) + : FilterTests(fixture) +{ + #region Equality with null + + public override Task Equal_with_null_reference_type() + => Assert.ThrowsAsync(() => base.Equal_with_null_reference_type()); + + public override Task Equal_with_null_captured() + => Assert.ThrowsAsync(() => base.Equal_with_null_captured()); + + public override Task NotEqual_with_null_reference_type() + => Assert.ThrowsAsync(() => base.Equal_with_null_reference_type()); + + public override Task NotEqual_with_null_captured() + => Assert.ThrowsAsync(() => base.NotEqual_with_null_captured()); + + public override Task Equal_int_property_with_null_nullable_int() + => Assert.ThrowsAsync(() => base.Equal_int_property_with_null_nullable_int()); + + #endregion + + #region Bool + + public override Task Bool() + => Assert.ThrowsAsync(() => base.Bool()); + + public override Task Not_over_bool() + => Assert.ThrowsAsync(() => base.Not_over_bool()); + + public override Task Bool_And_Bool() + => Assert.ThrowsAsync(() => base.Bool_And_Bool()); + + public override Task Bool_Or_Not_Bool() + => Assert.ThrowsAsync(() => base.Bool_Or_Not_Bool()); + + public override Task Not_over_bool_And_Comparison() + => Assert.ThrowsAsync(() => base.Not_over_bool_And_Comparison()); + + #endregion + + #region Contains + + public override Task Contains_over_inline_int_array() + => Assert.ThrowsAsync(() => base.Contains_over_inline_int_array()); + + public override Task Contains_over_inline_string_array() + => Assert.ThrowsAsync(() => base.Contains_over_inline_string_array()); + + public override Task Contains_over_inline_string_array_with_weird_chars() + => Assert.ThrowsAsync(() => base.Contains_over_inline_string_array_with_weird_chars()); + + public override Task Contains_over_captured_string_array() + => Assert.ThrowsAsync(() => base.Contains_over_captured_string_array()); + + #endregion +} + +public class RedisJsonFilterTests(RedisJsonFilterTests.Fixture fixture) + : RedisFilterTests(fixture), IClassFixture +{ + public new class Fixture : FilterTests.Fixture + { + public override TestStore TestStore => RedisTestStore.JsonInstance; + + protected override string CollectionNameBase => "JsonFilterTests"; + + // Override to remove the bool property, which isn't (currently) supported on Redis/JSON + public override VectorStoreCollectionDefinition CreateRecordDefinition() + => new() + { + Properties = base.CreateRecordDefinition().Properties.Where(p => p.Type != typeof(bool)).ToList() + }; + + protected override VectorStoreCollection GetCollection() + => new RedisJsonCollection( + RedisTestStore.JsonInstance.Database, + this.CollectionName, + new() { Definition = this.CreateRecordDefinition() }); + } +} + +public class RedisHashSetFilterTests(RedisHashSetFilterTests.Fixture fixture) + : RedisFilterTests(fixture), IClassFixture +{ + // Null values are not supported in Redis HashSet + public override Task Equal_with_null_reference_type() + => Assert.ThrowsAsync(() => base.Equal_with_null_reference_type()); + + public override Task Equal_with_null_captured() + => Assert.ThrowsAsync(() => base.Equal_with_null_captured()); + + public override Task NotEqual_with_null_reference_type() + => Assert.ThrowsAsync(() => base.NotEqual_with_null_reference_type()); + + public override Task NotEqual_with_null_captured() + => Assert.ThrowsAsync(() => base.NotEqual_with_null_captured()); + + // Array fields not supported on Redis HashSet + public override Task Contains_over_field_string_array() + => Assert.ThrowsAsync(() => base.Contains_over_field_string_array()); + + public override Task Contains_over_field_string_List() + => Assert.ThrowsAsync(() => base.Contains_over_field_string_List()); + + public override Task Contains_with_Enumerable_Contains() + => Assert.ThrowsAsync(() => base.Contains_with_Enumerable_Contains()); + +#if !NETFRAMEWORK + public override Task Contains_with_MemoryExtensions_Contains() + => Assert.ThrowsAsync(() => base.Contains_with_MemoryExtensions_Contains()); +#endif + +#if NET10_0_OR_GREATER + public override Task Contains_with_MemoryExtensions_Contains_with_null_comparer() + => Assert.ThrowsAsync(() => base.Contains_with_MemoryExtensions_Contains_with_null_comparer()); +#endif + + // Array fields not supported on Redis HashSet - Any tests + public override Task Any_with_Contains_over_inline_string_array() + => Assert.ThrowsAsync(() => base.Any_with_Contains_over_inline_string_array()); + + public override Task Any_with_Contains_over_captured_string_array() + => Assert.ThrowsAsync(() => base.Any_with_Contains_over_captured_string_array()); + + public override Task Any_with_Contains_over_captured_string_list() + => Assert.ThrowsAsync(() => base.Any_with_Contains_over_captured_string_list()); + + public override Task Any_over_List_with_Contains_over_captured_string_array() + => Assert.ThrowsAsync(() => base.Any_over_List_with_Contains_over_captured_string_array()); + + public new class Fixture : FilterTests.Fixture + { + public override TestStore TestStore => RedisTestStore.HashSetInstance; + + protected override string CollectionNameBase => "HashSetCollectionFilterTests"; + + // Override to remove the bool property, which isn't (currently) supported on Redis + public override VectorStoreCollectionDefinition CreateRecordDefinition() + => new() + { + Properties = base.CreateRecordDefinition().Properties.Where(p => + p.Type != typeof(bool) && + p.Type != typeof(string[]) && + p.Type != typeof(List)).ToList() + }; + + protected override VectorStoreCollection GetCollection() + => new RedisHashSetCollection( + RedisTestStore.HashSetInstance.Database, + this.CollectionName, + new() { Definition = this.CreateRecordDefinition() }); + + protected override List BuildTestData() + { + var testData = base.BuildTestData(); + + foreach (var record in testData) + { + // Null values are not supported in Redis hashsets + record.String ??= string.Empty; + } + + return testData; + } + } +} diff --git a/MEVD/test/Redis.ConformanceTests/RedisHashSetCollectionManagementTests.cs b/MEVD/test/Redis.ConformanceTests/RedisHashSetCollectionManagementTests.cs new file mode 100644 index 0000000..9e14bc0 --- /dev/null +++ b/MEVD/test/Redis.ConformanceTests/RedisHashSetCollectionManagementTests.cs @@ -0,0 +1,13 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Redis.ConformanceTests.Support; +using VectorData.ConformanceTests; +using Xunit; + +namespace Redis.ConformanceTests; + +public class RedisHashSetCollectionManagementTests(RedisHashSetFixture fixture) + : CollectionManagementTests(fixture), IClassFixture +{ +} diff --git a/MEVD/test/Redis.ConformanceTests/RedisHashSetDependencyInjectionTests.cs b/MEVD/test/Redis.ConformanceTests/RedisHashSetDependencyInjectionTests.cs new file mode 100644 index 0000000..45700b0 --- /dev/null +++ b/MEVD/test/Redis.ConformanceTests/RedisHashSetDependencyInjectionTests.cs @@ -0,0 +1,85 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using CommunityToolkit.VectorData.Redis; +using Redis.ConformanceTests.Support; +using StackExchange.Redis; +using VectorData.ConformanceTests; +using Xunit; + +namespace Redis.ConformanceTests; + +public class RedisHashSetDependencyInjectionTests + : DependencyInjectionTests.Record>, string, DependencyInjectionTests.Record> +{ + private const string ConnectionConfiguration = "localhost:6379"; + + protected override void PopulateConfiguration(ConfigurationManager configuration, object? serviceKey = null) + => configuration.AddInMemoryCollection( + [ + new(CreateConfigKey("RedisHashSet", serviceKey, "Configuration"), ConnectionConfiguration), + ]); + + private static string Provider(IServiceProvider sp, object? serviceKey = null) + => sp.GetRequiredService().GetRequiredSection(CreateConfigKey("RedisHashSet", serviceKey, "Configuration")).Value!; + + public override IEnumerable> CollectionDelegates + { + get + { + yield return (services, serviceKey, name, lifetime) => serviceKey is null + ? services + .AddRedisHashSetCollection(name, + sp => new FakeDatabase(Provider(sp)), lifetime: lifetime) + : services + .AddKeyedRedisHashSetCollection(serviceKey, name, + sp => new FakeDatabase(Provider(sp, serviceKey)), lifetime: lifetime); + + yield return (services, serviceKey, name, lifetime) => serviceKey is null + ? services + .AddSingleton(new FakeDatabase(ConnectionConfiguration)) + .AddRedisHashSetCollection(name, lifetime: lifetime) + : services + .AddSingleton(new FakeDatabase(ConnectionConfiguration)) + .AddKeyedRedisHashSetCollection(serviceKey, name, lifetime: lifetime); + + yield return (services, serviceKey, name, lifetime) => services + .AddKeyedSingleton(serviceKey, new FakeDatabase(ConnectionConfiguration)) + .AddKeyedRedisHashSetCollection(serviceKey, name, + sp => sp.GetRequiredKeyedService(serviceKey), lifetime: lifetime); + } + } + + public override IEnumerable> StoreDelegates + { + get + { + yield return (services, serviceKey, lifetime) => serviceKey is null + ? services + .AddSingleton(new FakeDatabase(ConnectionConfiguration)) + .AddRedisVectorStore(lifetime: lifetime) + : services + .AddSingleton(new FakeDatabase(ConnectionConfiguration)) + .AddKeyedRedisVectorStore(serviceKey, lifetime: lifetime); + } + } + + [Fact] + public void ConnectionConfigurationCantBeNullOrEmpty() + { + IServiceCollection services = new ServiceCollection(); + + Assert.Throws(() => services.AddRedisVectorStore(connectionConfiguration: null!)); + Assert.Throws(() => services.AddRedisVectorStore(connectionConfiguration: "")); + + Assert.Throws(() => services.AddKeyedRedisVectorStore("serviceKey", connectionConfiguration: null!)); + Assert.Throws(() => services.AddKeyedRedisVectorStore("serviceKey", connectionConfiguration: "")); + + Assert.Throws(() => services.AddRedisHashSetCollection( + name: "notNull", connectionConfiguration: null!)); + Assert.Throws(() => services.AddRedisHashSetCollection( + name: "notNull", connectionConfiguration: "")); + } +} diff --git a/MEVD/test/Redis.ConformanceTests/RedisHashSetDistanceFunctionTests.cs b/MEVD/test/Redis.ConformanceTests/RedisHashSetDistanceFunctionTests.cs new file mode 100644 index 0000000..08e3c93 --- /dev/null +++ b/MEVD/test/Redis.ConformanceTests/RedisHashSetDistanceFunctionTests.cs @@ -0,0 +1,34 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Redis.ConformanceTests.Support; +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.Support; +using Xunit; +using Xunit.Sdk; + +namespace Redis.ConformanceTests; + +public class RedisHashSetDistanceFunctionTests(RedisHashSetDistanceFunctionTests.Fixture fixture) + : DistanceFunctionTests(fixture), IClassFixture +{ + // Excluding DotProductSimilarity from the test even though Redis supports it, because the values that redis returns + // are neither DotProductSimilarity nor NegativeDotProduct, but rather 1 - DotProductSimilarity. + public override Task DotProductSimilarity() => Assert.ThrowsAsync(base.DotProductSimilarity); + + public override Task EuclideanDistance() => Assert.ThrowsAsync(base.EuclideanDistance); + public override Task NegativeDotProductSimilarity() => Assert.ThrowsAsync(base.NegativeDotProductSimilarity); + public override Task HammingDistance() => Assert.ThrowsAsync(base.HammingDistance); + public override Task ManhattanDistance() => Assert.ThrowsAsync(base.ManhattanDistance); + + public new class Fixture() : DistanceFunctionTests.Fixture + { + private int _collectionCounter; + + public override TestStore TestStore => RedisTestStore.HashSetInstance; + + // Redis doesn't seem to reliably delete the collection: when running multiple tests that delete and recreate the collection with different key types, + // we seem to get key values from the previous collection despite having deleted and recreated it. So we uniquify the collection name instead. + public override string CollectionName => "DistanceFunctionTests" + (++this._collectionCounter); + } +} diff --git a/MEVD/test/Redis.ConformanceTests/RedisHashSetEmbeddingGenerationTests.cs b/MEVD/test/Redis.ConformanceTests/RedisHashSetEmbeddingGenerationTests.cs new file mode 100644 index 0000000..993f629 --- /dev/null +++ b/MEVD/test/Redis.ConformanceTests/RedisHashSetEmbeddingGenerationTests.cs @@ -0,0 +1,61 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.VectorData; +using CommunityToolkit.VectorData.Redis; +using Redis.ConformanceTests.Support; +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace Redis.ConformanceTests; + +public class RedisHashSetEmbeddingGenerationTests(RedisHashSetEmbeddingGenerationTests.StringVectorFixture stringVectorFixture, RedisHashSetEmbeddingGenerationTests.RomOfFloatVectorFixture romOfFloatVectorFixture) + : EmbeddingGenerationTests(stringVectorFixture, romOfFloatVectorFixture), IClassFixture, IClassFixture +{ + public new class StringVectorFixture : EmbeddingGenerationTests.StringVectorFixture + { + public override TestStore TestStore => RedisTestStore.HashSetInstance; + + public override VectorStore CreateVectorStore(IEmbeddingGenerator? embeddingGenerator) + => RedisTestStore.HashSetInstance.GetVectorStore(new() { StorageType = RedisStorageType.HashSet, EmbeddingGenerator = embeddingGenerator }); + + public override Func[] DependencyInjectionStoreRegistrationDelegates => + [ + services => services + .AddSingleton(RedisTestStore.HashSetInstance.Database) + .AddRedisVectorStore(optionsProvider: _ => new RedisVectorStoreOptions() { StorageType = RedisStorageType.HashSet}) + ]; + + public override Func[] DependencyInjectionCollectionRegistrationDelegates => + [ + services => services + .AddSingleton(RedisTestStore.HashSetInstance.Database) + .AddRedisHashSetCollection(this.CollectionName) + ]; + } + + public new class RomOfFloatVectorFixture : EmbeddingGenerationTests.RomOfFloatVectorFixture + { + public override TestStore TestStore => RedisTestStore.HashSetInstance; + + public override VectorStore CreateVectorStore(IEmbeddingGenerator? embeddingGenerator) + => RedisTestStore.HashSetInstance.GetVectorStore(new() { StorageType = RedisStorageType.HashSet, EmbeddingGenerator = embeddingGenerator }); + + public override Func[] DependencyInjectionStoreRegistrationDelegates => + [ + services => services + .AddSingleton(RedisTestStore.HashSetInstance.Database) + .AddRedisVectorStore(optionsProvider: _ => new RedisVectorStoreOptions() { StorageType = RedisStorageType.HashSet}) + ]; + + public override Func[] DependencyInjectionCollectionRegistrationDelegates => + [ + services => services + .AddSingleton(RedisTestStore.HashSetInstance.Database) + .AddRedisHashSetCollection(this.CollectionName) + ]; + } +} diff --git a/MEVD/test/Redis.ConformanceTests/RedisHashSetIndexKindTests.cs b/MEVD/test/Redis.ConformanceTests/RedisHashSetIndexKindTests.cs new file mode 100644 index 0000000..9477ff0 --- /dev/null +++ b/MEVD/test/Redis.ConformanceTests/RedisHashSetIndexKindTests.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Redis.ConformanceTests.Support; +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace Redis.ConformanceTests; + +public class RedisHashSetIndexKindTests(RedisHashSetIndexKindTests.Fixture fixture) + : IndexKindTests(fixture), IClassFixture +{ + public new class Fixture() : IndexKindTests.Fixture + { + public override TestStore TestStore => RedisTestStore.HashSetInstance; + } +} diff --git a/MEVD/test/Redis.ConformanceTests/RedisJsonCollectionManagementTests.cs b/MEVD/test/Redis.ConformanceTests/RedisJsonCollectionManagementTests.cs new file mode 100644 index 0000000..33fddfe --- /dev/null +++ b/MEVD/test/Redis.ConformanceTests/RedisJsonCollectionManagementTests.cs @@ -0,0 +1,13 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Redis.ConformanceTests.Support; +using VectorData.ConformanceTests; +using Xunit; + +namespace Redis.ConformanceTests; + +public class RedisJsonCollectionManagementTests(RedisJsonFixture fixture) + : CollectionManagementTests(fixture), IClassFixture +{ +} diff --git a/MEVD/test/Redis.ConformanceTests/RedisJsonDependencyInjectionTests.cs b/MEVD/test/Redis.ConformanceTests/RedisJsonDependencyInjectionTests.cs new file mode 100644 index 0000000..bda32c7 --- /dev/null +++ b/MEVD/test/Redis.ConformanceTests/RedisJsonDependencyInjectionTests.cs @@ -0,0 +1,85 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using CommunityToolkit.VectorData.Redis; +using Redis.ConformanceTests.Support; +using StackExchange.Redis; +using VectorData.ConformanceTests; +using Xunit; + +namespace Redis.ConformanceTests; + +public class RedisJsonDependencyInjectionTests + : DependencyInjectionTests.Record>, string, DependencyInjectionTests.Record> +{ + private const string ConnectionConfiguration = "localhost:6379"; + + protected override void PopulateConfiguration(ConfigurationManager configuration, object? serviceKey = null) + => configuration.AddInMemoryCollection( + [ + new(CreateConfigKey("RedisJson", serviceKey, "Configuration"), ConnectionConfiguration), + ]); + + private static string Provider(IServiceProvider sp, object? serviceKey = null) + => sp.GetRequiredService().GetRequiredSection(CreateConfigKey("RedisJson", serviceKey, "Configuration")).Value!; + + public override IEnumerable> CollectionDelegates + { + get + { + yield return (services, serviceKey, name, lifetime) => serviceKey is null + ? services + .AddRedisJsonCollection(name, + sp => new FakeDatabase(Provider(sp)), lifetime: lifetime) + : services + .AddKeyedRedisJsonCollection(serviceKey, name, + sp => new FakeDatabase(Provider(sp, serviceKey)), lifetime: lifetime); + + yield return (services, serviceKey, name, lifetime) => serviceKey is null + ? services + .AddSingleton(new FakeDatabase(ConnectionConfiguration)) + .AddRedisJsonCollection(name, lifetime: lifetime) + : services + .AddSingleton(new FakeDatabase(ConnectionConfiguration)) + .AddKeyedRedisJsonCollection(serviceKey, name, lifetime: lifetime); + + yield return (services, serviceKey, name, lifetime) => services + .AddKeyedSingleton(serviceKey, new FakeDatabase(ConnectionConfiguration)) + .AddKeyedRedisJsonCollection(serviceKey, name, + sp => sp.GetRequiredKeyedService(serviceKey), lifetime: lifetime); + } + } + + public override IEnumerable> StoreDelegates + { + get + { + yield return (services, serviceKey, lifetime) => serviceKey is null + ? services + .AddSingleton(new FakeDatabase(ConnectionConfiguration)) + .AddRedisVectorStore(lifetime: lifetime) + : services + .AddSingleton(new FakeDatabase(ConnectionConfiguration)) + .AddKeyedRedisVectorStore(serviceKey, lifetime: lifetime); + } + } + + [Fact] + public void ConnectionConfigurationCantBeNullOrEmpty() + { + IServiceCollection services = new ServiceCollection(); + + Assert.Throws(() => services.AddRedisVectorStore(connectionConfiguration: null!)); + Assert.Throws(() => services.AddRedisVectorStore(connectionConfiguration: "")); + + Assert.Throws(() => services.AddKeyedRedisVectorStore("serviceKey", connectionConfiguration: null!)); + Assert.Throws(() => services.AddKeyedRedisVectorStore("serviceKey", connectionConfiguration: "")); + + Assert.Throws(() => services.AddRedisJsonCollection( + name: "notNull", connectionConfiguration: null!)); + Assert.Throws(() => services.AddRedisJsonCollection( + name: "notNull", connectionConfiguration: "")); + } +} diff --git a/MEVD/test/Redis.ConformanceTests/RedisJsonDistanceFunctionTests.cs b/MEVD/test/Redis.ConformanceTests/RedisJsonDistanceFunctionTests.cs new file mode 100644 index 0000000..3a87711 --- /dev/null +++ b/MEVD/test/Redis.ConformanceTests/RedisJsonDistanceFunctionTests.cs @@ -0,0 +1,34 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Redis.ConformanceTests.Support; +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.Support; +using Xunit; +using Xunit.Sdk; + +namespace Redis.ConformanceTests; + +public class RedisJsonDistanceFunctionTests(RedisJsonDistanceFunctionTests.Fixture fixture) + : DistanceFunctionTests(fixture), IClassFixture +{ + // Excluding DotProductSimilarity from the test even though Redis supports it, because the values that redis returns + // are neither DotProductSimilarity nor NegativeDotProduct, but rather 1 - DotProductSimilarity. + public override Task DotProductSimilarity() => Assert.ThrowsAsync(base.DotProductSimilarity); + + public override Task EuclideanDistance() => Assert.ThrowsAsync(base.EuclideanDistance); + public override Task NegativeDotProductSimilarity() => Assert.ThrowsAsync(base.NegativeDotProductSimilarity); + public override Task HammingDistance() => Assert.ThrowsAsync(base.HammingDistance); + public override Task ManhattanDistance() => Assert.ThrowsAsync(base.ManhattanDistance); + + public new class Fixture() : DistanceFunctionTests.Fixture + { + private int _collectionCounter; + + public override TestStore TestStore => RedisTestStore.JsonInstance; + + // Redis doesn't seem to reliably delete the collection: when running multiple tests that delete and recreate the collection with different key types, + // we seem to get key values from the previous collection despite having deleted and recreated it. So we uniquify the collection name instead. + public override string CollectionName => "DistanceFunctionTests" + (++this._collectionCounter); + } +} diff --git a/MEVD/test/Redis.ConformanceTests/RedisJsonEmbeddingGenerationTests.cs b/MEVD/test/Redis.ConformanceTests/RedisJsonEmbeddingGenerationTests.cs new file mode 100644 index 0000000..be5163c --- /dev/null +++ b/MEVD/test/Redis.ConformanceTests/RedisJsonEmbeddingGenerationTests.cs @@ -0,0 +1,60 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.VectorData; +using Redis.ConformanceTests.Support; +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace Redis.ConformanceTests; + +public class RedisJsonEmbeddingGenerationTests(RedisJsonEmbeddingGenerationTests.StringVectorFixture stringVectorFixture, RedisJsonEmbeddingGenerationTests.RomOfFloatVectorFixture romOfFloatVectorFixture) + : EmbeddingGenerationTests(stringVectorFixture, romOfFloatVectorFixture), IClassFixture, IClassFixture +{ + public new class StringVectorFixture : EmbeddingGenerationTests.StringVectorFixture + { + public override TestStore TestStore => RedisTestStore.JsonInstance; + + public override VectorStore CreateVectorStore(IEmbeddingGenerator? embeddingGenerator) + => RedisTestStore.JsonInstance.GetVectorStore(new() { EmbeddingGenerator = embeddingGenerator }); + + public override Func[] DependencyInjectionStoreRegistrationDelegates => + [ + services => services + .AddSingleton(RedisTestStore.JsonInstance.Database) + .AddRedisVectorStore() + ]; + + public override Func[] DependencyInjectionCollectionRegistrationDelegates => + [ + services => services + .AddSingleton(RedisTestStore.JsonInstance.Database) + .AddRedisJsonCollection(this.CollectionName) + ]; + } + + public new class RomOfFloatVectorFixture : EmbeddingGenerationTests.RomOfFloatVectorFixture + { + public override TestStore TestStore => RedisTestStore.JsonInstance; + + public override VectorStore CreateVectorStore(IEmbeddingGenerator? embeddingGenerator) + => RedisTestStore.JsonInstance.GetVectorStore(new() { EmbeddingGenerator = embeddingGenerator }); + + public override Func[] DependencyInjectionStoreRegistrationDelegates => + [ + services => services + .AddSingleton(RedisTestStore.JsonInstance.Database) + .AddRedisVectorStore() + ]; + + public override Func[] DependencyInjectionCollectionRegistrationDelegates => + [ + services => services + .AddSingleton(RedisTestStore.JsonInstance.Database) + .AddRedisJsonCollection(this.CollectionName) + ]; + } +} diff --git a/MEVD/test/Redis.ConformanceTests/RedisJsonIndexKindTests.cs b/MEVD/test/Redis.ConformanceTests/RedisJsonIndexKindTests.cs new file mode 100644 index 0000000..c5c31d6 --- /dev/null +++ b/MEVD/test/Redis.ConformanceTests/RedisJsonIndexKindTests.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Redis.ConformanceTests.Support; +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace Redis.ConformanceTests; + +public class RedisJsonIndexKindTests(RedisJsonIndexKindTests.Fixture fixture) + : IndexKindTests(fixture), IClassFixture +{ + public new class Fixture() : IndexKindTests.Fixture + { + public override TestStore TestStore => RedisTestStore.JsonInstance; + } +} diff --git a/MEVD/test/Redis.ConformanceTests/RedisJsonOptionsTests.cs b/MEVD/test/Redis.ConformanceTests/RedisJsonOptionsTests.cs new file mode 100644 index 0000000..98911ad --- /dev/null +++ b/MEVD/test/Redis.ConformanceTests/RedisJsonOptionsTests.cs @@ -0,0 +1,121 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text.Json; +using Microsoft.Extensions.VectorData; +using CommunityToolkit.VectorData.Redis; +using Redis.ConformanceTests.Support; +using StackExchange.Redis; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace Redis.ConformanceTests; + +public sealed class RedisJsonOptionsTests(RedisJsonOptionsTests.Fixture fixture) + : IClassFixture +{ + [Fact] + public async Task Json_collection_with_prefix_and_nested_address_roundtrips() + { + var store = (RedisTestStore)fixture.TestStore; + var collectionName = fixture.TestStore.AdjustCollectionName("jsonoptions"); + + var options = new RedisJsonCollectionOptions { PrefixCollectionNameToKeyNames = true }; + using var collection = new RedisJsonCollection(store.Database, collectionName, options); + + await collection.EnsureCollectionExistsAsync(); + + try + { + var record = new RedisJsonHotel + { + HotelId = "hotel-1", + HotelName = "Test Hotel", + ParkingIncluded = true, + Address = new RedisAddress { City = "Seattle", Country = "USA" }, + DescriptionEmbedding = new([30f, 31f, 32f, 33f]) + }; + + await collection.UpsertAsync(record); + var fetched = await collection.GetAsync(record.HotelId, new() { IncludeVectors = true }); + + Assert.NotNull(fetched); + Assert.Equal(record.HotelId, fetched!.HotelId); + Assert.Equal(record.HotelName, fetched.HotelName); + Assert.Equal(record.ParkingIncluded, fetched.ParkingIncluded); + Assert.NotNull(fetched.Address); + Assert.Equal(record.Address.City, fetched.Address.City); + Assert.Equal(record.Address.Country, fetched.Address.Country); + Assert.Equal(record.DescriptionEmbedding.ToArray(), fetched.DescriptionEmbedding.ToArray()); + } + finally + { + await collection.EnsureCollectionDeletedAsync(); + } + } + + [Fact] + public async Task Json_collection_get_throws_for_invalid_schema() + { + var store = (RedisTestStore)fixture.TestStore; + var collectionName = fixture.TestStore.AdjustCollectionName("jsoninvalidschema"); + + var options = new RedisJsonCollectionOptions { PrefixCollectionNameToKeyNames = true }; + using var collection = new RedisJsonCollection(store.Database, collectionName, options); + + await collection.EnsureCollectionExistsAsync(); + + try + { + var invalidDocument = new + { + HotelId = "another-id", + HotelName = "Invalid Hotel", + ParkingIncluded = false, + DescriptionEmbedding = new[] { 30f, 31f, 32f, 33f }, + Address = new { City = "Seattle", Country = "USA" } + }; + + var key = (RedisKey)$"{collectionName}:invalid"; + var json = JsonSerializer.Serialize(invalidDocument); + await store.Database.ExecuteAsync("JSON.SET", key, "$", json); + + await Assert.ThrowsAsync(async () => + await collection.GetAsync("invalid", new() { IncludeVectors = true })); + } + finally + { + await collection.EnsureCollectionDeletedAsync(); + } + } + + public sealed class Fixture : VectorStoreFixture + { + public override TestStore TestStore => RedisTestStore.JsonInstance; + } + + private sealed class RedisJsonHotel + { + [VectorStoreKey] + public string HotelId { get; set; } = string.Empty; + + [VectorStoreData(IsIndexed = true)] + public string HotelName { get; set; } = string.Empty; + + [VectorStoreData(StorageName = "parking_is_included")] + public bool ParkingIncluded { get; set; } + + [VectorStoreData] + public RedisAddress Address { get; set; } = new(); + + [VectorStoreVector(dimensions: 4)] + public ReadOnlyMemory DescriptionEmbedding { get; set; } + } + + private sealed class RedisAddress + { + public string City { get; set; } = string.Empty; + + public string Country { get; set; } = string.Empty; + } +} diff --git a/MEVD/test/Redis.ConformanceTests/RedisTestSuiteImplementationTests.cs b/MEVD/test/Redis.ConformanceTests/RedisTestSuiteImplementationTests.cs new file mode 100644 index 0000000..a4f39f9 --- /dev/null +++ b/MEVD/test/Redis.ConformanceTests/RedisTestSuiteImplementationTests.cs @@ -0,0 +1,15 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using VectorData.ConformanceTests; + +namespace Redis.ConformanceTests; + +public class RedisTestSuiteImplementationTests : TestSuiteImplementationTests +{ + protected override ICollection IgnoredTestBases { get; } = + [ + // Hybrid search not supported + typeof(HybridSearchTests<>) + ]; +} diff --git a/MEVD/test/Redis.ConformanceTests/Support/RedisFixture.cs b/MEVD/test/Redis.ConformanceTests/Support/RedisFixture.cs new file mode 100644 index 0000000..130b942 --- /dev/null +++ b/MEVD/test/Redis.ConformanceTests/Support/RedisFixture.cs @@ -0,0 +1,11 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using VectorData.ConformanceTests.Support; + +namespace Redis.ConformanceTests.Support; + +public class RedisFixture : VectorStoreFixture +{ + public override TestStore TestStore => RedisTestStore.JsonInstance; +} diff --git a/MEVD/test/Redis.ConformanceTests/Support/RedisHashSetFixture.cs b/MEVD/test/Redis.ConformanceTests/Support/RedisHashSetFixture.cs new file mode 100644 index 0000000..c520a1f --- /dev/null +++ b/MEVD/test/Redis.ConformanceTests/Support/RedisHashSetFixture.cs @@ -0,0 +1,11 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using VectorData.ConformanceTests.Support; + +namespace Redis.ConformanceTests.Support; + +public class RedisHashSetFixture : VectorStoreFixture +{ + public override TestStore TestStore => RedisTestStore.HashSetInstance; +} diff --git a/MEVD/test/Redis.ConformanceTests/Support/RedisJsonFixture.cs b/MEVD/test/Redis.ConformanceTests/Support/RedisJsonFixture.cs new file mode 100644 index 0000000..c42c329 --- /dev/null +++ b/MEVD/test/Redis.ConformanceTests/Support/RedisJsonFixture.cs @@ -0,0 +1,11 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using VectorData.ConformanceTests.Support; + +namespace Redis.ConformanceTests.Support; + +public class RedisJsonFixture : VectorStoreFixture +{ + public override TestStore TestStore => RedisTestStore.JsonInstance; +} diff --git a/MEVD/test/Redis.ConformanceTests/Support/RedisTestStore.cs b/MEVD/test/Redis.ConformanceTests/Support/RedisTestStore.cs new file mode 100644 index 0000000..665b4c8 --- /dev/null +++ b/MEVD/test/Redis.ConformanceTests/Support/RedisTestStore.cs @@ -0,0 +1,43 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using CommunityToolkit.VectorData.Redis; +using StackExchange.Redis; +using Testcontainers.Redis; +using VectorData.ConformanceTests.Support; + +namespace Redis.ConformanceTests.Support; + +#pragma warning restore CA2213 // Disposable fields should be disposed + +internal sealed class RedisTestStore : TestStore +{ + public static RedisTestStore JsonInstance { get; } = new(RedisStorageType.Json); + public static RedisTestStore HashSetInstance { get; } = new(RedisStorageType.HashSet); + + private readonly RedisContainer _container = new RedisBuilder("redis/redis-stack") + .WithPortBinding(6379, assignRandomHostPort: true) + .WithPortBinding(8001, assignRandomHostPort: true) + .Build(); + + private readonly RedisStorageType _storageType; + private IDatabase? _database; + + private RedisTestStore(RedisStorageType storageType) => this._storageType = storageType; + + public IDatabase Database => this._database ?? throw new InvalidOperationException("Not initialized"); + + public RedisVectorStore GetVectorStore(RedisVectorStoreOptions options) + => new(this.Database, options); + + protected override async Task StartAsync() + { + await this._container.StartAsync(); + var redis = await ConnectionMultiplexer.ConnectAsync($"{this._container.Hostname}:{this._container.GetMappedPublicPort(6379)},connectTimeout=60000,connectRetry=5"); + this._database = redis.GetDatabase(); + this.DefaultVectorStore = new RedisVectorStore(this._database, new() { StorageType = this._storageType }); + } + + protected override Task StopAsync() + => this._container.StopAsync(); +} diff --git a/MEVD/test/Redis.ConformanceTests/TypeTests/RedisHashSetDataTypeTests.cs b/MEVD/test/Redis.ConformanceTests/TypeTests/RedisHashSetDataTypeTests.cs new file mode 100644 index 0000000..beb7d31 --- /dev/null +++ b/MEVD/test/Redis.ConformanceTests/TypeTests/RedisHashSetDataTypeTests.cs @@ -0,0 +1,48 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Redis.ConformanceTests.Support; +using VectorData.ConformanceTests.Support; +using VectorData.ConformanceTests.TypeTests; +using Xunit; + +namespace Redis.ConformanceTests.TypeTests; + +public class RedisHashSetDataTypeTests(RedisHashSetDataTypeTests.Fixture fixture) + : DataTypeTests.DefaultRecord>(fixture), IClassFixture +{ + public override Task Bool() => Task.CompletedTask; + public override Task Decimal() => Task.CompletedTask; + public override Task DateTime() => Task.CompletedTask; + public override Task DateTimeOffset() => Task.CompletedTask; + public override Task DateOnly() => Task.CompletedTask; + public override Task TimeOnly() => Task.CompletedTask; + + public override Task String_array() + => this.Test( + "StringArray", + ["foo", "bar"], + ["foo", "baz"], + isFilterable: false); + + public new class Fixture : DataTypeTests.DefaultRecord>.Fixture + { + public override TestStore TestStore => RedisTestStore.JsonInstance; + + public override bool IsNullSupported => false; + public override bool IsNullFilteringSupported => false; + + public override Type[] UnsupportedDefaultTypes { get; } = + [ + typeof(bool), + typeof(decimal), + typeof(Guid), + typeof(DateTime), + typeof(DateTimeOffset), +#if NET + typeof(DateOnly), + typeof(TimeOnly) +#endif + ]; + } +} diff --git a/MEVD/test/Redis.ConformanceTests/TypeTests/RedisHashSetEmbeddingTypeTests.cs b/MEVD/test/Redis.ConformanceTests/TypeTests/RedisHashSetEmbeddingTypeTests.cs new file mode 100644 index 0000000..437adbf --- /dev/null +++ b/MEVD/test/Redis.ConformanceTests/TypeTests/RedisHashSetEmbeddingTypeTests.cs @@ -0,0 +1,41 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.AI; +using Redis.ConformanceTests.Support; +using VectorData.ConformanceTests.Support; +using VectorData.ConformanceTests.TypeTests; +using Xunit; + +#pragma warning disable CA2000 // Dispose objects before losing scope + +namespace Redis.ConformanceTests.TypeTests; + +public class RedisHashSetEmbeddingTypeTests(RedisHashSetEmbeddingTypeTests.Fixture fixture) + : EmbeddingTypeTests(fixture), IClassFixture +{ + [Fact] + public virtual Task ReadOnlyMemory_of_double() + => this.Test>( + new ReadOnlyMemory([1d, 2d, 3d]), + new ReadOnlyMemoryEmbeddingGenerator([1d, 2d, 3d]), + vectorEqualityAsserter: (e, a) => Assert.Equal(e.Span.ToArray(), a.Span.ToArray())); + + [Fact] + public virtual Task Embedding_of_double() + => this.Test>( + new Embedding(new ReadOnlyMemory([1, 2, 3])), + new ReadOnlyMemoryEmbeddingGenerator([1, 2, 3]), + vectorEqualityAsserter: (e, a) => Assert.Equal(e.Vector.Span.ToArray(), a.Vector.Span.ToArray())); + + [Fact] + public virtual Task Array_of_double() + => this.Test( + [1, 2, 3], + new ReadOnlyMemoryEmbeddingGenerator([1, 2, 3])); + + public new class Fixture : EmbeddingTypeTests.Fixture + { + public override TestStore TestStore => RedisTestStore.HashSetInstance; + } +} diff --git a/MEVD/test/Redis.ConformanceTests/TypeTests/RedisHashSetKeyTypeTests.cs b/MEVD/test/Redis.ConformanceTests/TypeTests/RedisHashSetKeyTypeTests.cs new file mode 100644 index 0000000..45f8710 --- /dev/null +++ b/MEVD/test/Redis.ConformanceTests/TypeTests/RedisHashSetKeyTypeTests.cs @@ -0,0 +1,32 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.VectorData; +using Redis.ConformanceTests.Support; +using VectorData.ConformanceTests.Support; +using VectorData.ConformanceTests.TypeTests; +using Xunit; + +namespace Redis.ConformanceTests.TypeTests; + +public class RedisHashSetKeyTypeTests(RedisHashSetKeyTypeTests.Fixture fixture) + : KeyTypeTests(fixture), IClassFixture +{ + [Fact] + public virtual Task String() => this.Test("foo", "bar"); + + public new class Fixture : KeyTypeTests.Fixture + { + private int _collectionCounter; + + public override TestStore TestStore => RedisTestStore.HashSetInstance; + + // Redis doesn't seem to reliably delete the collection: when running multiple tests that delete and recreate the collection with different key types, + // we seem to get key values from the previous collection despite having deleted and recreated it. So we uniquify the collection name instead. + public override VectorStoreCollection> CreateCollection(bool? withAutoGeneration) + => this.TestStore.DefaultVectorStore.GetCollection>(this.CollectionName + (++this._collectionCounter), this.CreateRecordDefinition(withAutoGeneration)); + + public override VectorStoreCollection> CreateDynamicCollection(bool withAutoGeneration) + => this.TestStore.DefaultVectorStore.GetDynamicCollection(this.CollectionName + (++this._collectionCounter), this.CreateRecordDefinition(withAutoGeneration)); + } +} diff --git a/MEVD/test/Redis.ConformanceTests/TypeTests/RedisJsonDataTypeTests.cs b/MEVD/test/Redis.ConformanceTests/TypeTests/RedisJsonDataTypeTests.cs new file mode 100644 index 0000000..8512dd4 --- /dev/null +++ b/MEVD/test/Redis.ConformanceTests/TypeTests/RedisJsonDataTypeTests.cs @@ -0,0 +1,41 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Redis.ConformanceTests.Support; +using VectorData.ConformanceTests.Support; +using VectorData.ConformanceTests.TypeTests; +using Xunit; + +namespace Redis.ConformanceTests.TypeTests; + +public class RedisJsonDataTypeTests(RedisJsonDataTypeTests.Fixture fixture) + : DataTypeTests.DefaultRecord>(fixture), IClassFixture +{ + public override Task String_array() + => this.Test( + "StringArray", + ["foo", "bar"], + ["foo", "baz"], + isFilterable: false); + + public new class Fixture : DataTypeTests.DefaultRecord>.Fixture + { + public override TestStore TestStore => RedisTestStore.JsonInstance; + + public override bool IsNullSupported => false; + public override bool IsNullFilteringSupported => false; + + public override Type[] UnsupportedDefaultTypes { get; } = + [ + typeof(bool), + typeof(decimal), + typeof(Guid), + typeof(DateTime), + typeof(DateTimeOffset), +#if NET + typeof(DateOnly), + typeof(TimeOnly) +#endif + ]; + } +} diff --git a/MEVD/test/Redis.ConformanceTests/TypeTests/RedisJsonEmbeddingTypeTests.cs b/MEVD/test/Redis.ConformanceTests/TypeTests/RedisJsonEmbeddingTypeTests.cs new file mode 100644 index 0000000..3d33bf3 --- /dev/null +++ b/MEVD/test/Redis.ConformanceTests/TypeTests/RedisJsonEmbeddingTypeTests.cs @@ -0,0 +1,41 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.AI; +using Redis.ConformanceTests.Support; +using VectorData.ConformanceTests.Support; +using VectorData.ConformanceTests.TypeTests; +using Xunit; + +#pragma warning disable CA2000 // Dispose objects before losing scope + +namespace Redis.ConformanceTests.TypeTests; + +public class RedisJsonEmbeddingTypeTests(RedisJsonEmbeddingTypeTests.Fixture fixture) + : EmbeddingTypeTests(fixture), IClassFixture +{ + [Fact] + public virtual Task ReadOnlyMemory_of_double() + => this.Test>( + new ReadOnlyMemory([1d, 2d, 3d]), + new ReadOnlyMemoryEmbeddingGenerator([1d, 2d, 3d]), + vectorEqualityAsserter: (e, a) => Assert.Equal(e.Span.ToArray(), a.Span.ToArray())); + + [Fact] + public virtual Task Embedding_of_double() + => this.Test>( + new Embedding(new ReadOnlyMemory([1, 2, 3])), + new ReadOnlyMemoryEmbeddingGenerator([1, 2, 3]), + vectorEqualityAsserter: (e, a) => Assert.Equal(e.Vector.Span.ToArray(), a.Vector.Span.ToArray())); + + [Fact] + public virtual Task Array_of_double() + => this.Test( + [1, 2, 3], + new ReadOnlyMemoryEmbeddingGenerator([1, 2, 3])); + + public new class Fixture : EmbeddingTypeTests.Fixture + { + public override TestStore TestStore => RedisTestStore.JsonInstance; + } +} diff --git a/MEVD/test/Redis.ConformanceTests/TypeTests/RedisJsonKeyTypeTests.cs b/MEVD/test/Redis.ConformanceTests/TypeTests/RedisJsonKeyTypeTests.cs new file mode 100644 index 0000000..1e2322d --- /dev/null +++ b/MEVD/test/Redis.ConformanceTests/TypeTests/RedisJsonKeyTypeTests.cs @@ -0,0 +1,32 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.VectorData; +using Redis.ConformanceTests.Support; +using VectorData.ConformanceTests.Support; +using VectorData.ConformanceTests.TypeTests; +using Xunit; + +namespace Redis.ConformanceTests.TypeTests; + +public class RedisJsonKeyTypeTests(RedisJsonKeyTypeTests.Fixture fixture) + : KeyTypeTests(fixture), IClassFixture +{ + [Fact] + public virtual Task String() => this.Test("foo", "bar"); + + public new class Fixture : KeyTypeTests.Fixture + { + private int _collectionCounter; + + public override TestStore TestStore => RedisTestStore.JsonInstance; + + // Redis doesn't seem to reliably delete the collection: when running multiple tests that delete and recreate the collection with different key types, + // we seem to get key values from the previous collection despite having deleted and recreated it. So we uniquify the collection name instead. + public override VectorStoreCollection> CreateCollection(bool? withAutoGeneration) + => this.TestStore.DefaultVectorStore.GetCollection>(this.CollectionName + (++this._collectionCounter), this.CreateRecordDefinition(withAutoGeneration)); + + public override VectorStoreCollection> CreateDynamicCollection(bool withAutoGeneration) + => this.TestStore.DefaultVectorStore.GetDynamicCollection(this.CollectionName + (++this._collectionCounter), this.CreateRecordDefinition(withAutoGeneration)); + } +} diff --git a/MEVD/test/Redis.UnitTests/.editorconfig b/MEVD/test/Redis.UnitTests/.editorconfig new file mode 100644 index 0000000..394eef6 --- /dev/null +++ b/MEVD/test/Redis.UnitTests/.editorconfig @@ -0,0 +1,6 @@ +# Suppressing errors for Test projects under dotnet folder +[*.cs] +dotnet_diagnostic.CA2007.severity = none # Do not directly await a Task +dotnet_diagnostic.VSTHRD111.severity = none # Use .ConfigureAwait(bool) is hidden by default, set to none to prevent IDE from changing on autosave +dotnet_diagnostic.CS1591.severity = none # Missing XML comment for publicly visible type or member +dotnet_diagnostic.IDE1006.severity = warning # Naming rule violations diff --git a/MEVD/test/Redis.UnitTests/Redis.UnitTests.csproj b/MEVD/test/Redis.UnitTests/Redis.UnitTests.csproj new file mode 100644 index 0000000..b23e6db --- /dev/null +++ b/MEVD/test/Redis.UnitTests/Redis.UnitTests.csproj @@ -0,0 +1,39 @@ + + + + CommunityToolkit.VectorData.Redis.UnitTests + CommunityToolkit.VectorData.Redis.UnitTests + net10.0 + true + enable + disable + false + $(NoWarn);CA2007,CA1806,CA1869,CA1861,IDE0300,VSTHRD111,SKEXP0001,SKEXP0010,SKEXP0050 + $(NoWarn);MEVD9001 + + + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + + + + + diff --git a/MEVD/test/Redis.UnitTests/RedisCollectionCreateMappingTests.cs b/MEVD/test/Redis.UnitTests/RedisCollectionCreateMappingTests.cs new file mode 100644 index 0000000..b6bfa2d --- /dev/null +++ b/MEVD/test/Redis.UnitTests/RedisCollectionCreateMappingTests.cs @@ -0,0 +1,128 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; +using NRedisStack.Search; +using Xunit; +using static NRedisStack.Search.Schema; + +namespace CommunityToolkit.VectorData.Redis.UnitTests; + +/// +/// Contains tests for the class. +/// +public class RedisCollectionCreateMappingTests +{ + [Theory] + [InlineData(true)] + [InlineData(false)] + public void MapToSchemaCreatesSchema(bool useDollarPrefix) + { + // Arrange. + PropertyModel[] properties = + [ + new KeyPropertyModel("Key", typeof(string)), + + new DataPropertyModel("FilterableString", typeof(string)) { IsIndexed = true }, + new DataPropertyModel("FullTextSearchableString", typeof(string)) { IsFullTextIndexed = true }, + new DataPropertyModel("FilterableStringEnumerable", typeof(string[])) { IsIndexed = true }, + new DataPropertyModel("FullTextSearchableStringEnumerable", typeof(string[])) { IsFullTextIndexed = true }, + + new DataPropertyModel("FilterableInt", typeof(int)) { IsIndexed = true }, + new DataPropertyModel("FilterableNullableInt", typeof(int)) { IsIndexed = true }, + + new DataPropertyModel("NonFilterableString", typeof(string)), + + new VectorPropertyModel("VectorDefaultIndexingOptions", typeof(ReadOnlyMemory)) { Dimensions = 10, EmbeddingType = typeof(ReadOnlyMemory) }, + new VectorPropertyModel("VectorSpecificIndexingOptions", typeof(ReadOnlyMemory)) + { + Dimensions = 20, + IndexKind = IndexKind.Flat, + DistanceFunction = DistanceFunction.EuclideanSquaredDistance, + StorageName = "vector_specific_indexing_options", + EmbeddingType = typeof(ReadOnlyMemory) + } + ]; + + // Act. + var schema = RedisCollectionCreateMapping.MapToSchema(properties, useDollarPrefix); + + // Assert. + Assert.NotNull(schema); + Assert.Equal(8, schema.Fields.Count); + + Assert.IsType(schema.Fields[0]); + Assert.IsType(schema.Fields[1]); + Assert.IsType(schema.Fields[2]); + Assert.IsType(schema.Fields[3]); + Assert.IsType(schema.Fields[4]); + Assert.IsType(schema.Fields[5]); + Assert.IsType(schema.Fields[6]); + Assert.IsType(schema.Fields[7]); + + if (useDollarPrefix) + { + VerifyFieldName(schema.Fields[0].FieldName, ["$.FilterableString", "AS", "FilterableString"]); + VerifyFieldName(schema.Fields[1].FieldName, ["$.FullTextSearchableString", "AS", "FullTextSearchableString"]); + VerifyFieldName(schema.Fields[2].FieldName, ["$.FilterableStringEnumerable.*", "AS", "FilterableStringEnumerable"]); + VerifyFieldName(schema.Fields[3].FieldName, ["$.FullTextSearchableStringEnumerable", "AS", "FullTextSearchableStringEnumerable"]); + + VerifyFieldName(schema.Fields[4].FieldName, ["$.FilterableInt", "AS", "FilterableInt"]); + VerifyFieldName(schema.Fields[5].FieldName, ["$.FilterableNullableInt", "AS", "FilterableNullableInt"]); + + VerifyFieldName(schema.Fields[6].FieldName, ["$.VectorDefaultIndexingOptions", "AS", "VectorDefaultIndexingOptions"]); + VerifyFieldName(schema.Fields[7].FieldName, ["$.vector_specific_indexing_options", "AS", "vector_specific_indexing_options"]); + } + else + { + VerifyFieldName(schema.Fields[0].FieldName, ["FilterableString", "AS", "FilterableString"]); + VerifyFieldName(schema.Fields[1].FieldName, ["FullTextSearchableString", "AS", "FullTextSearchableString"]); + VerifyFieldName(schema.Fields[2].FieldName, ["FilterableStringEnumerable.*", "AS", "FilterableStringEnumerable"]); + VerifyFieldName(schema.Fields[3].FieldName, ["FullTextSearchableStringEnumerable", "AS", "FullTextSearchableStringEnumerable"]); + + VerifyFieldName(schema.Fields[4].FieldName, ["FilterableInt", "AS", "FilterableInt"]); + VerifyFieldName(schema.Fields[5].FieldName, ["FilterableNullableInt", "AS", "FilterableNullableInt"]); + + VerifyFieldName(schema.Fields[6].FieldName, ["VectorDefaultIndexingOptions", "AS", "VectorDefaultIndexingOptions"]); + VerifyFieldName(schema.Fields[7].FieldName, ["vector_specific_indexing_options", "AS", "vector_specific_indexing_options"]); + } + + Assert.Equal("10", ((VectorField)schema.Fields[6]).Attributes!["DIM"]); + Assert.Equal("FLOAT32", ((VectorField)schema.Fields[6]).Attributes!["TYPE"]); + Assert.Equal("COSINE", ((VectorField)schema.Fields[6]).Attributes!["DISTANCE_METRIC"]); + + Assert.Equal("20", ((VectorField)schema.Fields[7]).Attributes!["DIM"]); + Assert.Equal("FLOAT32", ((VectorField)schema.Fields[7]).Attributes!["TYPE"]); + Assert.Equal("L2", ((VectorField)schema.Fields[7]).Attributes!["DISTANCE_METRIC"]); + } + + [Fact] + public void GetSDKIndexKindThrowsOnUnsupportedIndexKind() + { + // Arrange. + var vectorProperty = new VectorPropertyModel("VectorProperty", typeof(ReadOnlyMemory)) { IndexKind = "Unsupported" }; + + // Act and assert. + Assert.Throws(() => RedisCollectionCreateMapping.GetSDKIndexKind(vectorProperty)); + } + + [Fact] + public void GetSDKDistanceAlgorithmThrowsOnUnsupportedDistanceFunction() + { + // Arrange. + var vectorProperty = new VectorPropertyModel("VectorProperty", typeof(ReadOnlyMemory)) { DistanceFunction = "Unsupported" }; + + // Act and assert. + Assert.Throws(() => RedisCollectionCreateMapping.GetSDKDistanceAlgorithm(vectorProperty)); + } + + private static void VerifyFieldName(FieldName fieldName, List expected) + { + var args = new List(); + fieldName.AddCommandArguments(args); + Assert.Equal(expected, args); + } +} diff --git a/MEVD/test/Redis.UnitTests/RedisCollectionSearchMappingTests.cs b/MEVD/test/Redis.UnitTests/RedisCollectionSearchMappingTests.cs new file mode 100644 index 0000000..78a4cd1 --- /dev/null +++ b/MEVD/test/Redis.UnitTests/RedisCollectionSearchMappingTests.cs @@ -0,0 +1,157 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Runtime.InteropServices; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; +using Xunit; + +namespace CommunityToolkit.VectorData.Redis.UnitTests; + +/// +/// Contains tests for the class. +/// +public class RedisCollectionSearchMappingTests +{ + [Fact] + public void ValidateVectorAndConvertToBytesConvertsFloatVector() + { + // Arrange. + var floatVector = new ReadOnlyMemory(new float[] { 1.0f, 2.0f, 3.0f }); + + // Act. + var byteArray = RedisCollectionSearchMapping.ValidateVectorAndConvertToBytes(floatVector, "Test"); + + // Assert. + Assert.NotNull(byteArray); + Assert.Equal(12, byteArray.Length); + Assert.Equal(new byte[12] { 0, 0, 128, 63, 0, 0, 0, 64, 0, 0, 64, 64 }, byteArray); + } + + [Fact] + public void ValidateVectorAndConvertToBytesConvertsDoubleVector() + { + // Arrange. + var doubleVector = new ReadOnlyMemory(new double[] { 1.0, 2.0, 3.0 }); + + // Act. + var byteArray = RedisCollectionSearchMapping.ValidateVectorAndConvertToBytes(doubleVector, "Test"); + + // Assert. + Assert.NotNull(byteArray); + Assert.Equal(24, byteArray.Length); + Assert.Equal(new byte[24] { 0, 0, 0, 0, 0, 0, 240, 63, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 8, 64 }, byteArray); + } + + [Fact] + public void ValidateVectorAndConvertToBytesThrowsForUnsupportedType() + { + // Arrange. + var unsupportedVector = new ReadOnlyMemory(new int[] { 1, 2, 3 }); + + // Act & Assert. + var exception = Assert.Throws(() => + { + var byteArray = RedisCollectionSearchMapping.ValidateVectorAndConvertToBytes(unsupportedVector, "Test"); + }); + Assert.Equal("The provided vector type System.ReadOnlyMemory`1[[System.Int32, System.Private.CoreLib, Version=10.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e]] is not supported by the Redis Test connector.", exception.Message); + } + + [Fact] + public void BuildQueryBuildsRedisQueryWithDefaults() + { + // Arrange. + var floatVector = new ReadOnlyMemory(new float[] { 1.0f, 2.0f, 3.0f }); + var byteArray = MemoryMarshal.AsBytes(floatVector.Span).ToArray(); + var model = BuildModel( + [ + new VectorStoreKeyProperty("Key", typeof(string)), + new VectorStoreVectorProperty("Vector", typeof(ReadOnlyMemory), 10) + ]); + + // Act. + var query = RedisCollectionSearchMapping.BuildQuery(byteArray, top: 3, new VectorSearchOptions(), model, model.VectorProperty, null); + + // Assert. + Assert.NotNull(query); + Assert.Equal("*=>[KNN 3 @Vector $embedding AS vector_score]", query.QueryString); + Assert.Equal("vector_score", query.SortBy); + Assert.True(query.WithScores); + Assert.Equal(2, query.dialect); + } + + [Fact] + public void BuildQueryBuildsRedisQueryWithCustomVectorName() + { + // Arrange. + var floatVector = new ReadOnlyMemory(new float[] { 1.0f, 2.0f, 3.0f }); + var byteArray = MemoryMarshal.AsBytes(floatVector.Span).ToArray(); + var vectorSearchOptions = new VectorSearchOptions { Skip = 3 }; + var model = BuildModel( + [ + new VectorStoreKeyProperty("Key", typeof(string)), + new VectorStoreVectorProperty("Vector", typeof(ReadOnlyMemory), 10) { StorageName = "storage_Vector" } + ]); + var selectFields = new string[] { "storage_Field1", "storage_Field2" }; + + // Act. + var query = RedisCollectionSearchMapping.BuildQuery(byteArray, top: 5, vectorSearchOptions, model, model.VectorProperty, selectFields); + + // Assert. + Assert.NotNull(query); + Assert.Equal("*=>[KNN 8 @storage_Vector $embedding AS vector_score]", query.QueryString); + } + + [Fact] + public void ResolveDistanceFunctionReturnsCosineSimilarityIfNoDistanceFunctionSpecified() + { + var property = new VectorPropertyModel("Prop", typeof(ReadOnlyMemory)); + + // Act. + var resolvedDistanceFunction = RedisCollectionSearchMapping.ResolveDistanceFunction(property); + + // Assert. + Assert.Equal(DistanceFunction.CosineSimilarity, resolvedDistanceFunction); + } + + [Fact] + public void ResolveDistanceFunctionReturnsDistanceFunctionFromProvidedProperty() + { + var property = new VectorPropertyModel("Prop", typeof(ReadOnlyMemory)) { DistanceFunction = DistanceFunction.DotProductSimilarity }; + + // Act. + var resolvedDistanceFunction = RedisCollectionSearchMapping.ResolveDistanceFunction(property); + + // Assert. + Assert.Equal(DistanceFunction.DotProductSimilarity, resolvedDistanceFunction); + } + + [Fact] + public void GetOutputScoreFromRedisScoreConvertsCosineDistanceToSimilarity() + { + // Act & Assert. + Assert.Equal(-1, RedisCollectionSearchMapping.GetOutputScoreFromRedisScore(2, DistanceFunction.CosineSimilarity)); + Assert.Equal(0, RedisCollectionSearchMapping.GetOutputScoreFromRedisScore(1, DistanceFunction.CosineSimilarity)); + Assert.Equal(1, RedisCollectionSearchMapping.GetOutputScoreFromRedisScore(0, DistanceFunction.CosineSimilarity)); + } + + [Theory] + [InlineData(DistanceFunction.CosineDistance, 2)] + [InlineData(DistanceFunction.DotProductSimilarity, 2)] + [InlineData(DistanceFunction.EuclideanSquaredDistance, 2)] + public void GetOutputScoreFromRedisScoreLeavesNonConsineSimilarityUntouched(string distanceFunction, float score) + { + // Act & Assert. + Assert.Equal(score, RedisCollectionSearchMapping.GetOutputScoreFromRedisScore(score, distanceFunction)); + } + +#pragma warning disable CA1812 // An internal class that is apparently never instantiated. If so, remove the code from the assembly. + private sealed class DummyType; +#pragma warning restore CA1812 + + private static CollectionModel BuildModel(List properties) + => new RedisModelBuilder(RedisHashSetCollection.ModelBuildingOptions) + .BuildDynamic(new() { Properties = properties }, defaultEmbeddingGenerator: null); +} diff --git a/MEVD/test/Redis.UnitTests/RedisFilterTranslatorTests.cs b/MEVD/test/Redis.UnitTests/RedisFilterTranslatorTests.cs new file mode 100644 index 0000000..15bf362 --- /dev/null +++ b/MEVD/test/Redis.UnitTests/RedisFilterTranslatorTests.cs @@ -0,0 +1,193 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Linq; +using System.Linq.Expressions; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; +using Xunit; + +namespace CommunityToolkit.VectorData.Redis.UnitTests; + +public sealed class RedisFilterTranslatorTests +{ + [Fact] + public void Contains_with_simple_string() + { + var result = Translate(r => r.Tags.Contains("foo")); + Assert.Equal("""@Tags:{"foo"}""", result); + } + + [Fact] + public void Contains_with_curly_brace_in_value() + { + var result = Translate(r => r.Tags.Contains("foo}bar")); + Assert.Equal("""@Tags:{"foo}bar"}""", result); + } + + [Fact] + public void Contains_with_pipe_in_value() + { + var result = Translate(r => r.Tags.Contains("foo|bar")); + Assert.Equal("""@Tags:{"foo|bar"}""", result); + } + + [Fact] + public void Contains_with_double_quote_in_value() + { + var result = Translate(r => r.Tags.Contains("foo\"bar")); + Assert.Equal("""@Tags:{"foo\"bar"}""", result); + } + + [Fact] + public void Contains_with_asterisk_in_value() + { + var result = Translate(r => r.Tags.Contains("foo*bar")); + Assert.Equal("""@Tags:{"foo*bar"}""", result); + } + + [Fact] + public void Contains_with_at_sign_in_value() + { + var result = Translate(r => r.Tags.Contains("foo@bar")); + Assert.Equal("""@Tags:{"foo@bar"}""", result); + } + + [Fact] + public void Contains_with_injection_attempt() + { + var result = Translate(r => r.Tags.Contains("evil} | @secret:{*")); + Assert.Equal("""@Tags:{"evil} | @secret:{*"}""", result); + } + + [Fact] + public void Any_with_simple_strings() + { + var values = new[] { "a", "b" }; + var result = Translate(r => r.Tags.Any(t => values.Contains(t))); + Assert.Equal("""@Tags:{"a" | "b"}""", result); + } + + [Fact] + public void Any_with_metacharacters_in_values() + { + var values = new[] { "a|b", "c}d" }; + var result = Translate(r => r.Tags.Any(t => values.Contains(t))); + Assert.Equal("""@Tags:{"a|b" | "c}d"}""", result); + } + + [Fact] + public void Any_with_double_quotes_in_values() + { + var values = new[] { "a\"b", "c\"d" }; + var result = Translate(r => r.Tags.Any(t => values.Contains(t))); + Assert.Equal("""@Tags:{"a\"b" | "c\"d"}""", result); + } + + [Fact] + public void Any_with_injection_attempt() + { + var values = new[] { "safe", "x} | @admin:{true" }; + var result = Translate(r => r.Tags.Any(t => values.Contains(t))); + Assert.Equal("""@Tags:{"safe" | "x} | @admin:{true"}""", result); + } + + [Fact] + public void Equal_with_simple_string() + { + var result = Translate(r => r.Name == "foo"); + Assert.Equal("""@Name:{"foo"}""", result); + } + + [Fact] + public void Equal_with_double_quote_in_value() + { + var result = Translate(r => r.Name == "foo\"bar"); + Assert.Equal("""@Name:{"foo\"bar"}""", result); + } + + [Fact] + public void Equal_with_backslash_in_value() + { + var result = Translate(r => r.Name == "foo\\bar"); + Assert.Equal("""@Name:{"foo\\bar"}""", result); + } + + [Fact] + public void Equal_with_single_backslash() + { + var result = Translate(r => r.Name == "\\"); + Assert.Equal("""@Name:{"\\"}""", result); + } + + [Fact] + public void Equal_with_backslash_quote_injection_attempt() + { + // Input: \" which should NOT break out of the quoted string + var result = Translate(r => r.Name == "\\\""); + Assert.Equal("""@Name:{"\\\""}""", result); + } + + [Fact] + public void Equal_with_backslash_quote_wildcard_injection() + { + // The specific attack payload: \" | * | \"[m + var result = Translate(r => r.Name == "\\\" | * | \\\""); + Assert.Equal("""@Name:{"\\\" | * | \\\""}""", result); + } + + [Fact] + public void Contains_with_backslash_in_value() + { + var result = Translate(r => r.Tags.Contains("foo\\bar")); + Assert.Equal("""@Tags:{"foo\\bar"}""", result); + } + + [Fact] + public void Contains_with_backslash_quote_injection_attempt() + { + var result = Translate(r => r.Tags.Contains("\\\"")); + Assert.Equal("""@Tags:{"\\\""}""", result); + } + + [Fact] + public void Any_with_backslash_in_values() + { + var values = new[] { "a\\b", "c\\d" }; + var result = Translate(r => r.Tags.Any(t => values.Contains(t))); + Assert.Equal("""@Tags:{"a\\b" | "c\\d"}""", result); + } + + private static string Translate(Expression> filter) + { + var model = BuildModel(); + return new RedisFilterTranslator().Translate(filter, model); + } + + private static CollectionModel BuildModel() + { + var definition = new VectorStoreCollectionDefinition + { + Properties = + [ + new VectorStoreKeyProperty("Id", typeof(string)), + new VectorStoreDataProperty("Name", typeof(string)), + new VectorStoreDataProperty("Tags", typeof(string[])), + new VectorStoreVectorProperty("Embedding", typeof(ReadOnlyMemory), 10) + ] + }; + + return new RedisJsonModelBuilder(RedisJsonCollection.ModelBuildingOptions).BuildDynamic(definition, defaultEmbeddingGenerator: null); + } + +#pragma warning disable CA1812 + private sealed record TestRecord + { + public string Id { get; set; } = string.Empty; + public string Name { get; set; } = string.Empty; + public string[] Tags { get; set; } = []; + public ReadOnlyMemory Embedding { get; set; } + } +#pragma warning restore CA1812 +} diff --git a/MEVD/test/Redis.UnitTests/RedisHashSetCollectionTests.cs b/MEVD/test/Redis.UnitTests/RedisHashSetCollectionTests.cs new file mode 100644 index 0000000..1c0a395 --- /dev/null +++ b/MEVD/test/Redis.UnitTests/RedisHashSetCollectionTests.cs @@ -0,0 +1,563 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.InteropServices; +using System.Text.Json.Serialization; +using System.Threading.Tasks; +using Microsoft.Extensions.VectorData; +using Moq; +using NRedisStack; +using StackExchange.Redis; +using Xunit; + +namespace CommunityToolkit.VectorData.Redis.UnitTests; + +/// +/// Contains tests for the class. +/// +public class RedisHashSetCollectionTests +{ + private const string TestCollectionName = "testcollection"; + private const string TestRecordKey1 = "testid1"; + private const string TestRecordKey2 = "testid2"; + + private readonly Mock _redisDatabaseMock; + + public RedisHashSetCollectionTests() + { + this._redisDatabaseMock = new Mock(MockBehavior.Strict); + this._redisDatabaseMock.Setup(l => l.Database).Returns(0); + + var batchMock = new Mock(); + this._redisDatabaseMock.Setup(x => x.CreateBatch(It.IsAny())).Returns(batchMock.Object); + } + + [Theory] + [InlineData(TestCollectionName, true)] + [InlineData("nonexistentcollection", false)] + public async Task CollectionExistsReturnsCollectionStateAsync(string collectionName, bool expectedExists) + { + // Arrange + if (expectedExists) + { + SetupExecuteMock(this._redisDatabaseMock, ["index_name", collectionName]); + } + else + { + SetupExecuteMock(this._redisDatabaseMock, new RedisServerException("Unknown index name")); + } + using var sut = new RedisHashSetCollection( + this._redisDatabaseMock.Object, + collectionName); + + // Act + var actual = await sut.CollectionExistsAsync(); + + // Assert + var expectedArgs = new object[] { collectionName }; + this._redisDatabaseMock + .Verify( + x => x.ExecuteAsync( + "FT.INFO", + It.Is>(x => x.SequenceEqual(expectedArgs)), It.IsAny()), + Times.Once); + Assert.Equal(expectedExists, actual); + } + + [Fact] + public async Task CanEnsureCollectionExistsAsync() + { + // Arrange. + SetupExecuteMock(this._redisDatabaseMock, "FT.INFO", new RedisServerException("Unknown index name")); + SetupExecuteMock(this._redisDatabaseMock, "FT.CREATE", string.Empty); + using var sut = new RedisHashSetCollection(this._redisDatabaseMock.Object, TestCollectionName); + + // Act. + await sut.EnsureCollectionExistsAsync(); + + // Assert. + var expectedArgs = new object[] { + "testcollection", + "PREFIX", + 1, + "testcollection:", + "SCHEMA", + "OriginalNameData", + "AS", + "OriginalNameData", + "TAG", + "data_storage_name", + "AS", + "data_storage_name", + "TAG", + "vector_storage_name", + "AS", + "vector_storage_name", + "VECTOR", + "HNSW", + 6, + "TYPE", + "FLOAT32", + "DIM", + "4", + "DISTANCE_METRIC", + "COSINE" }; + this._redisDatabaseMock + .Verify( + x => x.ExecuteAsync( + "FT.CREATE", + It.Is>(x => x.SequenceEqual(expectedArgs)), It.IsAny()), + Times.Once); + } + + [Fact] + public async Task CanDeleteCollectionAsync() + { + // Arrange + SetupExecuteMock(this._redisDatabaseMock, string.Empty); + using var sut = this.CreateRecordCollection(false); + + // Act + await sut.EnsureCollectionDeletedAsync(); + + // Assert + var expectedArgs = new object[] { TestCollectionName }; + this._redisDatabaseMock + .Verify( + x => x.ExecuteAsync( + "FT.DROPINDEX", + It.Is>(x => x.SequenceEqual(expectedArgs)), It.IsAny()), + Times.Once); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task CanGetRecordWithVectorsAsync(bool useDefinition) + { + // Arrange + var hashEntries = new HashEntry[] + { + new("OriginalNameData", "data 1"), + new("data_storage_name", "data 1"), + new("vector_storage_name", MemoryMarshal.AsBytes(new ReadOnlySpan(new float[] { 1, 2, 3, 4 })).ToArray()) + }; + this._redisDatabaseMock.Setup(x => x.HashGetAllAsync(It.IsAny(), CommandFlags.None)).ReturnsAsync(hashEntries); + using var sut = this.CreateRecordCollection(useDefinition); + + // Act + var actual = await sut.GetAsync( + TestRecordKey1, + new() { IncludeVectors = true }); + + // Assert + this._redisDatabaseMock.Verify(x => x.HashGetAllAsync(TestRecordKey1, CommandFlags.None), Times.Once); + + Assert.NotNull(actual); + Assert.Equal(TestRecordKey1, actual.Key); + Assert.Equal("data 1", actual.OriginalNameData); + Assert.Equal("data 1", actual.Data); + Assert.Equal(new float[] { 1, 2, 3, 4 }, actual.Vector!.Value.ToArray()); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task CanGetRecordWithoutVectorsAsync(bool useDefinition) + { + // Arrange + var redisValues = new RedisValue[] { new("data 1"), new("data 1") }; + this._redisDatabaseMock.Setup(x => x.HashGetAsync(It.IsAny(), It.IsAny(), CommandFlags.None)).ReturnsAsync(redisValues); + using var sut = this.CreateRecordCollection(useDefinition); + + // Act + var actual = await sut.GetAsync( + TestRecordKey1, + new() { IncludeVectors = false }); + + // Assert + var fieldNames = new RedisValue[] { "OriginalNameData", "data_storage_name" }; + this._redisDatabaseMock.Verify(x => x.HashGetAsync(TestRecordKey1, fieldNames, CommandFlags.None), Times.Once); + + Assert.NotNull(actual); + Assert.Equal(TestRecordKey1, actual.Key); + Assert.Equal("data 1", actual.OriginalNameData); + Assert.Equal("data 1", actual.Data); + Assert.False(actual.Vector.HasValue); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task CanGetManyRecordsWithVectorsAsync(bool useDefinition) + { + // Arrange + var hashEntries1 = new HashEntry[] + { + new("OriginalNameData", "data 1"), + new("data_storage_name", "data 1"), + new("vector_storage_name", MemoryMarshal.AsBytes(new ReadOnlySpan(new float[] { 1, 2, 3, 4 })).ToArray()) + }; + var hashEntries2 = new HashEntry[] + { + new("OriginalNameData", "data 2"), + new("data_storage_name", "data 2"), + new("vector_storage_name", MemoryMarshal.AsBytes(new ReadOnlySpan(new float[] { 5, 6, 7, 8 })).ToArray()) + }; + this._redisDatabaseMock.Setup(x => x.HashGetAllAsync(It.IsAny(), CommandFlags.None)).Returns((RedisKey key, CommandFlags flags) => + { + return key switch + { + RedisKey k when k == TestRecordKey1 => Task.FromResult(hashEntries1), + RedisKey k when k == TestRecordKey2 => Task.FromResult(hashEntries2), + _ => throw new ArgumentException("Unexpected key."), + }; + }); + using var sut = this.CreateRecordCollection(useDefinition); + + // Act + var actual = await sut.GetAsync( + [TestRecordKey1, TestRecordKey2], + new() { IncludeVectors = true }).ToListAsync(); + + // Assert + this._redisDatabaseMock.Verify(x => x.HashGetAllAsync(TestRecordKey1, CommandFlags.None), Times.Once); + this._redisDatabaseMock.Verify(x => x.HashGetAllAsync(TestRecordKey2, CommandFlags.None), Times.Once); + + Assert.NotNull(actual); + Assert.Equal(2, actual.Count); + Assert.Equal(TestRecordKey1, actual[0].Key); + Assert.Equal("data 1", actual[0].OriginalNameData); + Assert.Equal("data 1", actual[0].Data); + Assert.Equal(new float[] { 1, 2, 3, 4 }, actual[0].Vector!.Value.ToArray()); + Assert.Equal(TestRecordKey2, actual[1].Key); + Assert.Equal("data 2", actual[1].OriginalNameData); + Assert.Equal("data 2", actual[1].Data); + Assert.Equal(new float[] { 5, 6, 7, 8 }, actual[1].Vector!.Value.ToArray()); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task CanDeleteRecordAsync(bool useDefinition) + { + // Arrange + this._redisDatabaseMock.Setup(x => x.KeyDeleteAsync(It.IsAny(), CommandFlags.None)).ReturnsAsync(true); + using var sut = this.CreateRecordCollection(useDefinition); + + // Act + await sut.DeleteAsync(TestRecordKey1); + + // Assert + this._redisDatabaseMock.Verify(x => x.KeyDeleteAsync(TestRecordKey1, CommandFlags.None), Times.Once); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task CanDeleteManyRecordsWithVectorsAsync(bool useDefinition) + { + // Arrange + this._redisDatabaseMock.Setup(x => x.KeyDeleteAsync(It.IsAny(), CommandFlags.None)).ReturnsAsync(true); + using var sut = this.CreateRecordCollection(useDefinition); + + // Act + await sut.DeleteAsync([TestRecordKey1, TestRecordKey2]); + + // Assert + this._redisDatabaseMock.Verify(x => x.KeyDeleteAsync(TestRecordKey1, CommandFlags.None), Times.Once); + this._redisDatabaseMock.Verify(x => x.KeyDeleteAsync(TestRecordKey2, CommandFlags.None), Times.Once); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task CanUpsertRecordAsync(bool useDefinition) + { + // Arrange + this._redisDatabaseMock.Setup(x => x.HashSetAsync(It.IsAny(), It.IsAny(), CommandFlags.None)).Returns(Task.CompletedTask); + using var sut = this.CreateRecordCollection(useDefinition); + var model = CreateModel(TestRecordKey1, true); + + // Act + await sut.UpsertAsync(model); + + // Assert + this._redisDatabaseMock.Verify( + x => x.HashSetAsync( + TestRecordKey1, + It.Is(x => x.Length == 3 && x[0].Name == "OriginalNameData" && x[1].Name == "data_storage_name" && x[2].Name == "vector_storage_name"), + CommandFlags.None), + Times.Once); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task CanUpsertManyRecordsAsync(bool useDefinition) + { + // Arrange + this._redisDatabaseMock.Setup(x => x.HashSetAsync(It.IsAny(), It.IsAny(), CommandFlags.None)).Returns(Task.CompletedTask); + using var sut = this.CreateRecordCollection(useDefinition); + + var model1 = CreateModel(TestRecordKey1, true); + var model2 = CreateModel(TestRecordKey2, true); + + // Act + await sut.UpsertAsync([model1, model2]); + + // Assert + this._redisDatabaseMock.Verify( + x => x.HashSetAsync( + TestRecordKey1, + It.Is(x => x.Length == 3 && x[0].Name == "OriginalNameData" && x[1].Name == "data_storage_name" && x[2].Name == "vector_storage_name"), + CommandFlags.None), + Times.Once); + this._redisDatabaseMock.Verify( + x => x.HashSetAsync( + TestRecordKey2, + It.Is(x => x.Length == 3 && x[0].Name == "OriginalNameData" && x[1].Name == "data_storage_name" && x[2].Name == "vector_storage_name"), + CommandFlags.None), + Times.Once); + } + + [Theory] + [InlineData(true, true)] + [InlineData(true, false)] + [InlineData(false, true)] + [InlineData(false, false)] + public async Task CanSearchWithVectorAndFilterAsync(bool useDefinition, bool includeVectors) + { + // Arrange + SetupExecuteMock(this._redisDatabaseMock, new RedisResult[] + { + RedisResult.Create(new RedisValue("1")), + RedisResult.Create(new RedisValue(TestRecordKey1)), + RedisResult.Create(new RedisValue("0.8")), + RedisResult.Create( + [ + new RedisValue("OriginalNameData"), + new RedisValue("original data 1"), + new RedisValue("data_storage_name"), + new RedisValue("data 1"), + new RedisValue("vector_storage_name"), + RedisValue.Unbox(MemoryMarshal.AsBytes(new ReadOnlySpan(new float[] { 1, 2, 3, 4 })).ToArray()), + new RedisValue("vector_score"), + new RedisValue("0.25"), + ]), + }); + using var sut = this.CreateRecordCollection(useDefinition); + + // Act. + var results = await sut.SearchAsync( + new ReadOnlyMemory(new[] { 1f, 2f, 3f, 4f }), + top: 5, + new() + { + IncludeVectors = includeVectors, + Filter = r => r.Data == "data 1", + Skip = 2 + }).ToListAsync(); + + // Assert. + var expectedArgsPart1 = new object[] + { + "testcollection", + "@data_storage_name:{\"data 1\"}=>[KNN 7 @vector_storage_name $embedding AS vector_score]", + "WITHSCORES", + "SORTBY", + "vector_score", + "LIMIT", + 2, + 7 + }; + var returnArgs = includeVectors ? Array.Empty() : new object[] + { + "RETURN", + 3, + "OriginalNameData", + "data_storage_name", + "vector_score" + }; + var expectedArgsPart2 = new object[] + { + "PARAMS", + 2, + "embedding", + MemoryMarshal.AsBytes(new ReadOnlySpan(new float[] { 1, 2, 3, 4 })).ToArray(), + "DIALECT", + 2 + }; + var expectedArgs = expectedArgsPart1.Concat(returnArgs).Concat(expectedArgsPart2).ToArray(); + + this._redisDatabaseMock + .Verify( + x => x.ExecuteAsync( + "FT.SEARCH", + It.Is>(x => x.Where(y => !(y is byte[])).SequenceEqual(expectedArgs.Where(y => !(y is byte[])))), It.IsAny()), + Times.Once); + + Assert.Single(results); + Assert.Equal(TestRecordKey1, results.First().Record.Key); + Assert.Equal(0.25d, results.First().Score); + Assert.Equal("original data 1", results.First().Record.OriginalNameData); + Assert.Equal("data 1", results.First().Record.Data); + if (includeVectors) + { + Assert.Equal(new float[] { 1, 2, 3, 4 }, results.First().Record.Vector!.Value.ToArray()); + } + else + { + Assert.False(results.First().Record.Vector.HasValue); + } + } + + /// + /// Tests that the collection can be created even if the definition and the type do not match. + /// In this case, the expectation is that a custom mapper will be provided to map between the + /// schema as defined by the definition and the different data model. + /// + [Fact] + public void CanCreateCollectionWithMismatchedDefinitionAndType() + { + // Arrange. + var definition = new VectorStoreCollectionDefinition() + { + Properties = + [ + new VectorStoreKeyProperty(nameof(SinglePropsModel.Key), typeof(string)), + new VectorStoreDataProperty(nameof(SinglePropsModel.OriginalNameData), typeof(string)), + new VectorStoreVectorProperty(nameof(SinglePropsModel.Vector), typeof(ReadOnlyMemory?), 4), + ] + }; + + // Act. + using var sut = new RedisHashSetCollection( + this._redisDatabaseMock.Object, + TestCollectionName, + new() { Definition = definition }); + } + + private RedisHashSetCollection CreateRecordCollection(bool useDefinition) + { + return new RedisHashSetCollection( + this._redisDatabaseMock.Object, + TestCollectionName, + new() + { + PrefixCollectionNameToKeyNames = false, + Definition = useDefinition ? this._singlePropsDefinition : null + }); + } + + private static void SetupExecuteMock(Mock redisDatabaseMock, Exception exception) + { + redisDatabaseMock + .Setup( + x => x.ExecuteAsync( + It.IsAny(), + It.IsAny>(), It.IsAny())) + .ThrowsAsync(exception); + } + + private static void SetupExecuteMock(Mock redisDatabaseMock, string command, Exception exception) + { + redisDatabaseMock + .Setup( + x => x.ExecuteAsync( + command, + It.IsAny>(), It.IsAny())) + .ThrowsAsync(exception); + } + + private static void SetupExecuteMock(Mock redisDatabaseMock, IEnumerable redisResultStrings) + { + var results = redisResultStrings + .Select(x => RedisResult.Create(new RedisValue(x))) + .ToArray(); + redisDatabaseMock + .Setup( + x => x.ExecuteAsync( + It.IsAny(), + It.IsAny>(), It.IsAny())) + .ReturnsAsync(RedisResult.Create(results)); + } + + private static void SetupExecuteMock(Mock redisDatabaseMock, IEnumerable redisResultStrings) + { + var results = redisResultStrings + .Select(x => x) + .ToArray(); + redisDatabaseMock + .Setup( + x => x.ExecuteAsync( + It.IsAny(), + It.IsAny>(), It.IsAny())) + .ReturnsAsync(RedisResult.Create(results)); + } + + private static void SetupExecuteMock(Mock redisDatabaseMock, string redisResultString) + { + redisDatabaseMock + .Setup( + x => x.ExecuteAsync( + It.IsAny(), + It.IsAny>(), It.IsAny())) + .ReturnsAsync(RedisResult.Create(new RedisValue(redisResultString))); + } + + private static void SetupExecuteMock(Mock redisDatabaseMock, string command, string redisResultString) + { + redisDatabaseMock + .Setup( + x => x.ExecuteAsync( + command, + It.IsAny>(), It.IsAny())) + .ReturnsAsync(RedisResult.Create(new RedisValue(redisResultString))); + } + + private static SinglePropsModel CreateModel(string key, bool withVectors) + { + return new SinglePropsModel + { + Key = key, + OriginalNameData = "data 1", + Data = "data 1", + Vector = withVectors ? new float[] { 1, 2, 3, 4 } : null, + NotAnnotated = null, + }; + } + + private readonly VectorStoreCollectionDefinition _singlePropsDefinition = new() + { + Properties = + [ + new VectorStoreKeyProperty("Key", typeof(string)), + new VectorStoreDataProperty("OriginalNameData", typeof(string)), + new VectorStoreDataProperty("Data", typeof(string)) { StorageName = "data_storage_name" }, + new VectorStoreVectorProperty("Vector", typeof(ReadOnlyMemory), 10) { StorageName = "vector_storage_name", DistanceFunction = DistanceFunction.CosineDistance } + ] + }; + + public sealed class SinglePropsModel + { + [VectorStoreKey] + public string Key { get; set; } = string.Empty; + + [VectorStoreData(IsIndexed = true)] + public string OriginalNameData { get; set; } = string.Empty; + + [JsonPropertyName("ignored_data_json_name")] + [VectorStoreData(IsIndexed = true, StorageName = "data_storage_name")] + public string Data { get; set; } = string.Empty; + + [JsonPropertyName("ignored_vector_json_name")] + [VectorStoreVector(dimensions: 4, DistanceFunction = DistanceFunction.CosineDistance, StorageName = "vector_storage_name")] + public ReadOnlyMemory? Vector { get; set; } + + public string? NotAnnotated { get; set; } + } +} diff --git a/MEVD/test/Redis.UnitTests/RedisHashSetDynamicMappingTests.cs b/MEVD/test/Redis.UnitTests/RedisHashSetDynamicMappingTests.cs new file mode 100644 index 0000000..58ead60 --- /dev/null +++ b/MEVD/test/Redis.UnitTests/RedisHashSetDynamicMappingTests.cs @@ -0,0 +1,209 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; +using StackExchange.Redis; +using Xunit; + +namespace CommunityToolkit.VectorData.Redis.UnitTests; + +/// +/// Contains dynamic mapping tests for the class. +/// +public class RedisHashSetDynamicMappingTests +{ + private static readonly CollectionModel s_model = BuildModel(RedisHashSetMappingTestHelpers.s_definition); + + private static readonly float[] s_floatVector = new float[] { 1.0f, 2.0f, 3.0f, 4.0f }; + private static readonly double[] s_doubleVector = new double[] { 5.0d, 6.0d, 7.0d, 8.0d }; + + [Fact] + public void MapFromDataToStorageModelMapsAllSupportedTypes() + { + // Arrange. + var sut = new RedisHashSetMapper>(s_model); + var dataModel = new Dictionary + { + ["Key"] = "key", + + ["StringData"] = "data 1", + ["IntData"] = 1, + ["UIntData"] = 2u, + ["LongData"] = 3L, + ["ULongData"] = 4ul, + ["DoubleData"] = 5.5d, + ["FloatData"] = 6.6f, + ["NullableIntData"] = 7, + ["NullableUIntData"] = 8u, + ["NullableLongData"] = 9L, + ["NullableULongData"] = 10ul, + ["NullableDoubleData"] = 11.1d, + ["NullableFloatData"] = 12.2f, + + ["FloatVector"] = new ReadOnlyMemory(s_floatVector), + ["DoubleVector"] = new ReadOnlyMemory(s_doubleVector), + }; + + // Act. + var storageModel = sut.MapFromDataToStorageModel(dataModel, recordIndex: 0, generatedEmbeddings: null); + + // Assert + Assert.Equal("key", storageModel.Key); + RedisHashSetMappingTestHelpers.VerifyHashSet(storageModel.HashEntries); + } + + [Fact] + public void MapFromDataToStorageModelMapsNullValues() + { + // Arrange + CollectionModel model = BuildModel(new() + { + Properties = + [ + new VectorStoreKeyProperty("Key", typeof(string)), + new VectorStoreDataProperty("StringData", typeof(string)) { StorageName = "storage_string_data" }, + new VectorStoreDataProperty("NullableIntData", typeof(int?)), + new VectorStoreVectorProperty("FloatVector", typeof(ReadOnlyMemory?), 10), + ], + }); + + var dataModel = new Dictionary + { + ["Key"] = "key", + ["StringData"] = null, + ["NullableIntData"] = null, + ["FloatVector"] = null, + }; + + var sut = new RedisHashSetMapper>(model); + + // Act + var storageModel = sut.MapFromDataToStorageModel(dataModel, recordIndex: 0, generatedEmbeddings: null); + + // Assert + Assert.Equal("key", storageModel.Key); + + Assert.Equal("storage_string_data", storageModel.HashEntries[0].Name.ToString()); + Assert.True(storageModel.HashEntries[0].Value.IsNull); + + Assert.Equal("NullableIntData", storageModel.HashEntries[1].Name.ToString()); + Assert.True(storageModel.HashEntries[1].Value.IsNull); + } + + [Fact] + public void MapFromStorageToDataModelMapsAllSupportedTypes() + { + // Arrange. + var hashSet = RedisHashSetMappingTestHelpers.CreateHashSet(); + var sut = new RedisHashSetMapper>(s_model); + + // Act. + var dataModel = sut.MapFromStorageToDataModel(("key", hashSet), includeVectors: true); + + // Assert. + Assert.Equal("key", dataModel["Key"]); + Assert.Equal("data 1", dataModel["StringData"]); + Assert.Equal(1, dataModel["IntData"]); + Assert.Equal(2u, dataModel["UIntData"]); + Assert.Equal(3L, dataModel["LongData"]); + Assert.Equal(4ul, dataModel["ULongData"]); + Assert.Equal(5.5d, dataModel["DoubleData"]); + Assert.Equal(6.6f, dataModel["FloatData"]); + Assert.Equal(7, dataModel["NullableIntData"]); + Assert.Equal(8u, dataModel["NullableUIntData"]); + Assert.Equal(9L, dataModel["NullableLongData"]); + Assert.Equal(10ul, dataModel["NullableULongData"]); + Assert.Equal(11.1d, dataModel["NullableDoubleData"]); + Assert.Equal(12.2f, dataModel["NullableFloatData"]); + Assert.Equal(new float[] { 1, 2, 3, 4 }, ((ReadOnlyMemory)dataModel["FloatVector"]!).ToArray()); + Assert.Equal(new double[] { 5, 6, 7, 8 }, ((ReadOnlyMemory)dataModel["DoubleVector"]!).ToArray()); + } + + [Fact] + public void MapFromStorageToDataModelMapsNullValues() + { + // Arrange + var model = BuildModel(new() + { + Properties = + [ + new VectorStoreKeyProperty("Key", typeof(string)), + new VectorStoreDataProperty("StringData", typeof(string)) { StorageName = "storage_string_data" }, + new VectorStoreDataProperty("NullableIntData", typeof(int?)), + new VectorStoreVectorProperty("FloatVector", typeof(ReadOnlyMemory?), 10), + ] + }); + + var hashSet = new HashEntry[] + { + new("storage_string_data", RedisValue.Null), + new("NullableIntData", RedisValue.Null), + new("FloatVector", RedisValue.Null), + }; + + var sut = new RedisHashSetMapper>(model); + + // Act + var dataModel = sut.MapFromStorageToDataModel(("key", hashSet), includeVectors: true); + + // Assert + Assert.Equal("key", dataModel["Key"]); + Assert.Null(dataModel["StringData"]); + Assert.Null(dataModel["NullableIntData"]); + Assert.Null(dataModel["FloatVector"]); + } + + [Fact] + public void MapFromDataToStorageModelSkipsMissingProperties() + { + // Arrange. + var model = BuildModel(new() + { + Properties = + [ + new VectorStoreKeyProperty("Key", typeof(string)), + new VectorStoreDataProperty("StringData", typeof(string)) { StorageName = "storage_string_data" }, + new VectorStoreDataProperty("NullableIntData", typeof(int?)), + new VectorStoreVectorProperty("FloatVector", typeof(ReadOnlyMemory?), 10), + ] + }); + + var sut = new RedisHashSetMapper>(model); + var dataModel = new Dictionary { ["Key"] = "key" }; + + // Act. + var storageModel = sut.MapFromDataToStorageModel(dataModel, recordIndex: 0, generatedEmbeddings: null); + + // Assert + Assert.Equal("key", storageModel.Key); + + Assert.Equal("storage_string_data", storageModel.HashEntries[0].Name.ToString()); + Assert.True(storageModel.HashEntries[0].Value.IsNull); + + Assert.Equal("NullableIntData", storageModel.HashEntries[1].Name.ToString()); + Assert.True(storageModel.HashEntries[1].Value.IsNull); + } + + [Fact] + public void MapFromStorageToDataModelSkipsMissingProperties() + { + // Arrange. + var hashSet = Array.Empty(); + + var sut = new RedisHashSetMapper>(s_model); + + // Act. + var dataModel = sut.MapFromStorageToDataModel(("key", hashSet), includeVectors: true); + + // Assert. + Assert.Single(dataModel); + Assert.Equal("key", dataModel["Key"]); + } + + private static CollectionModel BuildModel(VectorStoreCollectionDefinition definition) + => new RedisModelBuilder(RedisHashSetCollection>.ModelBuildingOptions) + .BuildDynamic(definition, defaultEmbeddingGenerator: null); +} diff --git a/MEVD/test/Redis.UnitTests/RedisHashSetMapperTests.cs b/MEVD/test/Redis.UnitTests/RedisHashSetMapperTests.cs new file mode 100644 index 0000000..d9db5c5 --- /dev/null +++ b/MEVD/test/Redis.UnitTests/RedisHashSetMapperTests.cs @@ -0,0 +1,143 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; +using CommunityToolkit.VectorData.Redis; +using CommunityToolkit.VectorData.Redis.UnitTests; +using Xunit; + +namespace SemanticKernel.Connectors.Redis.UnitTests; + +/// +/// Contains tests for the class. +/// +public sealed class RedisHashSetMapperTests +{ + private static readonly CollectionModel s_model + = new RedisModelBuilder(RedisHashSetCollection.ModelBuildingOptions) + .Build(typeof(AllTypesModel), typeof(string), RedisHashSetMappingTestHelpers.s_definition, defaultEmbeddingGenerator: null); + + [Fact] + public void MapsAllFieldsFromDataToStorageModel() + { + // Arrange. + var sut = new RedisHashSetMapper(s_model); + + // Act. + var actual = sut.MapFromDataToStorageModel(CreateModel("test key"), recordIndex: 0, generatedEmbeddings: null); + + // Assert. + Assert.NotNull(actual.HashEntries); + Assert.Equal("test key", actual.Key); + RedisHashSetMappingTestHelpers.VerifyHashSet(actual.HashEntries); + } + + [Fact] + public void MapsAllFieldsFromStorageToDataModel() + { + // Arrange. + var sut = new RedisHashSetMapper(s_model); + + // Act. + var actual = sut.MapFromStorageToDataModel(("test key", RedisHashSetMappingTestHelpers.CreateHashSet()), includeVectors: true); + + // Assert. + Assert.NotNull(actual); + Assert.Equal("test key", actual.Key); + Assert.Equal("data 1", actual.StringData); + Assert.Equal(1, actual.IntData); + Assert.Equal(2u, actual.UIntData); + Assert.Equal(3, actual.LongData); + Assert.Equal(4ul, actual.ULongData); + Assert.Equal(5.5d, actual.DoubleData); + Assert.Equal(6.6f, actual.FloatData); + Assert.Equal(7, actual.NullableIntData); + Assert.Equal(8u, actual.NullableUIntData); + Assert.Equal(9, actual.NullableLongData); + Assert.Equal(10ul, actual.NullableULongData); + Assert.Equal(11.1d, actual.NullableDoubleData); + Assert.Equal(12.2f, actual.NullableFloatData); + + Assert.Equal(new float[] { 1, 2, 3, 4 }, actual.FloatVector!.Value.ToArray()); + Assert.Equal(new double[] { 5, 6, 7, 8 }, actual.DoubleVector!.Value.ToArray()); + } + + private static AllTypesModel CreateModel(string key) + { + return new AllTypesModel + { + Key = key, + StringData = "data 1", + IntData = 1, + UIntData = 2, + LongData = 3, + ULongData = 4, + DoubleData = 5.5d, + FloatData = 6.6f, + NullableIntData = 7, + NullableUIntData = 8, + NullableLongData = 9, + NullableULongData = 10, + NullableDoubleData = 11.1d, + NullableFloatData = 12.2f, + FloatVector = new float[] { 1, 2, 3, 4 }, + DoubleVector = new double[] { 5, 6, 7, 8 }, + NotAnnotated = "notAnnotated", + }; + } + + private sealed class AllTypesModel + { + [VectorStoreKey] + public string Key { get; set; } = string.Empty; + + [VectorStoreData(StorageName = "storage_string_data")] + public string StringData { get; set; } = string.Empty; + + [VectorStoreData] + public int IntData { get; set; } + + [VectorStoreData] + public uint UIntData { get; set; } + + [VectorStoreData] + public long LongData { get; set; } + + [VectorStoreData] + public ulong ULongData { get; set; } + + [VectorStoreData] + public double DoubleData { get; set; } + + [VectorStoreData] + public float FloatData { get; set; } + + [VectorStoreData] + public int? NullableIntData { get; set; } + + [VectorStoreData] + public uint? NullableUIntData { get; set; } + + [VectorStoreData] + public long? NullableLongData { get; set; } + + [VectorStoreData] + public ulong? NullableULongData { get; set; } + + [VectorStoreData] + public double? NullableDoubleData { get; set; } + + [VectorStoreData] + public float? NullableFloatData { get; set; } + + [VectorStoreVector(dimensions: 10)] + public ReadOnlyMemory? FloatVector { get; set; } + + [VectorStoreVector(dimensions: 10)] + public ReadOnlyMemory? DoubleVector { get; set; } + + public string NotAnnotated { get; set; } = string.Empty; + } +} diff --git a/MEVD/test/Redis.UnitTests/RedisHashSetMappingTestHelpers.cs b/MEVD/test/Redis.UnitTests/RedisHashSetMappingTestHelpers.cs new file mode 100644 index 0000000..1b5369d --- /dev/null +++ b/MEVD/test/Redis.UnitTests/RedisHashSetMappingTestHelpers.cs @@ -0,0 +1,109 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Runtime.InteropServices; +using Microsoft.Extensions.VectorData; +using StackExchange.Redis; +using Xunit; + +namespace CommunityToolkit.VectorData.Redis.UnitTests; + +/// +/// Contains helper methods and data for testing the mapping of records between storage and data models. +/// These helpers are shared between the different mapping tests. +/// +internal static class RedisHashSetMappingTestHelpers +{ + public static readonly VectorStoreCollectionDefinition s_definition = new() + { + Properties = + [ + new VectorStoreKeyProperty("Key", typeof(string)), + new VectorStoreDataProperty("StringData", typeof(string)) { StorageName = "storage_string_data" }, + new VectorStoreDataProperty("IntData", typeof(int)), + new VectorStoreDataProperty("UIntData", typeof(uint)), + new VectorStoreDataProperty("LongData", typeof(long)), + new VectorStoreDataProperty("ULongData", typeof(ulong)), + new VectorStoreDataProperty("DoubleData", typeof(double)), + new VectorStoreDataProperty("FloatData", typeof(float)), + new VectorStoreDataProperty("NullableIntData", typeof(int?)), + new VectorStoreDataProperty("NullableUIntData", typeof(uint?)), + new VectorStoreDataProperty("NullableLongData", typeof(long?)), + new VectorStoreDataProperty("NullableULongData", typeof(ulong?)), + new VectorStoreDataProperty("NullableDoubleData", typeof(double?)), + new VectorStoreDataProperty("NullableFloatData", typeof(float?)), + new VectorStoreVectorProperty("FloatVector", typeof(ReadOnlyMemory), 10), + new VectorStoreVectorProperty("DoubleVector", typeof(ReadOnlyMemory), 10), + ] + }; + + public static HashEntry[] CreateHashSet() + { + var hashSet = new HashEntry[15]; + hashSet[0] = new HashEntry("storage_string_data", "data 1"); + hashSet[1] = new HashEntry("IntData", 1); + hashSet[2] = new HashEntry("UIntData", 2); + hashSet[3] = new HashEntry("LongData", 3); + hashSet[4] = new HashEntry("ULongData", 4); + hashSet[5] = new HashEntry("DoubleData", 5.5); + hashSet[6] = new HashEntry("FloatData", 6.6); + hashSet[7] = new HashEntry("NullableIntData", 7); + hashSet[8] = new HashEntry("NullableUIntData", 8); + hashSet[9] = new HashEntry("NullableLongData", 9); + hashSet[10] = new HashEntry("NullableULongData", 10); + hashSet[11] = new HashEntry("NullableDoubleData", 11.1); + hashSet[12] = new HashEntry("NullableFloatData", 12.2); + hashSet[13] = new HashEntry("FloatVector", MemoryMarshal.AsBytes(new ReadOnlySpan(new float[] { 1, 2, 3, 4 })).ToArray()); + hashSet[14] = new HashEntry("DoubleVector", MemoryMarshal.AsBytes(new ReadOnlySpan(new double[] { 5, 6, 7, 8 })).ToArray()); + return hashSet; + } + + public static void VerifyHashSet(HashEntry[] hashEntries) + { + Assert.Equal("storage_string_data", hashEntries[0].Name.ToString()); + Assert.Equal("data 1", hashEntries[0].Value.ToString()); + + Assert.Equal("IntData", hashEntries[1].Name.ToString()); + Assert.Equal(1, (int)hashEntries[1].Value); + + Assert.Equal("UIntData", hashEntries[2].Name.ToString()); + Assert.Equal(2u, (uint)hashEntries[2].Value); + + Assert.Equal("LongData", hashEntries[3].Name.ToString()); + Assert.Equal(3, (long)hashEntries[3].Value); + + Assert.Equal("ULongData", hashEntries[4].Name.ToString()); + Assert.Equal(4ul, (ulong)hashEntries[4].Value); + + Assert.Equal("DoubleData", hashEntries[5].Name.ToString()); + Assert.Equal(5.5d, (double)hashEntries[5].Value); + + Assert.Equal("FloatData", hashEntries[6].Name.ToString()); + Assert.Equal(6.6f, (float)hashEntries[6].Value); + + Assert.Equal("NullableIntData", hashEntries[7].Name.ToString()); + Assert.Equal(7, (int)hashEntries[7].Value); + + Assert.Equal("NullableUIntData", hashEntries[8].Name.ToString()); + Assert.Equal(8u, (uint)hashEntries[8].Value); + + Assert.Equal("NullableLongData", hashEntries[9].Name.ToString()); + Assert.Equal(9, (long)hashEntries[9].Value); + + Assert.Equal("NullableULongData", hashEntries[10].Name.ToString()); + Assert.Equal(10ul, (ulong)hashEntries[10].Value); + + Assert.Equal("NullableDoubleData", hashEntries[11].Name.ToString()); + Assert.Equal(11.1d, (double)hashEntries[11].Value); + + Assert.Equal("NullableFloatData", hashEntries[12].Name.ToString()); + Assert.Equal(12.2f, (float)hashEntries[12].Value); + + Assert.Equal("FloatVector", hashEntries[13].Name.ToString()); + Assert.Equal(new float[] { 1, 2, 3, 4 }, MemoryMarshal.Cast((byte[])hashEntries[13].Value!).ToArray()); + + Assert.Equal("DoubleVector", hashEntries[14].Name.ToString()); + Assert.Equal(new double[] { 5, 6, 7, 8 }, MemoryMarshal.Cast((byte[])hashEntries[14].Value!).ToArray()); + } +} diff --git a/MEVD/test/Redis.UnitTests/RedisJsonCollectionTests.cs b/MEVD/test/Redis.UnitTests/RedisJsonCollectionTests.cs new file mode 100644 index 0000000..0d4944a --- /dev/null +++ b/MEVD/test/Redis.UnitTests/RedisJsonCollectionTests.cs @@ -0,0 +1,558 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.InteropServices; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading.Tasks; +using Microsoft.Extensions.VectorData; +using Moq; +using NRedisStack; +using StackExchange.Redis; +using Xunit; + +namespace CommunityToolkit.VectorData.Redis.UnitTests; + +/// +/// Contains tests for the class. +/// +public class RedisJsonCollectionTests +{ + private const string TestCollectionName = "testcollection"; + private const string TestRecordKey1 = "testid1"; + private const string TestRecordKey2 = "testid2"; + + private readonly Mock _redisDatabaseMock; + + public RedisJsonCollectionTests() + { + this._redisDatabaseMock = new Mock(MockBehavior.Strict); + this._redisDatabaseMock.Setup(l => l.Database).Returns(0); + + var batchMock = new Mock(); + this._redisDatabaseMock.Setup(x => x.CreateBatch(It.IsAny())).Returns(batchMock.Object); + } + + [Theory] + [InlineData(TestCollectionName, true)] + [InlineData("nonexistentcollection", false)] + public async Task CollectionExistsReturnsCollectionStateAsync(string collectionName, bool expectedExists) + { + // Arrange + if (expectedExists) + { + SetupExecuteMock(this._redisDatabaseMock, ["index_name", collectionName]); + } + else + { + SetupExecuteMock(this._redisDatabaseMock, new RedisServerException("Unknown index name")); + } + using var sut = new RedisJsonCollection( + this._redisDatabaseMock.Object, + collectionName); + + // Act + var actual = await sut.CollectionExistsAsync(); + + // Assert + var expectedArgs = new object[] { collectionName }; + this._redisDatabaseMock + .Verify( + x => x.ExecuteAsync( + "FT.INFO", + It.Is>(x => x.SequenceEqual(expectedArgs)), It.IsAny()), + Times.Once); + Assert.Equal(expectedExists, actual); + } + + [Theory] + [InlineData(true, true, "data2", "vector2")] + [InlineData(true, false, "Data2", "Vector2")] + [InlineData(false, true, "data2", "vector2")] + [InlineData(false, false, "Data2", "Vector2")] + public async Task CanEnsureCollectionExistsAsync(bool useDefinition, bool useCustomJsonSerializerOptions, string expectedData2Name, string expectedVector2Name) + { + // Arrange. + SetupExecuteMock(this._redisDatabaseMock, "FT.INFO", new RedisServerException("Unknown index name")); + SetupExecuteMock(this._redisDatabaseMock, "FT.CREATE", string.Empty); + using var sut = this.CreateRecordCollection(useDefinition, useCustomJsonSerializerOptions); + + // Act. + await sut.EnsureCollectionExistsAsync(); + + // Assert. + var expectedArgs = new object[] { + "testcollection", + "ON", + "JSON", + "PREFIX", + 1, + "testcollection:", + "SCHEMA", + "$.data1_json_name", + "AS", + "data1_json_name", + "TAG", + $"$.{expectedData2Name}", + "AS", + expectedData2Name, + "TAG", + "$.vector1_json_name", + "AS", + "vector1_json_name", + "VECTOR", + "HNSW", + 6, + "TYPE", + "FLOAT32", + "DIM", + "4", + "DISTANCE_METRIC", + "COSINE", + $"$.{expectedVector2Name}", + "AS", + expectedVector2Name, + "VECTOR", + "HNSW", + 6, + "TYPE", + "FLOAT32", + "DIM", + "4", + "DISTANCE_METRIC", + "COSINE" }; + this._redisDatabaseMock + .Verify( + x => x.ExecuteAsync( + "FT.CREATE", + It.Is>(x => x.SequenceEqual(expectedArgs)), It.IsAny()), + Times.Once); + } + + [Fact] + public async Task CanDeleteCollectionAsync() + { + // Arrange + SetupExecuteMock(this._redisDatabaseMock, string.Empty); + using var sut = this.CreateRecordCollection(false); + + // Act + await sut.EnsureCollectionDeletedAsync(); + + // Assert + var expectedArgs = new object[] { TestCollectionName }; + this._redisDatabaseMock + .Verify( + x => x.ExecuteAsync( + "FT.DROPINDEX", + It.Is>(x => x.SequenceEqual(expectedArgs)), It.IsAny()), + Times.Once); + } + + [Theory] + [InlineData(true, true, """{ "data1_json_name": "data 1", "data2": "data 2", "vector1_json_name": [1, 2, 3, 4], "vector2": [1, 2, 3, 4] }""")] + [InlineData(true, false, """{ "data1_json_name": "data 1", "Data2": "data 2", "vector1_json_name": [1, 2, 3, 4], "Vector2": [1, 2, 3, 4] }""")] + [InlineData(false, true, """{ "data1_json_name": "data 1", "data2": "data 2", "vector1_json_name": [1, 2, 3, 4], "vector2": [1, 2, 3, 4] }""")] + [InlineData(false, false, """{ "data1_json_name": "data 1", "Data2": "data 2", "vector1_json_name": [1, 2, 3, 4], "Vector2": [1, 2, 3, 4] }""")] + public async Task CanGetRecordWithVectorsAsync(bool useDefinition, bool useCustomJsonSerializerOptions, string redisResultString) + { + // Arrange + SetupExecuteMock(this._redisDatabaseMock, redisResultString); + using var sut = this.CreateRecordCollection(useDefinition, useCustomJsonSerializerOptions); + + // Act + var actual = await sut.GetAsync( + TestRecordKey1, + new() { IncludeVectors = true }); + + // Assert + var expectedArgs = new object[] { TestRecordKey1 }; + this._redisDatabaseMock + .Verify( + x => x.ExecuteAsync( + "JSON.GET", + It.Is>(x => x.SequenceEqual(expectedArgs)), It.IsAny()), + Times.Once); + + Assert.NotNull(actual); + Assert.Equal(TestRecordKey1, actual.Key); + Assert.Equal("data 1", actual.Data1); + Assert.Equal("data 2", actual.Data2); + Assert.Equal(new float[] { 1, 2, 3, 4 }, actual.Vector1!.Value.ToArray()); + Assert.Equal(new float[] { 1, 2, 3, 4 }, actual.Vector2!.Value.ToArray()); + } + + [Theory] + [InlineData(true, true, """{ "data1_json_name": "data 1", "data2": "data 2" }""", "data2")] + [InlineData(true, false, """{ "data1_json_name": "data 1", "Data2": "data 2" }""", "Data2")] + [InlineData(false, true, """{ "data1_json_name": "data 1", "data2": "data 2" }""", "data2")] + [InlineData(false, false, """{ "data1_json_name": "data 1", "Data2": "data 2" }""", "Data2")] + public async Task CanGetRecordWithoutVectorsAsync(bool useDefinition, bool useCustomJsonSerializerOptions, string redisResultString, string expectedData2Name) + { + // Arrange + SetupExecuteMock(this._redisDatabaseMock, redisResultString); + using var sut = this.CreateRecordCollection(useDefinition, useCustomJsonSerializerOptions); + + // Act + var actual = await sut.GetAsync( + TestRecordKey1, + new() { IncludeVectors = false }); + + // Assert + var expectedArgs = new object[] { TestRecordKey1, "data1_json_name", expectedData2Name }; + this._redisDatabaseMock + .Verify( + x => x.ExecuteAsync( + "JSON.GET", + It.Is>(x => x.SequenceEqual(expectedArgs)), It.IsAny()), + Times.Once); + + Assert.NotNull(actual); + Assert.Equal(TestRecordKey1, actual.Key); + Assert.Equal("data 1", actual.Data1); + Assert.Equal("data 2", actual.Data2); + Assert.False(actual.Vector1.HasValue); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task CanGetManyRecordsWithVectorsAsync(bool useDefinition) + { + // Arrange + var redisResultString1 = """{ "data1_json_name": "data 1", "Data2": "data 2", "vector1_json_name": [1, 2, 3, 4], "Vector2": [1, 2, 3, 4] }"""; + var redisResultString2 = """{ "data1_json_name": "data 1", "Data2": "data 2", "vector1_json_name": [5, 6, 7, 8], "Vector2": [1, 2, 3, 4] }"""; + SetupExecuteMock(this._redisDatabaseMock, [redisResultString1, redisResultString2]); + using var sut = this.CreateRecordCollection(useDefinition); + + // Act + var actual = await sut.GetAsync( + [TestRecordKey1, TestRecordKey2], + new() { IncludeVectors = true }).ToListAsync(); + + // Assert + var expectedArgs = new object[] { TestRecordKey1, TestRecordKey2, "$" }; + this._redisDatabaseMock + .Verify( + x => x.ExecuteAsync( + "JSON.MGET", + It.Is>(x => x.SequenceEqual(expectedArgs)), It.IsAny()), + Times.Once); + + Assert.NotNull(actual); + Assert.Equal(2, actual.Count); + Assert.Equal(TestRecordKey1, actual[0].Key); + Assert.Equal("data 1", actual[0].Data1); + Assert.Equal("data 2", actual[0].Data2); + Assert.Equal(new float[] { 1, 2, 3, 4 }, actual[0].Vector1!.Value.ToArray()); + Assert.Equal(TestRecordKey2, actual[1].Key); + Assert.Equal("data 1", actual[1].Data1); + Assert.Equal("data 2", actual[1].Data2); + Assert.Equal(new float[] { 5, 6, 7, 8 }, actual[1].Vector1!.Value.ToArray()); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task CanDeleteRecordAsync(bool useDefinition) + { + // Arrange + SetupExecuteMock(this._redisDatabaseMock, "200"); + using var sut = this.CreateRecordCollection(useDefinition); + + // Act + await sut.DeleteAsync(TestRecordKey1); + + // Assert + var expectedArgs = new object[] { TestRecordKey1 }; + this._redisDatabaseMock + .Verify( + x => x.ExecuteAsync( + "JSON.DEL", + It.Is>(x => x.SequenceEqual(expectedArgs)), It.IsAny()), + Times.Once); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task CanDeleteManyRecordsWithVectorsAsync(bool useDefinition) + { + // Arrange + SetupExecuteMock(this._redisDatabaseMock, "200"); + using var sut = this.CreateRecordCollection(useDefinition); + + // Act + await sut.DeleteAsync([TestRecordKey1, TestRecordKey2]); + + // Assert + var expectedArgs1 = new object[] { TestRecordKey1 }; + var expectedArgs2 = new object[] { TestRecordKey2 }; + this._redisDatabaseMock + .Verify( + x => x.ExecuteAsync( + "JSON.DEL", + It.Is>(x => x.SequenceEqual(expectedArgs1)), It.IsAny()), + Times.Once); + this._redisDatabaseMock + .Verify( + x => x.ExecuteAsync( + "JSON.DEL", + It.Is>(x => x.SequenceEqual(expectedArgs2)), It.IsAny()), + Times.Once); + } + + [Theory] + [InlineData(true, true, """{"data1_json_name":"data 1","data2":"data 2","vector1_json_name":[1,2,3,4],"vector2":[1,2,3,4],"notAnnotated":null}""")] + [InlineData(true, false, """{"data1_json_name":"data 1","Data2":"data 2","vector1_json_name":[1,2,3,4],"Vector2":[1,2,3,4],"NotAnnotated":null}""")] + [InlineData(false, true, """{"data1_json_name":"data 1","data2":"data 2","vector1_json_name":[1,2,3,4],"vector2":[1,2,3,4],"notAnnotated":null}""")] + [InlineData(false, false, """{"data1_json_name":"data 1","Data2":"data 2","vector1_json_name":[1,2,3,4],"Vector2":[1,2,3,4],"NotAnnotated":null}""")] + public async Task CanUpsertRecordAsync(bool useDefinition, bool useCustomJsonSerializerOptions, string expectedUpsertedJson) + { + // Arrange + SetupExecuteMock(this._redisDatabaseMock, "OK"); + using var sut = this.CreateRecordCollection(useDefinition, useCustomJsonSerializerOptions); + var model = CreateModel(TestRecordKey1, true); + + // Act + await sut.UpsertAsync(model); + + // Assert + // TODO: Fix issue where NotAnnotated is being included in the JSON. + var expectedArgs = new object[] { TestRecordKey1, "$", expectedUpsertedJson }; + this._redisDatabaseMock + .Verify( + x => x.ExecuteAsync( + "JSON.SET", + It.Is>(x => x.SequenceEqual(expectedArgs)), It.IsAny()), + Times.Once); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task CanUpsertManyRecordsAsync(bool useDefinition) + { + // Arrange + SetupExecuteMock(this._redisDatabaseMock, "OK"); + using var sut = this.CreateRecordCollection(useDefinition); + + var model1 = CreateModel(TestRecordKey1, true); + var model2 = CreateModel(TestRecordKey2, true); + + // Act + await sut.UpsertAsync([model1, model2]); + + // Assert + // TODO: Fix issue where NotAnnotated is being included in the JSON. + var expectedArgs = new object[] { TestRecordKey1, "$", """{"data1_json_name":"data 1","Data2":"data 2","vector1_json_name":[1,2,3,4],"Vector2":[1,2,3,4],"NotAnnotated":null}""", TestRecordKey2, "$", """{"data1_json_name":"data 1","Data2":"data 2","vector1_json_name":[1,2,3,4],"Vector2":[1,2,3,4],"NotAnnotated":null}""" }; + this._redisDatabaseMock + .Verify( + x => x.ExecuteAsync( + "JSON.MSET", + It.Is>(x => x.SequenceEqual(expectedArgs)), It.IsAny()), + Times.Once); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task CanSearchWithVectorAndFilterAsync(bool useDefinition) + { + // Arrange + var jsonResult = """{ "data1_json_name": "data 1", "Data2": "data 2", "vector1_json_name": [1, 2, 3, 4], "Vector2": [1, 2, 3, 4] }"""; + SetupExecuteMock(this._redisDatabaseMock, new RedisResult[] + { + RedisResult.Create(new RedisValue("1")), + RedisResult.Create(new RedisValue(TestRecordKey1)), + RedisResult.Create(new RedisValue("0.8")), + RedisResult.Create( + [ + new RedisValue("$"), + new RedisValue(jsonResult), + new RedisValue("vector_score"), + new RedisValue("0.25") + ]), + }); + using var sut = this.CreateRecordCollection(useDefinition); + + // Act. + var results = await sut.SearchAsync( + new ReadOnlyMemory(new[] { 1f, 2f, 3f, 4f }), + top: 5, + new() + { + IncludeVectors = true, + Filter = r => r.Data1 == "data 1", + VectorProperty = r => r.Vector1, + Skip = 2 + }).ToListAsync(); + + // Assert. + var expectedArgs = new object[] + { + "testcollection", + "@data1_json_name:{\"data 1\"}=>[KNN 7 @vector1_json_name $embedding AS vector_score]", + "WITHSCORES", + "SORTBY", + "vector_score", + "LIMIT", + 2, + 7, + "PARAMS", + 2, + "embedding", + MemoryMarshal.AsBytes(new ReadOnlySpan(new float[] { 1, 2, 3, 4 })).ToArray(), + "DIALECT", + 2 + }; + this._redisDatabaseMock + .Verify( + x => x.ExecuteAsync( + "FT.SEARCH", + It.Is>(x => x.Where(y => !(y is byte[])).SequenceEqual(expectedArgs.Where(y => !(y is byte[])))), It.IsAny()), + Times.Once); + + Assert.Single(results); + Assert.Equal(TestRecordKey1, results.First().Record.Key); + Assert.Equal(0.25d, results.First().Score); + Assert.Equal("data 1", results.First().Record.Data1); + Assert.Equal("data 2", results.First().Record.Data2); + Assert.Equal(new float[] { 1, 2, 3, 4 }, results.First().Record.Vector1!.Value.ToArray()); + Assert.Equal(new float[] { 1, 2, 3, 4 }, results.First().Record.Vector2!.Value.ToArray()); + } + + private RedisJsonCollection CreateRecordCollection(bool useDefinition, bool useCustomJsonSerializerOptions = false) + { + return new RedisJsonCollection( + this._redisDatabaseMock.Object, + TestCollectionName, + new() + { + PrefixCollectionNameToKeyNames = false, + Definition = useDefinition ? this._multiPropsDefinition : null, + JsonSerializerOptions = useCustomJsonSerializerOptions ? this._customJsonSerializerOptions : null + }); + } + + private static void SetupExecuteMock(Mock redisDatabaseMock, Exception exception) + { + redisDatabaseMock + .Setup( + x => x.ExecuteAsync( + It.IsAny(), + It.IsAny>(), It.IsAny())) + .ThrowsAsync(exception); + } + + private static void SetupExecuteMock(Mock redisDatabaseMock, string command, Exception exception) + { + redisDatabaseMock + .Setup( + x => x.ExecuteAsync( + command, + It.IsAny>(), It.IsAny())) + .ThrowsAsync(exception); + } + + private static void SetupExecuteMock(Mock redisDatabaseMock, IEnumerable redisResultStrings) + { + var results = redisResultStrings + .Select(x => RedisResult.Create(new RedisValue(x))) + .ToArray(); + redisDatabaseMock + .Setup( + x => x.ExecuteAsync( + It.IsAny(), + It.IsAny>(), It.IsAny())) + .ReturnsAsync(RedisResult.Create(results)); + } + + private static void SetupExecuteMock(Mock redisDatabaseMock, IEnumerable redisResultStrings) + { + var results = redisResultStrings + .Select(x => x) + .ToArray(); + redisDatabaseMock + .Setup( + x => x.ExecuteAsync( + It.IsAny(), + It.IsAny>(), It.IsAny())) + .ReturnsAsync(RedisResult.Create(results)); + } + + private static void SetupExecuteMock(Mock redisDatabaseMock, string redisResultString) + { + redisDatabaseMock + .Setup( + x => x.ExecuteAsync( + It.IsAny(), + It.IsAny>(), It.IsAny())) + .ReturnsAsync(RedisResult.Create(new RedisValue(redisResultString))); + } + + private static void SetupExecuteMock(Mock redisDatabaseMock, string command, string redisResultString) + { + redisDatabaseMock + .Setup( + x => x.ExecuteAsync( + command, + It.IsAny>(), It.IsAny())) + .ReturnsAsync(RedisResult.Create(new RedisValue(redisResultString))); + } + + private static MultiPropsModel CreateModel(string key, bool withVectors) + { + return new MultiPropsModel + { + Key = key, + Data1 = "data 1", + Data2 = "data 2", + Vector1 = withVectors ? new float[] { 1, 2, 3, 4 } : null, + Vector2 = withVectors ? new float[] { 1, 2, 3, 4 } : null, + NotAnnotated = null, + }; + } + + private readonly JsonSerializerOptions _customJsonSerializerOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase + }; + + private readonly VectorStoreCollectionDefinition _multiPropsDefinition = new() + { + Properties = + [ + new VectorStoreKeyProperty("Key", typeof(string)), + new VectorStoreDataProperty("Data1", typeof(string)) { IsIndexed = true, StorageName = "ignored_data1_storage_name" }, + new VectorStoreDataProperty("Data2", typeof(string)) { IsIndexed = true }, + new VectorStoreVectorProperty("Vector1", typeof(ReadOnlyMemory), 4) { DistanceFunction = DistanceFunction.CosineDistance, StorageName = "ignored_vector1_storage_name" }, + new VectorStoreVectorProperty("Vector2", typeof(ReadOnlyMemory), 4) + ] + }; + + public sealed class MultiPropsModel + { + [VectorStoreKey] + public string Key { get; set; } = string.Empty; + + [JsonPropertyName("data1_json_name")] + [VectorStoreData(IsIndexed = true, StorageName = "ignored_data1_storage_name")] + public string Data1 { get; set; } = string.Empty; + + [VectorStoreData(IsIndexed = true)] + public string Data2 { get; set; } = string.Empty; + + [JsonPropertyName("vector1_json_name")] + [VectorStoreVector(dimensions: 4, DistanceFunction = DistanceFunction.CosineDistance, StorageName = "ignored_vector1_storage_name")] + public ReadOnlyMemory? Vector1 { get; set; } + + [VectorStoreVector(dimensions: 4)] + public ReadOnlyMemory? Vector2 { get; set; } + + public string? NotAnnotated { get; set; } + } +} diff --git a/MEVD/test/Redis.UnitTests/RedisJsonDynamicMapperTests.cs b/MEVD/test/Redis.UnitTests/RedisJsonDynamicMapperTests.cs new file mode 100644 index 0000000..d295639 --- /dev/null +++ b/MEVD/test/Redis.UnitTests/RedisJsonDynamicMapperTests.cs @@ -0,0 +1,182 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Nodes; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; +using Xunit; + +namespace CommunityToolkit.VectorData.Redis.UnitTests; + +/// +/// Contains tests for the class. +/// +public class RedisJsonDynamicMapperTests +{ + private static readonly float[] s_floatVector = new float[] { 1.0f, 2.0f, 3.0f, 4.0f }; + + private static readonly CollectionModel s_model + = new RedisJsonModelBuilder(RedisJsonCollection>.ModelBuildingOptions) + .BuildDynamic( + new() + { + Properties = + [ + new VectorStoreKeyProperty("Key", typeof(string)), + new VectorStoreDataProperty("StringData", typeof(string)), + new VectorStoreDataProperty("IntData", typeof(int)), + new VectorStoreDataProperty("NullableIntData", typeof(int?)), + new VectorStoreDataProperty("ComplexObjectData", typeof(ComplexObject)), + new VectorStoreVectorProperty("FloatVector", typeof(ReadOnlyMemory), 10), + ] + }, + defaultEmbeddingGenerator: null); + + [Fact] + public void MapFromDataToStorageModelMapsAllSupportedTypes() + { + // Arrange. + var sut = new RedisJsonDynamicMapper(s_model, JsonSerializerOptions.Default); + var dataModel = new Dictionary + { + ["Key"] = "key", + ["StringData"] = "data 1", + ["IntData"] = 1, + ["NullableIntData"] = 2, + ["ComplexObjectData"] = new ComplexObject { Prop1 = "prop 1", Prop2 = "prop 2" }, + ["FloatVector"] = new ReadOnlyMemory(s_floatVector) + }; + + // Act. + var storageModel = sut.MapFromDataToStorageModel(dataModel, recordIndex: 0, generatedEmbeddings: null); + + // Assert + Assert.Equal("key", storageModel.Key); + Assert.Equal("data 1", (string)storageModel.Node["StringData"]!); + Assert.Equal(1, (int)storageModel.Node["IntData"]!); + Assert.Equal(2, (int?)storageModel.Node["NullableIntData"]!); + Assert.Equal("prop 1", (string)storageModel.Node["ComplexObjectData"]!.AsObject()["Prop1"]!); + Assert.Equal(new float[] { 1, 2, 3, 4 }, storageModel.Node["FloatVector"]?.AsArray().GetValues().ToArray()); + } + + [Fact] + public void MapFromDataToStorageModelMapsNullValues() + { + // Arrange. + var sut = new RedisJsonDynamicMapper(s_model, JsonSerializerOptions.Default); + var dataModel = new Dictionary + { + ["Key"] = "key", + ["StringData"] = null, + ["IntData"] = null, + ["NullableIntData"] = null, + ["ComplexObjectData"] = null, + ["FloatVector"] = null, + }; + + // Act. + var storageModel = sut.MapFromDataToStorageModel(dataModel, recordIndex: 0, generatedEmbeddings: null); + + // Assert + Assert.Equal("key", storageModel.Key); + Assert.Null(storageModel.Node["storage_string_data"]); + Assert.Null(storageModel.Node["IntData"]); + Assert.Null(storageModel.Node["NullableIntData"]); + Assert.Null(storageModel.Node["ComplexObjectData"]); + Assert.Null(storageModel.Node["FloatVector"]); + } + + [Fact] + public void MapFromStorageToDataModelMapsAllSupportedTypes() + { + // Arrange. + var sut = new RedisJsonDynamicMapper(s_model, JsonSerializerOptions.Default); + var storageModel = new JsonObject + { + { "StringData", "data 1" }, + { "IntData", 1 }, + { "NullableIntData", 2 }, + { "ComplexObjectData", new JsonObject(new KeyValuePair[] { new("Prop1", JsonValue.Create("prop 1")), new("Prop2", JsonValue.Create("prop 2")) }) }, + { "FloatVector", new JsonArray(new float[] { 1, 2, 3, 4 }.Select(x => JsonValue.Create(x)).ToArray()) } + }; + + // Act. + var dataModel = sut.MapFromStorageToDataModel(("key", storageModel), includeVectors: true); + + // Assert. + Assert.Equal("key", dataModel["Key"]); + Assert.Equal("data 1", dataModel["StringData"]); + Assert.Equal(1, dataModel["IntData"]); + Assert.Equal(2, dataModel["NullableIntData"]); + Assert.Equal("prop 1", ((ComplexObject)dataModel["ComplexObjectData"]!).Prop1); + Assert.Equal(new float[] { 1, 2, 3, 4 }, ((ReadOnlyMemory)dataModel["FloatVector"]!).ToArray()); + } + + [Fact] + public void MapFromStorageToDataModelMapsNullValues() + { + // Arrange. + var sut = new RedisJsonDynamicMapper(s_model, JsonSerializerOptions.Default); + var storageModel = new JsonObject + { + { "StringData", null }, + { "IntData", null }, + { "NullableIntData", null }, + { "ComplexObjectData", null }, + { "FloatVector", null } + }; + + // Act. + var dataModel = sut.MapFromStorageToDataModel(("key", storageModel), includeVectors: true); + + // Assert. + Assert.Equal("key", dataModel["Key"]); + Assert.Null(dataModel["StringData"]); + Assert.Null(dataModel["IntData"]); + Assert.Null(dataModel["NullableIntData"]); + Assert.Null(dataModel["ComplexObjectData"]); + Assert.Null(dataModel["FloatVector"]); + } + + [Fact] + public void MapFromDataToStorageModelSkipsMissingProperties() + { + // Arrange. + var sut = new RedisJsonDynamicMapper(s_model, JsonSerializerOptions.Default); + var dataModel = new Dictionary { ["Key"] = "key" }; + + // Act. + var storageModel = sut.MapFromDataToStorageModel(dataModel, recordIndex: 0, generatedEmbeddings: null); + + // Assert + Assert.Equal("key", storageModel.Key); + Assert.Empty(storageModel.Node.AsObject()); + } + + [Fact] + public void MapFromStorageToDataModelSkipsMissingProperties() + { + // Arrange. + var storageModel = new JsonObject(); + + var sut = new RedisJsonDynamicMapper(s_model, JsonSerializerOptions.Default); + + // Act. + var dataModel = sut.MapFromStorageToDataModel(("key", storageModel), includeVectors: true); + + // Assert. + Assert.Equal("key", dataModel["Key"]); + Assert.Single(dataModel); + } + + private sealed class ComplexObject + { + public string Prop1 { get; set; } = string.Empty; + + public string Prop2 { get; set; } = string.Empty; + } +} diff --git a/MEVD/test/Redis.UnitTests/RedisJsonMapperTests.cs b/MEVD/test/Redis.UnitTests/RedisJsonMapperTests.cs new file mode 100644 index 0000000..41cd72b --- /dev/null +++ b/MEVD/test/Redis.UnitTests/RedisJsonMapperTests.cs @@ -0,0 +1,149 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Nodes; +using Microsoft.Extensions.VectorData; +using CommunityToolkit.VectorData.Redis; +using Xunit; + +namespace SemanticKernel.Connectors.Redis.UnitTests; + +/// +/// Contains tests for the class. +/// +public sealed class RedisJsonMapperTests +{ + [Fact] + public void MapsAllFieldsFromDataToStorageModel() + { + // Arrange. + var model = new RedisJsonModelBuilder(RedisJsonCollection.ModelBuildingOptions) + .Build(typeof(MultiPropsModel), typeof(string), definition: null, defaultEmbeddingGenerator: null, JsonSerializerOptions.Default); + var sut = new RedisJsonMapper(model, JsonSerializerOptions.Default); + + // Act. + var actual = sut.MapFromDataToStorageModel(CreateModel("test key"), recordIndex: 0, generatedEmbeddings: null); + + // Assert. + Assert.NotNull(actual.Node); + Assert.Equal("test key", actual.Key); + var jsonObject = actual.Node.AsObject(); + Assert.Equal("data 1", jsonObject?["Data1"]?.ToString()); + Assert.Equal("data 2", jsonObject?["Data2"]?.ToString()); + Assert.Equal(new float[] { 1, 2, 3, 4 }, jsonObject?["Vector1"]?.AsArray().GetValues().ToArray()); + Assert.Equal(new float[] { 5, 6, 7, 8 }, jsonObject?["Vector2"]?.AsArray().GetValues().ToArray()); + } + + [Fact] + public void MapsAllFieldsFromDataToStorageModelWithCustomSerializerOptions() + { + // Arrange. + var jsonSerializerOptions = new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }; + var model = new RedisJsonModelBuilder(RedisJsonCollection.ModelBuildingOptions) + .Build(typeof(MultiPropsModel), typeof(string), definition: null, defaultEmbeddingGenerator: null, jsonSerializerOptions); + var sut = new RedisJsonMapper(model, jsonSerializerOptions); + + // Act. + var actual = sut.MapFromDataToStorageModel(CreateModel("test key"), recordIndex: 0, generatedEmbeddings: null); + + // Assert. + Assert.NotNull(actual.Node); + Assert.Equal("test key", actual.Key); + var jsonObject = actual.Node.AsObject(); + Assert.Equal("data 1", jsonObject?["data1"]?.ToString()); + Assert.Equal("data 2", jsonObject?["data2"]?.ToString()); + Assert.Equal(new float[] { 1, 2, 3, 4 }, jsonObject?["vector1"]?.AsArray().GetValues().ToArray()); + Assert.Equal(new float[] { 5, 6, 7, 8 }, jsonObject?["vector2"]?.AsArray().GetValues().ToArray()); + } + + [Fact] + public void MapsAllFieldsFromStorageToDataModel() + { + // Arrange. + var model = new RedisJsonModelBuilder(RedisJsonCollection.ModelBuildingOptions) + .Build(typeof(MultiPropsModel), typeof(string), definition: null, defaultEmbeddingGenerator: null, JsonSerializerOptions.Default); + var sut = new RedisJsonMapper(model, JsonSerializerOptions.Default); + + // Act. + var jsonObject = new JsonObject + { + { "Data1", "data 1" }, + { "Data2", "data 2" }, + { "Vector1", new JsonArray(new[] { 1, 2, 3, 4 }.Select(x => JsonValue.Create(x)).ToArray()) }, + { "Vector2", new JsonArray(new[] { 5, 6, 7, 8 }.Select(x => JsonValue.Create(x)).ToArray()) } + }; + var actual = sut.MapFromStorageToDataModel(("test key", jsonObject), includeVectors: true); + + // Assert. + Assert.NotNull(actual); + Assert.Equal("test key", actual.Key); + Assert.Equal("data 1", actual.Data1); + Assert.Equal("data 2", actual.Data2); + Assert.Equal(new float[] { 1, 2, 3, 4 }, actual.Vector1!.Value.ToArray()); + Assert.Equal(new float[] { 5, 6, 7, 8 }, actual.Vector2!.Value.ToArray()); + } + + [Fact] + public void MapsAllFieldsFromStorageToDataModelWithCustomSerializerOptions() + { + // Arrange. + var jsonSerializerOptions = new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }; + var model = new RedisJsonModelBuilder(RedisJsonCollection.ModelBuildingOptions) + .Build(typeof(MultiPropsModel), typeof(string), definition: null, defaultEmbeddingGenerator: null, jsonSerializerOptions); + var sut = new RedisJsonMapper(model, jsonSerializerOptions); + + // Act. + var jsonObject = new JsonObject + { + { "data1", "data 1" }, + { "data2", "data 2" }, + { "vector1", new JsonArray(new[] { 1, 2, 3, 4 }.Select(x => JsonValue.Create(x)).ToArray()) }, + { "vector2", new JsonArray(new[] { 5, 6, 7, 8 }.Select(x => JsonValue.Create(x)).ToArray()) } + }; + var actual = sut.MapFromStorageToDataModel(("test key", jsonObject), includeVectors: true); + + // Assert. + Assert.NotNull(actual); + Assert.Equal("test key", actual.Key); + Assert.Equal("data 1", actual.Data1); + Assert.Equal("data 2", actual.Data2); + Assert.Equal(new float[] { 1, 2, 3, 4 }, actual.Vector1!.Value.ToArray()); + Assert.Equal(new float[] { 5, 6, 7, 8 }, actual.Vector2!.Value.ToArray()); + } + + private static MultiPropsModel CreateModel(string key) + { + return new MultiPropsModel + { + Key = key, + Data1 = "data 1", + Data2 = "data 2", + Vector1 = new float[] { 1, 2, 3, 4 }, + Vector2 = new float[] { 5, 6, 7, 8 }, + NotAnnotated = "notAnnotated", + }; + } + + private sealed class MultiPropsModel + { + [VectorStoreKey] + public string Key { get; set; } = string.Empty; + + [VectorStoreData] + public string Data1 { get; set; } = string.Empty; + + [VectorStoreData] + public string Data2 { get; set; } = string.Empty; + + [VectorStoreVector(dimensions: 10)] + public ReadOnlyMemory? Vector1 { get; set; } + + [VectorStoreVector(dimensions: 10)] + public ReadOnlyMemory? Vector2 { get; set; } + + public string NotAnnotated { get; set; } = string.Empty; + } +} diff --git a/MEVD/test/Redis.UnitTests/RedisVectorStoreTests.cs b/MEVD/test/Redis.UnitTests/RedisVectorStoreTests.cs new file mode 100644 index 0000000..51e629c --- /dev/null +++ b/MEVD/test/Redis.UnitTests/RedisVectorStoreTests.cs @@ -0,0 +1,108 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.VectorData; +using Moq; +using StackExchange.Redis; +using Xunit; + +namespace CommunityToolkit.VectorData.Redis.UnitTests; + +/// +/// Contains tests for the class. +/// +public class RedisVectorStoreTests +{ + private const string TestCollectionName = "testcollection"; + + private readonly Mock _redisDatabaseMock; + + public RedisVectorStoreTests() + { + this._redisDatabaseMock = new Mock(MockBehavior.Strict); + this._redisDatabaseMock.Setup(l => l.Database).Returns(0); + + var batchMock = new Mock(); + this._redisDatabaseMock.Setup(x => x.CreateBatch(It.IsAny())).Returns(batchMock.Object); + } + + [Fact] + public void GetCollectionReturnsJsonCollection() + { + // Arrange. + using var sut = new RedisVectorStore(this._redisDatabaseMock.Object); + + // Act. + var actual = sut.GetCollection>(TestCollectionName); + + // Assert. + Assert.NotNull(actual); + Assert.IsType>>(actual); + } + + [Fact] + public void GetCollectionReturnsHashSetCollection() + { + // Arrange. + using var sut = new RedisVectorStore(this._redisDatabaseMock.Object, new() { StorageType = RedisStorageType.HashSet }); + + // Act. + var actual = sut.GetCollection>(TestCollectionName); + + // Assert. + Assert.NotNull(actual); + Assert.IsType>>(actual); + } + + [Fact] + public void GetCollectionThrowsForInvalidKeyType() + { + // Arrange. + using var sut = new RedisVectorStore(this._redisDatabaseMock.Object); + + // Act & Assert. + Assert.Throws(() => sut.GetCollection>(TestCollectionName)); + } + + [Fact] + public async Task ListCollectionNamesCallsSDKAsync() + { + // Arrange. + var redisResultStrings = new string[] { "collection1", "collection2" }; + var results = redisResultStrings + .Select(x => RedisResult.Create(new RedisValue(x))) + .ToArray(); + this._redisDatabaseMock + .Setup( + x => x.ExecuteAsync( + It.IsAny(), + It.IsAny>(), It.IsAny())) + .ReturnsAsync(RedisResult.Create(results)); + using var sut = new RedisVectorStore(this._redisDatabaseMock.Object); + + // Act. + var collectionNames = sut.ListCollectionNamesAsync(); + + // Assert. + var collectionNamesList = await collectionNames.ToListAsync(); + Assert.Equal(new[] { "collection1", "collection2" }, collectionNamesList); + } + + public sealed class SinglePropsModel + { + [VectorStoreKey] + public required TKey Key { get; set; } + + [VectorStoreData] + public string Data { get; set; } = string.Empty; + + [VectorStoreVector(dimensions: 4)] + public ReadOnlyMemory? Vector { get; set; } + + public string? NotAnnotated { get; set; } + } +} diff --git a/MEVD/test/Shared/AssertExtensions.cs b/MEVD/test/Shared/AssertExtensions.cs new file mode 100644 index 0000000..dbd8628 --- /dev/null +++ b/MEVD/test/Shared/AssertExtensions.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using Assert = Xunit.Assert; + +namespace CommunityToolkit.VectorData.UnitTests; + +internal static class AssertExtensions +{ + /// Asserts that an exception is an with the specified values. + public static void AssertIsArgumentOutOfRange(Exception? e, string expectedParamName, string expectedActualValue) + { + ArgumentOutOfRangeException aoore = Assert.IsType(e); + Assert.Equal(expectedActualValue, aoore.ActualValue); + Assert.Equal(expectedParamName, aoore.ParamName); + } +} diff --git a/MEVD/test/Shared/HttpMessageHandlerStub.cs b/MEVD/test/Shared/HttpMessageHandlerStub.cs new file mode 100644 index 0000000..b999e34 --- /dev/null +++ b/MEVD/test/Shared/HttpMessageHandlerStub.cs @@ -0,0 +1,67 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Net.Mime; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +#pragma warning disable CA1812 // Internal class that is apparently never instantiated; this class is compiled in tests projects +internal sealed class HttpMessageHandlerStub : HttpMessageHandler +#pragma warning restore CA1812 // Internal class that is apparently never instantiated +{ + public HttpRequestHeaders? RequestHeaders { get; private set; } + + public HttpContentHeaders? ContentHeaders { get; private set; } + + public byte[]? RequestContent { get; private set; } + + public Uri? RequestUri { get; private set; } + + public HttpMethod? Method { get; private set; } + + public HttpResponseMessage ResponseToReturn { get; set; } + + public Queue ResponseQueue { get; } = new(); + public byte[]? FirstMultipartContent { get; private set; } + + public HttpMessageHandlerStub() + { + this.ResponseToReturn = new HttpResponseMessage(System.Net.HttpStatusCode.OK) + { + Content = new StringContent("{}", Encoding.UTF8, MediaTypeNames.Application.Json), + }; + } + +#pragma warning disable VSTHRD002 // Avoid problematic synchronous waits + protected override HttpResponseMessage Send(HttpRequestMessage request, CancellationToken cancellationToken) => + this.SendAsync(request, cancellationToken).GetAwaiter().GetResult(); +#pragma warning restore VSTHRD002 // Avoid problematic synchronous waits + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + this.Method = request.Method; + this.RequestUri = request.RequestUri; + this.RequestHeaders = request.Headers; + this.RequestContent = request.Content is null ? null : await request.Content.ReadAsByteArrayAsync(cancellationToken); + + if (request.Content is MultipartContent multipartContent) + { + this.FirstMultipartContent = await multipartContent.First().ReadAsByteArrayAsync(cancellationToken); + } + + this.ContentHeaders = request.Content?.Headers; + + HttpResponseMessage response = + this.ResponseQueue.Count == 0 ? + this.ResponseToReturn : + this.ResponseToReturn = this.ResponseQueue.Dequeue(); + + return response; + } +} diff --git a/MEVD/test/SqliteVec.ConformanceTests/ModelTests/SqliteBasicModelTests.cs b/MEVD/test/SqliteVec.ConformanceTests/ModelTests/SqliteBasicModelTests.cs new file mode 100644 index 0000000..a9e81f7 --- /dev/null +++ b/MEVD/test/SqliteVec.ConformanceTests/ModelTests/SqliteBasicModelTests.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using SqliteVec.ConformanceTests.Support; +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace SqliteVec.ConformanceTests.ModelTests; + +public class SqliteBasicModelTests(SqliteBasicModelTests.Fixture fixture) + : BasicModelTests(fixture), IClassFixture +{ + public new class Fixture : BasicModelTests.Fixture + { + public override TestStore TestStore => SqliteTestStore.Instance; + } +} diff --git a/MEVD/test/SqliteVec.ConformanceTests/ModelTests/SqliteMultiVectorModelTests.cs b/MEVD/test/SqliteVec.ConformanceTests/ModelTests/SqliteMultiVectorModelTests.cs new file mode 100644 index 0000000..e554ab6 --- /dev/null +++ b/MEVD/test/SqliteVec.ConformanceTests/ModelTests/SqliteMultiVectorModelTests.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using SqliteVec.ConformanceTests.Support; +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace SqliteVec.ConformanceTests.ModelTests; + +public class SqliteMultiVectorModelTests(SqliteMultiVectorModelTests.Fixture fixture) + : MultiVectorModelTests(fixture), IClassFixture +{ + public new class Fixture : MultiVectorModelTests.Fixture + { + public override TestStore TestStore => SqliteTestStore.Instance; + } +} diff --git a/MEVD/test/SqliteVec.ConformanceTests/ModelTests/SqliteNoDataModelTests.cs b/MEVD/test/SqliteVec.ConformanceTests/ModelTests/SqliteNoDataModelTests.cs new file mode 100644 index 0000000..38ccea1 --- /dev/null +++ b/MEVD/test/SqliteVec.ConformanceTests/ModelTests/SqliteNoDataModelTests.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using SqliteVec.ConformanceTests.Support; +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace SqliteVec.ConformanceTests.ModelTests; + +public class SqliteNoDataModelTests(SqliteNoDataModelTests.Fixture fixture) + : NoDataModelTests(fixture), IClassFixture +{ + public new class Fixture : NoDataModelTests.Fixture + { + public override TestStore TestStore => SqliteTestStore.Instance; + } +} diff --git a/MEVD/test/SqliteVec.ConformanceTests/ModelTests/SqliteNoVectorModelTests.cs b/MEVD/test/SqliteVec.ConformanceTests/ModelTests/SqliteNoVectorModelTests.cs new file mode 100644 index 0000000..bc392b8 --- /dev/null +++ b/MEVD/test/SqliteVec.ConformanceTests/ModelTests/SqliteNoVectorModelTests.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using SqliteVec.ConformanceTests.Support; +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace SqliteVec.ConformanceTests.ModelTests; + +public class SqliteNoVectorModelTests(SqliteNoVectorModelTests.Fixture fixture) + : NoVectorModelTests(fixture), IClassFixture +{ + public new class Fixture : NoVectorModelTests.Fixture + { + public override TestStore TestStore => SqliteTestStore.Instance; + } +} diff --git a/MEVD/test/SqliteVec.ConformanceTests/Properties/AssemblyAttributes.cs b/MEVD/test/SqliteVec.ConformanceTests/Properties/AssemblyAttributes.cs new file mode 100644 index 0000000..f1eb3bb --- /dev/null +++ b/MEVD/test/SqliteVec.ConformanceTests/Properties/AssemblyAttributes.cs @@ -0,0 +1,7 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Xunit; + +// Disable test parallelization in order to prevent from "database is locked" errors +[assembly: CollectionBehavior(DisableTestParallelization = true)] diff --git a/MEVD/test/SqliteVec.ConformanceTests/SqliteCollectionManagementTests.cs b/MEVD/test/SqliteVec.ConformanceTests/SqliteCollectionManagementTests.cs new file mode 100644 index 0000000..12273ef --- /dev/null +++ b/MEVD/test/SqliteVec.ConformanceTests/SqliteCollectionManagementTests.cs @@ -0,0 +1,13 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using SqliteVec.ConformanceTests.Support; +using VectorData.ConformanceTests; +using Xunit; + +namespace SqliteVec.ConformanceTests; + +public class SqliteCollectionManagementTests(SqliteFixture fixture) + : CollectionManagementTests(fixture), IClassFixture +{ +} diff --git a/MEVD/test/SqliteVec.ConformanceTests/SqliteDependencyInjectionTests.cs b/MEVD/test/SqliteVec.ConformanceTests/SqliteDependencyInjectionTests.cs new file mode 100644 index 0000000..d00dddd --- /dev/null +++ b/MEVD/test/SqliteVec.ConformanceTests/SqliteDependencyInjectionTests.cs @@ -0,0 +1,84 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using CommunityToolkit.VectorData.SqliteVec; +using VectorData.ConformanceTests; +using Xunit; + +namespace SqliteVec.ConformanceTests; + +public class SqliteDependencyInjectionTests + : DependencyInjectionTests.Record>, string, DependencyInjectionTests.Record> +{ + protected const string ConnectionString = "Data Source=:memory:"; + + protected override void PopulateConfiguration(ConfigurationManager configuration, object? serviceKey = null) + => configuration.AddInMemoryCollection( + [ + new(CreateConfigKey("Sqlite", serviceKey, "ConnectionString"), ConnectionString), + ]); + + private static string ConnectionStringProvider(IServiceProvider sp) + => sp.GetRequiredService().GetRequiredSection("Sqlite:ConnectionString").Value!; + + private static string ConnectionStringProvider(IServiceProvider sp, object serviceKey) + => sp.GetRequiredService().GetRequiredSection(CreateConfigKey("Sqlite", serviceKey, "ConnectionString")).Value!; + + public override IEnumerable> CollectionDelegates + { + get + { + yield return (services, serviceKey, name, lifetime) => serviceKey is null + ? services.AddSqliteCollection( + name, connectionString: ConnectionString, lifetime: lifetime) + : services.AddKeyedSqliteCollection( + serviceKey, name, connectionString: ConnectionString, lifetime: lifetime); + + yield return (services, serviceKey, name, lifetime) => serviceKey is null + ? services.AddSqliteCollection( + name, ConnectionStringProvider, lifetime: lifetime) + : services.AddKeyedSqliteCollection( + serviceKey, name, sp => ConnectionStringProvider(sp, serviceKey), lifetime: lifetime); + } + } + + public override IEnumerable> StoreDelegates + { + get + { + yield return (services, serviceKey, lifetime) => serviceKey is null + ? services.AddSqliteVectorStore( + ConnectionStringProvider, lifetime: lifetime) + : services.AddKeyedSqliteVectorStore( + serviceKey, sp => ConnectionStringProvider(sp, serviceKey), lifetime: lifetime); + } + } + + [Fact] + public void ConnectionStringProviderCantBeNull() + { + IServiceCollection services = new ServiceCollection(); + + Assert.Throws(() => services.AddSqliteVectorStore(connectionStringProvider: null!)); + Assert.Throws(() => services.AddKeyedSqliteVectorStore(serviceKey: "notNull", connectionStringProvider: null!)); + Assert.Throws(() => services.AddSqliteCollection(name: "notNull", connectionStringProvider: null!)); + Assert.Throws(() => services.AddKeyedSqliteCollection(serviceKey: "notNull", name: "notNull", connectionStringProvider: null!)); + } + + [Fact] + public void ConnectionStringCantBeNullOrEmpty() + { + IServiceCollection services = new ServiceCollection(); + + Assert.Throws(() => services.AddSqliteCollection( + name: "notNull", connectionString: null!)); + Assert.Throws(() => services.AddSqliteCollection( + name: "notNull", connectionString: "")); + Assert.Throws(() => services.AddKeyedSqliteCollection( + serviceKey: "notNull", name: "notNull", connectionString: null!)); + Assert.Throws(() => services.AddKeyedSqliteCollection( + serviceKey: "notNull", name: "notNull", connectionString: "")); + } +} diff --git a/MEVD/test/SqliteVec.ConformanceTests/SqliteDistanceFunctionTests.cs b/MEVD/test/SqliteVec.ConformanceTests/SqliteDistanceFunctionTests.cs new file mode 100644 index 0000000..29322cd --- /dev/null +++ b/MEVD/test/SqliteVec.ConformanceTests/SqliteDistanceFunctionTests.cs @@ -0,0 +1,24 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using SqliteVec.ConformanceTests.Support; +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace SqliteVec.ConformanceTests; + +public class SqliteDistanceFunctionTests(SqliteDistanceFunctionTests.Fixture fixture) + : DistanceFunctionTests(fixture), IClassFixture +{ + public override Task CosineSimilarity() => Assert.ThrowsAsync(base.CosineSimilarity); + public override Task DotProductSimilarity() => Assert.ThrowsAsync(base.DotProductSimilarity); + public override Task EuclideanSquaredDistance() => Assert.ThrowsAsync(base.EuclideanSquaredDistance); + public override Task HammingDistance() => Assert.ThrowsAsync(base.HammingDistance); + public override Task NegativeDotProductSimilarity() => Assert.ThrowsAsync(base.NegativeDotProductSimilarity); + + public new class Fixture() : DistanceFunctionTests.Fixture + { + public override TestStore TestStore => SqliteTestStore.Instance; + } +} diff --git a/MEVD/test/SqliteVec.ConformanceTests/SqliteEmbeddingGenerationTests.cs b/MEVD/test/SqliteVec.ConformanceTests/SqliteEmbeddingGenerationTests.cs new file mode 100644 index 0000000..ef9002f --- /dev/null +++ b/MEVD/test/SqliteVec.ConformanceTests/SqliteEmbeddingGenerationTests.cs @@ -0,0 +1,53 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.VectorData; +using SqliteVec.ConformanceTests.Support; +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace SqliteVec.ConformanceTests; + +public class SqliteEmbeddingGenerationTests(SqliteEmbeddingGenerationTests.StringVectorFixture stringVectorFixture, SqliteEmbeddingGenerationTests.RomOfFloatVectorFixture romOfFloatVectorFixture) + : EmbeddingGenerationTests(stringVectorFixture, romOfFloatVectorFixture), IClassFixture, IClassFixture +{ + public new class StringVectorFixture : EmbeddingGenerationTests.StringVectorFixture + { + public override TestStore TestStore => SqliteTestStore.Instance; + + public override VectorStore CreateVectorStore(IEmbeddingGenerator? embeddingGenerator) + => SqliteTestStore.Instance.GetVectorStore(new() { EmbeddingGenerator = embeddingGenerator }); + + public override Func[] DependencyInjectionStoreRegistrationDelegates => + [ + services => services.AddSqliteVectorStore(_ => SqliteTestStore.Instance.ConnectionString) + ]; + + public override Func[] DependencyInjectionCollectionRegistrationDelegates => + [ + services => services.AddSqliteCollection(this.CollectionName, SqliteTestStore.Instance.ConnectionString) + ]; + } + + public new class RomOfFloatVectorFixture : EmbeddingGenerationTests.RomOfFloatVectorFixture + { + public override TestStore TestStore => SqliteTestStore.Instance; + + public override VectorStore CreateVectorStore(IEmbeddingGenerator? embeddingGenerator) + => SqliteTestStore.Instance.GetVectorStore(new() { EmbeddingGenerator = embeddingGenerator }); + + public override Func[] DependencyInjectionStoreRegistrationDelegates => + [ + services => services.AddSqliteVectorStore(_ => SqliteTestStore.Instance.ConnectionString) + ]; + + public override Func[] DependencyInjectionCollectionRegistrationDelegates => + [ + services => services.AddSqliteCollection(this.CollectionName, SqliteTestStore.Instance.ConnectionString), + services => services.AddSqliteCollection(this.CollectionName, _ => SqliteTestStore.Instance.ConnectionString) + ]; + } +} diff --git a/MEVD/test/SqliteVec.ConformanceTests/SqliteFilterTests.cs b/MEVD/test/SqliteVec.ConformanceTests/SqliteFilterTests.cs new file mode 100644 index 0000000..c5f1e07 --- /dev/null +++ b/MEVD/test/SqliteVec.ConformanceTests/SqliteFilterTests.cs @@ -0,0 +1,87 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.VectorData; +using SqliteVec.ConformanceTests.Support; +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.Support; +using Xunit; +using Xunit.Sdk; + +namespace SqliteVec.ConformanceTests; + +#pragma warning disable CS0252 // Possible unintended reference comparison; left hand side needs cast + +public class SqliteFilterTests(SqliteFilterTests.Fixture fixture) + : FilterTests(fixture), IClassFixture +{ + public override async Task Not_over_Or() + { + // Test sends: WHERE (NOT (("Int" = 8) OR ("String" = 'foo'))) + // There's a NULL string in the database, and relational null semantics in conjunction with negation makes the default implementation fail. + await Assert.ThrowsAsync(base.Not_over_Or); + + // Compensate by adding a null check: + await this.TestFilterAsync( + r => r.String != null && !(r.Int == 8 || r.String == "foo"), + r => r["String"] != null && !((int)r["Int"]! == 8 || r["String"] == "foo")); + } + + public override async Task NotEqual_with_string() + { + // As above, null semantics + negation + await Assert.ThrowsAsync(base.NotEqual_with_string); + + await this.TestFilterAsync( + r => r.String != null && r.String != "foo", + r => r["String"] != null && r["String"] != "foo"); + } + + // Array fields not (currently) supported on SQLite (see #10343) + public override Task Contains_over_field_string_array() + => Assert.ThrowsAsync(base.Contains_over_field_string_array); + + // List fields not (currently) supported on SQLite (see #10343) + public override Task Contains_over_field_string_List() + => Assert.ThrowsAsync(base.Contains_over_field_string_List); + + // List fields not (currently) supported on SQLite (see #10343) + public override Task Contains_with_Enumerable_Contains() + => Assert.ThrowsAsync(base.Contains_with_Enumerable_Contains); + +#if !NETFRAMEWORK + // List fields not (currently) supported on SQLite (see #10343) + public override Task Contains_with_MemoryExtensions_Contains() + => Assert.ThrowsAsync(base.Contains_with_MemoryExtensions_Contains); +#endif + +#if NET10_0_OR_GREATER + public override Task Contains_with_MemoryExtensions_Contains_with_null_comparer() + => Assert.ThrowsAsync(base.Contains_with_MemoryExtensions_Contains_with_null_comparer); +#endif + + // Any over array fields not (currently) supported on SQLite (see #10343) + public override Task Any_with_Contains_over_inline_string_array() + => Assert.ThrowsAsync(base.Any_with_Contains_over_inline_string_array); + + public override Task Any_with_Contains_over_captured_string_array() + => Assert.ThrowsAsync(base.Any_with_Contains_over_captured_string_array); + + public override Task Any_with_Contains_over_captured_string_list() + => Assert.ThrowsAsync(base.Any_with_Contains_over_captured_string_list); + + public override Task Any_over_List_with_Contains_over_captured_string_array() + => Assert.ThrowsAsync(base.Any_over_List_with_Contains_over_captured_string_array); + + public new class Fixture : FilterTests.Fixture + { + public override TestStore TestStore => SqliteTestStore.Instance; + + // Override to remove the string array property, which isn't (currently) supported on SQLite + public override VectorStoreCollectionDefinition CreateRecordDefinition() + => new() + { + Properties = base.CreateRecordDefinition().Properties.Where(p => p.Type != typeof(string[]) && p.Type != typeof(List)).ToList() + }; + } +} diff --git a/MEVD/test/SqliteVec.ConformanceTests/SqliteIndexKindTests.cs b/MEVD/test/SqliteVec.ConformanceTests/SqliteIndexKindTests.cs new file mode 100644 index 0000000..5c98b4f --- /dev/null +++ b/MEVD/test/SqliteVec.ConformanceTests/SqliteIndexKindTests.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using SqliteVec.ConformanceTests.Support; +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace SqliteVec.ConformanceTests; + +public class SqliteIndexKindTests(SqliteIndexKindTests.Fixture fixture) + : IndexKindTests(fixture), IClassFixture +{ + public new class Fixture() : IndexKindTests.Fixture + { + public override TestStore TestStore => SqliteTestStore.Instance; + } +} diff --git a/MEVD/test/SqliteVec.ConformanceTests/SqliteTestSuiteImplementationTests.cs b/MEVD/test/SqliteVec.ConformanceTests/SqliteTestSuiteImplementationTests.cs new file mode 100644 index 0000000..32037b0 --- /dev/null +++ b/MEVD/test/SqliteVec.ConformanceTests/SqliteTestSuiteImplementationTests.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.ModelTests; + +namespace SqliteVec.ConformanceTests; + +public class SqliteTestSuiteImplementationTests : TestSuiteImplementationTests +{ + protected override ICollection IgnoredTestBases { get; } = + [ + typeof(DynamicModelTests<>), + + // Hybrid search not supported + typeof(HybridSearchTests<>) + ]; +} diff --git a/MEVD/test/SqliteVec.ConformanceTests/SqliteVec.ConformanceTests.csproj b/MEVD/test/SqliteVec.ConformanceTests/SqliteVec.ConformanceTests.csproj new file mode 100644 index 0000000..f5598fa --- /dev/null +++ b/MEVD/test/SqliteVec.ConformanceTests/SqliteVec.ConformanceTests.csproj @@ -0,0 +1,26 @@ + + + + net10.0;$(NetFrameworkTfm) + enable + enable + true + false + Sqlite.ConformanceTests + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + + + diff --git a/MEVD/test/SqliteVec.ConformanceTests/Support/SqliteFixture.cs b/MEVD/test/SqliteVec.ConformanceTests/Support/SqliteFixture.cs new file mode 100644 index 0000000..674c9fb --- /dev/null +++ b/MEVD/test/SqliteVec.ConformanceTests/Support/SqliteFixture.cs @@ -0,0 +1,11 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using VectorData.ConformanceTests.Support; + +namespace SqliteVec.ConformanceTests.Support; + +public class SqliteFixture : VectorStoreFixture +{ + public override TestStore TestStore => SqliteTestStore.Instance; +} diff --git a/MEVD/test/SqliteVec.ConformanceTests/Support/SqliteTestStore.cs b/MEVD/test/SqliteVec.ConformanceTests/Support/SqliteTestStore.cs new file mode 100644 index 0000000..b0b8ca6 --- /dev/null +++ b/MEVD/test/SqliteVec.ConformanceTests/Support/SqliteTestStore.cs @@ -0,0 +1,57 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using CommunityToolkit.VectorData.SqliteVec; +using Microsoft.Data.Sqlite; +using VectorData.ConformanceTests.Support; + +namespace SqliteVec.ConformanceTests.Support; + +internal sealed class SqliteTestStore : TestStore +{ + private string? _databasePath; + + private string? _connectionString; + public string ConnectionString => this._connectionString ?? throw new InvalidOperationException("Not initialized"); + + public static SqliteTestStore Instance { get; } = new(); + + public override string DefaultDistanceFunction => Microsoft.Extensions.VectorData.DistanceFunction.CosineDistance; + + public SqliteVectorStore GetVectorStore(SqliteVectorStoreOptions options) + => new(this.ConnectionString, options); + + private SqliteTestStore() + { + } + + protected override Task StartAsync() + { + // Verify that the sqlite_vec extension can be loaded; fail early with a clear message if not. + try + { + using var connection = new SqliteConnection("Data Source=:memory:"); + connection.Open(); + connection.LoadVector(); + } + catch (Exception ex) + { + throw new InvalidOperationException( + "The sqlite_vec native extension could not be loaded. " + + "Make sure the appropriate native dependencies are available.", + ex); + } + + this._databasePath = Path.GetTempFileName(); + this._connectionString = $"Data Source={this._databasePath};Pooling=false"; + this.DefaultVectorStore = new SqliteVectorStore(this._connectionString); + return Task.CompletedTask; + } + + protected override Task StopAsync() + { + File.Delete(this._databasePath!); + this._databasePath = null; + return Task.CompletedTask; + } +} diff --git a/MEVD/test/SqliteVec.ConformanceTests/TypeTests/SqliteDataTypeTests.cs b/MEVD/test/SqliteVec.ConformanceTests/TypeTests/SqliteDataTypeTests.cs new file mode 100644 index 0000000..99ded06 --- /dev/null +++ b/MEVD/test/SqliteVec.ConformanceTests/TypeTests/SqliteDataTypeTests.cs @@ -0,0 +1,25 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using SqliteVec.ConformanceTests.Support; +using VectorData.ConformanceTests.Support; +using VectorData.ConformanceTests.TypeTests; +using Xunit; + +namespace SqliteVec.ConformanceTests.TypeTests; + +public class SqliteDataTypeTests(SqliteDataTypeTests.Fixture fixture) + : DataTypeTests.DefaultRecord>(fixture), IClassFixture +{ + public new class Fixture : DataTypeTests.DefaultRecord>.Fixture + { + public override TestStore TestStore => SqliteTestStore.Instance; + + public override Type[] UnsupportedDefaultTypes { get; } = + [ + typeof(byte), + typeof(decimal), + typeof(string[]), + ]; + } +} diff --git a/MEVD/test/SqliteVec.ConformanceTests/TypeTests/SqliteEmbeddingTypeTests.cs b/MEVD/test/SqliteVec.ConformanceTests/TypeTests/SqliteEmbeddingTypeTests.cs new file mode 100644 index 0000000..08b6312 --- /dev/null +++ b/MEVD/test/SqliteVec.ConformanceTests/TypeTests/SqliteEmbeddingTypeTests.cs @@ -0,0 +1,20 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using SqliteVec.ConformanceTests.Support; +using VectorData.ConformanceTests.Support; +using VectorData.ConformanceTests.TypeTests; +using Xunit; + +#pragma warning disable CA2000 // Dispose objects before losing scope + +namespace SqliteVec.ConformanceTests; + +public class SqliteEmbeddingTypeTests(SqliteEmbeddingTypeTests.Fixture fixture) + : EmbeddingTypeTests(fixture), IClassFixture +{ + public new class Fixture : EmbeddingTypeTests.Fixture + { + public override TestStore TestStore => SqliteTestStore.Instance; + } +} diff --git a/MEVD/test/SqliteVec.ConformanceTests/TypeTests/SqliteKeyTypeTests.cs b/MEVD/test/SqliteVec.ConformanceTests/TypeTests/SqliteKeyTypeTests.cs new file mode 100644 index 0000000..4243133 --- /dev/null +++ b/MEVD/test/SqliteVec.ConformanceTests/TypeTests/SqliteKeyTypeTests.cs @@ -0,0 +1,27 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using SqliteVec.ConformanceTests.Support; +using VectorData.ConformanceTests.Support; +using VectorData.ConformanceTests.TypeTests; +using Xunit; + +namespace SqliteVec.ConformanceTests.TypeTests; + +public class SqliteKeyTypeTests(SqliteKeyTypeTests.Fixture fixture) + : KeyTypeTests(fixture), IClassFixture +{ + [Fact] + public virtual Task Int() => this.Test(8, supportsAutoGeneration: true); + + [Fact] + public virtual Task Long() => this.Test(8L, supportsAutoGeneration: true); + + [Fact] + public virtual Task String() => this.Test("foo", "bar"); + + public new class Fixture : KeyTypeTests.Fixture + { + public override TestStore TestStore => SqliteTestStore.Instance; + } +} diff --git a/MEVD/test/SqliteVec.UnitTests/SqliteCollectionTests.cs b/MEVD/test/SqliteVec.UnitTests/SqliteCollectionTests.cs new file mode 100644 index 0000000..8c96522 --- /dev/null +++ b/MEVD/test/SqliteVec.UnitTests/SqliteCollectionTests.cs @@ -0,0 +1,408 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +// TODO: Reimplement these as integration tests, #10464 + +#if DISABLED + +using System; +using System.Collections.Generic; +using System.Data.Common; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.VectorData; +using CommunityToolkit.VectorData.Sqlite; +using Moq; +using Xunit; + +namespace SemanticKernel.Connectors.Sqlite.UnitTests; + +/// +/// Unit tests for class. +/// +public sealed class SqliteCollectionTests +{ + [Theory] + [InlineData(1)] + [InlineData(0)] + public async Task CollectionExistsReturnsValidResultAsync(long tableCount) + { + // Arrange + const string TableName = "CollectionExists"; + + using var fakeCommand = new FakeDbCommand(scalarResult: tableCount); + using var fakeConnection = new FakeDBConnection(fakeCommand); + + var sut = new SqliteVectorStoreRecordCollection>(fakeConnection, TableName); + + // Act + var result = await sut.CollectionExistsAsync(); + + Assert.Equal(tableCount > 0, result); + } + + [Fact] + public async Task CreateCollectionCallsExecuteNonQueryMethodAsync() + { + // Arrange + const string TableName = "CreateCollection"; + + using var fakeCommand = new FakeDbCommand(); + using var fakeConnection = new FakeDBConnection(fakeCommand); + + var sut = new SqliteVectorStoreRecordCollection>(fakeConnection, TableName); + + // Act + await sut.CreateCollectionAsync(); + + // Assert + Assert.Equal(2, fakeCommand.ExecuteNonQueryCallCount); + } + + [Fact] + public async Task EnsureCollectionExistsCallsExecuteNonQueryMethodAsync() + { + // Arrange + const string TableName = "EnsureCollectionExists"; + + using var fakeCommand = new FakeDbCommand(); + using var fakeConnection = new FakeDBConnection(fakeCommand); + + var sut = new SqliteVectorStoreRecordCollection>(fakeConnection, TableName); + + // Act + await sut.EnsureCollectionExistsAsync(); + + // Assert + Assert.Equal(2, fakeCommand.ExecuteNonQueryCallCount); + } + + [Fact] + public async Task DeleteCollectionCallsExecuteNonQueryMethodAsync() + { + // Arrange + using var fakeCommandWithVectorProperty = new FakeDbCommand(); + using var fakeConnectionWithVectorProperty = new FakeDBConnection(fakeCommandWithVectorProperty); + + using var fakeCommandWithoutVectorProperty = new FakeDbCommand(); + using var fakeConnectionWithoutVectorProperty = new FakeDBConnection(fakeCommandWithoutVectorProperty); + + var collectionWithVectorProperty = new SqliteVectorStoreRecordCollection>(fakeConnectionWithVectorProperty, "WithVectorProperty"); + var collectionWithoutVectorProperty = new SqliteVectorStoreRecordCollection>(fakeConnectionWithoutVectorProperty, "WithoutVectorProperty"); + + // Act + await collectionWithVectorProperty.DeleteCollectionAsync(); + await collectionWithoutVectorProperty.DeleteCollectionAsync(); + + // Assert + Assert.Equal(2, fakeCommandWithVectorProperty.ExecuteNonQueryCallCount); + Assert.Equal(1, fakeCommandWithoutVectorProperty.ExecuteNonQueryCallCount); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task VectorizedSearchReturnsRecordAsync(bool includeVectors) + { + // Arrange + var expectedRecord = new TestRecord { Key = 1, Data = "Test data", Vector = new ReadOnlyMemory([1f, 2f, 3f, 4f]) }; + + var mockReader = new Mock(); + + SetupDbDataReaderGetBatch(mockReader, [expectedRecord]); + + mockReader.Setup(l => l.GetOrdinal("distance")).Returns(3); + mockReader.Setup(l => l.GetFieldValue(3)).Returns(5.5); + + using var fakeCommand = new FakeDbCommand(mockReader.Object); + using var fakeConnection = new FakeDBConnection(fakeCommand); + + var sut = new SqliteVectorStoreRecordCollection>(fakeConnection, "VectorizedSearch"); + + // Act + var result = await sut.VectorizedSearchAsync(expectedRecord.Vector, new() { IncludeVectors = includeVectors }).FirstOrDefaultAsync(); + + // Assert + Assert.NotNull(result); + + Assert.Equal(5.5, result.Score); + + AssertRecord(expectedRecord, result.Record, includeVectors); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task GetReturnsValidRecordAsync(bool includeVectors) + { + // Arrange + var expectedRecord = new TestRecord { Key = 1, Data = "Test data", Vector = new ReadOnlyMemory([1f, 2f, 3f, 4f]) }; + + var mockReader = new Mock(); + + SetupDbDataReaderGetBatch(mockReader, [expectedRecord]); + + using var fakeCommand = new FakeDbCommand(mockReader.Object); + using var fakeConnection = new FakeDBConnection(fakeCommand); + + var sut = new SqliteVectorStoreRecordCollection>(fakeConnection, "Get"); + + // Act + var actualRecord = await sut.GetAsync(expectedRecord.Key, new() { IncludeVectors = includeVectors }); + + // Assert + AssertRecord(expectedRecord, actualRecord, includeVectors); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task GetBatchReturnsValidRecordsAsync(bool includeVectors) + { + // Arrange + var expectedRecord1 = new TestRecord { Key = 1, Data = "Test data", Vector = new ReadOnlyMemory([1f, 2f, 3f, 4f]) }; + var expectedRecord2 = new TestRecord { Key = 2, Data = "Test data", Vector = new ReadOnlyMemory([1f, 2f, 3f, 4f]) }; + var expectedRecord3 = new TestRecord { Key = 3, Data = "Test data", Vector = new ReadOnlyMemory([1f, 2f, 3f, 4f]) }; + + var expectedRecords = new List> { expectedRecord1, expectedRecord2, expectedRecord3 }; + + var mockReader = new Mock(); + + SetupDbDataReaderGetBatch(mockReader, expectedRecords); + + using var fakeCommand = new FakeDbCommand(mockReader.Object); + using var fakeConnection = new FakeDBConnection(fakeCommand); + + var sut = new SqliteVectorStoreRecordCollection>(fakeConnection, "GetBatch"); + + // Act + var actualRecords = await sut + .GetBatchAsync(expectedRecords.Select(l => l.Key), new() { IncludeVectors = includeVectors }) + .ToListAsync(); + + // Assert + for (var i = 0; i < actualRecords.Count; i++) + { + AssertRecord(expectedRecords[i], actualRecords[i], includeVectors); + } + } + + [Fact] + public async Task UpsertReturnsKeyAsync() + { + // Arrange + var expectedRecord = new TestRecord { Key = 1, Data = "Test data", Vector = new ReadOnlyMemory([1f, 2f, 3f, 4f]) }; + + var mockReader = new Mock(); + + SetupDbDataReaderUpsertBatch(mockReader, [expectedRecord.Key]); + + using var fakeCommand = new FakeDbCommand(mockReader.Object); + using var fakeConnection = new FakeDBConnection(fakeCommand); + + var sut = new SqliteVectorStoreRecordCollection>(fakeConnection, "Upsert"); + + // Act + var actualKey = await sut.UpsertAsync(expectedRecord); + + // Assert + Assert.Equal(expectedRecord.Key, actualKey); + + Assert.Equal(2, fakeCommand.ExecuteNonQueryCallCount); + } + + [Fact] + public async Task UpsertWithoutVectorPropertyReturnsKeyAsync() + { + // Arrange + var expectedRecord = new TestRecordWithoutVectorProperty { Key = 1, Data = "Test data" }; + + var mockReader = new Mock(); + + SetupDbDataReaderUpsertBatch(mockReader, [expectedRecord.Key]); + + using var fakeCommand = new FakeDbCommand(mockReader.Object); + using var fakeConnection = new FakeDBConnection(fakeCommand); + + var sut = new SqliteVectorStoreRecordCollection>(fakeConnection, "UpsertWithoutVectorProperty"); + + // Act + var actualKey = await sut.UpsertAsync(expectedRecord); + + // Assert + Assert.Equal(expectedRecord.Key, actualKey); + + Assert.Equal(0, fakeCommand.ExecuteNonQueryCallCount); + } + + [Fact] + public async Task UpsertBatchReturnsKeysAsync() + { + // Arrange + var expectedRecord1 = new TestRecord { Key = 1, Data = "Test data", Vector = new ReadOnlyMemory([1f, 2f, 3f, 4f]) }; + var expectedRecord2 = new TestRecord { Key = 2, Data = "Test data", Vector = new ReadOnlyMemory([1f, 2f, 3f, 4f]) }; + var expectedRecord3 = new TestRecord { Key = 3, Data = "Test data", Vector = new ReadOnlyMemory([1f, 2f, 3f, 4f]) }; + + var expectedRecords = new List> { expectedRecord1, expectedRecord2, expectedRecord3 }; + + var mockReader = new Mock(); + + SetupDbDataReaderUpsertBatch(mockReader, expectedRecords.Select(l => l.Key)); + + using var fakeCommand = new FakeDbCommand(mockReader.Object); + using var fakeConnection = new FakeDBConnection(fakeCommand); + + var sut = new SqliteVectorStoreRecordCollection>(fakeConnection, "UpsertBatch"); + + // Act + var actualKeys = await sut.UpsertBatchAsync(expectedRecords).ToListAsync(); + + // Assert + Assert.Contains(expectedRecord1.Key, actualKeys); + Assert.Contains(expectedRecord2.Key, actualKeys); + Assert.Contains(expectedRecord3.Key, actualKeys); + } + + [Fact] + public async Task DeleteCallsExecuteNonQueryMethodAsync() + { + using var fakeCommandWithVectorProperty = new FakeDbCommand(); + using var fakeConnectionWithVectorProperty = new FakeDBConnection(fakeCommandWithVectorProperty); + + using var fakeCommandWithoutVectorProperty = new FakeDbCommand(); + using var fakeConnectionWithoutVectorProperty = new FakeDBConnection(fakeCommandWithoutVectorProperty); + + var collectionWithVectorProperty = new SqliteVectorStoreRecordCollection>(fakeConnectionWithVectorProperty, "WithVectorProperty"); + var collectionWithoutVectorProperty = new SqliteVectorStoreRecordCollection>(fakeConnectionWithoutVectorProperty, "WithoutVectorProperty"); + + // Act + await collectionWithVectorProperty.DeleteAsync(key: 1); + await collectionWithoutVectorProperty.DeleteAsync(key: 2); + + // Assert + Assert.Equal(2, fakeCommandWithVectorProperty.ExecuteNonQueryCallCount); + Assert.Equal(1, fakeCommandWithoutVectorProperty.ExecuteNonQueryCallCount); + } + + [Fact] + public async Task DeleteBatchCallsExecuteNonQueryMethodAsync() + { + using var fakeCommandWithVectorProperty = new FakeDbCommand(); + using var fakeConnectionWithVectorProperty = new FakeDBConnection(fakeCommandWithVectorProperty); + + using var fakeCommandWithoutVectorProperty = new FakeDbCommand(); + using var fakeConnectionWithoutVectorProperty = new FakeDBConnection(fakeCommandWithoutVectorProperty); + + var collectionWithVectorProperty = new SqliteVectorStoreRecordCollection>(fakeConnectionWithVectorProperty, "WithVectorProperty"); + var collectionWithoutVectorProperty = new SqliteVectorStoreRecordCollection>(fakeConnectionWithoutVectorProperty, "WithoutVectorProperty"); + + // Act + await collectionWithVectorProperty.DeleteBatchAsync(keys: [1, 2, 3]); + await collectionWithoutVectorProperty.DeleteBatchAsync(keys: [4, 5, 6]); + + // Assert + Assert.Equal(2, fakeCommandWithVectorProperty.ExecuteNonQueryCallCount); + Assert.Equal(1, fakeCommandWithoutVectorProperty.ExecuteNonQueryCallCount); + } + + #region private + + private static void SetupDbDataReaderGetBatch(Mock mockReader, List> records) + { + var readSequence = mockReader.SetupSequence(l => l.ReadAsync(It.IsAny())); + + var numericKeySequence = mockReader.SetupSequence(l => l.GetInt64(0)); + var stringKeySequence = mockReader.SetupSequence(l => l.GetString(0)); + + var dataSequence = mockReader.SetupSequence(l => l.GetString(1)); + var vectorSequence = mockReader.SetupSequence(l => l[2]); + + mockReader.Setup(l => l.IsDBNull(It.IsAny())).Returns(false); + + mockReader.Setup(l => l.GetOrdinal(nameof(TestRecord.Key))).Returns(0); + mockReader.Setup(l => l.GetOrdinal(nameof(TestRecord.Data))).Returns(1); + mockReader.Setup(l => l.GetOrdinal(nameof(TestRecord.Vector))).Returns(2); + + foreach (var record in records) + { + var vectorBytes = SqliteVectorStoreRecordPropertyMapping.MapVectorForStorageModel(record.Vector); + + if (record.Key is long numericKey) + { + numericKeySequence.Returns(numericKey); + } + + if (record.Key is string stringKey) + { + stringKeySequence.Returns(stringKey); + } + + dataSequence.Returns(record.Data!); + vectorSequence.Returns(vectorBytes); + + readSequence.ReturnsAsync(true); + } + + readSequence.ReturnsAsync(false); + } + + private static void SetupDbDataReaderUpsertBatch(Mock mockReader, IEnumerable keys) + { + var readSequence = mockReader.SetupSequence(l => l.ReadAsync(It.IsAny())); + var keySequence = mockReader.SetupSequence(l => l.GetFieldValue(0)); + + foreach (var key in keys) + { + keySequence.Returns(key); + readSequence.ReturnsAsync(true); + } + + readSequence.ReturnsAsync(false); + } + + private static void AssertRecord(TestRecord expectedRecord, TestRecord? actualRecord, bool includeVectors) + { + Assert.NotNull(actualRecord); + + Assert.Equal(expectedRecord.Key, actualRecord.Key); + Assert.Equal(expectedRecord.Data, actualRecord.Data); + + if (includeVectors) + { + Assert.NotNull(actualRecord.Vector); + Assert.Equal(expectedRecord.Vector!.Value.ToArray(), actualRecord.Vector.Value.Span.ToArray()); + } + else + { + Assert.Null(actualRecord.Vector); + } + } + +#pragma warning disable CA1812 + private sealed class TestRecord + { + [VectorStoreKey] + public TKey? Key { get; set; } + + [VectorStoreData] + public string? Data { get; set; } + + [VectorStoreVector(dimensions: 4, DistanceFunction = DistanceFunction.CosineDistance)] + public ReadOnlyMemory? Vector { get; set; } + } + + private sealed class TestRecordWithoutVectorProperty + { + [VectorStoreKey] + public TKey? Key { get; set; } + + [VectorStoreData] + public string? Data { get; set; } + } +#pragma warning restore CA1812 + + #endregion +} + +#endif diff --git a/MEVD/test/SqliteVec.UnitTests/SqliteCommandBuilderTests.cs b/MEVD/test/SqliteVec.UnitTests/SqliteCommandBuilderTests.cs new file mode 100644 index 0000000..2d7943f --- /dev/null +++ b/MEVD/test/SqliteVec.UnitTests/SqliteCommandBuilderTests.cs @@ -0,0 +1,388 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using Microsoft.Data.Sqlite; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; +using CommunityToolkit.VectorData.SqliteVec; +using Xunit; + +namespace SemanticKernel.Connectors.SqliteVec.UnitTests; + +/// +/// Unit tests for class. +/// +public sealed class SqliteCommandBuilderTests : IDisposable +{ + private readonly SqliteCommand _command; + private readonly SqliteConnection _connection; + + public SqliteCommandBuilderTests() + { + this._command = new() { Connection = this._connection }; + this._connection = new(); + } + + [Fact] + public void ItBuildsTableCountCommand() + { + // Arrange + const string TableName = "TestTable"; + + // Act + var command = SqliteCommandBuilder.BuildTableCountCommand(this._connection, TableName); + + // Assert + Assert.Equal("SELECT count(*) FROM sqlite_master WHERE type='table' AND name=@tableName;", command.CommandText); + Assert.Equal("@tableName", command.Parameters[0].ParameterName); + Assert.Equal(TableName, command.Parameters[0].Value); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void ItBuildsCreateTableCommand(bool ifNotExists) + { + // Arrange + const string TableName = "TestTable"; + + var columns = new List + { + new("Column1", "Type1", isPrimary: true), + new("Column2", "Type2", isPrimary: false) { IsNullable = true, Configuration = new() { ["distance_metric"] = "l2" } }, + new("Column3", "Type3", isPrimary: false) { IsNullable = false }, + new("Column4", "Type4", isPrimary: false) { IsNullable = true }, + }; + + // Act + var command = SqliteCommandBuilder.BuildCreateTableCommand(this._connection, TableName, columns, ifNotExists); + + // Assert + Assert.Contains("CREATE TABLE", command.CommandText); + Assert.Contains(TableName, command.CommandText); + + Assert.Equal(ifNotExists, command.CommandText.Contains("IF NOT EXISTS")); + + Assert.Contains("\"Column1\" Type1 PRIMARY KEY", command.CommandText); + Assert.Contains("\"Column2\" Type2 distance_metric=l2", command.CommandText); + Assert.Contains("\"Column3\" Type3 NOT NULL", command.CommandText); + Assert.Contains("\"Column4\" Type4", command.CommandText); + Assert.DoesNotContain("\"Column4\" Type4 NOT NULL", command.CommandText); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void ItBuildsCreateVirtualTableCommand(bool ifNotExists) + { + // Arrange + const string TableName = "TestTable"; + + var columns = new List + { + new("Column1", "Type1", isPrimary: true), + new("Column2", "Type2", isPrimary: false) { IsNullable = true, Configuration = new() { ["distance_metric"] = "l2" } }, + }; + + // Act + var command = SqliteCommandBuilder.BuildCreateVirtualTableCommand(this._connection, TableName, columns, ifNotExists); + + // Assert + Assert.Contains("CREATE VIRTUAL TABLE", command.CommandText); + Assert.Contains(TableName, command.CommandText); + Assert.Contains("USING vec0", command.CommandText); + + Assert.Equal(ifNotExists, command.CommandText.Contains("IF NOT EXISTS")); + + Assert.Contains("Column1 Type1 PRIMARY KEY", command.CommandText); + Assert.Contains("Column2 Type2 distance_metric=l2", command.CommandText); + } + + [Fact] + public void ItBuildsDropTableCommand() + { + // Arrange + const string TableName = "TestTable"; + + // Act + var command = SqliteCommandBuilder.BuildDropTableCommand(this._connection, TableName); + + // Assert + Assert.Equal("DROP TABLE IF EXISTS \"TestTable\";", command.CommandText); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void ItBuildsInsertCommand_without_autogenerated_key(bool replaceIfExists) + { + // Arrange + const string TableName = "TestTable"; + + var model = BuildModel( + [ + new VectorStoreKeyProperty("Id", typeof(int)), + new VectorStoreDataProperty("Name", typeof(string)), + new VectorStoreDataProperty("Age", typeof(string)), + new VectorStoreDataProperty("Address", typeof(string)), + ]); + + var records = new List> + { + new() { ["Id"] = 1, ["Name"] = "NameValue1", ["Age"] = "AgeValue1", ["Address"] = "AddressValue1" }, + new() { ["Id"] = 2, ["Name"] = "NameValue2", ["Age"] = "AgeValue2", ["Address"] = "AddressValue2" }, + }; + + // Act + var command = SqliteCommandBuilder.BuildInsertCommand( + this._connection, + TableName, + model, + records, + generatedEmbeddings: null, + data: true, + replaceIfExists: replaceIfExists); + + // Assert + Assert.Equal(replaceIfExists, command.CommandText.Contains("OR REPLACE")); + + Assert.Contains($"INTO \"{TableName}\" (\"Id\", \"Name\", \"Age\", \"Address\")", command.CommandText); + Assert.Contains("VALUES (@Id0, @Name0, @Age0, @Address0)", command.CommandText); + Assert.Contains("VALUES (@Id1, @Name1, @Age1, @Address1)", command.CommandText); + Assert.DoesNotContain("RETURNING", command.CommandText); + + Assert.Equal("@Id0", command.Parameters[0].ParameterName); + Assert.Equal(1, command.Parameters[0].Value); + + Assert.Equal("@Name0", command.Parameters[1].ParameterName); + Assert.Equal("NameValue1", command.Parameters[1].Value); + + Assert.Equal("@Age0", command.Parameters[2].ParameterName); + Assert.Equal("AgeValue1", command.Parameters[2].Value); + + Assert.Equal("@Address0", command.Parameters[3].ParameterName); + Assert.Equal("AddressValue1", command.Parameters[3].Value); + + Assert.Equal("@Id1", command.Parameters[4].ParameterName); + Assert.Equal(2, command.Parameters[4].Value); + + Assert.Equal("@Name1", command.Parameters[5].ParameterName); + Assert.Equal("NameValue2", command.Parameters[5].Value); + + Assert.Equal("@Age1", command.Parameters[6].ParameterName); + Assert.Equal("AgeValue2", command.Parameters[6].Value); + + Assert.Equal("@Address1", command.Parameters[7].ParameterName); + Assert.Equal("AddressValue2", command.Parameters[7].Value); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void ItBuildsInsertCommand_with_autogenerated_key(bool replaceIfExists) + { + // Arrange + const string TableName = "TestTable"; + + var model = BuildModel( + [ + new VectorStoreKeyProperty("Id", typeof(int)), + new VectorStoreDataProperty("Name", typeof(string)), + new VectorStoreDataProperty("Age", typeof(string)), + new VectorStoreDataProperty("Address", typeof(string)), + ]); + + var records = new List> + { + new() { ["Id"] = default(int), ["Name"] = "NameValue1", ["Age"] = "AgeValue1", ["Address"] = "AddressValue1" }, + new() { ["Id"] = default(int), ["Name"] = "NameValue2", ["Age"] = "AgeValue2", ["Address"] = "AddressValue2" }, + }; + + // Act + var command = SqliteCommandBuilder.BuildInsertCommand( + this._connection, + TableName, + model, + records, + generatedEmbeddings: null, + data: true, + replaceIfExists: replaceIfExists); + + // Assert + Assert.DoesNotContain("OR REPLACE", command.CommandText); + + Assert.Contains($"INTO \"{TableName}\" (\"Name\", \"Age\", \"Address\")", command.CommandText); + Assert.Contains("VALUES (@Name0, @Age0, @Address0)", command.CommandText); + Assert.Contains("VALUES (@Name1, @Age1, @Address1)", command.CommandText); + Assert.Contains("RETURNING \"Id\"", command.CommandText); + + Assert.Equal("@Name0", command.Parameters[0].ParameterName); + Assert.Equal("NameValue1", command.Parameters[0].Value); + + Assert.Equal("@Name1", command.Parameters[3].ParameterName); + Assert.Equal("NameValue2", command.Parameters[3].Value); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData("Age")] + public void ItBuildsSelectCommand(string? orderByPropertyName) + { + // Arrange + const string TableName = "TestTable"; + + var model = BuildModel( + [ + new VectorStoreKeyProperty("Id", typeof(string)), + new VectorStoreDataProperty("Name", typeof(string)), + new VectorStoreDataProperty("Age", typeof(string)), + new VectorStoreDataProperty("Address", typeof(string)), + ]); + var conditions = new List + { + new SqliteWhereEqualsCondition("Name", "NameValue"), + new SqliteWhereInCondition("Age", [10, 20, 30]), + }; + FilteredRecordRetrievalOptions> filterOptions = new(); + if (!string.IsNullOrWhiteSpace(orderByPropertyName)) + { + filterOptions.OrderBy = orderBy => orderBy.Ascending(record => record[orderByPropertyName]); + } + + // Act + var command = SqliteCommandBuilder.BuildSelectDataCommand>(this._connection, TableName, model, conditions, filterOptions); + + // Assert + Assert.Contains("SELECT \"Id\",\"Name\",\"Age\",\"Address\"", command.CommandText); + Assert.Contains($"FROM \"{TableName}\"", command.CommandText); + + Assert.Contains("\"Name\" = @Name0", command.CommandText); + Assert.Contains("\"Age\" IN (@Age0, @Age1, @Age2)", command.CommandText); + + Assert.Equal(!string.IsNullOrWhiteSpace(orderByPropertyName), command.CommandText.Contains($"ORDER BY \"{orderByPropertyName}\"")); + + Assert.Equal("@Name0", command.Parameters[0].ParameterName); + Assert.Equal("NameValue", command.Parameters[0].Value); + + Assert.Equal("@Age0", command.Parameters[1].ParameterName); + Assert.Equal(10, command.Parameters[1].Value); + + Assert.Equal("@Age1", command.Parameters[2].ParameterName); + Assert.Equal(20, command.Parameters[2].Value); + + Assert.Equal("@Age2", command.Parameters[3].ParameterName); + Assert.Equal(30, command.Parameters[3].Value); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData("Age")] + public void ItBuildsSelectInnerJoinCommand(string? orderByPropertyName) + { + // Arrange + const string DataTable = "DataTable"; + const string VectorTable = "VectorTable"; + const string JoinColumnName = "Id"; + + var model = BuildModel( + [ + new VectorStoreKeyProperty("Id", typeof(string)), + new VectorStoreDataProperty("Name", typeof(string)), + new VectorStoreVectorProperty("Age", typeof(ReadOnlyMemory), 10), + new VectorStoreVectorProperty("Address", typeof(ReadOnlyMemory), 10), + ]); + + var conditions = new List + { + new SqliteWhereEqualsCondition("Name", "NameValue"), + new SqliteWhereInCondition("Age", [10, 20, 30]), + }; + FilteredRecordRetrievalOptions> filterOptions = new(); + if (!string.IsNullOrWhiteSpace(orderByPropertyName)) + { + filterOptions.OrderBy = orderBy => orderBy.Ascending(record => record[orderByPropertyName]); + } + + // Act + var command = SqliteCommandBuilder.BuildSelectInnerJoinCommand( + this._connection, + VectorTable, + DataTable, + JoinColumnName, + model, + conditions, + true, + filterOptions); + + // Assert + Assert.Contains("SELECT \"DataTable\".\"Id\",\"DataTable\".\"Name\",\"VectorTable\".\"Age\",\"VectorTable\".\"Address\"", command.CommandText); + Assert.Contains("FROM \"VectorTable\"", command.CommandText); + + Assert.Contains("INNER JOIN \"DataTable\" ON \"VectorTable\".\"Id\" = \"DataTable\".\"Id\"", command.CommandText); + + Assert.Contains("\"Name\" = @Name0", command.CommandText); + Assert.Contains("\"Age\" IN (@Age0, @Age1, @Age2)", command.CommandText); + + Assert.Equal(!string.IsNullOrWhiteSpace(orderByPropertyName), command.CommandText.Contains($"ORDER BY \"DataTable\".\"{orderByPropertyName}\"")); + + Assert.Equal("@Name0", command.Parameters[0].ParameterName); + Assert.Equal("NameValue", command.Parameters[0].Value); + + Assert.Equal("@Age0", command.Parameters[1].ParameterName); + Assert.Equal(10, command.Parameters[1].Value); + + Assert.Equal("@Age1", command.Parameters[2].ParameterName); + Assert.Equal(20, command.Parameters[2].Value); + + Assert.Equal("@Age2", command.Parameters[3].ParameterName); + Assert.Equal(30, command.Parameters[3].Value); + } + + [Fact] + public void ItBuildsDeleteCommand() + { + // Arrange + const string TableName = "TestTable"; + + var conditions = new List + { + new SqliteWhereEqualsCondition("Name", "NameValue"), + new SqliteWhereInCondition("Age", [10, 20, 30]), + }; + + // Act + var command = SqliteCommandBuilder.BuildDeleteCommand(this._connection, TableName, conditions); + + // Assert + Assert.Contains("DELETE FROM \"TestTable\"", command.CommandText); + + Assert.Contains("\"Name\" = @Name0", command.CommandText); + Assert.Contains("\"Age\" IN (@Age0, @Age1, @Age2)", command.CommandText); + + Assert.Equal("@Name0", command.Parameters[0].ParameterName); + Assert.Equal("NameValue", command.Parameters[0].Value); + + Assert.Equal("@Age0", command.Parameters[1].ParameterName); + Assert.Equal(10, command.Parameters[1].Value); + + Assert.Equal("@Age1", command.Parameters[2].ParameterName); + Assert.Equal(20, command.Parameters[2].Value); + + Assert.Equal("@Age2", command.Parameters[3].ParameterName); + Assert.Equal(30, command.Parameters[3].Value); + } + + public void Dispose() + { + this._command.Dispose(); + this._connection.Dispose(); + } + + private static CollectionModel BuildModel(List properties) + => new SqliteModelBuilder() + .BuildDynamic(new() { Properties = properties }, defaultEmbeddingGenerator: null); +} diff --git a/MEVD/test/SqliteVec.UnitTests/SqliteConditionsTests.cs b/MEVD/test/SqliteVec.UnitTests/SqliteConditionsTests.cs new file mode 100644 index 0000000..35ec8b1 --- /dev/null +++ b/MEVD/test/SqliteVec.UnitTests/SqliteConditionsTests.cs @@ -0,0 +1,92 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using CommunityToolkit.VectorData.SqliteVec; +using Xunit; + +namespace SemanticKernel.Connectors.SqliteVec.UnitTests; + +/// +/// Unit tests for SQLite condition classes. +/// +public sealed class SqliteConditionsTests +{ + [Fact] + public void SqliteWhereEqualsConditionWithoutParameterNamesThrowsException() + { + // Arrange + var condition = new SqliteWhereEqualsCondition("Name", "Value"); + + // Act & Assert + Assert.Throws(() => condition.BuildQuery([])); + } + + [Theory] + [InlineData(null, "\"Name\" = @Name0")] + [InlineData("", "\"Name\" = @Name0")] + [InlineData("TableName", "\"TableName\".\"Name\" = @Name0")] + public void SqliteWhereEqualsConditionBuildsValidQuery(string? tableName, string expectedQuery) + { + // Arrange + var condition = new SqliteWhereEqualsCondition("Name", "Value") { TableName = tableName }; + + // Act + var query = condition.BuildQuery(["@Name0"]); + + // Assert + Assert.Equal(expectedQuery, query); + } + + [Fact] + public void SqliteWhereInConditionWithoutParameterNamesThrowsException() + { + // Arrange + var condition = new SqliteWhereInCondition("Name", ["Value1", "Value2"]); + + // Act & Assert + Assert.Throws(() => condition.BuildQuery([])); + } + + [Theory] + [InlineData(null, "\"Name\" IN (@Name0, @Name1)")] + [InlineData("", "\"Name\" IN (@Name0, @Name1)")] + [InlineData("TableName", "\"TableName\".\"Name\" IN (@Name0, @Name1)")] + public void SqliteWhereInConditionBuildsValidQuery(string? tableName, string expectedQuery) + { + // Arrange + var condition = new SqliteWhereInCondition("Name", ["Value1", "Value2"]) { TableName = tableName }; + + // Act + var query = condition.BuildQuery(["@Name0", "@Name1"]); + + // Assert + Assert.Equal(expectedQuery, query); + } + + [Fact] + public void SqliteWhereMatchConditionWithoutParameterNamesThrowsException() + { + // Arrange + var condition = new SqliteWhereMatchCondition("Name", "Value"); + + // Act & Assert + Assert.Throws(() => condition.BuildQuery([])); + } + + [Theory] + [InlineData(null, "\"Name\" MATCH @Name0")] + [InlineData("", "\"Name\" MATCH @Name0")] + [InlineData("TableName", "\"TableName\".\"Name\" MATCH @Name0")] + public void SqliteWhereMatchConditionBuildsValidQuery(string? tableName, string expectedQuery) + { + // Arrange + var condition = new SqliteWhereMatchCondition("Name", "Value") { TableName = tableName }; + + // Act + var query = condition.BuildQuery(["@Name0"]); + + // Assert + Assert.Equal(expectedQuery, query); + } +} diff --git a/MEVD/test/SqliteVec.UnitTests/SqliteHotel.cs b/MEVD/test/SqliteVec.UnitTests/SqliteHotel.cs new file mode 100644 index 0000000..619dd6c --- /dev/null +++ b/MEVD/test/SqliteVec.UnitTests/SqliteHotel.cs @@ -0,0 +1,38 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using Microsoft.Extensions.VectorData; + +namespace SemanticKernel.Connectors.SqliteVec.UnitTests; + +public class SqliteHotel() +{ + /// The key of the record. + [VectorStoreKey] + public TKey? HotelId { get; init; } + + /// A string metadata field. + [VectorStoreData(IsIndexed = true)] + public string? HotelName { get; set; } + + /// An int metadata field. + [VectorStoreData] + public int HotelCode { get; set; } + + /// A float metadata field. + [VectorStoreData] + public float? HotelRating { get; set; } + + /// A bool metadata field. + [VectorStoreData(StorageName = "parking_is_included")] + public bool ParkingIncluded { get; set; } + + /// A data field. + [VectorStoreData] + public string? Description { get; set; } + + /// A vector field. + [VectorStoreVector(dimensions: 4, DistanceFunction = DistanceFunction.EuclideanDistance)] + public ReadOnlyMemory? DescriptionEmbedding { get; set; } +} diff --git a/MEVD/test/SqliteVec.UnitTests/SqlitePropertyMappingTests.cs b/MEVD/test/SqliteVec.UnitTests/SqlitePropertyMappingTests.cs new file mode 100644 index 0000000..810a487 --- /dev/null +++ b/MEVD/test/SqliteVec.UnitTests/SqlitePropertyMappingTests.cs @@ -0,0 +1,97 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; +using CommunityToolkit.VectorData.SqliteVec; +using Xunit; + +namespace SemanticKernel.Connectors.SqliteVec.UnitTests; + +/// +/// Unit tests for class. +/// +public sealed class SqlitePropertyMappingTests +{ + [Fact] + public void MapVectorForStorageModelReturnsByteArray() + { + // Arrange + var vector = new ReadOnlyMemory([1.1f, 2.2f, 3.3f, 4.4f]); + + // Act + var storageModelVector = SqlitePropertyMapping.MapVectorForStorageModel(vector); + + // Assert + Assert.IsType(storageModelVector); + Assert.True(storageModelVector.Length > 0); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void GetColumnsReturnsCollectionOfColumns(bool data) + { + // Arrange + var properties = new List() + { + new KeyPropertyModel("Key", typeof(string)) { StorageName = "Key" }, + new DataPropertyModel("Data", typeof(int)) { StorageName = "my_data", IsIndexed = true }, + new DataPropertyModel("NullableData", typeof(int?)) { StorageName = "nullable_data" }, + new DataPropertyModel("StringData", typeof(string)) { StorageName = "string_data" }, + new VectorPropertyModel("Vector", typeof(ReadOnlyMemory)) + { + Dimensions = 4, + DistanceFunction = DistanceFunction.ManhattanDistance, + StorageName = "Vector" + } + }; + + // Act + var columns = SqlitePropertyMapping.GetColumns(properties, data: data); + + // Assert + Assert.Equal("Key", columns[0].Name); + Assert.Equal("TEXT", columns[0].Type); + Assert.True(columns[0].IsPrimary); + Assert.Null(columns[0].Configuration); + Assert.False(columns[0].HasIndex); + // string key is a reference type, so nullable + Assert.True(columns[0].IsNullable); + + if (data) + { + Assert.Equal("my_data", columns[1].Name); + Assert.Equal("INTEGER", columns[1].Type); + Assert.False(columns[1].IsPrimary); + Assert.Null(columns[1].Configuration); + Assert.True(columns[1].HasIndex); + // int is a non-nullable value type + Assert.False(columns[1].IsNullable); + + Assert.Equal("nullable_data", columns[2].Name); + Assert.Equal("INTEGER", columns[2].Type); + Assert.False(columns[2].IsPrimary); + // int? is a nullable value type + Assert.True(columns[2].IsNullable); + + Assert.Equal("string_data", columns[3].Name); + Assert.Equal("TEXT", columns[3].Type); + // string is a reference type, so nullable + Assert.True(columns[3].IsNullable); + } + else + { + Assert.Equal("Vector", columns[1].Name); + Assert.Equal("FLOAT[4]", columns[1].Type); + Assert.False(columns[1].IsPrimary); + Assert.NotNull(columns[1].Configuration); + Assert.False(columns[1].HasIndex); + Assert.Equal("l1", columns[1].Configuration!["distance_metric"]); + // ReadOnlyMemory is a non-nullable value type + Assert.False(columns[1].IsNullable); + } + } +} diff --git a/MEVD/test/SqliteVec.UnitTests/SqliteVec.UnitTests.csproj b/MEVD/test/SqliteVec.UnitTests/SqliteVec.UnitTests.csproj new file mode 100644 index 0000000..6636f99 --- /dev/null +++ b/MEVD/test/SqliteVec.UnitTests/SqliteVec.UnitTests.csproj @@ -0,0 +1,34 @@ + + + + CommunityToolkit.VectorData.SqliteVec.UnitTests + CommunityToolkit.VectorData.SqliteVec.UnitTests + net10.0 + true + enable + disable + false + $(NoWarn);SKEXP0001,VSTHRD111,CA2007,CS1591 + $(NoWarn);MEVD9001 + + + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + diff --git a/MEVD/test/SqliteVec.UnitTests/SqliteVectorStoreTests.cs b/MEVD/test/SqliteVec.UnitTests/SqliteVectorStoreTests.cs new file mode 100644 index 0000000..6966d98 --- /dev/null +++ b/MEVD/test/SqliteVec.UnitTests/SqliteVectorStoreTests.cs @@ -0,0 +1,81 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +// TODO: Reimplement these as integration tests, #10464 + +#if DISABLED + +using System; +using System.Data.Common; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Data.Sqlite; +using Microsoft.Extensions.VectorData; +using CommunityToolkit.VectorData.Sqlite; +using Moq; +using Xunit; + +namespace SemanticKernel.Connectors.Sqlite.UnitTests; + +/// +/// Unit tests for class. +/// +public sealed class SqliteVectorStoreTests +{ + [Fact] + public void GetCollectionWithNotSupportedKeyThrowsException() + { + // Arrange + var sut = new SqliteVectorStore(Mock.Of()); + + // Act & Assert + Assert.Throws(() => sut.GetCollection>("collection")); + } + + [Fact] + public void GetCollectionWithSupportedKeyReturnsCollection() + { + // Arrange + var sut = new SqliteVectorStore(Mock.Of()); + + // Act + var collectionWithNumericKey = sut.GetCollection>("collection1"); + var collectionWithStringKey = sut.GetCollection>("collection2"); + + // Assert + Assert.NotNull(collectionWithNumericKey); + Assert.NotNull(collectionWithStringKey); + } + + [Fact] + public async Task ListCollectionNamesReturnsCollectionNamesAsync() + { + // Arrange + var mockReader = new Mock(); + mockReader + .SetupSequence(l => l.ReadAsync(It.IsAny())) + .ReturnsAsync(true) + .ReturnsAsync(true) + .ReturnsAsync(false); + + mockReader + .SetupSequence(l => l.GetString(It.IsAny())) + .Returns("collection1") + .Returns("collection2"); + + using var fakeCommand = new FakeDbCommand(mockReader.Object); + using var fakeConnection = new FakeDBConnection(fakeCommand); + + var sut = new SqliteVectorStore(fakeConnection); + + // Act + var collections = await sut.ListCollectionNamesAsync().ToListAsync(); + + // Assert + Assert.Contains("collection1", collections); + Assert.Contains("collection2", collections); + } +} + +#endif diff --git a/MEVD/test/Weaviate.ConformanceTests/ModelTests/WeaviateBasicModelTests.cs b/MEVD/test/Weaviate.ConformanceTests/ModelTests/WeaviateBasicModelTests.cs new file mode 100644 index 0000000..ecb8f2f --- /dev/null +++ b/MEVD/test/Weaviate.ConformanceTests/ModelTests/WeaviateBasicModelTests.cs @@ -0,0 +1,27 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Weaviate.ConformanceTests.Support; +using Xunit; + +namespace Weaviate.ConformanceTests.ModelTests; + +public class WeaviateBasicModelTests_NamedVectors(WeaviateBasicModelTests_NamedVectors.Fixture fixture) + : BasicModelTests(fixture), IClassFixture +{ + public new class Fixture : BasicModelTests.Fixture + { + public override TestStore TestStore => WeaviateTestStore.NamedVectorsInstance; + } +} + +public class WeaviateBasicModelTests_UnnamedVectors(WeaviateBasicModelTests_UnnamedVectors.Fixture fixture) + : BasicModelTests(fixture), IClassFixture +{ + public new class Fixture : BasicModelTests.Fixture + { + public override TestStore TestStore => WeaviateTestStore.UnnamedVectorInstance; + } +} diff --git a/MEVD/test/Weaviate.ConformanceTests/ModelTests/WeaviateDynamicModelTests.cs b/MEVD/test/Weaviate.ConformanceTests/ModelTests/WeaviateDynamicModelTests.cs new file mode 100644 index 0000000..69a867e --- /dev/null +++ b/MEVD/test/Weaviate.ConformanceTests/ModelTests/WeaviateDynamicModelTests.cs @@ -0,0 +1,27 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Weaviate.ConformanceTests.Support; +using Xunit; + +namespace Weaviate.ConformanceTests.ModelTests; + +public class WeaviateDynamicModelTests_NamedVectors(WeaviateDynamicModelTests_NamedVectors.Fixture fixture) + : DynamicModelTests(fixture), IClassFixture +{ + public new class Fixture : DynamicModelTests.Fixture + { + public override TestStore TestStore => WeaviateTestStore.NamedVectorsInstance; + } +} + +public class WeaviateDynamicModelTests_UnnamedVectors(WeaviateDynamicModelTests_UnnamedVectors.Fixture fixture) + : DynamicModelTests(fixture), IClassFixture +{ + public new class Fixture : DynamicModelTests.Fixture + { + public override TestStore TestStore => WeaviateTestStore.UnnamedVectorInstance; + } +} diff --git a/MEVD/test/Weaviate.ConformanceTests/ModelTests/WeaviateMultiVectorModelTests.cs b/MEVD/test/Weaviate.ConformanceTests/ModelTests/WeaviateMultiVectorModelTests.cs new file mode 100644 index 0000000..11f36ad --- /dev/null +++ b/MEVD/test/Weaviate.ConformanceTests/ModelTests/WeaviateMultiVectorModelTests.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Weaviate.ConformanceTests.Support; +using Xunit; + +namespace Weaviate.ConformanceTests.ModelTests; + +public class WeaviateMultiVectorModelTests(WeaviateMultiVectorModelTests.Fixture fixture) + : MultiVectorModelTests(fixture), IClassFixture +{ + public new class Fixture : MultiVectorModelTests.Fixture + { + public override TestStore TestStore => WeaviateTestStore.NamedVectorsInstance; + } +} diff --git a/MEVD/test/Weaviate.ConformanceTests/ModelTests/WeaviateNoDataModelTests.cs b/MEVD/test/Weaviate.ConformanceTests/ModelTests/WeaviateNoDataModelTests.cs new file mode 100644 index 0000000..71dccb5 --- /dev/null +++ b/MEVD/test/Weaviate.ConformanceTests/ModelTests/WeaviateNoDataModelTests.cs @@ -0,0 +1,37 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Weaviate.ConformanceTests.Support; +using Xunit; + +namespace Weaviate.ConformanceTests.ModelTests; + +public class WeaviateNoDataModelTests_NamedVectors(WeaviateNoDataModelTests_NamedVectors.Fixture fixture) + : NoDataModelTests(fixture), IClassFixture +{ + public new class Fixture : NoDataModelTests.Fixture + { + public override TestStore TestStore => WeaviateTestStore.NamedVectorsInstance; + + /// + /// Weaviate collections must start with an uppercase letter. + /// + protected override string CollectionNameBase => "NoDataNamedCollection"; + } +} + +public class WeaviateNoDataModelTests_UnnamedVector(WeaviateNoDataModelTests_UnnamedVector.Fixture fixture) + : NoDataModelTests(fixture), IClassFixture +{ + public new class Fixture : NoDataModelTests.Fixture + { + public override TestStore TestStore => WeaviateTestStore.UnnamedVectorInstance; + + /// + /// Weaviate collections must start with an uppercase letter. + /// + protected override string CollectionNameBase => "NoDataUnnamedCollection"; + } +} diff --git a/MEVD/test/Weaviate.ConformanceTests/ModelTests/WeaviateNoVectorModelTests.cs b/MEVD/test/Weaviate.ConformanceTests/ModelTests/WeaviateNoVectorModelTests.cs new file mode 100644 index 0000000..f77cd56 --- /dev/null +++ b/MEVD/test/Weaviate.ConformanceTests/ModelTests/WeaviateNoVectorModelTests.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Weaviate.ConformanceTests.Support; +using Xunit; + +namespace Weaviate.ConformanceTests.ModelTests; + +public class WeaviateNoVectorModelTests(WeaviateNoVectorModelTests.Fixture fixture) + : NoVectorModelTests(fixture), IClassFixture +{ + public new class Fixture : NoVectorModelTests.Fixture + { + public override TestStore TestStore => WeaviateTestStore.NamedVectorsInstance; + } +} diff --git a/MEVD/test/Weaviate.ConformanceTests/Properties/AssemblyInfo.cs b/MEVD/test/Weaviate.ConformanceTests/Properties/AssemblyInfo.cs new file mode 100644 index 0000000..53616c2 --- /dev/null +++ b/MEVD/test/Weaviate.ConformanceTests/Properties/AssemblyInfo.cs @@ -0,0 +1,2 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. diff --git a/MEVD/test/Weaviate.ConformanceTests/Support/TestContainer/WeaviateBuilder.cs b/MEVD/test/Weaviate.ConformanceTests/Support/TestContainer/WeaviateBuilder.cs new file mode 100644 index 0000000..003d9f3 --- /dev/null +++ b/MEVD/test/Weaviate.ConformanceTests/Support/TestContainer/WeaviateBuilder.cs @@ -0,0 +1,49 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Docker.DotNet.Models; +using DotNet.Testcontainers.Builders; +using DotNet.Testcontainers.Configurations; + +namespace Weaviate.ConformanceTests.Support.TestContainer; + +public sealed class WeaviateBuilder : ContainerBuilder +{ + public const string WeaviateImage = "semitechnologies/weaviate:1.28.12"; + public const ushort WeaviateHttpPort = 8080; + public const ushort WeaviateGrpcPort = 50051; + + public WeaviateBuilder() : this(new WeaviateConfiguration()) => this.DockerResourceConfiguration = this.Init().DockerResourceConfiguration; + + private WeaviateBuilder(WeaviateConfiguration dockerResourceConfiguration) : base(dockerResourceConfiguration) + => this.DockerResourceConfiguration = dockerResourceConfiguration; + + public override WeaviateContainer Build() + { + this.Validate(); + return new WeaviateContainer(this.DockerResourceConfiguration); + } + + protected override WeaviateBuilder Init() + => base.Init() + .WithImage(WeaviateImage) + .WithPortBinding(WeaviateHttpPort, true) + .WithPortBinding(WeaviateGrpcPort, true) + .WithEnvironment("AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED", "true") + .WithEnvironment("PERSISTENCE_DATA_PATH", "/var/lib/weaviate") + .WithWaitStrategy(Wait.ForUnixContainer() + .UntilInternalTcpPortIsAvailable(WeaviateHttpPort) + .UntilInternalTcpPortIsAvailable(WeaviateGrpcPort) + .UntilHttpRequestIsSucceeded(r => r.ForPath("/v1/.well-known/ready").ForPort(WeaviateHttpPort))); + + protected override WeaviateBuilder Clone(IResourceConfiguration resourceConfiguration) + => this.Merge(this.DockerResourceConfiguration, new WeaviateConfiguration(resourceConfiguration)); + + protected override WeaviateBuilder Merge(WeaviateConfiguration oldValue, WeaviateConfiguration newValue) + => new(new WeaviateConfiguration(oldValue, newValue)); + + protected override WeaviateConfiguration DockerResourceConfiguration { get; } + + protected override WeaviateBuilder Clone(IContainerConfiguration resourceConfiguration) + => this.Merge(this.DockerResourceConfiguration, new WeaviateConfiguration(resourceConfiguration)); +} diff --git a/MEVD/test/Weaviate.ConformanceTests/Support/TestContainer/WeaviateConfiguration.cs b/MEVD/test/Weaviate.ConformanceTests/Support/TestContainer/WeaviateConfiguration.cs new file mode 100644 index 0000000..102c45d --- /dev/null +++ b/MEVD/test/Weaviate.ConformanceTests/Support/TestContainer/WeaviateConfiguration.cs @@ -0,0 +1,54 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Docker.DotNet.Models; +using DotNet.Testcontainers.Configurations; + +namespace Weaviate.ConformanceTests.Support.TestContainer; + +public sealed class WeaviateConfiguration : ContainerConfiguration +{ + /// + /// Initializes a new instance of the class. + /// + public WeaviateConfiguration() + { + } + + /// + /// Initializes a new instance of the class. + /// + /// The Docker resource configuration. + public WeaviateConfiguration(IResourceConfiguration resourceConfiguration) + : base(resourceConfiguration) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// The Docker resource configuration. + public WeaviateConfiguration(IContainerConfiguration resourceConfiguration) + : base(resourceConfiguration) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// The Docker resource configuration. + public WeaviateConfiguration(WeaviateConfiguration resourceConfiguration) + : this(new WeaviateConfiguration(), resourceConfiguration) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// The old Docker resource configuration. + /// The new Docker resource configuration. + public WeaviateConfiguration(WeaviateConfiguration oldValue, WeaviateConfiguration newValue) + : base(oldValue, newValue) + { + } +} diff --git a/MEVD/test/Weaviate.ConformanceTests/Support/TestContainer/WeaviateContainer.cs b/MEVD/test/Weaviate.ConformanceTests/Support/TestContainer/WeaviateContainer.cs new file mode 100644 index 0000000..edd4c26 --- /dev/null +++ b/MEVD/test/Weaviate.ConformanceTests/Support/TestContainer/WeaviateContainer.cs @@ -0,0 +1,8 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using DotNet.Testcontainers.Containers; + +namespace Weaviate.ConformanceTests.Support.TestContainer; + +public class WeaviateContainer(WeaviateConfiguration configuration) : DockerContainer(configuration); diff --git a/MEVD/test/Weaviate.ConformanceTests/Support/WeaviateTestStore.cs b/MEVD/test/Weaviate.ConformanceTests/Support/WeaviateTestStore.cs new file mode 100644 index 0000000..076c7c4 --- /dev/null +++ b/MEVD/test/Weaviate.ConformanceTests/Support/WeaviateTestStore.cs @@ -0,0 +1,42 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#if NET472 +using System.Net.Http; +#endif +using CommunityToolkit.VectorData.Weaviate; +using VectorData.ConformanceTests.Support; +using Weaviate.ConformanceTests.Support.TestContainer; + +namespace Weaviate.ConformanceTests.Support; + +#pragma warning restore CA2213 // Disposable fields should be disposed + +public sealed class WeaviateTestStore : TestStore +{ + public static WeaviateTestStore NamedVectorsInstance { get; } = new(hasNamedVectors: true); + public static WeaviateTestStore UnnamedVectorInstance { get; } = new(hasNamedVectors: false); + + public override string DefaultDistanceFunction => Microsoft.Extensions.VectorData.DistanceFunction.CosineDistance; + + private readonly WeaviateContainer _container = new WeaviateBuilder().Build(); + private readonly bool _hasNamedVectors; + public HttpClient? _httpClient { get; private set; } + + public HttpClient Client => this._httpClient ?? throw new InvalidOperationException("Not initialized"); + + public WeaviateVectorStore GetVectorStore(WeaviateVectorStoreOptions options) + => new(this.Client, options); + + private WeaviateTestStore(bool hasNamedVectors) => this._hasNamedVectors = hasNamedVectors; + + protected override async Task StartAsync() + { + await this._container.StartAsync(); + this._httpClient = new HttpClient { BaseAddress = new Uri($"http://localhost:{this._container.GetMappedPublicPort(WeaviateBuilder.WeaviateHttpPort)}/v1/") }; + this.DefaultVectorStore = new WeaviateVectorStore(this._httpClient, new() { HasNamedVectors = this._hasNamedVectors }); + } + + protected override Task StopAsync() + => this._container.StopAsync(); +} diff --git a/MEVD/test/Weaviate.ConformanceTests/TypeTests/WeaviateDataTypeTests.cs b/MEVD/test/Weaviate.ConformanceTests/TypeTests/WeaviateDataTypeTests.cs new file mode 100644 index 0000000..6328bee --- /dev/null +++ b/MEVD/test/Weaviate.ConformanceTests/TypeTests/WeaviateDataTypeTests.cs @@ -0,0 +1,35 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using VectorData.ConformanceTests.Support; +using VectorData.ConformanceTests.TypeTests; +using Weaviate.ConformanceTests.Support; +using Xunit; + +namespace Weaviate.ConformanceTests.TypeTests; + +public class WeaviateDataTypeTests(WeaviateDataTypeTests.Fixture fixture) + : DataTypeTests.DefaultRecord>(fixture), IClassFixture +{ + public override Task String_array() + => this.Test( + "StringArray", + ["foo", "bar"], + ["foo", "baz"], + isFilterable: false); // TODO: We don't currently support filtering on arrays + + public new class Fixture : DataTypeTests.DefaultRecord>.Fixture + { + public override TestStore TestStore => WeaviateTestStore.NamedVectorsInstance; + + // TODO: Weaviate requires special indexing for filtering on nulls, see #10358 + public override bool IsNullFilteringSupported => false; + + public override Type[] UnsupportedDefaultTypes { get; } = + [ +#if NET + typeof(TimeOnly) +#endif + ]; + } +} diff --git a/MEVD/test/Weaviate.ConformanceTests/TypeTests/WeaviateEmbeddingTypeTests.cs b/MEVD/test/Weaviate.ConformanceTests/TypeTests/WeaviateEmbeddingTypeTests.cs new file mode 100644 index 0000000..5101f32 --- /dev/null +++ b/MEVD/test/Weaviate.ConformanceTests/TypeTests/WeaviateEmbeddingTypeTests.cs @@ -0,0 +1,20 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using VectorData.ConformanceTests.Support; +using VectorData.ConformanceTests.TypeTests; +using Weaviate.ConformanceTests.Support; +using Xunit; + +#pragma warning disable CA2000 // Dispose objects before losing scope + +namespace Weaviate.ConformanceTests.TypeTests; + +public class WeaviateEmbeddingTypeTests(WeaviateEmbeddingTypeTests.Fixture fixture) + : EmbeddingTypeTests(fixture), IClassFixture +{ + public new class Fixture : EmbeddingTypeTests.Fixture + { + public override TestStore TestStore => WeaviateTestStore.NamedVectorsInstance; + } +} diff --git a/MEVD/test/Weaviate.ConformanceTests/TypeTests/WeaviateKeyTypeTests.cs b/MEVD/test/Weaviate.ConformanceTests/TypeTests/WeaviateKeyTypeTests.cs new file mode 100644 index 0000000..71069bb --- /dev/null +++ b/MEVD/test/Weaviate.ConformanceTests/TypeTests/WeaviateKeyTypeTests.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using VectorData.ConformanceTests.Support; +using VectorData.ConformanceTests.TypeTests; +using Weaviate.ConformanceTests.Support; +using Xunit; + +namespace Weaviate.ConformanceTests.TypeTests; + +public class WeaviateKeyTypeTests(WeaviateKeyTypeTests.Fixture fixture) + : KeyTypeTests(fixture), IClassFixture +{ + public new class Fixture : KeyTypeTests.Fixture + { + public override TestStore TestStore => WeaviateTestStore.NamedVectorsInstance; + } +} diff --git a/MEVD/test/Weaviate.ConformanceTests/Weaviate.ConformanceTests.csproj b/MEVD/test/Weaviate.ConformanceTests/Weaviate.ConformanceTests.csproj new file mode 100644 index 0000000..be2af6d --- /dev/null +++ b/MEVD/test/Weaviate.ConformanceTests/Weaviate.ConformanceTests.csproj @@ -0,0 +1,27 @@ + + + + net10.0;$(NetFrameworkTfm) + enable + enable + true + false + Weaviate.ConformanceTests + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + + + + diff --git a/MEVD/test/Weaviate.ConformanceTests/WeaviateCollectionManagementTests.cs b/MEVD/test/Weaviate.ConformanceTests/WeaviateCollectionManagementTests.cs new file mode 100644 index 0000000..db5164f --- /dev/null +++ b/MEVD/test/Weaviate.ConformanceTests/WeaviateCollectionManagementTests.cs @@ -0,0 +1,27 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.Support; +using Weaviate.ConformanceTests.Support; +using Xunit; + +namespace Weaviate.ConformanceTests; + +public class WeaviateCollectionManagementTests_NamedVectors(WeaviateCollectionManagementTests_NamedVectors.Fixture fixture) + : CollectionManagementTests(fixture), IClassFixture +{ + public class Fixture : VectorStoreFixture + { + public override TestStore TestStore => WeaviateTestStore.NamedVectorsInstance; + } +} + +public class WeaviateCollectionManagementTests_UnnamedVector(WeaviateCollectionManagementTests_UnnamedVector.Fixture fixture) + : CollectionManagementTests(fixture), IClassFixture +{ + public class Fixture : VectorStoreFixture + { + public override TestStore TestStore => WeaviateTestStore.UnnamedVectorInstance; + } +} diff --git a/MEVD/test/Weaviate.ConformanceTests/WeaviateDependencyInjectionTests.cs b/MEVD/test/Weaviate.ConformanceTests/WeaviateDependencyInjectionTests.cs new file mode 100644 index 0000000..2abfdd8 --- /dev/null +++ b/MEVD/test/Weaviate.ConformanceTests/WeaviateDependencyInjectionTests.cs @@ -0,0 +1,84 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using CommunityToolkit.VectorData.Weaviate; +using VectorData.ConformanceTests; +using Xunit; + +namespace Weaviate.ConformanceTests; + +public class WeaviateDependencyInjectionTests + : DependencyInjectionTests.Record>, Guid, DependencyInjectionTests.Record> +{ + private static readonly Uri s_endpoint = new("http://localhost"); + private const string ApiKey = "Fake API Key"; + + protected override string CollectionName => "Uppercase"; + + protected override void PopulateConfiguration(ConfigurationManager configuration, object? serviceKey = null) + => configuration.AddInMemoryCollection( + [ + new(CreateConfigKey("Weaviate", serviceKey, "Endpoint"), "http://localhost"), + new(CreateConfigKey("Weaviate", serviceKey, "ApiKey"), ApiKey), + ]); + + private static Uri EndpointProvider(IServiceProvider sp, object? serviceKey = null) + => new(sp.GetRequiredService().GetRequiredSection(CreateConfigKey("Weaviate", serviceKey, "Endpoint")).Value!); + + private static string ApiKeyProvider(IServiceProvider sp, object? serviceKey = null) + => sp.GetRequiredService().GetRequiredSection(CreateConfigKey("Weaviate", serviceKey, "ApiKey")).Value!; + + public override IEnumerable> CollectionDelegates + { + get + { + yield return (services, serviceKey, name, lifetime) => serviceKey is null + ? services + .AddSingleton(sp => new WeaviateCollectionOptions() { Endpoint = EndpointProvider(sp), ApiKey = ApiKeyProvider(sp) }) + .AddWeaviateCollection(name, lifetime: lifetime) + : services + .AddSingleton(sp => new WeaviateCollectionOptions() { Endpoint = EndpointProvider(sp, serviceKey), ApiKey = ApiKeyProvider(sp, serviceKey) }) + .AddKeyedWeaviateCollection(serviceKey, name, lifetime: lifetime); + + yield return (services, serviceKey, name, lifetime) => serviceKey is null + ? services.AddWeaviateCollection(name, s_endpoint, ApiKey, lifetime: lifetime) + : services.AddKeyedWeaviateCollection(serviceKey, name, s_endpoint, ApiKey, lifetime: lifetime); + + yield return (services, serviceKey, name, lifetime) => + services.AddKeyedWeaviateCollection(serviceKey, name, s_endpoint, ApiKey, lifetime: lifetime); + } + } + + public override IEnumerable> StoreDelegates + { + get + { + yield return (services, serviceKey, lifetime) => serviceKey is null + ? services.AddWeaviateVectorStore(s_endpoint, ApiKey, lifetime: lifetime) + : services.AddKeyedWeaviateVectorStore(serviceKey, s_endpoint, ApiKey, lifetime: lifetime); + + yield return (services, serviceKey, lifetime) => serviceKey is null + ? services + .AddSingleton(sp => new WeaviateVectorStoreOptions() { Endpoint = EndpointProvider(sp), ApiKey = ApiKeyProvider(sp) }) + .AddWeaviateVectorStore(lifetime: lifetime) + : services + .AddSingleton(sp => new WeaviateVectorStoreOptions() { Endpoint = EndpointProvider(sp, serviceKey), ApiKey = ApiKeyProvider(sp, serviceKey) }) + .AddKeyedWeaviateVectorStore(serviceKey, lifetime: lifetime); + } + } + + [Fact] + public void EndpointCantBeNull() + { + IServiceCollection services = new ServiceCollection(); + + Assert.Throws(() => services.AddWeaviateVectorStore(endpoint: null!, apiKey: ApiKey)); + Assert.Throws(() => services.AddKeyedWeaviateVectorStore(serviceKey: "notNull", endpoint: null!, apiKey: ApiKey)); + Assert.Throws(() => services.AddWeaviateCollection( + name: "notNull", endpoint: null!, apiKey: ApiKey)); + Assert.Throws(() => services.AddKeyedWeaviateCollection( + serviceKey: "notNull", name: "notNull", endpoint: null!, apiKey: ApiKey)); + } +} diff --git a/MEVD/test/Weaviate.ConformanceTests/WeaviateDistanceFunctionTests.cs b/MEVD/test/Weaviate.ConformanceTests/WeaviateDistanceFunctionTests.cs new file mode 100644 index 0000000..fb872bd --- /dev/null +++ b/MEVD/test/Weaviate.ConformanceTests/WeaviateDistanceFunctionTests.cs @@ -0,0 +1,22 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.Support; +using Weaviate.ConformanceTests.Support; +using Xunit; + +namespace Weaviate.ConformanceTests; + +public class WeaviateDistanceFunctionTests(WeaviateDistanceFunctionTests.Fixture fixture) + : DistanceFunctionTests(fixture), IClassFixture +{ + public override Task CosineSimilarity() => Assert.ThrowsAsync(base.CosineSimilarity); + public override Task DotProductSimilarity() => Assert.ThrowsAsync(base.DotProductSimilarity); + public override Task EuclideanDistance() => Assert.ThrowsAsync(base.EuclideanDistance); + + public new class Fixture() : DistanceFunctionTests.Fixture + { + public override TestStore TestStore => WeaviateTestStore.NamedVectorsInstance; + } +} diff --git a/MEVD/test/Weaviate.ConformanceTests/WeaviateEmbeddingGenerationTests.cs b/MEVD/test/Weaviate.ConformanceTests/WeaviateEmbeddingGenerationTests.cs new file mode 100644 index 0000000..6fd3c39 --- /dev/null +++ b/MEVD/test/Weaviate.ConformanceTests/WeaviateEmbeddingGenerationTests.cs @@ -0,0 +1,60 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.VectorData; +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.Support; +using Weaviate.ConformanceTests.Support; +using Xunit; + +namespace Weaviate.ConformanceTests; + +public class WeaviateEmbeddingGenerationTests(WeaviateEmbeddingGenerationTests.StringVectorFixture fixture, WeaviateEmbeddingGenerationTests.RomOfFloatVectorFixture romOfFloatVectorFixture) + : EmbeddingGenerationTests(fixture, romOfFloatVectorFixture), IClassFixture, IClassFixture +{ + public new class StringVectorFixture : EmbeddingGenerationTests.StringVectorFixture + { + public override TestStore TestStore => WeaviateTestStore.NamedVectorsInstance; + + public override VectorStore CreateVectorStore(IEmbeddingGenerator? embeddingGenerator) + => WeaviateTestStore.NamedVectorsInstance.GetVectorStore(new() { EmbeddingGenerator = embeddingGenerator }); + + public override Func[] DependencyInjectionStoreRegistrationDelegates => + [ + services => services + .AddSingleton(WeaviateTestStore.NamedVectorsInstance.Client) + .AddWeaviateVectorStore() + ]; + + public override Func[] DependencyInjectionCollectionRegistrationDelegates => + [ + services => services + .AddSingleton(WeaviateTestStore.NamedVectorsInstance.Client) + .AddWeaviateCollection(this.CollectionName) + ]; + } + + public new class RomOfFloatVectorFixture : EmbeddingGenerationTests.RomOfFloatVectorFixture + { + public override TestStore TestStore => WeaviateTestStore.NamedVectorsInstance; + + public override VectorStore CreateVectorStore(IEmbeddingGenerator? embeddingGenerator) + => WeaviateTestStore.NamedVectorsInstance.GetVectorStore(new() { EmbeddingGenerator = embeddingGenerator }); + + public override Func[] DependencyInjectionStoreRegistrationDelegates => + [ + services => services + .AddSingleton(WeaviateTestStore.NamedVectorsInstance.Client) + .AddWeaviateVectorStore() + ]; + + public override Func[] DependencyInjectionCollectionRegistrationDelegates => + [ + services => services + .AddSingleton(WeaviateTestStore.NamedVectorsInstance.Client) + .AddWeaviateCollection(this.CollectionName) + ]; + } +} diff --git a/MEVD/test/Weaviate.ConformanceTests/WeaviateFilterTests.cs b/MEVD/test/Weaviate.ConformanceTests/WeaviateFilterTests.cs new file mode 100644 index 0000000..ed230a6 --- /dev/null +++ b/MEVD/test/Weaviate.ConformanceTests/WeaviateFilterTests.cs @@ -0,0 +1,68 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.VectorData; +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.Support; +using Weaviate.ConformanceTests.Support; +using Xunit; + +namespace Weaviate.ConformanceTests; + +public class WeaviateFilterTests(WeaviateFilterTests.Fixture fixture) + : FilterTests(fixture), IClassFixture +{ + #region Filter by null + + // Null-state indexing needs to be set up, but that's not supported yet (#10358). + // We could interact with Weaviate directly (not via the abstraction) to do this. + + public override Task Equal_with_null_reference_type() + => Assert.ThrowsAsync(base.Equal_with_null_reference_type); + + public override Task Equal_with_null_captured() + => Assert.ThrowsAsync(base.Equal_with_null_captured); + + public override Task NotEqual_with_null_captured() + => Assert.ThrowsAsync(base.NotEqual_with_null_captured); + + public override Task NotEqual_with_null_reference_type() + => Assert.ThrowsAsync(base.NotEqual_with_null_reference_type); + + public override Task Equal_int_property_with_null_nullable_int() + => Assert.ThrowsAsync(base.Equal_int_property_with_null_nullable_int); + + #endregion + + #region Not + + // Weaviate currently doesn't support NOT (https://github.com/weaviate/weaviate/issues/3683) + public override Task Not_over_And() + => Assert.ThrowsAsync(base.Not_over_And); + + public override Task Not_over_Or() + => Assert.ThrowsAsync(base.Not_over_Or); + + #endregion + + #region Unsupported Contains scenarios + + public override Task Contains_over_captured_string_array() + => Assert.ThrowsAsync(base.Contains_over_captured_string_array); + + public override Task Contains_over_inline_int_array() + => Assert.ThrowsAsync(base.Contains_over_inline_int_array); + + public override Task Contains_over_inline_string_array() + => Assert.ThrowsAsync(base.Contains_over_inline_int_array); + + public override Task Contains_over_inline_string_array_with_weird_chars() + => Assert.ThrowsAsync(base.Contains_over_inline_string_array_with_weird_chars); + + #endregion + + public new class Fixture : FilterTests.Fixture + { + public override TestStore TestStore => WeaviateTestStore.NamedVectorsInstance; + } +} diff --git a/MEVD/test/Weaviate.ConformanceTests/WeaviateHybridSearchTests.cs b/MEVD/test/Weaviate.ConformanceTests/WeaviateHybridSearchTests.cs new file mode 100644 index 0000000..7d7af75 --- /dev/null +++ b/MEVD/test/Weaviate.ConformanceTests/WeaviateHybridSearchTests.cs @@ -0,0 +1,45 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.Support; +using Weaviate.ConformanceTests.Support; +using Xunit; + +namespace Weaviate.ConformanceTests; + +public class WeaviateHybridSearchTests_NamedVectors( + WeaviateHybridSearchTests_NamedVectors.VectorAndStringFixture vectorAndStringFixture, + WeaviateHybridSearchTests_NamedVectors.MultiTextFixture multiTextFixture) + : HybridSearchTests(vectorAndStringFixture, multiTextFixture), + IClassFixture, + IClassFixture +{ + public new class VectorAndStringFixture : HybridSearchTests.VectorAndStringFixture + { + public override TestStore TestStore => WeaviateTestStore.NamedVectorsInstance; + } + + public new class MultiTextFixture : HybridSearchTests.MultiTextFixture + { + public override TestStore TestStore => WeaviateTestStore.NamedVectorsInstance; + } +} + +public class WeaviateHybridSearchTests_UnnamedVector( + WeaviateHybridSearchTests_UnnamedVector.VectorAndStringFixture vectorAndStringFixture, + WeaviateHybridSearchTests_UnnamedVector.MultiTextFixture multiTextFixture) + : HybridSearchTests(vectorAndStringFixture, multiTextFixture), + IClassFixture, + IClassFixture +{ + public new class VectorAndStringFixture : HybridSearchTests.VectorAndStringFixture + { + public override TestStore TestStore => WeaviateTestStore.UnnamedVectorInstance; + } + + public new class MultiTextFixture : HybridSearchTests.MultiTextFixture + { + public override TestStore TestStore => WeaviateTestStore.UnnamedVectorInstance; + } +} diff --git a/MEVD/test/Weaviate.ConformanceTests/WeaviateIndexKindTests.cs b/MEVD/test/Weaviate.ConformanceTests/WeaviateIndexKindTests.cs new file mode 100644 index 0000000..d765d01 --- /dev/null +++ b/MEVD/test/Weaviate.ConformanceTests/WeaviateIndexKindTests.cs @@ -0,0 +1,27 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.VectorData; +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.Support; +using Weaviate.ConformanceTests.Support; +using Xunit; + +namespace Weaviate.ConformanceTests; + +public class WeaviateIndexKindTests(WeaviateIndexKindTests.Fixture fixture) + : IndexKindTests(fixture), IClassFixture +{ + [Fact] + public virtual Task Hnsw() + => this.Test(IndexKind.Hnsw); + + // The dynamic index requires the ASYNC_INDEXING Weaviate server environment variable, which decouples indexing + // from object creation. This causes eventual consistency issues for other tests (e.g. SearchAsync_with_Filter), + // so we can't enable it in the test container. + + public new class Fixture() : IndexKindTests.Fixture + { + public override TestStore TestStore => WeaviateTestStore.NamedVectorsInstance; + } +} diff --git a/MEVD/test/Weaviate.ConformanceTests/WeaviateTestSuiteImplementationTests.cs b/MEVD/test/Weaviate.ConformanceTests/WeaviateTestSuiteImplementationTests.cs new file mode 100644 index 0000000..84a2015 --- /dev/null +++ b/MEVD/test/Weaviate.ConformanceTests/WeaviateTestSuiteImplementationTests.cs @@ -0,0 +1,8 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using VectorData.ConformanceTests; + +namespace Weaviate.ConformanceTests; + +public class WeaviateTestSuiteImplementationTests : TestSuiteImplementationTests; diff --git a/MEVD/test/Weaviate.UnitTests/.editorconfig b/MEVD/test/Weaviate.UnitTests/.editorconfig new file mode 100644 index 0000000..394eef6 --- /dev/null +++ b/MEVD/test/Weaviate.UnitTests/.editorconfig @@ -0,0 +1,6 @@ +# Suppressing errors for Test projects under dotnet folder +[*.cs] +dotnet_diagnostic.CA2007.severity = none # Do not directly await a Task +dotnet_diagnostic.VSTHRD111.severity = none # Use .ConfigureAwait(bool) is hidden by default, set to none to prevent IDE from changing on autosave +dotnet_diagnostic.CS1591.severity = none # Missing XML comment for publicly visible type or member +dotnet_diagnostic.IDE1006.severity = warning # Naming rule violations diff --git a/MEVD/test/Weaviate.UnitTests/Weaviate.UnitTests.csproj b/MEVD/test/Weaviate.UnitTests/Weaviate.UnitTests.csproj new file mode 100644 index 0000000..b8e59cb --- /dev/null +++ b/MEVD/test/Weaviate.UnitTests/Weaviate.UnitTests.csproj @@ -0,0 +1,39 @@ + + + + CommunityToolkit.VectorData.Weaviate.UnitTests + CommunityToolkit.VectorData.Weaviate.UnitTests + net10.0 + true + enable + disable + false + $(NoWarn);SKEXP0001,VSTHRD111,CA2007 + $(NoWarn);MEVD9001 + + + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + + + + + \ No newline at end of file diff --git a/MEVD/test/Weaviate.UnitTests/WeaviateCollectionCreateMappingTests.cs b/MEVD/test/Weaviate.UnitTests/WeaviateCollectionCreateMappingTests.cs new file mode 100644 index 0000000..420c66c --- /dev/null +++ b/MEVD/test/Weaviate.UnitTests/WeaviateCollectionCreateMappingTests.cs @@ -0,0 +1,236 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Text.Json; +using Microsoft.Extensions.VectorData; +using CommunityToolkit.VectorData.Weaviate; +using Xunit; + +namespace SemanticKernel.Connectors.Weaviate.UnitTests; + +/// +/// Unit tests for class. +/// +public sealed class WeaviateCollectionCreateMappingTests +{ + private const bool HasNamedVectors = true; + + [Fact] + public void ItThrowsExceptionWithInvalidIndexKind() + { + // Arrange + var model = new WeaviateModelBuilder(HasNamedVectors) + .BuildDynamic( + new VectorStoreCollectionDefinition + { + Properties = + [ + new VectorStoreKeyProperty("Key", typeof(Guid)), + new VectorStoreVectorProperty("Vector", typeof(ReadOnlyMemory), 10) { IndexKind = "non-existent-index-kind" } + ] + }, + defaultEmbeddingGenerator: null); + + // Act & Assert + Assert.Throws(() => WeaviateCollectionCreateMapping.MapToSchema(collectionName: "CollectionName", HasNamedVectors, model)); + } + + [Theory] + [InlineData(IndexKind.Hnsw, "hnsw")] + [InlineData(IndexKind.Flat, "flat")] + [InlineData(IndexKind.Dynamic, "dynamic")] + public void ItReturnsCorrectSchemaWithValidIndexKind(string indexKind, string expectedIndexKind) + { + // Arrange + var model = new WeaviateModelBuilder(HasNamedVectors) + .BuildDynamic( + new VectorStoreCollectionDefinition + { + Properties = + [ + new VectorStoreKeyProperty("Key", typeof(Guid)), + new VectorStoreVectorProperty("Vector", typeof(ReadOnlyMemory), 10) { IndexKind = indexKind } + ] + }, + defaultEmbeddingGenerator: null); + + // Act + var schema = WeaviateCollectionCreateMapping.MapToSchema(collectionName: "CollectionName", HasNamedVectors, model); + var actualIndexKind = schema.VectorConfigurations["Vector"].VectorIndexType; + + // Assert + Assert.Equal(expectedIndexKind, actualIndexKind); + } + + [Fact] + public void ItThrowsExceptionWithUnsupportedDistanceFunction() + { + // Arrange + var model = new WeaviateModelBuilder(HasNamedVectors) + .BuildDynamic( + new VectorStoreCollectionDefinition + { + Properties = + [ + new VectorStoreKeyProperty("Key", typeof(Guid)), + new VectorStoreVectorProperty("Vector", typeof(ReadOnlyMemory), 10) { DistanceFunction = "unsupported-distance-function" } + ] + }, + defaultEmbeddingGenerator: null); + + // Act & Assert + Assert.Throws(() => WeaviateCollectionCreateMapping.MapToSchema(collectionName: "CollectionName", HasNamedVectors, model)); + } + + [Theory] + [InlineData(DistanceFunction.CosineDistance, "cosine")] + [InlineData(DistanceFunction.NegativeDotProductSimilarity, "dot")] + [InlineData(DistanceFunction.EuclideanSquaredDistance, "l2-squared")] + [InlineData(DistanceFunction.HammingDistance, "hamming")] + [InlineData(DistanceFunction.ManhattanDistance, "manhattan")] + public void ItReturnsCorrectSchemaWithValidDistanceFunction(string distanceFunction, string expectedDistanceFunction) + { + // Arrange + var model = new WeaviateModelBuilder(HasNamedVectors) + .BuildDynamic( + new VectorStoreCollectionDefinition + { + Properties = + [ + new VectorStoreKeyProperty("Key", typeof(Guid)), + new VectorStoreVectorProperty("Vector", typeof(ReadOnlyMemory), 10) { DistanceFunction = distanceFunction } + ] + }, + defaultEmbeddingGenerator: null); + + // Act + var schema = WeaviateCollectionCreateMapping.MapToSchema(collectionName: "CollectionName", HasNamedVectors, model); + + var actualDistanceFunction = schema.VectorConfigurations["Vector"].VectorIndexConfig?.Distance; + + // Assert + Assert.Equal(expectedDistanceFunction, actualDistanceFunction); + } + + [Theory] + [InlineData(typeof(string), "text")] + [InlineData(typeof(List), "text[]")] + [InlineData(typeof(int), "int")] + [InlineData(typeof(int?), "int")] + [InlineData(typeof(List), "int[]")] + [InlineData(typeof(List), "int[]")] + [InlineData(typeof(long), "int")] + [InlineData(typeof(long?), "int")] + [InlineData(typeof(List), "int[]")] + [InlineData(typeof(List), "int[]")] + [InlineData(typeof(short), "int")] + [InlineData(typeof(short?), "int")] + [InlineData(typeof(List), "int[]")] + [InlineData(typeof(List), "int[]")] + [InlineData(typeof(byte), "int")] + [InlineData(typeof(byte?), "int")] + [InlineData(typeof(List), "int[]")] + [InlineData(typeof(List), "int[]")] + [InlineData(typeof(float), "number")] + [InlineData(typeof(float?), "number")] + [InlineData(typeof(List), "number[]")] + [InlineData(typeof(List), "number[]")] + [InlineData(typeof(double), "number")] + [InlineData(typeof(double?), "number")] + [InlineData(typeof(List), "number[]")] + [InlineData(typeof(List), "number[]")] + [InlineData(typeof(decimal), "number")] + [InlineData(typeof(decimal?), "number")] + [InlineData(typeof(List), "number[]")] + [InlineData(typeof(List), "number[]")] + [InlineData(typeof(DateTime), "date")] + [InlineData(typeof(DateTime?), "date")] + [InlineData(typeof(List), "date[]")] + [InlineData(typeof(List), "date[]")] + [InlineData(typeof(DateTimeOffset), "date")] + [InlineData(typeof(DateTimeOffset?), "date")] + [InlineData(typeof(List), "date[]")] + [InlineData(typeof(List), "date[]")] + [InlineData(typeof(Guid), "uuid")] + [InlineData(typeof(Guid?), "uuid")] + [InlineData(typeof(List), "uuid[]")] + [InlineData(typeof(List), "uuid[]")] + [InlineData(typeof(bool), "boolean")] + [InlineData(typeof(bool?), "boolean")] + [InlineData(typeof(List), "boolean[]")] + [InlineData(typeof(List), "boolean[]")] + public void ItMapsPropertyCorrectly(Type propertyType, string expectedPropertyType) + { + // Arrange + var model = new WeaviateModelBuilder(HasNamedVectors) + .BuildDynamic( + new VectorStoreCollectionDefinition + { + Properties = + [ + new VectorStoreKeyProperty("Key", typeof(Guid)), + new VectorStoreDataProperty("PropertyName", propertyType) { IsIndexed = true, IsFullTextIndexed = true }, + new VectorStoreVectorProperty("Vector", typeof(ReadOnlyMemory), 10) + ] + }, + defaultEmbeddingGenerator: null, + new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }); + + // Act + var schema = WeaviateCollectionCreateMapping.MapToSchema(collectionName: "CollectionName", HasNamedVectors, model); + + var property = schema.Properties[0]; + + // Assert + Assert.Equal("propertyName", property.Name); + Assert.Equal(expectedPropertyType, property.DataType[0]); + Assert.True(property.IndexSearchable); + Assert.True(property.IndexFilterable); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void ItReturnsCorrectSchemaWithValidVectorConfiguration(bool hasNamedVectors) + { + // Arrange + var model = new WeaviateModelBuilder(hasNamedVectors) + .BuildDynamic( + new VectorStoreCollectionDefinition + { + Properties = + [ + new VectorStoreKeyProperty("Key", typeof(Guid)), + new VectorStoreVectorProperty("Vector", typeof(ReadOnlyMemory), 4) + { + DistanceFunction = DistanceFunction.CosineDistance, + IndexKind = IndexKind.Hnsw + } + ] + }, + defaultEmbeddingGenerator: null); + + // Act + var schema = WeaviateCollectionCreateMapping.MapToSchema(collectionName: "CollectionName", hasNamedVectors, model); + + // Assert + if (hasNamedVectors) + { + Assert.Null(schema.VectorIndexConfig?.Distance); + Assert.Null(schema.VectorIndexType); + Assert.True(schema.VectorConfigurations.ContainsKey("Vector")); + + Assert.Equal("cosine", schema.VectorConfigurations["Vector"].VectorIndexConfig?.Distance); + Assert.Equal("hnsw", schema.VectorConfigurations["Vector"].VectorIndexType); + } + else + { + Assert.False(schema.VectorConfigurations.ContainsKey("Vector")); + + Assert.Equal("cosine", schema.VectorIndexConfig?.Distance); + Assert.Equal("hnsw", schema.VectorIndexType); + } + } +} diff --git a/MEVD/test/Weaviate.UnitTests/WeaviateCollectionSearchMappingTests.cs b/MEVD/test/Weaviate.UnitTests/WeaviateCollectionSearchMappingTests.cs new file mode 100644 index 0000000..a44cd9d --- /dev/null +++ b/MEVD/test/Weaviate.UnitTests/WeaviateCollectionSearchMappingTests.cs @@ -0,0 +1,72 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Linq; +using System.Text.Json.Nodes; +using CommunityToolkit.VectorData.Weaviate; +using Xunit; + +namespace SemanticKernel.Connectors.Weaviate.UnitTests; + +/// +/// Unit tests for class. +/// +public sealed class WeaviateCollectionSearchMappingTests +{ + [Theory] + [InlineData(true)] + [InlineData(false)] + public void MapSearchResultByDefaultReturnsValidResult(bool hasNamedVectors) + { + // Arrange + var jsonObject = new JsonObject + { + ["_additional"] = new JsonObject + { + ["distance"] = 0.5, + ["id"] = "55555555-5555-5555-5555-555555555555" + }, + ["description"] = "This is a great hotel.", + ["hotelCode"] = 42, + ["hotelName"] = "My Hotel", + ["hotelRating"] = 4.5, + ["parking_is_included"] = true, + ["tags"] = new JsonArray(new List { "t1", "t2" }.Select(l => (JsonNode)l).ToArray()), + ["timestamp"] = "2024-08-28T10:11:12-07:00" + }; + + var vector = new JsonArray(new List { 30, 31, 32, 33 }.Select(l => (JsonNode)l).ToArray()); + + if (hasNamedVectors) + { + jsonObject["_additional"]!["vectors"] = new JsonObject + { + ["descriptionEmbedding"] = vector + }; + } + else + { + jsonObject["_additional"]!["vector"] = vector; + } + + // Act + var (storageModel, score) = WeaviateCollectionSearchMapping.MapSearchResult(jsonObject, "distance", hasNamedVectors); + + // Assert + Assert.Equal(0.5, score); + + Assert.Equal("55555555-5555-5555-5555-555555555555", storageModel["id"]!.GetValue()); + Assert.Equal("This is a great hotel.", storageModel["properties"]!["description"]!.GetValue()); + Assert.Equal(42, storageModel["properties"]!["hotelCode"]!.GetValue()); + Assert.Equal(4.5, storageModel["properties"]!["hotelRating"]!.GetValue()); + Assert.Equal("My Hotel", storageModel["properties"]!["hotelName"]!.GetValue()); + Assert.True(storageModel["properties"]!["parking_is_included"]!.GetValue()); + Assert.Equal(["t1", "t2"], storageModel["properties"]!["tags"]!.AsArray().Select(l => l!.GetValue())); + Assert.Equal("2024-08-28T10:11:12-07:00", storageModel["properties"]!["timestamp"]!.GetValue()); + + var vectorProperty = hasNamedVectors ? storageModel["vectors"]!["descriptionEmbedding"] : storageModel["vector"]; + + Assert.Equal([30f, 31f, 32f, 33f], vectorProperty!.AsArray().Select(l => l!.GetValue())); + } +} diff --git a/MEVD/test/Weaviate.UnitTests/WeaviateCollectionTests.cs b/MEVD/test/Weaviate.UnitTests/WeaviateCollectionTests.cs new file mode 100644 index 0000000..94d97d1 --- /dev/null +++ b/MEVD/test/Weaviate.UnitTests/WeaviateCollectionTests.cs @@ -0,0 +1,531 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Threading.Tasks; +using Microsoft.Extensions.VectorData; +using CommunityToolkit.VectorData.Weaviate; +using Xunit; + +namespace SemanticKernel.Connectors.Weaviate.UnitTests; + +/// +/// Unit tests for class. +/// +public sealed class WeaviateCollectionTests : IDisposable +{ + private readonly HttpMessageHandlerStub _messageHandlerStub = new(); + private readonly HttpClient _mockHttpClient; + + public WeaviateCollectionTests() + { + this._mockHttpClient = new(this._messageHandlerStub, false) { BaseAddress = new Uri("http://default-endpoint") }; + } + + [Fact] + public void ConstructorForModelWithoutKeyThrowsException() + { + // Act & Assert + var exception = Assert.Throws(() => new WeaviateCollection(this._mockHttpClient, "Collection")); + Assert.Contains("No key property found", exception.Message); + } + + [Fact] + public void ConstructorWithoutEndpointThrowsException() + { + // Arrange + using var httpClient = new HttpClient(); + + // Act & Assert + var exception = Assert.Throws(() => new WeaviateCollection(httpClient, "Collection")); + Assert.Contains("Weaviate endpoint should be provided", exception.Message); + } + + [Fact] + public void ConstructorWithDeclarativeModelInitializesCollection() + { + // Act & Assert + using var collection = new WeaviateCollection( + this._mockHttpClient, + "Collection"); + + Assert.NotNull(collection); + } + + [Fact] + public void ConstructorWithImperativeModelInitializesCollection() + { + // Arrange + var definition = new VectorStoreCollectionDefinition + { + Properties = [new VectorStoreKeyProperty("Id", typeof(Guid))] + }; + + // Act + using var collection = new WeaviateCollection( + this._mockHttpClient, + "Collection", + new() { Definition = definition }); + + // Assert + Assert.NotNull(collection); + } + + [Theory] + [MemberData(nameof(CollectionExistsData))] + public async Task CollectionExistsReturnsValidResultAsync(HttpResponseMessage responseMessage, bool expectedResult) + { + // Arrange + this._messageHandlerStub.ResponseToReturn = responseMessage; + + using var sut = new WeaviateCollection(this._mockHttpClient, "Collection"); + + // Act + var actualResult = await sut.CollectionExistsAsync(); + + // Assert + Assert.Equal(expectedResult, actualResult); + } + + [Theory] + [InlineData("notStartingWithCapitalLetter")] + [InlineData("0startingWithDigit")] + [InlineData("contains spaces")] + [InlineData("contains-dashes")] + [InlineData("contains_underscores")] + [InlineData("contains$specialCharacters")] + [InlineData("contains!specialCharacters")] + [InlineData("contains@specialCharacters")] + [InlineData("contains#specialCharacters")] + [InlineData("contains%specialCharacters")] + [InlineData("contains^specialCharacters")] + [InlineData("contains&specialCharacters")] + [InlineData("contains*specialCharacters")] + [InlineData("contains(specialCharacters")] + [InlineData("contains)specialCharacters")] + [InlineData("containsNonAsciiĄ")] + [InlineData("containsNonAsciią")] + public void CollectionCtorRejectsInvalidNames(string collectionName) + { + ArgumentException argumentException = Assert.Throws(() => new WeaviateCollection(this._mockHttpClient, collectionName)); + Assert.Equal("collectionName", argumentException.ParamName); + } + + [Fact] + public async Task EnsureCollectionExistsUsesValidCollectionSchemaAsync() + { + // Arrange + const string CollectionName = "Collection"; + using var sut = new WeaviateCollection(this._mockHttpClient, CollectionName); + + using var collectionExistsResponse = new HttpResponseMessage(HttpStatusCode.NotFound); + this._messageHandlerStub.ResponseQueue.Enqueue(collectionExistsResponse); + + using var createCollectionResponse = new HttpResponseMessage(HttpStatusCode.OK); + this._messageHandlerStub.ResponseQueue.Enqueue(createCollectionResponse); + + // Act + await sut.EnsureCollectionExistsAsync(); + + // Assert + var schemaRequest = JsonSerializer.Deserialize(this._messageHandlerStub.RequestContent); + + Assert.NotNull(schemaRequest); + + Assert.Equal(CollectionName, schemaRequest.CollectionName); + + Assert.NotNull(schemaRequest.VectorConfigurations); + Assert.Equal("descriptionEmbedding", schemaRequest.VectorConfigurations.Keys.First()); + + var vectorConfiguration = schemaRequest.VectorConfigurations["descriptionEmbedding"]; + + Assert.Equal("cosine", vectorConfiguration.VectorIndexConfig?.Distance); + Assert.Equal("hnsw", vectorConfiguration.VectorIndexType); + + Assert.NotNull(schemaRequest.Properties); + + this.AssertSchemaProperty(schemaRequest.Properties[0], "hotelName", "text", true, false); + this.AssertSchemaProperty(schemaRequest.Properties[1], "hotelCode", "int", false, false); + this.AssertSchemaProperty(schemaRequest.Properties[2], "hotelRating", "number", false, false); + this.AssertSchemaProperty(schemaRequest.Properties[3], "parking_is_included", "boolean", false, false); + this.AssertSchemaProperty(schemaRequest.Properties[4], "tags", "text[]", false, false); + this.AssertSchemaProperty(schemaRequest.Properties[5], "description", "text", false, true); + this.AssertSchemaProperty(schemaRequest.Properties[6], "timestamp", "date", false, false); + } + + [Fact] + public async Task DeleteCollectionSendsValidRequestAsync() + { + // Arrange + const string CollectionName = "Collection"; + using var sut = new WeaviateCollection(this._mockHttpClient, CollectionName); + + // Act + await sut.EnsureCollectionDeletedAsync(); + + // Assert + Assert.Equal("http://default-endpoint/schema/Collection", this._messageHandlerStub.RequestUri?.AbsoluteUri); + Assert.Equal(HttpMethod.Delete, this._messageHandlerStub.Method); + } + + [Fact] + public async Task DeleteSendsValidRequestAsync() + { + // Arrange + const string CollectionName = "Collection"; + var id = new Guid("55555555-5555-5555-5555-555555555555"); + + using var sut = new WeaviateCollection(this._mockHttpClient, CollectionName); + + // Act + await sut.DeleteAsync(id); + + // Assert + Assert.Equal("http://default-endpoint/objects/Collection/55555555-5555-5555-5555-555555555555", this._messageHandlerStub.RequestUri?.AbsoluteUri); + Assert.Equal(HttpMethod.Delete, this._messageHandlerStub.Method); + } + + [Fact] + public async Task DeleteBatchUsesValidQueryMatchAsync() + { + // Arrange + const string CollectionName = "Collection"; + List ids = [new Guid("11111111-1111-1111-1111-111111111111"), new Guid("22222222-2222-2222-2222-222222222222")]; + + using var sut = new WeaviateCollection(this._mockHttpClient, CollectionName); + + // Act + await sut.DeleteAsync(ids); + + // Assert + var request = JsonSerializer.Deserialize(this._messageHandlerStub.RequestContent); + + Assert.NotNull(request?.Match); + + Assert.Equal(CollectionName, request.Match.CollectionName); + + Assert.NotNull(request.Match.WhereClause); + + var clause = request.Match.WhereClause; + + Assert.Equal("ContainsAny", clause.Operator); + Assert.Equal(["id"], clause.Path); + Assert.Equal(["11111111-1111-1111-1111-111111111111", "22222222-2222-2222-2222-222222222222"], clause.Values); + } + + [Fact] + public async Task GetExistingRecordReturnsValidRecordAsync() + { + // Arrange + var id = new Guid("55555555-5555-5555-5555-555555555555"); + + var jsonObject = new JsonObject { ["id"] = id.ToString(), ["properties"] = new JsonObject() }; + + jsonObject["properties"]!["hotelName"] = "Test Name"; + + this._messageHandlerStub.ResponseToReturn = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(JsonSerializer.Serialize(jsonObject)) + }; + + using var sut = new WeaviateCollection(this._mockHttpClient, "Collection"); + + // Act + var result = await sut.GetAsync(id); + + // Assert + Assert.NotNull(result); + Assert.Equal(id, result.HotelId); + Assert.Equal("Test Name", result.HotelName); + } + + [Fact] + public async Task GetExistingBatchRecordsReturnsValidRecordsAsync() + { + // Arrange + var id1 = new Guid("11111111-1111-1111-1111-111111111111"); + var id2 = new Guid("22222222-2222-2222-2222-222222222222"); + + var jsonObject1 = new JsonObject { ["id"] = id1.ToString(), ["properties"] = new JsonObject() }; + var jsonObject2 = new JsonObject { ["id"] = id2.ToString(), ["properties"] = new JsonObject() }; + + jsonObject1["properties"]!["hotelName"] = "Test Name 1"; + jsonObject2["properties"]!["hotelName"] = "Test Name 2"; + + using var response1 = new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent(JsonSerializer.Serialize(jsonObject1)) }; + using var response2 = new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent(JsonSerializer.Serialize(jsonObject2)) }; + + this._messageHandlerStub.ResponseQueue.Enqueue(response1); + this._messageHandlerStub.ResponseQueue.Enqueue(response2); + + using var sut = new WeaviateCollection(this._mockHttpClient, "Collection"); + + // Act + var results = await sut.GetAsync([id1, id2]).ToListAsync(); + + // Assert + Assert.NotNull(results[0]); + Assert.Equal(id1, results[0].HotelId); + Assert.Equal("Test Name 1", results[0].HotelName); + + Assert.NotNull(results[1]); + Assert.Equal(id2, results[1].HotelId); + Assert.Equal("Test Name 2", results[1].HotelName); + } + + [Fact] + public async Task UpsertReturnsRecordKeyAsync() + { + // Arrange + var id = new Guid("11111111-1111-1111-1111-111111111111"); + var hotel = new WeaviateHotel { HotelId = id, HotelName = "Test Name" }; + + var batchResponse = new List { new() { Id = id, Result = new() { Status = "Success" } } }; + + this._messageHandlerStub.ResponseToReturn = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(JsonSerializer.Serialize(batchResponse)), + }; + + using var sut = new WeaviateCollection(this._mockHttpClient, "Collection"); + + // Act + await sut.UpsertAsync(hotel); + + // Assert + var request = JsonSerializer.Deserialize(this._messageHandlerStub.RequestContent); + + Assert.NotNull(request?.CollectionObjects); + + var jsonObject = request.CollectionObjects[0]; + + Assert.Equal("11111111-1111-1111-1111-111111111111", jsonObject["id"]?.GetValue()); + Assert.Equal("Test Name", jsonObject["properties"]?["hotelName"]?.GetValue()); + } + + [Fact] + public async Task UpsertReturnsRecordKeysAsync() + { + // Arrange + var id1 = new Guid("11111111-1111-1111-1111-111111111111"); + var id2 = new Guid("22222222-2222-2222-2222-222222222222"); + + var hotel1 = new WeaviateHotel { HotelId = id1, HotelName = "Test Name 1" }; + var hotel2 = new WeaviateHotel { HotelId = id2, HotelName = "Test Name 2" }; + + var batchResponse = new List + { + new() { Id = id1, Result = new() { Status = "Success" } }, + new() { Id = id2, Result = new() { Status = "Success" } } + }; + + this._messageHandlerStub.ResponseToReturn = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(JsonSerializer.Serialize(batchResponse)), + }; + + using var sut = new WeaviateCollection(this._mockHttpClient, "Collection"); + + // Act + await sut.UpsertAsync([hotel1, hotel2]); + + // Assert + var request = JsonSerializer.Deserialize(this._messageHandlerStub.RequestContent); + + Assert.NotNull(request?.CollectionObjects); + + var jsonObject1 = request.CollectionObjects[0]; + var jsonObject2 = request.CollectionObjects[1]; + + Assert.Equal("11111111-1111-1111-1111-111111111111", jsonObject1["id"]?.GetValue()); + Assert.Equal("Test Name 1", jsonObject1["properties"]?["hotelName"]?.GetValue()); + + Assert.Equal("22222222-2222-2222-2222-222222222222", jsonObject2["id"]?.GetValue()); + Assert.Equal("Test Name 2", jsonObject2["properties"]?["hotelName"]?.GetValue()); + } + + [Theory] + [InlineData(true, "http://test-endpoint/schema", "Bearer fake-key")] + [InlineData(false, "http://default-endpoint/schema", null)] + public async Task ItUsesHttpClientParametersAsync(bool initializeOptions, string expectedEndpoint, string? expectedHeader) + { + // Arrange + const string CollectionName = "Collection"; + + var options = initializeOptions ? + new WeaviateCollectionOptions() { Endpoint = new Uri("http://test-endpoint"), ApiKey = "fake-key" } : + null; + + using var collectionExistsResponse = new HttpResponseMessage(HttpStatusCode.NotFound); + this._messageHandlerStub.ResponseQueue.Enqueue(collectionExistsResponse); + + using var createCollectionResponse = new HttpResponseMessage(HttpStatusCode.OK); + this._messageHandlerStub.ResponseQueue.Enqueue(createCollectionResponse); + + using var sut = new WeaviateCollection(this._mockHttpClient, CollectionName, options); + + // Act + await sut.EnsureCollectionExistsAsync(); + + var headers = this._messageHandlerStub.RequestHeaders; + var endpoint = this._messageHandlerStub.RequestUri; + + // Assert + Assert.Equal(expectedEndpoint, endpoint?.AbsoluteUri); + Assert.Equal(expectedHeader, headers?.Authorization?.ToString()); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task SearchReturnsValidRecordAsync(bool includeVectors) + { + // Arrange + const string CollectionName = "SearchCollection"; + var id = new Guid("55555555-5555-5555-5555-555555555555"); + var vector = new ReadOnlyMemory([30f, 31f, 32f, 33f]); + + var jsonObject = new JsonObject + { + ["data"] = new JsonObject + { + ["Get"] = new JsonObject + { + [CollectionName] = new JsonArray + { + new JsonObject + { + ["_additional"] = new JsonObject + { + ["distance"] = 0.5, + ["id"] = id.ToString(), + ["vectors"] = new JsonObject + { + ["descriptionEmbedding"] = new JsonArray(new List {30, 31, 32, 33}.Select(l => (JsonNode)l).ToArray()) + } + }, + ["description"] = "This is a great hotel.", + ["hotelCode"] = 42, + ["hotelName"] = "My Hotel", + ["hotelRating"] = 4.5, + ["parking_is_included"] = true, + ["tags"] = new JsonArray(new List { "t1", "t2" }.Select(l => (JsonNode)l).ToArray()), + ["timestamp"] = "2024-08-28T10:11:12-07:00" + } + } + } + } + }; + + this._messageHandlerStub.ResponseToReturn = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(JsonSerializer.Serialize(jsonObject)) + }; + + using var sut = new WeaviateCollection(this._mockHttpClient, CollectionName); + + // Act + var results = await sut.SearchAsync(vector, top: 3, new() + { + IncludeVectors = includeVectors + }).ToListAsync(); + + // Assert + Assert.Single(results); + + var score = results[0].Score; + var record = results[0].Record; + + Assert.Equal(0.5, score); + + Assert.Equal(id, record.HotelId); + Assert.Equal("My Hotel", record.HotelName); + Assert.Equal("This is a great hotel.", record.Description); + Assert.Equal(42, record.HotelCode); + Assert.Equal(4.5f, record.HotelRating); + Assert.True(record.ParkingIncluded); + Assert.Equal(["t1", "t2"], record.Tags); + Assert.Equal(new DateTimeOffset(new DateTime(2024, 8, 28, 10, 11, 12), TimeSpan.FromHours(-7)), record.Timestamp); + + if (includeVectors) + { + Assert.True(record.DescriptionEmbedding.HasValue); + Assert.Equal(vector.ToArray(), record.DescriptionEmbedding.Value.ToArray()); + } + else + { + Assert.False(record.DescriptionEmbedding.HasValue); + } + } + + [Fact] + public async Task SearchWithUnsupportedVectorTypeThrowsExceptionAsync() + { + // Arrange + using var sut = new WeaviateCollection(this._mockHttpClient, "Collection"); + + // Act & Assert + await Assert.ThrowsAsync(async () => + await sut.SearchAsync(new List([1, 2, 3]), top: 3).ToListAsync()); + } + + [Fact] + public async Task SearchWithNonExistentVectorPropertyNameThrowsExceptionAsync() + { + // Arrange + using var sut = new WeaviateCollection(this._mockHttpClient, "Collection"); + + // Act & Assert + await Assert.ThrowsAsync(async () => + await sut.SearchAsync( + new ReadOnlyMemory([1f, 2f, 3f]), + top: 3, + new() { VectorProperty = r => "non-existent-property" }) + .ToListAsync()); + } + + public void Dispose() + { + this._mockHttpClient.Dispose(); + this._messageHandlerStub.Dispose(); + } + + public static TheoryData CollectionExistsData => new() + { + { new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent(JsonSerializer.Serialize(new WeaviateGetCollectionSchemaResponse { CollectionName = "Collection" })) }, true }, + { new HttpResponseMessage(HttpStatusCode.NotFound), false } + }; + + #region private + + private void AssertSchemaProperty( + WeaviateCollectionSchemaProperty property, + string propertyName, + string dataType, + bool indexFilterable, + bool indexSearchable) + { + Assert.NotNull(property); + Assert.Equal(propertyName, property.Name); + Assert.Equal(dataType, property.DataType[0]); + Assert.Equal(indexFilterable, property.IndexFilterable); + Assert.Equal(indexSearchable, property.IndexSearchable); + } + +#pragma warning disable CA1812 + private sealed class TestModel + { + public Guid Id { get; set; } + + public string? HotelName { get; set; } + } +#pragma warning restore CA1812 + + #endregion +} diff --git a/MEVD/test/Weaviate.UnitTests/WeaviateDynamicMapperTests.cs b/MEVD/test/Weaviate.UnitTests/WeaviateDynamicMapperTests.cs new file mode 100644 index 0000000..12a2174 --- /dev/null +++ b/MEVD/test/Weaviate.UnitTests/WeaviateDynamicMapperTests.cs @@ -0,0 +1,457 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.Json.Serialization; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; +using CommunityToolkit.VectorData.Weaviate; +using Xunit; + +namespace SemanticKernel.Connectors.Weaviate.UnitTests; + +/// +/// Unit tests for dynamic mapping. +/// +public sealed class WeaviateDynamicMapperTests +{ + private const bool HasNamedVectors = true; + + private static readonly JsonSerializerOptions s_jsonSerializerOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + Converters = + { + new WeaviateDateTimeOffsetConverter(), + new WeaviateNullableDateTimeOffsetConverter() + } + }; + + private static readonly CollectionModel s_model = new WeaviateModelBuilder(HasNamedVectors) + .BuildDynamic( + new VectorStoreCollectionDefinition + { + Properties = + [ + new VectorStoreKeyProperty("Key", typeof(Guid)), + + new VectorStoreDataProperty("StringDataProp", typeof(string)), + new VectorStoreDataProperty("BoolDataProp", typeof(bool)), + new VectorStoreDataProperty("NullableBoolDataProp", typeof(bool?)), + new VectorStoreDataProperty("IntDataProp", typeof(int)), + new VectorStoreDataProperty("NullableIntDataProp", typeof(int?)), + new VectorStoreDataProperty("LongDataProp", typeof(long)), + new VectorStoreDataProperty("NullableLongDataProp", typeof(long?)), + new VectorStoreDataProperty("ShortDataProp", typeof(short)), + new VectorStoreDataProperty("NullableShortDataProp", typeof(short?)), + new VectorStoreDataProperty("ByteDataProp", typeof(byte)), + new VectorStoreDataProperty("NullableByteDataProp", typeof(byte?)), + new VectorStoreDataProperty("FloatDataProp", typeof(float)), + new VectorStoreDataProperty("NullableFloatDataProp", typeof(float?)), + new VectorStoreDataProperty("DoubleDataProp", typeof(double)), + new VectorStoreDataProperty("NullableDoubleDataProp", typeof(double?)), + new VectorStoreDataProperty("DecimalDataProp", typeof(decimal)), + new VectorStoreDataProperty("NullableDecimalDataProp", typeof(decimal?)), + new VectorStoreDataProperty("DateTimeDataProp", typeof(DateTime)), + new VectorStoreDataProperty("NullableDateTimeDataProp", typeof(DateTime?)), + new VectorStoreDataProperty("DateTimeOffsetDataProp", typeof(DateTimeOffset)), + new VectorStoreDataProperty("NullableDateTimeOffsetDataProp", typeof(DateTimeOffset?)), + new VectorStoreDataProperty("GuidDataProp", typeof(Guid)), + new VectorStoreDataProperty("NullableGuidDataProp", typeof(Guid?)), + new VectorStoreDataProperty("TagListDataProp", typeof(List)), + + new VectorStoreVectorProperty("FloatVector", typeof(ReadOnlyMemory), 10), + new VectorStoreVectorProperty("NullableFloatVector", typeof(ReadOnlyMemory?), 10) + ] + }, + defaultEmbeddingGenerator: null, + s_jsonSerializerOptions); + + private static readonly float[] s_floatVector = [1.0f, 2.0f, 3.0f]; + private static readonly List s_taglist = ["tag1", "tag2"]; + + [Fact] + public void MapFromDataToStorageModelMapsAllSupportedTypes() + { + // Arrange + var key = new Guid("55555555-5555-5555-5555-555555555555"); + var sut = new WeaviateMapper>("Collection", HasNamedVectors, s_model, s_jsonSerializerOptions); + + var dataModel = new Dictionary + { + ["Key"] = key, + + ["StringDataProp"] = "string", + ["BoolDataProp"] = true, + ["NullableBoolDataProp"] = false, + ["IntDataProp"] = 1, + ["NullableIntDataProp"] = 2, + ["LongDataProp"] = 3L, + ["NullableLongDataProp"] = 4L, + ["ShortDataProp"] = (short)5, + ["NullableShortDataProp"] = (short)6, + ["ByteDataProp"] = (byte)7, + ["NullableByteDataProp"] = (byte)8, + ["FloatDataProp"] = 9.0f, + ["NullableFloatDataProp"] = 10.0f, + ["DoubleDataProp"] = 11.0, + ["NullableDoubleDataProp"] = 12.0, + ["DecimalDataProp"] = 13.99m, + ["NullableDecimalDataProp"] = 14.00m, + ["DateTimeDataProp"] = new DateTime(2021, 1, 1), + ["NullableDateTimeDataProp"] = new DateTime(2021, 1, 1), + ["DateTimeOffsetDataProp"] = new DateTimeOffset(2022, 1, 1, 0, 0, 0, TimeSpan.Zero), + ["NullableDateTimeOffsetDataProp"] = new DateTimeOffset(2022, 1, 1, 0, 0, 0, TimeSpan.Zero), + ["GuidDataProp"] = new Guid("11111111-1111-1111-1111-111111111111"), + ["NullableGuidDataProp"] = new Guid("22222222-2222-2222-2222-222222222222"), + ["TagListDataProp"] = s_taglist, + + ["FloatVector"] = new ReadOnlyMemory(s_floatVector), + ["NullableFloatVector"] = new ReadOnlyMemory(s_floatVector), + } + ; + + // Act + var storageModel = sut.MapFromDataToStorageModel(dataModel, recordIndex: 0, generatedEmbeddings: null); + + // Assert + Assert.Equal(key, (Guid?)storageModel["id"]); + Assert.Equal("Collection", (string?)storageModel["class"]); + Assert.Equal("string", (string?)storageModel["properties"]?["stringDataProp"]); + Assert.Equal(true, (bool?)storageModel["properties"]?["boolDataProp"]); + Assert.Equal(false, (bool?)storageModel["properties"]?["nullableBoolDataProp"]); + Assert.Equal(1, (int?)storageModel["properties"]?["intDataProp"]); + Assert.Equal(2, (int?)storageModel["properties"]?["nullableIntDataProp"]); + Assert.Equal(3L, (long?)storageModel["properties"]?["longDataProp"]); + Assert.Equal(4L, (long?)storageModel["properties"]?["nullableLongDataProp"]); + Assert.Equal((short)5, (short?)storageModel["properties"]?["shortDataProp"]); + Assert.Equal((short)6, (short?)storageModel["properties"]?["nullableShortDataProp"]); + Assert.Equal((byte)7, (byte?)storageModel["properties"]?["byteDataProp"]); + Assert.Equal((byte)8, (byte?)storageModel["properties"]?["nullableByteDataProp"]); + Assert.Equal(9.0f, (float?)storageModel["properties"]?["floatDataProp"]); + Assert.Equal(10.0f, (float?)storageModel["properties"]?["nullableFloatDataProp"]); + Assert.Equal(11.0, (double?)storageModel["properties"]?["doubleDataProp"]); + Assert.Equal(12.0, (double?)storageModel["properties"]?["nullableDoubleDataProp"]); + Assert.Equal(13.99m, (decimal?)storageModel["properties"]?["decimalDataProp"]); + Assert.Equal(14.00m, (decimal?)storageModel["properties"]?["nullableDecimalDataProp"]); + Assert.Equal(new DateTime(2021, 1, 1, 0, 0, 0), (DateTime?)storageModel["properties"]?["dateTimeDataProp"]); + Assert.Equal(new DateTime(2021, 1, 1, 0, 0, 0), (DateTime?)storageModel["properties"]?["nullableDateTimeDataProp"]); + Assert.Equal(new DateTimeOffset(2022, 1, 1, 0, 0, 0, TimeSpan.Zero), (DateTimeOffset?)storageModel["properties"]?["dateTimeOffsetDataProp"]); + Assert.Equal(new DateTimeOffset(2022, 1, 1, 0, 0, 0, TimeSpan.Zero), (DateTimeOffset?)storageModel["properties"]?["nullableDateTimeOffsetDataProp"]); + Assert.Equal(new Guid("11111111-1111-1111-1111-111111111111"), (Guid?)storageModel["properties"]?["guidDataProp"]); + Assert.Equal(new Guid("22222222-2222-2222-2222-222222222222"), (Guid?)storageModel["properties"]?["nullableGuidDataProp"]); + Assert.Equal(s_taglist, storageModel["properties"]?["tagListDataProp"]!.AsArray().GetValues().ToArray()); + Assert.Equal(s_floatVector, storageModel["vectors"]?["floatVector"]!.AsArray().GetValues().ToArray()); + Assert.Equal(s_floatVector, storageModel["vectors"]?["nullableFloatVector"]!.AsArray().GetValues().ToArray()); + } + + [Fact] + public void MapFromDataToStorageModelMapsNullValues() + { + // Arrange + var key = new Guid("55555555-5555-5555-5555-555555555555"); + var keyProperty = new VectorStoreKeyProperty("Key", typeof(Guid)); + + var dataProperties = new List + { + new("StringDataProp", typeof(string)), + new("NullableIntDataProp", typeof(int?)), + }; + + var vectorProperties = new List + { + new("NullableFloatVector", typeof(ReadOnlyMemory?), 10) + }; + + var dataModel = new Dictionary + { + ["Key"] = key, + + ["StringDataProp"] = null, + ["NullableIntDataProp"] = null, + + ["NullableFloatVector"] = null + }; + + var sut = new WeaviateMapper>("Collection", HasNamedVectors, s_model, s_jsonSerializerOptions); + + // Act + var storageModel = sut.MapFromDataToStorageModel(dataModel, recordIndex: 0, generatedEmbeddings: null); + + // Assert + Assert.Null(storageModel["StringDataProp"]); + Assert.Null(storageModel["NullableIntDataProp"]); + Assert.Null(storageModel["NullableFloatVector"]); + } + + [Fact] + public void MapFromStorageToDataModelMapsAllSupportedTypes() + { + // Arrange + var key = new Guid("55555555-5555-5555-5555-555555555555"); + var sut = new WeaviateMapper>("Collection", HasNamedVectors, s_model, s_jsonSerializerOptions); + + var storageModel = new JsonObject + { + ["id"] = key, + ["properties"] = new JsonObject + { + ["stringDataProp"] = "string", + ["boolDataProp"] = true, + ["nullableBoolDataProp"] = false, + ["intDataProp"] = 1, + ["nullableIntDataProp"] = 2, + ["longDataProp"] = 3L, + ["nullableLongDataProp"] = 4L, + ["shortDataProp"] = (short)5, + ["nullableShortDataProp"] = (short)6, + ["byteDataProp"] = (byte)7, + ["nullableByteDataProp"] = (byte)8, + ["floatDataProp"] = 9.0f, + ["nullableFloatDataProp"] = 10.0f, + ["doubleDataProp"] = 11.0, + ["nullableDoubleDataProp"] = 12.0, + ["decimalDataProp"] = 13.99m, + ["nullableDecimalDataProp"] = 14.00m, + ["dateTimeDataProp"] = new DateTime(2021, 1, 1), + ["nullableDateTimeDataProp"] = new DateTime(2021, 1, 1), + ["dateTimeOffsetDataProp"] = new DateTimeOffset(2022, 1, 1, 0, 0, 0, TimeSpan.Zero), + ["nullableDateTimeOffsetDataProp"] = new DateTimeOffset(2022, 1, 1, 0, 0, 0, TimeSpan.Zero), + ["guidDataProp"] = new Guid("11111111-1111-1111-1111-111111111111"), + ["nullableGuidDataProp"] = new Guid("22222222-2222-2222-2222-222222222222"), + ["tagListDataProp"] = new JsonArray(s_taglist.Select(l => (JsonValue)l).ToArray()) + }, + ["vectors"] = new JsonObject + { + ["floatVector"] = new JsonArray(s_floatVector.Select(l => (JsonValue)l).ToArray()), + ["nullableFloatVector"] = new JsonArray(s_floatVector.Select(l => (JsonValue)l).ToArray()), + } + }; + + // Act + var dataModel = sut.MapFromStorageToDataModel(storageModel, includeVectors: true); + + // Assert + Assert.Equal(key, dataModel["Key"]); + Assert.Equal("string", dataModel["StringDataProp"]); + Assert.Equal(true, dataModel["BoolDataProp"]); + Assert.Equal(false, dataModel["NullableBoolDataProp"]); + Assert.Equal(1, dataModel["IntDataProp"]); + Assert.Equal(2, dataModel["NullableIntDataProp"]); + Assert.Equal(3L, dataModel["LongDataProp"]); + Assert.Equal(4L, dataModel["NullableLongDataProp"]); + Assert.Equal((short)5, dataModel["ShortDataProp"]); + Assert.Equal((short)6, dataModel["NullableShortDataProp"]); + Assert.Equal((byte)7, dataModel["ByteDataProp"]); + Assert.Equal((byte)8, dataModel["NullableByteDataProp"]); + Assert.Equal(9.0f, dataModel["FloatDataProp"]); + Assert.Equal(10.0f, dataModel["NullableFloatDataProp"]); + Assert.Equal(11.0, dataModel["DoubleDataProp"]); + Assert.Equal(12.0, dataModel["NullableDoubleDataProp"]); + Assert.Equal(13.99m, dataModel["DecimalDataProp"]); + Assert.Equal(14.00m, dataModel["NullableDecimalDataProp"]); + Assert.Equal(new DateTime(2021, 1, 1, 0, 0, 0), dataModel["DateTimeDataProp"]); + Assert.Equal(new DateTime(2021, 1, 1, 0, 0, 0), dataModel["NullableDateTimeDataProp"]); + Assert.Equal(new DateTimeOffset(2022, 1, 1, 0, 0, 0, TimeSpan.Zero), dataModel["DateTimeOffsetDataProp"]); + Assert.Equal(new DateTimeOffset(2022, 1, 1, 0, 0, 0, TimeSpan.Zero), dataModel["NullableDateTimeOffsetDataProp"]); + Assert.Equal(new Guid("11111111-1111-1111-1111-111111111111"), dataModel["GuidDataProp"]); + Assert.Equal(new Guid("22222222-2222-2222-2222-222222222222"), dataModel["NullableGuidDataProp"]); + Assert.Equal(s_taglist, dataModel["TagListDataProp"]); + Assert.Equal(s_floatVector, ((ReadOnlyMemory)dataModel["FloatVector"]!).ToArray()); + Assert.Equal(s_floatVector, ((ReadOnlyMemory)dataModel["NullableFloatVector"]!)!.ToArray()); + } + + [Fact] + public void MapFromStorageToDataModelMapsNullValues() + { + // Arrange + var key = new Guid("55555555-5555-5555-5555-555555555555"); + var keyProperty = new VectorStoreKeyProperty("Key", typeof(Guid)); + + var storageModel = new JsonObject + { + ["id"] = key, + ["properties"] = new JsonObject + { + ["stringDataProp"] = null, + ["nullableIntDataProp"] = null, + }, + ["vectors"] = new JsonObject + { + ["nullableFloatVector"] = null + } + }; + + var sut = new WeaviateMapper>("Collection", HasNamedVectors, s_model, s_jsonSerializerOptions); + + // Act + var dataModel = sut.MapFromStorageToDataModel(storageModel, includeVectors: true); + + // Assert + Assert.Equal(key, dataModel["Key"]); + Assert.Null(dataModel["StringDataProp"]); + Assert.Null(dataModel["NullableIntDataProp"]); + Assert.Null(dataModel["NullableFloatVector"]); + } + + [Fact] + public void MapFromStorageToDataModelThrowsForMissingKey() + { + // Arrange + var sut = new WeaviateMapper>("Collection", HasNamedVectors, s_model, s_jsonSerializerOptions); + + var storageModel = new JsonObject(); + + // Act & Assert + var exception = Assert.Throws( + () => sut.MapFromStorageToDataModel(storageModel, includeVectors: true)); + } + + [Fact] + public void MapFromDataToStorageModelSkipsMissingProperties() + { + // Arrange + var recordDefinition = new VectorStoreCollectionDefinition + { + Properties = + [ + new VectorStoreKeyProperty("Key", typeof(Guid)), + new VectorStoreDataProperty("StringDataProp", typeof(string)), + new VectorStoreDataProperty("NullableIntDataProp", typeof(int?)), + new VectorStoreVectorProperty("FloatVector", typeof(ReadOnlyMemory), 10) + ] + }; + + var model = new WeaviateModelBuilder(HasNamedVectors).BuildDynamic(recordDefinition, defaultEmbeddingGenerator: null, s_jsonSerializerOptions); + + var key = new Guid("55555555-5555-5555-5555-555555555555"); + + var record = new Dictionary { ["Key"] = key }; + var sut = new WeaviateMapper>("Collection", HasNamedVectors, model, s_jsonSerializerOptions); + + // Act + var storageModel = sut.MapFromDataToStorageModel(record, recordIndex: 0, generatedEmbeddings: null); + + // Assert + Assert.Equal(key, (Guid?)storageModel["id"]); + Assert.False(storageModel.ContainsKey("StringDataProp")); + Assert.False(storageModel.ContainsKey("FloatVector")); + } + + [Fact] + public void MapFromStorageToDataModelSkipsMissingProperties() + { + // Arrange + var recordDefinition = new VectorStoreCollectionDefinition + { + Properties = + [ + new VectorStoreKeyProperty("Key", typeof(Guid)), + new VectorStoreDataProperty("StringDataProp", typeof(string)), + new VectorStoreDataProperty("NullableIntDataProp", typeof(int?)), + new VectorStoreVectorProperty("FloatVector", typeof(ReadOnlyMemory), 10) + ] + }; + + var model = new WeaviateModelBuilder(HasNamedVectors).BuildDynamic(recordDefinition, defaultEmbeddingGenerator: null, s_jsonSerializerOptions); + + var key = new Guid("55555555-5555-5555-5555-555555555555"); + + var sut = new WeaviateMapper>("Collection", HasNamedVectors, model, s_jsonSerializerOptions); + + var storageModel = new JsonObject + { + ["id"] = key + }; + + // Act + var dataModel = sut.MapFromStorageToDataModel(storageModel, includeVectors: true); + + // Assert + Assert.Equal(key, dataModel["Key"]); + Assert.False(dataModel.ContainsKey("StringDataProp")); + Assert.False(dataModel.ContainsKey("FloatVector")); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void MapFromDataToStorageModelMapsNamedVectorsCorrectly(bool hasNamedVectors) + { + // Arrange + var recordDefinition = new VectorStoreCollectionDefinition + { + Properties = + [ + new VectorStoreKeyProperty("Key", typeof(Guid)), + new VectorStoreVectorProperty("FloatVector", typeof(ReadOnlyMemory), 4) + ] + }; + + var model = new WeaviateModelBuilder(hasNamedVectors).BuildDynamic(recordDefinition, defaultEmbeddingGenerator: null, s_jsonSerializerOptions); + + var key = new Guid("55555555-5555-5555-5555-555555555555"); + + var record = new Dictionary { ["Key"] = key, ["FloatVector"] = new ReadOnlyMemory(s_floatVector) }; + var sut = new WeaviateMapper>("Collection", hasNamedVectors, model, s_jsonSerializerOptions); + + // Act + var storageModel = sut.MapFromDataToStorageModel(record, recordIndex: 0, generatedEmbeddings: null); + + // Assert + var vectorProperty = hasNamedVectors ? storageModel["vectors"]!["floatVector"] : storageModel["vector"]; + + Assert.Equal(key, (Guid?)storageModel["id"]); + Assert.Equal(s_floatVector, vectorProperty!.AsArray().GetValues().ToArray()); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void MapFromStorageToDataModelMapsNamedVectorsCorrectly(bool hasNamedVectors) + { + // Arrange + var recordDefinition = new VectorStoreCollectionDefinition + { + Properties = + [ + new VectorStoreKeyProperty("Key", typeof(Guid)), + new VectorStoreVectorProperty("FloatVector", typeof(ReadOnlyMemory), 4) + ] + }; + + var model = new WeaviateModelBuilder(hasNamedVectors).BuildDynamic(recordDefinition, defaultEmbeddingGenerator: null, s_jsonSerializerOptions); + + var key = new Guid("55555555-5555-5555-5555-555555555555"); + + var sut = new WeaviateMapper>("Collection", hasNamedVectors, model, s_jsonSerializerOptions); + + var storageModel = new JsonObject { ["id"] = key }; + + var vector = new JsonArray(s_floatVector.Select(l => (JsonValue)l).ToArray()); + + if (hasNamedVectors) + { + storageModel["vectors"] = new JsonObject + { + ["floatVector"] = vector + }; + } + else + { + storageModel["vector"] = vector; + } + + // Act + var dataModel = sut.MapFromStorageToDataModel(storageModel, includeVectors: true); + + // Assert + Assert.Equal(key, dataModel["Key"]); + Assert.Equal(s_floatVector, ((ReadOnlyMemory)dataModel["FloatVector"]!).ToArray()); + } +} diff --git a/MEVD/test/Weaviate.UnitTests/WeaviateHotel.cs b/MEVD/test/Weaviate.UnitTests/WeaviateHotel.cs new file mode 100644 index 0000000..c367cdc --- /dev/null +++ b/MEVD/test/Weaviate.UnitTests/WeaviateHotel.cs @@ -0,0 +1,50 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Text.Json.Serialization; +using Microsoft.Extensions.VectorData; + +namespace SemanticKernel.Connectors.Weaviate.UnitTests; + +#pragma warning disable CS8618 + +public sealed record WeaviateHotel +{ + /// The key of the record. + [VectorStoreKey] + public Guid HotelId { get; init; } + + /// A string metadata field. + [VectorStoreData(IsIndexed = true)] + public string? HotelName { get; set; } + + /// An int metadata field. + [VectorStoreData] + public int HotelCode { get; set; } + + /// A float metadata field. + [VectorStoreData] + public float? HotelRating { get; set; } + + /// A bool metadata field. + [JsonPropertyName("parking_is_included")] + [VectorStoreData] + public bool ParkingIncluded { get; set; } + + /// An array metadata field. + [VectorStoreData] + public List Tags { get; set; } = []; + + /// A data field. + [VectorStoreData(IsFullTextIndexed = true)] + public string Description { get; set; } + + [VectorStoreData] + public DateTimeOffset Timestamp { get; set; } + + /// A vector field. + [VectorStoreVector(dimensions: 4, DistanceFunction = DistanceFunction.CosineDistance, IndexKind = IndexKind.Hnsw)] + public ReadOnlyMemory? DescriptionEmbedding { get; set; } +} diff --git a/MEVD/test/Weaviate.UnitTests/WeaviateMapperTests.cs b/MEVD/test/Weaviate.UnitTests/WeaviateMapperTests.cs new file mode 100644 index 0000000..d3707d0 --- /dev/null +++ b/MEVD/test/Weaviate.UnitTests/WeaviateMapperTests.cs @@ -0,0 +1,131 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.Json.Serialization; +using Microsoft.Extensions.VectorData; +using CommunityToolkit.VectorData.Weaviate; +using Xunit; + +namespace SemanticKernel.Connectors.Weaviate.UnitTests; + +/// +/// Unit tests for class. +/// +public sealed class WeaviateMapperTests +{ + private static readonly JsonSerializerOptions s_jsonSerializerOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + Converters = + { + new WeaviateDateTimeOffsetConverter(), + new WeaviateNullableDateTimeOffsetConverter() + } + }; + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void MapFromDataToStorageModelReturnsValidObject(bool hasNamedVectors) + { + var key = new Guid("55555555-5555-5555-5555-555555555555"); + // Arrange + var hotel = new WeaviateHotel + { + HotelId = key, + HotelName = "Test Name", + Tags = ["tag1", "tag2"], + DescriptionEmbedding = new ReadOnlyMemory([1f, 2f, 3f]) + }; + + var sut = GetMapper(hasNamedVectors); + + // Act + var document = sut.MapFromDataToStorageModel(hotel, recordIndex: 0, generatedEmbeddings: null); + + // Assert + Assert.NotNull(document); + + Assert.Equal(key, document["id"]!.GetValue()); + Assert.Equal("Test Name", document["properties"]!["hotelName"]!.GetValue()); + Assert.Equal(["tag1", "tag2"], document["properties"]!["tags"]!.AsArray().Select(l => l!.GetValue())); + + var vectorNode = hasNamedVectors ? document["vectors"]!["descriptionEmbedding"] : document["vector"]; + + Assert.Equal([1f, 2f, 3f], vectorNode!.AsArray().Select(l => l!.GetValue())); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void MapFromStorageToDataModelReturnsValidObject(bool hasNamedVectors) + { + var key = new Guid("55555555-5555-5555-5555-555555555555"); + + // Arrange + var document = new JsonObject + { + ["id"] = key, + ["properties"] = new JsonObject(), + ["vectors"] = new JsonObject() + }; + + document["properties"]!["hotelName"] = "Test Name"; + document["properties"]!["tags"] = new JsonArray(new List { "tag1", "tag2" }.Select(l => JsonValue.Create(l)).ToArray()); + + var vectorNode = new JsonArray(new List { 1f, 2f, 3f }.Select(l => JsonValue.Create(l)).ToArray()); + + if (hasNamedVectors) + { + document["vectors"]!["descriptionEmbedding"] = vectorNode; + } + else + { + document["vector"] = vectorNode; + } + + var sut = GetMapper(hasNamedVectors); + + // Act + var hotel = sut.MapFromStorageToDataModel(document, includeVectors: true); + + // Assert + Assert.NotNull(hotel); + + Assert.Equal(key, hotel.HotelId); + Assert.Equal("Test Name", hotel.HotelName); + Assert.Equal(["tag1", "tag2"], hotel.Tags); + Assert.True(new ReadOnlyMemory([1f, 2f, 3f]).Span.SequenceEqual(hotel.DescriptionEmbedding!.Value.Span)); + } + + #region private + + private static WeaviateMapper GetMapper(bool hasNamedVectors) => new( + "CollectionName", + hasNamedVectors, + new WeaviateModelBuilder(hasNamedVectors) + .Build( + typeof(WeaviateHotel), + typeof(Guid), + new VectorStoreCollectionDefinition + { + Properties = + [ + new VectorStoreKeyProperty("HotelId", typeof(Guid)), + new VectorStoreDataProperty("HotelName", typeof(string)), + new VectorStoreDataProperty("Tags", typeof(List)), + new VectorStoreVectorProperty("DescriptionEmbedding", typeof(ReadOnlyMemory), 10) + ] + }, + defaultEmbeddingGenerator: null, + s_jsonSerializerOptions), + s_jsonSerializerOptions); + + #endregion +} diff --git a/MEVD/test/Weaviate.UnitTests/WeaviateQueryBuilderTests.cs b/MEVD/test/Weaviate.UnitTests/WeaviateQueryBuilderTests.cs new file mode 100644 index 0000000..dd04e40 --- /dev/null +++ b/MEVD/test/Weaviate.UnitTests/WeaviateQueryBuilderTests.cs @@ -0,0 +1,214 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Text.Json; +using System.Text.Json.Serialization; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; +using CommunityToolkit.VectorData.Weaviate; +using Xunit; + +namespace SemanticKernel.Connectors.Weaviate.UnitTests; + +/// +/// Unit tests for class. +/// +public sealed class WeaviateQueryBuilderTests +{ + private const string CollectionName = "Collection"; + private const string VectorPropertyName = "descriptionEmbedding"; + + private static readonly JsonSerializerOptions s_jsonSerializerOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + Converters = + { + new WeaviateDateTimeOffsetConverter(), + new WeaviateNullableDateTimeOffsetConverter() + } + }; + + private readonly CollectionModel _model = new WeaviateModelBuilder(hasNamedVectors: true) + .BuildDynamic( + new() + { + Properties = + [ + new VectorStoreKeyProperty("HotelId", typeof(Guid)) { StorageName = "hotelId" }, + new VectorStoreDataProperty("HotelName", typeof(string)) { StorageName = "hotelName" }, + new VectorStoreDataProperty("HotelCode", typeof(string)) { StorageName = "hotelCode" }, + new VectorStoreDataProperty("Tags", typeof(string[])) { StorageName = "tags" }, + new VectorStoreVectorProperty("DescriptionEmbedding", typeof(ReadOnlyMemory), 10) { StorageName = "descriptionEmbeddding" }, + ] + }, + defaultEmbeddingGenerator: null); + + private readonly ReadOnlyMemory _vector = new([31f, 32f, 33f, 34f]); + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void BuildSearchQueryByDefaultReturnsValidQuery(bool hasNamedVectors) + { + // Arrange + var expectedQuery = $$""" + { + Get { + Collection ( + limit: 3 + offset: 2 + {{string.Empty}} + nearVector: { + {{(hasNamedVectors ? "targetVectors: [\"descriptionEmbedding\"]" : string.Empty)}} + vector: [31,32,33,34] + {{string.Empty}} + } + ) { + HotelName HotelCode Tags + _additional { + id + distance + {{string.Empty}} + } + } + } + } + """; + + var searchOptions = new VectorSearchOptions + { + Skip = 2, + }; + + // Act + var query = WeaviateQueryBuilder.BuildSearchQuery( + this._vector, + CollectionName, + VectorPropertyName, + s_jsonSerializerOptions, + top: 3, + searchOptions, + this._model, + hasNamedVectors); + + // Assert + Assert.Equal(expectedQuery, query); + + Assert.DoesNotContain("vectors", query); + Assert.DoesNotContain("where", query); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void BuildSearchQueryWithIncludedVectorsReturnsValidQuery(bool hasNamedVectors) + { + // Arrange + var searchOptions = new VectorSearchOptions + { + Skip = 2, + IncludeVectors = true + }; + + // Act + var query = WeaviateQueryBuilder.BuildSearchQuery( + this._vector, + CollectionName, + VectorPropertyName, + s_jsonSerializerOptions, + top: 3, + searchOptions, + this._model, + hasNamedVectors); + + // Assert + var vectorQuery = hasNamedVectors ? "vectors { DescriptionEmbedding }" : "vector"; + + Assert.Contains(vectorQuery, query); + } + + [Fact] + public void BuildHybridSearchQueryEscapesDoubleQuotesInKeywords() + { + // Arrange + var searchOptions = new HybridSearchOptions { Skip = 0 }; + var vectorProperty = this._model.VectorProperties[0]; + var textProperty = this._model.DataProperties[0]; + + // Act + var query = WeaviateQueryBuilder.BuildHybridSearchQuery( + this._vector, + top: 3, + keywords: "test \"injection\"", + CollectionName, + this._model, + vectorProperty, + textProperty, + s_jsonSerializerOptions, + searchOptions, + hasNamedVectors: true); + + // Assert - the double quote must be escaped in the GraphQL string + Assert.Contains("query: \"test \\\"injection\\\"\"", query); + } + + [Fact] + public void BuildHybridSearchQueryEscapesBackslashInKeywords() + { + // Arrange + var searchOptions = new HybridSearchOptions { Skip = 0 }; + var vectorProperty = this._model.VectorProperties[0]; + var textProperty = this._model.DataProperties[0]; + + // Act + var query = WeaviateQueryBuilder.BuildHybridSearchQuery( + this._vector, + top: 3, + keywords: @"test\path", + CollectionName, + this._model, + vectorProperty, + textProperty, + s_jsonSerializerOptions, + searchOptions, + hasNamedVectors: true); + + // Assert - backslash must be escaped + Assert.Contains(@"query: ""test\\path""", query); + } + + [Fact] + public void BuildHybridSearchQueryWithPlainKeywordsWorks() + { + // Arrange + var searchOptions = new HybridSearchOptions { Skip = 0 }; + var vectorProperty = this._model.VectorProperties[0]; + var textProperty = this._model.DataProperties[0]; + + // Act + var query = WeaviateQueryBuilder.BuildHybridSearchQuery( + this._vector, + top: 3, + keywords: "hello world", + CollectionName, + this._model, + vectorProperty, + textProperty, + s_jsonSerializerOptions, + searchOptions, + hasNamedVectors: true); + + // Assert + Assert.Contains("query: \"hello world\"", query); + } + + #region private + +#pragma warning disable CA1812 // An internal class that is apparently never instantiated. If so, remove the code from the assembly. + private sealed class DummyType; +#pragma warning restore CA1812 + + #endregion +} diff --git a/MEVD/test/Weaviate.UnitTests/WeaviateVectorStoreTests.cs b/MEVD/test/Weaviate.UnitTests/WeaviateVectorStoreTests.cs new file mode 100644 index 0000000..ea250dd --- /dev/null +++ b/MEVD/test/Weaviate.UnitTests/WeaviateVectorStoreTests.cs @@ -0,0 +1,81 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Text.Json; +using System.Threading.Tasks; +using CommunityToolkit.VectorData.Weaviate; +using Xunit; + +namespace SemanticKernel.Connectors.Weaviate.UnitTests; + +/// +/// Unit tests for class. +/// +public sealed class WeaviateVectorStoreTests : IDisposable +{ + private readonly HttpMessageHandlerStub _messageHandlerStub = new(); + private readonly HttpClient _mockHttpClient; + + public WeaviateVectorStoreTests() + { + this._mockHttpClient = new(this._messageHandlerStub, false) { BaseAddress = new Uri("http://test") }; + } + + [Fact] + public void GetCollectionWithNotSupportedKeyThrowsException() + { + // Arrange + using var sut = new WeaviateVectorStore(this._mockHttpClient); + + // Act & Assert + Assert.Throws(() => sut.GetCollection("Collection")); + } + + [Fact] + public void GetCollectionWithSupportedKeyReturnsCollection() + { + // Arrange + using var sut = new WeaviateVectorStore(this._mockHttpClient); + + // Act + var collection = sut.GetCollection("Collection1"); + + // Assert + Assert.NotNull(collection); + } + + [Fact] + public async Task ListCollectionNamesReturnsCollectionNamesAsync() + { + // Arrange + var expectedCollectionNames = new List { "Collection1", "Collection2", "Collection3" }; + var response = new WeaviateGetCollectionsResponse + { + Collections = expectedCollectionNames.Select(name => new WeaviateCollectionSchema(name)).ToList() + }; + + this._messageHandlerStub.ResponseToReturn = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(JsonSerializer.Serialize(response)) + }; + + using var sut = new WeaviateVectorStore(this._mockHttpClient); + + // Act + var actualCollectionNames = await sut.ListCollectionNamesAsync().ToListAsync(); + + // Assert + Assert.Equal(expectedCollectionNames, actualCollectionNames); + } + + public void Dispose() + { + this._mockHttpClient.Dispose(); + this._messageHandlerStub.Dispose(); + } +} diff --git a/Shared/LegacySupport/CallerAttributes/CallerArgumentExpressionAttribute.cs b/Shared/LegacySupport/CallerAttributes/CallerArgumentExpressionAttribute.cs new file mode 100644 index 0000000..364f9ef --- /dev/null +++ b/Shared/LegacySupport/CallerAttributes/CallerArgumentExpressionAttribute.cs @@ -0,0 +1,32 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#pragma warning disable IDE0079 +#pragma warning disable SA1101 +#pragma warning disable SA1512 + +using System.Diagnostics.CodeAnalysis; + +namespace System.Runtime.CompilerServices; + +/// +/// Tags parameter that should be filled with specific caller name. +/// +[AttributeUsage(AttributeTargets.Parameter, AllowMultiple = false, Inherited = false)] +[ExcludeFromCodeCoverage] +internal sealed class CallerArgumentExpressionAttribute : Attribute +{ + /// + /// Initializes a new instance of the class. + /// + /// Function parameter to take the name from. + public CallerArgumentExpressionAttribute(string parameterName) + { + ParameterName = parameterName; + } + + /// + /// Gets name of the function parameter that name should be taken from. + /// + public string ParameterName { get; } +} diff --git a/Shared/LegacySupport/DiagnosticAttributes/NullableAttributes.cs b/Shared/LegacySupport/DiagnosticAttributes/NullableAttributes.cs new file mode 100644 index 0000000..300e045 --- /dev/null +++ b/Shared/LegacySupport/DiagnosticAttributes/NullableAttributes.cs @@ -0,0 +1,177 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#pragma warning disable IDE0079 +#pragma warning disable IDE0079 +#pragma warning disable SA1101 +#pragma warning disable SA1116 +#pragma warning disable SA1117 +#pragma warning disable SA1402 +#pragma warning disable SA1512 +#pragma warning disable SA1623 +#pragma warning disable SA1642 +#pragma warning disable SA1623 +#pragma warning disable SA1642 +#pragma warning disable SA1649 +#pragma warning disable S3903 +#pragma warning disable IDE0021 // Use block body for constructors +#pragma warning disable CA1019 + +namespace System.Diagnostics.CodeAnalysis; + +#if !NETCOREAPP3_1_OR_GREATER +/// Specifies that null is allowed as an input even if the corresponding type disallows it. +[AttributeUsage(AttributeTargets.Field | AttributeTargets.Parameter | AttributeTargets.Property, Inherited = false)] +[ExcludeFromCodeCoverage] +internal sealed class AllowNullAttribute : Attribute +{ +} + +/// Specifies that null is disallowed as an input even if the corresponding type allows it. +[AttributeUsage(AttributeTargets.Field | AttributeTargets.Parameter | AttributeTargets.Property, Inherited = false)] +[ExcludeFromCodeCoverage] +internal sealed class DisallowNullAttribute : Attribute +{ +} + +/// Specifies that an output may be null even if the corresponding type disallows it. +[AttributeUsage(AttributeTargets.Field | AttributeTargets.Parameter | AttributeTargets.Property | AttributeTargets.ReturnValue, Inherited = false)] +[ExcludeFromCodeCoverage] +internal sealed class MaybeNullAttribute : Attribute +{ +} + +/// Specifies that an output will not be null even if the corresponding type allows it. Specifies that an input argument was not null when the call returns. +[AttributeUsage(AttributeTargets.Field | AttributeTargets.Parameter | AttributeTargets.Property | AttributeTargets.ReturnValue, Inherited = false)] +[ExcludeFromCodeCoverage] +internal sealed class NotNullAttribute : Attribute +{ +} + +/// Specifies that when a method returns , the parameter may be null even if the corresponding type disallows it. +[AttributeUsage(AttributeTargets.Parameter, Inherited = false)] +[ExcludeFromCodeCoverage] +internal sealed class MaybeNullWhenAttribute : Attribute +{ + /// Initializes the attribute with the specified return value condition. + /// + /// The return value condition. If the method returns this value, the associated parameter may be . + /// + public MaybeNullWhenAttribute(bool returnValue) => ReturnValue = returnValue; + + /// Gets the return value condition. + public bool ReturnValue { get; } +} + +/// Specifies that when a method returns , the parameter will not be null even if the corresponding type allows it. +[AttributeUsage(AttributeTargets.Parameter, Inherited = false)] +[ExcludeFromCodeCoverage] +internal sealed class NotNullWhenAttribute : Attribute +{ + /// Initializes the attribute with the specified return value condition. + /// + /// The return value condition. If the method returns this value, the associated parameter will not be . + /// + public NotNullWhenAttribute(bool returnValue) => ReturnValue = returnValue; + + /// Gets the return value condition. + public bool ReturnValue { get; } +} + +/// Specifies that the output will be non-null if the named parameter is non-null. +[AttributeUsage(AttributeTargets.Parameter | AttributeTargets.Property | AttributeTargets.ReturnValue, AllowMultiple = true, Inherited = false)] +[ExcludeFromCodeCoverage] +internal sealed class NotNullIfNotNullAttribute : Attribute +{ + /// Initializes the attribute with the associated parameter name. + /// + /// The associated parameter name. The output will be non-null if the argument to the parameter specified is non-null. + /// + public NotNullIfNotNullAttribute(string parameterName) => ParameterName = parameterName; + + /// Gets the associated parameter name. + public string ParameterName { get; } +} + +/// Applied to a method that will never return under any circumstance. +[AttributeUsage(AttributeTargets.Method, Inherited = false)] +[ExcludeFromCodeCoverage] +internal sealed class DoesNotReturnAttribute : Attribute +{ +} + +/// Specifies that the method will not return if the associated Boolean parameter is passed the specified value. +[AttributeUsage(AttributeTargets.Parameter, Inherited = false)] +[ExcludeFromCodeCoverage] +internal sealed class DoesNotReturnIfAttribute : Attribute +{ + /// Initializes the attribute with the specified parameter value. + /// + /// The condition parameter value. Code after the method will be considered unreachable by diagnostics if the argument to + /// the associated parameter matches this value. + /// + public DoesNotReturnIfAttribute(bool parameterValue) => ParameterValue = parameterValue; + + /// Gets the condition parameter value. + public bool ParameterValue { get; } +} +#endif + +/// Specifies that the method or property will ensure that the listed field and property members have not-null values. +[AttributeUsage(AttributeTargets.Method | AttributeTargets.Property, Inherited = false, AllowMultiple = true)] +[ExcludeFromCodeCoverage] +internal sealed class MemberNotNullAttribute : Attribute +{ + /// Initializes the attribute with a field or property member. + /// + /// The field or property member that is promised to be not-null. + /// + public MemberNotNullAttribute(string member) => Members = new[] { member }; + + /// Initializes the attribute with the list of field and property members. + /// + /// The list of field and property members that are promised to be not-null. + /// + public MemberNotNullAttribute(params string[] members) => Members = members; + + /// Gets field or property member names. + public string[] Members { get; } +} + +/// Specifies that the method or property will ensure that the listed field and property members have not-null values when returning with the specified return value condition. +[AttributeUsage(AttributeTargets.Method | AttributeTargets.Property, Inherited = false, AllowMultiple = true)] +[ExcludeFromCodeCoverage] +internal sealed class MemberNotNullWhenAttribute : Attribute +{ + /// Initializes the attribute with the specified return value condition and a field or property member. + /// + /// The return value condition. If the method returns this value, the associated parameter will not be . + /// + /// + /// The field or property member that is promised to be not-null. + /// + public MemberNotNullWhenAttribute(bool returnValue, string member) + { + ReturnValue = returnValue; + Members = new[] { member }; + } + + /// Initializes the attribute with the specified return value condition and list of field and property members. + /// + /// The return value condition. If the method returns this value, the associated parameter will not be . + /// + /// + /// The list of field and property members that are promised to be not-null. + /// + public MemberNotNullWhenAttribute(bool returnValue, params string[] members) + { + ReturnValue = returnValue; + Members = members; + } + + /// Gets the return value condition. + public bool ReturnValue { get; } + + /// Gets field or property member names. + public string[] Members { get; } +} diff --git a/Shared/LegacySupport/IsExternalInit/IsExternalInit.cs b/Shared/LegacySupport/IsExternalInit/IsExternalInit.cs new file mode 100644 index 0000000..4e1b8ba --- /dev/null +++ b/Shared/LegacySupport/IsExternalInit/IsExternalInit.cs @@ -0,0 +1,13 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#pragma warning disable IDE0079 +#pragma warning disable S3903 + +/* This enables support for C# 9/10 records on older frameworks */ + +namespace System.Runtime.CompilerServices; + +internal static class IsExternalInit +{ +} diff --git a/Shared/LegacySupport/SystemIndex/Index.cs b/Shared/LegacySupport/SystemIndex/Index.cs new file mode 100644 index 0000000..7285d66 --- /dev/null +++ b/Shared/LegacySupport/SystemIndex/Index.cs @@ -0,0 +1,160 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; + +#pragma warning disable CS0436 // Type conflicts with imported type +#pragma warning disable S3427 // Method overloads with default parameter values should not overlap +#pragma warning disable SA1642 // Constructor summary documentation should begin with standard text +#pragma warning disable IDE0011 // Add braces +#pragma warning disable SA1623 // Property summary documentation should match accessors +#pragma warning disable IDE0023 // Use block body for conversion operator +#pragma warning disable S3928 // Parameter names used into ArgumentException constructors should match an existing one +#pragma warning disable LA0001 // Use the 'Microsoft.Shared.Diagnostics.Throws' class instead of explicitly throwing exception for improved performance +#pragma warning disable CA1305 // Specify IFormatProvider + +namespace System +{ + internal readonly struct Index : IEquatable + { + private readonly int _value; + + /// Construct an Index using a value and indicating if the index is from the start or from the end. + /// The index value. it has to be zero or positive number. + /// Indicating if the index is from the start or from the end. + /// + /// If the Index constructed from the end, index value 1 means pointing at the last element and index value 0 means pointing at beyond last element. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Index(int value, bool fromEnd = false) + { + if (value < 0) + { + ThrowValueArgumentOutOfRange_NeedNonNegNumException(); + } + + if (fromEnd) + _value = ~value; + else + _value = value; + } + + // The following private constructors mainly created for perf reason to avoid the checks + private Index(int value) + { + _value = value; + } + + /// Create an Index pointing at first element. + public static Index Start => new Index(0); + + /// Create an Index pointing at beyond last element. + public static Index End => new Index(~0); + + /// Create an Index from the start at the position indicated by the value. + /// The index value from the start. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Index FromStart(int value) + { + if (value < 0) + { + ThrowValueArgumentOutOfRange_NeedNonNegNumException(); + } + + return new Index(value); + } + + /// Create an Index from the end at the position indicated by the value. + /// The index value from the end. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Index FromEnd(int value) + { + if (value < 0) + { + ThrowValueArgumentOutOfRange_NeedNonNegNumException(); + } + + return new Index(~value); + } + + /// Returns the index value. + public int Value + { + get + { + if (_value < 0) + return ~_value; + else + return _value; + } + } + + /// Indicates whether the index is from the start or the end. + public bool IsFromEnd => _value < 0; + + /// Calculate the offset from the start using the giving collection length. + /// The length of the collection that the Index will be used with. length has to be a positive value. + /// + /// For performance reason, we don't validate the input length parameter and the returned offset value against negative values. + /// we don't validate either the returned offset is greater than the input length. + /// It is expected Index will be used with collections which always have non negative length/count. If the returned offset is negative and + /// then used to index a collection will get out of range exception which will be same affect as the validation. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public int GetOffset(int length) + { + int offset = _value; + if (IsFromEnd) + { + // offset = length - (~value) + // offset = length + (~(~value) + 1) + // offset = length + value + 1 + + offset += length + 1; + } + + return offset; + } + + /// Indicates whether the current Index object is equal to another object of the same type. + /// An object to compare with this object. + public override bool Equals([NotNullWhen(true)] object? value) => value is Index && _value == ((Index)value)._value; + + /// Indicates whether the current Index object is equal to another Index object. + /// An object to compare with this object. + public bool Equals(Index other) => _value == other._value; + + /// Returns the hash code for this instance. + public override int GetHashCode() => _value; + + /// Converts integer number to an Index. + public static implicit operator Index(int value) => FromStart(value); + + /// Converts the value of the current Index object to its equivalent string representation. + public override string ToString() + { + if (IsFromEnd) + return ToStringFromEnd(); + + return ((uint)Value).ToString(); + } + + private static void ThrowValueArgumentOutOfRange_NeedNonNegNumException() + { + throw new ArgumentOutOfRangeException("value", "value must be non-negative"); + } + + private string ToStringFromEnd() + { +#if (!NETSTANDARD2_0 && !NETFRAMEWORK) + Span span = stackalloc char[11]; // 1 for ^ and 10 for longest possible uint value + bool formatted = ((uint)Value).TryFormat(span.Slice(1), out int charsWritten); + span[0] = '^'; + return new string(span.Slice(0, charsWritten + 1)); +#else + return '^' + Value.ToString(); +#endif + } + } +} \ No newline at end of file diff --git a/Shared/LegacySupport/TrimAttributes/RequiresDynamicCodeAttribute.cs b/Shared/LegacySupport/TrimAttributes/RequiresDynamicCodeAttribute.cs new file mode 100644 index 0000000..072701f --- /dev/null +++ b/Shared/LegacySupport/TrimAttributes/RequiresDynamicCodeAttribute.cs @@ -0,0 +1,49 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#pragma warning disable IDE0079 +#pragma warning disable SA1101 +#pragma warning disable SA1116 +#pragma warning disable SA1117 +#pragma warning disable SA1512 +#pragma warning disable SA1623 +#pragma warning disable SA1642 +#pragma warning disable S3903 +#pragma warning disable S3996 + +namespace System.Diagnostics.CodeAnalysis; + +/// +/// Indicates that the specified method requires the ability to generate new code at runtime, +/// for example through . +/// +/// +/// This allows tools to understand which methods are unsafe to call when compiling ahead of time. +/// +[AttributeUsage(AttributeTargets.Method | AttributeTargets.Constructor | AttributeTargets.Class, Inherited = false)] +[ExcludeFromCodeCoverage] +internal sealed class RequiresDynamicCodeAttribute : Attribute +{ + /// + /// Initializes a new instance of the class + /// with the specified message. + /// + /// + /// A message that contains information about the usage of dynamic code. + /// + public RequiresDynamicCodeAttribute(string message) + { + Message = message; + } + + /// + /// Gets a message that contains information about the usage of dynamic code. + /// + public string Message { get; } + + /// + /// Gets or sets an optional URL that contains more information about the method, + /// why it requires dynamic code, and what options a consumer has to deal with it. + /// + public string? Url { get; set; } +} diff --git a/Shared/LegacySupport/TrimAttributes/RequiresUnreferencedCodeAttribute.cs b/Shared/LegacySupport/TrimAttributes/RequiresUnreferencedCodeAttribute.cs new file mode 100644 index 0000000..6ee4305 --- /dev/null +++ b/Shared/LegacySupport/TrimAttributes/RequiresUnreferencedCodeAttribute.cs @@ -0,0 +1,50 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#pragma warning disable IDE0079 +#pragma warning disable SA1101 +#pragma warning disable SA1116 +#pragma warning disable SA1117 +#pragma warning disable SA1512 +#pragma warning disable SA1623 +#pragma warning disable SA1642 +#pragma warning disable S3903 +#pragma warning disable S3996 + +namespace System.Diagnostics.CodeAnalysis; + +/// +/// /// Indicates that the specified method requires dynamic access to code that is not referenced +/// statically, for example through . +/// +/// +/// This allows tools to understand which methods are unsafe to call when removing unreferenced +/// code from an application. +/// +[AttributeUsage(AttributeTargets.Method | AttributeTargets.Constructor | AttributeTargets.Class, Inherited = false)] +[ExcludeFromCodeCoverage] +internal sealed class RequiresUnreferencedCodeAttribute : Attribute +{ + /// + /// Initializes a new instance of the class + /// with the specified message. + /// + /// + /// A message that contains information about the usage of unreferenced code. + /// + public RequiresUnreferencedCodeAttribute(string message) + { + Message = message; + } + + /// + /// Gets a message that contains information about the usage of unreferenced code. + /// + public string Message { get; } + + /// + /// Gets or sets an optional URL that contains more information about the method, + /// why it requires unreferenced code, and what options a consumer has to deal with it. + /// + public string? Url { get; set; } +} diff --git a/Shared/LegacySupport/UnreachableException/UnreachableException.cs b/Shared/LegacySupport/UnreachableException/UnreachableException.cs new file mode 100644 index 0000000..702dd43 --- /dev/null +++ b/Shared/LegacySupport/UnreachableException/UnreachableException.cs @@ -0,0 +1,48 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; + +#pragma warning disable CA1064 // Exceptions should be public +#pragma warning disable CA1812 // Internal class that is (sometimes) never instantiated. + +namespace System.Diagnostics; + +/// +/// Exception thrown when the program executes an instruction that was thought to be unreachable. +/// +[ExcludeFromCodeCoverage] +internal sealed class UnreachableException : Exception +{ + private const string MessageText = "The program executed an instruction that was thought to be unreachable."; + + /// + /// Initializes a new instance of the class with the default error message. + /// + public UnreachableException() + : base(MessageText) + { + } + + /// + /// Initializes a new instance of the + /// class with a specified error message. + /// + /// The error message that explains the reason for the exception. + public UnreachableException(string? message) + : base(message ?? MessageText) + { + } + + /// + /// Initializes a new instance of the + /// class with a specified error message and a reference to the inner exception that is the cause of + /// this exception. + /// + /// The error message that explains the reason for the exception. + /// The exception that is the cause of the current exception. + public UnreachableException(string? message, Exception? innerException) + : base(message ?? MessageText, innerException) + { + } +} diff --git a/Shared/Throw/Throw.cs b/Shared/Throw/Throw.cs new file mode 100644 index 0000000..9f3d6e8 --- /dev/null +++ b/Shared/Throw/Throw.cs @@ -0,0 +1,72 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; + +#pragma warning disable CA1716 +#pragma warning disable CS8777 // Parameter must have a non-null value when exiting +namespace Microsoft.Shared.Diagnostics; +#pragma warning restore CA1716 + +/// +/// Defines static methods used to throw exceptions. +/// +[ExcludeFromCodeCoverage] +internal static partial class Throw +{ + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [return: NotNull] + public static T IfNull([NotNull] T argument, [CallerArgumentExpression(nameof(argument))] string paramName = "") + { + if (argument is null) + { + ArgumentNullException(paramName); + } + + return argument; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [return: NotNull] + public static string IfNullOrWhitespace([NotNull] string? argument, [CallerArgumentExpression(nameof(argument))] string paramName = "") + { + if (string.IsNullOrWhiteSpace(argument)) + { + if (argument == null) + { + ArgumentNullException(paramName); + } + else + { + ArgumentException(paramName, "Argument is whitespace"); + } + } + + return argument!; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int IfLessThan(int argument, int min, [CallerArgumentExpression(nameof(argument))] string paramName = "") + { + if (argument < min) + { + ArgumentOutOfRangeException(paramName, argument, $"Argument less than minimum value {min}"); + } + + return argument; + } + + [DoesNotReturn] + public static void ArgumentNullException(string paramName) + => throw new ArgumentNullException(paramName); + + [DoesNotReturn] + public static void ArgumentOutOfRangeException(string paramName, object? actualValue, string? message) + => throw new ArgumentOutOfRangeException(paramName, actualValue, message); + + [DoesNotReturn] + public static void ArgumentException(string paramName, string? message) + => throw new ArgumentException(message, paramName); +} diff --git a/eng/LegacySupport.props b/eng/LegacySupport.props new file mode 100644 index 0000000..6c697a5 --- /dev/null +++ b/eng/LegacySupport.props @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/global.json b/global.json new file mode 100644 index 0000000..fd9feb1 --- /dev/null +++ b/global.json @@ -0,0 +1,7 @@ +{ + "sdk": { + "version": "10.0.202", + "rollForward": "latestMinor", + "allowPrerelease": true + } +}