diff --git a/dotnet/src/InternalUtilities/connectors/Memory/MongoDB/MongoDynamicMapper.cs b/dotnet/src/InternalUtilities/connectors/Memory/MongoDB/MongoDynamicMapper.cs index 91f732856047..71931f75ed1d 100644 --- a/dotnet/src/InternalUtilities/connectors/Memory/MongoDB/MongoDynamicMapper.cs +++ b/dotnet/src/InternalUtilities/connectors/Memory/MongoDB/MongoDynamicMapper.cs @@ -30,13 +30,15 @@ public BsonDocument MapFromDataToStorageModel(Dictionary dataMo : keyValue switch { string s => s, + Guid g => BsonValue.Create(g), + ObjectId o => o, + long i => i, + int i => i, + null => throw new InvalidOperationException($"Key property '{model.KeyProperty.ModelName}' is null."), _ => throw new InvalidCastException($"Key property '{model.KeyProperty.ModelName}' must be a string.") }; - document[MongoConstants.MongoReservedKeyPropertyName] = (string)(dataModel[model.KeyProperty.ModelName] - ?? throw new InvalidOperationException($"Key property '{model.KeyProperty.ModelName}' is null.")); - foreach (var property in model.DataProperties) { if (dataModel.TryGetValue(property.ModelName, out var dataValue)) @@ -88,9 +90,22 @@ Embedding e switch (property) { case KeyPropertyModel keyProperty: - result[keyProperty.ModelName] = storageModel.TryGetValue(MongoConstants.MongoReservedKeyPropertyName, out var keyValue) - ? keyValue.AsString - : throw new InvalidOperationException("No key property was found in the record retrieved from storage."); + if (!storageModel.TryGetValue(MongoConstants.MongoReservedKeyPropertyName, out var keyValue)) + { + throw new InvalidOperationException("No key property was found in the record retrieved from storage."); + } + + result[keyProperty.ModelName] = keyProperty.Type switch + { + var t when t == typeof(string) => keyValue.AsString, + var t when t == typeof(Guid) => keyValue.AsGuid, + var t when t == typeof(ObjectId) => keyValue.AsObjectId, + var t when t == typeof(long) => keyValue.AsInt64, + var t when t == typeof(int) => keyValue.AsInt32, + + _ => throw new UnreachableException() + }; + continue; case DataPropertyModel dataProperty: diff --git a/dotnet/src/VectorData/AzureAISearch/AzureAISearchDynamicMapper.cs b/dotnet/src/VectorData/AzureAISearch/AzureAISearchDynamicMapper.cs index ddf1366a2fe7..bf7ec744f269 100644 --- a/dotnet/src/VectorData/AzureAISearch/AzureAISearchDynamicMapper.cs +++ b/dotnet/src/VectorData/AzureAISearch/AzureAISearchDynamicMapper.cs @@ -29,8 +29,10 @@ public JsonObject MapFromDataToStorageModel(Dictionary dataMode : keyValue switch { string s => s, + Guid g => g.ToString(), + null => throw new InvalidOperationException($"Key property '{model.KeyProperty.ModelName}' is null."), - _ => throw new InvalidCastException($"Key property '{model.KeyProperty.ModelName}' must be a string.") + _ => throw new InvalidCastException($"Key property '{model.KeyProperty.ModelName}' must be a string or Guid.") }; foreach (var dataProperty in model.DataProperties) @@ -97,9 +99,16 @@ public JsonObject MapFromDataToStorageModel(Dictionary dataMode switch (property) { case KeyPropertyModel keyProperty: - result[keyProperty.ModelName] = (string?)storageModel[keyProperty.StorageName] + var key = (string?)storageModel[keyProperty.StorageName] ?? throw new InvalidOperationException($"The key property '{keyProperty.StorageName}' is missing from the record retrieved from storage."); + result[keyProperty.ModelName] = keyProperty.Type switch + { + var t when t == typeof(string) => key, + var t when t == typeof(Guid) => Guid.Parse(key), + _ => throw new UnreachableException() + }; + continue; case DataPropertyModel dataProperty: diff --git a/dotnet/src/VectorData/CosmosNoSql/CosmosNoSqlCollection.cs b/dotnet/src/VectorData/CosmosNoSql/CosmosNoSqlCollection.cs index 3f5ba4767771..2a22b855c408 100644 --- a/dotnet/src/VectorData/CosmosNoSql/CosmosNoSqlCollection.cs +++ b/dotnet/src/VectorData/CosmosNoSql/CosmosNoSqlCollection.cs @@ -860,7 +860,9 @@ private static IEnumerable GetCompositeKeys(IEnumerable IEnumerable k => k.Select(key => key switch { string s => new CosmosNoSqlCompositeKey(recordKey: s, partitionKey: s), + Guid g when g.ToString() is var guidString => new CosmosNoSqlCompositeKey(recordKey: guidString, partitionKey: guidString), CosmosNoSqlCompositeKey ck => ck, + _ => throw new ArgumentException($"Invalid key type '{key.GetType().Name}'.") }), diff --git a/dotnet/src/VectorData/CosmosNoSql/CosmosNoSqlDynamicMapper.cs b/dotnet/src/VectorData/CosmosNoSql/CosmosNoSqlDynamicMapper.cs index f76b3f651c7f..8f84593a5a6b 100644 --- a/dotnet/src/VectorData/CosmosNoSql/CosmosNoSqlDynamicMapper.cs +++ b/dotnet/src/VectorData/CosmosNoSql/CosmosNoSqlDynamicMapper.cs @@ -29,6 +29,8 @@ public JsonObject MapFromDataToStorageModel(Dictionary dataMode : keyValue switch { string s => s, + Guid g => g.ToString(), + null => throw new InvalidOperationException($"Key property '{model.KeyProperty.ModelName}' is null."), _ => throw new InvalidCastException($"Key property '{model.KeyProperty.ModelName}' must be a string.") }; @@ -121,9 +123,16 @@ static bool TryGetReadOnlyMemory(object value, [NotNullWhen(true)] out ReadOn switch (property) { case KeyPropertyModel keyProperty: - result[keyProperty.ModelName] = storageModel.TryGetPropertyValue(CosmosNoSqlConstants.ReservedKeyPropertyName, out var keyValue) - ? keyValue?.GetValue() - : throw new InvalidOperationException("No key property was found in the record retrieved from storage."); + var key = (string?)storageModel[CosmosNoSqlConstants.ReservedKeyPropertyName] + ?? throw new InvalidOperationException($"The key property '{keyProperty.StorageName}' is missing from the record retrieved from storage."); + + result[keyProperty.ModelName] = keyProperty.Type switch + { + var t when t == typeof(string) => key, + var t when t == typeof(Guid) => Guid.Parse(key), + _ => throw new UnreachableException() + }; + continue; case DataPropertyModel dataProperty: diff --git a/dotnet/src/VectorData/Redis/RedisJsonDynamicMapper.cs b/dotnet/src/VectorData/Redis/RedisJsonDynamicMapper.cs index 58673ef03078..6f6a38bac619 100644 --- a/dotnet/src/VectorData/Redis/RedisJsonDynamicMapper.cs +++ b/dotnet/src/VectorData/Redis/RedisJsonDynamicMapper.cs @@ -86,7 +86,15 @@ internal class RedisJsonDynamicMapper(CollectionModel model, JsonSerializerOptio } } - return ((string)dataModel[model.KeyProperty.ModelName]!, jsonObject); + var storageKey = dataModel[model.KeyProperty.ModelName] switch + { + string s => s, + Guid g => g.ToString(), + + _ => throw new UnreachableException() + }; + + return (storageKey, jsonObject); } /// @@ -94,14 +102,23 @@ internal class RedisJsonDynamicMapper(CollectionModel model, JsonSerializerOptio { var dataModel = new Dictionary { - [model.KeyProperty.ModelName] = storageModel.Key, + [model.KeyProperty.ModelName] = model.KeyProperty.Type switch + { + Type t when t == typeof(string) => storageModel.Key, + Type t when t == typeof(Guid) => Guid.Parse(storageModel.Key), + + _ => throw new UnreachableException() + }, }; // The redis result can be either a single object or an array with a single object in the case where we are doing an MGET. + // If there's a single data property, we get a simple value (no object wrapper). var jsonObject = storageModel.Node switch { - JsonObject topLevelJsonObject => topLevelJsonObject, - JsonArray jsonArray and [JsonObject arrayEntryJsonObject] => arrayEntryJsonObject, + JsonValue v when model.DataProperties is [var singleDataProperty] => new JsonObject([new(singleDataProperty.StorageName, v)]), + JsonObject o => o, + JsonArray a and [JsonObject arrayEntryJsonObject] => arrayEntryJsonObject, + _ => throw new InvalidOperationException($"Invalid data format for document with key '{storageModel.Key}'"), }; diff --git a/dotnet/src/VectorData/Redis/RedisJsonDynamicModelBuilder.cs b/dotnet/src/VectorData/Redis/RedisJsonDynamicModelBuilder.cs index b27cc1f570fd..5edc3e68d1ce 100644 --- a/dotnet/src/VectorData/Redis/RedisJsonDynamicModelBuilder.cs +++ b/dotnet/src/VectorData/Redis/RedisJsonDynamicModelBuilder.cs @@ -19,9 +19,9 @@ internal class RedisJsonDynamicModelBuilder(CollectionModelBuildingOptions optio protected override bool IsKeyPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes) { - supportedTypes = "string"; + supportedTypes = "string, Guid"; - return type == typeof(string); + return type == typeof(string) || type == typeof(Guid); } protected override bool IsDataPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes) diff --git a/dotnet/test/VectorData/InMemory.ConformanceTests/TypeTests/InMemoryKeyTypeTests.cs b/dotnet/test/VectorData/InMemory.ConformanceTests/TypeTests/InMemoryKeyTypeTests.cs index 1f092d34c59e..2739ddd93d0d 100644 --- a/dotnet/test/VectorData/InMemory.ConformanceTests/TypeTests/InMemoryKeyTypeTests.cs +++ b/dotnet/test/VectorData/InMemory.ConformanceTests/TypeTests/InMemoryKeyTypeTests.cs @@ -22,9 +22,9 @@ public class InMemoryKeyTypeTests(InMemoryKeyTypeTests.Fixture fixture) [ConditionalFact] public virtual Task String() => this.Test("foo"); - protected override async Task Test(TKey mainValue) + protected override async Task Test(TKey keyValue) { - await base.Test(mainValue); + await base.Test(keyValue); // For InMemory, delete the collection, otherwise the next test that runs will fail because the collection // already exists but with the previous key type. diff --git a/dotnet/test/VectorData/Redis.ConformanceTests/TypeTests/RedisHashSetKeyTypeTests.cs b/dotnet/test/VectorData/Redis.ConformanceTests/TypeTests/RedisHashSetKeyTypeTests.cs index 3cd6cb1c2e63..3bc84e2377e2 100644 --- a/dotnet/test/VectorData/Redis.ConformanceTests/TypeTests/RedisHashSetKeyTypeTests.cs +++ b/dotnet/test/VectorData/Redis.ConformanceTests/TypeTests/RedisHashSetKeyTypeTests.cs @@ -25,5 +25,8 @@ public class RedisHashSetKeyTypeTests(RedisHashSetKeyTypeTests.Fixture fixture) // we seem to get key values from the previous collection despite having deleted and recreated it. So we uniquify the collection name instead. public override VectorStoreCollection> CreateCollection() => this.TestStore.DefaultVectorStore.GetCollection>(this.CollectionName + (++this._collectionCounter), this.CreateRecordDefinition()); + + public override VectorStoreCollection> CreateDynamicCollection() + => this.TestStore.DefaultVectorStore.GetDynamicCollection(this.CollectionName + (++this._collectionCounter), this.CreateRecordDefinition()); } } diff --git a/dotnet/test/VectorData/Redis.ConformanceTests/TypeTests/RedisJsonKeyTypeTests.cs b/dotnet/test/VectorData/Redis.ConformanceTests/TypeTests/RedisJsonKeyTypeTests.cs index ddcbbaa77de4..8fc5f0b2c18a 100644 --- a/dotnet/test/VectorData/Redis.ConformanceTests/TypeTests/RedisJsonKeyTypeTests.cs +++ b/dotnet/test/VectorData/Redis.ConformanceTests/TypeTests/RedisJsonKeyTypeTests.cs @@ -25,5 +25,8 @@ public class RedisJsonKeyTypeTests(RedisJsonKeyTypeTests.Fixture fixture) // we seem to get key values from the previous collection despite having deleted and recreated it. So we uniquify the collection name instead. public override VectorStoreCollection> CreateCollection() => this.TestStore.DefaultVectorStore.GetCollection>(this.CollectionName + (++this._collectionCounter), this.CreateRecordDefinition()); + + public override VectorStoreCollection> CreateDynamicCollection() + => this.TestStore.DefaultVectorStore.GetDynamicCollection(this.CollectionName + (++this._collectionCounter), this.CreateRecordDefinition()); } } diff --git a/dotnet/test/VectorData/VectorData.ConformanceTests/TypeTests/KeyTypeTests.cs b/dotnet/test/VectorData/VectorData.ConformanceTests/TypeTests/KeyTypeTests.cs index 4608bc067cd6..5fca8934d188 100644 --- a/dotnet/test/VectorData/VectorData.ConformanceTests/TypeTests/KeyTypeTests.cs +++ b/dotnet/test/VectorData/VectorData.ConformanceTests/TypeTests/KeyTypeTests.cs @@ -14,7 +14,7 @@ public abstract class KeyTypeTests(KeyTypeTests.Fixture fixture) [ConditionalFact] public virtual Task Guid() => this.Test(new Guid("603840bf-cf91-4521-8b8e-8b6a2e75910a")); - protected virtual async Task Test(TKey mainValue) + protected virtual async Task Test(TKey keyValue) where TKey : notnull { using var collection = fixture.CreateCollection(); @@ -24,7 +24,7 @@ protected virtual async Task Test(TKey mainValue) var record = new Record { - Key = mainValue, + Key = keyValue, Int = 8, Vector = new ReadOnlyMemory([1, 2, 3]) }; @@ -32,10 +32,37 @@ protected virtual async Task Test(TKey mainValue) await collection.UpsertAsync(record); await fixture.TestStore.WaitForDataAsync(collection, recordCount: 1); - var result = await collection.GetAsync(mainValue); + var result = await collection.GetAsync(keyValue); Assert.NotNull(result); + Assert.Equal(keyValue, result.Key); Assert.Equal(8, result.Int); + + /////////////////////// + // Test dynamic mapping + /////////////////////// + await collection.DeleteAsync(keyValue); + await fixture.TestStore.WaitForDataAsync(collection, recordCount: 0); + + using var dynamicCollection = fixture.CreateDynamicCollection(); + await dynamicCollection.EnsureCollectionExistsAsync(); + + var dynamicRecord = new Dictionary + { + [nameof(Record.Key)] = keyValue, + [nameof(Record.Int)] = 8, + [nameof(Record.Vector)] = new ReadOnlyMemory([1, 2, 3]) + }; + + await dynamicCollection.UpsertAsync(dynamicRecord); + await fixture.TestStore.WaitForDataAsync(dynamicCollection, recordCount: 1); + + var dynamicResult = await dynamicCollection.GetAsync(keyValue); + + Assert.NotNull(dynamicResult); + Assert.IsType(dynamicResult[nameof(Record.Key)]); + Assert.Equal(keyValue, (TKey)dynamicResult[nameof(Record.Key)]!); + Assert.Equal(8, dynamicResult[nameof(Record.Int)]); } public abstract class Fixture : VectorStoreFixture @@ -47,6 +74,10 @@ public virtual VectorStoreCollection> CreateCollection( where TKey : notnull => this.TestStore.DefaultVectorStore.GetCollection>(this.CollectionName, this.CreateRecordDefinition()); + public virtual VectorStoreCollection> CreateDynamicCollection() + where TKey : notnull + => this.TestStore.DefaultVectorStore.GetDynamicCollection(this.CollectionName, this.CreateRecordDefinition()); + public virtual VectorStoreCollectionDefinition CreateRecordDefinition() where TKey : notnull => new()