From 8819fcfccb6bee2f6e99d212aa537611b98a3a32 Mon Sep 17 00:00:00 2001 From: Andriy Svyryd Date: Mon, 29 Jun 2026 18:08:54 -0700 Subject: [PATCH 1/5] Skip more flakey in-memory tests --- .../GraphUpdates/ProxyGraphUpdatesInMemoryTest.cs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test/EFCore.InMemory.FunctionalTests/GraphUpdates/ProxyGraphUpdatesInMemoryTest.cs b/test/EFCore.InMemory.FunctionalTests/GraphUpdates/ProxyGraphUpdatesInMemoryTest.cs index a4c6b7c47f9..41ffe841932 100644 --- a/test/EFCore.InMemory.FunctionalTests/GraphUpdates/ProxyGraphUpdatesInMemoryTest.cs +++ b/test/EFCore.InMemory.FunctionalTests/GraphUpdates/ProxyGraphUpdatesInMemoryTest.cs @@ -32,6 +32,12 @@ public override Task Optional_one_to_one_are_orphaned( CascadeTiming deleteOrphansTiming) => Task.CompletedTask; + // Cascade delete. + public override Task Optional_one_to_one_with_alternate_key_are_orphaned( + CascadeTiming cascadeDeleteTiming, + CascadeTiming deleteOrphansTiming) + => Task.CompletedTask; + // Cascade delete. public override Task Optional_many_to_one_dependents_are_orphaned_in_store( CascadeTiming cascadeDeleteTiming, @@ -115,6 +121,14 @@ public override Task Required_many_to_one_dependents_with_alternate_key_are_casc public override Task Can_attach_full_optional_graph_of_duplicates() => Task.CompletedTask; + // Cascade delete. + public override Task Can_attach_full_required_AK_graph_of_duplicates() + => Task.CompletedTask; + + // Graph fixup ordering is non-deterministic on InMemory. + public override Task No_fixup_to_Deleted_entities() + => Task.CompletedTask; + // Cascade delete. public override Task Required_non_PK_one_to_one_with_alternate_key_are_cascade_deleted( CascadeTiming cascadeDeleteTiming, From 45a5cfd9fe31ec03cec63148d80d0db300061a74 Mon Sep 17 00:00:00 2001 From: Andriy Svyryd Date: Tue, 30 Jun 2026 12:11:59 -0700 Subject: [PATCH 2/5] Fix reporting in Helix --- eng/helix.proj | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/eng/helix.proj b/eng/helix.proj index d68ccd054e0..d2aefb5c074 100644 --- a/eng/helix.proj +++ b/eng/helix.proj @@ -125,15 +125,21 @@ such as [ConditionalClass]/[ConditionalAssembly] add `category=failing` to tests whose condition evaluates to false; this filter is what actually skips them at the runner. + The 'report-trx' argument is required so each Helix work item produces a TRX file that Arcade's + Helix reporter uploads to Azure DevOps. The generic MTPProject work-item command only passes + 'results-directory', and the Microsoft.Testing.Extensions.TrxReport extension is not + auto-activated, so without this no per-test results would be reported (only work-item-level + pass/fail). + This must run AFTER any per-queue ItemGroup that re-Includes MTPProject items. --> - $(XUnitV3Arguments) --filter-not-trait category=failing --ignore-exit-code 8 + --report-trx --filter-not-trait category=failing --ignore-exit-code 8 - 03:00:00 + 03:00:00 From 0259fad5322456b28d63a18487877a3cdbbc3174 Mon Sep 17 00:00:00 2001 From: Andriy Svyryd Date: Tue, 30 Jun 2026 13:10:46 -0700 Subject: [PATCH 3/5] Also skip Save_changed_optional_one_to_one_with_alternate_key_in_store --- azure-pipelines-internal-tests.yml | 2 +- azure-pipelines-public.yml | 2 +- .../ProxyGraphUpdatesInMemoryTest.cs | 25 ++++++++++++------- ...orthwindAggregateOperatorsQueryTestBase.cs | 6 +++-- .../BuiltInDataTypesSqlServerTest.cs | 4 +-- 5 files changed, 24 insertions(+), 15 deletions(-) diff --git a/azure-pipelines-internal-tests.yml b/azure-pipelines-internal-tests.yml index ae6a00acb79..bb21351c0c5 100644 --- a/azure-pipelines-internal-tests.yml +++ b/azure-pipelines-internal-tests.yml @@ -137,7 +137,7 @@ extends: Test__Cosmos__DefaultConnection: $(_CosmosConnectionUrl) Test__SqlServer__DefaultConnection: "Data Source=" displayName: Build - - script: .dotnet\dotnet test test\EFCore.ApiBaseline.Tests -configuration $(_BuildConfig) + - script: .dotnet\dotnet test test\EFCore.ApiBaseline.Tests -p:Configuration=$(_BuildConfig) displayName: Test API Baselines condition: succeeded() - powershell: | diff --git a/azure-pipelines-public.yml b/azure-pipelines-public.yml index ced0274d3b9..87a1cf8e229 100644 --- a/azure-pipelines-public.yml +++ b/azure-pipelines-public.yml @@ -81,7 +81,7 @@ stages: Test__Cosmos__DefaultConnection: $(_CosmosConnectionUrl) Test__SqlServer__DefaultConnection: "Data Source=" name: Build - - script: .dotnet\dotnet test test\EFCore.ApiBaseline.Tests -configuration $(_BuildConfig) + - script: .dotnet\dotnet test test\EFCore.ApiBaseline.Tests -p:Configuration=$(_BuildConfig) displayName: Test API Baselines condition: succeeded() - powershell: | diff --git a/test/EFCore.InMemory.FunctionalTests/GraphUpdates/ProxyGraphUpdatesInMemoryTest.cs b/test/EFCore.InMemory.FunctionalTests/GraphUpdates/ProxyGraphUpdatesInMemoryTest.cs index 41ffe841932..682504b6b49 100644 --- a/test/EFCore.InMemory.FunctionalTests/GraphUpdates/ProxyGraphUpdatesInMemoryTest.cs +++ b/test/EFCore.InMemory.FunctionalTests/GraphUpdates/ProxyGraphUpdatesInMemoryTest.cs @@ -10,12 +10,10 @@ public abstract class ProxyGraphUpdatesInMemoryTestBase(TFixture fixtu where TFixture : ProxyGraphUpdatesInMemoryTestBase.ProxyGraphUpdatesInMemoryFixtureBase, new() { // FK constraint checking. - [Fact] public override Task Optional_one_to_one_relationships_are_one_to_one() => Assert.ThrowsAnyAsync(() => base.Optional_one_to_one_relationships_are_one_to_one()); // FK constraint checking. - [Fact] public override Task Optional_one_to_one_with_AK_relationships_are_one_to_one() => Assert.ThrowsAnyAsync(() => base.Optional_one_to_one_with_AK_relationships_are_one_to_one()); @@ -100,12 +98,10 @@ public override Task Required_many_to_one_dependents_with_alternate_key_are_casc => Task.CompletedTask; // FK constraint checking. - [Fact] public override Task Required_one_to_one_relationships_are_one_to_one() => Assert.ThrowsAnyAsync(() => base.Required_one_to_one_relationships_are_one_to_one()); // FK constraint checking. - [Fact] public override Task Required_one_to_one_with_AK_relationships_are_one_to_one() => Assert.ThrowsAnyAsync(() => base.Required_one_to_one_with_AK_relationships_are_one_to_one()); @@ -121,7 +117,7 @@ public override Task Required_many_to_one_dependents_with_alternate_key_are_casc public override Task Can_attach_full_optional_graph_of_duplicates() => Task.CompletedTask; - // Cascade delete. + // Graph fixup ordering is non-deterministic on InMemory. public override Task Can_attach_full_required_AK_graph_of_duplicates() => Task.CompletedTask; @@ -141,6 +137,10 @@ public override Task Reparent_required_non_PK_one_to_one_with_alternate_key( bool useExistingRoot) => Task.CompletedTask; + // Cascade delete. + public override Task Save_changed_optional_one_to_one_with_alternate_key_in_store() + => Task.CompletedTask; + // Cascade delete. public override Task Sever_required_one_to_one(ChangeMechanism changeMechanism) => Task.CompletedTask; @@ -171,10 +171,17 @@ protected override async Task ExecuteWithStrategyInTransactionAsync( Func nestedTestOperation2 = null, Func nestedTestOperation3 = null) { - await base.ExecuteWithStrategyInTransactionAsync( - testOperation, nestedTestOperation1, nestedTestOperation2, nestedTestOperation3); - - await Fixture.ReseedAsync(); + // InMemory has no real transactions, so the shared store is mutated directly by each test and must be + // reseeded afterwards. + try + { + await base.ExecuteWithStrategyInTransactionAsync( + testOperation, nestedTestOperation1, nestedTestOperation2, nestedTestOperation3); + } + finally + { + await Fixture.ReseedAsync(); + } } public abstract class ProxyGraphUpdatesInMemoryFixtureBase : ProxyGraphUpdatesFixtureBase diff --git a/test/EFCore.Specification.Tests/Query/NorthwindAggregateOperatorsQueryTestBase.cs b/test/EFCore.Specification.Tests/Query/NorthwindAggregateOperatorsQueryTestBase.cs index 00fa3990317..bb5e873c2cb 100644 --- a/test/EFCore.Specification.Tests/Query/NorthwindAggregateOperatorsQueryTestBase.cs +++ b/test/EFCore.Specification.Tests/Query/NorthwindAggregateOperatorsQueryTestBase.cs @@ -249,14 +249,16 @@ public virtual Task Average_over_nested_subquery(bool async) => AssertAverage( async, ss => ss.Set().OrderBy(c => c.CustomerID).Take(3), - selector: c => (decimal)c.Orders.Average(double (o) => 5 + o.OrderDetails.Average(int (od) => od.ProductID))); + selector: c => (decimal)c.Orders.Average(double (o) => 5 + o.OrderDetails.Average(int (od) => od.ProductID)), + asserter: (e, a) => Assert.Equal(e, a, 10)); [Theory, MemberData(nameof(IsAsyncData))] public virtual Task Average_over_max_subquery(bool async) => AssertAverage( async, ss => ss.Set().OrderBy(c => c.CustomerID).Take(3), - selector: c => (decimal)c.Orders.Average(int (o) => 5 + o.OrderDetails.Max(int (od) => od.ProductID))); + selector: c => (decimal)c.Orders.Average(int (o) => 5 + o.OrderDetails.Max(int (od) => od.ProductID)), + asserter: (e, a) => Assert.Equal(e, a, 10)); [Theory, MemberData(nameof(IsAsyncData))] public virtual Task Average_on_float_column(bool async) diff --git a/test/EFCore.SqlServer.FunctionalTests/BuiltInDataTypesSqlServerTest.cs b/test/EFCore.SqlServer.FunctionalTests/BuiltInDataTypesSqlServerTest.cs index 090bdf59a5b..86820a42e56 100644 --- a/test/EFCore.SqlServer.FunctionalTests/BuiltInDataTypesSqlServerTest.cs +++ b/test/EFCore.SqlServer.FunctionalTests/BuiltInDataTypesSqlServerTest.cs @@ -3859,8 +3859,8 @@ public async Task Xml_value_round_trips(string value, string expected) SELECT COALESCE([x].[Content], @value) FROM [XmlTestDocument] AS [x] WHERE [x].[Id] = @id -""", - query.ToQueryString()); +""".ReplaceLineEndings(), + query.ToQueryString().ReplaceLineEndings()); var roundTripped = await query.SingleAsync(); Assert.Equal(expected, roundTripped); From 40afd09a0692b07d5271871577b221f6cc363ab7 Mon Sep 17 00:00:00 2001 From: Andriy Svyryd Date: Wed, 1 Jul 2026 11:06:49 -0700 Subject: [PATCH 4/5] Handle stale cache in CosmosClient Add a retry for unintentional distributed transactions --- .../TestUtilities/CosmosTestStore.cs | 77 +++++++++++++++++ .../TransactionTestBase.cs | 84 ++++++++++++++----- 2 files changed, 141 insertions(+), 20 deletions(-) diff --git a/test/EFCore.Cosmos.FunctionalTests/TestUtilities/CosmosTestStore.cs b/test/EFCore.Cosmos.FunctionalTests/TestUtilities/CosmosTestStore.cs index 6a5057aa888..0690d052cd1 100644 --- a/test/EFCore.Cosmos.FunctionalTests/TestUtilities/CosmosTestStore.cs +++ b/test/EFCore.Cosmos.FunctionalTests/TestUtilities/CosmosTestStore.cs @@ -317,6 +317,10 @@ public override Task CleanAsync(DbContext context, bool createTables = true) private async Task CleanAsyncImpl(DbContext context, bool createTables) { var created = await EnsureCreatedAsync(context).ConfigureAwait(false); + + // Containers are deleted and recreated below only when the database already existed. A freshly-created + // database has brand-new containers whose metadata cannot be stale. + var containersRecreated = !created; try { if (!created) @@ -334,12 +338,23 @@ private async Task CleanAsyncImpl(DbContext context, bool createTables) created = await context.Database.EnsureCreatedAsync().ConfigureAwait(false); if (!created) { + if (containersRecreated) + { + await RefreshContainerMetadataAsync(context).ConfigureAwait(false); + } + await SeedAsync(context).ConfigureAwait(false); } } else { await CreateContainersAsync(context).ConfigureAwait(false); + + if (containersRecreated) + { + await RefreshContainerMetadataAsync(context).ConfigureAwait(false); + } + await SeedAsync(context).ConfigureAwait(false); } } @@ -357,6 +372,68 @@ private async Task CleanAsyncImpl(DbContext context, bool createTables) } } + // Deleting and recreating a container gives it a new resource id (_rid), but the shared CosmosClient caches each + // container's _rid (and the partition key ranges keyed by it) by container name. The first operation on the + // recreated container - whether it is the reseed below or the next test's first query - can then fail with + // "NotFound (404) ... GetTargetPartitionKeyRanges ... failed due to stale cache", and the query pipeline does not + // transparently refresh and retry. Prime the client's caches here, right after recreating the containers and before + // any test touches them. Each attempt re-reads the container by name (refreshing the collection cache with the new + // _rid) and then runs a query - the exact path that otherwise fails. Because the recreate can take a moment to + // become consistent on the emulator gateway, retry with a short delay until a query succeeds. This is the single + // place that handles the stale-metadata problem for the emulator; it is fully best-effort and must never fail the + // clean, so all errors are ultimately swallowed. + private async Task RefreshContainerMetadataAsync(DbContext context) + { + if (CosmosTestEnvironment.UseTokenCredential) + { + return; + } + + const int maxAttempts = 10; + + try + { + var cosmosClient = context.Database.GetCosmosClient(); + var database = cosmosClient.GetDatabase(Name); + var model = context.GetService().Model; + var countQuery = new QueryDefinition("SELECT VALUE COUNT(1) FROM c"); + + foreach (var containerProperties in GetContainersToCreate(model)) + { + var container = database.GetContainer(containerProperties.Id); + + for (var attempt = 1; attempt <= maxAttempts; attempt++) + { + try + { + await container.ReadContainerAsync().ConfigureAwait(false); + + using var iterator = container.GetItemQueryIterator(countQuery); + while (iterator.HasMoreResults) + { + await iterator.ReadNextAsync().ConfigureAwait(false); + } + + break; + } + catch when (attempt < maxAttempts) + { + // The recreate has not become consistent yet; wait for the gateway to catch up and retry. + await Task.Delay(200).ConfigureAwait(false); + } + catch + { + // Best-effort priming; never let it fail the clean. + } + } + } + } + catch + { + // Never let cache priming fail the clean. + } + } + private async Task CreateContainersAsync(DbContext context) { var databaseAccount = await GetDBAccount().ConfigureAwait(false); diff --git a/test/EFCore.Relational.Specification.Tests/TransactionTestBase.cs b/test/EFCore.Relational.Specification.Tests/TransactionTestBase.cs index 184aa8c5f3e..acfd34823f5 100644 --- a/test/EFCore.Relational.Specification.Tests/TransactionTestBase.cs +++ b/test/EFCore.Relational.Specification.Tests/TransactionTestBase.cs @@ -336,41 +336,85 @@ public virtual async Task SaveChanges_uses_ambient_transaction_with_connectionSt return; } - DbConnection connection; - using (var context = CreateContextWithConnectionString()) + DbConnection connection = null; + + await RetryOnDistributedTransactionNotSupportedAsync(async () => { - using (TestUtilities.TestStore.CreateTransactionScope()) + using (var context = CreateContextWithConnectionString()) { - context.Database.AutoTransactionBehavior = autoTransactionBehavior; + using (TestUtilities.TestStore.CreateTransactionScope()) + { + context.Database.AutoTransactionBehavior = autoTransactionBehavior; - connection = context.Database.GetDbConnection(); - Assert.Equal(ConnectionState.Closed, connection.State); + connection = context.Database.GetDbConnection(); + Assert.Equal(ConnectionState.Closed, connection.State); - await context.AddAsync( - new TransactionCustomer { Id = 77, Name = "Bobble" }); + await context.AddAsync( + new TransactionCustomer { Id = 77, Name = "Bobble" }); - context.Entry(context.Set().OrderBy(c => c.Id).Last()).State = EntityState.Added; + context.Entry(context.Set().OrderBy(c => c.Id).Last()).State = EntityState.Added; - if (async) - { - await Assert.ThrowsAsync(() => context.SaveChangesAsync()); - } - else - { - Assert.Throws(() => context.SaveChanges()); - } + // SaveChanges is expected to fail with DbUpdateException (duplicate key). + try + { + if (async) + { + await context.SaveChangesAsync(); + } + else + { + context.SaveChanges(); + } + + Assert.Fail("Expected a DbUpdateException to be thrown."); + } + catch (PlatformNotSupportedException) + { + // When the ambient transaction is promoted to a distributed one and throw PlatformNotSupportedException let it + // propagate so the whole operation is retried. + throw; + } + catch (Exception exception) + { + Assert.IsType(exception); + } - Assert.Equal(ConnectionState.Closed, connection.State); + Assert.Equal(ConnectionState.Closed, connection.State); - context.Database.AutoTransactionBehavior = AutoTransactionBehavior.WhenNeeded; + context.Database.AutoTransactionBehavior = AutoTransactionBehavior.WhenNeeded; + } } - } + }); Assert.Equal(ConnectionState.Closed, connection.State); AssertStoreInitialState(); } + // Sometimes an ambient/enlisted transaction ends up spanning more than one physical connection due to concurrency issues + // and is promoted to a distributed transaction, which throws PlatformNotSupportedException. + // Retry the operation a few times (with exponential back-off) so a run that keeps to a single connection (and + // therefore reaches the behavior under test) is used. + private static async Task RetryOnDistributedTransactionNotSupportedAsync(Func test) + { + const int maxAttempts = 3; + var delay = TimeSpan.FromSeconds(1); + + for (var attempt = 1; ; attempt++) + { + try + { + await test(); + return; + } + catch (PlatformNotSupportedException) when (attempt < maxAttempts) + { + await Task.Delay(delay); + delay *= 2; + } + } + } + [Theory, InlineData(true), InlineData(false)] public virtual void SaveChanges_throws_for_suppressed_ambient_transactions(bool connectionString) { From 43e625688c8657a25dd70ca3680ccc520b8b7a9a Mon Sep 17 00:00:00 2001 From: Andriy Svyryd Date: Thu, 2 Jul 2026 09:02:06 -0700 Subject: [PATCH 5/5] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- eng/helix.proj | 5 +---- .../TestUtilities/CosmosTestStore.cs | 9 ++------- .../TransactionTestBase.cs | 5 ++--- 3 files changed, 5 insertions(+), 14 deletions(-) diff --git a/eng/helix.proj b/eng/helix.proj index d2aefb5c074..3dfca72e27d 100644 --- a/eng/helix.proj +++ b/eng/helix.proj @@ -126,10 +126,7 @@ condition evaluates to false; this filter is what actually skips them at the runner. The 'report-trx' argument is required so each Helix work item produces a TRX file that Arcade's - Helix reporter uploads to Azure DevOps. The generic MTPProject work-item command only passes - 'results-directory', and the Microsoft.Testing.Extensions.TrxReport extension is not - auto-activated, so without this no per-test results would be reported (only work-item-level - pass/fail). + Helix reporter uploads to Azure DevOps. This must run AFTER any per-queue ItemGroup that re-Includes MTPProject items. --> diff --git a/test/EFCore.Cosmos.FunctionalTests/TestUtilities/CosmosTestStore.cs b/test/EFCore.Cosmos.FunctionalTests/TestUtilities/CosmosTestStore.cs index 0690d052cd1..acb1374b4e7 100644 --- a/test/EFCore.Cosmos.FunctionalTests/TestUtilities/CosmosTestStore.cs +++ b/test/EFCore.Cosmos.FunctionalTests/TestUtilities/CosmosTestStore.cs @@ -380,15 +380,10 @@ private async Task CleanAsyncImpl(DbContext context, bool createTables) // any test touches them. Each attempt re-reads the container by name (refreshing the collection cache with the new // _rid) and then runs a query - the exact path that otherwise fails. Because the recreate can take a moment to // become consistent on the emulator gateway, retry with a short delay until a query succeeds. This is the single - // place that handles the stale-metadata problem for the emulator; it is fully best-effort and must never fail the - // clean, so all errors are ultimately swallowed. + // place that handles the stale-metadata problem; it is fully best-effort and must never fail the clean, so all + // errors are ultimately swallowed. private async Task RefreshContainerMetadataAsync(DbContext context) { - if (CosmosTestEnvironment.UseTokenCredential) - { - return; - } - const int maxAttempts = 10; try diff --git a/test/EFCore.Relational.Specification.Tests/TransactionTestBase.cs b/test/EFCore.Relational.Specification.Tests/TransactionTestBase.cs index acfd34823f5..b3886a52788 100644 --- a/test/EFCore.Relational.Specification.Tests/TransactionTestBase.cs +++ b/test/EFCore.Relational.Specification.Tests/TransactionTestBase.cs @@ -370,13 +370,12 @@ await context.AddAsync( } catch (PlatformNotSupportedException) { - // When the ambient transaction is promoted to a distributed one and throw PlatformNotSupportedException let it + // When the ambient transaction is promoted to a distributed one and throws PlatformNotSupportedException, let it // propagate so the whole operation is retried. throw; } - catch (Exception exception) + catch (DbUpdateException) { - Assert.IsType(exception); } Assert.Equal(ConnectionState.Closed, connection.State);