diff --git a/src/libraries/Microsoft.Extensions.Hosting/tests/UnitTests/HostTests.cs b/src/libraries/Microsoft.Extensions.Hosting/tests/UnitTests/HostTests.cs
index f624d9901a209f..c6000f8a2457a9 100644
--- a/src/libraries/Microsoft.Extensions.Hosting/tests/UnitTests/HostTests.cs
+++ b/src/libraries/Microsoft.Extensions.Hosting/tests/UnitTests/HostTests.cs
@@ -183,40 +183,49 @@ public async Task CreateDefaultBuilder_ConfigJsonDoesReload()
             var reloadFlagConfig = new Dictionary<string, string>() { { "hostbuilder:reloadConfigOnChange", "true" } };
             var appSettingsPath = Path.Combine(Path.GetTempPath(), "appsettings.json");
 
-            string SaveRandomConfig()
+            try
             {
-                var newMessage = $"Hello ASP.NET Core: {Guid.NewGuid():N}";
-                File.WriteAllText(appSettingsPath, $"{{ \"Hello\": \"{newMessage}\" }}");
-                return newMessage;
-            }
+                static string SaveRandomConfig(string appSettingsPath)
+                {
+                    var newMessage = $"Hello ASP.NET Core: {Guid.NewGuid():N}";
+                    File.WriteAllText(appSettingsPath, $"{{ \"Hello\": \"{newMessage}\" }}");
+                    return newMessage;
+                }
 
-            var dynamicConfigMessage1 = SaveRandomConfig();
+                var dynamicConfigMessage1 = SaveRandomConfig(appSettingsPath);
 
-            using var host = Host.CreateDefaultBuilder()
-                .UseContentRoot(Path.GetDirectoryName(appSettingsPath))
-                .ConfigureHostConfiguration(builder =>
-                {
-                    builder.AddInMemoryCollection(reloadFlagConfig);
-                })
-                .Build();
+                using var host = Host.CreateDefaultBuilder()
+                    .UseContentRoot(Path.GetDirectoryName(appSettingsPath))
+                    .ConfigureHostConfiguration(builder =>
+                    {
+                        builder.AddInMemoryCollection(reloadFlagConfig);
+                    })
+                    .Build();
 
-            var config = host.Services.GetRequiredService<IConfiguration>();
+                var config = host.Services.GetRequiredService<IConfiguration>();
+                Assert.Equal(dynamicConfigMessage1, config["Hello"]);
 
-            Assert.Equal(dynamicConfigMessage1, config["Hello"]);
+                var configReloadedCancelTokenSource = new CancellationTokenSource();
+                var configReloadedCancelToken = configReloadedCancelTokenSource.Token;
 
-            var dynamicConfigMessage2 = SaveRandomConfig();
+                config.GetReloadToken().RegisterChangeCallback(
+                    _ => configReloadedCancelTokenSource.Cancel(), null);
 
-            var configReloadedCancelTokenSource = new CancellationTokenSource();
-            var configReloadedCancelToken = configReloadedCancelTokenSource.Token;
+                // Only update the config after we've registered the change callback
+                var dynamicConfigMessage2 = SaveRandomConfig(appSettingsPath);
 
-            config.GetReloadToken().RegisterChangeCallback(o =>
+                // Wait for up to 1 minute, if config reloads at any time, cancel the wait.
+                await Task.WhenAny(Task.Delay(TimeSpan.FromMinutes(1), configReloadedCancelToken)); // Task.WhenAny ignores the task throwing on cancellation.
+                Assert.NotEqual(dynamicConfigMessage1, dynamicConfigMessage2); // Messages are different.
+                Assert.Equal(dynamicConfigMessage2, config["Hello"]); // Config DID reload from disk
+            }
+            finally
             {
-                configReloadedCancelTokenSource.Cancel();
-            }, null);
-            // Wait for up to 1 minute, if config reloads at any time, cancel the wait.
-            await Task.WhenAny(Task.Delay(TimeSpan.FromMinutes(1), configReloadedCancelToken)); // Task.WhenAny ignores the task throwing on cancellation.
-            Assert.NotEqual(dynamicConfigMessage1, dynamicConfigMessage2); // Messages are different.
-            Assert.Equal(dynamicConfigMessage2, config["Hello"]); // Config DID reload from disk
+                if (File.Exists(appSettingsPath))
+                {
+                    File.Delete(appSettingsPath);
+                }
+            }
         }
 
         [Fact]
@@ -226,19 +235,18 @@ public async Task CreateDefaultBuilder_SecretsDoesReload()
             var secretId = Assembly.GetExecutingAssembly().GetName().Name;
             var reloadFlagConfig = new Dictionary<string, string>() { { "hostbuilder:reloadConfigOnChange", "true" } };
             var secretPath = PathHelper.GetSecretsPathFromSecretsId(secretId);
-
             var secretFileInfo = new FileInfo(secretPath);
+
             Directory.CreateDirectory(secretFileInfo.Directory.FullName);
 
-            string SaveRandomSecret()
+            static string SaveRandomSecret(string secretPath)
             {
                 var newMessage = $"Hello ASP.NET Core: {Guid.NewGuid():N}";
                 File.WriteAllText(secretPath, $"{{ \"Hello\": \"{newMessage}\" }}");
                 return newMessage;
             }
 
-            string dynamicSecretMessage1 = SaveRandomSecret();
-
+            var dynamicSecretMessage1 = SaveRandomSecret(secretPath);
             var host = Host.CreateDefaultBuilder(new[] { "environment=Development", $"applicationName={secretId}" })
                 .ConfigureHostConfiguration(builder =>
                 {
@@ -247,18 +255,17 @@ string SaveRandomSecret()
                 .Build();
 
             var config = host.Services.GetRequiredService<IConfiguration>();
-
             Assert.Equal(dynamicSecretMessage1, config["Hello"]);
 
-            string dynamicSecretMessage2 = SaveRandomSecret();
-
-            var configReloadedCancelTokenSource = new CancellationTokenSource();
+            using CancellationTokenSource configReloadedCancelTokenSource = new();
             var configReloadedCancelToken = configReloadedCancelTokenSource.Token;
 
-            config.GetReloadToken().RegisterChangeCallback(o =>
-            {
-                configReloadedCancelTokenSource.Cancel();
-            }, null);
+            config.GetReloadToken().RegisterChangeCallback(
+                _ => configReloadedCancelTokenSource.Cancel(), null);
+
+            // Only update the secrets after we've registered the change callback
+            var dynamicSecretMessage2 = SaveRandomSecret(secretPath);
+
             // Wait for up to 1 minute, if config reloads at any time, cancel the wait.
             await Task.WhenAny(Task.Delay(TimeSpan.FromMinutes(1), configReloadedCancelToken)); // Task.WhenAny ignores the task throwing on cancellation.
             Assert.NotEqual(dynamicSecretMessage1, dynamicSecretMessage2); // Messages are different.