Gracefully handle access token fetch failure in Eureka and Config Server

src/Configuration/src/ConfigServer/ConfigServerConfigurationProvider.cs

@@ -593,7 +593,7 @@ internal async Task ShutdownAsync(CancellationToken cancellationToken)
     /// <returns>
     /// The HttpRequestMessage built from the path.
     /// </returns>
-    internal async Task<HttpRequestMessage> GetRequestMessageAsync(ConfigServerClientOptions optionsSnapshot, Uri requestUri,
+    internal async Task<HttpRequestMessage> GetConfigServerRequestMessageAsync(ConfigServerClientOptions optionsSnapshot, Uri requestUri,
         CancellationToken cancellationToken)
     {
         var uriWithoutUserInfo = new Uri(requestUri.GetComponents(UriComponents.HttpRequestUrl, UriFormat.UriEscaped));
@@ -641,18 +641,34 @@ internal async Task<HttpRequestMessage> GetRequestMessageAsync(ConfigServerClien
 
         foreach (Uri requestUri in requestUris)
         {
-            // Make Config Server URI from settings
-            Uri uri = BuildConfigServerUri(optionsSnapshot, requestUri, label);
+            try
+            {
+                // Make Config Server URI from settings
+                Uri uri = BuildConfigServerUri(optionsSnapshot, requestUri, label);
+
+                LogTryingToConnect(uri.ToMaskedString());
+                HttpRequestMessage request;
+
+                try
+                {
+                    // Get the request message (potentially fetches access token)
+                    LogBuildingHttpRequest();
+                    request = await GetConfigServerRequestMessageAsync(optionsSnapshot, uri, cancellationToken);
+                }
+                catch (Exception exception) when (!exception.IsCancellation())
+                {
+                    if (!string.IsNullOrEmpty(optionsSnapshot.AccessTokenUri))
+                    {
+                        var accessTokenUri = new Uri(optionsSnapshot.AccessTokenUri);
+                        LogFailedToFetchAccessToken(exception, accessTokenUri.ToMaskedString());
 
-            LogTryingToConnect(uri.ToMaskedString());
+                        continue;
+                    }
 
-            // Get the request message
-            LogBuildingHttpRequest();
-            HttpRequestMessage request = await GetRequestMessageAsync(optionsSnapshot, uri, cancellationToken);
+                    throw;
+                }
 
-            // Invoke Config Server
-            try
-            {
+                // Invoke Config Server
                 LogSendingHttpRequest();
                 using HttpResponseMessage response = await httpClient.SendAsync(request, cancellationToken);
 
@@ -808,7 +824,7 @@ internal async Task RefreshVaultTokenAsync(ConfigServerClientOptions optionsSnap
         {
             using HttpClient httpClient = CreateHttpClient(optionsSnapshot);
 
-            Uri uri = GetVaultRenewUri(optionsSnapshot);
+            Uri uri = BuildVaultRenewUri(optionsSnapshot);
             HttpRequestMessage message = await GetVaultRenewRequestMessageAsync(optionsSnapshot, uri, cancellationToken);
 
             LogRenewingVaultToken(obscuredToken, optionsSnapshot.TokenTtl, uri.ToMaskedString());
@@ -825,7 +841,7 @@ internal async Task RefreshVaultTokenAsync(ConfigServerClientOptions optionsSnap
         }
     }
 
-    private static Uri GetVaultRenewUri(ConfigServerClientOptions optionsSnapshot)
+    private static Uri BuildVaultRenewUri(ConfigServerClientOptions optionsSnapshot)
     {
         string baseUri = optionsSnapshot.Uri!.Split(',')[0].Trim();
 
@@ -1022,6 +1038,9 @@ private void ShutdownTimers()
     [LoggerMessage(Level = LogLevel.Debug, Message = "Fetched access token from {AccessTokenUri}.")]
     private partial void LogAccessTokenFetched(string accessTokenUri);
 
+    [LoggerMessage(Level = LogLevel.Warning, Message = "Failed to fetch access token from '{AccessTokenUri}'.")]
+    private partial void LogFailedToFetchAccessToken(Exception exception, string accessTokenUri);
+
     [LoggerMessage(Level = LogLevel.Trace, Message = "Entered {Method}.")]
     private partial void LogRemoteLoadEntered(string method);
 

src/Configuration/test/ConfigServer.Test/ConfigServerConfigurationProviderTest.Loading.cs

@@ -8,6 +8,7 @@
 using System.Text;
 using FluentAssertions.Extensions;
 using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Logging.Abstractions;
 using RichardSzalay.MockHttp;
 using Steeltoe.Common.TestResources;
@@ -448,6 +449,38 @@ public void Load_MultipleConfigServers_SocketError_FallsBackToNextServer()
         value.Should().Be("value1");
     }
 
+    [Fact]
+    public void Load_MultipleConfigServers_SocketErrorFromAccessTokenUri_LogsWarnings()
+    {
+        using var loggerProvider = new CapturingLoggerProvider((_, level) => level == LogLevel.Warning);
+        using var loggerFactory = new LoggerFactory([loggerProvider]);
+
+        using var handler = new DelegateToMockHttpClientHandler();
+
+        handler.Mock.When(HttpMethod.Get, "http://auth-server.com")
+            .Throw(new HttpRequestException("Connection refused", new SocketException((int)SocketError.ConnectionRefused)));
+
+        var options = new ConfigServerClientOptions
+        {
+            Name = "myName",
+            AccessTokenUri = "http://auth-server.com",
+            Uri = "http://config-server1:8888,http://config-server2:8888"
+        };
+
+        using var provider = new ConfigServerConfigurationProvider(options, null, null, () => handler, loggerFactory);
+        provider.Load();
+
+        IList<string> logMessages = loggerProvider.GetAll();
+
+        logMessages.Should().BeEquivalentTo([
+            $"WARN {typeof(ConfigServerConfigurationProvider)}: Failed to fetch access token from 'http://auth-server.com/'.",
+            $"WARN {typeof(ConfigServerConfigurationProvider)}: Failed to fetch access token from 'http://auth-server.com/'.",
+            $"WARN {typeof(ConfigServerConfigurationProvider)}: Failed fetching remote configuration from server(s)."
+        ], assertionOptions => assertionOptions.WithStrictOrdering());
+
+        provider.InnerData.Should().BeEmpty();
+    }
+
     [Fact]
     public void Load_IdenticalData_DoesNotTriggerReload()
     {

src/Configuration/test/ConfigServer.Test/ConfigServerConfigurationProviderTest.cs

@@ -152,7 +152,9 @@ public async Task GetRequestMessage_AddsBasicAuthIfUserNameAndPasswordInURL()
         provider.Load();
 
         Uri requestUri = provider.BuildConfigServerUri(provider.ClientOptions, new Uri(options.Uri), null);
-        HttpRequestMessage request = await provider.GetRequestMessageAsync(provider.ClientOptions, requestUri, TestContext.Current.CancellationToken);
+
+        HttpRequestMessage request =
+            await provider.GetConfigServerRequestMessageAsync(provider.ClientOptions, requestUri, TestContext.Current.CancellationToken);
 
         request.Method.Should().Be(HttpMethod.Get);
         request.RequestUri.Should().Be(requestUri);
@@ -177,7 +179,9 @@ public async Task GetRequestMessage_AddsBasicAuthIfUserNameAndPasswordInSettings
         provider.Load();
 
         Uri requestUri = provider.BuildConfigServerUri(provider.ClientOptions, new Uri(options.Uri), null);
-        HttpRequestMessage request = await provider.GetRequestMessageAsync(provider.ClientOptions, requestUri, TestContext.Current.CancellationToken);
+
+        HttpRequestMessage request =
+            await provider.GetConfigServerRequestMessageAsync(provider.ClientOptions, requestUri, TestContext.Current.CancellationToken);
 
         request.Method.Should().Be(HttpMethod.Get);
         request.RequestUri.Should().Be(requestUri);
@@ -202,7 +206,9 @@ public async Task GetRequestMessage_BasicAuthInSettingsOverridesUserNameAndPassw
         provider.Load();
 
         Uri requestUri = provider.BuildConfigServerUri(provider.ClientOptions, new Uri(options.Uri), null);
-        HttpRequestMessage request = await provider.GetRequestMessageAsync(provider.ClientOptions, requestUri, TestContext.Current.CancellationToken);
+
+        HttpRequestMessage request =
+            await provider.GetConfigServerRequestMessageAsync(provider.ClientOptions, requestUri, TestContext.Current.CancellationToken);
 
         request.Method.Should().Be(HttpMethod.Get);
         request.RequestUri.Should().Be(requestUri);
@@ -225,7 +231,9 @@ public async Task GetRequestMessage_AddsVaultToken_IfNeeded()
         provider.Load();
 
         Uri requestUri = provider.BuildConfigServerUri(provider.ClientOptions, new Uri(options.Uri!), null);
-        HttpRequestMessage request = await provider.GetRequestMessageAsync(provider.ClientOptions, requestUri, TestContext.Current.CancellationToken);
+
+        HttpRequestMessage request =
+            await provider.GetConfigServerRequestMessageAsync(provider.ClientOptions, requestUri, TestContext.Current.CancellationToken);
 
         request.Method.Should().Be(HttpMethod.Get);
         request.RequestUri.Should().Be(requestUri);
@@ -260,7 +268,9 @@ public async Task GetRequestMessage_AddsBearerToken_WhenAccessTokenUriIsSet()
         using var provider = new ConfigServerConfigurationProvider(options, null, null, () => handler, NullLoggerFactory.Instance);
 
         Uri requestUri = provider.BuildConfigServerUri(provider.ClientOptions, new Uri(options.Uri), null);
-        HttpRequestMessage request = await provider.GetRequestMessageAsync(provider.ClientOptions, requestUri, TestContext.Current.CancellationToken);
+
+        HttpRequestMessage request =
+            await provider.GetConfigServerRequestMessageAsync(provider.ClientOptions, requestUri, TestContext.Current.CancellationToken);
 
         handler.Mock.VerifyNoOutstandingExpectation();
 

src/Discovery/src/Eureka/EurekaClient.cs

@@ -236,7 +236,24 @@ private async Task<TResult> ExecuteRequestAsync<TResult>(HttpMethod method, stri
             Uri requestUri = GetRequestUri(serviceUri, path, queryString);
 
             HttpContent? requestContent = requestBody != null ? new StringContent(requestBody, Encoding.UTF8, MediaType) : null;
-            HttpRequestMessage request = await GetRequestMessageAsync(method, requestUri, requestContent, cancellationToken);
+            HttpRequestMessage request;
+
+            try
+            {
+                request = await GetRequestMessageAsync(clientOptions, method, requestUri, requestContent, cancellationToken);
+            }
+            catch (Exception exception) when (!exception.IsCancellation())
+            {
+                if (!string.IsNullOrEmpty(clientOptions.AccessTokenUri))
+                {
+                    var accessTokenUri = new Uri(clientOptions.AccessTokenUri);
+                    LogFailedToFetchAccessToken(exception, accessTokenUri.ToMaskedString(), attempt);
+
+                    continue;
+                }
+
+                throw;
+            }
 
             if (!string.IsNullOrEmpty(requestBody))
             {
@@ -306,7 +323,8 @@ private static Uri GetRequestUri(Uri baseUri, string path, IDictionary<string, s
         return requestUri;
     }
 
-    private async Task<HttpRequestMessage> GetRequestMessageAsync(HttpMethod method, Uri requestUri, HttpContent? content, CancellationToken cancellationToken)
+    private async Task<HttpRequestMessage> GetRequestMessageAsync(EurekaClientOptions optionsSnapshot, HttpMethod method, Uri requestUri, HttpContent? content,
+        CancellationToken cancellationToken)
     {
         var uriWithoutUserInfo = new Uri(requestUri.GetComponents(UriComponents.HttpRequestUrl, UriFormat.UriEscaped));
         var requestMessage = new HttpRequestMessage(method, uriWithoutUserInfo);
@@ -320,15 +338,13 @@ private async Task<HttpRequestMessage> GetRequestMessageAsync(HttpMethod method,
         }
         else
         {
-            EurekaClientOptions clientOptions = _optionsMonitor.CurrentValue;
-
-            if (!string.IsNullOrEmpty(clientOptions.AccessTokenUri))
+            if (!string.IsNullOrEmpty(optionsSnapshot.AccessTokenUri))
             {
                 using HttpClient httpClient = CreateHttpClient("AccessTokenForEureka", GetAccessTokenTimeout);
-                var accessTokenUri = new Uri(clientOptions.AccessTokenUri);
+                var accessTokenUri = new Uri(optionsSnapshot.AccessTokenUri);
 
-                string accessToken = await httpClient.GetAccessTokenAsync(accessTokenUri, clientOptions.ClientId,
-                    clientOptions.ClientSecret, cancellationToken);
+                string accessToken =
+                    await httpClient.GetAccessTokenAsync(accessTokenUri, optionsSnapshot.ClientId, optionsSnapshot.ClientSecret, cancellationToken);
 
                 LogAccessTokenFetched(accessTokenUri.ToMaskedString());
                 requestMessage.Headers.Authorization = new AuthenticationHeaderValue("Bearer", accessToken);
@@ -371,4 +387,7 @@ private async Task<HttpRequestMessage> GetRequestMessageAsync(HttpMethod method,
 
     [LoggerMessage(Level = LogLevel.Debug, Message = "Fetched access token from '{AccessTokenUri}'.")]
     private partial void LogAccessTokenFetched(string accessTokenUri);
+
+    [LoggerMessage(Level = LogLevel.Warning, Message = "Failed to fetch access token from '{AccessTokenUri}' in attempt {Attempt}.")]
+    private partial void LogFailedToFetchAccessToken(Exception exception, string accessTokenUri, int attempt);
 }

src/Discovery/test/Eureka.Test/Transport/EurekaClientTest.cs

@@ -194,6 +194,42 @@ public async Task RegisterAsync_ThrowsOnUnreachableServer()
             $"WARN {typeof(EurekaClient)}: Failed to execute HTTP POST request to 'http://host-that-does-not-exist.net:9999/apps/FOOBAR' in attempt 1.");
     }
 
+    [Fact]
+    public async Task RegisterAsync_ThrowsOnUnreachableAccessTokenServer()
+    {
+        using var capturingLoggerProvider = new CapturingLoggerProvider(category => category.StartsWith("Steeltoe.", StringComparison.Ordinal));
+
+        var services = new ServiceCollection();
+        services.AddLogging(options => options.SetMinimumLevel(LogLevel.Trace).AddProvider(capturingLoggerProvider));
+        services.AddOptions<EurekaClientOptions>().Configure(options => options.AccessTokenUri = "http://host-that-does-not-exist.net:9999/");
+        services.AddSingleton<IHttpClientFactory>(new TestHttpClientFactory());
+        services.AddSingleton<EurekaServiceUriStateManager>();
+        services.AddSingleton<EurekaClient>();
+        services.AddSingleton(TimeProvider.System);
+
+        await using ServiceProvider serviceProvider = services.BuildServiceProvider(true);
+        var client = serviceProvider.GetRequiredService<EurekaClient>();
+
+        var instance = new InstanceInfo("some", "FOOBAR", "localhost", "127.0.0.1", new DataCenterInfo(), TimeProvider.System)
+        {
+            NonSecurePort = 8080,
+            IsNonSecurePortEnabled = true,
+            SecurePort = 9090,
+            IsSecurePortEnabled = false,
+            LastUpdatedTimeUtc = new DateTime(638_440_245_328_236_418, DateTimeKind.Utc),
+            LastDirtyTimeUtc = new DateTime(638_440_245_328_236_418, DateTimeKind.Utc)
+        };
+
+        Func<Task> asyncAction = async () => await client.RegisterAsync(instance, TestContext.Current.CancellationToken);
+
+        await asyncAction.Should().ThrowExactlyAsync<EurekaTransportException>().WithMessage("Failed to execute request on all known Eureka servers.");
+
+        IList<string> logMessages = capturingLoggerProvider.GetAll();
+
+        logMessages.Should().BeEquivalentTo(
+            $"WARN {typeof(EurekaClient)}: Failed to fetch access token from 'http://host-that-does-not-exist.net:9999/' in attempt 1.");
+    }
+
     [Fact]
     public async Task RegisterAsync_ThrowsOnErrorResponse()
     {