From afd4dde59456e172a35508335c52a0034e4e56ee Mon Sep 17 00:00:00 2001 From: Bart Koelman <104792814+bart-vmware@users.noreply.github.com> Date: Tue, 21 Apr 2026 00:46:33 +0200 Subject: [PATCH] Gracefully handle when unable to fetch access token in Eureka and Config Server --- .../ConfigServerConfigurationProvider.cs | 43 +++++++++++++------ ...ServerConfigurationProviderTest.Loading.cs | 33 ++++++++++++++ .../ConfigServerConfigurationProviderTest.cs | 20 ++++++--- src/Discovery/src/Eureka/EurekaClient.cs | 35 +++++++++++---- .../Eureka.Test/Transport/EurekaClientTest.cs | 36 ++++++++++++++++ 5 files changed, 142 insertions(+), 25 deletions(-) diff --git a/src/Configuration/src/ConfigServer/ConfigServerConfigurationProvider.cs b/src/Configuration/src/ConfigServer/ConfigServerConfigurationProvider.cs index 2246ff3e45..9a24046abe 100644 --- a/src/Configuration/src/ConfigServer/ConfigServerConfigurationProvider.cs +++ b/src/Configuration/src/ConfigServer/ConfigServerConfigurationProvider.cs @@ -593,7 +593,7 @@ internal async Task ShutdownAsync(CancellationToken cancellationToken) /// /// The HttpRequestMessage built from the path. /// - internal async Task GetRequestMessageAsync(ConfigServerClientOptions optionsSnapshot, Uri requestUri, + internal async Task GetConfigServerRequestMessageAsync(ConfigServerClientOptions optionsSnapshot, Uri requestUri, CancellationToken cancellationToken) { var uriWithoutUserInfo = new Uri(requestUri.GetComponents(UriComponents.HttpRequestUrl, UriFormat.UriEscaped)); @@ -641,18 +641,34 @@ internal async Task GetRequestMessageAsync(ConfigServerClien foreach (Uri requestUri in requestUris) { - // Make Config Server URI from settings - Uri uri = BuildConfigServerUri(optionsSnapshot, requestUri, label); + try + { + // Make Config Server URI from settings + Uri uri = BuildConfigServerUri(optionsSnapshot, requestUri, label); + + LogTryingToConnect(uri.ToMaskedString()); + HttpRequestMessage request; + + try + { + // Get the request message (potentially fetches access token) + LogBuildingHttpRequest(); + request = await GetConfigServerRequestMessageAsync(optionsSnapshot, uri, cancellationToken); + } + catch (Exception exception) when (!exception.IsCancellation()) + { + if (!string.IsNullOrEmpty(optionsSnapshot.AccessTokenUri)) + { + var accessTokenUri = new Uri(optionsSnapshot.AccessTokenUri); + LogFailedToFetchAccessToken(exception, accessTokenUri.ToMaskedString()); - LogTryingToConnect(uri.ToMaskedString()); + continue; + } - // Get the request message - LogBuildingHttpRequest(); - HttpRequestMessage request = await GetRequestMessageAsync(optionsSnapshot, uri, cancellationToken); + throw; + } - // Invoke Config Server - try - { + // Invoke Config Server LogSendingHttpRequest(); using HttpResponseMessage response = await httpClient.SendAsync(request, cancellationToken); @@ -808,7 +824,7 @@ internal async Task RefreshVaultTokenAsync(ConfigServerClientOptions optionsSnap { using HttpClient httpClient = CreateHttpClient(optionsSnapshot); - Uri uri = GetVaultRenewUri(optionsSnapshot); + Uri uri = BuildVaultRenewUri(optionsSnapshot); HttpRequestMessage message = await GetVaultRenewRequestMessageAsync(optionsSnapshot, uri, cancellationToken); LogRenewingVaultToken(obscuredToken, optionsSnapshot.TokenTtl, uri.ToMaskedString()); @@ -825,7 +841,7 @@ internal async Task RefreshVaultTokenAsync(ConfigServerClientOptions optionsSnap } } - private static Uri GetVaultRenewUri(ConfigServerClientOptions optionsSnapshot) + private static Uri BuildVaultRenewUri(ConfigServerClientOptions optionsSnapshot) { string baseUri = optionsSnapshot.Uri!.Split(',')[0].Trim(); @@ -1022,6 +1038,9 @@ private void ShutdownTimers() [LoggerMessage(Level = LogLevel.Debug, Message = "Fetched access token from {AccessTokenUri}.")] private partial void LogAccessTokenFetched(string accessTokenUri); + [LoggerMessage(Level = LogLevel.Warning, Message = "Failed to fetch access token from '{AccessTokenUri}'.")] + private partial void LogFailedToFetchAccessToken(Exception exception, string accessTokenUri); + [LoggerMessage(Level = LogLevel.Trace, Message = "Entered {Method}.")] private partial void LogRemoteLoadEntered(string method); diff --git a/src/Configuration/test/ConfigServer.Test/ConfigServerConfigurationProviderTest.Loading.cs b/src/Configuration/test/ConfigServer.Test/ConfigServerConfigurationProviderTest.Loading.cs index 257fdb3821..806e869a61 100644 --- a/src/Configuration/test/ConfigServer.Test/ConfigServerConfigurationProviderTest.Loading.cs +++ b/src/Configuration/test/ConfigServer.Test/ConfigServerConfigurationProviderTest.Loading.cs @@ -8,6 +8,7 @@ using System.Text; using FluentAssertions.Extensions; using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using RichardSzalay.MockHttp; using Steeltoe.Common.TestResources; @@ -448,6 +449,38 @@ public void Load_MultipleConfigServers_SocketError_FallsBackToNextServer() value.Should().Be("value1"); } + [Fact] + public void Load_MultipleConfigServers_SocketErrorFromAccessTokenUri_LogsWarnings() + { + using var loggerProvider = new CapturingLoggerProvider((_, level) => level == LogLevel.Warning); + using var loggerFactory = new LoggerFactory([loggerProvider]); + + using var handler = new DelegateToMockHttpClientHandler(); + + handler.Mock.When(HttpMethod.Get, "http://auth-server.com") + .Throw(new HttpRequestException("Connection refused", new SocketException((int)SocketError.ConnectionRefused))); + + var options = new ConfigServerClientOptions + { + Name = "myName", + AccessTokenUri = "http://auth-server.com", + Uri = "http://config-server1:8888,http://config-server2:8888" + }; + + using var provider = new ConfigServerConfigurationProvider(options, null, null, () => handler, loggerFactory); + provider.Load(); + + IList logMessages = loggerProvider.GetAll(); + + logMessages.Should().BeEquivalentTo([ + $"WARN {typeof(ConfigServerConfigurationProvider)}: Failed to fetch access token from 'http://auth-server.com/'.", + $"WARN {typeof(ConfigServerConfigurationProvider)}: Failed to fetch access token from 'http://auth-server.com/'.", + $"WARN {typeof(ConfigServerConfigurationProvider)}: Failed fetching remote configuration from server(s)." + ], assertionOptions => assertionOptions.WithStrictOrdering()); + + provider.InnerData.Should().BeEmpty(); + } + [Fact] public void Load_IdenticalData_DoesNotTriggerReload() { diff --git a/src/Configuration/test/ConfigServer.Test/ConfigServerConfigurationProviderTest.cs b/src/Configuration/test/ConfigServer.Test/ConfigServerConfigurationProviderTest.cs index 5cccbdb791..7b8c950a39 100644 --- a/src/Configuration/test/ConfigServer.Test/ConfigServerConfigurationProviderTest.cs +++ b/src/Configuration/test/ConfigServer.Test/ConfigServerConfigurationProviderTest.cs @@ -152,7 +152,9 @@ public async Task GetRequestMessage_AddsBasicAuthIfUserNameAndPasswordInURL() provider.Load(); Uri requestUri = provider.BuildConfigServerUri(provider.ClientOptions, new Uri(options.Uri), null); - HttpRequestMessage request = await provider.GetRequestMessageAsync(provider.ClientOptions, requestUri, TestContext.Current.CancellationToken); + + HttpRequestMessage request = + await provider.GetConfigServerRequestMessageAsync(provider.ClientOptions, requestUri, TestContext.Current.CancellationToken); request.Method.Should().Be(HttpMethod.Get); request.RequestUri.Should().Be(requestUri); @@ -177,7 +179,9 @@ public async Task GetRequestMessage_AddsBasicAuthIfUserNameAndPasswordInSettings provider.Load(); Uri requestUri = provider.BuildConfigServerUri(provider.ClientOptions, new Uri(options.Uri), null); - HttpRequestMessage request = await provider.GetRequestMessageAsync(provider.ClientOptions, requestUri, TestContext.Current.CancellationToken); + + HttpRequestMessage request = + await provider.GetConfigServerRequestMessageAsync(provider.ClientOptions, requestUri, TestContext.Current.CancellationToken); request.Method.Should().Be(HttpMethod.Get); request.RequestUri.Should().Be(requestUri); @@ -202,7 +206,9 @@ public async Task GetRequestMessage_BasicAuthInSettingsOverridesUserNameAndPassw provider.Load(); Uri requestUri = provider.BuildConfigServerUri(provider.ClientOptions, new Uri(options.Uri), null); - HttpRequestMessage request = await provider.GetRequestMessageAsync(provider.ClientOptions, requestUri, TestContext.Current.CancellationToken); + + HttpRequestMessage request = + await provider.GetConfigServerRequestMessageAsync(provider.ClientOptions, requestUri, TestContext.Current.CancellationToken); request.Method.Should().Be(HttpMethod.Get); request.RequestUri.Should().Be(requestUri); @@ -225,7 +231,9 @@ public async Task GetRequestMessage_AddsVaultToken_IfNeeded() provider.Load(); Uri requestUri = provider.BuildConfigServerUri(provider.ClientOptions, new Uri(options.Uri!), null); - HttpRequestMessage request = await provider.GetRequestMessageAsync(provider.ClientOptions, requestUri, TestContext.Current.CancellationToken); + + HttpRequestMessage request = + await provider.GetConfigServerRequestMessageAsync(provider.ClientOptions, requestUri, TestContext.Current.CancellationToken); request.Method.Should().Be(HttpMethod.Get); request.RequestUri.Should().Be(requestUri); @@ -260,7 +268,9 @@ public async Task GetRequestMessage_AddsBearerToken_WhenAccessTokenUriIsSet() using var provider = new ConfigServerConfigurationProvider(options, null, null, () => handler, NullLoggerFactory.Instance); Uri requestUri = provider.BuildConfigServerUri(provider.ClientOptions, new Uri(options.Uri), null); - HttpRequestMessage request = await provider.GetRequestMessageAsync(provider.ClientOptions, requestUri, TestContext.Current.CancellationToken); + + HttpRequestMessage request = + await provider.GetConfigServerRequestMessageAsync(provider.ClientOptions, requestUri, TestContext.Current.CancellationToken); handler.Mock.VerifyNoOutstandingExpectation(); diff --git a/src/Discovery/src/Eureka/EurekaClient.cs b/src/Discovery/src/Eureka/EurekaClient.cs index ad4b97e420..59bc33dc9d 100644 --- a/src/Discovery/src/Eureka/EurekaClient.cs +++ b/src/Discovery/src/Eureka/EurekaClient.cs @@ -236,7 +236,24 @@ private async Task ExecuteRequestAsync(HttpMethod method, stri Uri requestUri = GetRequestUri(serviceUri, path, queryString); HttpContent? requestContent = requestBody != null ? new StringContent(requestBody, Encoding.UTF8, MediaType) : null; - HttpRequestMessage request = await GetRequestMessageAsync(method, requestUri, requestContent, cancellationToken); + HttpRequestMessage request; + + try + { + request = await GetRequestMessageAsync(clientOptions, method, requestUri, requestContent, cancellationToken); + } + catch (Exception exception) when (!exception.IsCancellation()) + { + if (!string.IsNullOrEmpty(clientOptions.AccessTokenUri)) + { + var accessTokenUri = new Uri(clientOptions.AccessTokenUri); + LogFailedToFetchAccessToken(exception, accessTokenUri.ToMaskedString(), attempt); + + continue; + } + + throw; + } if (!string.IsNullOrEmpty(requestBody)) { @@ -306,7 +323,8 @@ private static Uri GetRequestUri(Uri baseUri, string path, IDictionary GetRequestMessageAsync(HttpMethod method, Uri requestUri, HttpContent? content, CancellationToken cancellationToken) + private async Task GetRequestMessageAsync(EurekaClientOptions optionsSnapshot, HttpMethod method, Uri requestUri, HttpContent? content, + CancellationToken cancellationToken) { var uriWithoutUserInfo = new Uri(requestUri.GetComponents(UriComponents.HttpRequestUrl, UriFormat.UriEscaped)); var requestMessage = new HttpRequestMessage(method, uriWithoutUserInfo); @@ -320,15 +338,13 @@ private async Task GetRequestMessageAsync(HttpMethod method, } else { - EurekaClientOptions clientOptions = _optionsMonitor.CurrentValue; - - if (!string.IsNullOrEmpty(clientOptions.AccessTokenUri)) + if (!string.IsNullOrEmpty(optionsSnapshot.AccessTokenUri)) { using HttpClient httpClient = CreateHttpClient("AccessTokenForEureka", GetAccessTokenTimeout); - var accessTokenUri = new Uri(clientOptions.AccessTokenUri); + var accessTokenUri = new Uri(optionsSnapshot.AccessTokenUri); - string accessToken = await httpClient.GetAccessTokenAsync(accessTokenUri, clientOptions.ClientId, - clientOptions.ClientSecret, cancellationToken); + string accessToken = + await httpClient.GetAccessTokenAsync(accessTokenUri, optionsSnapshot.ClientId, optionsSnapshot.ClientSecret, cancellationToken); LogAccessTokenFetched(accessTokenUri.ToMaskedString()); requestMessage.Headers.Authorization = new AuthenticationHeaderValue("Bearer", accessToken); @@ -371,4 +387,7 @@ private async Task GetRequestMessageAsync(HttpMethod method, [LoggerMessage(Level = LogLevel.Debug, Message = "Fetched access token from '{AccessTokenUri}'.")] private partial void LogAccessTokenFetched(string accessTokenUri); + + [LoggerMessage(Level = LogLevel.Warning, Message = "Failed to fetch access token from '{AccessTokenUri}' in attempt {Attempt}.")] + private partial void LogFailedToFetchAccessToken(Exception exception, string accessTokenUri, int attempt); } diff --git a/src/Discovery/test/Eureka.Test/Transport/EurekaClientTest.cs b/src/Discovery/test/Eureka.Test/Transport/EurekaClientTest.cs index b69254588b..af880a4275 100644 --- a/src/Discovery/test/Eureka.Test/Transport/EurekaClientTest.cs +++ b/src/Discovery/test/Eureka.Test/Transport/EurekaClientTest.cs @@ -194,6 +194,42 @@ public async Task RegisterAsync_ThrowsOnUnreachableServer() $"WARN {typeof(EurekaClient)}: Failed to execute HTTP POST request to 'http://host-that-does-not-exist.net:9999/apps/FOOBAR' in attempt 1."); } + [Fact] + public async Task RegisterAsync_ThrowsOnUnreachableAccessTokenServer() + { + using var capturingLoggerProvider = new CapturingLoggerProvider(category => category.StartsWith("Steeltoe.", StringComparison.Ordinal)); + + var services = new ServiceCollection(); + services.AddLogging(options => options.SetMinimumLevel(LogLevel.Trace).AddProvider(capturingLoggerProvider)); + services.AddOptions().Configure(options => options.AccessTokenUri = "http://host-that-does-not-exist.net:9999/"); + services.AddSingleton(new TestHttpClientFactory()); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(TimeProvider.System); + + await using ServiceProvider serviceProvider = services.BuildServiceProvider(true); + var client = serviceProvider.GetRequiredService(); + + var instance = new InstanceInfo("some", "FOOBAR", "localhost", "127.0.0.1", new DataCenterInfo(), TimeProvider.System) + { + NonSecurePort = 8080, + IsNonSecurePortEnabled = true, + SecurePort = 9090, + IsSecurePortEnabled = false, + LastUpdatedTimeUtc = new DateTime(638_440_245_328_236_418, DateTimeKind.Utc), + LastDirtyTimeUtc = new DateTime(638_440_245_328_236_418, DateTimeKind.Utc) + }; + + Func asyncAction = async () => await client.RegisterAsync(instance, TestContext.Current.CancellationToken); + + await asyncAction.Should().ThrowExactlyAsync().WithMessage("Failed to execute request on all known Eureka servers."); + + IList logMessages = capturingLoggerProvider.GetAll(); + + logMessages.Should().BeEquivalentTo( + $"WARN {typeof(EurekaClient)}: Failed to fetch access token from 'http://host-that-does-not-exist.net:9999/' in attempt 1."); + } + [Fact] public async Task RegisterAsync_ThrowsOnErrorResponse() {