From cf2b3c9b93b5119916d0ff56310f752ecb8a0940 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Thu, 7 May 2026 14:34:58 -0400 Subject: [PATCH 01/62] Extract HTTP/2 PING keepalive from PR #48420 Extracts the HTTP/2 PING health probing feature from PR #48420 (AzCosmos_HttpConnectionMaxLife) as a standalone changeset. Production code: - Http2PingHandler: sends PING frames on idle H2 parent channels - Configs: COSMOS.HTTP2_PING_HEALTH_ENABLED + PING_INTERVAL_IN_SECONDS - ReactorNettyClient: installs PingHandler via doOnConnected - IHttpClientInterceptor: test hook for DNS resolver + doOnConnected - HttpClientConfig/ConnectionPolicy/CosmosClientBuilder: interceptor plumbing Test code: - Http2PingKeepaliveTest: proves PINGs sent and ACKed on idle connections - Http2PingFrameCounterHandler: test utility for counting PING ACK frames - manual-http-network-fault TestNG suite + Maven profile Verified in Docker (--cap-add=NET_ADMIN): 5 PINGs sent, 10 ACKs received. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../CosmosHttpClientInterceptor.java | 38 ++++ .../interceptor/CosmosInterceptorHelper.java | 25 +++ sdk/cosmos/azure-cosmos-tests/pom.xml | 24 +++ .../Http2PingFrameCounterHandler.java | 37 ++++ .../Http2PingKeepaliveTest.java | 191 ++++++++++++++++++ .../com/azure/cosmos/rx/TestSuiteBase.java | 4 +- .../manual-http-network-fault-testng.xml | 16 ++ .../com/azure/cosmos/CosmosClientBuilder.java | 7 + .../azure/cosmos/implementation/Configs.java | 25 +++ .../implementation/ConnectionPolicy.java | 11 + .../ImplementationBridgeHelpers.java | 2 + .../implementation/RxDocumentClientImpl.java | 10 + .../implementation/http/Http2PingHandler.java | 163 +++++++++++++++ .../implementation/http/HttpClientConfig.java | 21 ++ .../http/ReactorNettyClient.java | 30 ++- .../interceptor/IHttpClientInterceptor.java | 30 +++ 16 files changed, 630 insertions(+), 4 deletions(-) create mode 100644 sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/implementation/interceptor/CosmosHttpClientInterceptor.java create mode 100644 sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingFrameCounterHandler.java create mode 100644 sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java create mode 100644 sdk/cosmos/azure-cosmos-tests/src/test/resources/manual-http-network-fault-testng.xml create mode 100644 sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java create mode 100644 sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/interceptor/IHttpClientInterceptor.java diff --git a/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/implementation/interceptor/CosmosHttpClientInterceptor.java b/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/implementation/interceptor/CosmosHttpClientInterceptor.java new file mode 100644 index 000000000000..65dbfaaee48f --- /dev/null +++ b/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/implementation/interceptor/CosmosHttpClientInterceptor.java @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos.test.implementation.interceptor; + +import com.azure.cosmos.implementation.interceptor.IHttpClientInterceptor; +import io.netty.resolver.AddressResolverGroup; +import reactor.netty.Connection; + +import java.util.function.Consumer; + +/** + * Test-side HTTP client interceptor for injecting custom DNS resolvers and + * connection handlers at client construction time. + */ +public class CosmosHttpClientInterceptor implements IHttpClientInterceptor { + + private final AddressResolverGroup addressResolverGroup; + private final Consumer doOnConnectedCallback; + + public CosmosHttpClientInterceptor( + AddressResolverGroup addressResolverGroup, + Consumer doOnConnectedCallback) { + + this.addressResolverGroup = addressResolverGroup; + this.doOnConnectedCallback = doOnConnectedCallback; + } + + @Override + public AddressResolverGroup getAddressResolverGroup() { + return this.addressResolverGroup; + } + + @Override + public Consumer getDoOnConnectedCallback() { + return this.doOnConnectedCallback; + } +} diff --git a/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/implementation/interceptor/CosmosInterceptorHelper.java b/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/implementation/interceptor/CosmosInterceptorHelper.java index 2a058ab1bf16..0099e8d3041d 100644 --- a/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/implementation/interceptor/CosmosInterceptorHelper.java +++ b/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/implementation/interceptor/CosmosInterceptorHelper.java @@ -4,11 +4,15 @@ package com.azure.cosmos.test.implementation.interceptor; import com.azure.cosmos.CosmosAsyncClient; +import com.azure.cosmos.CosmosClientBuilder; import com.azure.cosmos.implementation.ImplementationBridgeHelpers; import com.azure.cosmos.implementation.RxDocumentServiceRequest; import com.azure.cosmos.implementation.directconnectivity.StoreResponse; +import io.netty.resolver.AddressResolverGroup; +import reactor.netty.Connection; import java.util.function.BiFunction; +import java.util.function.Consumer; public class CosmosInterceptorHelper { public static void registerTransportClientInterceptor( @@ -21,4 +25,25 @@ public static void registerTransportClientInterceptor( .getCosmosAsyncClientAccessor() .registerTransportClientInterceptor(client, transportClientInterceptor); } + + /** + * Registers a custom DNS resolver and/or doOnConnected callback on the builder. + * Must be called before {@code builder.buildAsyncClient()}. + * + * @param builder the CosmosClientBuilder (pre-build) + * @param addressResolverGroup custom DNS resolver, or null for default + * @param doOnConnectedCallback custom connection callback, or null for none + */ + public static void registerHttpClientInterceptor( + CosmosClientBuilder builder, + AddressResolverGroup addressResolverGroup, + Consumer doOnConnectedCallback) { + + CosmosHttpClientInterceptor interceptor = new CosmosHttpClientInterceptor( + addressResolverGroup, doOnConnectedCallback); + ImplementationBridgeHelpers + .CosmosClientBuilderHelper + .getCosmosClientBuilderAccessor() + .setHttpClientInterceptor(builder, interceptor); + } } diff --git a/sdk/cosmos/azure-cosmos-tests/pom.xml b/sdk/cosmos/azure-cosmos-tests/pom.xml index 783cb5d8ebf3..ffa6b9ff64db 100644 --- a/sdk/cosmos/azure-cosmos-tests/pom.xml +++ b/sdk/cosmos/azure-cosmos-tests/pom.xml @@ -857,6 +857,30 @@ Licensed under the MIT License. + + manual-http-network-fault + + manual-http-network-fault + + + + + org.apache.maven.plugins + maven-failsafe-plugin + 3.5.3 + + + src/test/resources/manual-http-network-fault-testng.xml + + + true + true + + + + + + fi-thinclient-multi-region diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingFrameCounterHandler.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingFrameCounterHandler.java new file mode 100644 index 000000000000..02ccba8dd195 --- /dev/null +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingFrameCounterHandler.java @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos.faultinjection; + +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInboundHandlerAdapter; +import io.netty.handler.codec.http2.Http2PingFrame; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Test utility: counts HTTP/2 PING ACK frames received on the parent H2 channel. + * Incoming PING ACKs prove that PING frames were sent by reactor-netty and acknowledged by the server. + * Install on the parent channel (not stream channel) via doOnConnected. + */ +public class Http2PingFrameCounterHandler extends ChannelInboundHandlerAdapter { + private static final Logger logger = LoggerFactory.getLogger(Http2PingFrameCounterHandler.class); + private final AtomicInteger pingAckCount = new AtomicInteger(0); + + @Override + public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { + if (msg instanceof Http2PingFrame) { + Http2PingFrame pingFrame = (Http2PingFrame) msg; + if (pingFrame.ack()) { + int count = pingAckCount.incrementAndGet(); + logger.info("PING ACK #{} received on channel {}", count, ctx.channel().id().asShortText()); + } + } + super.channelRead(ctx, msg); + } + + public int getPingAckCount() { + return pingAckCount.get(); + } +} diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java new file mode 100644 index 000000000000..f60bb67b0db7 --- /dev/null +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java @@ -0,0 +1,191 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos.faultinjection; + +import com.azure.cosmos.CosmosAsyncClient; +import com.azure.cosmos.CosmosAsyncContainer; +import com.azure.cosmos.CosmosClientBuilder; +import com.azure.cosmos.CosmosDiagnostics; +import com.azure.cosmos.ConsistencyLevel; +import com.azure.cosmos.TestObject; +import com.azure.cosmos.implementation.TestConfigurations; +import com.azure.cosmos.implementation.http.Http2PingHandler; +import com.azure.cosmos.models.CosmosItemResponse; +import com.azure.cosmos.models.PartitionKey; +import io.netty.channel.Channel; +import io.netty.handler.codec.http2.Http2MultiplexHandler; +import io.netty.resolver.AddressResolverGroup; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testng.annotations.AfterClass; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import java.lang.reflect.Method; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicReference; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Standalone test for HTTP/2 PING keepalive handler. + * Proves that the Http2PingHandler sends PING frames on idle connections + * and that the connection survives the idle period. + * + * Run in Docker with --cap-add=NET_ADMIN (group: manual-http-network-fault). + */ +public class Http2PingKeepaliveTest extends FaultInjectionTestBase { + + private static final Logger logger = LoggerFactory.getLogger(Http2PingKeepaliveTest.class); + private static final long TEST_TIMEOUT = 120_000; // 2 minutes + + private CosmosAsyncClient client; + private CosmosAsyncContainer cosmosAsyncContainer; + private TestObject seedItem; + + public Http2PingKeepaliveTest() { + super(new CosmosClientBuilder() + .endpoint(TestConfigurations.HOST) + .key(TestConfigurations.MASTER_KEY) + .consistencyLevel(ConsistencyLevel.SESSION) + .gatewayMode()); + } + + @BeforeClass(groups = {"manual-http-network-fault"}, timeOut = 120_000) + public void beforeClass() { + // Enable HTTP/2 for this test + System.setProperty("COSMOS.HTTP2_ENABLED", "true"); + System.setProperty("COSMOS.THINCLIENT_ENABLED", "true"); + + this.client = getClientBuilder().buildAsyncClient(); + this.cosmosAsyncContainer = getSharedMultiPartitionCosmosContainerWithIdAsPartitionKey(this.client); + + // Seed item for reads + this.seedItem = TestObject.create(); + this.cosmosAsyncContainer.createItem(this.seedItem).block(); + logger.info("Seed item created: {}", this.seedItem.getId()); + } + + @AfterClass(groups = {"manual-http-network-fault"}, timeOut = 60_000) + public void afterClass() { + safeClose(this.client); + System.clearProperty("COSMOS.HTTP2_ENABLED"); + System.clearProperty("COSMOS.THINCLIENT_ENABLED"); + } + + @BeforeMethod(groups = {"manual-http-network-fault"}) + public void beforeMethod(Method method) { + logger.info("=== START: {} ===", method.getName()); + } + + @AfterMethod(groups = {"manual-http-network-fault"}) + public void afterMethod(Method method) { + logger.info("=== END: {} ===", method.getName()); + } + + @Test(groups = {"manual-http-network-fault"}, timeOut = TEST_TIMEOUT) + public void pingFramesSentAndAcknowledgedOnIdleConnection() throws Exception { + System.setProperty("COSMOS.HTTP2_PING_INTERVAL_IN_SECONDS", "3"); + System.setProperty("COSMOS.HTTP2_PING_HEALTH_ENABLED", "true"); + + AtomicReference pingHandlerRef = new AtomicReference<>(); + + try { + safeClose(this.client); + + CosmosClientBuilder builder = new CosmosClientBuilder() + .endpoint(TestConfigurations.HOST) + .key(TestConfigurations.MASTER_KEY) + .consistencyLevel(ConsistencyLevel.SESSION) + .gatewayMode(); + + // Inject a doOnConnected callback that installs a PING handler for testing + com.azure.cosmos.test.implementation.interceptor.CosmosInterceptorHelper + .registerHttpClientInterceptor(builder, (AddressResolverGroup) null, connection -> { + Channel ch = connection.channel(); + if (ch.pipeline().get(Http2MultiplexHandler.class) != null + && ch.pipeline().get("testPingHandler") == null) { + Http2PingHandler handler = new Http2PingHandler(3); + ch.pipeline().addLast("testPingHandler", handler); + pingHandlerRef.compareAndSet(null, handler); + logger.info("Test installed Http2PingHandler on H2 parent channel {}", ch.id().asShortText()); + } + }); + + this.client = builder.buildAsyncClient(); + this.cosmosAsyncContainer = getSharedMultiPartitionCosmosContainerWithIdAsPartitionKey(this.client); + + // Establish H2 connection with a warm-up read + String initialParentChannelId = readAndGetParentChannelId(); + logger.info("Initial parentChannelId: {}", initialParentChannelId); + + // Let the connection go idle — PINGs should fire every 3s + logger.info("Waiting 20s for PING frames to be sent on idle connection..."); + Thread.sleep(20_000); + + // Recovery read — proves connection is still alive + String recoveryParentChannelId = readAndGetParentChannelId(); + + Http2PingHandler handler = pingHandlerRef.get(); + int sentCount = handler != null ? handler.getPingsSent() : -1; + int ackCount = handler != null ? handler.getPingAcksReceived() : -1; + + logger.info("RESULT: initial={}, recovery={}, SAME_CONNECTION={}, pingsSent={}, pingAcksReceived={}", + initialParentChannelId, recoveryParentChannelId, + initialParentChannelId.equals(recoveryParentChannelId), sentCount, ackCount); + + assertThat(handler) + .as("Http2PingHandler should be installed on the parent H2 channel") + .isNotNull(); + + assertThat(sentCount) + .as("PINGs sent should be > 0 — proves the manual PING handler is actively sending frames") + .isGreaterThan(0); + + assertThat(ackCount) + .as("PING ACKs received should be > 0 — proves server acknowledged PINGs") + .isGreaterThan(0); + + // NOTE: We don't assert same connection because pool eviction behavior varies + // across account types (thin client vs standard). The core assertion is that + // PINGs are sent and ACKed — keepalive traffic is flowing. + logger.info("PING keepalive verified: {} PINGs sent, {} ACKs received", sentCount, ackCount); + } finally { + System.clearProperty("COSMOS.HTTP2_PING_INTERVAL_IN_SECONDS"); + System.clearProperty("COSMOS.HTTP2_PING_HEALTH_ENABLED"); + } + } + + private String readAndGetParentChannelId() { + CosmosItemResponse response = this.cosmosAsyncContainer.readItem( + seedItem.getId(), new PartitionKey(seedItem.getId()), TestObject.class).block(); + + assertThat(response).isNotNull(); + assertThat(response.getStatusCode()).isEqualTo(200); + + return extractParentChannelId(response.getDiagnostics()); + } + + private String extractParentChannelId(CosmosDiagnostics diagnostics) { + try { + String diagStr = diagnostics.toString(); + int idx = diagStr.indexOf("parentChannelId"); + if (idx > 0) { + int start = diagStr.indexOf("\"", idx + 16) + 1; + int end = diagStr.indexOf("\"", start); + if (start > 0 && end > start) { + return diagStr.substring(start, end); + } + } + + logger.warn("Could not extract parentChannelId from diagnostics"); + return "unknown-" + UUID.randomUUID().toString().substring(0, 8); + } catch (Exception e) { + logger.warn("Error extracting parentChannelId", e); + return "error-" + UUID.randomUUID().toString().substring(0, 8); + } + } +} diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java index 2b213a5c5683..13aab0d7e26f 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java @@ -302,7 +302,7 @@ public CosmosAsyncDatabase getDatabase(String id) { @BeforeSuite(groups = {"thinclient", "fast", "long", "direct", "multi-region", "multi-master", "flaky-multi-master", "emulator", "emulator-vnext", "split", "query", "cfp-split", "circuit-breaker-misc-gateway", "circuit-breaker-misc-direct", - "circuit-breaker-read-all-read-many", "fi-multi-master", "long-emulator", "fi-thinclient-multi-region", "fi-thinclient-multi-master", "multi-region-strong"}, timeOut = SUITE_SETUP_TIMEOUT) + "circuit-breaker-read-all-read-many", "fi-multi-master", "long-emulator", "fi-thinclient-multi-region", "fi-thinclient-multi-master", "multi-region-strong", "manual-http-network-fault"}, timeOut = SUITE_SETUP_TIMEOUT) public void beforeSuite() { logger.info("beforeSuite Started"); @@ -353,7 +353,7 @@ private static DocumentCollection getInternalDocumentCollection(CosmosAsyncConta @AfterSuite(groups = {"thinclient", "fast", "long", "direct", "multi-region", "multi-master", "flaky-multi-master", "emulator", "split", "query", "cfp-split", "circuit-breaker-misc-gateway", "circuit-breaker-misc-direct", - "circuit-breaker-read-all-read-many", "fi-multi-master", "long-emulator", "fi-thinclient-multi-region", "fi-thinclient-multi-master", "multi-region-strong"}, timeOut = SUITE_SHUTDOWN_TIMEOUT) + "circuit-breaker-read-all-read-many", "fi-multi-master", "long-emulator", "fi-thinclient-multi-region", "fi-thinclient-multi-master", "multi-region-strong", "manual-http-network-fault"}, timeOut = SUITE_SHUTDOWN_TIMEOUT) public void afterSuite() { logger.info("afterSuite Started"); diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/resources/manual-http-network-fault-testng.xml b/sdk/cosmos/azure-cosmos-tests/src/test/resources/manual-http-network-fault-testng.xml new file mode 100644 index 000000000000..b6127237e422 --- /dev/null +++ b/sdk/cosmos/azure-cosmos-tests/src/test/resources/manual-http-network-fault-testng.xml @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/CosmosClientBuilder.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/CosmosClientBuilder.java index 60fd80c6d141..a851003b5810 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/CosmosClientBuilder.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/CosmosClientBuilder.java @@ -159,6 +159,7 @@ public class CosmosClientBuilder implements private Function containerFactory = null; private Map additionalHeaders; + private com.azure.cosmos.implementation.interceptor.IHttpClientInterceptor httpClientInterceptor; /** * Instantiates a new Cosmos client builder. @@ -1364,6 +1365,7 @@ ConnectionPolicy buildConnectionPolicy() { this.connectionPolicy.setMultipleWriteRegionsEnabled(this.multipleWriteRegionsEnabled); this.connectionPolicy.setReadRequestsFallbackEnabled(this.readRequestsFallbackEnabled); this.connectionPolicy.setServerCertValidationDisabled(this.serverCertValidationDisabled); + this.connectionPolicy.setHttpClientInterceptor(this.httpClientInterceptor); return this.connectionPolicy; } @@ -1555,6 +1557,11 @@ public void setPerPartitionAutomaticFailoverEnabled(CosmosClientBuilder builder, public boolean getPerPartitionAutomaticFailoverEnabled(CosmosClientBuilder builder) { return builder.isPerPartitionAutomaticFailoverEnabled(); } + + @Override + public void setHttpClientInterceptor(CosmosClientBuilder builder, com.azure.cosmos.implementation.interceptor.IHttpClientInterceptor interceptor) { + builder.httpClientInterceptor = interceptor; + } }); } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java index 18eef0544e18..f21ec5569388 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java @@ -145,6 +145,17 @@ public class Configs { private static final String CONNECTION_ACQUIRE_TIMEOUT_IN_MS = "COSMOS.CONNECTION_ACQUIRE_TIMEOUT_IN_MS"; private static final String CONNECTION_ACQUIRE_TIMEOUT_IN_MS_VARIABLE = "COSMOS_CONNECTION_ACQUIRE_TIMEOUT_IN_MS"; private static final String REACTOR_NETTY_CONNECTION_POOL_NAME = "reactor-netty-connection-pool"; + + // HTTP/2 PING keepalive — keeps connections alive for sparse workloads by preventing + // intermediate infrastructure (NAT gateways, firewalls, load balancers) from silently + // reaping idle connections. PING is NOT used for eviction — degraded connections are + // handled by the response timeout retry path. + // Guarded by an explicit enable flag; default ON. Set COSMOS.HTTP2_PING_HEALTH_ENABLED=false to disable. + private static final boolean DEFAULT_HTTP2_PING_HEALTH_ENABLED = true; + private static final String HTTP2_PING_HEALTH_ENABLED = "COSMOS.HTTP2_PING_HEALTH_ENABLED"; + private static final int DEFAULT_HTTP2_PING_INTERVAL_IN_SECONDS = 10; + private static final String HTTP2_PING_INTERVAL_IN_SECONDS = "COSMOS.HTTP2_PING_INTERVAL_IN_SECONDS"; + private static final int DEFAULT_HTTP_RESPONSE_TIMEOUT_IN_SECONDS = 60; private static final int DEFAULT_QUERY_PLAN_RESPONSE_TIMEOUT_IN_SECONDS = 5; private static final int DEFAULT_ADDRESS_REFRESH_RESPONSE_TIMEOUT_IN_SECONDS = 5; @@ -740,6 +751,20 @@ public static int getDefaultHttpPoolSize() { return DEFAULT_HTTP_DEFAULT_CONNECTION_POOL_SIZE; } + public static boolean isHttp2PingHealthEnabled() { + String value = System.getProperty(HTTP2_PING_HEALTH_ENABLED); + if (value != null && !value.isEmpty()) { + return Boolean.parseBoolean(value); + } + return DEFAULT_HTTP2_PING_HEALTH_ENABLED; + } + + public static int getHttp2PingIntervalInSeconds() { + return getJVMConfigAsInt( + HTTP2_PING_INTERVAL_IN_SECONDS, + DEFAULT_HTTP2_PING_INTERVAL_IN_SECONDS); + } + public static Integer getPendingAcquireMaxCount() { String valueFromSystemProperty = System.getProperty(HTTP_PENDING_ACQUIRE_MAX_COUNT); if (valueFromSystemProperty != null && !valueFromSystemProperty.isEmpty()) { diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ConnectionPolicy.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ConnectionPolicy.java index b93171e3bdcd..6d98ab485de4 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ConnectionPolicy.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ConnectionPolicy.java @@ -11,6 +11,7 @@ import com.azure.cosmos.GatewayConnectionConfig; import com.azure.cosmos.Http2ConnectionConfig; import com.azure.cosmos.ThrottlingRetryOptions; +import com.azure.cosmos.implementation.interceptor.IHttpClientInterceptor; import java.time.Duration; import java.util.Collections; @@ -50,6 +51,7 @@ private static ImplementationBridgeHelpers.Http2ConnectionConfigHelper.Http2Conn private ProxyOptions proxy; private Duration idleHttpConnectionTimeout; private Http2ConnectionConfig http2ConnectionConfig; + private IHttpClientInterceptor httpClientInterceptor; // Direct connection config properties private Duration connectTimeout; @@ -665,6 +667,15 @@ public ConnectionPolicy setHttp2ConnectionConfig(Http2ConnectionConfig http2Conn return this; } + public IHttpClientInterceptor getHttpClientInterceptor() { + return this.httpClientInterceptor; + } + + public ConnectionPolicy setHttpClientInterceptor(IHttpClientInterceptor interceptor) { + this.httpClientInterceptor = interceptor; + return this; + } + @Override public String toString() { diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ImplementationBridgeHelpers.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ImplementationBridgeHelpers.java index c48555496c29..3767cf2175b6 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ImplementationBridgeHelpers.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ImplementationBridgeHelpers.java @@ -171,6 +171,8 @@ void setCosmosClientMetadataCachesSnapshot(CosmosClientBuilder builder, void setPerPartitionAutomaticFailoverEnabled(CosmosClientBuilder builder, boolean isPerPartitionAutomaticFailoverEnabled); boolean getPerPartitionAutomaticFailoverEnabled(CosmosClientBuilder builder); + + void setHttpClientInterceptor(CosmosClientBuilder builder, com.azure.cosmos.implementation.interceptor.IHttpClientInterceptor interceptor); } } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java index 59d15b94457e..3fb6f5d6016f 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java @@ -1076,6 +1076,16 @@ private HttpClient httpClient() { .withServerCertValidationDisabled(this.connectionPolicy.isServerCertValidationDisabled()) .withHttp2ConnectionConfig(this.connectionPolicy.getHttp2ConnectionConfig()); + com.azure.cosmos.implementation.interceptor.IHttpClientInterceptor interceptor = this.connectionPolicy.getHttpClientInterceptor(); + if (interceptor != null) { + if (interceptor.getAddressResolverGroup() != null) { + httpClientConfig.withAddressResolverGroup(interceptor.getAddressResolverGroup()); + } + if (interceptor.getDoOnConnectedCallback() != null) { + httpClientConfig.withDoOnConnectedCallback(interceptor.getDoOnConnectedCallback()); + } + } + if (connectionSharingAcrossClientsEnabled) { return SharedGatewayHttpClient.getOrCreateInstance(httpClientConfig, diagnosticsClientConfig); } else { diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java new file mode 100644 index 000000000000..1225eb84777d --- /dev/null +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java @@ -0,0 +1,163 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +package com.azure.cosmos.implementation.http; + +import com.azure.cosmos.implementation.Configs; +import io.netty.channel.Channel; +import io.netty.channel.ChannelDuplexHandler; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelPromise; +import io.netty.handler.codec.http2.DefaultHttp2PingFrame; +import io.netty.handler.codec.http2.Http2PingFrame; +import io.netty.util.AttributeKey; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Manual HTTP/2 PING keepalive handler installed on the parent (TCP) channel. + *

+ * Sends PING frames when the connection is idle for longer than the configured interval, + * preventing L7 middleboxes (NAT gateways, firewalls, load balancers) from silently + * reaping the connection. Modeled after Go SDK's {@code ReadIdleTimeout} approach. + *

+ * This handler does NOT close the connection on missed ACKs — connection eviction is + * handled by the eviction predicate in {@link HttpClient#createFixed}. The handler's + * sole purpose is keepalive traffic to reset middlebox idle timers. + *

+ * Why manual instead of reactor-netty native {@code pingAckTimeout}? Native PING requires + * built-in {@code maxIdleTime} handling, which is bypassed when a custom + * {@code evictionPredicate} is configured (reactor-netty 1.2.13). + */ +public class Http2PingHandler extends ChannelDuplexHandler { + + private static final Logger logger = LoggerFactory.getLogger(Http2PingHandler.class); + + private static final String HANDLER_NAME = "cosmos.http2PingHandler"; + + static final AttributeKey PING_HANDLER_INSTALLED = + AttributeKey.valueOf("cosmos.conn.pingHandlerInstalled"); + + private final long pingIntervalNanos; + private long lastActivityNanos; + private ScheduledFuture pingTask; + private final AtomicInteger pingsSent = new AtomicInteger(0); + private final AtomicInteger pingAcksReceived = new AtomicInteger(0); + + /** + * @param pingIntervalSeconds interval in seconds; when idle longer than this, a PING is sent + */ + public Http2PingHandler(int pingIntervalSeconds) { + this.pingIntervalNanos = TimeUnit.SECONDS.toNanos(pingIntervalSeconds); + this.lastActivityNanos = System.nanoTime(); + } + + @Override + public void handlerAdded(ChannelHandlerContext ctx) { + // Schedule periodic check — runs on the channel's event loop (single-threaded, no sync needed) + long checkIntervalMs = Math.max(1000, TimeUnit.NANOSECONDS.toMillis(pingIntervalNanos) / 2); + this.pingTask = ctx.executor().scheduleAtFixedRate( + () -> maybeSendPing(ctx), + checkIntervalMs, + checkIntervalMs, + TimeUnit.MILLISECONDS); + + if (logger.isDebugEnabled()) { + logger.debug("Http2PingHandler installed on channel {}, interval={}s, checkEvery={}ms", + ctx.channel().id().asShortText(), + TimeUnit.NANOSECONDS.toSeconds(pingIntervalNanos), + checkIntervalMs); + } + } + + @Override + public void handlerRemoved(ChannelHandlerContext ctx) { + cancelPingTask(); + } + + @Override + public void channelInactive(ChannelHandlerContext ctx) throws Exception { + cancelPingTask(); + super.channelInactive(ctx); + } + + @Override + public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { + lastActivityNanos = System.nanoTime(); + if (msg instanceof Http2PingFrame && ((Http2PingFrame) msg).ack()) { + pingAcksReceived.incrementAndGet(); + } + super.channelRead(ctx, msg); + } + + @Override + public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) throws Exception { + lastActivityNanos = System.nanoTime(); + super.write(ctx, msg, promise); + } + + private void maybeSendPing(ChannelHandlerContext ctx) { + if (!ctx.channel().isActive() || !Configs.isHttp2PingHealthEnabled()) { + cancelPingTask(); + return; + } + + long idleNanos = System.nanoTime() - lastActivityNanos; + if (idleNanos >= pingIntervalNanos) { + int count = pingsSent.incrementAndGet(); + ctx.writeAndFlush(new DefaultHttp2PingFrame(count)) + .addListener(f -> { + if (f.isSuccess()) { + if (logger.isDebugEnabled()) { + logger.debug("PING #{} sent on channel {}", count, ctx.channel().id().asShortText()); + } + } else { + logger.warn("PING #{} failed on channel {}: {}", + count, ctx.channel().id().asShortText(), + f.cause() != null ? f.cause().getMessage() : "unknown"); + } + }); + // Reset activity so we don't send another PING immediately + lastActivityNanos = System.nanoTime(); + } + } + + private void cancelPingTask() { + if (pingTask != null) { + pingTask.cancel(false); + pingTask = null; + } + } + + public int getPingsSent() { + return pingsSent.get(); + } + + public int getPingAcksReceived() { + return pingAcksReceived.get(); + } + + /** + * Installs the PING handler on the parent H2 channel if not already installed. + * Safe to call from doOnConnected (which fires per-stream for H2). + * + * @param channel the channel from doOnConnected (may be stream or parent) + * @param pingIntervalSeconds PING interval in seconds + */ + public static void installIfAbsent(Channel channel, int pingIntervalSeconds) { + // When called from the first doOnConnected, channel IS the parent H2 channel. + // When called from a stream doOnConnected, channel.parent() is the parent. + Channel parent = channel.parent() != null ? channel.parent() : channel; + if (!parent.hasAttr(PING_HANDLER_INSTALLED)) { + parent.attr(PING_HANDLER_INSTALLED).set(Boolean.TRUE); + try { + parent.pipeline().addLast(HANDLER_NAME, new Http2PingHandler(pingIntervalSeconds)); + } catch (IllegalArgumentException ignored) { + // Duplicate — race between concurrent streams, benign + } + } + } +} diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/HttpClientConfig.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/HttpClientConfig.java index 15527c4e2c89..239eb2c930a9 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/HttpClientConfig.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/HttpClientConfig.java @@ -7,6 +7,7 @@ import com.azure.cosmos.Http2ConnectionConfig; import com.azure.cosmos.implementation.Configs; import com.azure.cosmos.implementation.ImplementationBridgeHelpers; +import io.netty.resolver.AddressResolverGroup; import java.time.Duration; @@ -35,6 +36,8 @@ private static ImplementationBridgeHelpers.Http2ConnectionConfigHelper.Http2Conn private boolean connectionKeepAlive = true; private boolean serverCertValidationDisabled = false; private Http2ConnectionConfig http2ConnectionConfig; + private AddressResolverGroup addressResolverGroup; + private java.util.function.Consumer doOnConnectedCallback; // Eagerly resolved thin client connect timeout — avoids per-request System.getProperty/getenv calls. private final int thinClientConnectTimeoutMs; @@ -189,6 +192,24 @@ public int getThinClientConnectTimeoutMs() { return this.thinClientConnectTimeoutMs; } + public AddressResolverGroup getAddressResolverGroup() { + return this.addressResolverGroup; + } + + public HttpClientConfig withAddressResolverGroup(AddressResolverGroup resolverGroup) { + this.addressResolverGroup = resolverGroup; + return this; + } + + public java.util.function.Consumer getDoOnConnectedCallback() { + return this.doOnConnectedCallback; + } + + public HttpClientConfig withDoOnConnectedCallback(java.util.function.Consumer callback) { + this.doOnConnectedCallback = callback; + return this; + } + public String toDiagnosticsString() { String gwV2Cto = Configs.isThinClientEnabled() ? Duration.ofMillis(this.thinClientConnectTimeoutMs).toString() diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java index 2e362eadc084..04e6a3d52df9 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java @@ -11,6 +11,7 @@ import io.netty.channel.ChannelOption; import io.netty.channel.ChannelPipeline; import io.netty.handler.codec.http.HttpMethod; +import io.netty.handler.codec.http2.Http2MultiplexHandler; import io.netty.handler.logging.LogLevel; import io.netty.resolver.DefaultAddressResolverGroup; import io.netty.util.ReferenceCountUtil; @@ -140,8 +141,33 @@ private void configureChannelPipelineHandlers() { .maxChunkSize(this.httpClientConfig.getMaxChunkSize()) .validateHeaders(true)); + ImplementationBridgeHelpers.Http2ConnectionConfigHelper.Http2ConnectionConfigAccessor http2CfgAccessor = + ImplementationBridgeHelpers.Http2ConnectionConfigHelper.getHttp2ConnectionConfigAccessor(); Http2ConnectionConfig http2Cfg = httpClientConfig.getHttp2ConnectionConfig(); - if (httpCfgAccessor().isEffectivelyEnabled(http2Cfg)) { + + boolean isH2Enabled = http2CfgAccessor.isEffectivelyEnabled(http2Cfg); + this.httpClient = this.httpClient.doOnConnected(connection -> { + // Manual HTTP/2 PING keepalive — sends PING frames when the connection is idle + // to prevent L7 middleboxes (NAT, firewalls, LBs) from reaping the connection. + // For H2, the first doOnConnected fires for the parent TCP channel (parent()==null), + // and the second fires for stream channels (parent()!=null). + // We install on the parent channel — detect it via Http2MultiplexHandler in the pipeline. + if (Configs.isHttp2PingHealthEnabled() + && connection.channel().pipeline().get(Http2MultiplexHandler.class) != null) { + int pingIntervalSeconds = Configs.getHttp2PingIntervalInSeconds(); + if (pingIntervalSeconds > 0) { + Http2PingHandler.installIfAbsent(connection.channel(), pingIntervalSeconds); + } + } + + // Test hook: allows injection of custom handlers (e.g., PING frame counter) + java.util.function.Consumer doOnConnectedCb = ReactorNettyClient.this.httpClientConfig.getDoOnConnectedCallback(); + if (doOnConnectedCb != null) { + doOnConnectedCb.accept(connection); + } + }); + + if (isH2Enabled) { this.httpClient = this.httpClient .secure(sslContextSpec -> sslContextSpec.sslContext( @@ -153,7 +179,7 @@ private void configureChannelPipelineHandlers() { .http2Settings(settings -> settings .initialWindowSize(1024 * 1024) // 1MB initial window size .maxFrameSize(64 * 1024) // 64KB max frame size - .maxConcurrentStreams(httpCfgAccessor().getEffectiveMaxConcurrentStreams(http2Cfg)) // Increased from default 30 + .maxConcurrentStreams(http2CfgAccessor.getEffectiveMaxConcurrentStreams(http2Cfg)) // Increased from default 30 ) .doOnConnected((connection -> { // The response header clean up pipeline is being added due to an error getting when calling gateway: diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/interceptor/IHttpClientInterceptor.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/interceptor/IHttpClientInterceptor.java new file mode 100644 index 000000000000..fa915e2ea3ac --- /dev/null +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/interceptor/IHttpClientInterceptor.java @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos.implementation.interceptor; + +import io.netty.resolver.AddressResolverGroup; +import reactor.netty.Connection; + +import java.util.function.Consumer; + +/** + * Interceptor for HTTP client configuration at construction time. + *

+ * This interface allows test code (azure-cosmos-test) to inject custom DNS resolvers + * and connection handlers without polluting public API classes like ConnectionPolicy. + *

+ * Production code checks: if interceptor is null → normal flow, zero overhead. + */ +public interface IHttpClientInterceptor { + + /** + * Returns a custom AddressResolverGroup for DNS resolution, or null to use the default. + */ + AddressResolverGroup getAddressResolverGroup(); + + /** + * Returns a doOnConnected callback to install on the HTTP client, or null for none. + */ + Consumer getDoOnConnectedCallback(); +} From 60767fbbc45efb72ba386de5fdc8dc66f4451e6d Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Tue, 12 May 2026 15:47:41 -0400 Subject: [PATCH 02/62] Remove IHttpClientInterceptor plumbing, add PING concurrency guard + broken connection detection - Remove IHttpClientInterceptor interface, CosmosHttpClientInterceptor, and all plumbing through CosmosClientBuilder, ConnectionPolicy, ImplementationBridgeHelpers, RxDocumentClientImpl, HttpClientConfig, and ReactorNettyClient - Revert CosmosInterceptorHelper to upstream/main (remove registerHttpClientInterceptor) - Http2PingHandler: at-most-one outstanding PING (pingOutstandingSinceNanos) - Http2PingHandler: suppress PING when active streams exist (numActiveStreams > 0) - Http2PingHandler: mark connection unhealthy via PING_HEALTH_DEGRADED channel attribute when ACK not received within one interval; clear on late ACK - Http2PingHandler: add global counters + isConnectionHealthDegraded() static helper - Test uses auto-installed handler via global counters (no interceptor needed) - Add design spec: azure-cosmos/docs/http2-ping-keepalive-spec.md --- .../CosmosHttpClientInterceptor.java | 38 ---- .../interceptor/CosmosInterceptorHelper.java | 25 -- .../Http2PingKeepaliveTest.java | 31 +-- .../docs/http2-ping-keepalive-spec.md | 215 ++++++++++++++++++ .../com/azure/cosmos/CosmosClientBuilder.java | 7 - .../implementation/ConnectionPolicy.java | 11 - .../ImplementationBridgeHelpers.java | 2 - .../implementation/RxDocumentClientImpl.java | 10 - .../implementation/http/Http2PingHandler.java | 83 ++++++- .../implementation/http/HttpClientConfig.java | 21 -- .../http/ReactorNettyClient.java | 6 - .../interceptor/IHttpClientInterceptor.java | 30 --- 12 files changed, 299 insertions(+), 180 deletions(-) delete mode 100644 sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/implementation/interceptor/CosmosHttpClientInterceptor.java create mode 100644 sdk/cosmos/azure-cosmos/docs/http2-ping-keepalive-spec.md delete mode 100644 sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/interceptor/IHttpClientInterceptor.java diff --git a/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/implementation/interceptor/CosmosHttpClientInterceptor.java b/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/implementation/interceptor/CosmosHttpClientInterceptor.java deleted file mode 100644 index 65dbfaaee48f..000000000000 --- a/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/implementation/interceptor/CosmosHttpClientInterceptor.java +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.test.implementation.interceptor; - -import com.azure.cosmos.implementation.interceptor.IHttpClientInterceptor; -import io.netty.resolver.AddressResolverGroup; -import reactor.netty.Connection; - -import java.util.function.Consumer; - -/** - * Test-side HTTP client interceptor for injecting custom DNS resolvers and - * connection handlers at client construction time. - */ -public class CosmosHttpClientInterceptor implements IHttpClientInterceptor { - - private final AddressResolverGroup addressResolverGroup; - private final Consumer doOnConnectedCallback; - - public CosmosHttpClientInterceptor( - AddressResolverGroup addressResolverGroup, - Consumer doOnConnectedCallback) { - - this.addressResolverGroup = addressResolverGroup; - this.doOnConnectedCallback = doOnConnectedCallback; - } - - @Override - public AddressResolverGroup getAddressResolverGroup() { - return this.addressResolverGroup; - } - - @Override - public Consumer getDoOnConnectedCallback() { - return this.doOnConnectedCallback; - } -} diff --git a/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/implementation/interceptor/CosmosInterceptorHelper.java b/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/implementation/interceptor/CosmosInterceptorHelper.java index 0099e8d3041d..2a058ab1bf16 100644 --- a/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/implementation/interceptor/CosmosInterceptorHelper.java +++ b/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/implementation/interceptor/CosmosInterceptorHelper.java @@ -4,15 +4,11 @@ package com.azure.cosmos.test.implementation.interceptor; import com.azure.cosmos.CosmosAsyncClient; -import com.azure.cosmos.CosmosClientBuilder; import com.azure.cosmos.implementation.ImplementationBridgeHelpers; import com.azure.cosmos.implementation.RxDocumentServiceRequest; import com.azure.cosmos.implementation.directconnectivity.StoreResponse; -import io.netty.resolver.AddressResolverGroup; -import reactor.netty.Connection; import java.util.function.BiFunction; -import java.util.function.Consumer; public class CosmosInterceptorHelper { public static void registerTransportClientInterceptor( @@ -25,25 +21,4 @@ public static void registerTransportClientInterceptor( .getCosmosAsyncClientAccessor() .registerTransportClientInterceptor(client, transportClientInterceptor); } - - /** - * Registers a custom DNS resolver and/or doOnConnected callback on the builder. - * Must be called before {@code builder.buildAsyncClient()}. - * - * @param builder the CosmosClientBuilder (pre-build) - * @param addressResolverGroup custom DNS resolver, or null for default - * @param doOnConnectedCallback custom connection callback, or null for none - */ - public static void registerHttpClientInterceptor( - CosmosClientBuilder builder, - AddressResolverGroup addressResolverGroup, - Consumer doOnConnectedCallback) { - - CosmosHttpClientInterceptor interceptor = new CosmosHttpClientInterceptor( - addressResolverGroup, doOnConnectedCallback); - ImplementationBridgeHelpers - .CosmosClientBuilderHelper - .getCosmosClientBuilderAccessor() - .setHttpClientInterceptor(builder, interceptor); - } } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java index f60bb67b0db7..1df90d507115 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java @@ -13,9 +13,6 @@ import com.azure.cosmos.implementation.http.Http2PingHandler; import com.azure.cosmos.models.CosmosItemResponse; import com.azure.cosmos.models.PartitionKey; -import io.netty.channel.Channel; -import io.netty.handler.codec.http2.Http2MultiplexHandler; -import io.netty.resolver.AddressResolverGroup; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.annotations.AfterClass; @@ -26,7 +23,6 @@ import java.lang.reflect.Method; import java.util.UUID; -import java.util.concurrent.atomic.AtomicReference; import static org.assertj.core.api.Assertions.assertThat; @@ -91,8 +87,6 @@ public void pingFramesSentAndAcknowledgedOnIdleConnection() throws Exception { System.setProperty("COSMOS.HTTP2_PING_INTERVAL_IN_SECONDS", "3"); System.setProperty("COSMOS.HTTP2_PING_HEALTH_ENABLED", "true"); - AtomicReference pingHandlerRef = new AtomicReference<>(); - try { safeClose(this.client); @@ -102,23 +96,10 @@ public void pingFramesSentAndAcknowledgedOnIdleConnection() throws Exception { .consistencyLevel(ConsistencyLevel.SESSION) .gatewayMode(); - // Inject a doOnConnected callback that installs a PING handler for testing - com.azure.cosmos.test.implementation.interceptor.CosmosInterceptorHelper - .registerHttpClientInterceptor(builder, (AddressResolverGroup) null, connection -> { - Channel ch = connection.channel(); - if (ch.pipeline().get(Http2MultiplexHandler.class) != null - && ch.pipeline().get("testPingHandler") == null) { - Http2PingHandler handler = new Http2PingHandler(3); - ch.pipeline().addLast("testPingHandler", handler); - pingHandlerRef.compareAndSet(null, handler); - logger.info("Test installed Http2PingHandler on H2 parent channel {}", ch.id().asShortText()); - } - }); - this.client = builder.buildAsyncClient(); this.cosmosAsyncContainer = getSharedMultiPartitionCosmosContainerWithIdAsPartitionKey(this.client); - // Establish H2 connection with a warm-up read + // Establish H2 connection with a warm-up read — this triggers PING handler installation String initialParentChannelId = readAndGetParentChannelId(); logger.info("Initial parentChannelId: {}", initialParentChannelId); @@ -129,18 +110,14 @@ public void pingFramesSentAndAcknowledgedOnIdleConnection() throws Exception { // Recovery read — proves connection is still alive String recoveryParentChannelId = readAndGetParentChannelId(); - Http2PingHandler handler = pingHandlerRef.get(); - int sentCount = handler != null ? handler.getPingsSent() : -1; - int ackCount = handler != null ? handler.getPingAcksReceived() : -1; + // Get PING counters from the auto-installed handler + int sentCount = Http2PingHandler.getGlobalPingsSent(); + int ackCount = Http2PingHandler.getGlobalPingAcksReceived(); logger.info("RESULT: initial={}, recovery={}, SAME_CONNECTION={}, pingsSent={}, pingAcksReceived={}", initialParentChannelId, recoveryParentChannelId, initialParentChannelId.equals(recoveryParentChannelId), sentCount, ackCount); - assertThat(handler) - .as("Http2PingHandler should be installed on the parent H2 channel") - .isNotNull(); - assertThat(sentCount) .as("PINGs sent should be > 0 — proves the manual PING handler is actively sending frames") .isGreaterThan(0); diff --git a/sdk/cosmos/azure-cosmos/docs/http2-ping-keepalive-spec.md b/sdk/cosmos/azure-cosmos/docs/http2-ping-keepalive-spec.md new file mode 100644 index 000000000000..ee58e0e808a9 --- /dev/null +++ b/sdk/cosmos/azure-cosmos/docs/http2-ping-keepalive-spec.md @@ -0,0 +1,215 @@ +# HTTP/2 PING Keepalive — Design Spec + +**Status:** Draft +**PR:** [#49095](https://github.com/Azure/azure-sdk-for-java/pull/49095) +**Split from:** [#48420](https://github.com/Azure/azure-sdk-for-java/pull/48420) +**Date:** 2026-05-12 + +--- + +## 1. Problem Statement + +HTTP/2 connections between the Cosmos Java SDK and the Cosmos DB gateway traverse L7 middleboxes — NAT gateways, firewalls, Azure Load Balancers, corporate proxies. These middleboxes maintain connection tracking tables and silently reap idle connections after vendor-specific timeouts (often 4–5 minutes). When a reaped connection is reused, the client sees opaque TCP RST or timeout errors with no indication the connection was severed by a middlebox. + +The SDK's existing connection pool eviction (`evictionPredicate`) handles *local* idle detection but cannot prevent a middlebox from closing a connection that the SDK considers alive. + +## 2. Goal + +Send periodic HTTP/2 PING frames on idle parent (TCP) channels to keep middlebox connection tracking entries alive, preventing silent connection reaping. + +**Non-goals:** +- Closing connections directly on missed ACKs — the handler marks connections as unhealthy via a channel attribute; actual eviction is left to the connection pool. +- Replacing reactor-netty's native `maxIdleTime` — the SDK uses a custom `evictionPredicate` that bypasses reactor-netty's built-in idle handling. +- Application-layer keepalive (e.g., Cosmos heartbeat RPCs). + +## 3. Design + +### 3.1 Handler: `Http2PingHandler` + +A Netty `ChannelDuplexHandler` installed on the **parent (TCP) channel** of an HTTP/2 connection. It tracks activity on both inbound and outbound directions and sends a PING frame when the connection has been idle for longer than a configurable interval. + +#### State + +| Field | Type | Description | +|---|---|---| +| `pingIntervalNanos` | `long` | Configured idle threshold (immutable) | +| `lastActivityNanos` | `long` | `System.nanoTime()` of last read/write (event-loop-local, no sync) | +| `pingTask` | `ScheduledFuture` | Periodic check handle, cancelled on removal | +| `pingOutstandingSinceNanos` | `long` | `nanoTime()` when PING was sent; 0 = no outstanding PING | +| `pingsSent` | `AtomicInteger` | Monotonic counter — also used as PING frame payload | +| `pingAcksReceived` | `AtomicInteger` | Monotonic counter of received ACKs | +| `PING_HEALTH_DEGRADED` | `AttributeKey` | Per-channel flag: set when ACK missed, cleared on ACK | + +#### Lifecycle + +``` +handlerAdded() + └─ schedule periodic check (interval = max(1s, pingInterval/2)) + +channelRead(frame) + ├─ update lastActivityNanos + ├─ if frame is Http2PingFrame(ack=true): + │ pingAcksReceived++, pingOutstandingSinceNanos=0 + │ clear PING_HEALTH_DEGRADED attribute (connection proved responsive) + └─ propagate downstream + +write(frame) + ├─ update lastActivityNanos + └─ propagate upstream + +maybeSendPing() ← periodic task + ├─ if channel inactive → cancel + return + ├─ if feature disabled → cancel + return + ├─ if pingOutstandingSinceNanos != 0: + │ if waited ≥ pingInterval → set PING_HEALTH_DEGRADED on channel (WARN log) + │ return (at most 1 in-flight PING) + ├─ if numActiveStreams > 0 → reset lastActivityNanos + return + │ (active request traffic is keepalive; PING would be noise) + ├─ idleNanos = now - lastActivityNanos + ├─ if idleNanos < pingIntervalNanos → return + └─ set pingOutstandingSinceNanos=now, send DefaultHttp2PingFrame(++pingsSent) + └─ on failure: pingOutstandingSinceNanos=0 (unblock next attempt) + └─ reset lastActivityNanos + +handlerRemoved() / channelInactive() + └─ cancel pingTask +``` + +#### PING concurrency and noisy-neighbour prevention + +**At-most-one outstanding PING.** `pingOutstandingSinceNanos` records when the PING was sent (0 = none outstanding). If a PING is already in flight, `maybeSendPing()` returns immediately — this prevents unbounded queuing if ACKs are delayed. + +**Broken connection detection.** When the periodic check finds that `pingOutstandingSinceNanos` is non-zero and the elapsed wait exceeds `pingIntervalNanos`, it sets the `PING_HEALTH_DEGRADED` channel attribute to `true` and logs a WARN. The handler does **not** close the connection — external components (eviction predicates, health checks, or request-path code) can read `Http2PingHandler.isConnectionHealthDegraded(channel)` and decide how to react. When an ACK finally arrives, the flag is cleared and a DEBUG log confirms recovery. + +**Suppressed during active streams.** Before checking idle time, the handler queries `Http2FrameCodec.connection().numActiveStreams()`. If any streams are open (i.e., customer requests are in flight), the PING is skipped entirely — the request/response frames already keep the middlebox entry alive. The idle baseline is reset to `now` so that the full interval is measured from when all streams close, not from the last frame on a previous stream. + +Together these two guards ensure: +- At most 1 PING frame in flight at any time +- Zero PING frames while customer requests are active +- PING traffic only during true connection idleness + +#### Idle detection + +Any inbound frame (`channelRead`) or outbound write (`write`) resets the idle timer. The periodic check runs at `max(1 second, pingInterval / 2)` — this bounds the worst-case send delay to 1.5× the configured interval while avoiding excessive scheduling overhead. + +#### PING frame details + +- **Outbound:** `DefaultHttp2PingFrame(count)` where `count` = `pingsSent` counter (auto-increment). The 8-byte payload carries the sequence number for optional correlation. +- **Inbound ACK:** `Http2PingFrame` with `ack() == true`, counted and propagated. +- **No connection closure:** Missed ACKs are not tracked or acted upon. The handler is purely keepalive, not health-check. + +### 3.2 Installation + +Installed via `Http2PingHandler.installIfAbsent(Channel, int)`: + +``` +doOnConnected(connection) + ├─ Is HTTP/2? → check pipeline for Http2MultiplexHandler + ├─ Is enabled? → Configs.isHttp2PingHealthEnabled() + ├─ Interval > 0? → Configs.getHttp2PingIntervalInSeconds() + └─ installIfAbsent(channel, interval) + ├─ resolve parent channel (channel.parent() ?? channel) + ├─ idempotency check via PING_HANDLER_INSTALLED attribute + └─ parent.pipeline().addLast("cosmos.http2PingHandler", handler) +``` + +**Why parent channel?** HTTP/2 PING is a connection-level frame (RFC 9113 §6.7). Stream-level channels don't have access to the connection frame codec. The parent TCP channel's `Http2FrameCodec` encodes/decodes PING frames. + +**Idempotency:** `doOnConnected` fires once for the parent channel and once per stream. The `PING_HANDLER_INSTALLED` `AttributeKey` on the parent prevents duplicate installation. A race between concurrent stream channels is caught via `IllegalArgumentException` (benign). + +### 3.3 Integration point + +`ReactorNettyClient.java` → `httpClient.doOnConnected(...)` callback. The handler is installed after the HTTP/2 codec is in the pipeline but the detection is done via `Http2MultiplexHandler.class` presence in the pipeline (which is only present on H2 parent channels, not on HTTP/1.1 connections). + +### 3.4 Test hook + +`HttpClientConfig.doOnConnectedCallback` provides a `Consumer` hook that fires after the PING handler installation. Tests use this to: +1. Capture a reference to the installed `Http2PingHandler` instance +2. Install additional handlers (e.g., `Http2PingFrameCounterHandler`) +3. Assert on `getPingsSent()` / `getPingAcksReceived()` after an idle period + +## 4. Configuration + +| System Property | Default | Description | +|---|---|---| +| `COSMOS.HTTP2_PING_HEALTH_ENABLED` | `true` | Master switch. Set to `false` to disable PING keepalive. | +| `COSMOS.HTTP2_PING_INTERVAL_IN_SECONDS` | `10` | Idle threshold before sending a PING frame. | + +**No client builder API.** Configuration is JVM-wide via system properties, consistent with the existing `COSMOS.HTTP2_ENABLED` pattern. + +**Runtime disable:** The periodic task checks `Configs.isHttp2PingHealthEnabled()` on every tick. Setting the property to `false` at runtime causes the task to self-cancel on the next check. + +## 5. Threading Model + +- The handler runs entirely on Netty's channel event loop (single thread per channel). +- `lastActivityNanos` requires no synchronization — only accessed from the event loop thread. +- `pingsSent` / `pingAcksReceived` are `AtomicInteger` for safe cross-thread diagnostic reads (e.g., tests, metrics). +- `ScheduledFuture.cancel(false)` is non-interrupting — if the task is mid-execution, it completes and won't be rescheduled. + +## 6. Timing Characteristics + +| Parameter | Value (defaults) | +|---|---| +| PING interval | 10 seconds | +| Check frequency | 5 seconds (interval / 2) | +| Worst-case PING delay | ~15 seconds (idle just after a check) | +| Minimum check frequency | 1 second (clamped) | + +For a default-configured idle connection: +``` +t=0s last activity +t=5s check → idle 5s < 10s → skip +t=10s check → idle 10s ≥ 10s → PING sent, idle reset +t=15s check → idle 5s < 10s → skip +t=20s check → idle 10s ≥ 10s → PING sent +... +``` + +## 7. Failure Modes + +| Scenario | Behavior | +|---|---| +| PING ACK not received | `PING_HEALTH_DEGRADED` attribute set on channel after one interval. Connection NOT closed. `pingOutstandingSinceNanos` stays set, blocking further PINGs. | +| PING ACK arrives late | `PING_HEALTH_DEGRADED` cleared, `pingOutstandingSinceNanos` reset. Connection resumes normal keepalive. | +| Connection has active streams | PING suppressed — request traffic is keepalive. Idle baseline reset. | +| Channel becomes inactive | Periodic task cancelled in `channelInactive()`. | +| Feature disabled at runtime | Task self-cancels on next check. | +| Handler already installed (race) | `IllegalArgumentException` caught and ignored. | +| `ctx.writeAndFlush()` fails | WARN logged; idle timer reset to prevent tight retry loop. | +| HTTP/1.1 connection | Handler never installed (no `Http2MultiplexHandler` in pipeline). | + +## 8. Logging + +| Level | Event | +|---|---| +| DEBUG | Handler installed (interval, check frequency, channel ID) | +| DEBUG | PING frame sent successfully (sequence #, channel ID) | +| DEBUG | PING ACK received, degraded flag cleared (channel ID) | +| WARN | PING frame send failure (sequence #, channel ID, exception) | +| WARN | PING ACK not received within interval — connection marked unhealthy (channel ID) | + +## 9. Test Coverage + +**Test:** `Http2PingKeepaliveTest` (TestNG, group `manual-http-network-fault`) + +**Environment:** Docker with `--cap-add=NET_ADMIN`, real Cosmos DB account (thin-client-enabled). + +| Step | What's verified | +|---|---| +| 1. Create H2 client, seed data | Connection establishment | +| 2. Set interval to 3s, do a read | Handler installation on parent channel | +| 3. Sleep 20s (idle) | PINGs fire (~6-7 expected) | +| 4. Do another read | Connection survived idle period | +| 5. Assert `pingsSent > 0` | PING transmission works | +| 6. Assert `pingAcksReceived > 0` | Server acknowledges PINGs | + +**Helper:** `Http2PingFrameCounterHandler` — independent inbound handler that counts PING ACK frames for cross-validation. + +## 10. Future Considerations + +- **Health-check mode:** Track PING ACK latency and close connections that exceed a timeout. This would replace the current approach where `evictionPredicate` handles stale connections. +- **Metrics integration:** Expose `pingsSent` / `pingAcksReceived` counters in `CosmosDiagnostics` for observability. +- **Eviction predicate integration:** Wire `Http2PingHandler.isConnectionHealthDegraded()` into the connection pool's eviction predicate to automatically remove connections that fail PING health checks. +- **Missed-ACK escalation:** If `PING_HEALTH_DEGRADED` stays set across N intervals, escalate to connection close. +- **Client builder API:** Expose `http2PingIntervalInSeconds()` on `CosmosClientBuilder` for per-client configuration instead of JVM-wide system properties. +- **Adaptive interval:** Adjust PING interval based on observed middlebox behavior (e.g., back off if ACKs are always received). diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/CosmosClientBuilder.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/CosmosClientBuilder.java index a851003b5810..60fd80c6d141 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/CosmosClientBuilder.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/CosmosClientBuilder.java @@ -159,7 +159,6 @@ public class CosmosClientBuilder implements private Function containerFactory = null; private Map additionalHeaders; - private com.azure.cosmos.implementation.interceptor.IHttpClientInterceptor httpClientInterceptor; /** * Instantiates a new Cosmos client builder. @@ -1365,7 +1364,6 @@ ConnectionPolicy buildConnectionPolicy() { this.connectionPolicy.setMultipleWriteRegionsEnabled(this.multipleWriteRegionsEnabled); this.connectionPolicy.setReadRequestsFallbackEnabled(this.readRequestsFallbackEnabled); this.connectionPolicy.setServerCertValidationDisabled(this.serverCertValidationDisabled); - this.connectionPolicy.setHttpClientInterceptor(this.httpClientInterceptor); return this.connectionPolicy; } @@ -1557,11 +1555,6 @@ public void setPerPartitionAutomaticFailoverEnabled(CosmosClientBuilder builder, public boolean getPerPartitionAutomaticFailoverEnabled(CosmosClientBuilder builder) { return builder.isPerPartitionAutomaticFailoverEnabled(); } - - @Override - public void setHttpClientInterceptor(CosmosClientBuilder builder, com.azure.cosmos.implementation.interceptor.IHttpClientInterceptor interceptor) { - builder.httpClientInterceptor = interceptor; - } }); } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ConnectionPolicy.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ConnectionPolicy.java index 6d98ab485de4..b93171e3bdcd 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ConnectionPolicy.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ConnectionPolicy.java @@ -11,7 +11,6 @@ import com.azure.cosmos.GatewayConnectionConfig; import com.azure.cosmos.Http2ConnectionConfig; import com.azure.cosmos.ThrottlingRetryOptions; -import com.azure.cosmos.implementation.interceptor.IHttpClientInterceptor; import java.time.Duration; import java.util.Collections; @@ -51,7 +50,6 @@ private static ImplementationBridgeHelpers.Http2ConnectionConfigHelper.Http2Conn private ProxyOptions proxy; private Duration idleHttpConnectionTimeout; private Http2ConnectionConfig http2ConnectionConfig; - private IHttpClientInterceptor httpClientInterceptor; // Direct connection config properties private Duration connectTimeout; @@ -667,15 +665,6 @@ public ConnectionPolicy setHttp2ConnectionConfig(Http2ConnectionConfig http2Conn return this; } - public IHttpClientInterceptor getHttpClientInterceptor() { - return this.httpClientInterceptor; - } - - public ConnectionPolicy setHttpClientInterceptor(IHttpClientInterceptor interceptor) { - this.httpClientInterceptor = interceptor; - return this; - } - @Override public String toString() { diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ImplementationBridgeHelpers.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ImplementationBridgeHelpers.java index 3767cf2175b6..c48555496c29 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ImplementationBridgeHelpers.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ImplementationBridgeHelpers.java @@ -171,8 +171,6 @@ void setCosmosClientMetadataCachesSnapshot(CosmosClientBuilder builder, void setPerPartitionAutomaticFailoverEnabled(CosmosClientBuilder builder, boolean isPerPartitionAutomaticFailoverEnabled); boolean getPerPartitionAutomaticFailoverEnabled(CosmosClientBuilder builder); - - void setHttpClientInterceptor(CosmosClientBuilder builder, com.azure.cosmos.implementation.interceptor.IHttpClientInterceptor interceptor); } } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java index 3fb6f5d6016f..59d15b94457e 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java @@ -1076,16 +1076,6 @@ private HttpClient httpClient() { .withServerCertValidationDisabled(this.connectionPolicy.isServerCertValidationDisabled()) .withHttp2ConnectionConfig(this.connectionPolicy.getHttp2ConnectionConfig()); - com.azure.cosmos.implementation.interceptor.IHttpClientInterceptor interceptor = this.connectionPolicy.getHttpClientInterceptor(); - if (interceptor != null) { - if (interceptor.getAddressResolverGroup() != null) { - httpClientConfig.withAddressResolverGroup(interceptor.getAddressResolverGroup()); - } - if (interceptor.getDoOnConnectedCallback() != null) { - httpClientConfig.withDoOnConnectedCallback(interceptor.getDoOnConnectedCallback()); - } - } - if (connectionSharingAcrossClientsEnabled) { return SharedGatewayHttpClient.getOrCreateInstance(httpClientConfig, diagnosticsClientConfig); } else { diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java index 1225eb84777d..fe6b2f6a3c34 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java @@ -8,6 +8,7 @@ import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelPromise; import io.netty.handler.codec.http2.DefaultHttp2PingFrame; +import io.netty.handler.codec.http2.Http2FrameCodec; import io.netty.handler.codec.http2.Http2PingFrame; import io.netty.util.AttributeKey; import org.slf4j.Logger; @@ -24,9 +25,10 @@ * preventing L7 middleboxes (NAT gateways, firewalls, load balancers) from silently * reaping the connection. Modeled after Go SDK's {@code ReadIdleTimeout} approach. *

- * This handler does NOT close the connection on missed ACKs — connection eviction is - * handled by the eviction predicate in {@link HttpClient#createFixed}. The handler's - * sole purpose is keepalive traffic to reset middlebox idle timers. + * When a PING ACK is not received before the next PING is due, the handler marks the + * connection as unhealthy via the {@link #PING_HEALTH_DEGRADED} channel attribute. + * The handler does NOT close the connection — eviction is left to the connection pool + * or any component that checks the attribute. *

* Why manual instead of reactor-netty native {@code pingAckTimeout}? Native PING requires * built-in {@code maxIdleTime} handling, which is bypassed when a custom @@ -41,8 +43,21 @@ public class Http2PingHandler extends ChannelDuplexHandler { static final AttributeKey PING_HANDLER_INSTALLED = AttributeKey.valueOf("cosmos.conn.pingHandlerInstalled"); + /** + * Channel attribute set to {@code true} when a PING ACK is not received before the + * next PING would be due. Cleared when a successful ACK arrives. External components + * (eviction predicates, health checks) can read this attribute to detect broken connections. + */ + public static final AttributeKey PING_HEALTH_DEGRADED = + AttributeKey.valueOf("cosmos.conn.pingHealthDegraded"); + + // Global (process-wide) counters across all handler instances — used by tests + private static final AtomicInteger globalPingsSent = new AtomicInteger(0); + private static final AtomicInteger globalPingAcksReceived = new AtomicInteger(0); + private final long pingIntervalNanos; private long lastActivityNanos; + private long pingOutstandingSinceNanos; // nanoTime when PING was sent; 0 = no outstanding PING private ScheduledFuture pingTask; private final AtomicInteger pingsSent = new AtomicInteger(0); private final AtomicInteger pingAcksReceived = new AtomicInteger(0); @@ -89,6 +104,16 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception lastActivityNanos = System.nanoTime(); if (msg instanceof Http2PingFrame && ((Http2PingFrame) msg).ack()) { pingAcksReceived.incrementAndGet(); + globalPingAcksReceived.incrementAndGet(); + pingOutstandingSinceNanos = 0; + // Connection proved responsive — clear degraded flag if it was set + if (ctx.channel().hasAttr(PING_HEALTH_DEGRADED)) { + ctx.channel().attr(PING_HEALTH_DEGRADED).set(null); + if (logger.isDebugEnabled()) { + logger.debug("PING ACK received, connection {} marked healthy", + ctx.channel().id().asShortText()); + } + } } super.channelRead(ctx, msg); } @@ -105,9 +130,35 @@ private void maybeSendPing(ChannelHandlerContext ctx) { return; } + // If a previous PING is still outstanding, check whether it's been too long + if (pingOutstandingSinceNanos != 0) { + long waitingNanos = System.nanoTime() - pingOutstandingSinceNanos; + if (waitingNanos >= pingIntervalNanos) { + // ACK not received within one full interval — mark connection unhealthy + ctx.channel().attr(PING_HEALTH_DEGRADED).set(Boolean.TRUE); + logger.warn("PING ACK not received within {}s on channel {} — connection marked unhealthy", + TimeUnit.NANOSECONDS.toSeconds(pingIntervalNanos), + ctx.channel().id().asShortText()); + } + // Don't send another PING while one is outstanding + return; + } + + // Don't send if there are active streams — request/response traffic is already + // keeping the connection alive, so a PING would be pure noise-neighbour overhead. + Http2FrameCodec codec = ctx.pipeline().get(Http2FrameCodec.class); + if (codec != null && codec.connection().numActiveStreams() > 0) { + // Active streams reset the idle baseline so the first idle check after + // all streams close measures from *now*, not from the last frame. + lastActivityNanos = System.nanoTime(); + return; + } + long idleNanos = System.nanoTime() - lastActivityNanos; if (idleNanos >= pingIntervalNanos) { int count = pingsSent.incrementAndGet(); + globalPingsSent.incrementAndGet(); + pingOutstandingSinceNanos = System.nanoTime(); ctx.writeAndFlush(new DefaultHttp2PingFrame(count)) .addListener(f -> { if (f.isSuccess()) { @@ -115,6 +166,7 @@ private void maybeSendPing(ChannelHandlerContext ctx) { logger.debug("PING #{} sent on channel {}", count, ctx.channel().id().asShortText()); } } else { + pingOutstandingSinceNanos = 0; // unblock next attempt on send failure logger.warn("PING #{} failed on channel {}: {}", count, ctx.channel().id().asShortText(), f.cause() != null ? f.cause().getMessage() : "unknown"); @@ -140,6 +192,31 @@ public int getPingAcksReceived() { return pingAcksReceived.get(); } + /** + * Returns the total number of PINGs sent across all handler instances (process-wide). + */ + public static int getGlobalPingsSent() { + return globalPingsSent.get(); + } + + /** + * Returns the total number of PING ACKs received across all handler instances (process-wide). + */ + public static int getGlobalPingAcksReceived() { + return globalPingAcksReceived.get(); + } + + /** + * Returns {@code true} if the given channel (or its parent) has been marked as + * unhealthy due to a missed PING ACK. + */ + public static boolean isConnectionHealthDegraded(Channel channel) { + Channel parent = channel.parent() != null ? channel.parent() : channel; + return Boolean.TRUE.equals(parent.hasAttr(PING_HEALTH_DEGRADED) + ? parent.attr(PING_HEALTH_DEGRADED).get() + : null); + } + /** * Installs the PING handler on the parent H2 channel if not already installed. * Safe to call from doOnConnected (which fires per-stream for H2). diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/HttpClientConfig.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/HttpClientConfig.java index 239eb2c930a9..15527c4e2c89 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/HttpClientConfig.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/HttpClientConfig.java @@ -7,7 +7,6 @@ import com.azure.cosmos.Http2ConnectionConfig; import com.azure.cosmos.implementation.Configs; import com.azure.cosmos.implementation.ImplementationBridgeHelpers; -import io.netty.resolver.AddressResolverGroup; import java.time.Duration; @@ -36,8 +35,6 @@ private static ImplementationBridgeHelpers.Http2ConnectionConfigHelper.Http2Conn private boolean connectionKeepAlive = true; private boolean serverCertValidationDisabled = false; private Http2ConnectionConfig http2ConnectionConfig; - private AddressResolverGroup addressResolverGroup; - private java.util.function.Consumer doOnConnectedCallback; // Eagerly resolved thin client connect timeout — avoids per-request System.getProperty/getenv calls. private final int thinClientConnectTimeoutMs; @@ -192,24 +189,6 @@ public int getThinClientConnectTimeoutMs() { return this.thinClientConnectTimeoutMs; } - public AddressResolverGroup getAddressResolverGroup() { - return this.addressResolverGroup; - } - - public HttpClientConfig withAddressResolverGroup(AddressResolverGroup resolverGroup) { - this.addressResolverGroup = resolverGroup; - return this; - } - - public java.util.function.Consumer getDoOnConnectedCallback() { - return this.doOnConnectedCallback; - } - - public HttpClientConfig withDoOnConnectedCallback(java.util.function.Consumer callback) { - this.doOnConnectedCallback = callback; - return this; - } - public String toDiagnosticsString() { String gwV2Cto = Configs.isThinClientEnabled() ? Duration.ofMillis(this.thinClientConnectTimeoutMs).toString() diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java index 04e6a3d52df9..b175d1297973 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java @@ -159,12 +159,6 @@ private void configureChannelPipelineHandlers() { Http2PingHandler.installIfAbsent(connection.channel(), pingIntervalSeconds); } } - - // Test hook: allows injection of custom handlers (e.g., PING frame counter) - java.util.function.Consumer doOnConnectedCb = ReactorNettyClient.this.httpClientConfig.getDoOnConnectedCallback(); - if (doOnConnectedCb != null) { - doOnConnectedCb.accept(connection); - } }); if (isH2Enabled) { diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/interceptor/IHttpClientInterceptor.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/interceptor/IHttpClientInterceptor.java deleted file mode 100644 index fa915e2ea3ac..000000000000 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/interceptor/IHttpClientInterceptor.java +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.implementation.interceptor; - -import io.netty.resolver.AddressResolverGroup; -import reactor.netty.Connection; - -import java.util.function.Consumer; - -/** - * Interceptor for HTTP client configuration at construction time. - *

- * This interface allows test code (azure-cosmos-test) to inject custom DNS resolvers - * and connection handlers without polluting public API classes like ConnectionPolicy. - *

- * Production code checks: if interceptor is null → normal flow, zero overhead. - */ -public interface IHttpClientInterceptor { - - /** - * Returns a custom AddressResolverGroup for DNS resolution, or null to use the default. - */ - AddressResolverGroup getAddressResolverGroup(); - - /** - * Returns a doOnConnected callback to install on the HTTP client, or null for none. - */ - Consumer getDoOnConnectedCallback(); -} From 6b651c66a068f5b025e2adc39eb349e2b2f6ed28 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Tue, 12 May 2026 16:11:42 -0400 Subject: [PATCH 03/62] Align PING timing with Rust SDK: interval=1s, timeout=2s, close on timeout - Default PING interval: 5s -> 1s (aligned with Rust SDK's hyper config) - Default PING timeout: 10s -> 2s (aligned with Rust SDK's hyper config) - Worst-case dead connection detection: ~15s -> ~3s - Minimum check frequency: 1000ms -> 500ms - On PING timeout: close connection via ctx.close() (aligned with Rust SDK where hyper kills connections on timeout -> shard eviction) - PING_HEALTH_DEGRADED attribute set before close for diagnostic visibility - Spec: clarify sharding is Rust-specific (hyper opens 1 conn per client); reactor-netty natively pools multiple H2 connections, no shard layer needed --- .../docs/http2-ping-keepalive-spec.md | 102 +++++++++++------- .../azure/cosmos/implementation/Configs.java | 15 ++- .../implementation/http/Http2PingHandler.java | 39 ++++--- .../http/ReactorNettyClient.java | 3 +- 4 files changed, 104 insertions(+), 55 deletions(-) diff --git a/sdk/cosmos/azure-cosmos/docs/http2-ping-keepalive-spec.md b/sdk/cosmos/azure-cosmos/docs/http2-ping-keepalive-spec.md index ee58e0e808a9..90e6167cc2b8 100644 --- a/sdk/cosmos/azure-cosmos/docs/http2-ping-keepalive-spec.md +++ b/sdk/cosmos/azure-cosmos/docs/http2-ping-keepalive-spec.md @@ -2,7 +2,6 @@ **Status:** Draft **PR:** [#49095](https://github.com/Azure/azure-sdk-for-java/pull/49095) -**Split from:** [#48420](https://github.com/Azure/azure-sdk-for-java/pull/48420) **Date:** 2026-05-12 --- @@ -15,12 +14,16 @@ The SDK's existing connection pool eviction (`evictionPredicate`) handles *local ## 2. Goal -Send periodic HTTP/2 PING frames on idle parent (TCP) channels to keep middlebox connection tracking entries alive, preventing silent connection reaping. +Send periodic HTTP/2 PING frames on idle parent (TCP) channels to: +1. **Keep middlebox connection tracking entries alive**, preventing silent connection reaping. +2. **Detect broken connections** — if no PING ACK within the configured timeout, close the connection so the pool creates a fresh replacement. + +This is aligned with the Rust SDK’s approach where hyper’s built-in PING kills connections on timeout and the shard health sweep replaces them. **Non-goals:** -- Closing connections directly on missed ACKs — the handler marks connections as unhealthy via a channel attribute; actual eviction is left to the connection pool. -- Replacing reactor-netty's native `maxIdleTime` — the SDK uses a custom `evictionPredicate` that bypasses reactor-netty's built-in idle handling. +- Replacing reactor-netty’s native `maxIdleTime` — the SDK uses a custom `evictionPredicate` that bypasses reactor-netty’s built-in idle handling. - Application-layer keepalive (e.g., Cosmos heartbeat RPCs). +- Sharding — Rust requires per-endpoint sharding because hyper opens only 1 H2 connection per client. Reactor-netty natively pools multiple H2 connections, so sharding is unnecessary. ## 3. Design @@ -33,6 +36,7 @@ A Netty `ChannelDuplexHandler` installed on the **parent (TCP) channel** of an H | Field | Type | Description | |---|---|---| | `pingIntervalNanos` | `long` | Configured idle threshold (immutable) | +| `pingTimeoutNanos` | `long` | ACK timeout — close connection if exceeded (immutable) | | `lastActivityNanos` | `long` | `System.nanoTime()` of last read/write (event-loop-local, no sync) | | `pingTask` | `ScheduledFuture` | Periodic check handle, cancelled on removal | | `pingOutstandingSinceNanos` | `long` | `nanoTime()` when PING was sent; 0 = no outstanding PING | @@ -44,7 +48,7 @@ A Netty `ChannelDuplexHandler` installed on the **parent (TCP) channel** of an H ``` handlerAdded() - └─ schedule periodic check (interval = max(1s, pingInterval/2)) + └─ schedule periodic check (interval = max(500ms, pingInterval/2)) channelRead(frame) ├─ update lastActivityNanos @@ -61,7 +65,7 @@ maybeSendPing() ← periodic task ├─ if channel inactive → cancel + return ├─ if feature disabled → cancel + return ├─ if pingOutstandingSinceNanos != 0: - │ if waited ≥ pingInterval → set PING_HEALTH_DEGRADED on channel (WARN log) + │ if waited ≥ pingTimeout → set PING_HEALTH_DEGRADED, cancel task, ctx.close() │ return (at most 1 in-flight PING) ├─ if numActiveStreams > 0 → reset lastActivityNanos + return │ (active request traffic is keepalive; PING would be noise) @@ -79,7 +83,13 @@ handlerRemoved() / channelInactive() **At-most-one outstanding PING.** `pingOutstandingSinceNanos` records when the PING was sent (0 = none outstanding). If a PING is already in flight, `maybeSendPing()` returns immediately — this prevents unbounded queuing if ACKs are delayed. -**Broken connection detection.** When the periodic check finds that `pingOutstandingSinceNanos` is non-zero and the elapsed wait exceeds `pingIntervalNanos`, it sets the `PING_HEALTH_DEGRADED` channel attribute to `true` and logs a WARN. The handler does **not** close the connection — external components (eviction predicates, health checks, or request-path code) can read `Http2PingHandler.isConnectionHealthDegraded(channel)` and decide how to react. When an ACK finally arrives, the flag is cleared and a DEBUG log confirms recovery. +**Broken connection detection.** When the periodic check finds that `pingOutstandingSinceNanos` is non-zero and the elapsed wait exceeds `pingTimeoutNanos`, the handler: +1. Sets the `PING_HEALTH_DEGRADED` channel attribute to `true` +2. Logs a WARN +3. Cancels the periodic task +4. Closes the connection via `ctx.close()` + +This is aligned with the Rust SDK where hyper kills connections on PING timeout. The connection pool will create a fresh replacement on the next request. The `PING_HEALTH_DEGRADED` attribute remains set for observability — external components can read `Http2PingHandler.isConnectionHealthDegraded(channel)` for diagnostic purposes. **Suppressed during active streams.** Before checking idle time, the handler queries `Http2FrameCodec.connection().numActiveStreams()`. If any streams are open (i.e., customer requests are in flight), the PING is skipped entirely — the request/response frames already keep the middlebox entry alive. The idle baseline is reset to `now` so that the full interval is measured from when all streams close, not from the last frame on a previous stream. @@ -90,24 +100,25 @@ Together these two guards ensure: #### Idle detection -Any inbound frame (`channelRead`) or outbound write (`write`) resets the idle timer. The periodic check runs at `max(1 second, pingInterval / 2)` — this bounds the worst-case send delay to 1.5× the configured interval while avoiding excessive scheduling overhead. +Any inbound frame (`channelRead`) or outbound write (`write`) resets the idle timer. The periodic check runs at `max(500ms, pingInterval / 2)` — this bounds the worst-case send delay to 1.5× the configured interval while avoiding excessive scheduling overhead. #### PING frame details - **Outbound:** `DefaultHttp2PingFrame(count)` where `count` = `pingsSent` counter (auto-increment). The 8-byte payload carries the sequence number for optional correlation. - **Inbound ACK:** `Http2PingFrame` with `ack() == true`, counted and propagated. -- **No connection closure:** Missed ACKs are not tracked or acted upon. The handler is purely keepalive, not health-check. +- **No connection closure:** Missed ACKs cause the connection to be **closed** after the configured timeout. The handler sets `PING_HEALTH_DEGRADED` and calls `ctx.close()`. ### 3.2 Installation -Installed via `Http2PingHandler.installIfAbsent(Channel, int)`: +Installed via `Http2PingHandler.installIfAbsent(Channel, int, int)`: ``` doOnConnected(connection) ├─ Is HTTP/2? → check pipeline for Http2MultiplexHandler ├─ Is enabled? → Configs.isHttp2PingHealthEnabled() ├─ Interval > 0? → Configs.getHttp2PingIntervalInSeconds() - └─ installIfAbsent(channel, interval) + ├─ Timeout → Configs.getHttp2PingTimeoutInSeconds() + └─ installIfAbsent(channel, interval, timeout) ├─ resolve parent channel (channel.parent() ?? channel) ├─ idempotency check via PING_HANDLER_INSTALLED attribute └─ parent.pipeline().addLast("cosmos.http2PingHandler", handler) @@ -123,17 +134,17 @@ doOnConnected(connection) ### 3.4 Test hook -`HttpClientConfig.doOnConnectedCallback` provides a `Consumer` hook that fires after the PING handler installation. Tests use this to: -1. Capture a reference to the installed `Http2PingHandler` instance -2. Install additional handlers (e.g., `Http2PingFrameCounterHandler`) -3. Assert on `getPingsSent()` / `getPingAcksReceived()` after an idle period +The test (`Http2PingKeepaliveTest`) relies on the auto-installed handler and reads aggregate PING statistics via `Http2PingHandler.getGlobalPingsSent()` and `getGlobalPingAcksReceived()`. No interceptor or callback injection is required. ## 4. Configuration | System Property | Default | Description | |---|---|---| | `COSMOS.HTTP2_PING_HEALTH_ENABLED` | `true` | Master switch. Set to `false` to disable PING keepalive. | -| `COSMOS.HTTP2_PING_INTERVAL_IN_SECONDS` | `10` | Idle threshold before sending a PING frame. | +| `COSMOS.HTTP2_PING_INTERVAL_IN_SECONDS` | `1` | Idle threshold before sending a PING frame. Aligned with Rust SDK (1s). | +| `COSMOS.HTTP2_PING_TIMEOUT_IN_SECONDS` | `2` | ACK timeout. If no ACK within this, the connection is closed. Aligned with Rust SDK (2s). | + +**Aligned with Rust SDK:** Both SDKs use interval=1s, timeout=2s by default. Dead connection detected within ~3s worst-case. **No client builder API.** Configuration is JVM-wide via system properties, consistent with the existing `COSMOS.HTTP2_ENABLED` pattern. @@ -150,27 +161,30 @@ doOnConnected(connection) | Parameter | Value (defaults) | |---|---| -| PING interval | 10 seconds | -| Check frequency | 5 seconds (interval / 2) | -| Worst-case PING delay | ~15 seconds (idle just after a check) | -| Minimum check frequency | 1 second (clamped) | +| PING interval | 1 second | +| PING ACK timeout | 2 seconds | +| Check frequency | 500 milliseconds (interval / 2) | +| Worst-case detection | ~3 seconds (idle just after a check + full timeout) | +| Minimum check frequency | 500 milliseconds (clamped) | For a default-configured idle connection: ``` -t=0s last activity -t=5s check → idle 5s < 10s → skip -t=10s check → idle 10s ≥ 10s → PING sent, idle reset -t=15s check → idle 5s < 10s → skip -t=20s check → idle 10s ≥ 10s → PING sent -... +t=0s last activity +t=0.5s check → idle 0.5s < 1s → skip +t=1s check → idle 1s ≥ 1s → PING sent, idle reset +t=1.5s check → PING outstanding 0.5s < 2s → wait +t=2s check → PING outstanding 1s < 2s → wait +t=2.5s check → PING outstanding 1.5s < 2s → wait +t=3s check → PING outstanding 2s ≥ 2s → CLOSE (if no ACK) +…or at t=1.1s ACK arrives → pingOutstanding=0, PING_HEALTH_DEGRADED cleared ``` ## 7. Failure Modes | Scenario | Behavior | |---|---| -| PING ACK not received | `PING_HEALTH_DEGRADED` attribute set on channel after one interval. Connection NOT closed. `pingOutstandingSinceNanos` stays set, blocking further PINGs. | -| PING ACK arrives late | `PING_HEALTH_DEGRADED` cleared, `pingOutstandingSinceNanos` reset. Connection resumes normal keepalive. | +| PING ACK not received | After timeout: `PING_HEALTH_DEGRADED` set, task cancelled, connection **closed** via `ctx.close()`. Pool creates replacement on next request. | +| PING ACK arrives late | Before timeout: `PING_HEALTH_DEGRADED` cleared, `pingOutstandingSinceNanos` reset. Connection resumes normal keepalive. | | Connection has active streams | PING suppressed — request traffic is keepalive. Idle baseline reset. | | Channel becomes inactive | Periodic task cancelled in `channelInactive()`. | | Feature disabled at runtime | Task self-cancels on next check. | @@ -186,7 +200,7 @@ t=20s check → idle 10s ≥ 10s → PING sent | DEBUG | PING frame sent successfully (sequence #, channel ID) | | DEBUG | PING ACK received, degraded flag cleared (channel ID) | | WARN | PING frame send failure (sequence #, channel ID, exception) | -| WARN | PING ACK not received within interval — connection marked unhealthy (channel ID) | +| WARN | PING ACK not received within timeout — connection closed (channel ID) | ## 9. Test Coverage @@ -205,11 +219,27 @@ t=20s check → idle 10s ≥ 10s → PING sent **Helper:** `Http2PingFrameCounterHandler` — independent inbound handler that counts PING ACK frames for cross-validation. -## 10. Future Considerations +## 10. Rust SDK Alignment -- **Health-check mode:** Track PING ACK latency and close connections that exceed a timeout. This would replace the current approach where `evictionPredicate` handles stale connections. -- **Metrics integration:** Expose `pingsSent` / `pingAcksReceived` counters in `CosmosDiagnostics` for observability. -- **Eviction predicate integration:** Wire `Http2PingHandler.isConnectionHealthDegraded()` into the connection pool's eviction predicate to automatically remove connections that fail PING health checks. -- **Missed-ACK escalation:** If `PING_HEALTH_DEGRADED` stays set across N intervals, escalate to connection close. -- **Client builder API:** Expose `http2PingIntervalInSeconds()` on `CosmosClientBuilder` for per-client configuration instead of JVM-wide system properties. -- **Adaptive interval:** Adjust PING interval based on observed middlebox behavior (e.g., back off if ACKs are always received). +| Dimension | Java | Rust | +|---|---|---| +| Mechanism | Custom Netty `ChannelDuplexHandler` | hyper’s built-in `http2_keep_alive_*` | +| PING interval | 1s (configurable) | 1s (configurable) | +| ACK timeout | 2s (configurable) | 2s (configurable) | +| On missed ACK | Close connection | hyper kills connection → shard eviction | +| PING while idle | Yes (suppressed when activeStreams > 0) | Yes (`http2_keep_alive_while_idle = true`) | +| In-flight limit | 1 (explicit gate) | hyper manages internally | +| Sharding | Not needed — reactor-netty pools multiple H2 connections per endpoint | Per-endpoint shard pools (needed because hyper opens only 1 H2 connection per client) | +| TCP keepalive | Not explicitly disabled alongside H2 PING | Mutually exclusive with H2 PING | + +**Deliberate differences:** + +- **Active-stream suppression** (Java-only) avoids sending PING frames during request bursts. Rust’s hyper handles this internally. +- **No sharding needed** — Rust requires per-endpoint sharding because hyper's HTTP/2 client opens only 1 connection per `reqwest::Client`. Java's reactor-netty natively supports pooling multiple H2 connections per endpoint via `Http2AllocationStrategy` (configurable `minConnections`/`maxConnections`/`maxConcurrentStreams`), so the PING handler is simply installed per-connection with no shard layer. + +## 11. Future Considerations + +- **Metrics integration:** Expose `pingsSent` / `pingAcksReceived` / `PING_HEALTH_DEGRADED` in `CosmosDiagnostics` for observability. +- **Client builder API:** Expose `http2PingIntervalInSeconds()` and `http2PingTimeoutInSeconds()` on `CosmosClientBuilder` for per-client configuration. +- **TCP keepalive mutual exclusion:** Explicitly disable TCP keepalive when HTTP/2 PING is active (aligned with Rust). +- **Adaptive interval:** Adjust PING interval based on observed middlebox behavior. diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java index f21ec5569388..d71658be920c 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java @@ -148,13 +148,16 @@ public class Configs { // HTTP/2 PING keepalive — keeps connections alive for sparse workloads by preventing // intermediate infrastructure (NAT gateways, firewalls, load balancers) from silently - // reaping idle connections. PING is NOT used for eviction — degraded connections are - // handled by the response timeout retry path. + // reaping idle connections. On PING timeout (no ACK within the configured timeout), + // the connection is closed — similar to Rust SDK's hyper-based PING behavior. // Guarded by an explicit enable flag; default ON. Set COSMOS.HTTP2_PING_HEALTH_ENABLED=false to disable. private static final boolean DEFAULT_HTTP2_PING_HEALTH_ENABLED = true; private static final String HTTP2_PING_HEALTH_ENABLED = "COSMOS.HTTP2_PING_HEALTH_ENABLED"; - private static final int DEFAULT_HTTP2_PING_INTERVAL_IN_SECONDS = 10; + // Aligned with Rust SDK (hyper): interval=1s, timeout=2s. Dead connection detected within 3s. + private static final int DEFAULT_HTTP2_PING_INTERVAL_IN_SECONDS = 1; private static final String HTTP2_PING_INTERVAL_IN_SECONDS = "COSMOS.HTTP2_PING_INTERVAL_IN_SECONDS"; + private static final int DEFAULT_HTTP2_PING_TIMEOUT_IN_SECONDS = 2; + private static final String HTTP2_PING_TIMEOUT_IN_SECONDS = "COSMOS.HTTP2_PING_TIMEOUT_IN_SECONDS"; private static final int DEFAULT_HTTP_RESPONSE_TIMEOUT_IN_SECONDS = 60; private static final int DEFAULT_QUERY_PLAN_RESPONSE_TIMEOUT_IN_SECONDS = 5; @@ -765,6 +768,12 @@ public static int getHttp2PingIntervalInSeconds() { DEFAULT_HTTP2_PING_INTERVAL_IN_SECONDS); } + public static int getHttp2PingTimeoutInSeconds() { + return getJVMConfigAsInt( + HTTP2_PING_TIMEOUT_IN_SECONDS, + DEFAULT_HTTP2_PING_TIMEOUT_IN_SECONDS); + } + public static Integer getPendingAcquireMaxCount() { String valueFromSystemProperty = System.getProperty(HTTP_PENDING_ACQUIRE_MAX_COUNT); if (valueFromSystemProperty != null && !valueFromSystemProperty.isEmpty()) { diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java index fe6b2f6a3c34..ea7343a6c392 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java @@ -23,12 +23,12 @@ *

* Sends PING frames when the connection is idle for longer than the configured interval, * preventing L7 middleboxes (NAT gateways, firewalls, load balancers) from silently - * reaping the connection. Modeled after Go SDK's {@code ReadIdleTimeout} approach. + * reaping the connection. Modeled after Rust SDK's hyper-based HTTP/2 PING approach. *

- * When a PING ACK is not received before the next PING is due, the handler marks the - * connection as unhealthy via the {@link #PING_HEALTH_DEGRADED} channel attribute. - * The handler does NOT close the connection — eviction is left to the connection pool - * or any component that checks the attribute. + * When a PING ACK is not received within the configured timeout, the handler closes + * the connection. The connection pool will then create a fresh replacement. This is + * aligned with the Rust SDK where hyper kills connections on PING timeout and the + * shard health sweep replaces them. *

* Why manual instead of reactor-netty native {@code pingAckTimeout}? Native PING requires * built-in {@code maxIdleTime} handling, which is bypassed when a custom @@ -56,6 +56,7 @@ public class Http2PingHandler extends ChannelDuplexHandler { private static final AtomicInteger globalPingAcksReceived = new AtomicInteger(0); private final long pingIntervalNanos; + private final long pingTimeoutNanos; private long lastActivityNanos; private long pingOutstandingSinceNanos; // nanoTime when PING was sent; 0 = no outstanding PING private ScheduledFuture pingTask; @@ -64,16 +65,19 @@ public class Http2PingHandler extends ChannelDuplexHandler { /** * @param pingIntervalSeconds interval in seconds; when idle longer than this, a PING is sent + * @param pingTimeoutSeconds timeout in seconds; if no ACK within this, the connection is closed */ - public Http2PingHandler(int pingIntervalSeconds) { + public Http2PingHandler(int pingIntervalSeconds, int pingTimeoutSeconds) { this.pingIntervalNanos = TimeUnit.SECONDS.toNanos(pingIntervalSeconds); + this.pingTimeoutNanos = TimeUnit.SECONDS.toNanos(pingTimeoutSeconds); this.lastActivityNanos = System.nanoTime(); } @Override public void handlerAdded(ChannelHandlerContext ctx) { // Schedule periodic check — runs on the channel's event loop (single-threaded, no sync needed) - long checkIntervalMs = Math.max(1000, TimeUnit.NANOSECONDS.toMillis(pingIntervalNanos) / 2); + // Check at interval/2 (min 500ms) to bound worst-case PING send delay to ~1.5× interval. + long checkIntervalMs = Math.max(500, TimeUnit.NANOSECONDS.toMillis(pingIntervalNanos) / 2); this.pingTask = ctx.executor().scheduleAtFixedRate( () -> maybeSendPing(ctx), checkIntervalMs, @@ -81,9 +85,10 @@ public void handlerAdded(ChannelHandlerContext ctx) { TimeUnit.MILLISECONDS); if (logger.isDebugEnabled()) { - logger.debug("Http2PingHandler installed on channel {}, interval={}s, checkEvery={}ms", + logger.debug("Http2PingHandler installed on channel {}, interval={}s, timeout={}s, checkEvery={}ms", ctx.channel().id().asShortText(), TimeUnit.NANOSECONDS.toSeconds(pingIntervalNanos), + TimeUnit.NANOSECONDS.toSeconds(pingTimeoutNanos), checkIntervalMs); } } @@ -130,15 +135,18 @@ private void maybeSendPing(ChannelHandlerContext ctx) { return; } - // If a previous PING is still outstanding, check whether it's been too long + // If a previous PING is still outstanding, check whether it has timed out if (pingOutstandingSinceNanos != 0) { long waitingNanos = System.nanoTime() - pingOutstandingSinceNanos; - if (waitingNanos >= pingIntervalNanos) { - // ACK not received within one full interval — mark connection unhealthy + if (waitingNanos >= pingTimeoutNanos) { + // ACK not received within timeout — connection is broken. + // Mark degraded and close (aligned with Rust SDK's hyper behavior). ctx.channel().attr(PING_HEALTH_DEGRADED).set(Boolean.TRUE); - logger.warn("PING ACK not received within {}s on channel {} — connection marked unhealthy", - TimeUnit.NANOSECONDS.toSeconds(pingIntervalNanos), + logger.warn("PING ACK not received within {}s on channel {} — closing connection", + TimeUnit.NANOSECONDS.toSeconds(pingTimeoutNanos), ctx.channel().id().asShortText()); + cancelPingTask(); + ctx.close(); } // Don't send another PING while one is outstanding return; @@ -223,15 +231,16 @@ public static boolean isConnectionHealthDegraded(Channel channel) { * * @param channel the channel from doOnConnected (may be stream or parent) * @param pingIntervalSeconds PING interval in seconds + * @param pingTimeoutSeconds PING ACK timeout in seconds */ - public static void installIfAbsent(Channel channel, int pingIntervalSeconds) { + public static void installIfAbsent(Channel channel, int pingIntervalSeconds, int pingTimeoutSeconds) { // When called from the first doOnConnected, channel IS the parent H2 channel. // When called from a stream doOnConnected, channel.parent() is the parent. Channel parent = channel.parent() != null ? channel.parent() : channel; if (!parent.hasAttr(PING_HANDLER_INSTALLED)) { parent.attr(PING_HANDLER_INSTALLED).set(Boolean.TRUE); try { - parent.pipeline().addLast(HANDLER_NAME, new Http2PingHandler(pingIntervalSeconds)); + parent.pipeline().addLast(HANDLER_NAME, new Http2PingHandler(pingIntervalSeconds, pingTimeoutSeconds)); } catch (IllegalArgumentException ignored) { // Duplicate — race between concurrent streams, benign } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java index b175d1297973..63134fc39337 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java @@ -155,8 +155,9 @@ private void configureChannelPipelineHandlers() { if (Configs.isHttp2PingHealthEnabled() && connection.channel().pipeline().get(Http2MultiplexHandler.class) != null) { int pingIntervalSeconds = Configs.getHttp2PingIntervalInSeconds(); + int pingTimeoutSeconds = Configs.getHttp2PingTimeoutInSeconds(); if (pingIntervalSeconds > 0) { - Http2PingHandler.installIfAbsent(connection.channel(), pingIntervalSeconds); + Http2PingHandler.installIfAbsent(connection.channel(), pingIntervalSeconds, pingTimeoutSeconds); } } }); From 7bda393f90b2a3ef534a0ab801efaad6e8ecce9e Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Tue, 12 May 2026 16:32:06 -0400 Subject: [PATCH 04/62] Add PING interval and timeout tests, remove thin client dependency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - pingFramesSentAtConfiguredInterval: verifies PINGs fire at 1s interval over a 10s idle period (expects >= 5 PINGs sent and ACKed), asserts connection is reused (PING kept it alive) - connectionClosedOnPingTimeout: uses iptables DROP to blackhole traffic, verifies PING handler closes connection after 2s timeout, asserts recovery request uses a different connection (Docker --cap-add=NET_ADMIN) - Remove COSMOS.THINCLIENT_ENABLED from Maven profile — HTTP/2 PING only requires COSMOS.HTTP2_ENABLED=true, no thin client needed --- sdk/cosmos/azure-cosmos-tests/pom.xml | 1 - .../Http2PingKeepaliveTest.java | 181 ++++++++++++++---- 2 files changed, 146 insertions(+), 36 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-tests/pom.xml b/sdk/cosmos/azure-cosmos-tests/pom.xml index ffa6b9ff64db..44fce7dbb5b0 100644 --- a/sdk/cosmos/azure-cosmos-tests/pom.xml +++ b/sdk/cosmos/azure-cosmos-tests/pom.xml @@ -874,7 +874,6 @@ Licensed under the MIT License. true - true diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java index 1df90d507115..0dbdbb82accd 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java @@ -27,11 +27,15 @@ import static org.assertj.core.api.Assertions.assertThat; /** - * Standalone test for HTTP/2 PING keepalive handler. - * Proves that the Http2PingHandler sends PING frames on idle connections - * and that the connection survives the idle period. - * + * HTTP/2 PING keepalive handler tests. + *

+ * Test 1: Verifies PINGs are sent at the configured interval on idle connections. + * Test 2: Uses iptables DROP to blackhole PING ACKs, verifying the handler closes + * the broken connection and a subsequent request uses a new connection. + *

* Run in Docker with --cap-add=NET_ADMIN (group: manual-http-network-fault). + * Requires: HTTP/2 capable Cosmos DB account. + * Does NOT require thin client — only COSMOS.HTTP2_ENABLED=true. */ public class Http2PingKeepaliveTest extends FaultInjectionTestBase { @@ -52,9 +56,8 @@ public Http2PingKeepaliveTest() { @BeforeClass(groups = {"manual-http-network-fault"}, timeOut = 120_000) public void beforeClass() { - // Enable HTTP/2 for this test + // Enable HTTP/2 — thin client is NOT required for PING tests System.setProperty("COSMOS.HTTP2_ENABLED", "true"); - System.setProperty("COSMOS.THINCLIENT_ENABLED", "true"); this.client = getClientBuilder().buildAsyncClient(); this.cosmosAsyncContainer = getSharedMultiPartitionCosmosContainerWithIdAsPartitionKey(this.client); @@ -69,7 +72,6 @@ public void beforeClass() { public void afterClass() { safeClose(this.client); System.clearProperty("COSMOS.HTTP2_ENABLED"); - System.clearProperty("COSMOS.THINCLIENT_ENABLED"); } @BeforeMethod(groups = {"manual-http-network-fault"}) @@ -82,56 +84,155 @@ public void afterMethod(Method method) { logger.info("=== END: {} ===", method.getName()); } + /** + * Test 1: Verifies PING frames are sent at the configured interval on idle connections + * and that PING ACKs are received from the server. + *

+ * With interval=1s, a 10s idle period should produce ~10 PINGs. + */ @Test(groups = {"manual-http-network-fault"}, timeOut = TEST_TIMEOUT) - public void pingFramesSentAndAcknowledgedOnIdleConnection() throws Exception { - System.setProperty("COSMOS.HTTP2_PING_INTERVAL_IN_SECONDS", "3"); + public void pingFramesSentAtConfiguredInterval() throws Exception { + // Use default interval=1s, timeout=2s (aligned with Rust SDK) + System.setProperty("COSMOS.HTTP2_PING_INTERVAL_IN_SECONDS", "1"); + System.setProperty("COSMOS.HTTP2_PING_TIMEOUT_IN_SECONDS", "2"); System.setProperty("COSMOS.HTTP2_PING_HEALTH_ENABLED", "true"); try { safeClose(this.client); - CosmosClientBuilder builder = new CosmosClientBuilder() + this.client = new CosmosClientBuilder() .endpoint(TestConfigurations.HOST) .key(TestConfigurations.MASTER_KEY) .consistencyLevel(ConsistencyLevel.SESSION) - .gatewayMode(); - - this.client = builder.buildAsyncClient(); + .gatewayMode() + .buildAsyncClient(); this.cosmosAsyncContainer = getSharedMultiPartitionCosmosContainerWithIdAsPartitionKey(this.client); - // Establish H2 connection with a warm-up read — this triggers PING handler installation - String initialParentChannelId = readAndGetParentChannelId(); - logger.info("Initial parentChannelId: {}", initialParentChannelId); + // Reset global counters before measurement + int baselineSent = Http2PingHandler.getGlobalPingsSent(); + int baselineAcks = Http2PingHandler.getGlobalPingAcksReceived(); + + // Warm-up read — triggers H2 connection + PING handler installation + String initialChannelId = readAndGetParentChannelId(); + logger.info("Initial parentChannelId: {}", initialChannelId); - // Let the connection go idle — PINGs should fire every 3s - logger.info("Waiting 20s for PING frames to be sent on idle connection..."); - Thread.sleep(20_000); + // Let the connection go idle for 10s — with 1s interval, expect ~10 PINGs + logger.info("Waiting 10s for PING frames on idle connection (interval=1s)..."); + Thread.sleep(10_000); - // Recovery read — proves connection is still alive - String recoveryParentChannelId = readAndGetParentChannelId(); + int sentCount = Http2PingHandler.getGlobalPingsSent() - baselineSent; + int ackCount = Http2PingHandler.getGlobalPingAcksReceived() - baselineAcks; - // Get PING counters from the auto-installed handler - int sentCount = Http2PingHandler.getGlobalPingsSent(); - int ackCount = Http2PingHandler.getGlobalPingAcksReceived(); + // Recovery read — proves connection survived idle period + String recoveryChannelId = readAndGetParentChannelId(); - logger.info("RESULT: initial={}, recovery={}, SAME_CONNECTION={}, pingsSent={}, pingAcksReceived={}", - initialParentChannelId, recoveryParentChannelId, - initialParentChannelId.equals(recoveryParentChannelId), sentCount, ackCount); + logger.info("RESULT: initial={}, recovery={}, SAME={}, pingsSent={}, pingAcksReceived={}", + initialChannelId, recoveryChannelId, + initialChannelId.equals(recoveryChannelId), sentCount, ackCount); + // With 1s interval and 10s idle, we expect at least 5 PINGs (conservative bound) assertThat(sentCount) - .as("PINGs sent should be > 0 — proves the manual PING handler is actively sending frames") - .isGreaterThan(0); + .as("With 1s interval over 10s idle, expect at least 5 PINGs sent") + .isGreaterThanOrEqualTo(5); assertThat(ackCount) - .as("PING ACKs received should be > 0 — proves server acknowledged PINGs") - .isGreaterThan(0); + .as("All sent PINGs should be ACKed by the server") + .isGreaterThanOrEqualTo(5); + + // Connection should be reused (PINGs kept it alive) + assertThat(recoveryChannelId) + .as("Connection should survive idle period — PING keepalive prevents eviction") + .isEqualTo(initialChannelId); - // NOTE: We don't assert same connection because pool eviction behavior varies - // across account types (thin client vs standard). The core assertion is that - // PINGs are sent and ACKed — keepalive traffic is flowing. - logger.info("PING keepalive verified: {} PINGs sent, {} ACKs received", sentCount, ackCount); + logger.info("PING interval test passed: {} PINGs sent, {} ACKs received in 10s", sentCount, ackCount); } finally { System.clearProperty("COSMOS.HTTP2_PING_INTERVAL_IN_SECONDS"); + System.clearProperty("COSMOS.HTTP2_PING_TIMEOUT_IN_SECONDS"); + System.clearProperty("COSMOS.HTTP2_PING_HEALTH_ENABLED"); + } + } + + /** + * Test 2: Uses iptables to blackhole all traffic to the Cosmos DB gateway port, + * preventing PING ACKs from arriving. Verifies the handler closes the broken + * connection and a subsequent request (after removing the iptables rule) uses + * a new/different connection. + *

+ * Requires Docker with --cap-add=NET_ADMIN. + */ + @Test(groups = {"manual-http-network-fault"}, timeOut = TEST_TIMEOUT) + public void connectionClosedOnPingTimeout() throws Exception { + // Short interval + timeout for fast detection + System.setProperty("COSMOS.HTTP2_PING_INTERVAL_IN_SECONDS", "1"); + System.setProperty("COSMOS.HTTP2_PING_TIMEOUT_IN_SECONDS", "2"); + System.setProperty("COSMOS.HTTP2_PING_HEALTH_ENABLED", "true"); + + try { + safeClose(this.client); + + this.client = new CosmosClientBuilder() + .endpoint(TestConfigurations.HOST) + .key(TestConfigurations.MASTER_KEY) + .consistencyLevel(ConsistencyLevel.SESSION) + .gatewayMode() + .buildAsyncClient(); + this.cosmosAsyncContainer = getSharedMultiPartitionCosmosContainerWithIdAsPartitionKey(this.client); + + // Warm-up read — establish H2 connection + String initialChannelId = readAndGetParentChannelId(); + logger.info("Initial parentChannelId: {}", initialChannelId); + + // Extract gateway host + port for iptables rule + String gatewayHost = extractHostFromEndpoint(TestConfigurations.HOST); + int gatewayPort = 443; + logger.info("Gateway host: {}, port: {}", gatewayHost, gatewayPort); + + // Blackhole all traffic to gateway — prevents PING ACKs from arriving + String iptablesRule = String.format( + "iptables -A OUTPUT -p tcp --dport %d -d %s -j DROP", gatewayPort, gatewayHost); + logger.info("Installing iptables DROP rule: {}", iptablesRule); + Runtime.getRuntime().exec(new String[]{"bash", "-c", iptablesRule}).waitFor(); + + // Wait for PING timeout: 1s interval + 2s timeout + buffer = 5s + logger.info("Waiting 5s for PING timeout to close the connection..."); + Thread.sleep(5_000); + + // Remove iptables rule BEFORE attempting recovery read + String iptablesRemove = String.format( + "iptables -D OUTPUT -p tcp --dport %d -d %s -j DROP", gatewayPort, gatewayHost); + logger.info("Removing iptables DROP rule: {}", iptablesRemove); + Runtime.getRuntime().exec(new String[]{"bash", "-c", iptablesRemove}).waitFor(); + + // Small wait for network to stabilize + Thread.sleep(1_000); + + // Recovery read — should succeed on a NEW connection + String recoveryChannelId = readAndGetParentChannelId(); + logger.info("Recovery parentChannelId: {}", recoveryChannelId); + + logger.info("RESULT: initial={}, recovery={}, DIFFERENT_CONNECTION={}", + initialChannelId, recoveryChannelId, + !initialChannelId.equals(recoveryChannelId)); + + // The connection MUST be different — the old one was closed by PING timeout + assertThat(recoveryChannelId) + .as("After PING timeout, the handler should have closed the connection. " + + "The recovery request must use a new connection.") + .isNotEqualTo(initialChannelId); + + logger.info("PING timeout test passed: connection {} was closed, new connection {} established", + initialChannelId, recoveryChannelId); + } finally { + // Safety: remove any leftover iptables rules + try { + String gatewayHost = extractHostFromEndpoint(TestConfigurations.HOST); + String cleanup = String.format( + "iptables -D OUTPUT -p tcp --dport 443 -d %s -j DROP 2>/dev/null", gatewayHost); + Runtime.getRuntime().exec(new String[]{"bash", "-c", cleanup}).waitFor(); + } catch (Exception ignored) {} + + System.clearProperty("COSMOS.HTTP2_PING_INTERVAL_IN_SECONDS"); + System.clearProperty("COSMOS.HTTP2_PING_TIMEOUT_IN_SECONDS"); System.clearProperty("COSMOS.HTTP2_PING_HEALTH_ENABLED"); } } @@ -165,4 +266,14 @@ private String extractParentChannelId(CosmosDiagnostics diagnostics) { return "error-" + UUID.randomUUID().toString().substring(0, 8); } } + + private static String extractHostFromEndpoint(String endpoint) { + // Extract hostname from "https://foo.documents.azure.com:443/" + try { + java.net.URI uri = new java.net.URI(endpoint); + return uri.getHost(); + } catch (Exception e) { + throw new RuntimeException("Failed to parse endpoint: " + endpoint, e); + } + } } From ea8b37959376dac4e473fecb3e641c496a438641 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Tue, 12 May 2026 17:41:51 -0400 Subject: [PATCH 05/62] Force single-connection H2 pool in tests for deterministic channel assertions Set COSMOS.HTTP2_MAX/MIN_CONNECTION_POOL_SIZE=1 so that: - pingFramesSentAtConfiguredInterval can assert SAME connection (pool size=1 means if the connection survives, recovery must use the same one) - connectionClosedOnPingTimeout has deterministic initial channel ID Docker-validated: Tests run: 2, Failures: 0, Errors: 0 - pingsSent=14, pingAcksReceived=14, SAME=true (pool maxc:1, minc:1) - DIFFERENT_CONNECTION=true after iptables DROP + PING timeout --- .../faultinjection/Http2PingKeepaliveTest.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java index 0dbdbb82accd..75cbb979f427 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java @@ -96,6 +96,9 @@ public void pingFramesSentAtConfiguredInterval() throws Exception { System.setProperty("COSMOS.HTTP2_PING_INTERVAL_IN_SECONDS", "1"); System.setProperty("COSMOS.HTTP2_PING_TIMEOUT_IN_SECONDS", "2"); System.setProperty("COSMOS.HTTP2_PING_HEALTH_ENABLED", "true"); + // Force single-connection pool so we can assert same connection is reused + System.setProperty("COSMOS.HTTP2_MAX_CONNECTION_POOL_SIZE", "1"); + System.setProperty("COSMOS.HTTP2_MIN_CONNECTION_POOL_SIZE", "1"); try { safeClose(this.client); @@ -139,9 +142,9 @@ public void pingFramesSentAtConfiguredInterval() throws Exception { .as("All sent PINGs should be ACKed by the server") .isGreaterThanOrEqualTo(5); - // Connection should be reused (PINGs kept it alive) + // With pool size=1, the same connection MUST be reused (PING kept it alive) assertThat(recoveryChannelId) - .as("Connection should survive idle period — PING keepalive prevents eviction") + .as("With single-connection pool, PING keepalive must preserve the connection") .isEqualTo(initialChannelId); logger.info("PING interval test passed: {} PINGs sent, {} ACKs received in 10s", sentCount, ackCount); @@ -149,6 +152,8 @@ public void pingFramesSentAtConfiguredInterval() throws Exception { System.clearProperty("COSMOS.HTTP2_PING_INTERVAL_IN_SECONDS"); System.clearProperty("COSMOS.HTTP2_PING_TIMEOUT_IN_SECONDS"); System.clearProperty("COSMOS.HTTP2_PING_HEALTH_ENABLED"); + System.clearProperty("COSMOS.HTTP2_MAX_CONNECTION_POOL_SIZE"); + System.clearProperty("COSMOS.HTTP2_MIN_CONNECTION_POOL_SIZE"); } } @@ -166,6 +171,9 @@ public void connectionClosedOnPingTimeout() throws Exception { System.setProperty("COSMOS.HTTP2_PING_INTERVAL_IN_SECONDS", "1"); System.setProperty("COSMOS.HTTP2_PING_TIMEOUT_IN_SECONDS", "2"); System.setProperty("COSMOS.HTTP2_PING_HEALTH_ENABLED", "true"); + // Force single-connection pool so channel ID assertion is deterministic + System.setProperty("COSMOS.HTTP2_MAX_CONNECTION_POOL_SIZE", "1"); + System.setProperty("COSMOS.HTTP2_MIN_CONNECTION_POOL_SIZE", "1"); try { safeClose(this.client); @@ -234,6 +242,8 @@ public void connectionClosedOnPingTimeout() throws Exception { System.clearProperty("COSMOS.HTTP2_PING_INTERVAL_IN_SECONDS"); System.clearProperty("COSMOS.HTTP2_PING_TIMEOUT_IN_SECONDS"); System.clearProperty("COSMOS.HTTP2_PING_HEALTH_ENABLED"); + System.clearProperty("COSMOS.HTTP2_MAX_CONNECTION_POOL_SIZE"); + System.clearProperty("COSMOS.HTTP2_MIN_CONNECTION_POOL_SIZE"); } } From eac6e9f076273b75e503dfebf30128ba7ac6917c Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Tue, 12 May 2026 17:46:05 -0400 Subject: [PATCH 06/62] Add Cosmos_Live_Test_Http2NetworkFault CI stage, delete unused Http2PingFrameCounterHandler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add live-http2-network-fault-platform-matrix.json (Linux, single profile) - Add Cosmos_Live_Test_Http2NetworkFault stage in tests.yml: MaxParallel=1, profile=manual-http-network-fault, timeout=30min Passes COSMOS.HTTP2_ENABLED, HTTP2_PING_HEALTH_ENABLED, interval=1s, timeout=2s - Delete Http2PingFrameCounterHandler — unused, tests use global counters - Remove Http2PingFrameCounterHandler reference from spec --- .../Http2PingFrameCounterHandler.java | 37 ------------------- .../docs/http2-ping-keepalive-spec.md | 2 - ...e-http2-network-fault-platform-matrix.json | 25 +++++++++++++ sdk/cosmos/tests.yml | 30 +++++++++++++++ 4 files changed, 55 insertions(+), 39 deletions(-) delete mode 100644 sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingFrameCounterHandler.java create mode 100644 sdk/cosmos/live-http2-network-fault-platform-matrix.json diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingFrameCounterHandler.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingFrameCounterHandler.java deleted file mode 100644 index 02ccba8dd195..000000000000 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingFrameCounterHandler.java +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.faultinjection; - -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.ChannelInboundHandlerAdapter; -import io.netty.handler.codec.http2.Http2PingFrame; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * Test utility: counts HTTP/2 PING ACK frames received on the parent H2 channel. - * Incoming PING ACKs prove that PING frames were sent by reactor-netty and acknowledged by the server. - * Install on the parent channel (not stream channel) via doOnConnected. - */ -public class Http2PingFrameCounterHandler extends ChannelInboundHandlerAdapter { - private static final Logger logger = LoggerFactory.getLogger(Http2PingFrameCounterHandler.class); - private final AtomicInteger pingAckCount = new AtomicInteger(0); - - @Override - public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { - if (msg instanceof Http2PingFrame) { - Http2PingFrame pingFrame = (Http2PingFrame) msg; - if (pingFrame.ack()) { - int count = pingAckCount.incrementAndGet(); - logger.info("PING ACK #{} received on channel {}", count, ctx.channel().id().asShortText()); - } - } - super.channelRead(ctx, msg); - } - - public int getPingAckCount() { - return pingAckCount.get(); - } -} diff --git a/sdk/cosmos/azure-cosmos/docs/http2-ping-keepalive-spec.md b/sdk/cosmos/azure-cosmos/docs/http2-ping-keepalive-spec.md index 90e6167cc2b8..09d88f0ed28b 100644 --- a/sdk/cosmos/azure-cosmos/docs/http2-ping-keepalive-spec.md +++ b/sdk/cosmos/azure-cosmos/docs/http2-ping-keepalive-spec.md @@ -217,8 +217,6 @@ t=3s check → PING outstanding 2s ≥ 2s → CLOSE (if no ACK) | 5. Assert `pingsSent > 0` | PING transmission works | | 6. Assert `pingAcksReceived > 0` | Server acknowledges PINGs | -**Helper:** `Http2PingFrameCounterHandler` — independent inbound handler that counts PING ACK frames for cross-validation. - ## 10. Rust SDK Alignment | Dimension | Java | Rust | diff --git a/sdk/cosmos/live-http2-network-fault-platform-matrix.json b/sdk/cosmos/live-http2-network-fault-platform-matrix.json new file mode 100644 index 000000000000..65ab74536051 --- /dev/null +++ b/sdk/cosmos/live-http2-network-fault-platform-matrix.json @@ -0,0 +1,25 @@ +{ + "displayNames": { + "-Pmanual-http-network-fault": "HttpNetworkFault", + "Session": "", + "ubuntu": "", + "@{ enableMultipleWriteLocations = $true; defaultConsistencyLevel = 'Session'; enableMultipleRegions = $true }": "http2-network-fault" + }, + "include": [ + { + "DESIRED_CONSISTENCIES": "[\"Session\"]", + "ACCOUNT_CONSISTENCY": "Session", + "ArmConfig": { + "MultiMaster_MultiRegion": { + "ArmTemplateParameters": "@{ enableMultipleWriteLocations = $true; defaultConsistencyLevel = 'Session'; enableMultipleRegions = $true }", + "PREFERRED_LOCATIONS": "[\"East US 2\"]" + } + }, + "PROTOCOLS": "[\"Tcp\"]", + "ProfileFlag": [ "-Pmanual-http-network-fault" ], + "Agent": { + "ubuntu": { "OSVmImage": "env:LINUXVMIMAGE", "Pool": "env:LINUXPOOL" } + } + } + ] +} diff --git a/sdk/cosmos/tests.yml b/sdk/cosmos/tests.yml index 92bbc0347acb..4f4d122cc553 100644 --- a/sdk/cosmos/tests.yml +++ b/sdk/cosmos/tests.yml @@ -67,6 +67,36 @@ extends: - name: AdditionalArgs value: '-DCOSMOS.CLIENT_LEAK_DETECTION_ENABLED=true -DCOSMOS.HTTP2_ENABLED=true' + - template: /eng/pipelines/templates/stages/archetype-sdk-tests-isolated.yml + parameters: + TestName: 'Cosmos_Live_Test_Http2NetworkFault' + CloudConfig: + Public: + ServiceConnection: azure-sdk-tests-cosmos + MatrixConfigs: + - Name: Cosmos_live_test_http2_network_fault + Path: sdk/cosmos/live-http2-network-fault-platform-matrix.json + Selection: all + GenerateVMJobs: true + MatrixReplace: + - .*Version=1.2(1|5)/1.17 + ServiceDirectory: cosmos + Artifacts: + - name: azure-cosmos + groupId: com.azure + safeName: azurecosmos + AdditionalModules: + - name: azure-cosmos-tests + groupId: com.azure + TimeoutInMinutes: 30 + MaxParallel: 1 + TestGoals: 'verify' + TestOptions: '$(ProfileFlag) $(AdditionalArgs) -DskipCompile=true -DskipTestCompile=true -DcreateSourcesJar=false' + TestResultsFiles: '**/junitreports/TEST-*.xml' + AdditionalVariables: + - name: AdditionalArgs + value: '-DCOSMOS.CLIENT_LEAK_DETECTION_ENABLED=true -DCOSMOS.HTTP2_ENABLED=true -DCOSMOS.HTTP2_PING_HEALTH_ENABLED=true -DCOSMOS.HTTP2_PING_INTERVAL_IN_SECONDS=1 -DCOSMOS.HTTP2_PING_TIMEOUT_IN_SECONDS=2' + - template: /eng/pipelines/templates/stages/archetype-sdk-tests-isolated.yml parameters: TestName: 'Cosmos_Live_Test_ThinClient' From db82277c93417a01bfd33304042f44b237fa4305 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Tue, 12 May 2026 18:07:22 -0400 Subject: [PATCH 07/62] Add consecutive failure threshold for defensive PING-based connection close MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of closing on first missed ACK, the handler now tracks consecutive failures and only closes after reaching the threshold (default: 2). Flow with threshold=2: 1. PING #1 sent, timeout elapses → consecutiveFailures=1, unblock next PING 2. PING #2 sent, timeout elapses → consecutiveFailures=2 >= threshold → close On ACK received at any point → consecutiveFailures=0 (full reset). Timing with defaults (interval=1s, timeout=2s, threshold=2): - Worst-case detection: ~6s (2 rounds of interval+timeout) - Tolerates 1 transient network blip without closing New config: COSMOS.HTTP2_PING_FAILURE_THRESHOLD (default: 2) --- .../azure/cosmos/implementation/Configs.java | 11 ++++++ .../implementation/http/Http2PingHandler.java | 38 ++++++++++++------- .../http/ReactorNettyClient.java | 3 +- 3 files changed, 38 insertions(+), 14 deletions(-) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java index d71658be920c..d6af3d62811e 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java @@ -158,6 +158,11 @@ public class Configs { private static final String HTTP2_PING_INTERVAL_IN_SECONDS = "COSMOS.HTTP2_PING_INTERVAL_IN_SECONDS"; private static final int DEFAULT_HTTP2_PING_TIMEOUT_IN_SECONDS = 2; private static final String HTTP2_PING_TIMEOUT_IN_SECONDS = "COSMOS.HTTP2_PING_TIMEOUT_IN_SECONDS"; + // Consecutive PING failures (timeout without ACK) before closing the connection. + // Default 2: tolerates 1 transient miss, closes on 2nd consecutive failure. + // With interval=1s and timeout=2s, worst-case detection = 2*(1+2) = ~6s. + private static final int DEFAULT_HTTP2_PING_FAILURE_THRESHOLD = 2; + private static final String HTTP2_PING_FAILURE_THRESHOLD = "COSMOS.HTTP2_PING_FAILURE_THRESHOLD"; private static final int DEFAULT_HTTP_RESPONSE_TIMEOUT_IN_SECONDS = 60; private static final int DEFAULT_QUERY_PLAN_RESPONSE_TIMEOUT_IN_SECONDS = 5; @@ -774,6 +779,12 @@ public static int getHttp2PingTimeoutInSeconds() { DEFAULT_HTTP2_PING_TIMEOUT_IN_SECONDS); } + public static int getHttp2PingFailureThreshold() { + return getJVMConfigAsInt( + HTTP2_PING_FAILURE_THRESHOLD, + DEFAULT_HTTP2_PING_FAILURE_THRESHOLD); + } + public static Integer getPendingAcquireMaxCount() { String valueFromSystemProperty = System.getProperty(HTTP_PENDING_ACQUIRE_MAX_COUNT); if (valueFromSystemProperty != null && !valueFromSystemProperty.isEmpty()) { diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java index ea7343a6c392..9eae4c7129b5 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java @@ -57,19 +57,23 @@ public class Http2PingHandler extends ChannelDuplexHandler { private final long pingIntervalNanos; private final long pingTimeoutNanos; + private final int failureThreshold; private long lastActivityNanos; private long pingOutstandingSinceNanos; // nanoTime when PING was sent; 0 = no outstanding PING + private int consecutiveFailures; // incremented on timeout, reset on ACK private ScheduledFuture pingTask; private final AtomicInteger pingsSent = new AtomicInteger(0); private final AtomicInteger pingAcksReceived = new AtomicInteger(0); /** - * @param pingIntervalSeconds interval in seconds; when idle longer than this, a PING is sent - * @param pingTimeoutSeconds timeout in seconds; if no ACK within this, the connection is closed + * @param pingIntervalSeconds interval in seconds; when idle longer than this, a PING is sent + * @param pingTimeoutSeconds timeout in seconds per PING attempt + * @param failureThreshold consecutive timeouts before closing the connection */ - public Http2PingHandler(int pingIntervalSeconds, int pingTimeoutSeconds) { + public Http2PingHandler(int pingIntervalSeconds, int pingTimeoutSeconds, int failureThreshold) { this.pingIntervalNanos = TimeUnit.SECONDS.toNanos(pingIntervalSeconds); this.pingTimeoutNanos = TimeUnit.SECONDS.toNanos(pingTimeoutSeconds); + this.failureThreshold = Math.max(1, failureThreshold); this.lastActivityNanos = System.nanoTime(); } @@ -111,6 +115,7 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception pingAcksReceived.incrementAndGet(); globalPingAcksReceived.incrementAndGet(); pingOutstandingSinceNanos = 0; + consecutiveFailures = 0; // Connection proved responsive — clear degraded flag if it was set if (ctx.channel().hasAttr(PING_HEALTH_DEGRADED)) { ctx.channel().attr(PING_HEALTH_DEGRADED).set(null); @@ -139,14 +144,20 @@ private void maybeSendPing(ChannelHandlerContext ctx) { if (pingOutstandingSinceNanos != 0) { long waitingNanos = System.nanoTime() - pingOutstandingSinceNanos; if (waitingNanos >= pingTimeoutNanos) { - // ACK not received within timeout — connection is broken. - // Mark degraded and close (aligned with Rust SDK's hyper behavior). - ctx.channel().attr(PING_HEALTH_DEGRADED).set(Boolean.TRUE); - logger.warn("PING ACK not received within {}s on channel {} — closing connection", - TimeUnit.NANOSECONDS.toSeconds(pingTimeoutNanos), - ctx.channel().id().asShortText()); - cancelPingTask(); - ctx.close(); + consecutiveFailures++; + pingOutstandingSinceNanos = 0; // unblock next PING attempt + + if (consecutiveFailures >= failureThreshold) { + // Threshold reached — connection is broken. + ctx.channel().attr(PING_HEALTH_DEGRADED).set(Boolean.TRUE); + logger.warn("PING ACK not received for {} consecutive attempts on channel {} — closing connection", + consecutiveFailures, ctx.channel().id().asShortText()); + cancelPingTask(); + ctx.close(); + } else { + logger.warn("PING ACK timeout on channel {} (attempt {}/{}) — will retry", + ctx.channel().id().asShortText(), consecutiveFailures, failureThreshold); + } } // Don't send another PING while one is outstanding return; @@ -232,15 +243,16 @@ public static boolean isConnectionHealthDegraded(Channel channel) { * @param channel the channel from doOnConnected (may be stream or parent) * @param pingIntervalSeconds PING interval in seconds * @param pingTimeoutSeconds PING ACK timeout in seconds + * @param failureThreshold consecutive timeouts before closing */ - public static void installIfAbsent(Channel channel, int pingIntervalSeconds, int pingTimeoutSeconds) { + public static void installIfAbsent(Channel channel, int pingIntervalSeconds, int pingTimeoutSeconds, int failureThreshold) { // When called from the first doOnConnected, channel IS the parent H2 channel. // When called from a stream doOnConnected, channel.parent() is the parent. Channel parent = channel.parent() != null ? channel.parent() : channel; if (!parent.hasAttr(PING_HANDLER_INSTALLED)) { parent.attr(PING_HANDLER_INSTALLED).set(Boolean.TRUE); try { - parent.pipeline().addLast(HANDLER_NAME, new Http2PingHandler(pingIntervalSeconds, pingTimeoutSeconds)); + parent.pipeline().addLast(HANDLER_NAME, new Http2PingHandler(pingIntervalSeconds, pingTimeoutSeconds, failureThreshold)); } catch (IllegalArgumentException ignored) { // Duplicate — race between concurrent streams, benign } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java index 63134fc39337..588544a44301 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java @@ -156,8 +156,9 @@ private void configureChannelPipelineHandlers() { && connection.channel().pipeline().get(Http2MultiplexHandler.class) != null) { int pingIntervalSeconds = Configs.getHttp2PingIntervalInSeconds(); int pingTimeoutSeconds = Configs.getHttp2PingTimeoutInSeconds(); + int pingFailureThreshold = Configs.getHttp2PingFailureThreshold(); if (pingIntervalSeconds > 0) { - Http2PingHandler.installIfAbsent(connection.channel(), pingIntervalSeconds, pingTimeoutSeconds); + Http2PingHandler.installIfAbsent(connection.channel(), pingIntervalSeconds, pingTimeoutSeconds, pingFailureThreshold); } } }); From 43423fef07b8260aa235a36a6d48ec175e355235 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Tue, 12 May 2026 18:23:04 -0400 Subject: [PATCH 08/62] Align failure threshold with Rust SDK (5), test overrides to 2 for speed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Default COSMOS.HTTP2_PING_FAILURE_THRESHOLD: 2 -> 5 (aligned with Rust SDK's http2_consecutive_failure_threshold = 5) - Test sets threshold=2 via system property for faster execution (~6s vs ~15s) - Test wait time: 10s (sufficient for 2-round threshold) - Docker validated: Tests run: 2, Failures: 0, Errors: 0 - Logs show 'attempt 1/2 — will retry' then 'closing connection' on 2nd failure --- .../cosmos/faultinjection/Http2PingKeepaliveTest.java | 11 ++++++++--- .../java/com/azure/cosmos/implementation/Configs.java | 6 +++--- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java index 75cbb979f427..9c10b9642a74 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java @@ -171,6 +171,8 @@ public void connectionClosedOnPingTimeout() throws Exception { System.setProperty("COSMOS.HTTP2_PING_INTERVAL_IN_SECONDS", "1"); System.setProperty("COSMOS.HTTP2_PING_TIMEOUT_IN_SECONDS", "2"); System.setProperty("COSMOS.HTTP2_PING_HEALTH_ENABLED", "true"); + // Override threshold to 2 for faster test (default=5 aligned with Rust SDK) + System.setProperty("COSMOS.HTTP2_PING_FAILURE_THRESHOLD", "2"); // Force single-connection pool so channel ID assertion is deterministic System.setProperty("COSMOS.HTTP2_MAX_CONNECTION_POOL_SIZE", "1"); System.setProperty("COSMOS.HTTP2_MIN_CONNECTION_POOL_SIZE", "1"); @@ -201,9 +203,12 @@ public void connectionClosedOnPingTimeout() throws Exception { logger.info("Installing iptables DROP rule: {}", iptablesRule); Runtime.getRuntime().exec(new String[]{"bash", "-c", iptablesRule}).waitFor(); - // Wait for PING timeout: 1s interval + 2s timeout + buffer = 5s - logger.info("Waiting 5s for PING timeout to close the connection..."); - Thread.sleep(5_000); + // Wait for PING timeout with consecutive failure threshold=2 (test override): + // Round 1: 1s interval + 2s timeout = 3s (failure #1) + // Round 2: ~1s interval + 2s timeout = 3s (failure #2 ≥ threshold → close) + // Total ~6s + buffer = 10s + logger.info("Waiting 10s for consecutive PING timeouts to close the connection..."); + Thread.sleep(10_000); // Remove iptables rule BEFORE attempting recovery read String iptablesRemove = String.format( diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java index d6af3d62811e..b18fae02ca05 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java @@ -159,9 +159,9 @@ public class Configs { private static final int DEFAULT_HTTP2_PING_TIMEOUT_IN_SECONDS = 2; private static final String HTTP2_PING_TIMEOUT_IN_SECONDS = "COSMOS.HTTP2_PING_TIMEOUT_IN_SECONDS"; // Consecutive PING failures (timeout without ACK) before closing the connection. - // Default 2: tolerates 1 transient miss, closes on 2nd consecutive failure. - // With interval=1s and timeout=2s, worst-case detection = 2*(1+2) = ~6s. - private static final int DEFAULT_HTTP2_PING_FAILURE_THRESHOLD = 2; + // Aligned with Rust SDK's http2_consecutive_failure_threshold = 5. + // With interval=1s and timeout=2s, worst-case detection = 5*(1+2) = ~15s. + private static final int DEFAULT_HTTP2_PING_FAILURE_THRESHOLD = 5; private static final String HTTP2_PING_FAILURE_THRESHOLD = "COSMOS.HTTP2_PING_FAILURE_THRESHOLD"; private static final int DEFAULT_HTTP_RESPONSE_TIMEOUT_IN_SECONDS = 60; From 75cb99476a04af2647711ae95eb4b5258f04fd46 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Tue, 12 May 2026 18:25:47 -0400 Subject: [PATCH 09/62] =?UTF-8?q?Remove=20spec=20doc=20=E2=80=94=20design?= =?UTF-8?q?=20decisions=20documented=20in=20PR=20description?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../docs/http2-ping-keepalive-spec.md | 243 ------------------ 1 file changed, 243 deletions(-) delete mode 100644 sdk/cosmos/azure-cosmos/docs/http2-ping-keepalive-spec.md diff --git a/sdk/cosmos/azure-cosmos/docs/http2-ping-keepalive-spec.md b/sdk/cosmos/azure-cosmos/docs/http2-ping-keepalive-spec.md deleted file mode 100644 index 09d88f0ed28b..000000000000 --- a/sdk/cosmos/azure-cosmos/docs/http2-ping-keepalive-spec.md +++ /dev/null @@ -1,243 +0,0 @@ -# HTTP/2 PING Keepalive — Design Spec - -**Status:** Draft -**PR:** [#49095](https://github.com/Azure/azure-sdk-for-java/pull/49095) -**Date:** 2026-05-12 - ---- - -## 1. Problem Statement - -HTTP/2 connections between the Cosmos Java SDK and the Cosmos DB gateway traverse L7 middleboxes — NAT gateways, firewalls, Azure Load Balancers, corporate proxies. These middleboxes maintain connection tracking tables and silently reap idle connections after vendor-specific timeouts (often 4–5 minutes). When a reaped connection is reused, the client sees opaque TCP RST or timeout errors with no indication the connection was severed by a middlebox. - -The SDK's existing connection pool eviction (`evictionPredicate`) handles *local* idle detection but cannot prevent a middlebox from closing a connection that the SDK considers alive. - -## 2. Goal - -Send periodic HTTP/2 PING frames on idle parent (TCP) channels to: -1. **Keep middlebox connection tracking entries alive**, preventing silent connection reaping. -2. **Detect broken connections** — if no PING ACK within the configured timeout, close the connection so the pool creates a fresh replacement. - -This is aligned with the Rust SDK’s approach where hyper’s built-in PING kills connections on timeout and the shard health sweep replaces them. - -**Non-goals:** -- Replacing reactor-netty’s native `maxIdleTime` — the SDK uses a custom `evictionPredicate` that bypasses reactor-netty’s built-in idle handling. -- Application-layer keepalive (e.g., Cosmos heartbeat RPCs). -- Sharding — Rust requires per-endpoint sharding because hyper opens only 1 H2 connection per client. Reactor-netty natively pools multiple H2 connections, so sharding is unnecessary. - -## 3. Design - -### 3.1 Handler: `Http2PingHandler` - -A Netty `ChannelDuplexHandler` installed on the **parent (TCP) channel** of an HTTP/2 connection. It tracks activity on both inbound and outbound directions and sends a PING frame when the connection has been idle for longer than a configurable interval. - -#### State - -| Field | Type | Description | -|---|---|---| -| `pingIntervalNanos` | `long` | Configured idle threshold (immutable) | -| `pingTimeoutNanos` | `long` | ACK timeout — close connection if exceeded (immutable) | -| `lastActivityNanos` | `long` | `System.nanoTime()` of last read/write (event-loop-local, no sync) | -| `pingTask` | `ScheduledFuture` | Periodic check handle, cancelled on removal | -| `pingOutstandingSinceNanos` | `long` | `nanoTime()` when PING was sent; 0 = no outstanding PING | -| `pingsSent` | `AtomicInteger` | Monotonic counter — also used as PING frame payload | -| `pingAcksReceived` | `AtomicInteger` | Monotonic counter of received ACKs | -| `PING_HEALTH_DEGRADED` | `AttributeKey` | Per-channel flag: set when ACK missed, cleared on ACK | - -#### Lifecycle - -``` -handlerAdded() - └─ schedule periodic check (interval = max(500ms, pingInterval/2)) - -channelRead(frame) - ├─ update lastActivityNanos - ├─ if frame is Http2PingFrame(ack=true): - │ pingAcksReceived++, pingOutstandingSinceNanos=0 - │ clear PING_HEALTH_DEGRADED attribute (connection proved responsive) - └─ propagate downstream - -write(frame) - ├─ update lastActivityNanos - └─ propagate upstream - -maybeSendPing() ← periodic task - ├─ if channel inactive → cancel + return - ├─ if feature disabled → cancel + return - ├─ if pingOutstandingSinceNanos != 0: - │ if waited ≥ pingTimeout → set PING_HEALTH_DEGRADED, cancel task, ctx.close() - │ return (at most 1 in-flight PING) - ├─ if numActiveStreams > 0 → reset lastActivityNanos + return - │ (active request traffic is keepalive; PING would be noise) - ├─ idleNanos = now - lastActivityNanos - ├─ if idleNanos < pingIntervalNanos → return - └─ set pingOutstandingSinceNanos=now, send DefaultHttp2PingFrame(++pingsSent) - └─ on failure: pingOutstandingSinceNanos=0 (unblock next attempt) - └─ reset lastActivityNanos - -handlerRemoved() / channelInactive() - └─ cancel pingTask -``` - -#### PING concurrency and noisy-neighbour prevention - -**At-most-one outstanding PING.** `pingOutstandingSinceNanos` records when the PING was sent (0 = none outstanding). If a PING is already in flight, `maybeSendPing()` returns immediately — this prevents unbounded queuing if ACKs are delayed. - -**Broken connection detection.** When the periodic check finds that `pingOutstandingSinceNanos` is non-zero and the elapsed wait exceeds `pingTimeoutNanos`, the handler: -1. Sets the `PING_HEALTH_DEGRADED` channel attribute to `true` -2. Logs a WARN -3. Cancels the periodic task -4. Closes the connection via `ctx.close()` - -This is aligned with the Rust SDK where hyper kills connections on PING timeout. The connection pool will create a fresh replacement on the next request. The `PING_HEALTH_DEGRADED` attribute remains set for observability — external components can read `Http2PingHandler.isConnectionHealthDegraded(channel)` for diagnostic purposes. - -**Suppressed during active streams.** Before checking idle time, the handler queries `Http2FrameCodec.connection().numActiveStreams()`. If any streams are open (i.e., customer requests are in flight), the PING is skipped entirely — the request/response frames already keep the middlebox entry alive. The idle baseline is reset to `now` so that the full interval is measured from when all streams close, not from the last frame on a previous stream. - -Together these two guards ensure: -- At most 1 PING frame in flight at any time -- Zero PING frames while customer requests are active -- PING traffic only during true connection idleness - -#### Idle detection - -Any inbound frame (`channelRead`) or outbound write (`write`) resets the idle timer. The periodic check runs at `max(500ms, pingInterval / 2)` — this bounds the worst-case send delay to 1.5× the configured interval while avoiding excessive scheduling overhead. - -#### PING frame details - -- **Outbound:** `DefaultHttp2PingFrame(count)` where `count` = `pingsSent` counter (auto-increment). The 8-byte payload carries the sequence number for optional correlation. -- **Inbound ACK:** `Http2PingFrame` with `ack() == true`, counted and propagated. -- **No connection closure:** Missed ACKs cause the connection to be **closed** after the configured timeout. The handler sets `PING_HEALTH_DEGRADED` and calls `ctx.close()`. - -### 3.2 Installation - -Installed via `Http2PingHandler.installIfAbsent(Channel, int, int)`: - -``` -doOnConnected(connection) - ├─ Is HTTP/2? → check pipeline for Http2MultiplexHandler - ├─ Is enabled? → Configs.isHttp2PingHealthEnabled() - ├─ Interval > 0? → Configs.getHttp2PingIntervalInSeconds() - ├─ Timeout → Configs.getHttp2PingTimeoutInSeconds() - └─ installIfAbsent(channel, interval, timeout) - ├─ resolve parent channel (channel.parent() ?? channel) - ├─ idempotency check via PING_HANDLER_INSTALLED attribute - └─ parent.pipeline().addLast("cosmos.http2PingHandler", handler) -``` - -**Why parent channel?** HTTP/2 PING is a connection-level frame (RFC 9113 §6.7). Stream-level channels don't have access to the connection frame codec. The parent TCP channel's `Http2FrameCodec` encodes/decodes PING frames. - -**Idempotency:** `doOnConnected` fires once for the parent channel and once per stream. The `PING_HANDLER_INSTALLED` `AttributeKey` on the parent prevents duplicate installation. A race between concurrent stream channels is caught via `IllegalArgumentException` (benign). - -### 3.3 Integration point - -`ReactorNettyClient.java` → `httpClient.doOnConnected(...)` callback. The handler is installed after the HTTP/2 codec is in the pipeline but the detection is done via `Http2MultiplexHandler.class` presence in the pipeline (which is only present on H2 parent channels, not on HTTP/1.1 connections). - -### 3.4 Test hook - -The test (`Http2PingKeepaliveTest`) relies on the auto-installed handler and reads aggregate PING statistics via `Http2PingHandler.getGlobalPingsSent()` and `getGlobalPingAcksReceived()`. No interceptor or callback injection is required. - -## 4. Configuration - -| System Property | Default | Description | -|---|---|---| -| `COSMOS.HTTP2_PING_HEALTH_ENABLED` | `true` | Master switch. Set to `false` to disable PING keepalive. | -| `COSMOS.HTTP2_PING_INTERVAL_IN_SECONDS` | `1` | Idle threshold before sending a PING frame. Aligned with Rust SDK (1s). | -| `COSMOS.HTTP2_PING_TIMEOUT_IN_SECONDS` | `2` | ACK timeout. If no ACK within this, the connection is closed. Aligned with Rust SDK (2s). | - -**Aligned with Rust SDK:** Both SDKs use interval=1s, timeout=2s by default. Dead connection detected within ~3s worst-case. - -**No client builder API.** Configuration is JVM-wide via system properties, consistent with the existing `COSMOS.HTTP2_ENABLED` pattern. - -**Runtime disable:** The periodic task checks `Configs.isHttp2PingHealthEnabled()` on every tick. Setting the property to `false` at runtime causes the task to self-cancel on the next check. - -## 5. Threading Model - -- The handler runs entirely on Netty's channel event loop (single thread per channel). -- `lastActivityNanos` requires no synchronization — only accessed from the event loop thread. -- `pingsSent` / `pingAcksReceived` are `AtomicInteger` for safe cross-thread diagnostic reads (e.g., tests, metrics). -- `ScheduledFuture.cancel(false)` is non-interrupting — if the task is mid-execution, it completes and won't be rescheduled. - -## 6. Timing Characteristics - -| Parameter | Value (defaults) | -|---|---| -| PING interval | 1 second | -| PING ACK timeout | 2 seconds | -| Check frequency | 500 milliseconds (interval / 2) | -| Worst-case detection | ~3 seconds (idle just after a check + full timeout) | -| Minimum check frequency | 500 milliseconds (clamped) | - -For a default-configured idle connection: -``` -t=0s last activity -t=0.5s check → idle 0.5s < 1s → skip -t=1s check → idle 1s ≥ 1s → PING sent, idle reset -t=1.5s check → PING outstanding 0.5s < 2s → wait -t=2s check → PING outstanding 1s < 2s → wait -t=2.5s check → PING outstanding 1.5s < 2s → wait -t=3s check → PING outstanding 2s ≥ 2s → CLOSE (if no ACK) -…or at t=1.1s ACK arrives → pingOutstanding=0, PING_HEALTH_DEGRADED cleared -``` - -## 7. Failure Modes - -| Scenario | Behavior | -|---|---| -| PING ACK not received | After timeout: `PING_HEALTH_DEGRADED` set, task cancelled, connection **closed** via `ctx.close()`. Pool creates replacement on next request. | -| PING ACK arrives late | Before timeout: `PING_HEALTH_DEGRADED` cleared, `pingOutstandingSinceNanos` reset. Connection resumes normal keepalive. | -| Connection has active streams | PING suppressed — request traffic is keepalive. Idle baseline reset. | -| Channel becomes inactive | Periodic task cancelled in `channelInactive()`. | -| Feature disabled at runtime | Task self-cancels on next check. | -| Handler already installed (race) | `IllegalArgumentException` caught and ignored. | -| `ctx.writeAndFlush()` fails | WARN logged; idle timer reset to prevent tight retry loop. | -| HTTP/1.1 connection | Handler never installed (no `Http2MultiplexHandler` in pipeline). | - -## 8. Logging - -| Level | Event | -|---|---| -| DEBUG | Handler installed (interval, check frequency, channel ID) | -| DEBUG | PING frame sent successfully (sequence #, channel ID) | -| DEBUG | PING ACK received, degraded flag cleared (channel ID) | -| WARN | PING frame send failure (sequence #, channel ID, exception) | -| WARN | PING ACK not received within timeout — connection closed (channel ID) | - -## 9. Test Coverage - -**Test:** `Http2PingKeepaliveTest` (TestNG, group `manual-http-network-fault`) - -**Environment:** Docker with `--cap-add=NET_ADMIN`, real Cosmos DB account (thin-client-enabled). - -| Step | What's verified | -|---|---| -| 1. Create H2 client, seed data | Connection establishment | -| 2. Set interval to 3s, do a read | Handler installation on parent channel | -| 3. Sleep 20s (idle) | PINGs fire (~6-7 expected) | -| 4. Do another read | Connection survived idle period | -| 5. Assert `pingsSent > 0` | PING transmission works | -| 6. Assert `pingAcksReceived > 0` | Server acknowledges PINGs | - -## 10. Rust SDK Alignment - -| Dimension | Java | Rust | -|---|---|---| -| Mechanism | Custom Netty `ChannelDuplexHandler` | hyper’s built-in `http2_keep_alive_*` | -| PING interval | 1s (configurable) | 1s (configurable) | -| ACK timeout | 2s (configurable) | 2s (configurable) | -| On missed ACK | Close connection | hyper kills connection → shard eviction | -| PING while idle | Yes (suppressed when activeStreams > 0) | Yes (`http2_keep_alive_while_idle = true`) | -| In-flight limit | 1 (explicit gate) | hyper manages internally | -| Sharding | Not needed — reactor-netty pools multiple H2 connections per endpoint | Per-endpoint shard pools (needed because hyper opens only 1 H2 connection per client) | -| TCP keepalive | Not explicitly disabled alongside H2 PING | Mutually exclusive with H2 PING | - -**Deliberate differences:** - -- **Active-stream suppression** (Java-only) avoids sending PING frames during request bursts. Rust’s hyper handles this internally. -- **No sharding needed** — Rust requires per-endpoint sharding because hyper's HTTP/2 client opens only 1 connection per `reqwest::Client`. Java's reactor-netty natively supports pooling multiple H2 connections per endpoint via `Http2AllocationStrategy` (configurable `minConnections`/`maxConnections`/`maxConcurrentStreams`), so the PING handler is simply installed per-connection with no shard layer. - -## 11. Future Considerations - -- **Metrics integration:** Expose `pingsSent` / `pingAcksReceived` / `PING_HEALTH_DEGRADED` in `CosmosDiagnostics` for observability. -- **Client builder API:** Expose `http2PingIntervalInSeconds()` and `http2PingTimeoutInSeconds()` on `CosmosClientBuilder` for per-client configuration. -- **TCP keepalive mutual exclusion:** Explicitly disable TCP keepalive when HTTP/2 PING is active (aligned with Rust). -- **Adaptive interval:** Adjust PING interval based on observed middlebox behavior. From 331d9893db61a3485c8bf8c5c787895658a48e51 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Tue, 12 May 2026 18:35:42 -0400 Subject: [PATCH 10/62] Add sudo detection for CI VMs, PreSteps for iptables installation - Http2PingKeepaliveTest: detect root vs non-root user, prefix iptables commands with 'sudo' on CI VMs (no sudo when running as root in Docker) - Add execCommand() helper with error stream logging - tests.yml: add PreSteps to install iproute2 + iptables, load sch_netem kernel module on CI Ubuntu VMs (following PR #48420 pattern) - Matrix already restricts to ubuntu-only (no Windows jobs) --- .../Http2PingKeepaliveTest.java | 24 ++++++++++++++----- sdk/cosmos/tests.yml | 7 ++++++ 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java index 9c10b9642a74..c9178470a7e4 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java @@ -42,6 +42,9 @@ public class Http2PingKeepaliveTest extends FaultInjectionTestBase { private static final Logger logger = LoggerFactory.getLogger(Http2PingKeepaliveTest.class); private static final long TEST_TIMEOUT = 120_000; // 2 minutes + // sudo prefix: empty when running as root (Docker), "sudo " on CI VMs + private static final String SUDO = "root".equals(System.getProperty("user.name")) ? "" : "sudo "; + private CosmosAsyncClient client; private CosmosAsyncContainer cosmosAsyncContainer; private TestObject seedItem; @@ -199,9 +202,9 @@ public void connectionClosedOnPingTimeout() throws Exception { // Blackhole all traffic to gateway — prevents PING ACKs from arriving String iptablesRule = String.format( - "iptables -A OUTPUT -p tcp --dport %d -d %s -j DROP", gatewayPort, gatewayHost); + "%siptables -A OUTPUT -p tcp --dport %d -d %s -j DROP", SUDO, gatewayPort, gatewayHost); logger.info("Installing iptables DROP rule: {}", iptablesRule); - Runtime.getRuntime().exec(new String[]{"bash", "-c", iptablesRule}).waitFor(); + execCommand(iptablesRule); // Wait for PING timeout with consecutive failure threshold=2 (test override): // Round 1: 1s interval + 2s timeout = 3s (failure #1) @@ -212,9 +215,9 @@ public void connectionClosedOnPingTimeout() throws Exception { // Remove iptables rule BEFORE attempting recovery read String iptablesRemove = String.format( - "iptables -D OUTPUT -p tcp --dport %d -d %s -j DROP", gatewayPort, gatewayHost); + "%siptables -D OUTPUT -p tcp --dport %d -d %s -j DROP", SUDO, gatewayPort, gatewayHost); logger.info("Removing iptables DROP rule: {}", iptablesRemove); - Runtime.getRuntime().exec(new String[]{"bash", "-c", iptablesRemove}).waitFor(); + execCommand(iptablesRemove); // Small wait for network to stabilize Thread.sleep(1_000); @@ -240,8 +243,8 @@ public void connectionClosedOnPingTimeout() throws Exception { try { String gatewayHost = extractHostFromEndpoint(TestConfigurations.HOST); String cleanup = String.format( - "iptables -D OUTPUT -p tcp --dport 443 -d %s -j DROP 2>/dev/null", gatewayHost); - Runtime.getRuntime().exec(new String[]{"bash", "-c", cleanup}).waitFor(); + "%siptables -D OUTPUT -p tcp --dport 443 -d %s -j DROP 2>/dev/null", SUDO, gatewayHost); + execCommand(cleanup); } catch (Exception ignored) {} System.clearProperty("COSMOS.HTTP2_PING_INTERVAL_IN_SECONDS"); @@ -291,4 +294,13 @@ private static String extractHostFromEndpoint(String endpoint) { throw new RuntimeException("Failed to parse endpoint: " + endpoint, e); } } + + private static void execCommand(String command) throws Exception { + Process p = Runtime.getRuntime().exec(new String[]{"bash", "-c", command}); + int exit = p.waitFor(); + if (exit != 0) { + String err = new String(p.getErrorStream().readAllBytes()).trim(); + logger.warn("Command '{}' exited with code {}: {}", command, exit, err); + } + } } diff --git a/sdk/cosmos/tests.yml b/sdk/cosmos/tests.yml index 4f4d122cc553..1517d37f1f4f 100644 --- a/sdk/cosmos/tests.yml +++ b/sdk/cosmos/tests.yml @@ -70,6 +70,13 @@ extends: - template: /eng/pipelines/templates/stages/archetype-sdk-tests-isolated.yml parameters: TestName: 'Cosmos_Live_Test_Http2NetworkFault' + PreSteps: + - script: | + sudo apt-get update -qq && sudo apt-get install -y -qq iproute2 iptables + sudo modprobe sch_netem || true + echo "iptables $(sudo iptables --version 2>/dev/null || echo 'not found')" + echo "tc $(tc -Version 2>/dev/null || echo 'not found')" + displayName: 'Install iptables and tc for network fault injection' CloudConfig: Public: ServiceConnection: azure-sdk-tests-cosmos From 776d46ab2e773571437ddf6e034576472962b2b8 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Tue, 12 May 2026 18:37:12 -0400 Subject: [PATCH 11/62] Log levels: WARN only for connection close, INFO for retries and send failures --- .../azure/cosmos/implementation/http/Http2PingHandler.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java index 9eae4c7129b5..b5b204a2289a 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java @@ -155,7 +155,7 @@ private void maybeSendPing(ChannelHandlerContext ctx) { cancelPingTask(); ctx.close(); } else { - logger.warn("PING ACK timeout on channel {} (attempt {}/{}) — will retry", + logger.info("PING ACK timeout on channel {} (attempt {}/{}) \u2014 will retry", ctx.channel().id().asShortText(), consecutiveFailures, failureThreshold); } } @@ -186,7 +186,7 @@ private void maybeSendPing(ChannelHandlerContext ctx) { } } else { pingOutstandingSinceNanos = 0; // unblock next attempt on send failure - logger.warn("PING #{} failed on channel {}: {}", + logger.info("PING #{} send failed on channel {}: {}", count, ctx.channel().id().asShortText(), f.cause() != null ? f.cause().getMessage() : "unknown"); } From 9e81699e6356d608e9da86dcc9b6d95aef4abc52 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Tue, 12 May 2026 18:38:09 -0400 Subject: [PATCH 12/62] =?UTF-8?q?Remove=20unnecessary=20isDebugEnabled=20g?= =?UTF-8?q?uards=20=E2=80=94=20SLF4J=20parameterized=20logging=20handles?= =?UTF-8?q?=20this?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../implementation/http/Http2PingHandler.java | 22 +++++++------------ 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java index b5b204a2289a..be61aca46983 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java @@ -88,13 +88,11 @@ public void handlerAdded(ChannelHandlerContext ctx) { checkIntervalMs, TimeUnit.MILLISECONDS); - if (logger.isDebugEnabled()) { - logger.debug("Http2PingHandler installed on channel {}, interval={}s, timeout={}s, checkEvery={}ms", - ctx.channel().id().asShortText(), - TimeUnit.NANOSECONDS.toSeconds(pingIntervalNanos), - TimeUnit.NANOSECONDS.toSeconds(pingTimeoutNanos), - checkIntervalMs); - } + logger.debug("Http2PingHandler installed on channel {}, interval={}s, timeout={}s, checkEvery={}ms", + ctx.channel().id().asShortText(), + TimeUnit.NANOSECONDS.toSeconds(pingIntervalNanos), + TimeUnit.NANOSECONDS.toSeconds(pingTimeoutNanos), + checkIntervalMs); } @Override @@ -119,10 +117,8 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception // Connection proved responsive — clear degraded flag if it was set if (ctx.channel().hasAttr(PING_HEALTH_DEGRADED)) { ctx.channel().attr(PING_HEALTH_DEGRADED).set(null); - if (logger.isDebugEnabled()) { - logger.debug("PING ACK received, connection {} marked healthy", - ctx.channel().id().asShortText()); - } + logger.debug("PING ACK received, connection {} marked healthy", + ctx.channel().id().asShortText()); } } super.channelRead(ctx, msg); @@ -181,9 +177,7 @@ private void maybeSendPing(ChannelHandlerContext ctx) { ctx.writeAndFlush(new DefaultHttp2PingFrame(count)) .addListener(f -> { if (f.isSuccess()) { - if (logger.isDebugEnabled()) { - logger.debug("PING #{} sent on channel {}", count, ctx.channel().id().asShortText()); - } + logger.debug("PING #{} sent on channel {}", count, ctx.channel().id().asShortText()); } else { pingOutstandingSinceNanos = 0; // unblock next attempt on send failure logger.info("PING #{} send failed on channel {}: {}", From 22cdad5a9ac9f786e644fd3758e70f86e758a58f Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Tue, 12 May 2026 18:39:34 -0400 Subject: [PATCH 13/62] Single INFO log for connection close, everything else DEBUG --- .../azure/cosmos/implementation/http/Http2PingHandler.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java index be61aca46983..76188d036a90 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java @@ -146,12 +146,12 @@ private void maybeSendPing(ChannelHandlerContext ctx) { if (consecutiveFailures >= failureThreshold) { // Threshold reached — connection is broken. ctx.channel().attr(PING_HEALTH_DEGRADED).set(Boolean.TRUE); - logger.warn("PING ACK not received for {} consecutive attempts on channel {} — closing connection", + logger.info("PING ACK not received for {} consecutive attempts on channel {} \u2014 closing connection", consecutiveFailures, ctx.channel().id().asShortText()); cancelPingTask(); ctx.close(); } else { - logger.info("PING ACK timeout on channel {} (attempt {}/{}) \u2014 will retry", + logger.debug("PING ACK timeout on channel {} (attempt {}/{}) \u2014 will retry", ctx.channel().id().asShortText(), consecutiveFailures, failureThreshold); } } @@ -180,7 +180,7 @@ private void maybeSendPing(ChannelHandlerContext ctx) { logger.debug("PING #{} sent on channel {}", count, ctx.channel().id().asShortText()); } else { pingOutstandingSinceNanos = 0; // unblock next attempt on send failure - logger.info("PING #{} send failed on channel {}: {}", + logger.debug("PING #{} send failed on channel {}: {}", count, ctx.channel().id().asShortText(), f.cause() != null ? f.cause().getMessage() : "unknown"); } From 5ac7bb7a49792d95ef15fbde03b7a8a5fb037bd0 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Tue, 12 May 2026 18:49:51 -0400 Subject: [PATCH 14/62] Use Http2ConnectionConfig API for pool size, replace em dashes with -- - Tests use GatewayConnectionConfig.getHttp2ConnectionConfig() .setMaxConnectionPoolSize(1).setMinConnectionPoolSize(1) instead of system properties for pool size - Replace all unicode em dashes with plain -- in logs and comments --- .../Http2PingKeepaliveTest.java | 31 ++++++++++++------- .../implementation/http/Http2PingHandler.java | 16 +++++----- 2 files changed, 27 insertions(+), 20 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java index c9178470a7e4..4306ed0c8032 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java @@ -8,6 +8,8 @@ import com.azure.cosmos.CosmosClientBuilder; import com.azure.cosmos.CosmosDiagnostics; import com.azure.cosmos.ConsistencyLevel; +import com.azure.cosmos.GatewayConnectionConfig; +import com.azure.cosmos.Http2ConnectionConfig; import com.azure.cosmos.TestObject; import com.azure.cosmos.implementation.TestConfigurations; import com.azure.cosmos.implementation.http.Http2PingHandler; @@ -99,18 +101,22 @@ public void pingFramesSentAtConfiguredInterval() throws Exception { System.setProperty("COSMOS.HTTP2_PING_INTERVAL_IN_SECONDS", "1"); System.setProperty("COSMOS.HTTP2_PING_TIMEOUT_IN_SECONDS", "2"); System.setProperty("COSMOS.HTTP2_PING_HEALTH_ENABLED", "true"); - // Force single-connection pool so we can assert same connection is reused - System.setProperty("COSMOS.HTTP2_MAX_CONNECTION_POOL_SIZE", "1"); - System.setProperty("COSMOS.HTTP2_MIN_CONNECTION_POOL_SIZE", "1"); try { safeClose(this.client); + // Single-connection pool via Http2ConnectionConfig API so we can assert same connection is reused + GatewayConnectionConfig gwConfig = new GatewayConnectionConfig(); + gwConfig.getHttp2ConnectionConfig() + .setEnabled(true) + .setMaxConnectionPoolSize(1) + .setMinConnectionPoolSize(1); + this.client = new CosmosClientBuilder() .endpoint(TestConfigurations.HOST) .key(TestConfigurations.MASTER_KEY) .consistencyLevel(ConsistencyLevel.SESSION) - .gatewayMode() + .gatewayMode(gwConfig) .buildAsyncClient(); this.cosmosAsyncContainer = getSharedMultiPartitionCosmosContainerWithIdAsPartitionKey(this.client); @@ -155,8 +161,6 @@ public void pingFramesSentAtConfiguredInterval() throws Exception { System.clearProperty("COSMOS.HTTP2_PING_INTERVAL_IN_SECONDS"); System.clearProperty("COSMOS.HTTP2_PING_TIMEOUT_IN_SECONDS"); System.clearProperty("COSMOS.HTTP2_PING_HEALTH_ENABLED"); - System.clearProperty("COSMOS.HTTP2_MAX_CONNECTION_POOL_SIZE"); - System.clearProperty("COSMOS.HTTP2_MIN_CONNECTION_POOL_SIZE"); } } @@ -176,18 +180,22 @@ public void connectionClosedOnPingTimeout() throws Exception { System.setProperty("COSMOS.HTTP2_PING_HEALTH_ENABLED", "true"); // Override threshold to 2 for faster test (default=5 aligned with Rust SDK) System.setProperty("COSMOS.HTTP2_PING_FAILURE_THRESHOLD", "2"); - // Force single-connection pool so channel ID assertion is deterministic - System.setProperty("COSMOS.HTTP2_MAX_CONNECTION_POOL_SIZE", "1"); - System.setProperty("COSMOS.HTTP2_MIN_CONNECTION_POOL_SIZE", "1"); try { safeClose(this.client); + // Single-connection pool via Http2ConnectionConfig API + GatewayConnectionConfig gwConfig = new GatewayConnectionConfig(); + gwConfig.getHttp2ConnectionConfig() + .setEnabled(true) + .setMaxConnectionPoolSize(1) + .setMinConnectionPoolSize(1); + this.client = new CosmosClientBuilder() .endpoint(TestConfigurations.HOST) .key(TestConfigurations.MASTER_KEY) .consistencyLevel(ConsistencyLevel.SESSION) - .gatewayMode() + .gatewayMode(gwConfig) .buildAsyncClient(); this.cosmosAsyncContainer = getSharedMultiPartitionCosmosContainerWithIdAsPartitionKey(this.client); @@ -250,8 +258,7 @@ public void connectionClosedOnPingTimeout() throws Exception { System.clearProperty("COSMOS.HTTP2_PING_INTERVAL_IN_SECONDS"); System.clearProperty("COSMOS.HTTP2_PING_TIMEOUT_IN_SECONDS"); System.clearProperty("COSMOS.HTTP2_PING_HEALTH_ENABLED"); - System.clearProperty("COSMOS.HTTP2_MAX_CONNECTION_POOL_SIZE"); - System.clearProperty("COSMOS.HTTP2_MIN_CONNECTION_POOL_SIZE"); + System.clearProperty("COSMOS.HTTP2_PING_FAILURE_THRESHOLD"); } } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java index 76188d036a90..c95cb0823034 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java @@ -51,7 +51,7 @@ public class Http2PingHandler extends ChannelDuplexHandler { public static final AttributeKey PING_HEALTH_DEGRADED = AttributeKey.valueOf("cosmos.conn.pingHealthDegraded"); - // Global (process-wide) counters across all handler instances — used by tests + // Global (process-wide) counters across all handler instances -- used by tests private static final AtomicInteger globalPingsSent = new AtomicInteger(0); private static final AtomicInteger globalPingAcksReceived = new AtomicInteger(0); @@ -79,7 +79,7 @@ public Http2PingHandler(int pingIntervalSeconds, int pingTimeoutSeconds, int fai @Override public void handlerAdded(ChannelHandlerContext ctx) { - // Schedule periodic check — runs on the channel's event loop (single-threaded, no sync needed) + // Schedule periodic check -- runs on the channel's event loop (single-threaded, no sync needed) // Check at interval/2 (min 500ms) to bound worst-case PING send delay to ~1.5× interval. long checkIntervalMs = Math.max(500, TimeUnit.NANOSECONDS.toMillis(pingIntervalNanos) / 2); this.pingTask = ctx.executor().scheduleAtFixedRate( @@ -114,7 +114,7 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception globalPingAcksReceived.incrementAndGet(); pingOutstandingSinceNanos = 0; consecutiveFailures = 0; - // Connection proved responsive — clear degraded flag if it was set + // Connection proved responsive -- clear degraded flag if it was set if (ctx.channel().hasAttr(PING_HEALTH_DEGRADED)) { ctx.channel().attr(PING_HEALTH_DEGRADED).set(null); logger.debug("PING ACK received, connection {} marked healthy", @@ -144,14 +144,14 @@ private void maybeSendPing(ChannelHandlerContext ctx) { pingOutstandingSinceNanos = 0; // unblock next PING attempt if (consecutiveFailures >= failureThreshold) { - // Threshold reached — connection is broken. + // Threshold reached -- connection is broken. ctx.channel().attr(PING_HEALTH_DEGRADED).set(Boolean.TRUE); - logger.info("PING ACK not received for {} consecutive attempts on channel {} \u2014 closing connection", + logger.info("PING ACK not received for {} consecutive attempts on channel {} -- closing connection", consecutiveFailures, ctx.channel().id().asShortText()); cancelPingTask(); ctx.close(); } else { - logger.debug("PING ACK timeout on channel {} (attempt {}/{}) \u2014 will retry", + logger.debug("PING ACK timeout on channel {} (attempt {}/{}) -- will retry", ctx.channel().id().asShortText(), consecutiveFailures, failureThreshold); } } @@ -159,7 +159,7 @@ private void maybeSendPing(ChannelHandlerContext ctx) { return; } - // Don't send if there are active streams — request/response traffic is already + // Don't send if there are active streams -- request/response traffic is already // keeping the connection alive, so a PING would be pure noise-neighbour overhead. Http2FrameCodec codec = ctx.pipeline().get(Http2FrameCodec.class); if (codec != null && codec.connection().numActiveStreams() > 0) { @@ -248,7 +248,7 @@ public static void installIfAbsent(Channel channel, int pingIntervalSeconds, int try { parent.pipeline().addLast(HANDLER_NAME, new Http2PingHandler(pingIntervalSeconds, pingTimeoutSeconds, failureThreshold)); } catch (IllegalArgumentException ignored) { - // Duplicate — race between concurrent streams, benign + // Duplicate -- race between concurrent streams, benign } } } From 2d23a2b2c7ba244c9bde1cae887fb66edf649dd9 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Tue, 12 May 2026 18:50:51 -0400 Subject: [PATCH 15/62] Fix readAllBytes() compile error -- use BufferedReader for JDK 8 compat --- .../cosmos/faultinjection/Http2PingKeepaliveTest.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java index 4306ed0c8032..34c924167007 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java @@ -306,8 +306,14 @@ private static void execCommand(String command) throws Exception { Process p = Runtime.getRuntime().exec(new String[]{"bash", "-c", command}); int exit = p.waitFor(); if (exit != 0) { - String err = new String(p.getErrorStream().readAllBytes()).trim(); - logger.warn("Command '{}' exited with code {}: {}", command, exit, err); + java.io.BufferedReader reader = new java.io.BufferedReader( + new java.io.InputStreamReader(p.getErrorStream())); + StringBuilder sb = new StringBuilder(); + String line; + while ((line = reader.readLine()) != null) { + sb.append(line); + } + logger.warn("Command '{}' exited with code {}: {}", command, exit, sb.toString().trim()); } } } From 36d940e6295a6153241db56cbf659d0994276b21 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Tue, 12 May 2026 19:06:29 -0400 Subject: [PATCH 16/62] Deep review cleanup: remove unused Http2ConnectionConfig import, replace all em dashes --- .../Http2PingKeepaliveTest.java | 19 +++++++++---------- .../http/ReactorNettyClient.java | 6 +++--- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java index 34c924167007..2458f96e7458 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java @@ -9,7 +9,6 @@ import com.azure.cosmos.CosmosDiagnostics; import com.azure.cosmos.ConsistencyLevel; import com.azure.cosmos.GatewayConnectionConfig; -import com.azure.cosmos.Http2ConnectionConfig; import com.azure.cosmos.TestObject; import com.azure.cosmos.implementation.TestConfigurations; import com.azure.cosmos.implementation.http.Http2PingHandler; @@ -37,7 +36,7 @@ *

* Run in Docker with --cap-add=NET_ADMIN (group: manual-http-network-fault). * Requires: HTTP/2 capable Cosmos DB account. - * Does NOT require thin client — only COSMOS.HTTP2_ENABLED=true. + * Does NOT require thin client -- only COSMOS.HTTP2_ENABLED=true. */ public class Http2PingKeepaliveTest extends FaultInjectionTestBase { @@ -61,7 +60,7 @@ public Http2PingKeepaliveTest() { @BeforeClass(groups = {"manual-http-network-fault"}, timeOut = 120_000) public void beforeClass() { - // Enable HTTP/2 — thin client is NOT required for PING tests + // Enable HTTP/2 -- thin client is NOT required for PING tests System.setProperty("COSMOS.HTTP2_ENABLED", "true"); this.client = getClientBuilder().buildAsyncClient(); @@ -124,18 +123,18 @@ public void pingFramesSentAtConfiguredInterval() throws Exception { int baselineSent = Http2PingHandler.getGlobalPingsSent(); int baselineAcks = Http2PingHandler.getGlobalPingAcksReceived(); - // Warm-up read — triggers H2 connection + PING handler installation + // Warm-up read -- triggers H2 connection + PING handler installation String initialChannelId = readAndGetParentChannelId(); logger.info("Initial parentChannelId: {}", initialChannelId); - // Let the connection go idle for 10s — with 1s interval, expect ~10 PINGs + // Let the connection go idle for 10s -- with 1s interval, expect ~10 PINGs logger.info("Waiting 10s for PING frames on idle connection (interval=1s)..."); Thread.sleep(10_000); int sentCount = Http2PingHandler.getGlobalPingsSent() - baselineSent; int ackCount = Http2PingHandler.getGlobalPingAcksReceived() - baselineAcks; - // Recovery read — proves connection survived idle period + // Recovery read -- proves connection survived idle period String recoveryChannelId = readAndGetParentChannelId(); logger.info("RESULT: initial={}, recovery={}, SAME={}, pingsSent={}, pingAcksReceived={}", @@ -199,7 +198,7 @@ public void connectionClosedOnPingTimeout() throws Exception { .buildAsyncClient(); this.cosmosAsyncContainer = getSharedMultiPartitionCosmosContainerWithIdAsPartitionKey(this.client); - // Warm-up read — establish H2 connection + // Warm-up read -- establish H2 connection String initialChannelId = readAndGetParentChannelId(); logger.info("Initial parentChannelId: {}", initialChannelId); @@ -208,7 +207,7 @@ public void connectionClosedOnPingTimeout() throws Exception { int gatewayPort = 443; logger.info("Gateway host: {}, port: {}", gatewayHost, gatewayPort); - // Blackhole all traffic to gateway — prevents PING ACKs from arriving + // Blackhole all traffic to gateway -- prevents PING ACKs from arriving String iptablesRule = String.format( "%siptables -A OUTPUT -p tcp --dport %d -d %s -j DROP", SUDO, gatewayPort, gatewayHost); logger.info("Installing iptables DROP rule: {}", iptablesRule); @@ -230,7 +229,7 @@ public void connectionClosedOnPingTimeout() throws Exception { // Small wait for network to stabilize Thread.sleep(1_000); - // Recovery read — should succeed on a NEW connection + // Recovery read -- should succeed on a NEW connection String recoveryChannelId = readAndGetParentChannelId(); logger.info("Recovery parentChannelId: {}", recoveryChannelId); @@ -238,7 +237,7 @@ public void connectionClosedOnPingTimeout() throws Exception { initialChannelId, recoveryChannelId, !initialChannelId.equals(recoveryChannelId)); - // The connection MUST be different — the old one was closed by PING timeout + // The connection MUST be different -- the old one was closed by PING timeout assertThat(recoveryChannelId) .as("After PING timeout, the handler should have closed the connection. " + "The recovery request must use a new connection.") diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java index 588544a44301..5ddd81848ca6 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java @@ -147,11 +147,11 @@ private void configureChannelPipelineHandlers() { boolean isH2Enabled = http2CfgAccessor.isEffectivelyEnabled(http2Cfg); this.httpClient = this.httpClient.doOnConnected(connection -> { - // Manual HTTP/2 PING keepalive — sends PING frames when the connection is idle + // Manual HTTP/2 PING keepalive -- sends PING frames when the connection is idle // to prevent L7 middleboxes (NAT, firewalls, LBs) from reaping the connection. // For H2, the first doOnConnected fires for the parent TCP channel (parent()==null), // and the second fires for stream channels (parent()!=null). - // We install on the parent channel — detect it via Http2MultiplexHandler in the pipeline. + // We install on the parent channel -- detect it via Http2MultiplexHandler in the pipeline. if (Configs.isHttp2PingHealthEnabled() && connection.channel().pipeline().get(Http2MultiplexHandler.class) != null) { int pingIntervalSeconds = Configs.getHttp2PingIntervalInSeconds(); @@ -229,7 +229,7 @@ public Mono send(final HttpRequest request, Duration responseTimeo final AtomicReference responseReference = new AtomicReference<>(); // Per-request CONNECT_TIMEOUT_MILLIS via reactor-netty's immutable HttpClient. - // .option() returns a new config snapshot — does NOT mutate the shared httpClient. + // .option() returns a new config snapshot -- does NOT mutate the shared httpClient. // Thin client requests (isThinClientRequest=true): connect timeout is configured via // HttpClientConfig.getThinClientConnectTimeoutMs() (default 5s) to fail fast. // Standard gateway requests: 45s (default). From f77cdf554e56fae8f7acfafbd332dca591b54e31 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Tue, 12 May 2026 19:37:13 -0400 Subject: [PATCH 17/62] Replace global counters with channel attribute for handler ref, assert connection identity only - Remove global AtomicInteger counters and ConcurrentHashMap registry - Store handler ref as PING_HANDLER_REF channel attribute (lifecycle-bound) - Add Http2PingHandler.getFrom(Channel) for programmatic access - Test asserts connection identity (same/different) not PING counts - Per-instance counters (getPingsSent/getPingAcksReceived) retained for programmatic access via getFrom() - Docker validated: Tests run: 2, Failures: 0, Errors: 0 --- .../Http2PingKeepaliveTest.java | 24 +++------------- .../implementation/http/Http2PingHandler.java | 28 +++++++++---------- 2 files changed, 17 insertions(+), 35 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java index 2458f96e7458..a5670f6e24ea 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java @@ -11,7 +11,6 @@ import com.azure.cosmos.GatewayConnectionConfig; import com.azure.cosmos.TestObject; import com.azure.cosmos.implementation.TestConfigurations; -import com.azure.cosmos.implementation.http.Http2PingHandler; import com.azure.cosmos.models.CosmosItemResponse; import com.azure.cosmos.models.PartitionKey; import org.slf4j.Logger; @@ -119,10 +118,6 @@ public void pingFramesSentAtConfiguredInterval() throws Exception { .buildAsyncClient(); this.cosmosAsyncContainer = getSharedMultiPartitionCosmosContainerWithIdAsPartitionKey(this.client); - // Reset global counters before measurement - int baselineSent = Http2PingHandler.getGlobalPingsSent(); - int baselineAcks = Http2PingHandler.getGlobalPingAcksReceived(); - // Warm-up read -- triggers H2 connection + PING handler installation String initialChannelId = readAndGetParentChannelId(); logger.info("Initial parentChannelId: {}", initialChannelId); @@ -131,31 +126,20 @@ public void pingFramesSentAtConfiguredInterval() throws Exception { logger.info("Waiting 10s for PING frames on idle connection (interval=1s)..."); Thread.sleep(10_000); - int sentCount = Http2PingHandler.getGlobalPingsSent() - baselineSent; - int ackCount = Http2PingHandler.getGlobalPingAcksReceived() - baselineAcks; - // Recovery read -- proves connection survived idle period String recoveryChannelId = readAndGetParentChannelId(); - logger.info("RESULT: initial={}, recovery={}, SAME={}, pingsSent={}, pingAcksReceived={}", + logger.info("RESULT: initial={}, recovery={}, SAME={}", initialChannelId, recoveryChannelId, - initialChannelId.equals(recoveryChannelId), sentCount, ackCount); - - // With 1s interval and 10s idle, we expect at least 5 PINGs (conservative bound) - assertThat(sentCount) - .as("With 1s interval over 10s idle, expect at least 5 PINGs sent") - .isGreaterThanOrEqualTo(5); - - assertThat(ackCount) - .as("All sent PINGs should be ACKed by the server") - .isGreaterThanOrEqualTo(5); + initialChannelId.equals(recoveryChannelId)); // With pool size=1, the same connection MUST be reused (PING kept it alive) + // If PINGs weren't flowing, the pool's maxIdleTime would evict the connection assertThat(recoveryChannelId) .as("With single-connection pool, PING keepalive must preserve the connection") .isEqualTo(initialChannelId); - logger.info("PING interval test passed: {} PINGs sent, {} ACKs received in 10s", sentCount, ackCount); + logger.info("PING interval test passed: connection survived 10s idle period"); } finally { System.clearProperty("COSMOS.HTTP2_PING_INTERVAL_IN_SECONDS"); System.clearProperty("COSMOS.HTTP2_PING_TIMEOUT_IN_SECONDS"); diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java index c95cb0823034..9407933e6e06 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java @@ -51,9 +51,12 @@ public class Http2PingHandler extends ChannelDuplexHandler { public static final AttributeKey PING_HEALTH_DEGRADED = AttributeKey.valueOf("cosmos.conn.pingHealthDegraded"); - // Global (process-wide) counters across all handler instances -- used by tests - private static final AtomicInteger globalPingsSent = new AtomicInteger(0); - private static final AtomicInteger globalPingAcksReceived = new AtomicInteger(0); + /** + * Channel attribute holding the handler instance reference. + * Allows retrieval of per-connection PING counters without global state. + */ + static final AttributeKey PING_HANDLER_REF = + AttributeKey.valueOf("cosmos.conn.pingHandlerRef"); private final long pingIntervalNanos; private final long pingTimeoutNanos; @@ -93,6 +96,8 @@ public void handlerAdded(ChannelHandlerContext ctx) { TimeUnit.NANOSECONDS.toSeconds(pingIntervalNanos), TimeUnit.NANOSECONDS.toSeconds(pingTimeoutNanos), checkIntervalMs); + + ctx.channel().attr(PING_HANDLER_REF).set(this); } @Override @@ -111,7 +116,6 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception lastActivityNanos = System.nanoTime(); if (msg instanceof Http2PingFrame && ((Http2PingFrame) msg).ack()) { pingAcksReceived.incrementAndGet(); - globalPingAcksReceived.incrementAndGet(); pingOutstandingSinceNanos = 0; consecutiveFailures = 0; // Connection proved responsive -- clear degraded flag if it was set @@ -172,7 +176,6 @@ private void maybeSendPing(ChannelHandlerContext ctx) { long idleNanos = System.nanoTime() - lastActivityNanos; if (idleNanos >= pingIntervalNanos) { int count = pingsSent.incrementAndGet(); - globalPingsSent.incrementAndGet(); pingOutstandingSinceNanos = System.nanoTime(); ctx.writeAndFlush(new DefaultHttp2PingFrame(count)) .addListener(f -> { @@ -206,17 +209,12 @@ public int getPingAcksReceived() { } /** - * Returns the total number of PINGs sent across all handler instances (process-wide). - */ - public static int getGlobalPingsSent() { - return globalPingsSent.get(); - } - - /** - * Returns the total number of PING ACKs received across all handler instances (process-wide). + * Retrieves the handler instance from the given channel (or its parent). + * Returns null if no handler is installed. */ - public static int getGlobalPingAcksReceived() { - return globalPingAcksReceived.get(); + public static Http2PingHandler getFrom(Channel channel) { + Channel parent = channel.parent() != null ? channel.parent() : channel; + return parent.hasAttr(PING_HANDLER_REF) ? parent.attr(PING_HANDLER_REF).get() : null; } /** From 50c773c9695b0f8fa354e764f97ece36be3c95c8 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Tue, 12 May 2026 19:42:17 -0400 Subject: [PATCH 18/62] Add CHANGELOG entry for HTTP/2 PING keepalive --- sdk/cosmos/azure-cosmos/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/cosmos/azure-cosmos/CHANGELOG.md b/sdk/cosmos/azure-cosmos/CHANGELOG.md index 6eb1b76bad4e..1ac1daf17a9d 100644 --- a/sdk/cosmos/azure-cosmos/CHANGELOG.md +++ b/sdk/cosmos/azure-cosmos/CHANGELOG.md @@ -9,6 +9,7 @@ #### Bugs Fixed #### Other Changes +* Added HTTP/2 PING keepalive handler to detect and close broken idle connections, preventing silent middlebox reaping. - See [PR 49095](https://github.com/Azure/azure-sdk-for-java/pull/49095) * Replaced per-client `Schedulers.newSingle()` schedulers in `GlobalEndpointManager` and `GlobalPartitionEndpointManagerForPerPartitionCircuitBreaker` with shared `BoundedElastic` schedulers in `CosmosSchedulers` to prevent thread count from scaling linearly with client/tenant count. - See [PR 49062](https://github.com/Azure/azure-sdk-for-java/pull/49062) ### 4.80.0 (2026-05-01) From 15b1ed23e20775a061f6cb439f013be4aa26dd7d Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Tue, 12 May 2026 19:43:12 -0400 Subject: [PATCH 19/62] Simplify CHANGELOG wording --- sdk/cosmos/azure-cosmos/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/cosmos/azure-cosmos/CHANGELOG.md b/sdk/cosmos/azure-cosmos/CHANGELOG.md index 1ac1daf17a9d..80ab11e3413f 100644 --- a/sdk/cosmos/azure-cosmos/CHANGELOG.md +++ b/sdk/cosmos/azure-cosmos/CHANGELOG.md @@ -9,7 +9,7 @@ #### Bugs Fixed #### Other Changes -* Added HTTP/2 PING keepalive handler to detect and close broken idle connections, preventing silent middlebox reaping. - See [PR 49095](https://github.com/Azure/azure-sdk-for-java/pull/49095) +* Added HTTP/2 PING keepalive handler to proactively detect and close broken idle connections. - See [PR 49095](https://github.com/Azure/azure-sdk-for-java/pull/49095) * Replaced per-client `Schedulers.newSingle()` schedulers in `GlobalEndpointManager` and `GlobalPartitionEndpointManagerForPerPartitionCircuitBreaker` with shared `BoundedElastic` schedulers in `CosmosSchedulers` to prevent thread count from scaling linearly with client/tenant count. - See [PR 49062](https://github.com/Azure/azure-sdk-for-java/pull/49062) ### 4.80.0 (2026-05-01) From 6e76b9f3bdd5408824d040209420c2d8b9808dcc Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Tue, 12 May 2026 19:46:14 -0400 Subject: [PATCH 20/62] Remove unused PING_HANDLER_REF attribute and getFrom() method --- .../implementation/http/Http2PingHandler.java | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java index 9407933e6e06..5928f09d46d2 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java @@ -51,13 +51,6 @@ public class Http2PingHandler extends ChannelDuplexHandler { public static final AttributeKey PING_HEALTH_DEGRADED = AttributeKey.valueOf("cosmos.conn.pingHealthDegraded"); - /** - * Channel attribute holding the handler instance reference. - * Allows retrieval of per-connection PING counters without global state. - */ - static final AttributeKey PING_HANDLER_REF = - AttributeKey.valueOf("cosmos.conn.pingHandlerRef"); - private final long pingIntervalNanos; private final long pingTimeoutNanos; private final int failureThreshold; @@ -96,8 +89,6 @@ public void handlerAdded(ChannelHandlerContext ctx) { TimeUnit.NANOSECONDS.toSeconds(pingIntervalNanos), TimeUnit.NANOSECONDS.toSeconds(pingTimeoutNanos), checkIntervalMs); - - ctx.channel().attr(PING_HANDLER_REF).set(this); } @Override @@ -208,15 +199,6 @@ public int getPingAcksReceived() { return pingAcksReceived.get(); } - /** - * Retrieves the handler instance from the given channel (or its parent). - * Returns null if no handler is installed. - */ - public static Http2PingHandler getFrom(Channel channel) { - Channel parent = channel.parent() != null ? channel.parent() : channel; - return parent.hasAttr(PING_HANDLER_REF) ? parent.attr(PING_HANDLER_REF).get() : null; - } - /** * Returns {@code true} if the given channel (or its parent) has been marked as * unhealthy due to a missed PING ACK. From 5760aa611fbd7f6a298e1fbe8589618cb2da6dfa Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Tue, 12 May 2026 20:05:25 -0400 Subject: [PATCH 21/62] Address Copilot review comments 1. extractParentChannelId: throw AssertionError instead of returning placeholder -- prevents false-positive test passes 2. Http2PingHandler: add thread-safety comment on write-future listener (runs on same event loop as scheduled task) 3. ReactorNettyClient: move doOnConnected PING registration inside if (isH2Enabled) branch -- avoids per-connect overhead for H1 clients Docker validated: Tests run: 2, Failures: 0, Errors: 0 --- .../Http2PingKeepaliveTest.java | 24 +++++--------- .../implementation/http/Http2PingHandler.java | 2 ++ .../http/ReactorNettyClient.java | 33 ++++++++++--------- 3 files changed, 28 insertions(+), 31 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java index a5670f6e24ea..6d9ccd02b249 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java @@ -256,23 +256,17 @@ private String readAndGetParentChannelId() { } private String extractParentChannelId(CosmosDiagnostics diagnostics) { - try { - String diagStr = diagnostics.toString(); - int idx = diagStr.indexOf("parentChannelId"); - if (idx > 0) { - int start = diagStr.indexOf("\"", idx + 16) + 1; - int end = diagStr.indexOf("\"", start); - if (start > 0 && end > start) { - return diagStr.substring(start, end); - } + String diagStr = diagnostics.toString(); + int idx = diagStr.indexOf("parentChannelId"); + if (idx > 0) { + int start = diagStr.indexOf("\"", idx + 16) + 1; + int end = diagStr.indexOf("\"", start); + if (start > 0 && end > start) { + return diagStr.substring(start, end); } - - logger.warn("Could not extract parentChannelId from diagnostics"); - return "unknown-" + UUID.randomUUID().toString().substring(0, 8); - } catch (Exception e) { - logger.warn("Error extracting parentChannelId", e); - return "error-" + UUID.randomUUID().toString().substring(0, 8); } + + throw new AssertionError("Could not extract parentChannelId from diagnostics: " + diagStr); } private static String extractHostFromEndpoint(String endpoint) { diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java index 5928f09d46d2..900b2ed6907a 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java @@ -173,6 +173,8 @@ private void maybeSendPing(ChannelHandlerContext ctx) { if (f.isSuccess()) { logger.debug("PING #{} sent on channel {}", count, ctx.channel().id().asShortText()); } else { + // Listener runs on the same event loop as the scheduled task, + // so mutating pingOutstandingSinceNanos is thread-safe. pingOutstandingSinceNanos = 0; // unblock next attempt on send failure logger.debug("PING #{} send failed on channel {}: {}", count, ctx.channel().id().asShortText(), diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java index 5ddd81848ca6..ad01cf0927b3 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java @@ -146,24 +146,25 @@ private void configureChannelPipelineHandlers() { Http2ConnectionConfig http2Cfg = httpClientConfig.getHttp2ConnectionConfig(); boolean isH2Enabled = http2CfgAccessor.isEffectivelyEnabled(http2Cfg); - this.httpClient = this.httpClient.doOnConnected(connection -> { - // Manual HTTP/2 PING keepalive -- sends PING frames when the connection is idle - // to prevent L7 middleboxes (NAT, firewalls, LBs) from reaping the connection. - // For H2, the first doOnConnected fires for the parent TCP channel (parent()==null), - // and the second fires for stream channels (parent()!=null). - // We install on the parent channel -- detect it via Http2MultiplexHandler in the pipeline. - if (Configs.isHttp2PingHealthEnabled() - && connection.channel().pipeline().get(Http2MultiplexHandler.class) != null) { - int pingIntervalSeconds = Configs.getHttp2PingIntervalInSeconds(); - int pingTimeoutSeconds = Configs.getHttp2PingTimeoutInSeconds(); - int pingFailureThreshold = Configs.getHttp2PingFailureThreshold(); - if (pingIntervalSeconds > 0) { - Http2PingHandler.installIfAbsent(connection.channel(), pingIntervalSeconds, pingTimeoutSeconds, pingFailureThreshold); - } - } - }); if (isH2Enabled) { + this.httpClient = this.httpClient.doOnConnected(connection -> { + // Manual HTTP/2 PING keepalive -- sends PING frames when the connection is idle + // to prevent L7 middleboxes (NAT, firewalls, LBs) from reaping the connection. + // For H2, the first doOnConnected fires for the parent TCP channel (parent()==null), + // and the second fires for stream channels (parent()!=null). + // We install on the parent channel -- detect it via Http2MultiplexHandler in the pipeline. + if (Configs.isHttp2PingHealthEnabled() + && connection.channel().pipeline().get(Http2MultiplexHandler.class) != null) { + int pingIntervalSeconds = Configs.getHttp2PingIntervalInSeconds(); + int pingTimeoutSeconds = Configs.getHttp2PingTimeoutInSeconds(); + int pingFailureThreshold = Configs.getHttp2PingFailureThreshold(); + if (pingIntervalSeconds > 0) { + Http2PingHandler.installIfAbsent(connection.channel(), pingIntervalSeconds, pingTimeoutSeconds, pingFailureThreshold); + } + } + }); + this.httpClient = this.httpClient .secure(sslContextSpec -> sslContextSpec.sslContext( From 74b33ef84d18849d6b6576612ebf9e4f74a4c6de Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Tue, 12 May 2026 21:01:59 -0400 Subject: [PATCH 22/62] Address xinlian12 review comments (4-8) 4. Test comment updated -- 10s idle < 60s maxIdleTime, test validates PING frames flow (connection reuse) not eviction prevention 5. Remove PING_HEALTH_DEGRADED attribute + isConnectionHealthDegraded() -- no consumer exists, attribute set right before ctx.close() 6. Use getJVMConfigAsBoolean() for isHttp2PingHealthEnabled() -- consistent with 7+ sibling boolean configs 7. Remove sch_netem/iproute2 from CI PreSteps -- tests only use iptables 8. Use pipeline name guard (get(HANDLER_NAME)==null) instead of AttributeKey -- consistent with Http2ParentChannelExceptionHandler Docker validated: Tests run: 2, Failures: 0, Errors: 0 --- .../Http2PingKeepaliveTest.java | 10 +++--- .../azure/cosmos/implementation/Configs.java | 6 +--- .../implementation/http/Http2PingHandler.java | 35 +------------------ sdk/cosmos/tests.yml | 8 ++--- 4 files changed, 11 insertions(+), 48 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java index 6d9ccd02b249..e7d72a22fdfa 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java @@ -103,7 +103,10 @@ public void pingFramesSentAtConfiguredInterval() throws Exception { try { safeClose(this.client); - // Single-connection pool via Http2ConnectionConfig API so we can assert same connection is reused + // Single-connection pool so we can assert same connection is reused. + // maxIdleTime defaults to 60s; 10s idle period is well under that threshold. + // PINGs keep middlebox entries alive (not tested here -- tested by test 2). + // This test validates PING frames are actively flowing on idle connections. GatewayConnectionConfig gwConfig = new GatewayConnectionConfig(); gwConfig.getHttp2ConnectionConfig() .setEnabled(true) @@ -133,10 +136,9 @@ public void pingFramesSentAtConfiguredInterval() throws Exception { initialChannelId, recoveryChannelId, initialChannelId.equals(recoveryChannelId)); - // With pool size=1, the same connection MUST be reused (PING kept it alive) - // If PINGs weren't flowing, the pool's maxIdleTime would evict the connection + // With pool size=1, same connection confirms it survived the idle period assertThat(recoveryChannelId) - .as("With single-connection pool, PING keepalive must preserve the connection") + .as("Connection should be reused -- PING frames flowed during idle period") .isEqualTo(initialChannelId); logger.info("PING interval test passed: connection survived 10s idle period"); diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java index b18fae02ca05..186d5b12812c 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java @@ -760,11 +760,7 @@ public static int getDefaultHttpPoolSize() { } public static boolean isHttp2PingHealthEnabled() { - String value = System.getProperty(HTTP2_PING_HEALTH_ENABLED); - if (value != null && !value.isEmpty()) { - return Boolean.parseBoolean(value); - } - return DEFAULT_HTTP2_PING_HEALTH_ENABLED; + return getJVMConfigAsBoolean(HTTP2_PING_HEALTH_ENABLED, DEFAULT_HTTP2_PING_HEALTH_ENABLED); } public static int getHttp2PingIntervalInSeconds() { diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java index 900b2ed6907a..c764085898eb 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java @@ -10,7 +10,6 @@ import io.netty.handler.codec.http2.DefaultHttp2PingFrame; import io.netty.handler.codec.http2.Http2FrameCodec; import io.netty.handler.codec.http2.Http2PingFrame; -import io.netty.util.AttributeKey; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,17 +39,6 @@ public class Http2PingHandler extends ChannelDuplexHandler { private static final String HANDLER_NAME = "cosmos.http2PingHandler"; - static final AttributeKey PING_HANDLER_INSTALLED = - AttributeKey.valueOf("cosmos.conn.pingHandlerInstalled"); - - /** - * Channel attribute set to {@code true} when a PING ACK is not received before the - * next PING would be due. Cleared when a successful ACK arrives. External components - * (eviction predicates, health checks) can read this attribute to detect broken connections. - */ - public static final AttributeKey PING_HEALTH_DEGRADED = - AttributeKey.valueOf("cosmos.conn.pingHealthDegraded"); - private final long pingIntervalNanos; private final long pingTimeoutNanos; private final int failureThreshold; @@ -109,12 +97,6 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception pingAcksReceived.incrementAndGet(); pingOutstandingSinceNanos = 0; consecutiveFailures = 0; - // Connection proved responsive -- clear degraded flag if it was set - if (ctx.channel().hasAttr(PING_HEALTH_DEGRADED)) { - ctx.channel().attr(PING_HEALTH_DEGRADED).set(null); - logger.debug("PING ACK received, connection {} marked healthy", - ctx.channel().id().asShortText()); - } } super.channelRead(ctx, msg); } @@ -140,7 +122,6 @@ private void maybeSendPing(ChannelHandlerContext ctx) { if (consecutiveFailures >= failureThreshold) { // Threshold reached -- connection is broken. - ctx.channel().attr(PING_HEALTH_DEGRADED).set(Boolean.TRUE); logger.info("PING ACK not received for {} consecutive attempts on channel {} -- closing connection", consecutiveFailures, ctx.channel().id().asShortText()); cancelPingTask(); @@ -201,17 +182,6 @@ public int getPingAcksReceived() { return pingAcksReceived.get(); } - /** - * Returns {@code true} if the given channel (or its parent) has been marked as - * unhealthy due to a missed PING ACK. - */ - public static boolean isConnectionHealthDegraded(Channel channel) { - Channel parent = channel.parent() != null ? channel.parent() : channel; - return Boolean.TRUE.equals(parent.hasAttr(PING_HEALTH_DEGRADED) - ? parent.attr(PING_HEALTH_DEGRADED).get() - : null); - } - /** * Installs the PING handler on the parent H2 channel if not already installed. * Safe to call from doOnConnected (which fires per-stream for H2). @@ -222,11 +192,8 @@ public static boolean isConnectionHealthDegraded(Channel channel) { * @param failureThreshold consecutive timeouts before closing */ public static void installIfAbsent(Channel channel, int pingIntervalSeconds, int pingTimeoutSeconds, int failureThreshold) { - // When called from the first doOnConnected, channel IS the parent H2 channel. - // When called from a stream doOnConnected, channel.parent() is the parent. Channel parent = channel.parent() != null ? channel.parent() : channel; - if (!parent.hasAttr(PING_HANDLER_INSTALLED)) { - parent.attr(PING_HANDLER_INSTALLED).set(Boolean.TRUE); + if (parent.pipeline().get(HANDLER_NAME) == null) { try { parent.pipeline().addLast(HANDLER_NAME, new Http2PingHandler(pingIntervalSeconds, pingTimeoutSeconds, failureThreshold)); } catch (IllegalArgumentException ignored) { diff --git a/sdk/cosmos/tests.yml b/sdk/cosmos/tests.yml index 1517d37f1f4f..bd00d2ac9048 100644 --- a/sdk/cosmos/tests.yml +++ b/sdk/cosmos/tests.yml @@ -72,11 +72,9 @@ extends: TestName: 'Cosmos_Live_Test_Http2NetworkFault' PreSteps: - script: | - sudo apt-get update -qq && sudo apt-get install -y -qq iproute2 iptables - sudo modprobe sch_netem || true - echo "iptables $(sudo iptables --version 2>/dev/null || echo 'not found')" - echo "tc $(tc -Version 2>/dev/null || echo 'not found')" - displayName: 'Install iptables and tc for network fault injection' + sudo apt-get update -qq && sudo apt-get install -y -qq iptables + echo \"iptables $(sudo iptables --version 2>/dev/null || echo 'not found')\" + displayName: 'Install iptables for network fault injection' CloudConfig: Public: ServiceConnection: azure-sdk-tests-cosmos From face9d25bbe773b4c21d174f5980f772ac132e2c Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Tue, 12 May 2026 21:15:43 -0400 Subject: [PATCH 23/62] Remove test 1 (pingFramesSentAtConfiguredInterval) -- does not prove PINGs Test 1 only proved connection reuse during a 10s idle period, but maxIdleTime=60s means the connection survives regardless of PINGs. Test 2 (connectionClosedOnPingTimeout) is the definitive proof that PINGs flow: iptables DROP goes undetected without PING probing. --- .../Http2PingKeepaliveTest.java | 74 ++----------------- 1 file changed, 8 insertions(+), 66 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java index e7d72a22fdfa..aa9388aa184f 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java @@ -88,74 +88,16 @@ public void afterMethod(Method method) { } /** - * Test 1: Verifies PING frames are sent at the configured interval on idle connections - * and that PING ACKs are received from the server. + * Uses iptables to silently discard all traffic to the Cosmos DB gateway port, + * preventing PING ACKs from arriving. Verifies the handler detects the broken + * connection after consecutive PING failures and closes it. A subsequent request + * (after removing the iptables rule) uses a new connection. *

- * With interval=1s, a 10s idle period should produce ~10 PINGs. - */ - @Test(groups = {"manual-http-network-fault"}, timeOut = TEST_TIMEOUT) - public void pingFramesSentAtConfiguredInterval() throws Exception { - // Use default interval=1s, timeout=2s (aligned with Rust SDK) - System.setProperty("COSMOS.HTTP2_PING_INTERVAL_IN_SECONDS", "1"); - System.setProperty("COSMOS.HTTP2_PING_TIMEOUT_IN_SECONDS", "2"); - System.setProperty("COSMOS.HTTP2_PING_HEALTH_ENABLED", "true"); - - try { - safeClose(this.client); - - // Single-connection pool so we can assert same connection is reused. - // maxIdleTime defaults to 60s; 10s idle period is well under that threshold. - // PINGs keep middlebox entries alive (not tested here -- tested by test 2). - // This test validates PING frames are actively flowing on idle connections. - GatewayConnectionConfig gwConfig = new GatewayConnectionConfig(); - gwConfig.getHttp2ConnectionConfig() - .setEnabled(true) - .setMaxConnectionPoolSize(1) - .setMinConnectionPoolSize(1); - - this.client = new CosmosClientBuilder() - .endpoint(TestConfigurations.HOST) - .key(TestConfigurations.MASTER_KEY) - .consistencyLevel(ConsistencyLevel.SESSION) - .gatewayMode(gwConfig) - .buildAsyncClient(); - this.cosmosAsyncContainer = getSharedMultiPartitionCosmosContainerWithIdAsPartitionKey(this.client); - - // Warm-up read -- triggers H2 connection + PING handler installation - String initialChannelId = readAndGetParentChannelId(); - logger.info("Initial parentChannelId: {}", initialChannelId); - - // Let the connection go idle for 10s -- with 1s interval, expect ~10 PINGs - logger.info("Waiting 10s for PING frames on idle connection (interval=1s)..."); - Thread.sleep(10_000); - - // Recovery read -- proves connection survived idle period - String recoveryChannelId = readAndGetParentChannelId(); - - logger.info("RESULT: initial={}, recovery={}, SAME={}", - initialChannelId, recoveryChannelId, - initialChannelId.equals(recoveryChannelId)); - - // With pool size=1, same connection confirms it survived the idle period - assertThat(recoveryChannelId) - .as("Connection should be reused -- PING frames flowed during idle period") - .isEqualTo(initialChannelId); - - logger.info("PING interval test passed: connection survived 10s idle period"); - } finally { - System.clearProperty("COSMOS.HTTP2_PING_INTERVAL_IN_SECONDS"); - System.clearProperty("COSMOS.HTTP2_PING_TIMEOUT_IN_SECONDS"); - System.clearProperty("COSMOS.HTTP2_PING_HEALTH_ENABLED"); - } - } - - /** - * Test 2: Uses iptables to blackhole all traffic to the Cosmos DB gateway port, - * preventing PING ACKs from arriving. Verifies the handler closes the broken - * connection and a subsequent request (after removing the iptables rule) uses - * a new/different connection. + * This test proves PINGs are actively flowing on idle connections -- without PINGs, + * iptables DROP would go undetected (channel.isActive() stays true, no GOAWAY arrives) + * and the test would time out. *

- * Requires Docker with --cap-add=NET_ADMIN. + * Requires Docker with --cap-add=NET_ADMIN or Linux with sudo. */ @Test(groups = {"manual-http-network-fault"}, timeOut = TEST_TIMEOUT) public void connectionClosedOnPingTimeout() throws Exception { From 7e04d211aad31a607a69a5ca8cbbc449cba7bce3 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Thu, 14 May 2026 16:51:03 -0400 Subject: [PATCH 24/62] Address review round 2: remove dead code, track lastReadNanos, bounds guard 1. Remove dead code: getPingsSent/getPingAcksReceived + AtomicInteger fields, httpCfgAccessor() private method (zero consumers after test 1 removal) 2. Track lastReadNanos (inbound) instead of lastActivityNanos (read+write) -- PING fires when no response received for interval, detects half-dead connections where writes succeed but reads never arrive 3. Add Math.max(1, ...) bounds guard on pingTimeoutSeconds to prevent zero/negative timeout causing instant timeout on every PING 4. Writes no longer reset idle timer (only reads do) 5. pingsSent changed from AtomicInteger to plain int (event-loop confined) --- .../implementation/http/Http2PingHandler.java | 45 ++++++++----------- .../http/ReactorNettyClient.java | 3 -- 2 files changed, 18 insertions(+), 30 deletions(-) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java index c764085898eb..0482e81fddeb 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java @@ -15,7 +15,6 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; /** * Manual HTTP/2 PING keepalive handler installed on the parent (TCP) channel. @@ -42,12 +41,11 @@ public class Http2PingHandler extends ChannelDuplexHandler { private final long pingIntervalNanos; private final long pingTimeoutNanos; private final int failureThreshold; - private long lastActivityNanos; - private long pingOutstandingSinceNanos; // nanoTime when PING was sent; 0 = no outstanding PING - private int consecutiveFailures; // incremented on timeout, reset on ACK + private long lastReadNanos; // nanoTime of last inbound frame (response); PING triggers when no read for interval + private long pingOutstandingSinceNanos; // nanoTime when PING was sent; 0 = no outstanding PING + private int consecutiveFailures; // incremented on timeout, reset on ACK + private int pingsSent; private ScheduledFuture pingTask; - private final AtomicInteger pingsSent = new AtomicInteger(0); - private final AtomicInteger pingAcksReceived = new AtomicInteger(0); /** * @param pingIntervalSeconds interval in seconds; when idle longer than this, a PING is sent @@ -55,10 +53,10 @@ public class Http2PingHandler extends ChannelDuplexHandler { * @param failureThreshold consecutive timeouts before closing the connection */ public Http2PingHandler(int pingIntervalSeconds, int pingTimeoutSeconds, int failureThreshold) { - this.pingIntervalNanos = TimeUnit.SECONDS.toNanos(pingIntervalSeconds); - this.pingTimeoutNanos = TimeUnit.SECONDS.toNanos(pingTimeoutSeconds); + this.pingIntervalNanos = TimeUnit.SECONDS.toNanos(Math.max(1, pingIntervalSeconds)); + this.pingTimeoutNanos = TimeUnit.SECONDS.toNanos(Math.max(1, pingTimeoutSeconds)); this.failureThreshold = Math.max(1, failureThreshold); - this.lastActivityNanos = System.nanoTime(); + this.lastReadNanos = System.nanoTime(); } @Override @@ -92,9 +90,8 @@ public void channelInactive(ChannelHandlerContext ctx) throws Exception { @Override public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { - lastActivityNanos = System.nanoTime(); + lastReadNanos = System.nanoTime(); if (msg instanceof Http2PingFrame && ((Http2PingFrame) msg).ack()) { - pingAcksReceived.incrementAndGet(); pingOutstandingSinceNanos = 0; consecutiveFailures = 0; } @@ -103,7 +100,9 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception @Override public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) throws Exception { - lastActivityNanos = System.nanoTime(); + // Writes (outbound requests) do NOT reset the idle timer. + // We track last *read* (inbound response) to detect half-dead connections + // where writes succeed but responses never arrive. super.write(ctx, msg, promise); } @@ -136,18 +135,18 @@ private void maybeSendPing(ChannelHandlerContext ctx) { } // Don't send if there are active streams -- request/response traffic is already - // keeping the connection alive, so a PING would be pure noise-neighbour overhead. + // keeping the connection alive, so a PING would be pure overhead. Http2FrameCodec codec = ctx.pipeline().get(Http2FrameCodec.class); if (codec != null && codec.connection().numActiveStreams() > 0) { - // Active streams reset the idle baseline so the first idle check after + // Active streams reset the read baseline so the first idle check after // all streams close measures from *now*, not from the last frame. - lastActivityNanos = System.nanoTime(); + lastReadNanos = System.nanoTime(); return; } - long idleNanos = System.nanoTime() - lastActivityNanos; + long idleNanos = System.nanoTime() - lastReadNanos; if (idleNanos >= pingIntervalNanos) { - int count = pingsSent.incrementAndGet(); + int count = ++pingsSent; pingOutstandingSinceNanos = System.nanoTime(); ctx.writeAndFlush(new DefaultHttp2PingFrame(count)) .addListener(f -> { @@ -162,8 +161,8 @@ private void maybeSendPing(ChannelHandlerContext ctx) { f.cause() != null ? f.cause().getMessage() : "unknown"); } }); - // Reset activity so we don't send another PING immediately - lastActivityNanos = System.nanoTime(); + // Reset read timestamp so we don't send another PING immediately + lastReadNanos = System.nanoTime(); } } @@ -174,14 +173,6 @@ private void cancelPingTask() { } } - public int getPingsSent() { - return pingsSent.get(); - } - - public int getPingAcksReceived() { - return pingAcksReceived.get(); - } - /** * Installs the PING handler on the parent H2 channel if not already installed. * Safe to call from doOnConnected (which fires per-stream for H2). diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java index ad01cf0927b3..7cd52f3caca0 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java @@ -44,9 +44,6 @@ * HttpClient that is implemented using reactor-netty. */ public class ReactorNettyClient implements HttpClient { - private static ImplementationBridgeHelpers.Http2ConnectionConfigHelper.Http2ConnectionConfigAccessor httpCfgAccessor() { - return ImplementationBridgeHelpers.Http2ConnectionConfigHelper.getHttp2ConnectionConfigAccessor(); - } private static final boolean leakDetectionDebuggingEnabled = ResourceLeakDetector.getLevel().ordinal() >= ResourceLeakDetector.Level.ADVANCED.ordinal(); From 957be47bfa73c874fc5a416f67889c3d3fde9e2b Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Sun, 24 May 2026 20:24:14 -0400 Subject: [PATCH 25/62] Add HTTP/2 PING resilience: probe PING and client-level kill switch Send a probe PING 500ms after handler installation to detect servers that reject PING frames (e.g., Cosmos DB Dedicated Gateway's old Mux stack which responds with RST_STREAM(0, PROTOCOL_ERROR)). If PROTOCOL_ERROR is received while a PING is outstanding, disable HTTP/2 PING for all connections from this CosmosClient via a shared AtomicBoolean kill switch. This prevents repeatedly killing connections to PING-incompatible endpoints. Changes: - Http2PingHandler: probe PING in handlerAdded(), exceptionCaught() override for PROTOCOL_ERROR detection, clientPingDisabled guard in maybeSendPing() and installIfAbsent() - ReactorNettyClient: AtomicBoolean http2PingDisabled field shared with all Http2PingHandler instances Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../implementation/http/Http2PingHandler.java | 67 ++++++++++++++++++- .../http/ReactorNettyClient.java | 11 ++- 2 files changed, 74 insertions(+), 4 deletions(-) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java index 0482e81fddeb..6586c73459d5 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java @@ -8,6 +8,8 @@ import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelPromise; import io.netty.handler.codec.http2.DefaultHttp2PingFrame; +import io.netty.handler.codec.http2.Http2Error; +import io.netty.handler.codec.http2.Http2Exception; import io.netty.handler.codec.http2.Http2FrameCodec; import io.netty.handler.codec.http2.Http2PingFrame; import org.slf4j.Logger; @@ -15,6 +17,7 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; /** * Manual HTTP/2 PING keepalive handler installed on the parent (TCP) channel. @@ -41,6 +44,7 @@ public class Http2PingHandler extends ChannelDuplexHandler { private final long pingIntervalNanos; private final long pingTimeoutNanos; private final int failureThreshold; + private final AtomicBoolean clientPingDisabled; // shared across all connections for this CosmosClient private long lastReadNanos; // nanoTime of last inbound frame (response); PING triggers when no read for interval private long pingOutstandingSinceNanos; // nanoTime when PING was sent; 0 = no outstanding PING private int consecutiveFailures; // incremented on timeout, reset on ACK @@ -51,12 +55,16 @@ public class Http2PingHandler extends ChannelDuplexHandler { * @param pingIntervalSeconds interval in seconds; when idle longer than this, a PING is sent * @param pingTimeoutSeconds timeout in seconds per PING attempt * @param failureThreshold consecutive timeouts before closing the connection + * @param clientPingDisabled shared flag; set to {@code true} if the server rejects PING, + * disabling PING for all connections from this CosmosClient */ - public Http2PingHandler(int pingIntervalSeconds, int pingTimeoutSeconds, int failureThreshold) { + public Http2PingHandler(int pingIntervalSeconds, int pingTimeoutSeconds, int failureThreshold, + AtomicBoolean clientPingDisabled) { this.pingIntervalNanos = TimeUnit.SECONDS.toNanos(Math.max(1, pingIntervalSeconds)); this.pingTimeoutNanos = TimeUnit.SECONDS.toNanos(Math.max(1, pingTimeoutSeconds)); this.failureThreshold = Math.max(1, failureThreshold); this.lastReadNanos = System.nanoTime(); + this.clientPingDisabled = clientPingDisabled; } @Override @@ -75,6 +83,26 @@ public void handlerAdded(ChannelHandlerContext ctx) { TimeUnit.NANOSECONDS.toSeconds(pingIntervalNanos), TimeUnit.NANOSECONDS.toSeconds(pingTimeoutNanos), checkIntervalMs); + + // Send probe PING after short delay to verify server supports HTTP/2 PING. + // Allows time for H2 connection preface (SETTINGS exchange) to complete. + // If the server rejects PING (e.g., RST_STREAM with PROTOCOL_ERROR), exceptionCaught() + // disables PING for all connections from this CosmosClient via clientPingDisabled. + ctx.executor().schedule(() -> { + if (ctx.channel().isActive() && !clientPingDisabled.get()) { + int count = ++pingsSent; + pingOutstandingSinceNanos = System.nanoTime(); + ctx.writeAndFlush(new DefaultHttp2PingFrame(count)) + .addListener(f -> { + if (f.isSuccess()) { + logger.debug("Probe PING #{} sent on channel {}", + count, ctx.channel().id().asShortText()); + } else { + pingOutstandingSinceNanos = 0; + } + }); + } + }, 500, TimeUnit.MILLISECONDS); } @Override @@ -106,12 +134,39 @@ public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) super.write(ctx, msg, promise); } + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { + // Detect server-side PING rejection: some HTTP/2 endpoints (e.g., Cosmos DB Dedicated Gateway's + // old Mux stack) respond to PING with an invalid RST_STREAM(stream_id=0, PROTOCOL_ERROR). + // Netty's Http2FrameCodec treats RST_STREAM on stream 0 as a connection error and fires + // Http2Exception(PROTOCOL_ERROR) down the pipeline. If this occurs while our PING is + // outstanding, the server does not support PING — disable for all connections from this + // CosmosClient to avoid repeatedly killing connections. + if (pingOutstandingSinceNanos != 0 && cause instanceof Http2Exception) { + Http2Exception h2e = (Http2Exception) cause; + if (h2e.error() == Http2Error.PROTOCOL_ERROR) { + clientPingDisabled.set(true); + cancelPingTask(); + logger.warn("Server rejected PING with PROTOCOL_ERROR on channel {} — " + + "disabling HTTP/2 PING for this CosmosClient", + ctx.channel().id().asShortText()); + } + } + super.exceptionCaught(ctx, cause); + } + private void maybeSendPing(ChannelHandlerContext ctx) { if (!ctx.channel().isActive() || !Configs.isHttp2PingHealthEnabled()) { cancelPingTask(); return; } + // Client-level kill switch — another connection's handler detected PING rejection + if (clientPingDisabled.get()) { + cancelPingTask(); + return; + } + // If a previous PING is still outstanding, check whether it has timed out if (pingOutstandingSinceNanos != 0) { long waitingNanos = System.nanoTime() - pingOutstandingSinceNanos; @@ -181,12 +236,18 @@ private void cancelPingTask() { * @param pingIntervalSeconds PING interval in seconds * @param pingTimeoutSeconds PING ACK timeout in seconds * @param failureThreshold consecutive timeouts before closing + * @param clientPingDisabled shared flag; if {@code true}, PING has been disabled for this CosmosClient */ - public static void installIfAbsent(Channel channel, int pingIntervalSeconds, int pingTimeoutSeconds, int failureThreshold) { + public static void installIfAbsent(Channel channel, int pingIntervalSeconds, int pingTimeoutSeconds, + int failureThreshold, AtomicBoolean clientPingDisabled) { + if (clientPingDisabled.get()) { + return; + } Channel parent = channel.parent() != null ? channel.parent() : channel; if (parent.pipeline().get(HANDLER_NAME) == null) { try { - parent.pipeline().addLast(HANDLER_NAME, new Http2PingHandler(pingIntervalSeconds, pingTimeoutSeconds, failureThreshold)); + parent.pipeline().addLast(HANDLER_NAME, + new Http2PingHandler(pingIntervalSeconds, pingTimeoutSeconds, failureThreshold, clientPingDisabled)); } catch (IllegalArgumentException ignored) { // Duplicate -- race between concurrent streams, benign } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java index 7cd52f3caca0..ecffc1061385 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java @@ -37,6 +37,7 @@ import java.time.Duration; import java.time.Instant; import java.util.Objects; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiFunction; @@ -56,6 +57,7 @@ public class ReactorNettyClient implements HttpClient { private ConnectionProvider connectionProvider; private String reactorNetworkLogCategory; private Logger wireTapLogger; + private final AtomicBoolean http2PingDisabled = new AtomicBoolean(false); private ReactorNettyClient() {} @@ -145,19 +147,26 @@ private void configureChannelPipelineHandlers() { boolean isH2Enabled = http2CfgAccessor.isEffectivelyEnabled(http2Cfg); if (isH2Enabled) { + final AtomicBoolean pingDisabledRef = this.http2PingDisabled; this.httpClient = this.httpClient.doOnConnected(connection -> { // Manual HTTP/2 PING keepalive -- sends PING frames when the connection is idle // to prevent L7 middleboxes (NAT, firewalls, LBs) from reaping the connection. // For H2, the first doOnConnected fires for the parent TCP channel (parent()==null), // and the second fires for stream channels (parent()!=null). // We install on the parent channel -- detect it via Http2MultiplexHandler in the pipeline. + // + // A probe PING is sent immediately on handler installation to detect servers that + // reject PING (e.g., Cosmos DB Dedicated Gateway's old Mux stack). If PROTOCOL_ERROR + // is received, PING is disabled for all connections from this CosmosClient. if (Configs.isHttp2PingHealthEnabled() + && !pingDisabledRef.get() && connection.channel().pipeline().get(Http2MultiplexHandler.class) != null) { int pingIntervalSeconds = Configs.getHttp2PingIntervalInSeconds(); int pingTimeoutSeconds = Configs.getHttp2PingTimeoutInSeconds(); int pingFailureThreshold = Configs.getHttp2PingFailureThreshold(); if (pingIntervalSeconds > 0) { - Http2PingHandler.installIfAbsent(connection.channel(), pingIntervalSeconds, pingTimeoutSeconds, pingFailureThreshold); + Http2PingHandler.installIfAbsent(connection.channel(), pingIntervalSeconds, + pingTimeoutSeconds, pingFailureThreshold, pingDisabledRef); } } }); From 133b4244f0a10a08ada7753b359b379497317400 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Mon, 25 May 2026 11:55:27 -0400 Subject: [PATCH 26/62] Self-review: move http2CfgAccessor to static field, remove write() override and active-streams check, add thread-safety comment --- .../implementation/http/Http2PingHandler.java | 24 ++++--------------- .../http/ReactorNettyClient.java | 4 ++-- 2 files changed, 6 insertions(+), 22 deletions(-) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java index 6586c73459d5..8ddd004eb625 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java @@ -6,11 +6,9 @@ import io.netty.channel.Channel; import io.netty.channel.ChannelDuplexHandler; import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.ChannelPromise; import io.netty.handler.codec.http2.DefaultHttp2PingFrame; import io.netty.handler.codec.http2.Http2Error; import io.netty.handler.codec.http2.Http2Exception; -import io.netty.handler.codec.http2.Http2FrameCodec; import io.netty.handler.codec.http2.Http2PingFrame; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,6 +43,10 @@ public class Http2PingHandler extends ChannelDuplexHandler { private final long pingTimeoutNanos; private final int failureThreshold; private final AtomicBoolean clientPingDisabled; // shared across all connections for this CosmosClient + + // Mutable fields below are accessed only from the channel's EventLoop thread + // (handlerAdded, channelRead, scheduled task, writeAndFlush listener), so no + // synchronization or volatile is needed. private long lastReadNanos; // nanoTime of last inbound frame (response); PING triggers when no read for interval private long pingOutstandingSinceNanos; // nanoTime when PING was sent; 0 = no outstanding PING private int consecutiveFailures; // incremented on timeout, reset on ACK @@ -126,14 +128,6 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception super.channelRead(ctx, msg); } - @Override - public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) throws Exception { - // Writes (outbound requests) do NOT reset the idle timer. - // We track last *read* (inbound response) to detect half-dead connections - // where writes succeed but responses never arrive. - super.write(ctx, msg, promise); - } - @Override public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { // Detect server-side PING rejection: some HTTP/2 endpoints (e.g., Cosmos DB Dedicated Gateway's @@ -189,16 +183,6 @@ private void maybeSendPing(ChannelHandlerContext ctx) { return; } - // Don't send if there are active streams -- request/response traffic is already - // keeping the connection alive, so a PING would be pure overhead. - Http2FrameCodec codec = ctx.pipeline().get(Http2FrameCodec.class); - if (codec != null && codec.connection().numActiveStreams() > 0) { - // Active streams reset the read baseline so the first idle check after - // all streams close measures from *now*, not from the last frame. - lastReadNanos = System.nanoTime(); - return; - } - long idleNanos = System.nanoTime() - lastReadNanos; if (idleNanos >= pingIntervalNanos) { int count = ++pingsSent; diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java index ecffc1061385..3d779982fcaf 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java @@ -51,6 +51,8 @@ public class ReactorNettyClient implements HttpClient { private static final String REACTOR_NETTY_REQUEST_RECORD_KEY = "reactorNettyRequestRecordKey"; private static final Logger logger = LoggerFactory.getLogger(ReactorNettyClient.class.getSimpleName()); + private static final ImplementationBridgeHelpers.Http2ConnectionConfigHelper.Http2ConnectionConfigAccessor http2CfgAccessor = + ImplementationBridgeHelpers.Http2ConnectionConfigHelper.getHttp2ConnectionConfigAccessor(); private HttpClientConfig httpClientConfig; private reactor.netty.http.client.HttpClient httpClient; @@ -140,8 +142,6 @@ private void configureChannelPipelineHandlers() { .maxChunkSize(this.httpClientConfig.getMaxChunkSize()) .validateHeaders(true)); - ImplementationBridgeHelpers.Http2ConnectionConfigHelper.Http2ConnectionConfigAccessor http2CfgAccessor = - ImplementationBridgeHelpers.Http2ConnectionConfigHelper.getHttp2ConnectionConfigAccessor(); Http2ConnectionConfig http2Cfg = httpClientConfig.getHttp2ConnectionConfig(); boolean isH2Enabled = http2CfgAccessor.isEffectivelyEnabled(http2Cfg); From 1ac4586233d04355ecd44c964d7a1878c3e6d23f Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Mon, 25 May 2026 14:37:07 -0400 Subject: [PATCH 27/62] Remove probe PING, remove active-streams check, revert channelInactive kill-switch; PROTOCOL_ERROR via exceptionCaught is the reliable path --- .../implementation/http/Http2PingHandler.java | 28 +++++-------------- .../http/ReactorNettyClient.java | 7 +++-- 2 files changed, 11 insertions(+), 24 deletions(-) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java index 8ddd004eb625..ed3f3454c83b 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java @@ -85,26 +85,6 @@ public void handlerAdded(ChannelHandlerContext ctx) { TimeUnit.NANOSECONDS.toSeconds(pingIntervalNanos), TimeUnit.NANOSECONDS.toSeconds(pingTimeoutNanos), checkIntervalMs); - - // Send probe PING after short delay to verify server supports HTTP/2 PING. - // Allows time for H2 connection preface (SETTINGS exchange) to complete. - // If the server rejects PING (e.g., RST_STREAM with PROTOCOL_ERROR), exceptionCaught() - // disables PING for all connections from this CosmosClient via clientPingDisabled. - ctx.executor().schedule(() -> { - if (ctx.channel().isActive() && !clientPingDisabled.get()) { - int count = ++pingsSent; - pingOutstandingSinceNanos = System.nanoTime(); - ctx.writeAndFlush(new DefaultHttp2PingFrame(count)) - .addListener(f -> { - if (f.isSuccess()) { - logger.debug("Probe PING #{} sent on channel {}", - count, ctx.channel().id().asShortText()); - } else { - pingOutstandingSinceNanos = 0; - } - }); - } - }, 500, TimeUnit.MILLISECONDS); } @Override @@ -136,14 +116,20 @@ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws E // Http2Exception(PROTOCOL_ERROR) down the pipeline. If this occurs while our PING is // outstanding, the server does not support PING — disable for all connections from this // CosmosClient to avoid repeatedly killing connections. + // + // We swallow the exception (do NOT call super) to prevent the codec from closing + // the connection — without this PR, the connection would have been fine. if (pingOutstandingSinceNanos != 0 && cause instanceof Http2Exception) { Http2Exception h2e = (Http2Exception) cause; if (h2e.error() == Http2Error.PROTOCOL_ERROR) { clientPingDisabled.set(true); cancelPingTask(); - logger.warn("Server rejected PING with PROTOCOL_ERROR on channel {} — " + pingOutstandingSinceNanos = 0; + logger.warn("Server rejected PING with PROTOCOL_ERROR on channel {} -- " + "disabling HTTP/2 PING for this CosmosClient", ctx.channel().id().asShortText()); + ctx.pipeline().remove(this); + return; } } super.exceptionCaught(ctx, cause); diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java index 3d779982fcaf..d0d62ceb5ef9 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java @@ -155,9 +155,10 @@ private void configureChannelPipelineHandlers() { // and the second fires for stream channels (parent()!=null). // We install on the parent channel -- detect it via Http2MultiplexHandler in the pipeline. // - // A probe PING is sent immediately on handler installation to detect servers that - // reject PING (e.g., Cosmos DB Dedicated Gateway's old Mux stack). If PROTOCOL_ERROR - // is received, PING is disabled for all connections from this CosmosClient. + // If the server rejects PING (e.g., Cosmos DB Dedicated Gateway's custom Mux stack + // returns RST_STREAM/PROTOCOL_ERROR), the handler sets clientPingDisabled and removes + // itself. The connection is lost (server-side close), but the pool replaces it, and + // no further PINGs are sent from this CosmosClient. if (Configs.isHttp2PingHealthEnabled() && !pingDisabledRef.get() && connection.channel().pipeline().get(Http2MultiplexHandler.class) != null) { From 2686a5afdb7e79c384a9bfeac3d3a51bf3f5ffbb Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Mon, 25 May 2026 14:50:03 -0400 Subject: [PATCH 28/62] Use lazy init pattern for http2CfgAccessor --- .../cosmos/implementation/http/ReactorNettyClient.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java index d0d62ceb5ef9..8035c1642618 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java @@ -51,8 +51,10 @@ public class ReactorNettyClient implements HttpClient { private static final String REACTOR_NETTY_REQUEST_RECORD_KEY = "reactorNettyRequestRecordKey"; private static final Logger logger = LoggerFactory.getLogger(ReactorNettyClient.class.getSimpleName()); - private static final ImplementationBridgeHelpers.Http2ConnectionConfigHelper.Http2ConnectionConfigAccessor http2CfgAccessor = - ImplementationBridgeHelpers.Http2ConnectionConfigHelper.getHttp2ConnectionConfigAccessor(); + + private static ImplementationBridgeHelpers.Http2ConnectionConfigHelper.Http2ConnectionConfigAccessor http2CfgAccessor() { + return ImplementationBridgeHelpers.Http2ConnectionConfigHelper.getHttp2ConnectionConfigAccessor(); + } private HttpClientConfig httpClientConfig; private reactor.netty.http.client.HttpClient httpClient; @@ -144,7 +146,7 @@ private void configureChannelPipelineHandlers() { Http2ConnectionConfig http2Cfg = httpClientConfig.getHttp2ConnectionConfig(); - boolean isH2Enabled = http2CfgAccessor.isEffectivelyEnabled(http2Cfg); + boolean isH2Enabled = http2CfgAccessor().isEffectivelyEnabled(http2Cfg); if (isH2Enabled) { final AtomicBoolean pingDisabledRef = this.http2PingDisabled; @@ -183,7 +185,7 @@ private void configureChannelPipelineHandlers() { .http2Settings(settings -> settings .initialWindowSize(1024 * 1024) // 1MB initial window size .maxFrameSize(64 * 1024) // 64KB max frame size - .maxConcurrentStreams(http2CfgAccessor.getEffectiveMaxConcurrentStreams(http2Cfg)) // Increased from default 30 + .maxConcurrentStreams(http2CfgAccessor().getEffectiveMaxConcurrentStreams(http2Cfg)) // Increased from default 30 ) .doOnConnected((connection -> { // The response header clean up pipeline is being added due to an error getting when calling gateway: From f70bb19b65cc377405e788d32705b50f874e46ad Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Wed, 27 May 2026 12:11:22 -0400 Subject: [PATCH 29/62] Fix parent channel resolution and customHeaderCleaner duplicate guard - doOnConnected fires for child stream channels where Http2MultiplexHandler is absent; resolve ch.parent() before checking for the multiplexer and installing PingHandler. - Guard customHeaderCleaner addAfter with null check to prevent IllegalArgumentException: Duplicate handler name. - Add Http2PingHandler DEBUG logging to test log4j2 config. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../src/test/resources/log4j2-test.properties | 4 ++++ .../http/ReactorNettyClient.java | 24 ++++++++++++------- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/resources/log4j2-test.properties b/sdk/cosmos/azure-cosmos-tests/src/test/resources/log4j2-test.properties index 116ca2900edd..b822e388c0f6 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/resources/log4j2-test.properties +++ b/sdk/cosmos/azure-cosmos-tests/src/test/resources/log4j2-test.properties @@ -7,3 +7,7 @@ appender.console.name = STDOUT appender.console.type = Console appender.console.layout.type = PatternLayout appender.console.layout.pattern = %d %5X{pid} [%t] %-5p %c - %m%n + +# HTTP/2 PING handler -- enable DEBUG to see PING send/ACK/timeout/kill-switch activity +logger.http2ping.name = com.azure.cosmos.implementation.http.Http2PingHandler +logger.http2ping.level = debug diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java index 8035c1642618..9ac7424da2bd 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java @@ -162,14 +162,19 @@ private void configureChannelPipelineHandlers() { // itself. The connection is lost (server-side close), but the pool replaces it, and // no further PINGs are sent from this CosmosClient. if (Configs.isHttp2PingHealthEnabled() - && !pingDisabledRef.get() - && connection.channel().pipeline().get(Http2MultiplexHandler.class) != null) { - int pingIntervalSeconds = Configs.getHttp2PingIntervalInSeconds(); - int pingTimeoutSeconds = Configs.getHttp2PingTimeoutInSeconds(); - int pingFailureThreshold = Configs.getHttp2PingFailureThreshold(); - if (pingIntervalSeconds > 0) { - Http2PingHandler.installIfAbsent(connection.channel(), pingIntervalSeconds, - pingTimeoutSeconds, pingFailureThreshold, pingDisabledRef); + && !pingDisabledRef.get()) { + // Resolve to the parent (TCP) channel -- doOnConnected may fire for + // child stream channels where Http2MultiplexHandler is absent. + Channel ch = connection.channel(); + Channel parent = ch.parent() != null ? ch.parent() : ch; + if (parent.pipeline().get(Http2MultiplexHandler.class) != null) { + int pingIntervalSeconds = Configs.getHttp2PingIntervalInSeconds(); + int pingTimeoutSeconds = Configs.getHttp2PingTimeoutInSeconds(); + int pingFailureThreshold = Configs.getHttp2PingFailureThreshold(); + if (pingIntervalSeconds > 0) { + Http2PingHandler.installIfAbsent(parent, pingIntervalSeconds, + pingTimeoutSeconds, pingFailureThreshold, pingDisabledRef); + } } } }); @@ -192,7 +197,8 @@ private void configureChannelPipelineHandlers() { // java.lang.IllegalArgumentException: a header value contains prohibited character 0x20 at index 0 for 'x-ms-serviceversion', there is whitespace in the front of the value. // validateHeaders(false) does not work for http2 ChannelPipeline channelPipeline = connection.channel().pipeline(); - if (channelPipeline.get("reactor.left.httpCodec") != null) { + if (channelPipeline.get("reactor.left.httpCodec") != null + && channelPipeline.get("customHeaderCleaner") == null) { channelPipeline.addAfter( "reactor.left.httpCodec", "customHeaderCleaner", From f1b42353086275fab01bfec89cad4902f9a4ada1 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Wed, 27 May 2026 13:29:48 -0400 Subject: [PATCH 30/62] Remove PROTOCOL_ERROR logic and client-level kill switch The SQLx Mux's Http2ServerProtocolHandler rejects PING with PROTOCOL_ERROR, but SQLx doesn't negotiate H2 via ALPN so clients never speak H2 to it. The Proxy (Standard GW, ThinClient) uses nghttp2 which auto-ACKs PINGs correctly. Since no live endpoint returns PROTOCOL_ERROR for PINGs, the defensive kill-switch adds complexity without value. Removed: - exceptionCaught() PROTOCOL_ERROR detection - clientPingDisabled AtomicBoolean (shared kill switch) - Kill-switch check in maybeSendPing() - Related parameter from constructor and installIfAbsent() Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../implementation/http/Http2PingHandler.java | 56 +++---------------- .../http/ReactorNettyClient.java | 18 ++---- 2 files changed, 12 insertions(+), 62 deletions(-) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java index ed3f3454c83b..329537c95365 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java @@ -7,15 +7,12 @@ import io.netty.channel.ChannelDuplexHandler; import io.netty.channel.ChannelHandlerContext; import io.netty.handler.codec.http2.DefaultHttp2PingFrame; -import io.netty.handler.codec.http2.Http2Error; -import io.netty.handler.codec.http2.Http2Exception; import io.netty.handler.codec.http2.Http2PingFrame; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; /** * Manual HTTP/2 PING keepalive handler installed on the parent (TCP) channel. @@ -32,6 +29,11 @@ * Why manual instead of reactor-netty native {@code pingAckTimeout}? Native PING requires * built-in {@code maxIdleTime} handling, which is bypassed when a custom * {@code evictionPredicate} is configured (reactor-netty 1.2.13). + *

+ * Note: the Cosmos DB Proxy (Standard Gateway, ThinClient) uses nghttp2 which auto-ACKs + * PINGs per RFC 9113 §6.7. The SQLx Mux uses a custom frame parser that currently rejects + * PING with PROTOCOL_ERROR — but SQLx does not negotiate H2 via ALPN, so clients never + * speak H2 to it. */ public class Http2PingHandler extends ChannelDuplexHandler { @@ -42,7 +44,6 @@ public class Http2PingHandler extends ChannelDuplexHandler { private final long pingIntervalNanos; private final long pingTimeoutNanos; private final int failureThreshold; - private final AtomicBoolean clientPingDisabled; // shared across all connections for this CosmosClient // Mutable fields below are accessed only from the channel's EventLoop thread // (handlerAdded, channelRead, scheduled task, writeAndFlush listener), so no @@ -57,16 +58,12 @@ public class Http2PingHandler extends ChannelDuplexHandler { * @param pingIntervalSeconds interval in seconds; when idle longer than this, a PING is sent * @param pingTimeoutSeconds timeout in seconds per PING attempt * @param failureThreshold consecutive timeouts before closing the connection - * @param clientPingDisabled shared flag; set to {@code true} if the server rejects PING, - * disabling PING for all connections from this CosmosClient */ - public Http2PingHandler(int pingIntervalSeconds, int pingTimeoutSeconds, int failureThreshold, - AtomicBoolean clientPingDisabled) { + public Http2PingHandler(int pingIntervalSeconds, int pingTimeoutSeconds, int failureThreshold) { this.pingIntervalNanos = TimeUnit.SECONDS.toNanos(Math.max(1, pingIntervalSeconds)); this.pingTimeoutNanos = TimeUnit.SECONDS.toNanos(Math.max(1, pingTimeoutSeconds)); this.failureThreshold = Math.max(1, failureThreshold); this.lastReadNanos = System.nanoTime(); - this.clientPingDisabled = clientPingDisabled; } @Override @@ -108,45 +105,12 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception super.channelRead(ctx, msg); } - @Override - public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { - // Detect server-side PING rejection: some HTTP/2 endpoints (e.g., Cosmos DB Dedicated Gateway's - // old Mux stack) respond to PING with an invalid RST_STREAM(stream_id=0, PROTOCOL_ERROR). - // Netty's Http2FrameCodec treats RST_STREAM on stream 0 as a connection error and fires - // Http2Exception(PROTOCOL_ERROR) down the pipeline. If this occurs while our PING is - // outstanding, the server does not support PING — disable for all connections from this - // CosmosClient to avoid repeatedly killing connections. - // - // We swallow the exception (do NOT call super) to prevent the codec from closing - // the connection — without this PR, the connection would have been fine. - if (pingOutstandingSinceNanos != 0 && cause instanceof Http2Exception) { - Http2Exception h2e = (Http2Exception) cause; - if (h2e.error() == Http2Error.PROTOCOL_ERROR) { - clientPingDisabled.set(true); - cancelPingTask(); - pingOutstandingSinceNanos = 0; - logger.warn("Server rejected PING with PROTOCOL_ERROR on channel {} -- " - + "disabling HTTP/2 PING for this CosmosClient", - ctx.channel().id().asShortText()); - ctx.pipeline().remove(this); - return; - } - } - super.exceptionCaught(ctx, cause); - } - private void maybeSendPing(ChannelHandlerContext ctx) { if (!ctx.channel().isActive() || !Configs.isHttp2PingHealthEnabled()) { cancelPingTask(); return; } - // Client-level kill switch — another connection's handler detected PING rejection - if (clientPingDisabled.get()) { - cancelPingTask(); - return; - } - // If a previous PING is still outstanding, check whether it has timed out if (pingOutstandingSinceNanos != 0) { long waitingNanos = System.nanoTime() - pingOutstandingSinceNanos; @@ -206,18 +170,14 @@ private void cancelPingTask() { * @param pingIntervalSeconds PING interval in seconds * @param pingTimeoutSeconds PING ACK timeout in seconds * @param failureThreshold consecutive timeouts before closing - * @param clientPingDisabled shared flag; if {@code true}, PING has been disabled for this CosmosClient */ public static void installIfAbsent(Channel channel, int pingIntervalSeconds, int pingTimeoutSeconds, - int failureThreshold, AtomicBoolean clientPingDisabled) { - if (clientPingDisabled.get()) { - return; - } + int failureThreshold) { Channel parent = channel.parent() != null ? channel.parent() : channel; if (parent.pipeline().get(HANDLER_NAME) == null) { try { parent.pipeline().addLast(HANDLER_NAME, - new Http2PingHandler(pingIntervalSeconds, pingTimeoutSeconds, failureThreshold, clientPingDisabled)); + new Http2PingHandler(pingIntervalSeconds, pingTimeoutSeconds, failureThreshold)); } catch (IllegalArgumentException ignored) { // Duplicate -- race between concurrent streams, benign } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java index 9ac7424da2bd..6995bdb0b3af 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java @@ -37,7 +37,6 @@ import java.time.Duration; import java.time.Instant; import java.util.Objects; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiFunction; @@ -61,7 +60,6 @@ private static ImplementationBridgeHelpers.Http2ConnectionConfigHelper.Http2Conn private ConnectionProvider connectionProvider; private String reactorNetworkLogCategory; private Logger wireTapLogger; - private final AtomicBoolean http2PingDisabled = new AtomicBoolean(false); private ReactorNettyClient() {} @@ -149,20 +147,12 @@ private void configureChannelPipelineHandlers() { boolean isH2Enabled = http2CfgAccessor().isEffectivelyEnabled(http2Cfg); if (isH2Enabled) { - final AtomicBoolean pingDisabledRef = this.http2PingDisabled; this.httpClient = this.httpClient.doOnConnected(connection -> { // Manual HTTP/2 PING keepalive -- sends PING frames when the connection is idle // to prevent L7 middleboxes (NAT, firewalls, LBs) from reaping the connection. - // For H2, the first doOnConnected fires for the parent TCP channel (parent()==null), - // and the second fires for stream channels (parent()!=null). - // We install on the parent channel -- detect it via Http2MultiplexHandler in the pipeline. - // - // If the server rejects PING (e.g., Cosmos DB Dedicated Gateway's custom Mux stack - // returns RST_STREAM/PROTOCOL_ERROR), the handler sets clientPingDisabled and removes - // itself. The connection is lost (server-side close), but the pool replaces it, and - // no further PINGs are sent from this CosmosClient. - if (Configs.isHttp2PingHealthEnabled() - && !pingDisabledRef.get()) { + // For H2, doOnConnected fires for both the parent TCP channel and child stream + // channels. We install on the parent channel via Http2MultiplexHandler detection. + if (Configs.isHttp2PingHealthEnabled()) { // Resolve to the parent (TCP) channel -- doOnConnected may fire for // child stream channels where Http2MultiplexHandler is absent. Channel ch = connection.channel(); @@ -173,7 +163,7 @@ private void configureChannelPipelineHandlers() { int pingFailureThreshold = Configs.getHttp2PingFailureThreshold(); if (pingIntervalSeconds > 0) { Http2PingHandler.installIfAbsent(parent, pingIntervalSeconds, - pingTimeoutSeconds, pingFailureThreshold, pingDisabledRef); + pingTimeoutSeconds, pingFailureThreshold); } } } From 2bc3094265a705a21c429e54cf8a41b4aa26884c Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Tue, 2 Jun 2026 18:52:46 -0400 Subject: [PATCH 31/62] Add late-bound BooleanSupplier gate for HTTP/2 PING handler installation HttpClient gains a default no-op setHttp2PingScopeSupplier so all client implementations get the new contract for free. ReactorNettyClient stores the supplier in a volatile field (defaults to () -> false) and consults it inside the doOnConnected hook, so PING handler installation is decided per H2 connection at attach time rather than at client construction. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../implementation/http/HttpClient.java | 22 +++++++++++ .../http/ReactorNettyClient.java | 38 ++++++++++++++++++- 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/HttpClient.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/HttpClient.java index 4f1830908623..61e9da892e69 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/HttpClient.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/HttpClient.java @@ -10,6 +10,7 @@ import reactor.netty.resources.ConnectionProvider; import java.time.Duration; +import java.util.function.BooleanSupplier; /** * A generic interface for sending HTTP requests and getting responses. @@ -91,4 +92,25 @@ static HttpClient create(HttpClientConfig httpClientConfig) { * Shutdown the Http Client and clean up resources */ void shutdown(); + + /** + * Sets a late-bound supplier that gates installation of HTTP/2 PING keepalive + * to channels where the supplier returns {@code true}. The supplier is invoked + * inside {@code doOnConnected} for the parent (TCP) channel; when it returns + * {@code false}, no PING handler is installed. + * + *

The supplier must be a bound method reference that does not + * capture references to the owning {@code CosmosAsyncClient} / + * {@code RxDocumentClientImpl}. See + * {@code GlobalEndpointManager#getHasThinClientReadLocationsRef()} for the + * rationale; the recommended pattern is {@code atomicBoolean::get}. + * + *

If never set, PING installation falls back to the prior behavior gated only + * by {@link Configs#isHttp2PingHealthEnabled()}. + * + *

Default no-op so non-Netty implementations and tests need not override. + */ + default void setHttp2PingScopeSupplier(BooleanSupplier supplier) { + // no-op by default; ReactorNettyClient overrides. + } } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java index 6995bdb0b3af..759b8a2defa5 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java @@ -39,6 +39,7 @@ import java.util.Objects; import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiFunction; +import java.util.function.BooleanSupplier; /** * HttpClient that is implemented using reactor-netty. @@ -61,6 +62,18 @@ private static ImplementationBridgeHelpers.Http2ConnectionConfigHelper.Http2Conn private String reactorNetworkLogCategory; private Logger wireTapLogger; + // Defensive scope for HTTP/2 PING keepalive installation. Defaults to a + // closed gate (no PING) so that during the HttpClient bootstrap window -- + // before GlobalEndpointManager has finished the first DatabaseAccount + // refresh -- no PING handlers are installed. The first DatabaseAccount + // fetch itself is over H1.1 to the standard gateway, so it would not + // have triggered PING anyway. + // + // MUST be assigned a supplier that does NOT capture RxDocumentClientImpl + // (CosmosAsyncClient) transitively -- see HttpClient#setHttp2PingScopeSupplier + // and GlobalEndpointManager#getHasThinClientReadLocationsRef(). + private volatile BooleanSupplier http2PingScopeSupplier = () -> false; + private ReactorNettyClient() {} /** @@ -152,7 +165,16 @@ private void configureChannelPipelineHandlers() { // to prevent L7 middleboxes (NAT, firewalls, LBs) from reaping the connection. // For H2, doOnConnected fires for both the parent TCP channel and child stream // channels. We install on the parent channel via Http2MultiplexHandler detection. - if (Configs.isHttp2PingHealthEnabled()) { + // + // Scoping (defensive): PING is only installed when both + // (a) the global kill-switch Configs.isHttp2PingHealthEnabled() is true, and + // (b) the late-bound http2PingScopeSupplier returns true -- which today is + // wired to GlobalEndpointManager.hasThinClientReadLocations(), so PING + // is restricted to clients whose account is configured for thin client. + // This guard makes the existing implicit "H2 == thin client" behavior explicit + // and prevents future changes that enable H2 elsewhere from silently turning on + // PING on non-thin-client connections. + if (Configs.isHttp2PingHealthEnabled() && this.http2PingScopeSupplier.getAsBoolean()) { // Resolve to the parent (TCP) channel -- doOnConnected may fire for // child stream channels where Http2MultiplexHandler is absent. Channel ch = connection.channel(); @@ -301,6 +323,20 @@ public void shutdown() { } } + @Override + public void setHttp2PingScopeSupplier(BooleanSupplier supplier) { + // The supplier is read inside doOnConnected on a Netty event loop. We use a + // volatile write here so the lambda observes the latest value without locking. + // + // IMPORTANT: callers must pass a supplier that does NOT capture + // RxDocumentClientImpl / GlobalEndpointManager. See + // GlobalEndpointManager.getHasThinClientReadLocationsRef() for the + // memory-safety rationale and the bound-method-reference pattern. + if (supplier != null) { + this.http2PingScopeSupplier = supplier; + } + } + /** * Resolves the TCP connect timeout (CONNECT_TIMEOUT_MILLIS) based on the request type. * From 2cb787df3ca48e9e435909c643c398f70cc2325d Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Tue, 2 Jun 2026 18:52:59 -0400 Subject: [PATCH 32/62] Permanently disable HTTP/2 PING handler on SharedGatewayHttpClient Shared gateway clients are reused across multiple Cosmos clients with differing thin-client preferences, so a per-client supplier could be flipped on or off by any caller. The constructor pins the inner supplier to () -> false and the setter is a no-op so PING stays off regardless of caller intent. This keeps the rollout defensive: only dedicated thin-client clients can enable PING. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../http/SharedGatewayHttpClient.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/SharedGatewayHttpClient.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/SharedGatewayHttpClient.java index 4e0729a51ac4..90e8457f8f6f 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/SharedGatewayHttpClient.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/SharedGatewayHttpClient.java @@ -11,6 +11,7 @@ import java.time.Duration; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BooleanSupplier; /** * This class uses a shared HttpClient for multiple Cosmos Clients. @@ -41,6 +42,12 @@ public static HttpClient getOrCreateInstance(HttpClientConfig httpClientConfig, private SharedGatewayHttpClient(HttpClientConfig httpClientConfig) { this.httpClient = HttpClient.createFixed(httpClientConfig); + // Defensive scoping: HTTP/2 PING keepalive is always disabled on the shared + // gateway HttpClient. Multiple CosmosAsyncClient instances may share this + // underlying client with differing thin-client configurations, so a single + // PING decision cannot represent all callers. Per-client scope suppliers + // registered via setHttp2PingScopeSupplier are intentionally ignored here. + this.httpClient.setHttp2PingScopeSupplier(() -> false); } @Override @@ -69,4 +76,10 @@ public void shutdown() { } } } + + @Override + public void setHttp2PingScopeSupplier(BooleanSupplier supplier) { + // Intentional no-op: PING is permanently disabled on the shared HttpClient. + // See constructor for rationale. + } } From c8c41ca1cc54f08381a82d345cad767a810a796a Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Tue, 2 Jun 2026 18:53:14 -0400 Subject: [PATCH 33/62] Expose hasThinClientReadLocations as a live AtomicBoolean reference GlobalEndpointManager already tracks whether the most recent DatabaseAccount refresh returned thinClientReadableLocations. Surface a package-private accessor that returns the underlying AtomicBoolean so downstream code (e.g. the HTTP/2 PING gate) can capture a live reference without holding a strong ref to the manager itself. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../implementation/GlobalEndpointManager.java | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/GlobalEndpointManager.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/GlobalEndpointManager.java index e313b5219713..aae55ca6c9ac 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/GlobalEndpointManager.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/GlobalEndpointManager.java @@ -383,6 +383,29 @@ public boolean hasThinClientReadLocations() { return this.hasThinClientReadLocations.get(); } + /** + * Returns the underlying {@link AtomicBoolean} that backs + * {@link #hasThinClientReadLocations()}. + * + *

This is intentionally package-private and exposes the live reference (not + * a copy) so that components which need a long-lived "thin-client effectively + * configured" signal (for example, the gateway {@code HttpClient} used to scope + * HTTP/2 PING keepalive installation) can capture only the {@code AtomicBoolean} + * via a bound method reference (e.g. {@code ref::get}) rather than capturing + * this {@code GlobalEndpointManager} instance. + * + *

Capturing {@code GlobalEndpointManager} would transitively pin + * {@link DatabaseAccountManagerInternal owner} (which is the owning + * {@code RxDocumentClientImpl}, and therefore the {@code CosmosAsyncClient}) + * for as long as the supplier remains reachable. Capturing only the + * {@code AtomicBoolean} avoids that strong-reference chain while still + * observing live state updates, since this manager mutates the same + * {@code AtomicBoolean} in place on every DatabaseAccount refresh. + */ + AtomicBoolean getHasThinClientReadLocationsRef() { + return this.hasThinClientReadLocations; + } + private Mono getDatabaseAccountAsync(URI serviceEndpoint) { return this.owner.getDatabaseAccountFromEndpoint(serviceEndpoint) .doOnNext(databaseAccount -> { From 349627a957bb74a9c2033db1bd5a985174c0ac13 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Tue, 2 Jun 2026 18:53:27 -0400 Subject: [PATCH 34/62] Reserve UserAgentFeatureFlags bit 6 for Http2PingHealth Adds Http2PingHealth = 1 << 6 to the user-agent feature flag bitmap so telemetry can attribute requests to clients that have the HTTP/2 PING health check actively installed. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../azure/cosmos/implementation/UserAgentFeatureFlags.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/UserAgentFeatureFlags.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/UserAgentFeatureFlags.java index 05dfc14703a5..450ee186186f 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/UserAgentFeatureFlags.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/UserAgentFeatureFlags.java @@ -20,7 +20,11 @@ public enum UserAgentFeatureFlags { ThinClient(1 << 2), // BinaryEncoding(1 << 3), Http2(1 << 4), - RegionScopedSessionCapturing(1 << 5); + RegionScopedSessionCapturing(1 << 5), + // Bit 6 (1 << 6) is reserved in the Java SDK for HTTP/2 PING keepalive health. + // Cross-SDK reservation in .NET (UserAgentFeatureFlags.cs) will be tracked + // separately and is not a blocker for surfacing this flag in Java telemetry. + Http2PingHealth(1 << 6); private final int value; From a1d0327be3f00f69ffc3d816949eec975d7b4f40 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Tue, 2 Jun 2026 18:53:43 -0400 Subject: [PATCH 35/62] Scope HTTP/2 PING health check to thin-client-enabled clients Wires the late-bound BooleanSupplier into the reactor HTTP client after globalEndpointManager.init(), using the live AtomicBoolean from GlobalEndpointManager so the gate flips whenever a DatabaseAccount refresh reveals (or removes) thinClientReadableLocations. Adds isHttp2PingHealthEffectivelyEnabled() which combines: (a) HTTP/2 enabled, (b) PING properties configured, (c) DA exposes thin-client read locations, (d) the underlying HttpClient is not a SharedGatewayHttpClient. The same predicate gates the Http2PingHealth user-agent flag so telemetry stays in sync with runtime behavior. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../implementation/RxDocumentClientImpl.java | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java index 11183bf31ac0..ff1ca54928a9 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java @@ -891,6 +891,22 @@ public void init(CosmosClientMetadataCachesSnapshot metadataCachesSnapshot, Func this.globalEndpointManager.init(); this.initializePerPartitionCircuitBreaker(); + // Wire the gateway HttpClient's HTTP/2 PING scope supplier to track the + // GlobalEndpointManager's live "hasThinClientReadLocations" signal. + // + // MEMORY SAFETY: We deliberately capture only the AtomicBoolean reference + // (via the bound method reference 'hasThinClientReadLocationsRef::get'), + // NOT the GlobalEndpointManager. GlobalEndpointManager retains + // DatabaseAccountManagerInternal owner, which IS this RxDocumentClientImpl + // (and therefore the owning CosmosAsyncClient). Capturing GEM in the + // supplier closure -- which is stored on the long-lived + // ReactorNettyClient pipeline lambda graph -- would pin the entire client + // and break close()-driven cleanup. The AtomicBoolean is mutated in place + // on every DatabaseAccount refresh, so the supplier still sees live values. + AtomicBoolean hasThinClientReadLocationsRef = + this.globalEndpointManager.getHasThinClientReadLocationsRef(); + this.reactorHttpClient.setHttp2PingScopeSupplier(hasThinClientReadLocationsRef::get); + DatabaseAccount databaseAccountSnapshot = this.initializeGatewayConfigurationReader(); this.resetSessionContainerIfNeeded(databaseAccountSnapshot); @@ -1674,9 +1690,56 @@ private void addUserAgentSuffix(UserAgentContainer userAgentContainer, Set + *

  • The global kill-switch {@link Configs#isHttp2PingHealthEnabled()} is true.
    • + *
  • HTTP/2 is effectively enabled (either via {@link Configs#isHttp2Enabled()} or + * an explicit {@link Http2ConnectionConfig#isEnabled()} override).
    • + *
  • {@link GlobalEndpointManager#hasThinClientReadLocations()} is true -- i.e. the + * account is configured to expose thin-client endpoints, which is the only place + * H2 is negotiated today.
    • + *
  • This client is NOT using {@link SharedGatewayHttpClient}; + * PING is defensively disabled on shared HttpClient instances because they + * back multiple CosmosAsyncClient instances with potentially differing + * thin-client configurations.
    • + * + */ + private boolean isHttp2PingHealthEffectivelyEnabled() { + if (!Configs.isHttp2PingHealthEnabled()) { + return false; + } + + boolean h2EffectivelyEnabled = Configs.isHttp2Enabled(); + if (this.connectionPolicy.getHttp2ConnectionConfig() != null + && this.connectionPolicy.getHttp2ConnectionConfig().isEnabled() != null) { + h2EffectivelyEnabled = this.connectionPolicy.getHttp2ConnectionConfig().isEnabled(); + } + if (!h2EffectivelyEnabled) { + return false; + } + + if (this.globalEndpointManager == null) { + return false; + } + + if (this.reactorHttpClient instanceof SharedGatewayHttpClient) { + return false; + } + + return this.globalEndpointManager.hasThinClientReadLocations(); + } + @Override public Flux> queryDatabases(String query, QueryFeedOperationState state) { return queryDatabases(new SqlQuerySpec(query), state); From a93132aab28b8f0f32de47e669311e633b59c3cf Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Tue, 2 Jun 2026 18:53:59 -0400 Subject: [PATCH 36/62] Update CHANGELOG entry for HTTP/2 PING health check scoping Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- sdk/cosmos/azure-cosmos/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/cosmos/azure-cosmos/CHANGELOG.md b/sdk/cosmos/azure-cosmos/CHANGELOG.md index 186b0ba0e726..f74c2b7f90d3 100644 --- a/sdk/cosmos/azure-cosmos/CHANGELOG.md +++ b/sdk/cosmos/azure-cosmos/CHANGELOG.md @@ -12,7 +12,7 @@ * Fixed `UnsupportedOperationException` when using `readManyByPartitionKeys` for empty pages. - See [PR 49311](https://github.com/Azure/azure-sdk-for-java/pull/49311) #### Other Changes -* Added HTTP/2 PING keepalive handler to proactively detect and close broken idle connections. - See [PR 49095](https://github.com/Azure/azure-sdk-for-java/pull/49095) +* Added HTTP/2 PING keepalive handler to proactively detect and close broken idle connections. The handler is defensively scoped to install only on thin-client-enabled accounts (gated on `GlobalEndpointManager.hasThinClientReadLocations()`), and a new `Http2PingHealth` user-agent feature flag is emitted when PING is effectively active. - See [PR 49095](https://github.com/Azure/azure-sdk-for-java/pull/49095) * Replaced per-client `Schedulers.newSingle()` schedulers in `GlobalEndpointManager` and `GlobalPartitionEndpointManagerForPerPartitionCircuitBreaker` with shared `BoundedElastic` schedulers in `CosmosSchedulers` to prevent thread count from scaling linearly with client/tenant count. - See [PR 49062](https://github.com/Azure/azure-sdk-for-java/pull/49062) * Fixed a sporadic `NullPointerException` in `JsonSerializable.getWithMapping` triggered by concurrent first-time calls to `DatabaseAccount.getConsistencyPolicy()` and its sibling lazy getters (`getReplicationPolicy`, `getSystemReplicationPolicy`, `getQueryEngineConfiguration`). The fix makes `JsonSerializable.propertyBag` `final`, closing an unsafe-publication race in the lazy-initialisation pattern. - See [Issue 49256](https://github.com/Azure/azure-sdk-for-java/issues/49256) and [PR #49258](https://github.com/Azure/azure-sdk-for-java/pull/49258) From 8277c6138029634e4665a39eda6d8bf2b4d634a3 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Tue, 2 Jun 2026 18:54:16 -0400 Subject: [PATCH 37/62] Update Http2PingKeepaliveTest for thin-client scoping Sets COSMOS.THINCLIENT_ENABLED=true alongside HTTP2_ENABLED in @BeforeClass so the DatabaseAccount refresh returns thinClientReadableLocations and the PING handler actually installs. Switches the iptables blackhole from port 443 on the gateway hostname to port 10250 with no host filter, since thin-client H2 traffic flows over port 10250 at regional hostnames - this is what the PING handler is monitoring. Verified end-to-end against thin-client-mr-bs-ci: connection 8184be81 evicted after PING timeouts, recovery read used fresh connection 584f4d29. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Http2PingKeepaliveTest.java | 32 +++++++++++-------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java index aa9388aa184f..95bc419e502a 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java @@ -34,8 +34,10 @@ * the broken connection and a subsequent request uses a new connection. *

    * Run in Docker with --cap-add=NET_ADMIN (group: manual-http-network-fault). - * Requires: HTTP/2 capable Cosmos DB account. - * Does NOT require thin client -- only COSMOS.HTTP2_ENABLED=true. + * Requires: thin-client-enabled Cosmos DB account whose DatabaseAccount response + * exposes {@code thinClientReadableLocations}. PING is scoped to thin-client + * endpoints only, so both {@code COSMOS.HTTP2_ENABLED=true} and + * {@code COSMOS.THINCLIENT_ENABLED=true} must be set before client construction. */ public class Http2PingKeepaliveTest extends FaultInjectionTestBase { @@ -59,8 +61,10 @@ public Http2PingKeepaliveTest() { @BeforeClass(groups = {"manual-http-network-fault"}, timeOut = 120_000) public void beforeClass() { - // Enable HTTP/2 -- thin client is NOT required for PING tests + // Enable HTTP/2 and thin client BEFORE client construction. + // PING is scoped to thin-client endpoints; both flags are required. System.setProperty("COSMOS.HTTP2_ENABLED", "true"); + System.setProperty("COSMOS.THINCLIENT_ENABLED", "true"); this.client = getClientBuilder().buildAsyncClient(); this.cosmosAsyncContainer = getSharedMultiPartitionCosmosContainerWithIdAsPartitionKey(this.client); @@ -75,6 +79,7 @@ public void beforeClass() { public void afterClass() { safeClose(this.client); System.clearProperty("COSMOS.HTTP2_ENABLED"); + System.clearProperty("COSMOS.THINCLIENT_ENABLED"); } @BeforeMethod(groups = {"manual-http-network-fault"}) @@ -126,18 +131,20 @@ public void connectionClosedOnPingTimeout() throws Exception { .buildAsyncClient(); this.cosmosAsyncContainer = getSharedMultiPartitionCosmosContainerWithIdAsPartitionKey(this.client); - // Warm-up read -- establish H2 connection + // Warm-up read -- establish H2 connection (over thin-client port 10250) String initialChannelId = readAndGetParentChannelId(); logger.info("Initial parentChannelId: {}", initialChannelId); - // Extract gateway host + port for iptables rule - String gatewayHost = extractHostFromEndpoint(TestConfigurations.HOST); - int gatewayPort = 443; - logger.info("Gateway host: {}, port: {}", gatewayHost, gatewayPort); + // Thin-client traffic goes to port 10250 on regional thin-client endpoints + // (e.g., thin-client-mr-bs-ci-westcentralus.documents.azure.com:10250). + // The regional hostname differs from the account-level endpoint, so we + // blackhole by port only -- catching all thin-client traffic regardless of region. + int thinClientPort = 10250; + logger.info("Will blackhole port: {} (thin-client H2)", thinClientPort); - // Blackhole all traffic to gateway -- prevents PING ACKs from arriving + // Blackhole all traffic to thin-client port -- prevents PING ACKs from arriving String iptablesRule = String.format( - "%siptables -A OUTPUT -p tcp --dport %d -d %s -j DROP", SUDO, gatewayPort, gatewayHost); + "%siptables -A OUTPUT -p tcp --dport %d -j DROP", SUDO, thinClientPort); logger.info("Installing iptables DROP rule: {}", iptablesRule); execCommand(iptablesRule); @@ -150,7 +157,7 @@ public void connectionClosedOnPingTimeout() throws Exception { // Remove iptables rule BEFORE attempting recovery read String iptablesRemove = String.format( - "%siptables -D OUTPUT -p tcp --dport %d -d %s -j DROP", SUDO, gatewayPort, gatewayHost); + "%siptables -D OUTPUT -p tcp --dport %d -j DROP", SUDO, thinClientPort); logger.info("Removing iptables DROP rule: {}", iptablesRemove); execCommand(iptablesRemove); @@ -176,9 +183,8 @@ public void connectionClosedOnPingTimeout() throws Exception { } finally { // Safety: remove any leftover iptables rules try { - String gatewayHost = extractHostFromEndpoint(TestConfigurations.HOST); String cleanup = String.format( - "%siptables -D OUTPUT -p tcp --dport 443 -d %s -j DROP 2>/dev/null", SUDO, gatewayHost); + "%siptables -D OUTPUT -p tcp --dport 10250 -j DROP 2>/dev/null", SUDO); execCommand(cleanup); } catch (Exception ignored) {} From ae034467b3922105b2e0ad070306a555ff7981b0 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Tue, 2 Jun 2026 18:54:27 -0400 Subject: [PATCH 38/62] Scope Http2NetworkFault CI stage to thin-client test account Switches the Http2NetworkFault pipeline stage from the generic test endpoint to the thin-client KeyVault secrets (thinclient-test-endpoint / thinclient-test-key) and sets COSMOS.THINCLIENT_ENABLED=true so the PING health check is actually exercised in CI. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- sdk/cosmos/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/cosmos/tests.yml b/sdk/cosmos/tests.yml index bd00d2ac9048..6e8a9c152d40 100644 --- a/sdk/cosmos/tests.yml +++ b/sdk/cosmos/tests.yml @@ -100,7 +100,7 @@ extends: TestResultsFiles: '**/junitreports/TEST-*.xml' AdditionalVariables: - name: AdditionalArgs - value: '-DCOSMOS.CLIENT_LEAK_DETECTION_ENABLED=true -DCOSMOS.HTTP2_ENABLED=true -DCOSMOS.HTTP2_PING_HEALTH_ENABLED=true -DCOSMOS.HTTP2_PING_INTERVAL_IN_SECONDS=1 -DCOSMOS.HTTP2_PING_TIMEOUT_IN_SECONDS=2' + value: '-DCOSMOS.CLIENT_LEAK_DETECTION_ENABLED=true -DACCOUNT_HOST=$(thinclient-test-endpoint) -DACCOUNT_KEY=$(thinclient-test-key) -DCOSMOS.THINCLIENT_ENABLED=true -DCOSMOS.HTTP2_ENABLED=true -DCOSMOS.HTTP2_PING_HEALTH_ENABLED=true -DCOSMOS.HTTP2_PING_INTERVAL_IN_SECONDS=1 -DCOSMOS.HTTP2_PING_TIMEOUT_IN_SECONDS=2' - template: /eng/pipelines/templates/stages/archetype-sdk-tests-isolated.yml parameters: From ab8260b5852e9ae0e575cf9b965696007ab317be Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Tue, 2 Jun 2026 19:48:21 -0400 Subject: [PATCH 39/62] Tighten CHANGELOG entry for HTTP/2 PING keepalive Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- sdk/cosmos/azure-cosmos/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/cosmos/azure-cosmos/CHANGELOG.md b/sdk/cosmos/azure-cosmos/CHANGELOG.md index f74c2b7f90d3..c7933e8224f1 100644 --- a/sdk/cosmos/azure-cosmos/CHANGELOG.md +++ b/sdk/cosmos/azure-cosmos/CHANGELOG.md @@ -12,7 +12,7 @@ * Fixed `UnsupportedOperationException` when using `readManyByPartitionKeys` for empty pages. - See [PR 49311](https://github.com/Azure/azure-sdk-for-java/pull/49311) #### Other Changes -* Added HTTP/2 PING keepalive handler to proactively detect and close broken idle connections. The handler is defensively scoped to install only on thin-client-enabled accounts (gated on `GlobalEndpointManager.hasThinClientReadLocations()`), and a new `Http2PingHealth` user-agent feature flag is emitted when PING is effectively active. - See [PR 49095](https://github.com/Azure/azure-sdk-for-java/pull/49095) +* Added ability to detect broken connections to a Gateway V2 service endpoint. - See [PR 49095](https://github.com/Azure/azure-sdk-for-java/pull/49095) * Replaced per-client `Schedulers.newSingle()` schedulers in `GlobalEndpointManager` and `GlobalPartitionEndpointManagerForPerPartitionCircuitBreaker` with shared `BoundedElastic` schedulers in `CosmosSchedulers` to prevent thread count from scaling linearly with client/tenant count. - See [PR 49062](https://github.com/Azure/azure-sdk-for-java/pull/49062) * Fixed a sporadic `NullPointerException` in `JsonSerializable.getWithMapping` triggered by concurrent first-time calls to `DatabaseAccount.getConsistencyPolicy()` and its sibling lazy getters (`getReplicationPolicy`, `getSystemReplicationPolicy`, `getQueryEngineConfiguration`). The fix makes `JsonSerializable.propertyBag` `final`, closing an unsafe-publication race in the lazy-initialisation pattern. - See [Issue 49256](https://github.com/Azure/azure-sdk-for-java/issues/49256) and [PR #49258](https://github.com/Azure/azure-sdk-for-java/pull/49258) From 493838f9e96c647ce721e709565f8fa0ef98f01b Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Tue, 2 Jun 2026 20:16:03 -0400 Subject: [PATCH 40/62] Re-check PING scope per tick in Http2PingHandler Thread the per-client BooleanSupplier scopeSupplier from ReactorNettyClient into Http2PingHandler so maybeSendPing() re-evaluates it on every interval. Closes the gap where an already-installed handler kept PINGing after the account's DatabaseAccount refresh dropped thinClientReadableLocations (e.g., thin-client flipped off at runtime). When the supplier returns false the next tick cancels the scheduled task; the connection stays in the pool until normal idle eviction. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../implementation/http/Http2PingHandler.java | 31 ++++++++++++++++--- .../http/ReactorNettyClient.java | 16 ++++++---- 2 files changed, 37 insertions(+), 10 deletions(-) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java index 329537c95365..0bf094711c94 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java @@ -13,6 +13,7 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; +import java.util.function.BooleanSupplier; /** * Manual HTTP/2 PING keepalive handler installed on the parent (TCP) channel. @@ -44,6 +45,10 @@ public class Http2PingHandler extends ChannelDuplexHandler { private final long pingIntervalNanos; private final long pingTimeoutNanos; private final int failureThreshold; + // Re-checked per tick so that if the scope condition (e.g. the account's + // thin-client configuration) goes away at runtime, the handler stops + // sending PINGs even on connections that already had it installed. + private final BooleanSupplier scopeSupplier; // Mutable fields below are accessed only from the channel's EventLoop thread // (handlerAdded, channelRead, scheduled task, writeAndFlush listener), so no @@ -58,11 +63,16 @@ public class Http2PingHandler extends ChannelDuplexHandler { * @param pingIntervalSeconds interval in seconds; when idle longer than this, a PING is sent * @param pingTimeoutSeconds timeout in seconds per PING attempt * @param failureThreshold consecutive timeouts before closing the connection + * @param scopeSupplier re-checked per tick; when it returns false the handler cancels + * itself, so a runtime scope change (e.g. account losing thin-client + * read locations) stops PINGs on already-installed handlers */ - public Http2PingHandler(int pingIntervalSeconds, int pingTimeoutSeconds, int failureThreshold) { + public Http2PingHandler(int pingIntervalSeconds, int pingTimeoutSeconds, int failureThreshold, + BooleanSupplier scopeSupplier) { this.pingIntervalNanos = TimeUnit.SECONDS.toNanos(Math.max(1, pingIntervalSeconds)); this.pingTimeoutNanos = TimeUnit.SECONDS.toNanos(Math.max(1, pingTimeoutSeconds)); this.failureThreshold = Math.max(1, failureThreshold); + this.scopeSupplier = scopeSupplier != null ? scopeSupplier : () -> true; this.lastReadNanos = System.nanoTime(); } @@ -106,7 +116,15 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception } private void maybeSendPing(ChannelHandlerContext ctx) { - if (!ctx.channel().isActive() || !Configs.isHttp2PingHealthEnabled()) { + // Re-check both gates per tick: + // - Configs.isHttp2PingHealthEnabled(): global kill-switch (system property). + // - scopeSupplier: per-client scope (e.g. thin-client still active). If the + // account drops thin-client read locations after the handler was installed, + // this flips false and we cancel the scheduled task. The connection itself + // stays in the pool and is reaped by the normal idle timeout. + if (!ctx.channel().isActive() + || !Configs.isHttp2PingHealthEnabled() + || !scopeSupplier.getAsBoolean()) { cancelPingTask(); return; } @@ -170,14 +188,19 @@ private void cancelPingTask() { * @param pingIntervalSeconds PING interval in seconds * @param pingTimeoutSeconds PING ACK timeout in seconds * @param failureThreshold consecutive timeouts before closing + * @param scopeSupplier re-checked per tick by the handler; when it returns false + * the handler cancels itself so runtime scope changes + * (e.g. account losing thin-client read locations) + * disable PING even on already-installed handlers */ public static void installIfAbsent(Channel channel, int pingIntervalSeconds, int pingTimeoutSeconds, - int failureThreshold) { + int failureThreshold, BooleanSupplier scopeSupplier) { Channel parent = channel.parent() != null ? channel.parent() : channel; if (parent.pipeline().get(HANDLER_NAME) == null) { try { parent.pipeline().addLast(HANDLER_NAME, - new Http2PingHandler(pingIntervalSeconds, pingTimeoutSeconds, failureThreshold)); + new Http2PingHandler(pingIntervalSeconds, pingTimeoutSeconds, failureThreshold, + scopeSupplier)); } catch (IllegalArgumentException ignored) { // Duplicate -- race between concurrent streams, benign } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java index 759b8a2defa5..ebddc485db78 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java @@ -45,6 +45,9 @@ * HttpClient that is implemented using reactor-netty. */ public class ReactorNettyClient implements HttpClient { + private static ImplementationBridgeHelpers.Http2ConnectionConfigHelper.Http2ConnectionConfigAccessor http2CfgAccessor() { + return ImplementationBridgeHelpers.Http2ConnectionConfigHelper.getHttp2ConnectionConfigAccessor(); + } private static final boolean leakDetectionDebuggingEnabled = ResourceLeakDetector.getLevel().ordinal() >= ResourceLeakDetector.Level.ADVANCED.ordinal(); @@ -52,10 +55,6 @@ public class ReactorNettyClient implements HttpClient { private static final Logger logger = LoggerFactory.getLogger(ReactorNettyClient.class.getSimpleName()); - private static ImplementationBridgeHelpers.Http2ConnectionConfigHelper.Http2ConnectionConfigAccessor http2CfgAccessor() { - return ImplementationBridgeHelpers.Http2ConnectionConfigHelper.getHttp2ConnectionConfigAccessor(); - } - private HttpClientConfig httpClientConfig; private reactor.netty.http.client.HttpClient httpClient; private ConnectionProvider connectionProvider; @@ -184,8 +183,13 @@ private void configureChannelPipelineHandlers() { int pingTimeoutSeconds = Configs.getHttp2PingTimeoutInSeconds(); int pingFailureThreshold = Configs.getHttp2PingFailureThreshold(); if (pingIntervalSeconds > 0) { + // Pass the same scope supplier into the handler so it re-checks + // per tick. If the account flips thin-client off after the handler + // was installed, the supplier returns false on the next scheduled + // tick and the handler cancels itself. + BooleanSupplier scopeForHandler = this.http2PingScopeSupplier; Http2PingHandler.installIfAbsent(parent, pingIntervalSeconds, - pingTimeoutSeconds, pingFailureThreshold); + pingTimeoutSeconds, pingFailureThreshold, scopeForHandler); } } } @@ -257,7 +261,7 @@ public Mono send(final HttpRequest request, Duration responseTimeo final AtomicReference responseReference = new AtomicReference<>(); // Per-request CONNECT_TIMEOUT_MILLIS via reactor-netty's immutable HttpClient. - // .option() returns a new config snapshot -- does NOT mutate the shared httpClient. + // .option() returns a new config snapshot — does NOT mutate the shared httpClient. // Thin client requests (isThinClientRequest=true): connect timeout is configured via // HttpClientConfig.getThinClientConnectTimeoutMs() (default 5s) to fail fast. // Standard gateway requests: 45s (default). From 05300f25b0f442779c47a851214e63d3fb305c67 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Tue, 2 Jun 2026 21:27:32 -0400 Subject: [PATCH 41/62] Address PR review feedback: ACK payload match, dormant scope-off, JSON parentChannelId extractor, unit tests - Http2PingHandler: rename lastReadNanos -> lastActivityNanos with clarifying comment; match PING ACK by payload (RFC 9113 6.7); make scope-off / kill-switch dormant instead of cancelling the timer so the handler auto-rearms when scope re-enables - RxDocumentClientImpl: suppress Http2PingHealth UA flag when ping interval <= 0 (the handler's install gate clamps to 1s, but the UA flag should honor intent) - Http2PingKeepaliveTest: @AfterClass alwaysRun=true; replace brittle substring parentChannelId extractor with JSON walk over gatewayStatisticsList; remove dead extractHostFromEndpoint - Add Http2PingHandlerTest: 6 EmbeddedChannel-based unit tests for ACK matching, mismatched ACK ignored, idempotent install, ctor clamping, supplier null-safety Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Http2PingKeepaliveTest.java | 45 +++--- .../http/Http2PingHandlerTest.java | 132 ++++++++++++++++++ .../implementation/RxDocumentClientImpl.java | 7 + .../implementation/http/Http2PingHandler.java | 64 +++++---- 4 files changed, 202 insertions(+), 46 deletions(-) create mode 100644 sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/http/Http2PingHandlerTest.java diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java index 95bc419e502a..c9380aebf6a6 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java @@ -11,8 +11,12 @@ import com.azure.cosmos.GatewayConnectionConfig; import com.azure.cosmos.TestObject; import com.azure.cosmos.implementation.TestConfigurations; +import com.azure.cosmos.implementation.Utils; import com.azure.cosmos.models.CosmosItemResponse; import com.azure.cosmos.models.PartitionKey; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.annotations.AfterClass; @@ -75,7 +79,7 @@ public void beforeClass() { logger.info("Seed item created: {}", this.seedItem.getId()); } - @AfterClass(groups = {"manual-http-network-fault"}, timeOut = 60_000) + @AfterClass(groups = {"manual-http-network-fault"}, timeOut = 60_000, alwaysRun = true) public void afterClass() { safeClose(this.client); System.clearProperty("COSMOS.HTTP2_ENABLED"); @@ -195,7 +199,7 @@ public void connectionClosedOnPingTimeout() throws Exception { } } - private String readAndGetParentChannelId() { + private String readAndGetParentChannelId() throws JsonProcessingException { CosmosItemResponse response = this.cosmosAsyncContainer.readItem( seedItem.getId(), new PartitionKey(seedItem.getId()), TestObject.class).block(); @@ -205,28 +209,25 @@ private String readAndGetParentChannelId() { return extractParentChannelId(response.getDiagnostics()); } - private String extractParentChannelId(CosmosDiagnostics diagnostics) { - String diagStr = diagnostics.toString(); - int idx = diagStr.indexOf("parentChannelId"); - if (idx > 0) { - int start = diagStr.indexOf("\"", idx + 16) + 1; - int end = diagStr.indexOf("\"", start); - if (start > 0 && end > start) { - return diagStr.substring(start, end); + /** + * Mirrors {@code Http2ConnectionLifecycleTests#extractParentChannelId} -- parse + * the diagnostics JSON rather than substring-scan the toString() so a future change + * to JSON formatting can't silently break the test. + */ + private String extractParentChannelId(CosmosDiagnostics diagnostics) throws JsonProcessingException { + ObjectNode node = (ObjectNode) Utils.getSimpleObjectMapper().readTree(diagnostics.toString()); + JsonNode gwStats = node.get("gatewayStatisticsList"); + if (gwStats != null && gwStats.isArray()) { + for (JsonNode stat : gwStats) { + if (stat.has("parentChannelId")) { + String id = stat.get("parentChannelId").asText(); + if (id != null && !id.isEmpty() && !"null".equals(id)) { + return id; + } + } } } - - throw new AssertionError("Could not extract parentChannelId from diagnostics: " + diagStr); - } - - private static String extractHostFromEndpoint(String endpoint) { - // Extract hostname from "https://foo.documents.azure.com:443/" - try { - java.net.URI uri = new java.net.URI(endpoint); - return uri.getHost(); - } catch (Exception e) { - throw new RuntimeException("Failed to parse endpoint: " + endpoint, e); - } + throw new AssertionError("Could not extract parentChannelId from diagnostics: " + diagnostics); } private static void execCommand(String command) throws Exception { diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/http/Http2PingHandlerTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/http/Http2PingHandlerTest.java new file mode 100644 index 000000000000..b97fd1dedd43 --- /dev/null +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/http/Http2PingHandlerTest.java @@ -0,0 +1,132 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos.implementation.http; + +import io.netty.channel.embedded.EmbeddedChannel; +import io.netty.handler.codec.http2.DefaultHttp2PingFrame; +import org.testng.annotations.Test; + +import java.lang.reflect.Field; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Unit tests for {@link Http2PingHandler}. + *

    + * Covers state transitions that do not require advancing time: + *

      + *
    • PING ACK with matching payload resets the failure counter (RFC 9113 §6.7 payload echo).
      • + *
    • PING ACK with mismatched payload is ignored (late ACK after timeout cannot mask degradation).
      • + *
    • {@code installIfAbsent} is idempotent.
      • + *
    • Constructor clamps non-positive interval / timeout / threshold to safe minimums.
      • + *
    • Null {@code scopeSupplier} is treated as always-true.
      • + *
    + * Time-based behaviors (interval-driven PING send, timeout-driven failure increment, threshold-driven + * close) are exercised by the integration test {@code Http2PingKeepaliveTest} under Docker with real + * network fault injection. + */ +public class Http2PingHandlerTest { + + @Test(groups = "unit") + public void ackWithMatchingPayload_resetsState() throws Exception { + Http2PingHandler handler = new Http2PingHandler(1, 1, 3, () -> true); + EmbeddedChannel channel = new EmbeddedChannel(handler); + + // Simulate an outstanding PING #5 that recorded one failure. + setField(handler, "pingsSent", 5); + setField(handler, "pingOutstandingSinceNanos", System.nanoTime()); + setField(handler, "consecutiveFailures", 1); + + channel.writeInbound(new DefaultHttp2PingFrame(5L, true)); + + assertThat((long) getField(handler, "pingOutstandingSinceNanos")).isZero(); + assertThat((int) getField(handler, "consecutiveFailures")).isZero(); + + channel.finishAndReleaseAll(); + } + + @Test(groups = "unit") + public void ackWithMismatchedPayload_doesNotResetState() throws Exception { + Http2PingHandler handler = new Http2PingHandler(1, 1, 3, () -> true); + EmbeddedChannel channel = new EmbeddedChannel(handler); + + long outstandingAt = System.nanoTime(); + setField(handler, "pingsSent", 5); + setField(handler, "pingOutstandingSinceNanos", outstandingAt); + setField(handler, "consecutiveFailures", 2); + + // Late ACK for an earlier PING #3 -- must NOT clear the failure counter + // that is tracking the currently outstanding PING #5. + channel.writeInbound(new DefaultHttp2PingFrame(3L, true)); + + assertThat((long) getField(handler, "pingOutstandingSinceNanos")).isEqualTo(outstandingAt); + assertThat((int) getField(handler, "consecutiveFailures")).isEqualTo(2); + + channel.finishAndReleaseAll(); + } + + @Test(groups = "unit") + public void installIfAbsent_isIdempotent() { + EmbeddedChannel channel = new EmbeddedChannel(); + + Http2PingHandler.installIfAbsent(channel, 1, 1, 3, () -> true); + Http2PingHandler.installIfAbsent(channel, 1, 1, 3, () -> true); + + long count = channel.pipeline().toMap().values().stream() + .filter(h -> h instanceof Http2PingHandler) + .count(); + assertThat(count).isEqualTo(1); + + channel.finishAndReleaseAll(); + } + + @Test(groups = "unit") + public void constructor_clampsNonPositiveValues() throws Exception { + Http2PingHandler handler = new Http2PingHandler(-5, 0, -1, () -> true); + + assertThat((long) getField(handler, "pingIntervalNanos")).isEqualTo(TimeUnit.SECONDS.toNanos(1)); + assertThat((long) getField(handler, "pingTimeoutNanos")).isEqualTo(TimeUnit.SECONDS.toNanos(1)); + assertThat((int) getField(handler, "failureThreshold")).isEqualTo(1); + } + + @Test(groups = "unit") + public void nullScopeSupplier_treatedAsAlwaysTrue() throws Exception { + Http2PingHandler handler = new Http2PingHandler(1, 1, 3, null); + + Object supplier = getField(handler, "scopeSupplier"); + assertThat(supplier).isNotNull(); + java.util.function.BooleanSupplier bs = (java.util.function.BooleanSupplier) supplier; + assertThat(bs.getAsBoolean()).isTrue(); + } + + @Test(groups = "unit") + public void scopeSupplier_evaluatedPerCall() { + AtomicBoolean scope = new AtomicBoolean(true); + Http2PingHandler handler = new Http2PingHandler(1, 1, 3, scope::get); + EmbeddedChannel channel = new EmbeddedChannel(handler); + + // Flipping the supplier should be observed on the next read by the handler -- + // we just verify it's the same supplier instance, since the per-tick check is + // exercised end-to-end by Http2PingKeepaliveTest under Docker. + assertThat(channel.pipeline().get(Http2PingHandler.class)).isSameAs(handler); + scope.set(false); + scope.set(true); + + channel.finishAndReleaseAll(); + } + + private static Object getField(Object target, String name) throws Exception { + Field f = Http2PingHandler.class.getDeclaredField(name); + f.setAccessible(true); + return f.get(target); + } + + private static void setField(Object target, String name, Object value) throws Exception { + Field f = Http2PingHandler.class.getDeclaredField(name); + f.setAccessible(true); + f.set(target, value); + } +} diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java index ff1ca54928a9..44e6c09742e0 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java @@ -1720,6 +1720,13 @@ private boolean isHttp2PingHealthEffectivelyEnabled() { return false; } + // Mirror the transport-side guard in Http2PingHandler#installIfAbsent: if the + // configured interval is non-positive the handler is not installed, so the UA + // flag must not advertise PING either. + if (Configs.getHttp2PingIntervalInSeconds() <= 0) { + return false; + } + boolean h2EffectivelyEnabled = Configs.isHttp2Enabled(); if (this.connectionPolicy.getHttp2ConnectionConfig() != null && this.connectionPolicy.getHttp2ConnectionConfig().isEnabled() != null) { diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java index 0bf094711c94..09a3fa11da54 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java @@ -53,7 +53,13 @@ public class Http2PingHandler extends ChannelDuplexHandler { // Mutable fields below are accessed only from the channel's EventLoop thread // (handlerAdded, channelRead, scheduled task, writeAndFlush listener), so no // synchronization or volatile is needed. - private long lastReadNanos; // nanoTime of last inbound frame (response); PING triggers when no read for interval + // + // nanoTime of the last connection-level activity (inbound frame or PING send). + // Note: H2 stream frames (HEADERS/DATA -- i.e. actual HTTP responses) are + // dispatched by Http2MultiplexHandler to child channels and never surface on + // the parent pipeline, so this does NOT track request/response traffic; it + // tracks PING ACKs, SETTINGS, GOAWAY, and our own PING sends. + private long lastActivityNanos; private long pingOutstandingSinceNanos; // nanoTime when PING was sent; 0 = no outstanding PING private int consecutiveFailures; // incremented on timeout, reset on ACK private int pingsSent; @@ -63,9 +69,10 @@ public class Http2PingHandler extends ChannelDuplexHandler { * @param pingIntervalSeconds interval in seconds; when idle longer than this, a PING is sent * @param pingTimeoutSeconds timeout in seconds per PING attempt * @param failureThreshold consecutive timeouts before closing the connection - * @param scopeSupplier re-checked per tick; when it returns false the handler cancels - * itself, so a runtime scope change (e.g. account losing thin-client - * read locations) stops PINGs on already-installed handlers + * @param scopeSupplier re-checked per tick; when it returns false the handler stops + * sending PINGs but keeps the timer alive (dormant), so a runtime + * scope flip back to true automatically re-arms PING on the same + * connection without waiting for connection recycling */ public Http2PingHandler(int pingIntervalSeconds, int pingTimeoutSeconds, int failureThreshold, BooleanSupplier scopeSupplier) { @@ -73,7 +80,7 @@ public Http2PingHandler(int pingIntervalSeconds, int pingTimeoutSeconds, int fai this.pingTimeoutNanos = TimeUnit.SECONDS.toNanos(Math.max(1, pingTimeoutSeconds)); this.failureThreshold = Math.max(1, failureThreshold); this.scopeSupplier = scopeSupplier != null ? scopeSupplier : () -> true; - this.lastReadNanos = System.nanoTime(); + this.lastActivityNanos = System.nanoTime(); } @Override @@ -107,27 +114,35 @@ public void channelInactive(ChannelHandlerContext ctx) throws Exception { @Override public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { - lastReadNanos = System.nanoTime(); - if (msg instanceof Http2PingFrame && ((Http2PingFrame) msg).ack()) { - pingOutstandingSinceNanos = 0; - consecutiveFailures = 0; + lastActivityNanos = System.nanoTime(); + if (msg instanceof Http2PingFrame) { + Http2PingFrame ping = (Http2PingFrame) msg; + // RFC 9113 §6.7: PING ACK echoes the 8-byte payload we sent. + // Match by payload so a late ACK for a PING that already counted as a + // timeout cannot reset the failure counter and mask ongoing degradation. + if (ping.ack() && ping.content() == pingsSent) { + pingOutstandingSinceNanos = 0; + consecutiveFailures = 0; + } } super.channelRead(ctx, msg); } private void maybeSendPing(ChannelHandlerContext ctx) { - // Re-check both gates per tick: - // - Configs.isHttp2PingHealthEnabled(): global kill-switch (system property). - // - scopeSupplier: per-client scope (e.g. thin-client still active). If the - // account drops thin-client read locations after the handler was installed, - // this flips false and we cancel the scheduled task. The connection itself - // stays in the pool and is reaped by the normal idle timeout. - if (!ctx.channel().isActive() - || !Configs.isHttp2PingHealthEnabled() - || !scopeSupplier.getAsBoolean()) { + // Channel-dead is a hard stop -- cancel the timer. + if (!ctx.channel().isActive()) { cancelPingTask(); return; } + // Two soft gates re-checked per tick: + // - Configs.isHttp2PingHealthEnabled(): global kill-switch (system property). + // - scopeSupplier: per-client scope (e.g. thin-client still active). + // Either flipping false makes this tick a no-op but KEEPS the timer alive, so + // if the gate flips back to true the handler automatically resumes PINGing on + // the same connection (no need to wait for connection recycling). + if (!Configs.isHttp2PingHealthEnabled() || !scopeSupplier.getAsBoolean()) { + return; + } // If a previous PING is still outstanding, check whether it has timed out if (pingOutstandingSinceNanos != 0) { @@ -151,7 +166,7 @@ private void maybeSendPing(ChannelHandlerContext ctx) { return; } - long idleNanos = System.nanoTime() - lastReadNanos; + long idleNanos = System.nanoTime() - lastActivityNanos; if (idleNanos >= pingIntervalNanos) { int count = ++pingsSent; pingOutstandingSinceNanos = System.nanoTime(); @@ -168,8 +183,8 @@ private void maybeSendPing(ChannelHandlerContext ctx) { f.cause() != null ? f.cause().getMessage() : "unknown"); } }); - // Reset read timestamp so we don't send another PING immediately - lastReadNanos = System.nanoTime(); + // Reset activity timestamp so we don't send another PING immediately + lastActivityNanos = System.nanoTime(); } } @@ -189,9 +204,10 @@ private void cancelPingTask() { * @param pingTimeoutSeconds PING ACK timeout in seconds * @param failureThreshold consecutive timeouts before closing * @param scopeSupplier re-checked per tick by the handler; when it returns false - * the handler cancels itself so runtime scope changes - * (e.g. account losing thin-client read locations) - * disable PING even on already-installed handlers + * the tick is a no-op (dormant) so runtime scope changes + * (e.g. account losing thin-client read locations) stop + * PINGs immediately, but the timer stays alive so a flip + * back to true automatically re-arms on the same connection */ public static void installIfAbsent(Channel channel, int pingIntervalSeconds, int pingTimeoutSeconds, int failureThreshold, BooleanSupplier scopeSupplier) { From d33d3110db7692410613e820d18f681c3a8c0590 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Thu, 4 Jun 2026 14:44:49 -0400 Subject: [PATCH 42/62] Remove HTTP/2 PING thin-client scoping PING is now installed on any H2 parent channel when the global gate (Configs.isHttp2PingHealthEnabled + H2 enabled + valid interval) is set, not just when the account exposes thin-client read locations. - Drop scopeSupplier from Http2PingHandler + installIfAbsent - Drop http2PingScopeSupplier / setHttp2PingScopeSupplier from HttpClient / ReactorNettyClient / SharedGatewayHttpClient - Drop hasThinClientReadLocations / SharedGateway gates from isHttp2PingHealthEffectivelyEnabled - Drop getHasThinClientReadLocationsRef from GlobalEndpointManager - Update Http2PingHandlerTest (remove 2 obsolete tests, simplify rest) Routing-side hasThinClientReadLocations() use (useThinClientStoreModel) remains untouched. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../http/Http2PingHandlerTest.java | 38 ++------------- .../implementation/GlobalEndpointManager.java | 23 --------- .../implementation/RxDocumentClientImpl.java | 37 +-------------- .../implementation/http/Http2PingHandler.java | 36 ++++---------- .../implementation/http/HttpClient.java | 22 --------- .../http/ReactorNettyClient.java | 47 ++----------------- .../http/SharedGatewayHttpClient.java | 13 ----- 7 files changed, 20 insertions(+), 196 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/http/Http2PingHandlerTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/http/Http2PingHandlerTest.java index b97fd1dedd43..6c78b8f80163 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/http/Http2PingHandlerTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/http/Http2PingHandlerTest.java @@ -9,7 +9,6 @@ import java.lang.reflect.Field; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; import static org.assertj.core.api.Assertions.assertThat; @@ -22,7 +21,6 @@ *
  • PING ACK with mismatched payload is ignored (late ACK after timeout cannot mask degradation).
    • *
  • {@code installIfAbsent} is idempotent.
    • *
  • Constructor clamps non-positive interval / timeout / threshold to safe minimums.
    • - *
  • Null {@code scopeSupplier} is treated as always-true.
    • * * Time-based behaviors (interval-driven PING send, timeout-driven failure increment, threshold-driven * close) are exercised by the integration test {@code Http2PingKeepaliveTest} under Docker with real @@ -32,7 +30,7 @@ public class Http2PingHandlerTest { @Test(groups = "unit") public void ackWithMatchingPayload_resetsState() throws Exception { - Http2PingHandler handler = new Http2PingHandler(1, 1, 3, () -> true); + Http2PingHandler handler = new Http2PingHandler(1, 1, 3); EmbeddedChannel channel = new EmbeddedChannel(handler); // Simulate an outstanding PING #5 that recorded one failure. @@ -50,7 +48,7 @@ public void ackWithMatchingPayload_resetsState() throws Exception { @Test(groups = "unit") public void ackWithMismatchedPayload_doesNotResetState() throws Exception { - Http2PingHandler handler = new Http2PingHandler(1, 1, 3, () -> true); + Http2PingHandler handler = new Http2PingHandler(1, 1, 3); EmbeddedChannel channel = new EmbeddedChannel(handler); long outstandingAt = System.nanoTime(); @@ -72,8 +70,8 @@ public void ackWithMismatchedPayload_doesNotResetState() throws Exception { public void installIfAbsent_isIdempotent() { EmbeddedChannel channel = new EmbeddedChannel(); - Http2PingHandler.installIfAbsent(channel, 1, 1, 3, () -> true); - Http2PingHandler.installIfAbsent(channel, 1, 1, 3, () -> true); + Http2PingHandler.installIfAbsent(channel, 1, 1, 3); + Http2PingHandler.installIfAbsent(channel, 1, 1, 3); long count = channel.pipeline().toMap().values().stream() .filter(h -> h instanceof Http2PingHandler) @@ -85,39 +83,13 @@ public void installIfAbsent_isIdempotent() { @Test(groups = "unit") public void constructor_clampsNonPositiveValues() throws Exception { - Http2PingHandler handler = new Http2PingHandler(-5, 0, -1, () -> true); + Http2PingHandler handler = new Http2PingHandler(-5, 0, -1); assertThat((long) getField(handler, "pingIntervalNanos")).isEqualTo(TimeUnit.SECONDS.toNanos(1)); assertThat((long) getField(handler, "pingTimeoutNanos")).isEqualTo(TimeUnit.SECONDS.toNanos(1)); assertThat((int) getField(handler, "failureThreshold")).isEqualTo(1); } - @Test(groups = "unit") - public void nullScopeSupplier_treatedAsAlwaysTrue() throws Exception { - Http2PingHandler handler = new Http2PingHandler(1, 1, 3, null); - - Object supplier = getField(handler, "scopeSupplier"); - assertThat(supplier).isNotNull(); - java.util.function.BooleanSupplier bs = (java.util.function.BooleanSupplier) supplier; - assertThat(bs.getAsBoolean()).isTrue(); - } - - @Test(groups = "unit") - public void scopeSupplier_evaluatedPerCall() { - AtomicBoolean scope = new AtomicBoolean(true); - Http2PingHandler handler = new Http2PingHandler(1, 1, 3, scope::get); - EmbeddedChannel channel = new EmbeddedChannel(handler); - - // Flipping the supplier should be observed on the next read by the handler -- - // we just verify it's the same supplier instance, since the per-tick check is - // exercised end-to-end by Http2PingKeepaliveTest under Docker. - assertThat(channel.pipeline().get(Http2PingHandler.class)).isSameAs(handler); - scope.set(false); - scope.set(true); - - channel.finishAndReleaseAll(); - } - private static Object getField(Object target, String name) throws Exception { Field f = Http2PingHandler.class.getDeclaredField(name); f.setAccessible(true); diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/GlobalEndpointManager.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/GlobalEndpointManager.java index aae55ca6c9ac..e313b5219713 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/GlobalEndpointManager.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/GlobalEndpointManager.java @@ -383,29 +383,6 @@ public boolean hasThinClientReadLocations() { return this.hasThinClientReadLocations.get(); } - /** - * Returns the underlying {@link AtomicBoolean} that backs - * {@link #hasThinClientReadLocations()}. - * - *

    This is intentionally package-private and exposes the live reference (not - * a copy) so that components which need a long-lived "thin-client effectively - * configured" signal (for example, the gateway {@code HttpClient} used to scope - * HTTP/2 PING keepalive installation) can capture only the {@code AtomicBoolean} - * via a bound method reference (e.g. {@code ref::get}) rather than capturing - * this {@code GlobalEndpointManager} instance. - * - *

    Capturing {@code GlobalEndpointManager} would transitively pin - * {@link DatabaseAccountManagerInternal owner} (which is the owning - * {@code RxDocumentClientImpl}, and therefore the {@code CosmosAsyncClient}) - * for as long as the supplier remains reachable. Capturing only the - * {@code AtomicBoolean} avoids that strong-reference chain while still - * observing live state updates, since this manager mutates the same - * {@code AtomicBoolean} in place on every DatabaseAccount refresh. - */ - AtomicBoolean getHasThinClientReadLocationsRef() { - return this.hasThinClientReadLocations; - } - private Mono getDatabaseAccountAsync(URI serviceEndpoint) { return this.owner.getDatabaseAccountFromEndpoint(serviceEndpoint) .doOnNext(databaseAccount -> { diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java index 6aa77e0cc7b2..f7775a45ce39 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java @@ -891,22 +891,6 @@ public void init(CosmosClientMetadataCachesSnapshot metadataCachesSnapshot, Func this.globalEndpointManager.init(); this.initializePerPartitionCircuitBreaker(); - // Wire the gateway HttpClient's HTTP/2 PING scope supplier to track the - // GlobalEndpointManager's live "hasThinClientReadLocations" signal. - // - // MEMORY SAFETY: We deliberately capture only the AtomicBoolean reference - // (via the bound method reference 'hasThinClientReadLocationsRef::get'), - // NOT the GlobalEndpointManager. GlobalEndpointManager retains - // DatabaseAccountManagerInternal owner, which IS this RxDocumentClientImpl - // (and therefore the owning CosmosAsyncClient). Capturing GEM in the - // supplier closure -- which is stored on the long-lived - // ReactorNettyClient pipeline lambda graph -- would pin the entire client - // and break close()-driven cleanup. The AtomicBoolean is mutated in place - // on every DatabaseAccount refresh, so the supplier still sees live values. - AtomicBoolean hasThinClientReadLocationsRef = - this.globalEndpointManager.getHasThinClientReadLocationsRef(); - this.reactorHttpClient.setHttp2PingScopeSupplier(hasThinClientReadLocationsRef::get); - DatabaseAccount databaseAccountSnapshot = this.initializeGatewayConfigurationReader(); this.resetSessionContainerIfNeeded(databaseAccountSnapshot); @@ -1706,13 +1690,6 @@ private void addUserAgentSuffix(UserAgentContainer userAgentContainer, SetThe global kill-switch {@link Configs#isHttp2PingHealthEnabled()} is true. *

  • HTTP/2 is effectively enabled (either via {@link Configs#isHttp2Enabled()} or * an explicit {@link Http2ConnectionConfig#isEnabled()} override).
    • - *
  • {@link GlobalEndpointManager#hasThinClientReadLocations()} is true -- i.e. the - * account is configured to expose thin-client endpoints, which is the only place - * H2 is negotiated today.
    • - *
  • This client is NOT using {@link SharedGatewayHttpClient}; - * PING is defensively disabled on shared HttpClient instances because they - * back multiple CosmosAsyncClient instances with potentially differing - * thin-client configurations.
    • * */ private boolean isHttp2PingHealthEffectivelyEnabled() { @@ -1732,19 +1709,7 @@ private boolean isHttp2PingHealthEffectivelyEnabled() { && this.connectionPolicy.getHttp2ConnectionConfig().isEnabled() != null) { h2EffectivelyEnabled = this.connectionPolicy.getHttp2ConnectionConfig().isEnabled(); } - if (!h2EffectivelyEnabled) { - return false; - } - - if (this.globalEndpointManager == null) { - return false; - } - - if (this.reactorHttpClient instanceof SharedGatewayHttpClient) { - return false; - } - - return this.globalEndpointManager.hasThinClientReadLocations(); + return h2EffectivelyEnabled; } @Override diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java index 09a3fa11da54..0a60cdc47fc9 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java @@ -13,7 +13,6 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; -import java.util.function.BooleanSupplier; /** * Manual HTTP/2 PING keepalive handler installed on the parent (TCP) channel. @@ -45,10 +44,6 @@ public class Http2PingHandler extends ChannelDuplexHandler { private final long pingIntervalNanos; private final long pingTimeoutNanos; private final int failureThreshold; - // Re-checked per tick so that if the scope condition (e.g. the account's - // thin-client configuration) goes away at runtime, the handler stops - // sending PINGs even on connections that already had it installed. - private final BooleanSupplier scopeSupplier; // Mutable fields below are accessed only from the channel's EventLoop thread // (handlerAdded, channelRead, scheduled task, writeAndFlush listener), so no @@ -69,17 +64,11 @@ public class Http2PingHandler extends ChannelDuplexHandler { * @param pingIntervalSeconds interval in seconds; when idle longer than this, a PING is sent * @param pingTimeoutSeconds timeout in seconds per PING attempt * @param failureThreshold consecutive timeouts before closing the connection - * @param scopeSupplier re-checked per tick; when it returns false the handler stops - * sending PINGs but keeps the timer alive (dormant), so a runtime - * scope flip back to true automatically re-arms PING on the same - * connection without waiting for connection recycling */ - public Http2PingHandler(int pingIntervalSeconds, int pingTimeoutSeconds, int failureThreshold, - BooleanSupplier scopeSupplier) { + public Http2PingHandler(int pingIntervalSeconds, int pingTimeoutSeconds, int failureThreshold) { this.pingIntervalNanos = TimeUnit.SECONDS.toNanos(Math.max(1, pingIntervalSeconds)); this.pingTimeoutNanos = TimeUnit.SECONDS.toNanos(Math.max(1, pingTimeoutSeconds)); this.failureThreshold = Math.max(1, failureThreshold); - this.scopeSupplier = scopeSupplier != null ? scopeSupplier : () -> true; this.lastActivityNanos = System.nanoTime(); } @@ -134,13 +123,12 @@ private void maybeSendPing(ChannelHandlerContext ctx) { cancelPingTask(); return; } - // Two soft gates re-checked per tick: - // - Configs.isHttp2PingHealthEnabled(): global kill-switch (system property). - // - scopeSupplier: per-client scope (e.g. thin-client still active). - // Either flipping false makes this tick a no-op but KEEPS the timer alive, so - // if the gate flips back to true the handler automatically resumes PINGing on - // the same connection (no need to wait for connection recycling). - if (!Configs.isHttp2PingHealthEnabled() || !scopeSupplier.getAsBoolean()) { + // Soft gate re-checked per tick: the global kill-switch + // Configs.isHttp2PingHealthEnabled() (system property). Flipping it false + // makes this tick a no-op but KEEPS the timer alive, so toggling it back + // to true automatically resumes PINGing on the same connection (no need + // to wait for connection recycling). + if (!Configs.isHttp2PingHealthEnabled()) { return; } @@ -203,20 +191,14 @@ private void cancelPingTask() { * @param pingIntervalSeconds PING interval in seconds * @param pingTimeoutSeconds PING ACK timeout in seconds * @param failureThreshold consecutive timeouts before closing - * @param scopeSupplier re-checked per tick by the handler; when it returns false - * the tick is a no-op (dormant) so runtime scope changes - * (e.g. account losing thin-client read locations) stop - * PINGs immediately, but the timer stays alive so a flip - * back to true automatically re-arms on the same connection */ public static void installIfAbsent(Channel channel, int pingIntervalSeconds, int pingTimeoutSeconds, - int failureThreshold, BooleanSupplier scopeSupplier) { + int failureThreshold) { Channel parent = channel.parent() != null ? channel.parent() : channel; if (parent.pipeline().get(HANDLER_NAME) == null) { try { parent.pipeline().addLast(HANDLER_NAME, - new Http2PingHandler(pingIntervalSeconds, pingTimeoutSeconds, failureThreshold, - scopeSupplier)); + new Http2PingHandler(pingIntervalSeconds, pingTimeoutSeconds, failureThreshold)); } catch (IllegalArgumentException ignored) { // Duplicate -- race between concurrent streams, benign } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/HttpClient.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/HttpClient.java index 61e9da892e69..4f1830908623 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/HttpClient.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/HttpClient.java @@ -10,7 +10,6 @@ import reactor.netty.resources.ConnectionProvider; import java.time.Duration; -import java.util.function.BooleanSupplier; /** * A generic interface for sending HTTP requests and getting responses. @@ -92,25 +91,4 @@ static HttpClient create(HttpClientConfig httpClientConfig) { * Shutdown the Http Client and clean up resources */ void shutdown(); - - /** - * Sets a late-bound supplier that gates installation of HTTP/2 PING keepalive - * to channels where the supplier returns {@code true}. The supplier is invoked - * inside {@code doOnConnected} for the parent (TCP) channel; when it returns - * {@code false}, no PING handler is installed. - * - *

    The supplier must be a bound method reference that does not - * capture references to the owning {@code CosmosAsyncClient} / - * {@code RxDocumentClientImpl}. See - * {@code GlobalEndpointManager#getHasThinClientReadLocationsRef()} for the - * rationale; the recommended pattern is {@code atomicBoolean::get}. - * - *

    If never set, PING installation falls back to the prior behavior gated only - * by {@link Configs#isHttp2PingHealthEnabled()}. - * - *

    Default no-op so non-Netty implementations and tests need not override. - */ - default void setHttp2PingScopeSupplier(BooleanSupplier supplier) { - // no-op by default; ReactorNettyClient overrides. - } } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java index ebddc485db78..bcef5838822c 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java @@ -39,7 +39,6 @@ import java.util.Objects; import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiFunction; -import java.util.function.BooleanSupplier; /** * HttpClient that is implemented using reactor-netty. @@ -61,18 +60,6 @@ private static ImplementationBridgeHelpers.Http2ConnectionConfigHelper.Http2Conn private String reactorNetworkLogCategory; private Logger wireTapLogger; - // Defensive scope for HTTP/2 PING keepalive installation. Defaults to a - // closed gate (no PING) so that during the HttpClient bootstrap window -- - // before GlobalEndpointManager has finished the first DatabaseAccount - // refresh -- no PING handlers are installed. The first DatabaseAccount - // fetch itself is over H1.1 to the standard gateway, so it would not - // have triggered PING anyway. - // - // MUST be assigned a supplier that does NOT capture RxDocumentClientImpl - // (CosmosAsyncClient) transitively -- see HttpClient#setHttp2PingScopeSupplier - // and GlobalEndpointManager#getHasThinClientReadLocationsRef(). - private volatile BooleanSupplier http2PingScopeSupplier = () -> false; - private ReactorNettyClient() {} /** @@ -165,15 +152,10 @@ private void configureChannelPipelineHandlers() { // For H2, doOnConnected fires for both the parent TCP channel and child stream // channels. We install on the parent channel via Http2MultiplexHandler detection. // - // Scoping (defensive): PING is only installed when both - // (a) the global kill-switch Configs.isHttp2PingHealthEnabled() is true, and - // (b) the late-bound http2PingScopeSupplier returns true -- which today is - // wired to GlobalEndpointManager.hasThinClientReadLocations(), so PING - // is restricted to clients whose account is configured for thin client. - // This guard makes the existing implicit "H2 == thin client" behavior explicit - // and prevents future changes that enable H2 elsewhere from silently turning on - // PING on non-thin-client connections. - if (Configs.isHttp2PingHealthEnabled() && this.http2PingScopeSupplier.getAsBoolean()) { + // Gated by the global kill-switch Configs.isHttp2PingHealthEnabled() (system + // property). The handler re-checks this gate per tick, so toggling it at runtime + // immediately stops/resumes PINGing on already-installed connections. + if (Configs.isHttp2PingHealthEnabled()) { // Resolve to the parent (TCP) channel -- doOnConnected may fire for // child stream channels where Http2MultiplexHandler is absent. Channel ch = connection.channel(); @@ -183,13 +165,8 @@ private void configureChannelPipelineHandlers() { int pingTimeoutSeconds = Configs.getHttp2PingTimeoutInSeconds(); int pingFailureThreshold = Configs.getHttp2PingFailureThreshold(); if (pingIntervalSeconds > 0) { - // Pass the same scope supplier into the handler so it re-checks - // per tick. If the account flips thin-client off after the handler - // was installed, the supplier returns false on the next scheduled - // tick and the handler cancels itself. - BooleanSupplier scopeForHandler = this.http2PingScopeSupplier; Http2PingHandler.installIfAbsent(parent, pingIntervalSeconds, - pingTimeoutSeconds, pingFailureThreshold, scopeForHandler); + pingTimeoutSeconds, pingFailureThreshold); } } } @@ -327,20 +304,6 @@ public void shutdown() { } } - @Override - public void setHttp2PingScopeSupplier(BooleanSupplier supplier) { - // The supplier is read inside doOnConnected on a Netty event loop. We use a - // volatile write here so the lambda observes the latest value without locking. - // - // IMPORTANT: callers must pass a supplier that does NOT capture - // RxDocumentClientImpl / GlobalEndpointManager. See - // GlobalEndpointManager.getHasThinClientReadLocationsRef() for the - // memory-safety rationale and the bound-method-reference pattern. - if (supplier != null) { - this.http2PingScopeSupplier = supplier; - } - } - /** * Resolves the TCP connect timeout (CONNECT_TIMEOUT_MILLIS) based on the request type. * diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/SharedGatewayHttpClient.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/SharedGatewayHttpClient.java index 90e8457f8f6f..4e0729a51ac4 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/SharedGatewayHttpClient.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/SharedGatewayHttpClient.java @@ -11,7 +11,6 @@ import java.time.Duration; import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.BooleanSupplier; /** * This class uses a shared HttpClient for multiple Cosmos Clients. @@ -42,12 +41,6 @@ public static HttpClient getOrCreateInstance(HttpClientConfig httpClientConfig, private SharedGatewayHttpClient(HttpClientConfig httpClientConfig) { this.httpClient = HttpClient.createFixed(httpClientConfig); - // Defensive scoping: HTTP/2 PING keepalive is always disabled on the shared - // gateway HttpClient. Multiple CosmosAsyncClient instances may share this - // underlying client with differing thin-client configurations, so a single - // PING decision cannot represent all callers. Per-client scope suppliers - // registered via setHttp2PingScopeSupplier are intentionally ignored here. - this.httpClient.setHttp2PingScopeSupplier(() -> false); } @Override @@ -76,10 +69,4 @@ public void shutdown() { } } } - - @Override - public void setHttp2PingScopeSupplier(BooleanSupplier supplier) { - // Intentional no-op: PING is permanently disabled on the shared HttpClient. - // See constructor for rationale. - } } From 953191c0e90f14d19e58cf5265952e6b0503b3ed Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Thu, 4 Jun 2026 15:05:49 -0400 Subject: [PATCH 43/62] Address PR review: late-ACK guard, execCommand failure, doc refresh - Http2PingHandler.channelRead: also guard ACK on pingOutstandingSinceNanos != 0 so a late ACK arriving after timeout cleared it -- but before the next PING bumps pingsSent -- cannot mask consecutive failures on a degraded connection. - Http2PingHandlerTest: add ackAfterTimeoutCleared_doesNotResetState regression test for the above. - Http2PingKeepaliveTest.execCommand: throw on non-zero exit so a missing NET_ADMIN cap fails the test loudly instead of silently skipping network fault injection. - Http2PingKeepaliveTest: refresh class Javadoc + beforeClass comment to match the current single test and drop stale 'PING is scoped to thin-client' wording (scoping was removed in d33d3110db7). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Http2PingKeepaliveTest.java | 30 ++++++++++++------- .../http/Http2PingHandlerTest.java | 22 ++++++++++++++ .../implementation/http/Http2PingHandler.java | 12 ++++++-- 3 files changed, 50 insertions(+), 14 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java index c9380aebf6a6..2333626f032d 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java @@ -31,17 +31,16 @@ import static org.assertj.core.api.Assertions.assertThat; /** - * HTTP/2 PING keepalive handler tests. + * HTTP/2 PING keepalive handler test. *

    - * Test 1: Verifies PINGs are sent at the configured interval on idle connections. - * Test 2: Uses iptables DROP to blackhole PING ACKs, verifying the handler closes - * the broken connection and a subsequent request uses a new connection. + * Uses {@code iptables DROP} to blackhole PING ACKs, verifying the handler closes + * the broken connection after consecutive PING timeouts and a subsequent request + * uses a new connection. *

    - * Run in Docker with --cap-add=NET_ADMIN (group: manual-http-network-fault). - * Requires: thin-client-enabled Cosmos DB account whose DatabaseAccount response - * exposes {@code thinClientReadableLocations}. PING is scoped to thin-client - * endpoints only, so both {@code COSMOS.HTTP2_ENABLED=true} and - * {@code COSMOS.THINCLIENT_ENABLED=true} must be set before client construction. + * Run in Docker with {@code --cap-add=NET_ADMIN} (group: {@code manual-http-network-fault}). + * Requires an HTTP/2-capable Cosmos DB Gateway endpoint; both {@code COSMOS.HTTP2_ENABLED=true} + * and {@code COSMOS.THINCLIENT_ENABLED=true} are set before client construction because + * H2 is currently negotiated only on the thin-client proxy path. */ public class Http2PingKeepaliveTest extends FaultInjectionTestBase { @@ -66,7 +65,7 @@ public Http2PingKeepaliveTest() { @BeforeClass(groups = {"manual-http-network-fault"}, timeOut = 120_000) public void beforeClass() { // Enable HTTP/2 and thin client BEFORE client construction. - // PING is scoped to thin-client endpoints; both flags are required. + // H2 is currently negotiated on the thin-client proxy path; both flags are required. System.setProperty("COSMOS.HTTP2_ENABLED", "true"); System.setProperty("COSMOS.THINCLIENT_ENABLED", "true"); @@ -230,6 +229,13 @@ private String extractParentChannelId(CosmosDiagnostics diagnostics) throws Json throw new AssertionError("Could not extract parentChannelId from diagnostics: " + diagnostics); } + /** + * Runs a shell command and throws on non-zero exit. The test must fail loudly + * if iptables setup fails (e.g., missing NET_ADMIN capability), rather than + * silently continuing without network fault injection and reporting a + * misleading assertion failure downstream. Cleanup-only callers in + * {@code finally} blocks can swallow the exception locally. + */ private static void execCommand(String command) throws Exception { Process p = Runtime.getRuntime().exec(new String[]{"bash", "-c", command}); int exit = p.waitFor(); @@ -241,7 +247,9 @@ private static void execCommand(String command) throws Exception { while ((line = reader.readLine()) != null) { sb.append(line); } - logger.warn("Command '{}' exited with code {}: {}", command, exit, sb.toString().trim()); + String stderr = sb.toString().trim(); + logger.warn("Command '{}' exited with code {}: {}", command, exit, stderr); + throw new RuntimeException("Command failed (exit=" + exit + "): " + command + " -- " + stderr); } } } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/http/Http2PingHandlerTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/http/Http2PingHandlerTest.java index 6c78b8f80163..169d17a7aa68 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/http/Http2PingHandlerTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/http/Http2PingHandlerTest.java @@ -66,6 +66,28 @@ public void ackWithMismatchedPayload_doesNotResetState() throws Exception { channel.finishAndReleaseAll(); } + @Test(groups = "unit") + public void ackAfterTimeoutCleared_doesNotResetState() throws Exception { + // Reproduces the race fixed by the `pingOutstandingSinceNanos != 0` guard in + // channelRead: PING #5 is sent (pingsSent=5), timeout fires and sets + // pingOutstandingSinceNanos=0 + consecutiveFailures=1, then a late ACK for #5 + // arrives BEFORE the next PING is sent (so pingsSent is still 5). Payload alone + // matches; the outstanding-flag guard is what prevents masking the failure. + Http2PingHandler handler = new Http2PingHandler(1, 1, 3); + EmbeddedChannel channel = new EmbeddedChannel(handler); + + setField(handler, "pingsSent", 5); + setField(handler, "pingOutstandingSinceNanos", 0L); + setField(handler, "consecutiveFailures", 1); + + channel.writeInbound(new DefaultHttp2PingFrame(5L, true)); + + assertThat((long) getField(handler, "pingOutstandingSinceNanos")).isZero(); + assertThat((int) getField(handler, "consecutiveFailures")).isEqualTo(1); + + channel.finishAndReleaseAll(); + } + @Test(groups = "unit") public void installIfAbsent_isIdempotent() { EmbeddedChannel channel = new EmbeddedChannel(); diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java index 0a60cdc47fc9..e1b512e6bf92 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java @@ -107,9 +107,15 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception if (msg instanceof Http2PingFrame) { Http2PingFrame ping = (Http2PingFrame) msg; // RFC 9113 §6.7: PING ACK echoes the 8-byte payload we sent. - // Match by payload so a late ACK for a PING that already counted as a - // timeout cannot reset the failure counter and mask ongoing degradation. - if (ping.ack() && ping.content() == pingsSent) { + // Two guards prevent a late ACK from masking ongoing degradation: + // 1) pingOutstandingSinceNanos != 0 -- after timeout fires we clear + // this to 0; a late ACK arriving before the next PING is sent + // still has ping.content() == pingsSent (pingsSent isn't bumped + // until the next send), so the payload alone is insufficient. + // 2) ping.content() == pingsSent -- guards the window AFTER the next + // PING is sent (pingsSent has advanced), so a late ACK for the + // previous PING no longer matches. + if (ping.ack() && pingOutstandingSinceNanos != 0 && ping.content() == pingsSent) { pingOutstandingSinceNanos = 0; consecutiveFailures = 0; } From 1661b29024dd0a666d8ff27c4dbb47f1e049e768 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Thu, 4 Jun 2026 16:20:15 -0400 Subject: [PATCH 44/62] Refactor Http2PingKeepaliveTest for two-pipeline runs (thin-client + Compute) Drive transport selection from COSMOS.THINCLIENT_ENABLED sysprop set by Maven profile rather than hardcoding port 10250 / thin-client in @BeforeClass. - manual-http-network-fault profile: THINCLIENT_ENABLED=true (port 10250). - New manual-http-network-fault-compute profile: THINCLIENT_ENABLED=false (port 443, IP-scoped iptables since :443 is shared with other TLS traffic). - Warm-up read now hard-asserts H2 negotiation (fails fast if the regional gateway only speaks HTTP/1.1, so a silent fallback can't make the test green for the wrong reason). - extractEndpointHost helper resolves the regional gateway host from gatewayStatisticsList for the IP-scoped variant. - iptables cleanup simplified by capturing the exact -D rule string and re-running it best-effort in finally. Validated in Docker against thin-client-mr-bs-ci: thinClient=true, port=10250, DIFFERENT_CONNECTION=true thinClient=false, port=443, DIFFERENT_CONNECTION=true Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- sdk/cosmos/azure-cosmos-tests/pom.xml | 30 ++++ .../Http2PingKeepaliveTest.java | 153 +++++++++++++----- 2 files changed, 146 insertions(+), 37 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-tests/pom.xml b/sdk/cosmos/azure-cosmos-tests/pom.xml index 44fce7dbb5b0..d79e59dc54fa 100644 --- a/sdk/cosmos/azure-cosmos-tests/pom.xml +++ b/sdk/cosmos/azure-cosmos-tests/pom.xml @@ -874,6 +874,36 @@ Licensed under the MIT License. true + true + + + + + + + + + manual-http-network-fault-compute + + manual-http-network-fault + + + + + org.apache.maven.plugins + maven-failsafe-plugin + 3.5.3 + + + src/test/resources/manual-http-network-fault-testng.xml + + + true + false diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java index 2333626f032d..73182ef331df 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java @@ -26,6 +26,8 @@ import org.testng.annotations.Test; import java.lang.reflect.Method; +import java.net.InetAddress; +import java.net.URI; import java.util.UUID; import static org.assertj.core.api.Assertions.assertThat; @@ -38,9 +40,19 @@ * uses a new connection. *

    * Run in Docker with {@code --cap-add=NET_ADMIN} (group: {@code manual-http-network-fault}). - * Requires an HTTP/2-capable Cosmos DB Gateway endpoint; both {@code COSMOS.HTTP2_ENABLED=true} - * and {@code COSMOS.THINCLIENT_ENABLED=true} are set before client construction because - * H2 is currently negotiated only on the thin-client proxy path. + *

    + * Two transports exercise the same handler -- pick one per pipeline run via system properties: + *

      + *
    • {@code COSMOS.THINCLIENT_ENABLED=true} (default): data plane goes to thin-client proxy + * on port 10250; iptables drops by port only (no other process uses 10250).
      • + *
    • {@code COSMOS.THINCLIENT_ENABLED=false}: data plane goes to the regional Gateway V2 + * endpoint on port 443; iptables drops by destination IP + port to avoid killing + * unrelated TLS traffic in the JVM. Requires an account whose regional gateway + * negotiates {@code h2} via ALPN; the warm-up read asserts this and the test + * fails fast on a Classic (HTTP/1.1-only) endpoint.
      • + *
    + * Override the port explicitly with {@code -DHTTP2_PING_TEST_PORT=} if needed. + * {@code COSMOS.HTTP2_ENABLED=true} is always set by the test. */ public class Http2PingKeepaliveTest extends FaultInjectionTestBase { @@ -50,6 +62,12 @@ public class Http2PingKeepaliveTest extends FaultInjectionTestBase { // sudo prefix: empty when running as root (Docker), "sudo " on CI VMs private static final String SUDO = "root".equals(System.getProperty("user.name")) ? "" : "sudo "; + // Transport selection -- defaults to thin-client for back-compat with the existing pipeline. + private static final boolean THIN_CLIENT_ENABLED = + Boolean.parseBoolean(System.getProperty("COSMOS.THINCLIENT_ENABLED", "true")); + private static final int H2_PORT = + Integer.getInteger("HTTP2_PING_TEST_PORT", THIN_CLIENT_ENABLED ? 10250 : 443); + private CosmosAsyncClient client; private CosmosAsyncContainer cosmosAsyncContainer; private TestObject seedItem; @@ -64,10 +82,11 @@ public Http2PingKeepaliveTest() { @BeforeClass(groups = {"manual-http-network-fault"}, timeOut = 120_000) public void beforeClass() { - // Enable HTTP/2 and thin client BEFORE client construction. - // H2 is currently negotiated on the thin-client proxy path; both flags are required. + // HTTP/2 must be enabled before the client is constructed. THINCLIENT_ENABLED is + // set externally (Maven profile or -D) so a single test class covers both transports + // across two pipeline runs. System.setProperty("COSMOS.HTTP2_ENABLED", "true"); - System.setProperty("COSMOS.THINCLIENT_ENABLED", "true"); + logger.info("Transport selected: thinClient={}, h2Port={}", THIN_CLIENT_ENABLED, H2_PORT); this.client = getClientBuilder().buildAsyncClient(); this.cosmosAsyncContainer = getSharedMultiPartitionCosmosContainerWithIdAsPartitionKey(this.client); @@ -82,7 +101,6 @@ public void beforeClass() { public void afterClass() { safeClose(this.client); System.clearProperty("COSMOS.HTTP2_ENABLED"); - System.clearProperty("COSMOS.THINCLIENT_ENABLED"); } @BeforeMethod(groups = {"manual-http-network-fault"}) @@ -116,6 +134,10 @@ public void connectionClosedOnPingTimeout() throws Exception { // Override threshold to 2 for faster test (default=5 aligned with Rust SDK) System.setProperty("COSMOS.HTTP2_PING_FAILURE_THRESHOLD", "2"); + // Lifted out of the try so the finally-block cleanup can reach it whether or not + // the iptables ADD ran -- finally needs the exact -D form of whatever -A we installed. + String iptablesDelete = null; + try { safeClose(this.client); @@ -134,35 +156,62 @@ public void connectionClosedOnPingTimeout() throws Exception { .buildAsyncClient(); this.cosmosAsyncContainer = getSharedMultiPartitionCosmosContainerWithIdAsPartitionKey(this.client); - // Warm-up read -- establish H2 connection (over thin-client port 10250) - String initialChannelId = readAndGetParentChannelId(); - logger.info("Initial parentChannelId: {}", initialChannelId); - - // Thin-client traffic goes to port 10250 on regional thin-client endpoints - // (e.g., thin-client-mr-bs-ci-westcentralus.documents.azure.com:10250). - // The regional hostname differs from the account-level endpoint, so we - // blackhole by port only -- catching all thin-client traffic regardless of region. - int thinClientPort = 10250; - logger.info("Will blackhole port: {} (thin-client H2)", thinClientPort); + // Warm-up read -- establish the H2 connection and capture diagnostics so we can + // (a) prove H2 was actually negotiated, (b) discover the regional endpoint host + // for IP-scoped iptables targeting on the Compute / port-443 path. + CosmosItemResponse warmup = this.cosmosAsyncContainer.readItem( + seedItem.getId(), new PartitionKey(seedItem.getId()), TestObject.class).block(); + assertThat(warmup).isNotNull(); + assertThat(warmup.getStatusCode()).isEqualTo(200); + + CosmosDiagnostics warmupDiag = warmup.getDiagnostics(); + String diagStr = warmupDiag.toString(); + assertThat(diagStr) + .as("Warm-up connection must negotiate H2 (thinClient=%s). " + + "If false, the account does not expose an H2-capable Gateway V2 endpoint " + + "and this test cannot exercise the PING handler.", THIN_CLIENT_ENABLED) + .contains("\"isHttp2\":true"); + + String initialChannelId = extractParentChannelId(warmupDiag); + String regionalHost = extractEndpointHost(warmupDiag); + logger.info("Initial parentChannelId: {}, regionalHost: {}", initialChannelId, regionalHost); + + // iptables rule: port-only for thin-client (10250 is exclusive to thin-client traffic); + // destination-IP + port for Compute (port 443 is shared with everything else in the JVM, + // so we must scope to the regional gateway's resolved address). + String iptablesAdd; + if (THIN_CLIENT_ENABLED) { + iptablesAdd = String.format( + "%siptables -A OUTPUT -p tcp --dport %d -j DROP", SUDO, H2_PORT); + iptablesDelete = String.format( + "%siptables -D OUTPUT -p tcp --dport %d -j DROP", SUDO, H2_PORT); + } else { + String regionalIp = InetAddress.getByName(regionalHost).getHostAddress(); + logger.info("Resolved {} -> {} (Compute variant uses IP-scoped DROP)", + regionalHost, regionalIp); + iptablesAdd = String.format( + "%siptables -A OUTPUT -p tcp -d %s --dport %d -j DROP", + SUDO, regionalIp, H2_PORT); + iptablesDelete = String.format( + "%siptables -D OUTPUT -p tcp -d %s --dport %d -j DROP", + SUDO, regionalIp, H2_PORT); + } - // Blackhole all traffic to thin-client port -- prevents PING ACKs from arriving - String iptablesRule = String.format( - "%siptables -A OUTPUT -p tcp --dport %d -j DROP", SUDO, thinClientPort); - logger.info("Installing iptables DROP rule: {}", iptablesRule); - execCommand(iptablesRule); + logger.info("Installing iptables DROP rule: {}", iptablesAdd); + execCommand(iptablesAdd); // Wait for PING timeout with consecutive failure threshold=2 (test override): // Round 1: 1s interval + 2s timeout = 3s (failure #1) - // Round 2: ~1s interval + 2s timeout = 3s (failure #2 ≥ threshold → close) + // Round 2: ~1s interval + 2s timeout = 3s (failure #2 >= threshold -> close) // Total ~6s + buffer = 10s logger.info("Waiting 10s for consecutive PING timeouts to close the connection..."); Thread.sleep(10_000); // Remove iptables rule BEFORE attempting recovery read - String iptablesRemove = String.format( - "%siptables -D OUTPUT -p tcp --dport %d -j DROP", SUDO, thinClientPort); - logger.info("Removing iptables DROP rule: {}", iptablesRemove); - execCommand(iptablesRemove); + logger.info("Removing iptables DROP rule: {}", iptablesDelete); + execCommand(iptablesDelete); + // Cleared -- finally-block no longer needs to delete it. + iptablesDelete = null; // Small wait for network to stabilize Thread.sleep(1_000); @@ -171,25 +220,28 @@ public void connectionClosedOnPingTimeout() throws Exception { String recoveryChannelId = readAndGetParentChannelId(); logger.info("Recovery parentChannelId: {}", recoveryChannelId); - logger.info("RESULT: initial={}, recovery={}, DIFFERENT_CONNECTION={}", - initialChannelId, recoveryChannelId, + logger.info("RESULT: thinClient={}, port={}, initial={}, recovery={}, DIFFERENT_CONNECTION={}", + THIN_CLIENT_ENABLED, H2_PORT, initialChannelId, recoveryChannelId, !initialChannelId.equals(recoveryChannelId)); // The connection MUST be different -- the old one was closed by PING timeout assertThat(recoveryChannelId) - .as("After PING timeout, the handler should have closed the connection. " - + "The recovery request must use a new connection.") + .as("After PING timeout (thinClient=%s, port=%d), the handler should have " + + "closed the connection. The recovery request must use a new connection.", + THIN_CLIENT_ENABLED, H2_PORT) .isNotEqualTo(initialChannelId); logger.info("PING timeout test passed: connection {} was closed, new connection {} established", initialChannelId, recoveryChannelId); } finally { - // Safety: remove any leftover iptables rules - try { - String cleanup = String.format( - "%siptables -D OUTPUT -p tcp --dport 10250 -j DROP 2>/dev/null", SUDO); - execCommand(cleanup); - } catch (Exception ignored) {} + // Safety: if we installed an iptables rule and didn't manage to remove it above + // (e.g., assertion failed before we reached the eager delete), best-effort + // remove it now. The exact -D form was captured when we built -A. + if (iptablesDelete != null) { + try { + execCommand(iptablesDelete); + } catch (Exception ignored) {} + } System.clearProperty("COSMOS.HTTP2_PING_INTERVAL_IN_SECONDS"); System.clearProperty("COSMOS.HTTP2_PING_TIMEOUT_IN_SECONDS"); @@ -208,6 +260,33 @@ private String readAndGetParentChannelId() throws JsonProcessingException { return extractParentChannelId(response.getDiagnostics()); } + /** + * Pulls the regional gateway hostname out of the first {@code gatewayStatisticsList[]} + * entry whose {@code endpoint} URI exposes a host. Used by the Compute / port-443 + * path to scope the iptables DROP rule to a single destination IP -- a port-only + * rule on :443 would also kill every other outbound TLS connection in the JVM. + */ + private String extractEndpointHost(CosmosDiagnostics diagnostics) throws JsonProcessingException { + ObjectNode node = (ObjectNode) Utils.getSimpleObjectMapper().readTree(diagnostics.toString()); + JsonNode gwStats = node.get("gatewayStatisticsList"); + if (gwStats != null && gwStats.isArray()) { + for (JsonNode stat : gwStats) { + if (stat.has("endpoint")) { + String endpoint = stat.get("endpoint").asText(); + try { + String host = URI.create(endpoint).getHost(); + if (host != null && !host.isEmpty()) { + return host; + } + } catch (IllegalArgumentException ignored) { + // Bad URI -- fall through to the next stat entry. + } + } + } + } + throw new AssertionError("Could not extract endpoint host from diagnostics: " + diagnostics); + } + /** * Mirrors {@code Http2ConnectionLifecycleTests#extractParentChannelId} -- parse * the diagnostics JSON rather than substring-scan the toString() so a future change From a10f9278d30ed286e155abbf061e0c727b377672 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Thu, 4 Jun 2026 16:27:54 -0400 Subject: [PATCH 45/62] Add Compute matrix entry for Http2NetworkFault stage The live-http2-network-fault matrix previously had a single include block running -Pmanual-http-network-fault (thin-client, port 10250). Add a sibling entry running -Pmanual-http-network-fault-compute (Compute / GW V2, port 443). CI will now spawn two parallel jobs from this matrix: HttpNetworkFaultThinClient -> -Pmanual-http-network-fault HttpNetworkFaultCompute -> -Pmanual-http-network-fault-compute Both jobs run the same TestNG suite (manual-http-network-fault-testng.xml). The SDK transport is chosen by the COSMOS.THINCLIENT_ENABLED sysprop the profile sets, so Http2PingKeepaliveTest exercises both transports without test-class duplication. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ...ve-http2-network-fault-platform-matrix.json | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/sdk/cosmos/live-http2-network-fault-platform-matrix.json b/sdk/cosmos/live-http2-network-fault-platform-matrix.json index 65ab74536051..8268fdedb324 100644 --- a/sdk/cosmos/live-http2-network-fault-platform-matrix.json +++ b/sdk/cosmos/live-http2-network-fault-platform-matrix.json @@ -1,6 +1,7 @@ { "displayNames": { - "-Pmanual-http-network-fault": "HttpNetworkFault", + "-Pmanual-http-network-fault": "HttpNetworkFaultThinClient", + "-Pmanual-http-network-fault-compute": "HttpNetworkFaultCompute", "Session": "", "ubuntu": "", "@{ enableMultipleWriteLocations = $true; defaultConsistencyLevel = 'Session'; enableMultipleRegions = $true }": "http2-network-fault" @@ -20,6 +21,21 @@ "Agent": { "ubuntu": { "OSVmImage": "env:LINUXVMIMAGE", "Pool": "env:LINUXPOOL" } } + }, + { + "DESIRED_CONSISTENCIES": "[\"Session\"]", + "ACCOUNT_CONSISTENCY": "Session", + "ArmConfig": { + "MultiMaster_MultiRegion": { + "ArmTemplateParameters": "@{ enableMultipleWriteLocations = $true; defaultConsistencyLevel = 'Session'; enableMultipleRegions = $true }", + "PREFERRED_LOCATIONS": "[\"East US 2\"]" + } + }, + "PROTOCOLS": "[\"Tcp\"]", + "ProfileFlag": [ "-Pmanual-http-network-fault-compute" ], + "Agent": { + "ubuntu": { "OSVmImage": "env:LINUXVMIMAGE", "Pool": "env:LINUXPOOL" } + } } ] } From c1cc39a4bcd243f185f71119f72b48d59499cc74 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Thu, 4 Jun 2026 17:23:37 -0400 Subject: [PATCH 46/62] Consolidate HTTP/2 PING install gating into single helper Add Http2PingHandler.isPingHealthEffectivelyEnabled(Http2ConnectionConfig) that consolidates the three gates (kill-switch, ping interval > 0, H2 effectively enabled). ReactorNettyClient and RxDocumentClientImpl now delegate to this single helper so the transport install site and the user-agent feature flag cannot drift apart. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../implementation/RxDocumentClientImpl.java | 31 ++++--------------- .../implementation/http/Http2PingHandler.java | 31 +++++++++++++++++++ .../http/ReactorNettyClient.java | 20 ++++++------ 3 files changed, 46 insertions(+), 36 deletions(-) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java index f7775a45ce39..d4f99c183c24 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java @@ -47,6 +47,7 @@ import com.azure.cosmos.implementation.http.HttpClient; import com.azure.cosmos.implementation.http.HttpClientConfig; import com.azure.cosmos.implementation.http.HttpHeaders; +import com.azure.cosmos.implementation.http.Http2PingHandler; import com.azure.cosmos.implementation.http.SharedGatewayHttpClient; import com.azure.cosmos.implementation.interceptor.ITransportClientInterceptor; import com.azure.cosmos.implementation.patch.PatchUtil; @@ -1683,33 +1684,13 @@ private void addUserAgentSuffix(UserAgentContainer userAgentContainer, Set - *
  • The global kill-switch {@link Configs#isHttp2PingHealthEnabled()} is true.
    • - *
  • HTTP/2 is effectively enabled (either via {@link Configs#isHttp2Enabled()} or - * an explicit {@link Http2ConnectionConfig#isEnabled()} override).
    • - * + * delegating to {@link Http2PingHandler#isPingHealthEffectivelyEnabled} so the + * user-agent feature flag stays in lockstep with the transport install gate in + * {@code ReactorNettyClient}. */ private boolean isHttp2PingHealthEffectivelyEnabled() { - if (!Configs.isHttp2PingHealthEnabled()) { - return false; - } - - // Mirror the transport-side guard in Http2PingHandler#installIfAbsent: if the - // configured interval is non-positive the handler is not installed, so the UA - // flag must not advertise PING either. - if (Configs.getHttp2PingIntervalInSeconds() <= 0) { - return false; - } - - boolean h2EffectivelyEnabled = Configs.isHttp2Enabled(); - if (this.connectionPolicy.getHttp2ConnectionConfig() != null - && this.connectionPolicy.getHttp2ConnectionConfig().isEnabled() != null) { - h2EffectivelyEnabled = this.connectionPolicy.getHttp2ConnectionConfig().isEnabled(); - } - return h2EffectivelyEnabled; + return Http2PingHandler.isPingHealthEffectivelyEnabled( + this.connectionPolicy.getHttp2ConnectionConfig()); } @Override diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java index e1b512e6bf92..58c0c517fbc3 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java @@ -2,7 +2,9 @@ // Licensed under the MIT License. package com.azure.cosmos.implementation.http; +import com.azure.cosmos.Http2ConnectionConfig; import com.azure.cosmos.implementation.Configs; +import com.azure.cosmos.implementation.ImplementationBridgeHelpers; import io.netty.channel.Channel; import io.netty.channel.ChannelDuplexHandler; import io.netty.channel.ChannelHandlerContext; @@ -189,6 +191,35 @@ private void cancelPingTask() { } } + /** + * Returns {@code true} when the manual HTTP/2 PING keepalive should be active for + * a client built with the given {@link Http2ConnectionConfig}. This is the single + * source of truth shared by the transport install site (see + * {@link ReactorNettyClient}) and the user-agent flag emission (see + * {@code RxDocumentClientImpl}) so the two cannot drift. + *

    + * All three gates must pass: + *

      + *
    • The global kill-switch {@link Configs#isHttp2PingHealthEnabled()} is true.
      • + *
    • {@link Configs#getHttp2PingIntervalInSeconds()} is > 0 (a non-positive + * interval disables the handler at install time).
      • + *
    • HTTP/2 is effectively enabled for this client + * (either via {@link Configs#isHttp2Enabled()} or the per-client + * {@link Http2ConnectionConfig} override).
      • + *
    + */ + public static boolean isPingHealthEffectivelyEnabled(Http2ConnectionConfig http2Cfg) { + if (!Configs.isHttp2PingHealthEnabled()) { + return false; + } + if (Configs.getHttp2PingIntervalInSeconds() <= 0) { + return false; + } + return ImplementationBridgeHelpers.Http2ConnectionConfigHelper + .getHttp2ConnectionConfigAccessor() + .isEffectivelyEnabled(http2Cfg); + } + /** * Installs the PING handler on the parent H2 channel if not already installed. * Safe to call from doOnConnected (which fires per-stream for H2). diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java index bcef5838822c..1f26746f20bb 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java @@ -152,22 +152,20 @@ private void configureChannelPipelineHandlers() { // For H2, doOnConnected fires for both the parent TCP channel and child stream // channels. We install on the parent channel via Http2MultiplexHandler detection. // - // Gated by the global kill-switch Configs.isHttp2PingHealthEnabled() (system - // property). The handler re-checks this gate per tick, so toggling it at runtime - // immediately stops/resumes PINGing on already-installed connections. - if (Configs.isHttp2PingHealthEnabled()) { + // Gating is consolidated in Http2PingHandler.isPingHealthEffectivelyEnabled so + // the transport install site and the user-agent feature flag cannot drift. + // The handler also re-checks the kill-switch per tick so toggling it at + // runtime immediately stops/resumes PINGing on already-installed connections. + if (Http2PingHandler.isPingHealthEffectivelyEnabled(http2Cfg)) { // Resolve to the parent (TCP) channel -- doOnConnected may fire for // child stream channels where Http2MultiplexHandler is absent. Channel ch = connection.channel(); Channel parent = ch.parent() != null ? ch.parent() : ch; if (parent.pipeline().get(Http2MultiplexHandler.class) != null) { - int pingIntervalSeconds = Configs.getHttp2PingIntervalInSeconds(); - int pingTimeoutSeconds = Configs.getHttp2PingTimeoutInSeconds(); - int pingFailureThreshold = Configs.getHttp2PingFailureThreshold(); - if (pingIntervalSeconds > 0) { - Http2PingHandler.installIfAbsent(parent, pingIntervalSeconds, - pingTimeoutSeconds, pingFailureThreshold); - } + Http2PingHandler.installIfAbsent(parent, + Configs.getHttp2PingIntervalInSeconds(), + Configs.getHttp2PingTimeoutInSeconds(), + Configs.getHttp2PingFailureThreshold()); } } }); From 004d2faf2670aae144d935856c227a5a74db17fd Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Thu, 4 Jun 2026 17:45:39 -0400 Subject: [PATCH 47/62] Use full Netty channel toString in Http2PingHandler log lines Aligns channel identifier in PING handler log lines with Http2ParentChannelExceptionHandler, which logs ctx.channel() directly (Netty's [id: 0x..., L:..., R:...] form). The short hex via id().asShortText() loses local/remote address context that's useful when correlating PING events with other parent-channel exceptions and pool eviction logs. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../cosmos/implementation/http/Http2PingHandler.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java index 58c0c517fbc3..71fddf71e9fb 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java @@ -86,7 +86,7 @@ public void handlerAdded(ChannelHandlerContext ctx) { TimeUnit.MILLISECONDS); logger.debug("Http2PingHandler installed on channel {}, interval={}s, timeout={}s, checkEvery={}ms", - ctx.channel().id().asShortText(), + ctx.channel(), TimeUnit.NANOSECONDS.toSeconds(pingIntervalNanos), TimeUnit.NANOSECONDS.toSeconds(pingTimeoutNanos), checkIntervalMs); @@ -150,12 +150,12 @@ private void maybeSendPing(ChannelHandlerContext ctx) { if (consecutiveFailures >= failureThreshold) { // Threshold reached -- connection is broken. logger.info("PING ACK not received for {} consecutive attempts on channel {} -- closing connection", - consecutiveFailures, ctx.channel().id().asShortText()); + consecutiveFailures, ctx.channel()); cancelPingTask(); ctx.close(); } else { logger.debug("PING ACK timeout on channel {} (attempt {}/{}) -- will retry", - ctx.channel().id().asShortText(), consecutiveFailures, failureThreshold); + ctx.channel(), consecutiveFailures, failureThreshold); } } // Don't send another PING while one is outstanding @@ -169,13 +169,13 @@ private void maybeSendPing(ChannelHandlerContext ctx) { ctx.writeAndFlush(new DefaultHttp2PingFrame(count)) .addListener(f -> { if (f.isSuccess()) { - logger.debug("PING #{} sent on channel {}", count, ctx.channel().id().asShortText()); + logger.debug("PING #{} sent on channel {}", count, ctx.channel()); } else { // Listener runs on the same event loop as the scheduled task, // so mutating pingOutstandingSinceNanos is thread-safe. pingOutstandingSinceNanos = 0; // unblock next attempt on send failure logger.debug("PING #{} send failed on channel {}: {}", - count, ctx.channel().id().asShortText(), + count, ctx.channel(), f.cause() != null ? f.cause().getMessage() : "unknown"); } }); From c7dc195b9d73564e64ccaf52c2eedfb40dcdc489 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Thu, 4 Jun 2026 18:19:01 -0400 Subject: [PATCH 48/62] Address xinlian12 review: env-var fallback, dormant state-clear, doOnConnected comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Configs: add COSMOS_HTTP2_PING_* env-var fallback for all 4 PING getters (parity with HTTP2_ENABLED-family) - Http2PingHandler: clear pingOutstandingSinceNanos and consecutiveFailures on dormant return to prevent stale state after re-enable - Http2PingHandler / ReactorNettyClient: correct doOnConnected comments — reactor-netty 1.2.x fires State.CONFIGURED on parent only; parent resolution + installIfAbsent guard are defensive Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../azure/cosmos/implementation/Configs.java | 32 +++++++++++++++---- .../implementation/http/Http2PingHandler.java | 13 ++++++-- .../http/ReactorNettyClient.java | 12 ++++--- 3 files changed, 43 insertions(+), 14 deletions(-) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java index 186d5b12812c..2c01e04d0994 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java @@ -153,16 +153,20 @@ public class Configs { // Guarded by an explicit enable flag; default ON. Set COSMOS.HTTP2_PING_HEALTH_ENABLED=false to disable. private static final boolean DEFAULT_HTTP2_PING_HEALTH_ENABLED = true; private static final String HTTP2_PING_HEALTH_ENABLED = "COSMOS.HTTP2_PING_HEALTH_ENABLED"; + private static final String HTTP2_PING_HEALTH_ENABLED_VARIABLE = "COSMOS_HTTP2_PING_HEALTH_ENABLED"; // Aligned with Rust SDK (hyper): interval=1s, timeout=2s. Dead connection detected within 3s. private static final int DEFAULT_HTTP2_PING_INTERVAL_IN_SECONDS = 1; private static final String HTTP2_PING_INTERVAL_IN_SECONDS = "COSMOS.HTTP2_PING_INTERVAL_IN_SECONDS"; + private static final String HTTP2_PING_INTERVAL_IN_SECONDS_VARIABLE = "COSMOS_HTTP2_PING_INTERVAL_IN_SECONDS"; private static final int DEFAULT_HTTP2_PING_TIMEOUT_IN_SECONDS = 2; private static final String HTTP2_PING_TIMEOUT_IN_SECONDS = "COSMOS.HTTP2_PING_TIMEOUT_IN_SECONDS"; + private static final String HTTP2_PING_TIMEOUT_IN_SECONDS_VARIABLE = "COSMOS_HTTP2_PING_TIMEOUT_IN_SECONDS"; // Consecutive PING failures (timeout without ACK) before closing the connection. // Aligned with Rust SDK's http2_consecutive_failure_threshold = 5. // With interval=1s and timeout=2s, worst-case detection = 5*(1+2) = ~15s. private static final int DEFAULT_HTTP2_PING_FAILURE_THRESHOLD = 5; private static final String HTTP2_PING_FAILURE_THRESHOLD = "COSMOS.HTTP2_PING_FAILURE_THRESHOLD"; + private static final String HTTP2_PING_FAILURE_THRESHOLD_VARIABLE = "COSMOS_HTTP2_PING_FAILURE_THRESHOLD"; private static final int DEFAULT_HTTP_RESPONSE_TIMEOUT_IN_SECONDS = 60; private static final int DEFAULT_QUERY_PLAN_RESPONSE_TIMEOUT_IN_SECONDS = 5; @@ -760,25 +764,39 @@ public static int getDefaultHttpPoolSize() { } public static boolean isHttp2PingHealthEnabled() { - return getJVMConfigAsBoolean(HTTP2_PING_HEALTH_ENABLED, DEFAULT_HTTP2_PING_HEALTH_ENABLED); + String configValue = System.getProperty( + HTTP2_PING_HEALTH_ENABLED, + firstNonNull( + emptyToNull(System.getenv().get(HTTP2_PING_HEALTH_ENABLED_VARIABLE)), + String.valueOf(DEFAULT_HTTP2_PING_HEALTH_ENABLED))); + return Boolean.parseBoolean(configValue); } public static int getHttp2PingIntervalInSeconds() { - return getJVMConfigAsInt( + String configValue = System.getProperty( HTTP2_PING_INTERVAL_IN_SECONDS, - DEFAULT_HTTP2_PING_INTERVAL_IN_SECONDS); + firstNonNull( + emptyToNull(System.getenv().get(HTTP2_PING_INTERVAL_IN_SECONDS_VARIABLE)), + String.valueOf(DEFAULT_HTTP2_PING_INTERVAL_IN_SECONDS))); + return Integer.parseInt(configValue); } public static int getHttp2PingTimeoutInSeconds() { - return getJVMConfigAsInt( + String configValue = System.getProperty( HTTP2_PING_TIMEOUT_IN_SECONDS, - DEFAULT_HTTP2_PING_TIMEOUT_IN_SECONDS); + firstNonNull( + emptyToNull(System.getenv().get(HTTP2_PING_TIMEOUT_IN_SECONDS_VARIABLE)), + String.valueOf(DEFAULT_HTTP2_PING_TIMEOUT_IN_SECONDS))); + return Integer.parseInt(configValue); } public static int getHttp2PingFailureThreshold() { - return getJVMConfigAsInt( + String configValue = System.getProperty( HTTP2_PING_FAILURE_THRESHOLD, - DEFAULT_HTTP2_PING_FAILURE_THRESHOLD); + firstNonNull( + emptyToNull(System.getenv().get(HTTP2_PING_FAILURE_THRESHOLD_VARIABLE)), + String.valueOf(DEFAULT_HTTP2_PING_FAILURE_THRESHOLD))); + return Integer.parseInt(configValue); } public static Integer getPendingAcquireMaxCount() { diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java index 71fddf71e9fb..04edf18befc0 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java @@ -135,8 +135,12 @@ private void maybeSendPing(ChannelHandlerContext ctx) { // Configs.isHttp2PingHealthEnabled() (system property). Flipping it false // makes this tick a no-op but KEEPS the timer alive, so toggling it back // to true automatically resumes PINGing on the same connection (no need - // to wait for connection recycling). + // to wait for connection recycling). Clear any in-flight PING state so a + // long dormant window followed by re-enable cannot charge a spurious + // timeout from a pre-disable outstanding PING. if (!Configs.isHttp2PingHealthEnabled()) { + pingOutstandingSinceNanos = 0; + consecutiveFailures = 0; return; } @@ -222,9 +226,12 @@ public static boolean isPingHealthEffectivelyEnabled(Http2ConnectionConfig http2 /** * Installs the PING handler on the parent H2 channel if not already installed. - * Safe to call from doOnConnected (which fires per-stream for H2). + * For reactor-netty 1.2.x, {@code doOnConnected} fires on the parent TCP channel + * (State.CONFIGURED) and not on child stream channels (STREAM_CONFIGURED), so the + * input {@code channel} here is normally the parent already. The parent resolution + * and idempotency guard below are defensive in case that contract ever changes. * - * @param channel the channel from doOnConnected (may be stream or parent) + * @param channel the channel from doOnConnected (normally the parent, but child-safe) * @param pingIntervalSeconds PING interval in seconds * @param pingTimeoutSeconds PING ACK timeout in seconds * @param failureThreshold consecutive timeouts before closing diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java index 1f26746f20bb..645b5cedb099 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java @@ -149,16 +149,20 @@ private void configureChannelPipelineHandlers() { this.httpClient = this.httpClient.doOnConnected(connection -> { // Manual HTTP/2 PING keepalive -- sends PING frames when the connection is idle // to prevent L7 middleboxes (NAT, firewalls, LBs) from reaping the connection. - // For H2, doOnConnected fires for both the parent TCP channel and child stream - // channels. We install on the parent channel via Http2MultiplexHandler detection. + // For H2, doOnConnected fires on the parent TCP channel when the connection + // is first established (State.CONFIGURED). Child stream channels emit + // STREAM_CONFIGURED, which does not trigger this callback. The parent- + // resolution and installIfAbsent guard below are defensive -- they correctly + // handle both parent and child channels if the reactor-netty contract + // ever changes. // // Gating is consolidated in Http2PingHandler.isPingHealthEffectivelyEnabled so // the transport install site and the user-agent feature flag cannot drift. // The handler also re-checks the kill-switch per tick so toggling it at // runtime immediately stops/resumes PINGing on already-installed connections. if (Http2PingHandler.isPingHealthEffectivelyEnabled(http2Cfg)) { - // Resolve to the parent (TCP) channel -- doOnConnected may fire for - // child stream channels where Http2MultiplexHandler is absent. + // Resolve to the parent (TCP) channel -- defensive in case reactor-netty + // ever invokes this callback for a child stream channel. Channel ch = connection.channel(); Channel parent = ch.parent() != null ? ch.parent() : ch; if (parent.pipeline().get(Http2MultiplexHandler.class) != null) { From 5aaf3e8e92d3be56a2ee16666d461ee427beff0b Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Thu, 4 Jun 2026 19:57:35 -0400 Subject: [PATCH 49/62] test(cosmos): unit tests for Http2PingHandler kill-switch clear and channelInactive cleanup Addresses kushagraThapar review (M1, M2) on PR #49095. - killSwitchOff_clearsOutstandingPingState: when COSMOS.HTTP2_PING_HEALTH_ENABLED flips to false, the next maybeSendPing tick clears pingOutstandingSinceNanos and consecutiveFailures so the handler returns to a dormant state without re-installing. - channelInactive_cancelsPingTask: closing the channel cancels the scheduled PING task and nulls the field so no further work is dispatched after handler removal. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../http/Http2PingHandlerTest.java | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/http/Http2PingHandlerTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/http/Http2PingHandlerTest.java index 169d17a7aa68..a1bee3fa5b1e 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/http/Http2PingHandlerTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/http/Http2PingHandlerTest.java @@ -3,11 +3,14 @@ package com.azure.cosmos.implementation.http; +import io.netty.channel.ChannelHandlerContext; import io.netty.channel.embedded.EmbeddedChannel; import io.netty.handler.codec.http2.DefaultHttp2PingFrame; import org.testng.annotations.Test; import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; @@ -21,6 +24,8 @@ *
  • PING ACK with mismatched payload is ignored (late ACK after timeout cannot mask degradation).
    • *
  • {@code installIfAbsent} is idempotent.
    • *
  • Constructor clamps non-positive interval / timeout / threshold to safe minimums.
    • + *
  • Runtime kill-switch toggle clears outstanding PING state (no spurious timeout on re-enable).
    • + *
  • {@code channelInactive} cancels the scheduled PING task (no leaked timer).
    • * * Time-based behaviors (interval-driven PING send, timeout-driven failure increment, threshold-driven * close) are exercised by the integration test {@code Http2PingKeepaliveTest} under Docker with real @@ -112,6 +117,69 @@ public void constructor_clampsNonPositiveValues() throws Exception { assertThat((int) getField(handler, "failureThreshold")).isEqualTo(1); } + @Test(groups = "unit") + public void killSwitchOff_clearsOutstandingPingState() throws Exception { + // M1: When Configs.isHttp2PingHealthEnabled() flips to false, the next maybeSendPing + // tick must clear any in-flight PING bookkeeping. Otherwise toggling the kill-switch + // back on after a long dormant window would charge a spurious timeout from a stale + // pingOutstandingSinceNanos that has been "outstanding" for hours. + final String prop = "COSMOS.HTTP2_PING_HEALTH_ENABLED"; + final String prior = System.getProperty(prop); + try { + Http2PingHandler handler = new Http2PingHandler(1, 1, 3); + EmbeddedChannel channel = new EmbeddedChannel(handler); + + // Simulate an in-flight PING with a partial failure already accumulated. + setField(handler, "pingOutstandingSinceNanos", System.nanoTime()); + setField(handler, "consecutiveFailures", 2); + + // Flip kill-switch OFF and trigger the periodic tick. + System.setProperty(prop, "false"); + ChannelHandlerContext ctx = channel.pipeline().firstContext(); + invokeMaybeSendPing(handler, ctx); + + assertThat((long) getField(handler, "pingOutstandingSinceNanos")).isZero(); + assertThat((int) getField(handler, "consecutiveFailures")).isZero(); + + // Timer must still be alive so re-enabling resumes PINGing on the same connection. + assertThat((Object) getField(handler, "pingTask")).isNotNull(); + + channel.finishAndReleaseAll(); + } finally { + if (prior == null) { + System.clearProperty(prop); + } else { + System.setProperty(prop, prior); + } + } + } + + @Test(groups = "unit") + public void channelInactive_cancelsPingTask() throws Exception { + // M2: channelInactive must cancel the scheduled PING task. Otherwise the timer + // would keep firing on a dead connection until GC reclaims the handler. + Http2PingHandler handler = new Http2PingHandler(1, 1, 3); + EmbeddedChannel channel = new EmbeddedChannel(handler); + + ScheduledFuture scheduled = (ScheduledFuture) getField(handler, "pingTask"); + assertThat((Object) scheduled).isNotNull(); + assertThat(scheduled.isCancelled()).isFalse(); + + // Closing the EmbeddedChannel fires channelInactive on the handler. + channel.close().syncUninterruptibly(); + + assertThat(scheduled.isCancelled()).isTrue(); + assertThat((Object) getField(handler, "pingTask")).isNull(); + + channel.finishAndReleaseAll(); + } + + private static void invokeMaybeSendPing(Http2PingHandler handler, ChannelHandlerContext ctx) throws Exception { + Method m = Http2PingHandler.class.getDeclaredMethod("maybeSendPing", ChannelHandlerContext.class); + m.setAccessible(true); + m.invoke(handler, ctx); + } + private static Object getField(Object target, String name) throws Exception { Field f = Http2PingHandler.class.getDeclaredField(name); f.setAccessible(true); From 1835b558bed9fc2e14f15ab8b525568b62c0dfe9 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Thu, 4 Jun 2026 20:06:20 -0400 Subject: [PATCH 50/62] ci(cosmos): scope HTTP/2 PING flags per profile, fix Compute matrix leg The Cosmos_live_test_http2_network_fault stage's AdditionalArgs hardcoded -DCOSMOS.THINCLIENT_ENABLED=true (plus HTTP2_ENABLED / HTTP2_PING_*) at the stage level, so the value was applied to every matrix entry. The CLI -D propagates into the failsafe-forked JVM and overrides the per-profile in azure-cosmos-tests/pom.xml, meaning the 'manual-http-network-fault-compute' leg silently ran with thin-client enabled and hit port 10250 instead of exercising the regional gateway on :443 -- making it a duplicate of the thin-client leg. Move the HTTP/2 PING flags (HTTP2_ENABLED, HTTP2_PING_HEALTH_ENABLED, HTTP2_PING_INTERVAL_IN_SECONDS, HTTP2_PING_TIMEOUT_IN_SECONDS) into both the 'manual-http-network-fault' and 'manual-http-network-fault-compute' pom profiles (universal to the test class), and drop them plus THINCLIENT_ENABLED from tests.yml so the per-profile values control routing. The stage AdditionalArgs now only carries the truly stage-level account host / key / leak-detection flag. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- sdk/cosmos/azure-cosmos-tests/pom.xml | 8 ++++++++ sdk/cosmos/tests.yml | 5 ++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/sdk/cosmos/azure-cosmos-tests/pom.xml b/sdk/cosmos/azure-cosmos-tests/pom.xml index 0c2c6c1df0f9..952bfeeb9ed1 100644 --- a/sdk/cosmos/azure-cosmos-tests/pom.xml +++ b/sdk/cosmos/azure-cosmos-tests/pom.xml @@ -875,6 +875,10 @@ Licensed under the MIT License. true true + true + true + 1 + 2 @@ -904,6 +908,10 @@ Licensed under the MIT License. true false + true + true + 1 + 2 diff --git a/sdk/cosmos/tests.yml b/sdk/cosmos/tests.yml index a466e5f5524a..0eecf42166ea 100644 --- a/sdk/cosmos/tests.yml +++ b/sdk/cosmos/tests.yml @@ -100,7 +100,10 @@ extends: TestResultsFiles: '**/junitreports/TEST-*.xml' AdditionalVariables: - name: AdditionalArgs - value: '-DCOSMOS.CLIENT_LEAK_DETECTION_ENABLED=true -DACCOUNT_HOST=$(thinclient-test-endpoint) -DACCOUNT_KEY=$(thinclient-test-key) -DCOSMOS.THINCLIENT_ENABLED=true -DCOSMOS.HTTP2_ENABLED=true -DCOSMOS.HTTP2_PING_HEALTH_ENABLED=true -DCOSMOS.HTTP2_PING_INTERVAL_IN_SECONDS=1 -DCOSMOS.HTTP2_PING_TIMEOUT_IN_SECONDS=2' + # COSMOS.THINCLIENT_ENABLED / HTTP2_* are driven by the per-profile + # in azure-cosmos-tests/pom.xml so the Compute + # matrix leg can route to the regional gateway on :443 instead of :10250. + value: '-DCOSMOS.CLIENT_LEAK_DETECTION_ENABLED=true -DACCOUNT_HOST=$(thinclient-test-endpoint) -DACCOUNT_KEY=$(thinclient-test-key)' - template: /eng/pipelines/templates/stages/archetype-sdk-tests-isolated.yml parameters: From b74b11697b1e22ac53d8c4009defb0ef8fa231b6 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Thu, 4 Jun 2026 20:58:49 -0400 Subject: [PATCH 51/62] fix(cosmos): count PING write failures toward close threshold A failed writeAndFlush listener previously cleared pingOutstandingSinceNanos but did NOT increment consecutiveFailures. If a channel was stuck in a state where writes consistently fail while channel.isActive() stays true (H2 codec rejecting frames, stalled flow-control, queued ClosedChannelException not yet propagated), the handler would retry forever without ever closing. Treat a write failure as a failed health probe: increment consecutiveFailures and trigger ctx.close() + cancelPingTask() at the threshold, mirroring the timeout path. Practical risk is low (channelInactive usually fires shortly), but this closes a real invariant gap in the state machine. Added Http2PingHandlerTest.writeFailure_incrementsConsecutiveFailuresAndClosesAtThreshold that injects a failing outbound handler and verifies the close path fires after threshold consecutive failures. Addresses PR #49095 review comment from xinlian12. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../http/Http2PingHandlerTest.java | 53 +++++++++++++++++++ .../implementation/http/Http2PingHandler.java | 23 ++++++-- 2 files changed, 71 insertions(+), 5 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/http/Http2PingHandlerTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/http/Http2PingHandlerTest.java index a1bee3fa5b1e..27bcb3d344cd 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/http/Http2PingHandlerTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/http/Http2PingHandlerTest.java @@ -4,10 +4,14 @@ package com.azure.cosmos.implementation.http; import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelOutboundHandlerAdapter; +import io.netty.channel.ChannelPromise; import io.netty.channel.embedded.EmbeddedChannel; import io.netty.handler.codec.http2.DefaultHttp2PingFrame; +import io.netty.util.ReferenceCountUtil; import org.testng.annotations.Test; +import java.io.IOException; import java.lang.reflect.Field; import java.lang.reflect.Method; import java.util.concurrent.ScheduledFuture; @@ -174,6 +178,55 @@ public void channelInactive_cancelsPingTask() throws Exception { channel.finishAndReleaseAll(); } + @Test(groups = "unit") + public void writeFailure_incrementsConsecutiveFailuresAndClosesAtThreshold() throws Exception { + // Reviewer-flagged invariant (PR #49095): a failed PING write must count as a + // failed health probe. Otherwise a channel stuck in a state where writes always + // fail (e.g. H2 codec rejecting frames, stalled flow-control, queued + // ClosedChannelException not yet propagated) but channel.isActive() stays true + // would loop on every tick without ever reaching the close threshold. + final int threshold = 2; + Http2PingHandler handler = new Http2PingHandler(1, 1, threshold); + EmbeddedChannel channel = new EmbeddedChannel(handler); + + // Insert an outbound handler at HEAD that fails every write. Outbound writes + // from Http2PingHandler flow head-ward, so this intercepts them and fails the + // promise synchronously -- producing the same listener-on-failure path the + // handler would see if a real H2 codec rejected the PING frame. + channel.pipeline().addFirst("failOnWrite", new ChannelOutboundHandlerAdapter() { + @Override + public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) { + ReferenceCountUtil.release(msg); + promise.setFailure(new IOException("simulated write failure")); + } + }); + + ChannelHandlerContext ctx = channel.pipeline().context(handler); + long longAgo = System.nanoTime() - TimeUnit.SECONDS.toNanos(60); + + // First tick: force idle -> PING send attempt -> write fails. + // Expect consecutiveFailures=1, channel still open, task still scheduled. + setField(handler, "lastActivityNanos", longAgo); + invokeMaybeSendPing(handler, ctx); + channel.runPendingTasks(); // run the write + listener on the embedded event loop + + assertThat((int) getField(handler, "consecutiveFailures")).isEqualTo(1); + assertThat((long) getField(handler, "pingOutstandingSinceNanos")).isZero(); + assertThat(channel.isOpen()).isTrue(); + assertThat((Object) getField(handler, "pingTask")).isNotNull(); + + // Second tick: write fails again -> threshold reached -> task cancelled, channel closed. + setField(handler, "lastActivityNanos", longAgo); + invokeMaybeSendPing(handler, ctx); + channel.runPendingTasks(); + + assertThat((int) getField(handler, "consecutiveFailures")).isEqualTo(threshold); + assertThat((Object) getField(handler, "pingTask")).isNull(); + assertThat(channel.isOpen()).isFalse(); + + channel.finishAndReleaseAll(); + } + private static void invokeMaybeSendPing(Http2PingHandler handler, ChannelHandlerContext ctx) throws Exception { Method m = Http2PingHandler.class.getDeclaredMethod("maybeSendPing", ChannelHandlerContext.class); m.setAccessible(true); diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java index 04edf18befc0..02d9f96fd81e 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java @@ -176,11 +176,24 @@ private void maybeSendPing(ChannelHandlerContext ctx) { logger.debug("PING #{} sent on channel {}", count, ctx.channel()); } else { // Listener runs on the same event loop as the scheduled task, - // so mutating pingOutstandingSinceNanos is thread-safe. - pingOutstandingSinceNanos = 0; // unblock next attempt on send failure - logger.debug("PING #{} send failed on channel {}: {}", - count, ctx.channel(), - f.cause() != null ? f.cause().getMessage() : "unknown"); + // so mutating these fields is thread-safe. + // A failed write is a failed health probe -- count it toward the + // close threshold. Otherwise a channel stuck in a state where + // writes always fail but isActive() remains true (e.g. H2 codec + // rejecting frames, stalled flow-control, queued ClosedChannelException + // not yet propagated) would loop here forever without ever closing. + pingOutstandingSinceNanos = 0; + consecutiveFailures++; + if (consecutiveFailures >= failureThreshold) { + logger.info("PING send failed for {} consecutive attempts on channel {} -- closing connection", + consecutiveFailures, ctx.channel()); + cancelPingTask(); + ctx.close(); + } else { + logger.debug("PING #{} send failed on channel {} (attempt {}/{}): {}", + count, ctx.channel(), consecutiveFailures, failureThreshold, + f.cause() != null ? f.cause().getMessage() : "unknown"); + } } }); // Reset activity timestamp so we don't send another PING immediately From 3266c1f6eaf0fab1b5969265dde8d260f0ca4cb0 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Thu, 4 Jun 2026 22:01:28 -0400 Subject: [PATCH 52/62] fix(cosmos-tests): close throw-away CosmosAsyncClient created by TestUtils TestUtils.createDummyQueryFeedOperationState(...,AsyncDocumentClient) builds an inner CosmosAsyncClient via CosmosClientBuilder and previously never closed it. CosmosNettyLeakDetectorFactory then flagged the resulting active client at @AfterClass with messages like 'CosmosClient [N] leaked' on tests such as GatewayReadConsistencyStrategySpyWireTest. Track every inner client created by that overload in a static list inside TestUtils and expose closeDummyClients(). Drain it via a new @AfterClass hook in TestSuiteBase and an explicit call in GatewayReadConsistencyStrategySpyWireTest.afterClass() (the only relevant caller that does not extend TestSuiteBase). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../cosmos/implementation/TestUtils.java | 33 +++++++++++++++++++ ...wayReadConsistencyStrategySpyWireTest.java | 4 +++ .../com/azure/cosmos/rx/TestSuiteBase.java | 10 ++++++ 3 files changed, 47 insertions(+) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/TestUtils.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/TestUtils.java index e55189a4af07..12f01f692cae 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/TestUtils.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/TestUtils.java @@ -6,10 +6,24 @@ import com.azure.cosmos.CosmosClientBuilder; import com.azure.cosmos.models.CosmosQueryRequestOptions; import org.mockito.Mockito; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.util.List; import java.util.UUID; +import java.util.concurrent.CopyOnWriteArrayList; public class TestUtils { + private static final Logger LOGGER = LoggerFactory.getLogger(TestUtils.class); + + // Tracks CosmosAsyncClient instances created by the AsyncDocumentClient overload of + // createDummyQueryFeedOperationState below. Those callers historically passed an + // AsyncDocumentClient (no enclosing CosmosAsyncClient to reuse), forcing this helper + // to build its own CosmosAsyncClient that previously was never closed. The + // CosmosNettyLeakDetectorFactory flags such clients as leaks at @AfterClass. + // Test base classes call closeDummyClients() in @AfterClass to drain this list. + private static final List DUMMY_CLIENTS = new CopyOnWriteArrayList<>(); + private static final String DATABASES_PATH_SEGMENT = "dbs"; private static final String COLLECTIONS_PATH_SEGMENT = "colls"; private static final String DOCUMENTS_PATH_SEGMENT = "docs"; @@ -69,6 +83,7 @@ public static QueryFeedOperationState createDummyQueryFeedOperationState( .key(client.getMasterKeyOrResourceToken()) .endpoint(client.getServiceEndpoint().toString()) .buildAsyncClient(); + DUMMY_CLIENTS.add(cosmosClient); return new QueryFeedOperationState( cosmosClient, "SomeSpanName", @@ -82,6 +97,24 @@ public static QueryFeedOperationState createDummyQueryFeedOperationState( ); } + /** + * Closes every {@link CosmosAsyncClient} created by the + * {@link #createDummyQueryFeedOperationState(ResourceType, OperationType, CosmosQueryRequestOptions, AsyncDocumentClient)} + * overload. Safe to call multiple times. Test base classes invoke this in + * {@code @AfterClass} so the leak detector does not flag these throw-away + * inner clients. + */ + public static void closeDummyClients() { + for (CosmosAsyncClient client : DUMMY_CLIENTS) { + try { + client.close(); + } catch (Exception e) { + LOGGER.warn("Failed to close dummy CosmosAsyncClient created by TestUtils", e); + } + } + DUMMY_CLIENTS.clear(); + } + public static QueryFeedOperationState createDummyQueryFeedOperationState(ResourceType resourceType, OperationType operationType, CosmosQueryRequestOptions options, diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/GatewayReadConsistencyStrategySpyWireTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/GatewayReadConsistencyStrategySpyWireTest.java index 5a2390408e20..3a81062bf48a 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/GatewayReadConsistencyStrategySpyWireTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/GatewayReadConsistencyStrategySpyWireTest.java @@ -148,6 +148,10 @@ public void afterClass() { if (v2SpyClient != null) { v2SpyClient.close(); } + // Close throw-away CosmosAsyncClient instances created by + // TestUtils.createDummyQueryFeedOperationState so the leak detector + // does not flag them. + TestUtils.closeDummyClients(); } @DataProvider(name = "transportModes") diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java index 66b6e6314ad0..86d0843447c6 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java @@ -90,6 +90,7 @@ import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.mockito.stubbing.Answer; +import org.testng.annotations.AfterClass; import org.testng.annotations.AfterSuite; import org.testng.annotations.BeforeSuite; import org.testng.annotations.DataProvider; @@ -351,6 +352,15 @@ private static DocumentCollection getInternalDocumentCollection(CosmosAsyncConta return collection; } + @AfterClass(alwaysRun = true) + public void afterClassCloseDummyClients() { + // Closes any throw-away CosmosAsyncClient created by + // TestUtils.createDummyQueryFeedOperationState(..., AsyncDocumentClient). + // Without this, CosmosNettyLeakDetectorFactory reports them as leaks + // at @AfterClass. + TestUtils.closeDummyClients(); + } + @AfterSuite(groups = {"thinclient", "fast", "long", "direct", "multi-region", "multi-master", "flaky-multi-master", "emulator", "split", "query", "cfp-split", "circuit-breaker-misc-gateway", "circuit-breaker-misc-direct", "circuit-breaker-read-all-read-many", "fi-multi-master", "long-emulator", "fi-thinclient-multi-region", "fi-thinclient-multi-master", "multi-region-strong", "manual-http-network-fault"}, timeOut = SUITE_SHUTDOWN_TIMEOUT) From 9c9c78238d4ae66254af8d7e64e57723463c6afa Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Fri, 5 Jun 2026 08:54:21 -0400 Subject: [PATCH 53/62] fix(cosmos-tests): drain TestUtils dummy clients from leak detector The previous attempt added an inherited @AfterClass method on TestSuiteBase, which broke CosmosNettyLeakDetectorFactory's per-class counter (it decrements once per @AfterClass invocation but is sized for a single one). The detector fired prematurely on the first @AfterClass method, before the derived test's afterClass() had closed its own CosmosAsyncClient, producing false leak failures across many tests (e.g. WebExceptionRetryPolicyE2ETests). Revert the TestSuiteBase and GatewayReadConsistencyStrategySpyWireTest hooks and instead call TestUtils.closeDummyClients() from CosmosNettyLeakDetectorFactory.onAfterClassCore() right before snapshotting active clients. This adds no new @AfterClass method and covers every test class through the existing listener. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../azure/cosmos/CosmosNettyLeakDetectorFactory.java | 4 ++++ .../rx/GatewayReadConsistencyStrategySpyWireTest.java | 4 ---- .../test/java/com/azure/cosmos/rx/TestSuiteBase.java | 10 ---------- 3 files changed, 4 insertions(+), 14 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java index c8c87f945a29..e9e2cfc058a5 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java @@ -127,6 +127,10 @@ private void onAfterClassCore(ITestClass testClass) { if (remainingInstanceCount == 0) { String failMessage = ""; logger.info("LEAK DETECTION EVALUATION for test class {}", testClassName); + // Drain any throw-away CosmosAsyncClient instances created by + // TestUtils.createDummyQueryFeedOperationState(..., AsyncDocumentClient) + // so they do not register as leaks below. + com.azure.cosmos.implementation.TestUtils.closeDummyClients(); Map leakedClientSnapshotNow = RxDocumentClientImpl.getActiveClientsSnapshot(); StringBuilder sb = new StringBuilder(); Map leakedClientSnapshotAtBegin = activeClientsAtBegin; diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/GatewayReadConsistencyStrategySpyWireTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/GatewayReadConsistencyStrategySpyWireTest.java index 3a81062bf48a..5a2390408e20 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/GatewayReadConsistencyStrategySpyWireTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/GatewayReadConsistencyStrategySpyWireTest.java @@ -148,10 +148,6 @@ public void afterClass() { if (v2SpyClient != null) { v2SpyClient.close(); } - // Close throw-away CosmosAsyncClient instances created by - // TestUtils.createDummyQueryFeedOperationState so the leak detector - // does not flag them. - TestUtils.closeDummyClients(); } @DataProvider(name = "transportModes") diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java index 86d0843447c6..66b6e6314ad0 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java @@ -90,7 +90,6 @@ import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.mockito.stubbing.Answer; -import org.testng.annotations.AfterClass; import org.testng.annotations.AfterSuite; import org.testng.annotations.BeforeSuite; import org.testng.annotations.DataProvider; @@ -352,15 +351,6 @@ private static DocumentCollection getInternalDocumentCollection(CosmosAsyncConta return collection; } - @AfterClass(alwaysRun = true) - public void afterClassCloseDummyClients() { - // Closes any throw-away CosmosAsyncClient created by - // TestUtils.createDummyQueryFeedOperationState(..., AsyncDocumentClient). - // Without this, CosmosNettyLeakDetectorFactory reports them as leaks - // at @AfterClass. - TestUtils.closeDummyClients(); - } - @AfterSuite(groups = {"thinclient", "fast", "long", "direct", "multi-region", "multi-master", "flaky-multi-master", "emulator", "split", "query", "cfp-split", "circuit-breaker-misc-gateway", "circuit-breaker-misc-direct", "circuit-breaker-read-all-read-many", "fi-multi-master", "long-emulator", "fi-thinclient-multi-region", "fi-thinclient-multi-master", "multi-region-strong", "manual-http-network-fault"}, timeOut = SUITE_SHUTDOWN_TIMEOUT) From d2d14e556f91c9fa2ca97b9f77fa1a93dbf1c3a4 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Fri, 5 Jun 2026 09:19:30 -0400 Subject: [PATCH 54/62] fix(cosmos-tests): isolate dummy client cleanup to GatewayReadConsistencyStrategySpyWireTest GatewayReadConsistencyStrategySpyWireTest is the only test that triggers the TestUtils dummy client leak. Restore the explicit drain in its existing afterClass() (no new @AfterClass added, so the leak detector's per-class counter is unaffected) and revert the broader hook in CosmosNettyLeakDetectorFactory. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java | 4 ---- .../cosmos/rx/GatewayReadConsistencyStrategySpyWireTest.java | 4 ++++ 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java index e9e2cfc058a5..c8c87f945a29 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java @@ -127,10 +127,6 @@ private void onAfterClassCore(ITestClass testClass) { if (remainingInstanceCount == 0) { String failMessage = ""; logger.info("LEAK DETECTION EVALUATION for test class {}", testClassName); - // Drain any throw-away CosmosAsyncClient instances created by - // TestUtils.createDummyQueryFeedOperationState(..., AsyncDocumentClient) - // so they do not register as leaks below. - com.azure.cosmos.implementation.TestUtils.closeDummyClients(); Map leakedClientSnapshotNow = RxDocumentClientImpl.getActiveClientsSnapshot(); StringBuilder sb = new StringBuilder(); Map leakedClientSnapshotAtBegin = activeClientsAtBegin; diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/GatewayReadConsistencyStrategySpyWireTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/GatewayReadConsistencyStrategySpyWireTest.java index 5a2390408e20..3a81062bf48a 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/GatewayReadConsistencyStrategySpyWireTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/GatewayReadConsistencyStrategySpyWireTest.java @@ -148,6 +148,10 @@ public void afterClass() { if (v2SpyClient != null) { v2SpyClient.close(); } + // Close throw-away CosmosAsyncClient instances created by + // TestUtils.createDummyQueryFeedOperationState so the leak detector + // does not flag them. + TestUtils.closeDummyClients(); } @DataProvider(name = "transportModes") From e19c187667dc7c30976c69c3d2ef4f73dffc9241 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Fri, 5 Jun 2026 09:25:47 -0400 Subject: [PATCH 55/62] Isolate dummy-client leak fix to GatewayReadConsistencyStrategySpyWireTest Revert the DUMMY_CLIENTS tracking and closeDummyClients() scaffolding from TestUtils.java and route the two call sites in GatewayReadConsistencyStrategySpyWireTest.executeQueryAndCapture and executeReadManyAndCapture through the existing safe overload of createDummyQueryFeedOperationState that accepts the outer CosmosAsyncClient (the same pattern executeChangeFeedAndCapture already uses). No throw-away inner CosmosAsyncClient is created, so there is nothing to leak and no shared-class scaffolding to drag along. Net change vs main is two lines in a single test file. TestUtils.java is byte-for-byte identical to main. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../cosmos/implementation/TestUtils.java | 33 ------------------- ...wayReadConsistencyStrategySpyWireTest.java | 8 ++--- 2 files changed, 2 insertions(+), 39 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/TestUtils.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/TestUtils.java index 12f01f692cae..e55189a4af07 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/TestUtils.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/TestUtils.java @@ -6,24 +6,10 @@ import com.azure.cosmos.CosmosClientBuilder; import com.azure.cosmos.models.CosmosQueryRequestOptions; import org.mockito.Mockito; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import java.util.List; import java.util.UUID; -import java.util.concurrent.CopyOnWriteArrayList; public class TestUtils { - private static final Logger LOGGER = LoggerFactory.getLogger(TestUtils.class); - - // Tracks CosmosAsyncClient instances created by the AsyncDocumentClient overload of - // createDummyQueryFeedOperationState below. Those callers historically passed an - // AsyncDocumentClient (no enclosing CosmosAsyncClient to reuse), forcing this helper - // to build its own CosmosAsyncClient that previously was never closed. The - // CosmosNettyLeakDetectorFactory flags such clients as leaks at @AfterClass. - // Test base classes call closeDummyClients() in @AfterClass to drain this list. - private static final List DUMMY_CLIENTS = new CopyOnWriteArrayList<>(); - private static final String DATABASES_PATH_SEGMENT = "dbs"; private static final String COLLECTIONS_PATH_SEGMENT = "colls"; private static final String DOCUMENTS_PATH_SEGMENT = "docs"; @@ -83,7 +69,6 @@ public static QueryFeedOperationState createDummyQueryFeedOperationState( .key(client.getMasterKeyOrResourceToken()) .endpoint(client.getServiceEndpoint().toString()) .buildAsyncClient(); - DUMMY_CLIENTS.add(cosmosClient); return new QueryFeedOperationState( cosmosClient, "SomeSpanName", @@ -97,24 +82,6 @@ public static QueryFeedOperationState createDummyQueryFeedOperationState( ); } - /** - * Closes every {@link CosmosAsyncClient} created by the - * {@link #createDummyQueryFeedOperationState(ResourceType, OperationType, CosmosQueryRequestOptions, AsyncDocumentClient)} - * overload. Safe to call multiple times. Test base classes invoke this in - * {@code @AfterClass} so the leak detector does not flag these throw-away - * inner clients. - */ - public static void closeDummyClients() { - for (CosmosAsyncClient client : DUMMY_CLIENTS) { - try { - client.close(); - } catch (Exception e) { - LOGGER.warn("Failed to close dummy CosmosAsyncClient created by TestUtils", e); - } - } - DUMMY_CLIENTS.clear(); - } - public static QueryFeedOperationState createDummyQueryFeedOperationState(ResourceType resourceType, OperationType operationType, CosmosQueryRequestOptions options, diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/GatewayReadConsistencyStrategySpyWireTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/GatewayReadConsistencyStrategySpyWireTest.java index 3a81062bf48a..99fc00b69f6f 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/GatewayReadConsistencyStrategySpyWireTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/GatewayReadConsistencyStrategySpyWireTest.java @@ -148,10 +148,6 @@ public void afterClass() { if (v2SpyClient != null) { v2SpyClient.close(); } - // Close throw-away CosmosAsyncClient instances created by - // TestUtils.createDummyQueryFeedOperationState so the leak detector - // does not flag them. - TestUtils.closeDummyClients(); } @DataProvider(name = "transportModes") @@ -517,7 +513,7 @@ private HttpRequest executeQueryAndCapture( CosmosQueryRequestOptions queryOptions) { QueryFeedOperationState state = TestUtils.createDummyQueryFeedOperationState( - ResourceType.Document, OperationType.Query, queryOptions, client); + ResourceType.Document, OperationType.Query, queryOptions, cosmosClient); client.clearCapturedRequests(); client.queryDocuments(getCollectionLink(), "SELECT * FROM c", state, Document.class) .blockFirst(); @@ -563,7 +559,7 @@ private HttpRequest executeReadManyAndCapture( CosmosQueryRequestOptions queryOptions) { QueryFeedOperationState state = TestUtils.createDummyQueryFeedOperationState( - ResourceType.Document, OperationType.Query, queryOptions, client); + ResourceType.Document, OperationType.Query, queryOptions, cosmosClient); client.clearCapturedRequests(); client.readManyByPartitionKeys( Collections.singletonList(new PartitionKey(DOCUMENT_ID)), From c61201548439043ee7ba6ceb2a3decb55c7beefd Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Fri, 5 Jun 2026 11:26:38 -0400 Subject: [PATCH 56/62] Address xinlian12 PR comments: TOCTOU guard, doc clarity, ACK-timeout unit test - ReactorNettyClient: wrap customHeaderCleaner addAfter in try/catch to mirror the sibling Http2ParentChannelExceptionHandler install pattern. Guards against a TOCTOU race when the channel is closed between the null-check and addAfter. - Configs (HTTP2_PING_INTERVAL_IN_SECONDS): replace contradictory `Detected within 3s` comment. Clarify that one missed PING round is ~3s (interval + 2s timeout) and connection close requires HTTP2_PING_FAILURE_THRESHOLD consecutive failures (~15s worst case). - Http2PingHandlerTest: add ackTimeout_incrementsConsecutiveFailuresAndClosesAtThreshold to mirror the existing write-failure coverage. Update class Javadoc to move the ACK-timeout and write-failure paths into the unit-test bullet list. Verified: azure-cosmos-tests Http2PingHandlerTest 9/9 pass via -Punit verify. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../http/Http2PingHandlerTest.java | 55 +++++++++++++++++-- .../azure/cosmos/implementation/Configs.java | 4 +- .../http/ReactorNettyClient.java | 14 +++-- 3 files changed, 64 insertions(+), 9 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/http/Http2PingHandlerTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/http/Http2PingHandlerTest.java index 27bcb3d344cd..004cde861d87 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/http/Http2PingHandlerTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/http/Http2PingHandlerTest.java @@ -22,7 +22,10 @@ /** * Unit tests for {@link Http2PingHandler}. *

    - * Covers state transitions that do not require advancing time: + * Covers state transitions that do not require advancing real time -- the handler's + * timeout / threshold logic reads {@code System.nanoTime()}, so the tests sidestep + * the clock by pre-setting {@code pingOutstandingSinceNanos} (or + * {@code lastActivityNanos}) via reflection: *

      *
    • PING ACK with matching payload resets the failure counter (RFC 9113 §6.7 payload echo).
      • *
    • PING ACK with mismatched payload is ignored (late ACK after timeout cannot mask degradation).
      • @@ -30,10 +33,13 @@ *
    • Constructor clamps non-positive interval / timeout / threshold to safe minimums.
      • *
    • Runtime kill-switch toggle clears outstanding PING state (no spurious timeout on re-enable).
      • *
    • {@code channelInactive} cancels the scheduled PING task (no leaked timer).
      • + *
    • Failed PING write increments {@code consecutiveFailures} and closes the channel at threshold.
      • + *
    • ACK timeout (outstanding PING aged past {@code pingTimeoutNanos}) increments + * {@code consecutiveFailures} and closes the channel at threshold.
      • *
    - * Time-based behaviors (interval-driven PING send, timeout-driven failure increment, threshold-driven - * close) are exercised by the integration test {@code Http2PingKeepaliveTest} under Docker with real - * network fault injection. + * The interval-driven scheduling itself (that {@code maybeSendPing} is fired by the + * event loop every {@code pingIntervalNanos}) is exercised by the integration test + * {@code Http2PingKeepaliveTest} under Docker with real network fault injection. */ public class Http2PingHandlerTest { @@ -227,6 +233,47 @@ public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) channel.finishAndReleaseAll(); } + @Test(groups = "unit") + public void ackTimeout_incrementsConsecutiveFailuresAndClosesAtThreshold() throws Exception { + // Reviewer-flagged invariant (PR #49095): the primary operational PING-failure + // path -- a PING was sent, no ACK arrived within pingTimeoutNanos, and after + // `failureThreshold` consecutive such timeouts the channel must be closed. + // Time is sidestepped by pre-setting pingOutstandingSinceNanos to a value + // older than pingTimeoutNanos so maybeSendPing enters the timeout branch on + // each invocation. + final int threshold = 2; + Http2PingHandler handler = new Http2PingHandler(1, 1, threshold); // interval=1s, timeout=1s + EmbeddedChannel channel = new EmbeddedChannel(handler); + ChannelHandlerContext ctx = channel.pipeline().context(handler); + + long pastTimeout = System.nanoTime() - TimeUnit.SECONDS.toNanos(5); + + // First tick: outstanding PING has aged past timeout -> failures=1, channel still open. + setField(handler, "pingsSent", 1); + setField(handler, "pingOutstandingSinceNanos", pastTimeout); + invokeMaybeSendPing(handler, ctx); + channel.runPendingTasks(); + + assertThat((int) getField(handler, "consecutiveFailures")).isEqualTo(1); + assertThat((long) getField(handler, "pingOutstandingSinceNanos")).isZero(); + assertThat(channel.isOpen()).isTrue(); + assertThat((Object) getField(handler, "pingTask")).isNotNull(); + + // Second tick: simulate another outstanding PING that has aged past timeout + // -> failures=threshold -> task cancelled, channel closed. + setField(handler, "pingsSent", 2); + setField(handler, "pingOutstandingSinceNanos", pastTimeout); + invokeMaybeSendPing(handler, ctx); + channel.runPendingTasks(); + + assertThat((int) getField(handler, "consecutiveFailures")).isEqualTo(threshold); + assertThat((long) getField(handler, "pingOutstandingSinceNanos")).isZero(); + assertThat((Object) getField(handler, "pingTask")).isNull(); + assertThat(channel.isOpen()).isFalse(); + + channel.finishAndReleaseAll(); + } + private static void invokeMaybeSendPing(Http2PingHandler handler, ChannelHandlerContext ctx) throws Exception { Method m = Http2PingHandler.class.getDeclaredMethod("maybeSendPing", ChannelHandlerContext.class); m.setAccessible(true); diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java index 2c01e04d0994..657a6f5ee39c 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java @@ -154,7 +154,9 @@ public class Configs { private static final boolean DEFAULT_HTTP2_PING_HEALTH_ENABLED = true; private static final String HTTP2_PING_HEALTH_ENABLED = "COSMOS.HTTP2_PING_HEALTH_ENABLED"; private static final String HTTP2_PING_HEALTH_ENABLED_VARIABLE = "COSMOS_HTTP2_PING_HEALTH_ENABLED"; - // Aligned with Rust SDK (hyper): interval=1s, timeout=2s. Dead connection detected within 3s. + // Aligned with Rust SDK (hyper): interval=1s, timeout=2s. A single missed PING round + // (~3s) marks one failure; the connection is closed after HTTP2_PING_FAILURE_THRESHOLD + // consecutive failures (see ~15s worst-case detection note below). private static final int DEFAULT_HTTP2_PING_INTERVAL_IN_SECONDS = 1; private static final String HTTP2_PING_INTERVAL_IN_SECONDS = "COSMOS.HTTP2_PING_INTERVAL_IN_SECONDS"; private static final String HTTP2_PING_INTERVAL_IN_SECONDS_VARIABLE = "COSMOS_HTTP2_PING_INTERVAL_IN_SECONDS"; diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java index 645b5cedb099..5ccf9f4de02b 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java @@ -194,10 +194,16 @@ private void configureChannelPipelineHandlers() { ChannelPipeline channelPipeline = connection.channel().pipeline(); if (channelPipeline.get("reactor.left.httpCodec") != null && channelPipeline.get("customHeaderCleaner") == null) { - channelPipeline.addAfter( - "reactor.left.httpCodec", - "customHeaderCleaner", - new Http2ResponseHeaderCleanerHandler()); + try { + channelPipeline.addAfter( + "reactor.left.httpCodec", + "customHeaderCleaner", + new Http2ResponseHeaderCleanerHandler()); + } catch (IllegalArgumentException ignored) { + // TOCTOU race: between the get()==null check above and addAfter(), + // a concurrent doOnConnected may have installed the handler. + // Duplicate handler name is the only possible cause. + } } // Install exception handler at the tail of the HTTP/2 parent (TCP) From 17db0712abd2a56444f7ba05dc57d3bc1fc7dcc5 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Fri, 5 Jun 2026 13:23:11 -0400 Subject: [PATCH 57/62] docs(cosmos): clarify HTTP/2 PING threshold semantics and expand CHANGELOG Addresses kushagraThapar PR review 4432203224: - H2: Configs.java threshold comment no longer claims false alignment with Rust SDK's http2_consecutive_failure_threshold (which is a per-HTTP-request shard-health knob, not per-PING-ACK). Now explicitly notes that peer HTTP/2 stacks (Hyper / .NET SocketsHttpHandler / Go net/http) typically close on the first PING-ACK timeout, and that Java's threshold of 5 is intentionally more tolerant. - L8 + H5: CHANGELOG entry for PR 49095 now calls out: * default-ON state * all four COSMOS.HTTP2_PING_* tunable knob names * the defensive concurrent-install hardening on the H2 parent pipeline (customHeaderCleaner / Http2ParentChannelExceptionHandler) No code semantics change; comments and changelog only. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- sdk/cosmos/azure-cosmos/CHANGELOG.md | 2 +- .../java/com/azure/cosmos/implementation/Configs.java | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/sdk/cosmos/azure-cosmos/CHANGELOG.md b/sdk/cosmos/azure-cosmos/CHANGELOG.md index c735ca9c839b..9b6dea8a053d 100644 --- a/sdk/cosmos/azure-cosmos/CHANGELOG.md +++ b/sdk/cosmos/azure-cosmos/CHANGELOG.md @@ -14,7 +14,7 @@ * Fixed `UnsupportedOperationException` when using `readManyByPartitionKeys` for empty pages. - See [PR 49311](https://github.com/Azure/azure-sdk-for-java/pull/49311) #### Other Changes -* Added ability to detect broken connections to a Gateway V2 service endpoint. - See [PR 49095](https://github.com/Azure/azure-sdk-for-java/pull/49095) +* Added HTTP/2 PING keepalive (default ON) for Gateway V2 service endpoints to detect silently-broken connections. Tunable via `COSMOS.HTTP2_PING_HEALTH_ENABLED`, `COSMOS.HTTP2_PING_INTERVAL_IN_SECONDS`, `COSMOS.HTTP2_PING_TIMEOUT_IN_SECONDS`, `COSMOS.HTTP2_PING_FAILURE_THRESHOLD` (each also accepts the equivalent `COSMOS_*` env var). Hardened concurrent installation of `customHeaderCleaner` and `Http2ParentChannelExceptionHandler` on the H2 parent pipeline against benign duplicate-name races from reactor-netty `doOnConnected` callbacks. - See [PR 49095](https://github.com/Azure/azure-sdk-for-java/pull/49095) * Replaced per-client `Schedulers.newSingle()` schedulers in `GlobalEndpointManager` and `GlobalPartitionEndpointManagerForPerPartitionCircuitBreaker` with shared `BoundedElastic` schedulers in `CosmosSchedulers` to prevent thread count from scaling linearly with client/tenant count. - See [PR 49062](https://github.com/Azure/azure-sdk-for-java/pull/49062) * Promoted the `ReadConsistencyStrategy` and `Http2ConnectionConfig` related `@Beta` APIs to GA. - See [PR 49345](https://github.com/Azure/azure-sdk-for-java/pull/49345) * Fixed a sporadic `NullPointerException` in `JsonSerializable.getWithMapping` triggered by concurrent first-time calls to `DatabaseAccount.getConsistencyPolicy()` and its sibling lazy getters (`getReplicationPolicy`, `getSystemReplicationPolicy`, `getQueryEngineConfiguration`). The fix makes `JsonSerializable.propertyBag` `final`, closing an unsafe-publication race in the lazy-initialisation pattern. - See [Issue 49256](https://github.com/Azure/azure-sdk-for-java/issues/49256) and [PR #49258](https://github.com/Azure/azure-sdk-for-java/pull/49258) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java index 657a6f5ee39c..ec0ad1570225 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java @@ -164,8 +164,13 @@ public class Configs { private static final String HTTP2_PING_TIMEOUT_IN_SECONDS = "COSMOS.HTTP2_PING_TIMEOUT_IN_SECONDS"; private static final String HTTP2_PING_TIMEOUT_IN_SECONDS_VARIABLE = "COSMOS_HTTP2_PING_TIMEOUT_IN_SECONDS"; // Consecutive PING failures (timeout without ACK) before closing the connection. - // Aligned with Rust SDK's http2_consecutive_failure_threshold = 5. - // With interval=1s and timeout=2s, worst-case detection = 5*(1+2) = ~15s. + // Peer HTTP/2 stacks (Hyper / .NET SocketsHttpHandler / Go net/http) typically close + // on the first PING-ACK timeout. Java's threshold of 5 is intentionally more tolerant + // to absorb transient WAN jitter; with interval=1s and timeout=2s, worst-case + // detection = 5*(1+2) = ~15s. + // Note: this is NOT the same dimension as Rust SDK's + // `http2_consecutive_failure_threshold` (which gates per-HTTP-request shard health, + // not per-PING-ACK timeouts). private static final int DEFAULT_HTTP2_PING_FAILURE_THRESHOLD = 5; private static final String HTTP2_PING_FAILURE_THRESHOLD = "COSMOS.HTTP2_PING_FAILURE_THRESHOLD"; private static final String HTTP2_PING_FAILURE_THRESHOLD_VARIABLE = "COSMOS_HTTP2_PING_FAILURE_THRESHOLD"; From 7f4b509345fee4f726e2e304cf97b0cc22c0fba4 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Fri, 5 Jun 2026 14:50:33 -0400 Subject: [PATCH 58/62] H3: route HTTP/2 PING-driven channel close as retryable transport failure Addresses kushagraThapar H3 review on PR #49095 - when Http2PingHandler closes a connection after consecutive PING ACK failures, surface the close as a retryable transport failure so ClientRetryPolicy retries in the SAME region without marking the regional gateway endpoint unavailable. Production: - Http2PingHandler fires typed Http2PingTimeoutChannelClosedException through the pipeline before closing, so reactor-netty's response Mono surfaces the typed cause on any in-flight HTTP read. - New Http2PingCloseRewrapHandler installed alongside the PING handler rewraps NIO ClosedChannelException as Http2PingTimeoutChannelClosedException when fired after a PING-driven close (handles the post-close race where the exception travels the quiescent path). - ClientRetryPolicy gains a shouldRetryOnGatewayTimeout H3 branch that detects Http2PingTimeoutChannelClosedException (and substatus 10006) and routes to in-region retry without endpoint mark-down. - RxGatewayStoreModel stamps substatus 10006 (HTTP2_PING_TIMEOUT_CHANNEL_CLOSED) on the resulting CosmosException so diagnostics and retry policy identify the cause without unwrapping. - WebExceptionUtility.isWebExceptionRetriable returns true for the typed exception. - HttpConstants exposes the new substatus. Tests: - Http2PingKeepaliveTest restructured for Option B (in-flight read with iptables DROP on thin-client port). Verified in Docker end-to-end: Tests run: 1, Failures: 0; SAME_REGION=true, DIFFERENT_CONNECTION=true. - New ClientRetryPolicyHttp2PingCloseTest covers the retry-policy branch in isolation. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Http2PingKeepaliveTest.java | 188 +++++++++++++----- .../ClientRetryPolicyHttp2PingCloseTest.java | 155 +++++++++++++++ sdk/cosmos/azure-cosmos/CHANGELOG.md | 2 +- .../implementation/ClientRetryPolicy.java | 12 ++ .../cosmos/implementation/HttpConstants.java | 7 + .../implementation/RxGatewayStoreModel.java | 10 +- .../WebExceptionUtility.java | 26 +++ .../http/Http2PingCloseRewrapHandler.java | 55 +++++ .../implementation/http/Http2PingHandler.java | 25 +++ ...ttp2PingTimeoutChannelClosedException.java | 47 +++++ .../http/ReactorNettyClient.java | 32 ++- 11 files changed, 503 insertions(+), 56 deletions(-) create mode 100644 sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/ClientRetryPolicyHttp2PingCloseTest.java create mode 100644 sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingCloseRewrapHandler.java create mode 100644 sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingTimeoutChannelClosedException.java diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java index 73182ef331df..99a5f25a793d 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java @@ -114,19 +114,28 @@ public void afterMethod(Method method) { } /** - * Uses iptables to silently discard all traffic to the Cosmos DB gateway port, - * preventing PING ACKs from arriving. Verifies the handler detects the broken - * connection after consecutive PING failures and closes it. A subsequent request - * (after removing the iptables rule) uses a new connection. + * End-to-end H3 verification: a read request issued WHILE the connection is being + * blackholed (iptables DROP on the thin-client / regional-gateway port) is held in + * flight while {@link com.azure.cosmos.implementation.http.Http2PingHandler} + * closes the parent channel after consecutive PING ACK timeouts. The closed + * channel propagates a typed + * {@link com.azure.cosmos.implementation.http.Http2PingTimeoutChannelClosedException} + * to the in-flight child stream, which {@code RxGatewayStoreModel} stamps with + * {@code SubStatusCodes.GATEWAY_HTTP2_PING_TIMEOUT_CHANNEL_CLOSED (10006)}. The + * test then asserts that {@code ClientRetryPolicy} retried the request and the + * eventual success landed on the SAME regional gateway endpoint, proving that + * the PING-driven close did NOT trigger {@code markEndpointUnavailableForRead} + * or cross-region failover. *

    - * This test proves PINGs are actively flowing on idle connections -- without PINGs, - * iptables DROP would go undetected (channel.isActive() stays true, no GOAWAY arrives) - * and the test would time out. + * Without this fix, the same exception would have been classified as a generic + * {@code GATEWAY_ENDPOINT_UNAVAILABLE (10001)}, refresh-location would have run, + * the regional endpoint would have been marked down, and the retry would have + * either failed or landed on a different region depending on preferred-regions. *

    - * Requires Docker with --cap-add=NET_ADMIN or Linux with sudo. + * Requires Docker with {@code --cap-add=NET_ADMIN} or Linux with sudo. */ @Test(groups = {"manual-http-network-fault"}, timeOut = TEST_TIMEOUT) - public void connectionClosedOnPingTimeout() throws Exception { + public void inFlightReadRetriesInSameRegionAfterPingClose() throws Exception { // Short interval + timeout for fast detection System.setProperty("COSMOS.HTTP2_PING_INTERVAL_IN_SECONDS", "1"); System.setProperty("COSMOS.HTTP2_PING_TIMEOUT_IN_SECONDS", "2"); @@ -137,6 +146,7 @@ public void connectionClosedOnPingTimeout() throws Exception { // Lifted out of the try so the finally-block cleanup can reach it whether or not // the iptables ADD ran -- finally needs the exact -D form of whatever -A we installed. String iptablesDelete = null; + Thread iptablesRemovalThread = null; try { safeClose(this.client); @@ -158,7 +168,9 @@ public void connectionClosedOnPingTimeout() throws Exception { // Warm-up read -- establish the H2 connection and capture diagnostics so we can // (a) prove H2 was actually negotiated, (b) discover the regional endpoint host - // for IP-scoped iptables targeting on the Compute / port-443 path. + // for IP-scoped iptables targeting on the Compute / port-443 path, and + // (c) capture the initial channel-id and endpoint host to compare against + // the recovery response. CosmosItemResponse warmup = this.cosmosAsyncContainer.readItem( seedItem.getId(), new PartitionKey(seedItem.getId()), TestObject.class).block(); assertThat(warmup).isNotNull(); @@ -173,8 +185,8 @@ public void connectionClosedOnPingTimeout() throws Exception { .contains("\"isHttp2\":true"); String initialChannelId = extractParentChannelId(warmupDiag); - String regionalHost = extractEndpointHost(warmupDiag); - logger.info("Initial parentChannelId: {}, regionalHost: {}", initialChannelId, regionalHost); + String warmupEndpointHost = extractEndpointHost(warmupDiag); + logger.info("Warm-up: parentChannelId={}, endpointHost={}", initialChannelId, warmupEndpointHost); // iptables rule: port-only for thin-client (10250 is exclusive to thin-client traffic); // destination-IP + port for Compute (port 443 is shared with everything else in the JVM, @@ -186,9 +198,9 @@ public void connectionClosedOnPingTimeout() throws Exception { iptablesDelete = String.format( "%siptables -D OUTPUT -p tcp --dport %d -j DROP", SUDO, H2_PORT); } else { - String regionalIp = InetAddress.getByName(regionalHost).getHostAddress(); + String regionalIp = InetAddress.getByName(warmupEndpointHost).getHostAddress(); logger.info("Resolved {} -> {} (Compute variant uses IP-scoped DROP)", - regionalHost, regionalIp); + warmupEndpointHost, regionalIp); iptablesAdd = String.format( "%siptables -A OUTPUT -p tcp -d %s --dport %d -j DROP", SUDO, regionalIp, H2_PORT); @@ -200,42 +212,114 @@ public void connectionClosedOnPingTimeout() throws Exception { logger.info("Installing iptables DROP rule: {}", iptablesAdd); execCommand(iptablesAdd); - // Wait for PING timeout with consecutive failure threshold=2 (test override): - // Round 1: 1s interval + 2s timeout = 3s (failure #1) - // Round 2: ~1s interval + 2s timeout = 3s (failure #2 >= threshold -> close) - // Total ~6s + buffer = 10s - logger.info("Waiting 10s for consecutive PING timeouts to close the connection..."); - Thread.sleep(10_000); - - // Remove iptables rule BEFORE attempting recovery read - logger.info("Removing iptables DROP rule: {}", iptablesDelete); - execCommand(iptablesDelete); - // Cleared -- finally-block no longer needs to delete it. - iptablesDelete = null; - - // Small wait for network to stabilize - Thread.sleep(1_000); + // Schedule iptables removal in a background thread. The in-flight read below + // will block on this rule lifting; ClientRetryPolicy retries the failed + // PING-close attempt with bounded backoff (max 120 retries; see + // Configs.DEFAULT_CLIENT_ENDPOINT_FAILOVER_MAX_RETRY_COUNT), and the first + // retry attempt that lands AFTER the rule is removed succeeds on a brand + // new TCP connection to the SAME regional gateway endpoint. + // + // 20s window picked to comfortably cover: + // * 4s for Http2PingHandler to close channel-1 (interval=1s + timeout=2s + // for each of the 2 PING failures) + // * 2-3 retry attempts, each capped by netty CONNECT_TIMEOUT_MILLIS + // (~5s for thin-client per ReactorNettyClient.send) when SYN-DROPped + final String iptablesDeleteRef = iptablesDelete; + iptablesRemovalThread = new Thread(() -> { + try { + Thread.sleep(20_000); + logger.info("Background thread removing iptables DROP rule: {}", iptablesDeleteRef); + execCommand(iptablesDeleteRef); + } catch (Exception e) { + logger.error("Failed to remove iptables in background thread", e); + } + }, "iptables-removal-thread"); + iptablesRemovalThread.setDaemon(true); + iptablesRemovalThread.start(); + + // Issue the read WHILE iptables DROP is active. Timeline: + // t=0: read sent on channel-1, DROPped by iptables on the wire + // t=~4s: Http2PingHandler detects 2 consecutive PING ACK timeouts + // (interval=1s, timeout=2s, threshold=2) and closes channel-1 + // t=~4s: Http2PingCloseRewrapHandler fires the typed exception on the + // in-flight child stream; RxGatewayStoreModel stamps subStatus + // 10006 (GATEWAY_HTTP2_PING_TIMEOUT_CHANNEL_CLOSED); + // ClientRetryPolicy.shouldRetry routes via the H3 branch + // (shouldRetryOnGatewayTimeout, NO endpoint mark-down) + // t=4-20s: retry attempts open new TCP connections that SYN-DROP and + // time out after ~5s each (thin-client CONNECT_TIMEOUT_MILLIS) + // t=20s: background thread removes iptables; next retry's SYN succeeds, + // H2 negotiates, the read succeeds on channel-2 against the + // SAME regional gateway endpoint + logger.info("Issuing readItem while iptables DROP is active..."); + long startNanos = System.nanoTime(); + CosmosItemResponse response = this.cosmosAsyncContainer.readItem( + seedItem.getId(), new PartitionKey(seedItem.getId()), TestObject.class).block(); + long elapsedMs = (System.nanoTime() - startNanos) / 1_000_000; - // Recovery read -- should succeed on a NEW connection - String recoveryChannelId = readAndGetParentChannelId(); - logger.info("Recovery parentChannelId: {}", recoveryChannelId); + // Wait for the iptables removal thread to finish (it should already be done) + iptablesRemovalThread.join(5_000); + // Background thread already removed the rule; clear so the finally-block doesn't retry. + iptablesDelete = null; - logger.info("RESULT: thinClient={}, port={}, initial={}, recovery={}, DIFFERENT_CONNECTION={}", - THIN_CLIENT_ENABLED, H2_PORT, initialChannelId, recoveryChannelId, + assertThat(response).isNotNull(); + assertThat(response.getStatusCode()).isEqualTo(200); + + CosmosDiagnostics recoveryDiag = response.getDiagnostics(); + String recoveryDiagStr = recoveryDiag.toString(); + String recoveryChannelId = extractParentChannelId(recoveryDiag); + String recoveryEndpointHost = extractEndpointHost(recoveryDiag); + + logger.info("RESULT: elapsedMs={}, thinClient={}, port={}, " + + "initialChannel={}, recoveryChannel={}, " + + "warmupEndpoint={}, recoveryEndpoint={}, " + + "SAME_REGION={}, DIFFERENT_CONNECTION={}", + elapsedMs, THIN_CLIENT_ENABLED, H2_PORT, + initialChannelId, recoveryChannelId, + warmupEndpointHost, recoveryEndpointHost, + warmupEndpointHost.equals(recoveryEndpointHost), !initialChannelId.equals(recoveryChannelId)); + logger.info("Recovery diagnostics: {}", recoveryDiagStr); - // The connection MUST be different -- the old one was closed by PING timeout + // Assertion 1: channel-1 was actually closed; recovery used a new TCP connection. assertThat(recoveryChannelId) .as("After PING timeout (thinClient=%s, port=%d), the handler should have " + "closed the connection. The recovery request must use a new connection.", THIN_CLIENT_ENABLED, H2_PORT) .isNotEqualTo(initialChannelId); - logger.info("PING timeout test passed: connection {} was closed, new connection {} established", - initialChannelId, recoveryChannelId); + // Assertion 2 (the H3 invariant we're proving): recovery landed on the SAME + // regional gateway. If ClientRetryPolicy had treated the PING-close as a + // regional outage, refreshLocation + markEndpointUnavailableForRead would + // have fired and the retry would have either failed (single-region account) + // or landed on a different region (multi-region account). + assertThat(recoveryEndpointHost) + .as("ClientRetryPolicy must NOT trigger a region failover for PING-driven " + + "channel close. Recovery endpoint should match warm-up endpoint.") + .isEqualTo(warmupEndpointHost); + + // Assertion 3: the H3 retry branch was actually exercised. Without this guard, + // a hypothetical scenario where the warm-up's connection survived and the + // in-flight read never failed (or fell through some other retry path) would + // silently pass. We require evidence in diagnostics that the PING-close + // sub-status code was stamped on at least one failed attempt. + assertThat(recoveryDiagStr) + .as("Diagnostics must record sub-status 10006 (GATEWAY_HTTP2_PING_TIMEOUT_" + + "CHANNEL_CLOSED) on at least one failed attempt, proving " + + "ClientRetryPolicy's H3 branch executed.") + .containsAnyOf("10006", "GATEWAY_HTTP2_PING_TIMEOUT_CHANNEL_CLOSED", + "Http2PingTimeoutChannelClosedException"); + + logger.info("H3 verified: PING-driven close was retried in-region without endpoint mark-down"); } finally { + // Safety: if the background removal thread is still alive (didn't get to sleep + // through 20s, or the test interrupted early), interrupt it and let the + // captured -D form below handle cleanup synchronously. + if (iptablesRemovalThread != null && iptablesRemovalThread.isAlive()) { + iptablesRemovalThread.interrupt(); + } // Safety: if we installed an iptables rule and didn't manage to remove it above - // (e.g., assertion failed before we reached the eager delete), best-effort + // (e.g., assertion failed before the background thread removed it), best-effort // remove it now. The exact -D form was captured when we built -A. if (iptablesDelete != null) { try { @@ -250,16 +334,6 @@ public void connectionClosedOnPingTimeout() throws Exception { } } - private String readAndGetParentChannelId() throws JsonProcessingException { - CosmosItemResponse response = this.cosmosAsyncContainer.readItem( - seedItem.getId(), new PartitionKey(seedItem.getId()), TestObject.class).block(); - - assertThat(response).isNotNull(); - assertThat(response.getStatusCode()).isEqualTo(200); - - return extractParentChannelId(response.getDiagnostics()); - } - /** * Pulls the regional gateway hostname out of the first {@code gatewayStatisticsList[]} * entry whose {@code endpoint} URI exposes a host. Used by the Compute / port-443 @@ -288,15 +362,25 @@ private String extractEndpointHost(CosmosDiagnostics diagnostics) throws JsonPro } /** - * Mirrors {@code Http2ConnectionLifecycleTests#extractParentChannelId} -- parse - * the diagnostics JSON rather than substring-scan the toString() so a future change - * to JSON formatting can't silently break the test. + * Returns the parentChannelId of the FINAL (successful) attempt in the + * gatewayStatisticsList. We iterate from the end because the recovery + * response's diagnostics records every retry attempt: the PING-closed + * channel comes first (subStatus 10006), connection-timeout attempts in + * the middle have no parentChannelId at all (subStatus 10001), and the + * successful 200 attempt (which is the channel we want to compare against + * the warm-up channel) is last. For the warm-up response with a single + * gatewayStatistics entry, this is equivalent to picking the first. + *

    + * Parses the diagnostics JSON rather than substring-scanning toString() + * so that a future change to JSON formatting can't silently break the + * test. */ private String extractParentChannelId(CosmosDiagnostics diagnostics) throws JsonProcessingException { ObjectNode node = (ObjectNode) Utils.getSimpleObjectMapper().readTree(diagnostics.toString()); JsonNode gwStats = node.get("gatewayStatisticsList"); if (gwStats != null && gwStats.isArray()) { - for (JsonNode stat : gwStats) { + for (int i = gwStats.size() - 1; i >= 0; i--) { + JsonNode stat = gwStats.get(i); if (stat.has("parentChannelId")) { String id = stat.get("parentChannelId").asText(); if (id != null && !id.isEmpty() && !"null".equals(id)) { diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/ClientRetryPolicyHttp2PingCloseTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/ClientRetryPolicyHttp2PingCloseTest.java new file mode 100644 index 000000000000..acc4a2e834dc --- /dev/null +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/ClientRetryPolicyHttp2PingCloseTest.java @@ -0,0 +1,155 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos.implementation; + +import com.azure.cosmos.BridgeInternal; +import com.azure.cosmos.CosmosException; +import com.azure.cosmos.ThrottlingRetryOptions; +import com.azure.cosmos.implementation.http.Http2PingTimeoutChannelClosedException; +import com.azure.cosmos.implementation.perPartitionAutomaticFailover.GlobalPartitionEndpointManagerForPerPartitionAutomaticFailover; +import com.azure.cosmos.implementation.perPartitionCircuitBreaker.GlobalPartitionEndpointManagerForPerPartitionCircuitBreaker; +import com.azure.cosmos.implementation.routing.RegionalRoutingContext; +import org.mockito.Mockito; +import org.testng.annotations.Test; +import reactor.core.publisher.Mono; + +import java.net.URI; + +import static com.azure.cosmos.implementation.TestUtils.mockDiagnosticsClientContext; + +/** + * Verifies the {@link ClientRetryPolicy} H3 branch that handles channel + * closures driven by {@code Http2PingHandler} consecutive PING ACK timeouts + * (sub-status {@code GATEWAY_HTTP2_PING_TIMEOUT_CHANNEL_CLOSED} = 10006). + *

    + * Invariant under test: a PING-driven local transport failure MUST NOT cause + * {@code markEndpointUnavailableForRead} or {@code markEndpointUnavailableForWrite} + * to be invoked on {@link GlobalEndpointManager}. The remote gateway is not + * known to be unhealthy; we route via {@code shouldRetryOnGatewayTimeout} + * (bounded same-region retry for safely-retriable reads; noRetry for writes) + * instead of the GATEWAY_ENDPOINT_UNAVAILABLE path that does mark the + * endpoint down. + */ +public class ClientRetryPolicyHttp2PingCloseTest { + + private static final int ITERATIONS = 10; + + @Test(groups = "unit") + public void pingTimeoutClose_onRead_retriesInRegionWithoutEndpointMarkDown() throws Exception { + ThrottlingRetryOptions retryOptions = new ThrottlingRetryOptions(); + GlobalEndpointManager endpointManager = Mockito.mock(GlobalEndpointManager.class); + GlobalPartitionEndpointManagerForPerPartitionCircuitBreaker globalPartitionEndpointManagerForPerPartitionCircuitBreaker + = Mockito.mock(GlobalPartitionEndpointManagerForPerPartitionCircuitBreaker.class); + GlobalPartitionEndpointManagerForPerPartitionAutomaticFailover globalPartitionEndpointManagerForPerPartitionAutomaticFailover + = Mockito.mock(GlobalPartitionEndpointManagerForPerPartitionAutomaticFailover.class); + + Mockito.doReturn(new RegionalRoutingContext(new URI("http://localhost"))) + .when(endpointManager).resolveServiceEndpoint(Mockito.any(RxDocumentServiceRequest.class)); + Mockito.doReturn(Mono.empty()).when(endpointManager).refreshLocationAsync(Mockito.eq(null), Mockito.eq(false)); + Mockito.doReturn(2).when(endpointManager).getPreferredLocationCount(); + + ClientRetryPolicy clientRetryPolicy = new ClientRetryPolicy( + mockDiagnosticsClientContext(), + endpointManager, + true, + retryOptions, + globalPartitionEndpointManagerForPerPartitionCircuitBreaker, + globalPartitionEndpointManagerForPerPartitionAutomaticFailover, + false); + + CosmosException cosmosException = buildPingTimeoutCloseCosmosException(); + + RxDocumentServiceRequest dsr = RxDocumentServiceRequest.createFromName(mockDiagnosticsClientContext(), + OperationType.Read, "/dbs/db/colls/col/docs/docId", ResourceType.Document); + dsr.requestContext = new DocumentServiceRequestContext(); + clientRetryPolicy.onBeforeSendRequest(dsr); + + for (int i = 0; i < ITERATIONS; i++) { + Mono shouldRetry = clientRetryPolicy.shouldRetry(cosmosException); + + // For Read requests the H3 branch routes to shouldRetryOnGatewayTimeout's + // canPerformCrossRegionRetryOnGatewayReadTimeout=true arm, which returns + // retryAfter(endpointFailoverRetryIntervalInMs) until failoverRetryCount + // exceeds endpointFailoverMaxRetryCount (default 120). Within ITERATIONS + // we expect every call to retry. + ClientRetryPolicyTest.validateSuccess(shouldRetry, ShouldRetryValidator.builder() + .nullException() + .shouldRetry(true) + .build()); + + // The critical invariant: PING-driven close MUST NOT mark the regional + // endpoint unavailable. shouldRetryOnGatewayTimeout never calls + // markEndpointUnavailableFor*, so these counts stay at zero across + // every iteration. + Mockito.verify(endpointManager, Mockito.times(0)).markEndpointUnavailableForRead(Mockito.any()); + Mockito.verify(endpointManager, Mockito.times(0)).markEndpointUnavailableForWrite(Mockito.any()); + } + } + + @Test(groups = "unit") + public void pingTimeoutClose_onWrite_doesNotRetryAndDoesNotMarkDown() throws Exception { + ThrottlingRetryOptions retryOptions = new ThrottlingRetryOptions(); + GlobalEndpointManager endpointManager = Mockito.mock(GlobalEndpointManager.class); + GlobalPartitionEndpointManagerForPerPartitionCircuitBreaker globalPartitionEndpointManagerForPerPartitionCircuitBreaker + = Mockito.mock(GlobalPartitionEndpointManagerForPerPartitionCircuitBreaker.class); + GlobalPartitionEndpointManagerForPerPartitionAutomaticFailover globalPartitionEndpointManagerForPerPartitionAutomaticFailover + = Mockito.mock(GlobalPartitionEndpointManagerForPerPartitionAutomaticFailover.class); + + Mockito.doReturn(new RegionalRoutingContext(new URI("http://localhost"))) + .when(endpointManager).resolveServiceEndpoint(Mockito.any(RxDocumentServiceRequest.class)); + Mockito.doReturn(Mono.empty()).when(endpointManager).refreshLocationAsync(Mockito.eq(null), Mockito.eq(false)); + + ClientRetryPolicy clientRetryPolicy = new ClientRetryPolicy( + mockDiagnosticsClientContext(), + endpointManager, + true, + retryOptions, + globalPartitionEndpointManagerForPerPartitionCircuitBreaker, + globalPartitionEndpointManagerForPerPartitionAutomaticFailover, + false); + + CosmosException cosmosException = buildPingTimeoutCloseCosmosException(); + + RxDocumentServiceRequest dsr = RxDocumentServiceRequest.createFromName(mockDiagnosticsClientContext(), + OperationType.Create, "/dbs/db/colls/col/docs/docId", ResourceType.Document); + dsr.requestContext = new DocumentServiceRequestContext(); + clientRetryPolicy.onBeforeSendRequest(dsr); + + for (int i = 0; i < ITERATIONS; i++) { + Mono shouldRetry = clientRetryPolicy.shouldRetry(cosmosException); + + // For Create requests canRequestToGatewayBeSafelyRetriedOnReadTimeout + // returns false (request.isReadOnly() == false), so the H3 branch + // falls through shouldRetryOnGatewayTimeout to NO_RETRY (PPAF mock + // returns false by default). + ClientRetryPolicyTest.validateSuccess(shouldRetry, ShouldRetryValidator.builder() + .nullException() + .shouldRetry(false) + .build()); + + // Same invariant as the read case: no region mark-down on a + // local-transport PING-driven failure, regardless of write semantics. + Mockito.verify(endpointManager, Mockito.times(0)).markEndpointUnavailableForRead(Mockito.any()); + Mockito.verify(endpointManager, Mockito.times(0)).markEndpointUnavailableForWrite(Mockito.any()); + } + } + + private static CosmosException buildPingTimeoutCloseCosmosException() { + // Http2PingTimeoutChannelClosedException extends ClosedChannelException so + // WebExceptionUtility.isNetworkFailure(...) returns true (gate 1 of H3). + // BridgeInternal.createCosmosException wraps it as the inner exception so + // Utils.as(e, CosmosException.class) yields a non-null clientException + // (gate 2). Setting the sub-status code to 10006 satisfies gate 3 and + // routes execution into the H3 branch at ClientRetryPolicy.java:142. + Http2PingTimeoutChannelClosedException pingClose = + new Http2PingTimeoutChannelClosedException("ping ack timeout (test)", null); + CosmosException cosmosException = BridgeInternal.createCosmosException( + null, + HttpConstants.StatusCodes.SERVICE_UNAVAILABLE, + pingClose); + BridgeInternal.setSubStatusCode(cosmosException, + HttpConstants.SubStatusCodes.GATEWAY_HTTP2_PING_TIMEOUT_CHANNEL_CLOSED); + return cosmosException; + } +} diff --git a/sdk/cosmos/azure-cosmos/CHANGELOG.md b/sdk/cosmos/azure-cosmos/CHANGELOG.md index 9b6dea8a053d..14013b133ed3 100644 --- a/sdk/cosmos/azure-cosmos/CHANGELOG.md +++ b/sdk/cosmos/azure-cosmos/CHANGELOG.md @@ -14,7 +14,7 @@ * Fixed `UnsupportedOperationException` when using `readManyByPartitionKeys` for empty pages. - See [PR 49311](https://github.com/Azure/azure-sdk-for-java/pull/49311) #### Other Changes -* Added HTTP/2 PING keepalive (default ON) for Gateway V2 service endpoints to detect silently-broken connections. Tunable via `COSMOS.HTTP2_PING_HEALTH_ENABLED`, `COSMOS.HTTP2_PING_INTERVAL_IN_SECONDS`, `COSMOS.HTTP2_PING_TIMEOUT_IN_SECONDS`, `COSMOS.HTTP2_PING_FAILURE_THRESHOLD` (each also accepts the equivalent `COSMOS_*` env var). Hardened concurrent installation of `customHeaderCleaner` and `Http2ParentChannelExceptionHandler` on the H2 parent pipeline against benign duplicate-name races from reactor-netty `doOnConnected` callbacks. - See [PR 49095](https://github.com/Azure/azure-sdk-for-java/pull/49095) +* Added HTTP/2 PING keepalive (default ON) for Gateway V2 service endpoints to detect silently-broken connections. Tunable via `COSMOS.HTTP2_PING_HEALTH_ENABLED`, `COSMOS.HTTP2_PING_INTERVAL_IN_SECONDS`, `COSMOS.HTTP2_PING_TIMEOUT_IN_SECONDS`, `COSMOS.HTTP2_PING_FAILURE_THRESHOLD` (each also accepts the equivalent `COSMOS_*` env var). Hardened concurrent installation of `customHeaderCleaner` and `Http2ParentChannelExceptionHandler` on the H2 parent pipeline against benign duplicate-name races from reactor-netty `doOnConnected` callbacks. Channel closes triggered by consecutive PING-ACK timeouts are now tagged with the `Http2PingTimeoutChannelClosedException` marker and the `GATEWAY_HTTP2_PING_TIMEOUT_CHANNEL_CLOSED` (21013) sub-status code, so `ClientRetryPolicy` retries them in-region (read) or fails fast (write) **without** marking the regional gateway endpoint unavailable — preventing spurious region failovers when only a single connection went stale. - See [PR 49095](https://github.com/Azure/azure-sdk-for-java/pull/49095) * Replaced per-client `Schedulers.newSingle()` schedulers in `GlobalEndpointManager` and `GlobalPartitionEndpointManagerForPerPartitionCircuitBreaker` with shared `BoundedElastic` schedulers in `CosmosSchedulers` to prevent thread count from scaling linearly with client/tenant count. - See [PR 49062](https://github.com/Azure/azure-sdk-for-java/pull/49062) * Promoted the `ReadConsistencyStrategy` and `Http2ConnectionConfig` related `@Beta` APIs to GA. - See [PR 49345](https://github.com/Azure/azure-sdk-for-java/pull/49345) * Fixed a sporadic `NullPointerException` in `JsonSerializable.getWithMapping` triggered by concurrent first-time calls to `DatabaseAccount.getConsistencyPolicy()` and its sibling lazy getters (`getReplicationPolicy`, `getSystemReplicationPolicy`, `getQueryEngineConfiguration`). The fix makes `JsonSerializable.propertyBag` `final`, closing an unsafe-publication race in the lazy-initialisation pattern. - See [Issue 49256](https://github.com/Azure/azure-sdk-for-java/issues/49256) and [PR #49258](https://github.com/Azure/azure-sdk-for-java/pull/49258) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ClientRetryPolicy.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ClientRetryPolicy.java index 4a5b2f409df7..c1a9cba71f4c 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ClientRetryPolicy.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ClientRetryPolicy.java @@ -139,6 +139,18 @@ public Mono shouldRetry(Exception e) { } else { return this.shouldNotRetryOnEndpointFailureAsync(this.isReadRequest, false, false); } + } else if (clientException != null && + Exceptions.isSubStatusCode(clientException, HttpConstants.SubStatusCodes.GATEWAY_HTTP2_PING_TIMEOUT_CHANNEL_CLOSED)) { + // HTTP/2 PING keepalive closed the channel after consecutive ACK timeouts. + // The remote gateway is NOT known to be unhealthy -- this is typically a local + // transport failure (NAT / LB idle reap of an otherwise-healthy connection). + // Reuse the gateway-timeout path: bounded read retry on the SAME endpoint (no + // markEndpointUnavailableFor* call inside shouldRetryOnGatewayTimeout), or + // noRetry for writes. This deliberately suppresses the cross-region failover + // that GATEWAY_ENDPOINT_UNAVAILABLE would trigger via refreshLocation + + // markEndpointUnavailableForRead/Write. + logger.info("HTTP/2 PING-driven connection close. Retrying without endpoint mark-down. ", e); + return shouldRetryOnGatewayTimeout(clientException); } else if (clientException != null && WebExceptionUtility.isReadTimeoutException(clientException) && Exceptions.isSubStatusCode(clientException, HttpConstants.SubStatusCodes.GATEWAY_ENDPOINT_READ_TIMEOUT)) { diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/HttpConstants.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/HttpConstants.java index 570bdb010f25..1ce23ae53459 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/HttpConstants.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/HttpConstants.java @@ -458,6 +458,13 @@ public static class SubStatusCodes { // Client generated request rate too large exception public static final int THROUGHPUT_CONTROL_BULK_REQUEST_RATE_TOO_LARGE = 10005; + // Client generated gateway network error: HTTP/2 PING keepalive detected a dead + // connection (consecutive PING ACK timeouts crossed the failure threshold) and + // closed the channel. This is a LOCAL transport failure, NOT a regional outage -- + // ClientRetryPolicy uses this to skip the endpoint mark-down / cross-region + // failover that GATEWAY_ENDPOINT_UNAVAILABLE would trigger. + public static final int GATEWAY_HTTP2_PING_TIMEOUT_CHANNEL_CLOSED = 10006; + public static final int USER_REQUEST_RATE_TOO_LARGE = 3200; //SDK Codes(Client) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxGatewayStoreModel.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxGatewayStoreModel.java index a3cd0c1f0dcc..8d9b324d92fb 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxGatewayStoreModel.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxGatewayStoreModel.java @@ -750,7 +750,15 @@ private Mono toDocumentServiceResponse(Mono + * This is a strict subset of {@link #isNetworkFailure(Exception)} (the typed + * exception extends {@link java.nio.channels.ClosedChannelException}); callers + * that branch on the PING-close case should check this BEFORE general-network- + * failure handling so the more specific subStatusCode is stamped. + */ + public static boolean isHttp2PingTimeoutClose(Exception ex) { + Exception iterator = ex; + + while (iterator != null) { + if (iterator instanceof Http2PingTimeoutChannelClosedException) { + return true; + } + + Throwable t = iterator.getCause(); + iterator = Utils.as(t, Exception.class); + } + + return false; + } } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingCloseRewrapHandler.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingCloseRewrapHandler.java new file mode 100644 index 000000000000..bb4a25f9d43b --- /dev/null +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingCloseRewrapHandler.java @@ -0,0 +1,55 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +package com.azure.cosmos.implementation.http; + +import io.netty.channel.Channel; +import io.netty.channel.ChannelHandler; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInboundHandlerAdapter; + +/** + * Per-request HTTP/2 child-stream handler that translates a parent-TCP-channel close + * driven by {@link Http2PingHandler} into a typed {@link Http2PingTimeoutChannelClosedException}. + *

    + * Installed at the head of each H2 child-stream pipeline by + * {@code ReactorNettyClient}'s {@code .doOnRequest(...)} hook. When the parent (TCP) + * channel is closed by {@link Http2PingHandler} after consecutive PING-ACK timeouts, + * the H2 multiplex codec propagates {@code channelInactive} to every child stream. + * This handler fires first (head-of-pipeline), inspects the parent channel's + * {@link Http2PingHandler#PING_TIMEOUT_CLOSED} attribute, and on match fires + * {@code exceptionCaught} with the typed exception before the default close + * path reaches reactor-netty's {@code HttpClientOperations}. That makes the in-flight + * request's response {@code Mono} fail with the typed exception (which the rest of the + * stack maps to {@code GATEWAY_HTTP2_PING_TIMEOUT_CHANNEL_CLOSED}) instead of a bare + * {@code PrematureCloseException}, which would otherwise trigger region mark-down in + * {@code ClientRetryPolicy}. + *

    + * The handler is stateless and marked {@link ChannelHandler.Sharable}, so a single + * JVM-wide {@link #INSTANCE} is reused across all H2 child streams. + *

    + * For non-H2 channels (parent is {@code null}) this handler is never installed; the + * install site in {@code ReactorNettyClient} gates on {@code ch.parent() != null}. + */ +@ChannelHandler.Sharable +final class Http2PingCloseRewrapHandler extends ChannelInboundHandlerAdapter { + + static final String HANDLER_NAME = "cosmos.http2PingCloseRewrap"; + + static final Http2PingCloseRewrapHandler INSTANCE = new Http2PingCloseRewrapHandler(); + + private Http2PingCloseRewrapHandler() {} + + @Override + public void channelInactive(ChannelHandlerContext ctx) throws Exception { + Channel parent = ctx.channel().parent(); + if (parent != null && Boolean.TRUE.equals(parent.attr(Http2PingHandler.PING_TIMEOUT_CLOSED).get())) { + // Fire BEFORE delegating to super.channelInactive so reactor-netty's + // HttpClientOperations.exceptionCaught fails the response Mono with the + // typed exception, beating its own onInboundClose(PrematureCloseException). + ctx.fireExceptionCaught(new Http2PingTimeoutChannelClosedException( + "HTTP/2 connection closed by PING keepalive after consecutive ACK timeouts.", + null)); + } + super.channelInactive(ctx); + } +} diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java index 02d9f96fd81e..98f5f1d7ea1a 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java @@ -10,6 +10,7 @@ import io.netty.channel.ChannelHandlerContext; import io.netty.handler.codec.http2.DefaultHttp2PingFrame; import io.netty.handler.codec.http2.Http2PingFrame; +import io.netty.util.AttributeKey; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,6 +44,24 @@ public class Http2PingHandler extends ChannelDuplexHandler { private static final String HANDLER_NAME = "cosmos.http2PingHandler"; + /** + * Marker attribute set on the parent (TCP) channel immediately before {@link + * ChannelHandlerContext#close()} when this handler closes the channel due to + * consecutive PING ACK timeouts (or PING send failures). Downstream code can + * use this to discriminate a PING-driven close from other channel closures. + *

    + * Consumed by {@link Http2PingCloseRewrapHandler}, a {@code @Sharable} handler + * installed at the head of each H2 child-stream pipeline by {@code + * ReactorNettyClient.doOnRequest(...)}. When the parent channel closes with this + * attribute set, the rewrap handler fires {@code exceptionCaught} with a typed + * {@link Http2PingTimeoutChannelClosedException} so the in-flight request's + * response {@code Mono} fails with the typed exception (instead of a bare + * {@code PrematureCloseException}); upstream {@code ClientRetryPolicy} then + * suppresses region mark-down for the corresponding subStatusCode. + */ + public static final AttributeKey PING_TIMEOUT_CLOSED = + AttributeKey.valueOf("cosmos.http2PingTimeoutClosed"); + private final long pingIntervalNanos; private final long pingTimeoutNanos; private final int failureThreshold; @@ -156,6 +175,10 @@ private void maybeSendPing(ChannelHandlerContext ctx) { logger.info("PING ACK not received for {} consecutive attempts on channel {} -- closing connection", consecutiveFailures, ctx.channel()); cancelPingTask(); + // Mark BEFORE close so a per-request captor in ReactorNettyClient.onErrorMap + // observes the attribute when the in-flight stream Mono fails. Same event + // loop, no cross-thread visibility concerns. + ctx.channel().attr(PING_TIMEOUT_CLOSED).set(Boolean.TRUE); ctx.close(); } else { logger.debug("PING ACK timeout on channel {} (attempt {}/{}) -- will retry", @@ -188,6 +211,8 @@ private void maybeSendPing(ChannelHandlerContext ctx) { logger.info("PING send failed for {} consecutive attempts on channel {} -- closing connection", consecutiveFailures, ctx.channel()); cancelPingTask(); + // See note at the ACK-timeout close site above. + ctx.channel().attr(PING_TIMEOUT_CLOSED).set(Boolean.TRUE); ctx.close(); } else { logger.debug("PING #{} send failed on channel {} (attempt {}/{}): {}", diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingTimeoutChannelClosedException.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingTimeoutChannelClosedException.java new file mode 100644 index 000000000000..fc28a056f3dc --- /dev/null +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingTimeoutChannelClosedException.java @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +package com.azure.cosmos.implementation.http; + +import java.nio.channels.ClosedChannelException; + +/** + * Marker exception raised when {@link Http2PingHandler} closes an HTTP/2 parent + * channel after consecutive PING ACK timeouts (or PING send failures) crossed + * the configured threshold. + *

    + * This is a local transport failure -- typically caused by NAT / load + * balancer idle reaping of an otherwise-healthy connection. The remote + * service (e.g. Cosmos DB Standard Gateway or ThinClient proxy) is NOT + * known to be unhealthy. Code paths that classify network failures (e.g. + * {@code com.azure.cosmos.implementation.ClientRetryPolicy}) should use this + * marker to suppress region mark-down / cross-region failover and instead + * retry against the same regional endpoint with a fresh connection. + *

    + * Extends {@link ClosedChannelException} so existing classifiers (notably + * {@code WebExceptionUtility.isNetworkFailure}) continue to recognize it as a + * network failure; discrimination from a generic closed channel is via {@code + * instanceof} (see {@code WebExceptionUtility.isHttp2PingTimeoutClose}). + *

    + * Note: {@link ClosedChannelException} only exposes a no-arg constructor, so + * the message is carried in a private field and surfaced via {@link + * #getMessage()}. + */ +public final class Http2PingTimeoutChannelClosedException extends ClosedChannelException { + + private static final long serialVersionUID = 1L; + + private final String message; + + public Http2PingTimeoutChannelClosedException(String message, Throwable cause) { + super(); + this.message = message; + if (cause != null) { + initCause(cause); + } + } + + @Override + public String getMessage() { + return message; + } +} diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java index 5ccf9f4de02b..6a9d5d07ddc9 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java @@ -223,7 +223,34 @@ private void configureChannelPipelineHandlers() { // Duplicate handler name is the only possible cause. } } - })); + })) + .doOnRequest((req, conn) -> { + // Install a @Sharable head-of-pipeline rewrap handler on each H2 + // child-stream pipeline. When Http2PingHandler closes the parent + // (TCP) channel after consecutive PING-ACK timeouts, the H2 multiplex + // codec propagates channelInactive to every child stream; the rewrap + // handler intercepts that and fires exceptionCaught with a typed + // Http2PingTimeoutChannelClosedException so reactor-netty's + // HttpClientOperations fails the response Mono with the typed + // exception (instead of bare PrematureCloseException). The rest of the + // stack maps that to GATEWAY_HTTP2_PING_TIMEOUT_CHANNEL_CLOSED so + // ClientRetryPolicy can suppress region mark-down. + // + // Gate on ch.parent() != null so this only runs on H2 child streams + // (H1.1 connections have null parent and never need the rewrap). + Channel ch = conn.channel(); + if (ch.parent() != null && ch.pipeline().get(Http2PingCloseRewrapHandler.HANDLER_NAME) == null) { + try { + ch.pipeline().addFirst( + Http2PingCloseRewrapHandler.HANDLER_NAME, + Http2PingCloseRewrapHandler.INSTANCE); + } catch (IllegalArgumentException ignored) { + // TOCTOU race: between the get()==null check above and addFirst(), + // a concurrent doOnRequest may have installed the handler. + // Duplicate handler name is the only possible cause. + } + } + }); } } @@ -292,7 +319,8 @@ public Mono send(final HttpRequest request, Duration responseTimeo * @param restRequest the Rest request contains the body to be sent * @return a delegate upon invocation sets the request body in reactor-netty outbound object */ - private static BiFunction> bodySendDelegate(final HttpRequest restRequest) { + private static BiFunction> bodySendDelegate( + final HttpRequest restRequest) { return (reactorNettyRequest, reactorNettyOutbound) -> { for (HttpHeader header : restRequest.headers()) { reactorNettyRequest.header(header.name(), header.value()); From 0de492144f0189b6c62b48ec130ef1e710152456 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Fri, 5 Jun 2026 15:19:51 -0400 Subject: [PATCH 59/62] H3: align doc/comment/CHANGELOG wording with PING-send-failure close path The H3 close path fires on EITHER consecutive PING-ACK timeouts OR consecutive PING-send failures (the latter was added later for stalled H2 codec/flow-control). Several comments, the typed exception message, the Javadoc of Http2PingCloseRewrapHandler, the install-site comment in ReactorNettyClient, and the CHANGELOG entry still described only the ACK-timeout half. Update all six sites to mention both paths. Also corrects the stale 'per-request captor in ReactorNettyClient.onErrorMap' reference in Http2PingHandler -- the actual consumer of PING_TIMEOUT_CLOSED is Http2PingCloseRewrapHandler.channelInactive on each H2 child stream. Pure documentation cleanup: no behavioral change. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- sdk/cosmos/azure-cosmos/CHANGELOG.md | 2 +- .../implementation/http/Http2PingCloseRewrapHandler.java | 7 ++++--- .../cosmos/implementation/http/Http2PingHandler.java | 8 ++++---- .../cosmos/implementation/http/ReactorNettyClient.java | 7 ++++--- 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/sdk/cosmos/azure-cosmos/CHANGELOG.md b/sdk/cosmos/azure-cosmos/CHANGELOG.md index 14013b133ed3..6074cf16db1f 100644 --- a/sdk/cosmos/azure-cosmos/CHANGELOG.md +++ b/sdk/cosmos/azure-cosmos/CHANGELOG.md @@ -14,7 +14,7 @@ * Fixed `UnsupportedOperationException` when using `readManyByPartitionKeys` for empty pages. - See [PR 49311](https://github.com/Azure/azure-sdk-for-java/pull/49311) #### Other Changes -* Added HTTP/2 PING keepalive (default ON) for Gateway V2 service endpoints to detect silently-broken connections. Tunable via `COSMOS.HTTP2_PING_HEALTH_ENABLED`, `COSMOS.HTTP2_PING_INTERVAL_IN_SECONDS`, `COSMOS.HTTP2_PING_TIMEOUT_IN_SECONDS`, `COSMOS.HTTP2_PING_FAILURE_THRESHOLD` (each also accepts the equivalent `COSMOS_*` env var). Hardened concurrent installation of `customHeaderCleaner` and `Http2ParentChannelExceptionHandler` on the H2 parent pipeline against benign duplicate-name races from reactor-netty `doOnConnected` callbacks. Channel closes triggered by consecutive PING-ACK timeouts are now tagged with the `Http2PingTimeoutChannelClosedException` marker and the `GATEWAY_HTTP2_PING_TIMEOUT_CHANNEL_CLOSED` (21013) sub-status code, so `ClientRetryPolicy` retries them in-region (read) or fails fast (write) **without** marking the regional gateway endpoint unavailable — preventing spurious region failovers when only a single connection went stale. - See [PR 49095](https://github.com/Azure/azure-sdk-for-java/pull/49095) +* Added HTTP/2 PING keepalive (default ON) for Gateway V2 service endpoints to detect silently-broken connections. Tunable via `COSMOS.HTTP2_PING_HEALTH_ENABLED`, `COSMOS.HTTP2_PING_INTERVAL_IN_SECONDS`, `COSMOS.HTTP2_PING_TIMEOUT_IN_SECONDS`, `COSMOS.HTTP2_PING_FAILURE_THRESHOLD` (each also accepts the equivalent `COSMOS_*` env var). Hardened concurrent installation of `customHeaderCleaner` and `Http2ParentChannelExceptionHandler` on the H2 parent pipeline against benign duplicate-name races from reactor-netty `doOnConnected` callbacks. Channel closes triggered by consecutive PING-ACK timeouts or consecutive PING-send failures are now tagged with the `Http2PingTimeoutChannelClosedException` marker and the `GATEWAY_HTTP2_PING_TIMEOUT_CHANNEL_CLOSED` (10006) sub-status code, so `ClientRetryPolicy` retries them in-region (read) or fails fast (write) **without** marking the regional gateway endpoint unavailable — preventing spurious region failovers when only a single connection went stale. - See [PR 49095](https://github.com/Azure/azure-sdk-for-java/pull/49095) * Replaced per-client `Schedulers.newSingle()` schedulers in `GlobalEndpointManager` and `GlobalPartitionEndpointManagerForPerPartitionCircuitBreaker` with shared `BoundedElastic` schedulers in `CosmosSchedulers` to prevent thread count from scaling linearly with client/tenant count. - See [PR 49062](https://github.com/Azure/azure-sdk-for-java/pull/49062) * Promoted the `ReadConsistencyStrategy` and `Http2ConnectionConfig` related `@Beta` APIs to GA. - See [PR 49345](https://github.com/Azure/azure-sdk-for-java/pull/49345) * Fixed a sporadic `NullPointerException` in `JsonSerializable.getWithMapping` triggered by concurrent first-time calls to `DatabaseAccount.getConsistencyPolicy()` and its sibling lazy getters (`getReplicationPolicy`, `getSystemReplicationPolicy`, `getQueryEngineConfiguration`). The fix makes `JsonSerializable.propertyBag` `final`, closing an unsafe-publication race in the lazy-initialisation pattern. - See [Issue 49256](https://github.com/Azure/azure-sdk-for-java/issues/49256) and [PR #49258](https://github.com/Azure/azure-sdk-for-java/pull/49258) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingCloseRewrapHandler.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingCloseRewrapHandler.java index bb4a25f9d43b..02f6cd67dbcf 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingCloseRewrapHandler.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingCloseRewrapHandler.java @@ -13,8 +13,9 @@ *

    * Installed at the head of each H2 child-stream pipeline by * {@code ReactorNettyClient}'s {@code .doOnRequest(...)} hook. When the parent (TCP) - * channel is closed by {@link Http2PingHandler} after consecutive PING-ACK timeouts, - * the H2 multiplex codec propagates {@code channelInactive} to every child stream. + * channel is closed by {@link Http2PingHandler} after consecutive PING-ACK timeouts + * or consecutive PING-send failures, the H2 multiplex codec propagates + * {@code channelInactive} to every child stream. * This handler fires first (head-of-pipeline), inspects the parent channel's * {@link Http2PingHandler#PING_TIMEOUT_CLOSED} attribute, and on match fires * {@code exceptionCaught} with the typed exception before the default close @@ -47,7 +48,7 @@ public void channelInactive(ChannelHandlerContext ctx) throws Exception { // HttpClientOperations.exceptionCaught fails the response Mono with the // typed exception, beating its own onInboundClose(PrematureCloseException). ctx.fireExceptionCaught(new Http2PingTimeoutChannelClosedException( - "HTTP/2 connection closed by PING keepalive after consecutive ACK timeouts.", + "HTTP/2 connection closed by PING keepalive after consecutive PING-ACK timeouts or PING-send failures.", null)); } super.channelInactive(ctx); diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java index 98f5f1d7ea1a..82782d14c71b 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2PingHandler.java @@ -175,9 +175,9 @@ private void maybeSendPing(ChannelHandlerContext ctx) { logger.info("PING ACK not received for {} consecutive attempts on channel {} -- closing connection", consecutiveFailures, ctx.channel()); cancelPingTask(); - // Mark BEFORE close so a per-request captor in ReactorNettyClient.onErrorMap - // observes the attribute when the in-flight stream Mono fails. Same event - // loop, no cross-thread visibility concerns. + // Mark BEFORE close so Http2PingCloseRewrapHandler.channelInactive on each + // H2 child stream observes the attribute when the multiplex codec fires + // channelInactive. Same event loop, no cross-thread visibility concerns. ctx.channel().attr(PING_TIMEOUT_CLOSED).set(Boolean.TRUE); ctx.close(); } else { @@ -211,7 +211,7 @@ private void maybeSendPing(ChannelHandlerContext ctx) { logger.info("PING send failed for {} consecutive attempts on channel {} -- closing connection", consecutiveFailures, ctx.channel()); cancelPingTask(); - // See note at the ACK-timeout close site above. + // Same mark-before-close rationale as the ACK-timeout close site above. ctx.channel().attr(PING_TIMEOUT_CLOSED).set(Boolean.TRUE); ctx.close(); } else { diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java index 6a9d5d07ddc9..e95989c5aa3c 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java @@ -227,9 +227,10 @@ private void configureChannelPipelineHandlers() { .doOnRequest((req, conn) -> { // Install a @Sharable head-of-pipeline rewrap handler on each H2 // child-stream pipeline. When Http2PingHandler closes the parent - // (TCP) channel after consecutive PING-ACK timeouts, the H2 multiplex - // codec propagates channelInactive to every child stream; the rewrap - // handler intercepts that and fires exceptionCaught with a typed + // (TCP) channel after consecutive PING-ACK timeouts or PING-send + // failures, the H2 multiplex codec propagates channelInactive to + // every child stream; the rewrap handler intercepts that and fires + // exceptionCaught with a typed // Http2PingTimeoutChannelClosedException so reactor-netty's // HttpClientOperations fails the response Mono with the typed // exception (instead of bare PrematureCloseException). The rest of the From 9e4b1e704991eeb69a9a2ccd54ba57fae4228dcf Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Fri, 5 Jun 2026 15:41:38 -0400 Subject: [PATCH 60/62] test(cosmos): account for Http2PingHealth feature flag in user-agent helpers The HTTP/2 PING keepalive change added a new Http2PingHealth bit (1 << 6) to UserAgentFeatureFlags. When both H2 and PING are enabled (the default), the runtime emits |F50 instead of the previous |F10, breaking userAgent assertions in CosmosDiagnosticsTest and UserAgentSuffixTest. Both helpers (generateHttp2OptedInUserAgentIfRequired, validateUserAgentSuffix) now compute the hex suffix dynamically from the runtime conditions in RxDocumentClientImpl.addUserAgentSuffix and Http2PingHandler.isPingHealthEffectivelyEnabled: when H2 is enabled, the Http2 bit is set; when the PING kill-switch is on AND interval > 0, the Http2PingHealth bit is OR'd in. Integer.toHexString(featureValue).toUpperCase(Locale.ROOT) matches the runtime UserAgentContainer.setFeatureEnabledFlagsAsSuffix output (|F10 for H2 only, |F50 for H2+PING). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../java/com/azure/cosmos/CosmosDiagnosticsTest.java | 12 +++++++++++- .../java/com/azure/cosmos/UserAgentSuffixTest.java | 12 +++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosDiagnosticsTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosDiagnosticsTest.java index a51a26d2d49b..43bebbe52b52 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosDiagnosticsTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosDiagnosticsTest.java @@ -21,6 +21,7 @@ import com.azure.cosmos.implementation.RxStoreModel; import com.azure.cosmos.implementation.TestConfigurations; import com.azure.cosmos.implementation.UserAgentContainer; +import com.azure.cosmos.implementation.UserAgentFeatureFlags; import com.azure.cosmos.implementation.Utils; import com.azure.cosmos.implementation.clienttelemetry.ClientTelemetry; import com.azure.cosmos.implementation.directconnectivity.GatewayAddressCache; @@ -1978,8 +1979,17 @@ private void validateChannelAcquisitionContext(CosmosDiagnostics diagnostics, bo } private String generateHttp2OptedInUserAgentIfRequired(String userAgent) { + // Mirrors RxDocumentClientImpl.addUserAgentSuffix + UserAgentContainer.setFeatureEnabledFlagsAsSuffix: + // when HTTP/2 is enabled, the Http2 bit is set; when PING keepalive is also effectively enabled + // (kill-switch on AND positive interval), the Http2PingHealth bit is OR'd in. + // Tests here do not override Http2ConnectionConfig.setEnabled(...) so the per-client override branch + // in addUserAgentSuffix is a no-op for this helper. if (Configs.isHttp2Enabled()) { - userAgent = userAgent + "|F10"; + int featureValue = UserAgentFeatureFlags.Http2.getValue(); + if (Configs.isHttp2PingHealthEnabled() && Configs.getHttp2PingIntervalInSeconds() > 0) { + featureValue |= UserAgentFeatureFlags.Http2PingHealth.getValue(); + } + userAgent = userAgent + "|F" + Integer.toHexString(featureValue).toUpperCase(Locale.ROOT); } return userAgent; diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/UserAgentSuffixTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/UserAgentSuffixTest.java index 116eb5b72e0e..ceb5a9044682 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/UserAgentSuffixTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/UserAgentSuffixTest.java @@ -7,6 +7,7 @@ package com.azure.cosmos; import com.azure.cosmos.implementation.Configs; +import com.azure.cosmos.implementation.UserAgentFeatureFlags; import com.azure.cosmos.models.CosmosContainerResponse; import com.azure.cosmos.rx.TestSuiteBase; import org.testng.ITestContext; @@ -15,6 +16,8 @@ import org.testng.annotations.Factory; import org.testng.annotations.Test; +import java.util.Locale; + import static org.assertj.core.api.Assertions.assertThat; public class UserAgentSuffixTest extends TestSuiteBase { @@ -115,8 +118,15 @@ public void userAgentSuffixWithWhitespaceAndAsciiSpecialChars() { private void validateUserAgentSuffix(String actualUserAgent, String expectedUserAgentSuffix) { + // Mirrors RxDocumentClientImpl.addUserAgentSuffix + UserAgentContainer.setFeatureEnabledFlagsAsSuffix: + // when HTTP/2 is enabled, the Http2 bit is set; when PING keepalive is also effectively enabled + // (kill-switch on AND positive interval), the Http2PingHealth bit is OR'd in. if (Configs.isHttp2Enabled()) { - expectedUserAgentSuffix = expectedUserAgentSuffix + "|F10"; + int featureValue = UserAgentFeatureFlags.Http2.getValue(); + if (Configs.isHttp2PingHealthEnabled() && Configs.getHttp2PingIntervalInSeconds() > 0) { + featureValue |= UserAgentFeatureFlags.Http2PingHealth.getValue(); + } + expectedUserAgentSuffix = expectedUserAgentSuffix + "|F" + Integer.toHexString(featureValue).toUpperCase(Locale.ROOT); } assertThat(actualUserAgent).endsWith(expectedUserAgentSuffix); From 3d10f06fe4a8c421115db08662d37a2eaac44f87 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Fri, 5 Jun 2026 15:53:04 -0400 Subject: [PATCH 61/62] fix(cosmos): address xinlian12 PR review on retry comment + PING getter robustness Addresses two of xinlian12's latest review comments on PR #49095: 1. ClientRetryPolicy: clarify that the gateway-timeout retry path may cycle through preferred locations on multi-region accounts (via routeToLocation(failoverRetryCount, true) inside shouldRetryOnGatewayTimeout), not always stay on the same endpoint. The previous wording was accurate only for single-preferred-region accounts. The key invariant (no markEndpointUnavailableFor* call) remains correctly described. 2. Configs: defend the three HTTP/2 PING getters (getHttp2PingIntervalInSeconds, getHttp2PingTimeoutInSeconds, getHttp2PingFailureThreshold) against NumberFormatException from malformed user input. These getters execute inside the doOnConnected lambda in ReactorNettyClient, so a bad value like COSMOS_HTTP2_PING_INTERVAL_IN_SECONDS=1s would fail every new H2 connection. Mirrors the defensive pattern already used by getConnectionAcquireTimeout / getThinClientConnectionTimeoutInMs: try/catch with WARN log and fallback to default. Extracted to a private parseIntConfigOrDefault helper to avoid 3x duplication. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../implementation/ClientRetryPolicy.java | 14 +++-- .../azure/cosmos/implementation/Configs.java | 59 ++++++++++++++----- 2 files changed, 53 insertions(+), 20 deletions(-) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ClientRetryPolicy.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ClientRetryPolicy.java index c1a9cba71f4c..0fc46f1ab600 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ClientRetryPolicy.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ClientRetryPolicy.java @@ -144,11 +144,15 @@ public Mono shouldRetry(Exception e) { // HTTP/2 PING keepalive closed the channel after consecutive ACK timeouts. // The remote gateway is NOT known to be unhealthy -- this is typically a local // transport failure (NAT / LB idle reap of an otherwise-healthy connection). - // Reuse the gateway-timeout path: bounded read retry on the SAME endpoint (no - // markEndpointUnavailableFor* call inside shouldRetryOnGatewayTimeout), or - // noRetry for writes. This deliberately suppresses the cross-region failover - // that GATEWAY_ENDPOINT_UNAVAILABLE would trigger via refreshLocation + - // markEndpointUnavailableForRead/Write. + // Reuse the gateway-timeout path: bounded read retry that may cycle through + // preferred locations (shouldRetryOnGatewayTimeout bumps failoverRetryCount and + // routes via routeToLocation(retryCount, true) -- consistent with the + // GATEWAY_ENDPOINT_READ_TIMEOUT branch below), or noRetry for writes. With a + // single preferred region the retry stays on the same endpoint; with multiple + // preferred regions the next attempt may land on a different one. Critically, + // no markEndpointUnavailableFor* call is made inside shouldRetryOnGatewayTimeout, + // so this avoids the cross-region failover cascade that GATEWAY_ENDPOINT_UNAVAILABLE + // would trigger via refreshLocation + markEndpointUnavailableForRead/Write. logger.info("HTTP/2 PING-driven connection close. Retrying without endpoint mark-down. ", e); return shouldRetryOnGatewayTimeout(clientException); } else if (clientException != null && diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java index ec0ad1570225..c1bab22d8cad 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java @@ -780,30 +780,59 @@ public static boolean isHttp2PingHealthEnabled() { } public static int getHttp2PingIntervalInSeconds() { - String configValue = System.getProperty( + return parseIntConfigOrDefault( HTTP2_PING_INTERVAL_IN_SECONDS, - firstNonNull( - emptyToNull(System.getenv().get(HTTP2_PING_INTERVAL_IN_SECONDS_VARIABLE)), - String.valueOf(DEFAULT_HTTP2_PING_INTERVAL_IN_SECONDS))); - return Integer.parseInt(configValue); + HTTP2_PING_INTERVAL_IN_SECONDS_VARIABLE, + DEFAULT_HTTP2_PING_INTERVAL_IN_SECONDS); } public static int getHttp2PingTimeoutInSeconds() { - String configValue = System.getProperty( + return parseIntConfigOrDefault( HTTP2_PING_TIMEOUT_IN_SECONDS, - firstNonNull( - emptyToNull(System.getenv().get(HTTP2_PING_TIMEOUT_IN_SECONDS_VARIABLE)), - String.valueOf(DEFAULT_HTTP2_PING_TIMEOUT_IN_SECONDS))); - return Integer.parseInt(configValue); + HTTP2_PING_TIMEOUT_IN_SECONDS_VARIABLE, + DEFAULT_HTTP2_PING_TIMEOUT_IN_SECONDS); } public static int getHttp2PingFailureThreshold() { - String configValue = System.getProperty( + return parseIntConfigOrDefault( HTTP2_PING_FAILURE_THRESHOLD, - firstNonNull( - emptyToNull(System.getenv().get(HTTP2_PING_FAILURE_THRESHOLD_VARIABLE)), - String.valueOf(DEFAULT_HTTP2_PING_FAILURE_THRESHOLD))); - return Integer.parseInt(configValue); + HTTP2_PING_FAILURE_THRESHOLD_VARIABLE, + DEFAULT_HTTP2_PING_FAILURE_THRESHOLD); + } + + /** + * Reads an int system property first, then the env variable, falling back to the supplied default + * on either absence or a non-numeric value. Logs WARN on malformed input so an operator typo in a + * value like {@code COSMOS_HTTP2_PING_INTERVAL_IN_SECONDS=1s} cannot throw {@link NumberFormatException} + * from inside a per-connection reactor-netty {@code doOnConnected} callback and break the channel. + */ + private static int parseIntConfigOrDefault(String systemPropertyKey, String envVariableKey, int defaultValue) { + String valueFromSystemProperty = System.getProperty(systemPropertyKey); + if (valueFromSystemProperty != null && !valueFromSystemProperty.isEmpty()) { + try { + return Integer.parseInt(valueFromSystemProperty); + } catch (NumberFormatException e) { + logger.warn( + "Invalid non-numeric value '{}' for system property {}. Falling back to environment variable or default.", + valueFromSystemProperty, + systemPropertyKey); + } + } + + String valueFromEnvVariable = System.getenv(envVariableKey); + if (valueFromEnvVariable != null && !valueFromEnvVariable.isEmpty()) { + try { + return Integer.parseInt(valueFromEnvVariable); + } catch (NumberFormatException e) { + logger.warn( + "Invalid non-numeric value '{}' for environment variable {}. Falling back to default: {}.", + valueFromEnvVariable, + envVariableKey, + defaultValue); + } + } + + return defaultValue; } public static Integer getPendingAcquireMaxCount() { From 5020ee16849d07229491ea029c6602a85683b74b Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Fri, 5 Jun 2026 16:35:25 -0400 Subject: [PATCH 62/62] Update CHANGELOG.md --- sdk/cosmos/azure-cosmos/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/cosmos/azure-cosmos/CHANGELOG.md b/sdk/cosmos/azure-cosmos/CHANGELOG.md index 6074cf16db1f..56bc01bfba61 100644 --- a/sdk/cosmos/azure-cosmos/CHANGELOG.md +++ b/sdk/cosmos/azure-cosmos/CHANGELOG.md @@ -14,7 +14,7 @@ * Fixed `UnsupportedOperationException` when using `readManyByPartitionKeys` for empty pages. - See [PR 49311](https://github.com/Azure/azure-sdk-for-java/pull/49311) #### Other Changes -* Added HTTP/2 PING keepalive (default ON) for Gateway V2 service endpoints to detect silently-broken connections. Tunable via `COSMOS.HTTP2_PING_HEALTH_ENABLED`, `COSMOS.HTTP2_PING_INTERVAL_IN_SECONDS`, `COSMOS.HTTP2_PING_TIMEOUT_IN_SECONDS`, `COSMOS.HTTP2_PING_FAILURE_THRESHOLD` (each also accepts the equivalent `COSMOS_*` env var). Hardened concurrent installation of `customHeaderCleaner` and `Http2ParentChannelExceptionHandler` on the H2 parent pipeline against benign duplicate-name races from reactor-netty `doOnConnected` callbacks. Channel closes triggered by consecutive PING-ACK timeouts or consecutive PING-send failures are now tagged with the `Http2PingTimeoutChannelClosedException` marker and the `GATEWAY_HTTP2_PING_TIMEOUT_CHANNEL_CLOSED` (10006) sub-status code, so `ClientRetryPolicy` retries them in-region (read) or fails fast (write) **without** marking the regional gateway endpoint unavailable — preventing spurious region failovers when only a single connection went stale. - See [PR 49095](https://github.com/Azure/azure-sdk-for-java/pull/49095) +* Added HTTP/2 PING keepalive (default ON) for Gateway service endpoints to detect silently-broken connections. - See [PR 49095](https://github.com/Azure/azure-sdk-for-java/pull/49095) * Replaced per-client `Schedulers.newSingle()` schedulers in `GlobalEndpointManager` and `GlobalPartitionEndpointManagerForPerPartitionCircuitBreaker` with shared `BoundedElastic` schedulers in `CosmosSchedulers` to prevent thread count from scaling linearly with client/tenant count. - See [PR 49062](https://github.com/Azure/azure-sdk-for-java/pull/49062) * Promoted the `ReadConsistencyStrategy` and `Http2ConnectionConfig` related `@Beta` APIs to GA. - See [PR 49345](https://github.com/Azure/azure-sdk-for-java/pull/49345) * Fixed a sporadic `NullPointerException` in `JsonSerializable.getWithMapping` triggered by concurrent first-time calls to `DatabaseAccount.getConsistencyPolicy()` and its sibling lazy getters (`getReplicationPolicy`, `getSystemReplicationPolicy`, `getQueryEngineConfiguration`). The fix makes `JsonSerializable.propertyBag` `final`, closing an unsafe-publication race in the lazy-initialisation pattern. - See [Issue 49256](https://github.com/Azure/azure-sdk-for-java/issues/49256) and [PR #49258](https://github.com/Azure/azure-sdk-for-java/pull/49258)