Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ Only `iptables DROP SYN` prevents the TCP handshake, triggering the real netty `
- Docker Desktop with Linux containers
- Docker memory: **8 GB+**
- A Cosmos DB account with thin client enabled
- System properties: `COSMOS.THINCLIENT_ENABLED=true`, `COSMOS.THINCLIENT_CONNECTION_TIMEOUT_IN_SECONDS=5` (default; override to 1 for fast-fail local tests)
- System properties: `COSMOS.THINCLIENT_ENABLED=true`, `COSMOS.THINCLIENT_CONNECTION_TIMEOUT_IN_MS=5000` (default; override to 1000 for fast-fail local tests)
- Credentials in `sdk/cosmos/cosmos-v4.properties`

## Build & Run
Expand All @@ -36,7 +36,7 @@ docker run --rm --cap-add=NET_ADMIN --memory 8g \
cosmos-netem-test bash -c '
cd /workspace && \
java -DCOSMOS.THINCLIENT_ENABLED=true \
-DCOSMOS.THINCLIENT_CONNECTION_TIMEOUT_IN_SECONDS=1 \
-DCOSMOS.THINCLIENT_CONNECTION_TIMEOUT_IN_MS=1000 \
-DCOSMOS.HTTP2_ENABLED=true \
org.testng.TestNG /workspace/azure-cosmos-tests/src/test/resources/manual-thinclient-network-delay-testng.xml \
-verbose 2
Expand Down Expand Up @@ -111,4 +111,4 @@ is required to route non-SYN traffic to band 3 (no delay).
- Tests run **sequentially** — tc/iptables are interface-global
- `--cap-add=NET_ADMIN` required for both `tc` and `iptables`
- `@AfterClass` removes all iptables rules (`alwaysRun=true`)
- System property `COSMOS.THINCLIENT_CONNECTION_TIMEOUT_IN_SECONDS=1` sets the 1s bifurcated timeout
- System property `COSMOS.THINCLIENT_CONNECTION_TIMEOUT_IN_MS=1000` sets the 1s bifurcated timeout
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ public Http2ConnectTimeoutBifurcationTests(CosmosClientBuilder clientBuilder) {
@BeforeClass(groups = {TEST_GROUP}, timeOut = TIMEOUT)
public void beforeClass() {
System.setProperty("COSMOS.THINCLIENT_ENABLED", "true");
// Use the default THINCLIENT_CONNECTION_TIMEOUT_IN_SECONDS (5s) — no override.
// Use the default THINCLIENT_CONNECTION_TIMEOUT_IN_MS (5000ms) — no override.
// Tests are designed around the 5s default to match production behavior.

this.client = getClientBuilder().buildAsyncClient();
Expand Down Expand Up @@ -394,7 +394,7 @@ public void connectTimeout_GwV1_Metadata_UnaffectedByGwV2Drop() throws Exception
/**
* Measures the precise connect timeout boundary.
*
* With THINCLIENT_CONNECTION_TIMEOUT_IN_SECONDS=5 (default), a single TCP connect attempt
* With THINCLIENT_CONNECTION_TIMEOUT_IN_MS=5000 (default), a single TCP connect attempt
* to a blackholed port 10250 should fail in ~5s. We measure multiple individual
* attempts by using an e2e timeout of 12s (enough for 2 connect attempts at 5s each).
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import org.testng.annotations.Test;

import java.net.URI;
import java.time.Duration;
import java.util.EnumSet;

import static com.azure.cosmos.implementation.Configs.isThinClientEnabled;
Expand Down Expand Up @@ -182,42 +183,58 @@ public void thinClientEnabledTest() {

@Test(groups = { "unit" })
public void thinClientConnectionTimeoutDefaultTest() {
// Default thin client connection timeout should be 5 seconds
System.clearProperty("COSMOS.THINCLIENT_CONNECTION_TIMEOUT_IN_SECONDS");
// Default thin client connection timeout should be 5000 milliseconds
System.clearProperty("COSMOS.THINCLIENT_CONNECTION_TIMEOUT_IN_MS");
try {
assertThat(Configs.getThinClientConnectionTimeoutInSeconds()).isEqualTo(5);
assertThat(Configs.getThinClientConnectionTimeoutInMs()).isEqualTo(5_000);
} finally {
System.clearProperty("COSMOS.THINCLIENT_CONNECTION_TIMEOUT_IN_SECONDS");
System.clearProperty("COSMOS.THINCLIENT_CONNECTION_TIMEOUT_IN_MS");
}
}

@Test(groups = { "unit" })
public void thinClientConnectionTimeoutOverrideTest() {
System.clearProperty("COSMOS.THINCLIENT_CONNECTION_TIMEOUT_IN_SECONDS");
System.setProperty("COSMOS.THINCLIENT_CONNECTION_TIMEOUT_IN_SECONDS", "3");
System.clearProperty("COSMOS.THINCLIENT_CONNECTION_TIMEOUT_IN_MS");
System.setProperty("COSMOS.THINCLIENT_CONNECTION_TIMEOUT_IN_MS", "3000");
try {
assertThat(Configs.getThinClientConnectionTimeoutInSeconds()).isEqualTo(3);
assertThat(Configs.getThinClientConnectionTimeoutInMs()).isEqualTo(3_000);
} finally {
System.clearProperty("COSMOS.THINCLIENT_CONNECTION_TIMEOUT_IN_SECONDS");
System.clearProperty("COSMOS.THINCLIENT_CONNECTION_TIMEOUT_IN_MS");
}
}

@Test(groups = { "unit" })
public void thinClientConnectionTimeoutRejectsZeroAndNegative() {
// Zero should fall back to default (5s)
System.setProperty("COSMOS.THINCLIENT_CONNECTION_TIMEOUT_IN_SECONDS", "0");
public void thinClientConnectionTimeoutRejectsInvalidValues() {
// Zero should fall back to default (5000ms)
System.setProperty("COSMOS.THINCLIENT_CONNECTION_TIMEOUT_IN_MS", "0");
try {
assertThat(Configs.getThinClientConnectionTimeoutInSeconds()).isEqualTo(5);
assertThat(Configs.getThinClientConnectionTimeoutInMs()).isEqualTo(5_000);
} finally {
System.clearProperty("COSMOS.THINCLIENT_CONNECTION_TIMEOUT_IN_SECONDS");
System.clearProperty("COSMOS.THINCLIENT_CONNECTION_TIMEOUT_IN_MS");
}

// Negative should fall back to default (5s)
System.setProperty("COSMOS.THINCLIENT_CONNECTION_TIMEOUT_IN_SECONDS", "-1");
// Negative should fall back to default (5000ms)
System.setProperty("COSMOS.THINCLIENT_CONNECTION_TIMEOUT_IN_MS", "-1");
try {
assertThat(Configs.getThinClientConnectionTimeoutInSeconds()).isEqualTo(5);
assertThat(Configs.getThinClientConnectionTimeoutInMs()).isEqualTo(5_000);
} finally {
System.clearProperty("COSMOS.THINCLIENT_CONNECTION_TIMEOUT_IN_SECONDS");
System.clearProperty("COSMOS.THINCLIENT_CONNECTION_TIMEOUT_IN_MS");
}

// Below 500ms should fall back to default (5000ms)
System.setProperty("COSMOS.THINCLIENT_CONNECTION_TIMEOUT_IN_MS", "499");
try {
assertThat(Configs.getThinClientConnectionTimeoutInMs()).isEqualTo(5_000);
} finally {
System.clearProperty("COSMOS.THINCLIENT_CONNECTION_TIMEOUT_IN_MS");
}

// Exactly 500ms should be accepted
System.setProperty("COSMOS.THINCLIENT_CONNECTION_TIMEOUT_IN_MS", "500");
try {
assertThat(Configs.getThinClientConnectionTimeoutInMs()).isEqualTo(500);
} finally {
System.clearProperty("COSMOS.THINCLIENT_CONNECTION_TIMEOUT_IN_MS");
}
}

Expand Down Expand Up @@ -259,4 +276,72 @@ public void thinClientEndpointTest() {
System.clearProperty("COSMOS.THINCLIENT_ENDPOINT");
}
}

@Test(groups = { "unit" })
public void connectionAcquireTimeoutDefaultTest() {
// Default connection acquire timeout should be 45000 milliseconds
System.clearProperty("COSMOS.CONNECTION_ACQUIRE_TIMEOUT_IN_MS");
try {
assertThat(Configs.getConnectionAcquireTimeout()).isEqualTo(Duration.ofMillis(45_000));
} finally {
System.clearProperty("COSMOS.CONNECTION_ACQUIRE_TIMEOUT_IN_MS");
}
}

@Test(groups = { "unit" })
public void connectionAcquireTimeoutOverrideTest() {
System.clearProperty("COSMOS.CONNECTION_ACQUIRE_TIMEOUT_IN_MS");
System.setProperty("COSMOS.CONNECTION_ACQUIRE_TIMEOUT_IN_MS", "30000");
try {
assertThat(Configs.getConnectionAcquireTimeout()).isEqualTo(Duration.ofMillis(30_000));
} finally {
System.clearProperty("COSMOS.CONNECTION_ACQUIRE_TIMEOUT_IN_MS");
}
Comment thread
xinlian12 marked this conversation as resolved.
}

@Test(groups = { "unit" })
public void connectionAcquireTimeoutRejectsInvalidValues() {
// Zero should fall back to default (45000ms)
System.setProperty("COSMOS.CONNECTION_ACQUIRE_TIMEOUT_IN_MS", "0");
try {
assertThat(Configs.getConnectionAcquireTimeout()).isEqualTo(Duration.ofMillis(45_000));
} finally {
System.clearProperty("COSMOS.CONNECTION_ACQUIRE_TIMEOUT_IN_MS");
}

// Negative should fall back to default (45000ms)
System.setProperty("COSMOS.CONNECTION_ACQUIRE_TIMEOUT_IN_MS", "-1");
try {
assertThat(Configs.getConnectionAcquireTimeout()).isEqualTo(Duration.ofMillis(45_000));
} finally {
System.clearProperty("COSMOS.CONNECTION_ACQUIRE_TIMEOUT_IN_MS");
}

// Below 500ms should fall back to default (45000ms)
System.setProperty("COSMOS.CONNECTION_ACQUIRE_TIMEOUT_IN_MS", "499");
try {
assertThat(Configs.getConnectionAcquireTimeout()).isEqualTo(Duration.ofMillis(45_000));
} finally {
System.clearProperty("COSMOS.CONNECTION_ACQUIRE_TIMEOUT_IN_MS");
}

// Exactly 500ms should be accepted
System.setProperty("COSMOS.CONNECTION_ACQUIRE_TIMEOUT_IN_MS", "500");
try {
assertThat(Configs.getConnectionAcquireTimeout()).isEqualTo(Duration.ofMillis(500));
} finally {
System.clearProperty("COSMOS.CONNECTION_ACQUIRE_TIMEOUT_IN_MS");
}
}

@Test(groups = { "unit" })
public void connectionAcquireTimeoutRejectsNonNumericValue() {
// Non-numeric value should fall back to default (45s)
System.setProperty("COSMOS.CONNECTION_ACQUIRE_TIMEOUT_IN_SECONDS", "abc");
try {
assertThat(Configs.getConnectionAcquireTimeout()).isEqualTo(Duration.ofSeconds(45));
} finally {
System.clearProperty("COSMOS.CONNECTION_ACQUIRE_TIMEOUT_IN_SECONDS");
}
}
}
2 changes: 2 additions & 0 deletions sdk/cosmos/azure-cosmos/CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@
* Added `appendUserAgentSuffix` method to `AsyncDocumentClient` to allow downstream libraries to append to the user agent after client construction. - See [PR 48505](https://github.com/Azure/azure-sdk-for-java/pull/48505)
* Added aggressive HTTP timeout policies for document operations routed to Gateway V2. - [PR 47879](https://github.com/Azure/azure-sdk-for-java/pull/47879)
* Added a default connect timeout of 5s for Gateway V2 (thin client) data-plane endpoints. - See [PR 48174](https://github.com/Azure/azure-sdk-for-java/pull/48174)
* Added system property `COSMOS.CONNECTION_ACQUIRE_TIMEOUT_IN_MS` and environment variable `COSMOS_CONNECTION_ACQUIRE_TIMEOUT_IN_MS` to allow overriding the gateway connection acquire timeout in milliseconds (default 45000ms). Minimum accepted value is 500ms. Replaces the previous `_IN_SECONDS` variants. - See [PR 48580](https://github.com/Azure/azure-sdk-for-java/pull/48580)
* Changed system property for thin client connection timeout from `COSMOS.THINCLIENT_CONNECTION_TIMEOUT_IN_SECONDS` to `COSMOS.THINCLIENT_CONNECTION_TIMEOUT_IN_MS` (default 5000ms, minimum 500ms). - See [PR 48580](https://github.com/Azure/azure-sdk-for-java/pull/48580)

### 4.78.0 (2026-02-10)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -62,9 +62,9 @@ public class Configs {
// Data plane requests are routed to the thin client regional endpoint (from RegionalRoutingContext)
// which uses a non-443 port. These get a shorter 5s connect/acquire timeout.
// Metadata requests target Gateway V1 endpoint (port 443) and retain the full 45s/60s timeout (unchanged).
private static final int DEFAULT_THINCLIENT_CONNECTION_TIMEOUT_IN_SECONDS = 5;
private static final String THINCLIENT_CONNECTION_TIMEOUT_IN_SECONDS = "COSMOS.THINCLIENT_CONNECTION_TIMEOUT_IN_SECONDS";
private static final String THINCLIENT_CONNECTION_TIMEOUT_IN_SECONDS_VARIABLE = "COSMOS_THINCLIENT_CONNECTION_TIMEOUT_IN_SECONDS";
private static final int DEFAULT_THINCLIENT_CONNECTION_TIMEOUT_IN_MS = 5_000;
private static final String THINCLIENT_CONNECTION_TIMEOUT_IN_MS = "COSMOS.THINCLIENT_CONNECTION_TIMEOUT_IN_MS";
private static final String THINCLIENT_CONNECTION_TIMEOUT_IN_MS_VARIABLE = "COSMOS_THINCLIENT_CONNECTION_TIMEOUT_IN_MS";

private static final String MAX_HTTP_BODY_LENGTH_IN_BYTES = "COSMOS.MAX_HTTP_BODY_LENGTH_IN_BYTES";
private static final String MAX_HTTP_INITIAL_LINE_LENGTH_IN_BYTES = "COSMOS.MAX_HTTP_INITIAL_LINE_LENGTH_IN_BYTES";
Expand Down Expand Up @@ -139,7 +139,9 @@ public class Configs {

// Reactor Netty Constants
private static final Duration MAX_IDLE_CONNECTION_TIMEOUT = Duration.ofSeconds(60);
private static final Duration CONNECTION_ACQUIRE_TIMEOUT = Duration.ofSeconds(45);
private static final int DEFAULT_CONNECTION_ACQUIRE_TIMEOUT_IN_MS = 45_000;
private static final String CONNECTION_ACQUIRE_TIMEOUT_IN_MS = "COSMOS.CONNECTION_ACQUIRE_TIMEOUT_IN_MS";
private static final String CONNECTION_ACQUIRE_TIMEOUT_IN_MS_VARIABLE = "COSMOS_CONNECTION_ACQUIRE_TIMEOUT_IN_MS";
private static final String REACTOR_NETTY_CONNECTION_POOL_NAME = "reactor-netty-connection-pool";
private static final int DEFAULT_HTTP_RESPONSE_TIMEOUT_IN_SECONDS = 60;
private static final int DEFAULT_QUERY_PLAN_RESPONSE_TIMEOUT_IN_SECONDS = 5;
Expand Down Expand Up @@ -594,57 +596,95 @@ public static Duration getMaxIdleConnectionTimeout() {
}

public static Duration getConnectionAcquireTimeout() {
return CONNECTION_ACQUIRE_TIMEOUT;
int timeoutInMs = DEFAULT_CONNECTION_ACQUIRE_TIMEOUT_IN_MS;

String valueFromSystemProperty = System.getProperty(CONNECTION_ACQUIRE_TIMEOUT_IN_MS);
if (valueFromSystemProperty != null && !valueFromSystemProperty.isEmpty()) {
try {
timeoutInMs = Integer.parseInt(valueFromSystemProperty);
} catch (NumberFormatException e) {
logger.warn(
"Invalid non-numeric value '{}' for system property {}. Falling back to environment variable or default.",
valueFromSystemProperty,
CONNECTION_ACQUIRE_TIMEOUT_IN_MS);
valueFromSystemProperty = null;
}
}

if (valueFromSystemProperty == null || valueFromSystemProperty.isEmpty()) {
String valueFromEnvVariable = System.getenv(CONNECTION_ACQUIRE_TIMEOUT_IN_MS_VARIABLE);
if (valueFromEnvVariable != null && !valueFromEnvVariable.isEmpty()) {
try {
timeoutInMs = Integer.parseInt(valueFromEnvVariable);
} catch (NumberFormatException e) {
logger.warn(
"Invalid non-numeric value '{}' for environment variable {}. Falling back to default: {}ms.",
valueFromEnvVariable,
CONNECTION_ACQUIRE_TIMEOUT_IN_MS_VARIABLE,
DEFAULT_CONNECTION_ACQUIRE_TIMEOUT_IN_MS);
}
}
}

if (timeoutInMs < 500) {
logger.warn(
Comment thread
xinlian12 marked this conversation as resolved.
"Invalid connection acquire timeout: {}ms. Must be >= 500. Falling back to default: {}ms.",
timeoutInMs,
DEFAULT_CONNECTION_ACQUIRE_TIMEOUT_IN_MS);
timeoutInMs = DEFAULT_CONNECTION_ACQUIRE_TIMEOUT_IN_MS;
}

return Duration.ofMillis(timeoutInMs);
}

/**
* Returns the TCP connect timeout for thin client data plane endpoints.
* Returns the TCP connect timeout for thin client data plane endpoints in milliseconds.
* Data plane requests routed via thinclientRegionalEndpoint (from RegionalRoutingContext)
* use this aggressive timeout to fail fast when the proxy is unreachable.
* Metadata requests on port 443 are unaffected and retain the full 45s timeout.
*
* Configurable via system property COSMOS.THINCLIENT_CONNECTION_TIMEOUT_IN_SECONDS
* or environment variable COSMOS_THINCLIENT_CONNECTION_TIMEOUT_IN_SECONDS.
* Default: 5 seconds.
* Configurable via system property COSMOS.THINCLIENT_CONNECTION_TIMEOUT_IN_MS
* or environment variable COSMOS_THINCLIENT_CONNECTION_TIMEOUT_IN_MS.
* Default: 5000 milliseconds.
*/
public static int getThinClientConnectionTimeoutInSeconds() {
int value = DEFAULT_THINCLIENT_CONNECTION_TIMEOUT_IN_SECONDS;
public static int getThinClientConnectionTimeoutInMs() {
int value = DEFAULT_THINCLIENT_CONNECTION_TIMEOUT_IN_MS;

String valueFromSystemProperty = System.getProperty(THINCLIENT_CONNECTION_TIMEOUT_IN_SECONDS);
String valueFromSystemProperty = System.getProperty(THINCLIENT_CONNECTION_TIMEOUT_IN_MS);
if (valueFromSystemProperty != null && !valueFromSystemProperty.isEmpty()) {
try {
value = Integer.parseInt(valueFromSystemProperty);
} catch (NumberFormatException e) {
logger.warn(
"Invalid non-numeric value '{}' for system property {}. Falling back to environment variable or default.",
valueFromSystemProperty,
THINCLIENT_CONNECTION_TIMEOUT_IN_SECONDS);
THINCLIENT_CONNECTION_TIMEOUT_IN_MS);
valueFromSystemProperty = null;
}
}

if (valueFromSystemProperty == null || valueFromSystemProperty.isEmpty()) {
String valueFromEnvVariable = System.getenv(THINCLIENT_CONNECTION_TIMEOUT_IN_SECONDS_VARIABLE);
String valueFromEnvVariable = System.getenv(THINCLIENT_CONNECTION_TIMEOUT_IN_MS_VARIABLE);
if (valueFromEnvVariable != null && !valueFromEnvVariable.isEmpty()) {
try {
value = Integer.parseInt(valueFromEnvVariable);
} catch (NumberFormatException e) {
logger.warn(
"Invalid non-numeric value '{}' for environment variable {}. Falling back to default: {}s.",
"Invalid non-numeric value '{}' for environment variable {}. Falling back to default: {}ms.",
valueFromEnvVariable,
THINCLIENT_CONNECTION_TIMEOUT_IN_SECONDS_VARIABLE,
DEFAULT_THINCLIENT_CONNECTION_TIMEOUT_IN_SECONDS);
THINCLIENT_CONNECTION_TIMEOUT_IN_MS_VARIABLE,
DEFAULT_THINCLIENT_CONNECTION_TIMEOUT_IN_MS);
}
}
}

// Guard against invalid values — timeout must be at least 1 second
if (value <= 0) {
// Guard against invalid values — timeout must be at least 500ms
if (value < 500) {
logger.warn(
"Invalid thin client connection timeout: {}s. Must be > 0. Falling back to default: {}s.",
"Invalid thin client connection timeout: {}ms. Must be >= 500. Falling back to default: {}ms.",
value,
DEFAULT_THINCLIENT_CONNECTION_TIMEOUT_IN_SECONDS);
return DEFAULT_THINCLIENT_CONNECTION_TIMEOUT_IN_SECONDS;
DEFAULT_THINCLIENT_CONNECTION_TIMEOUT_IN_MS);
return DEFAULT_THINCLIENT_CONNECTION_TIMEOUT_IN_MS;
}

return value;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ public class HttpClientConfig {
public HttpClientConfig(Configs configs) {
this.configs = configs;
this.http2ConnectionConfig = new Http2ConnectionConfig();
this.thinClientConnectTimeoutMs = Configs.getThinClientConnectionTimeoutInSeconds() * 1000;
this.thinClientConnectTimeoutMs = Configs.getThinClientConnectionTimeoutInMs();
}

public HttpClientConfig withMaxHeaderSize(int maxHeaderSize) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -257,7 +257,7 @@ public void shutdown() {
* Resolves the TCP connect timeout (CONNECT_TIMEOUT_MILLIS) based on the request type.
*
* Thin client requests (identified by {@link HttpRequest#isThinClientRequest()}) use the thin
* client connection timeout configured via {@link Configs#getThinClientConnectionTimeoutInSeconds()}
* client connection timeout configured via {@link Configs#getThinClientConnectionTimeoutInMs()}
* (default 5s) to fail fast when the thin client proxy is unreachable.
* Standard gateway requests use the configured connection acquire timeout (default 45s).
*
Expand Down
Loading