From 6789796d1ebb18203e6580f66d34d4df4d211a3d Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Mon, 22 Apr 2024 13:01:46 +0000 Subject: [PATCH 01/10] Add capability to use (and enforce) native netty transport in azure-cosmos-spark --- .../docs/configuration-reference.md | 1 + sdk/cosmos/azure-cosmos-spark_3_2-12/pom.xml | 32 ++++++++++++++++++- .../cosmos/spark/CosmosClientCache.scala | 9 ++++++ .../spark/CosmosClientConfiguration.scala | 2 ++ .../com/azure/cosmos/spark/CosmosConfig.scala | 11 +++++++ .../cosmos/spark/CosmosClientCacheITest.scala | 5 +++ .../spark/CosmosPartitionPlannerSpec.scala | 6 ++++ .../cosmos/spark/PartitionMetadataSpec.scala | 12 +++++++ .../notebooks/basicScenario.scala | 6 ++-- .../notebooks/basicScenarioAad.scala | 6 ++-- .../com/azure/cosmos/CosmosClientBuilder.java | 9 ++++-- 11 files changed, 91 insertions(+), 8 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-spark_3_2-12/docs/configuration-reference.md b/sdk/cosmos/azure-cosmos-spark_3_2-12/docs/configuration-reference.md index b00d75cfdc73..5d1c9be3f836 100644 --- a/sdk/cosmos/azure-cosmos-spark_3_2-12/docs/configuration-reference.md +++ b/sdk/cosmos/azure-cosmos-spark_3_2-12/docs/configuration-reference.md @@ -24,6 +24,7 @@ | Config Property Name | Default | Description | |:------------------------------------------------------------------|:--------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `spark.cosmos.useGatewayMode` | `false` | Use gateway mode for the client operations | +| `spark.cosmos.enforceNativeTransport` | `false` | Indicates whether native netty transport should be enforced. If true, on clinet creation an exception will be trown if netty is not able to use native transport. The native transport is more efficient especiall when there is a high number of connections. | | `spark.cosmos.http.connectionPoolSize` | `1000` | Gateway mode connection pool size | | `spark.cosmos.read.forceEventualConsistency` | `true` | Makes the client use Eventual consistency for read operations instead of using the default account level consistency | | `spark.cosmos.applicationName` | None | Application name | diff --git a/sdk/cosmos/azure-cosmos-spark_3_2-12/pom.xml b/sdk/cosmos/azure-cosmos-spark_3_2-12/pom.xml index 07377decbbad..0228ac404b0a 100644 --- a/sdk/cosmos/azure-cosmos-spark_3_2-12/pom.xml +++ b/sdk/cosmos/azure-cosmos-spark_3_2-12/pom.xml @@ -24,6 +24,7 @@ 11 true azure_cosmos_spark + azurecosmosspark @@ -427,7 +428,7 @@ io.netty - ${shadingPrefix}.io.netty + ${shadingPrefixNetty}.io.netty org.codehaus @@ -604,6 +605,35 @@ + org.codehaus.mojo xml-maven-plugin diff --git a/sdk/cosmos/azure-cosmos-spark_3_2-12/src/main/scala/com/azure/cosmos/spark/CosmosClientCache.scala b/sdk/cosmos/azure-cosmos-spark_3_2-12/src/main/scala/com/azure/cosmos/spark/CosmosClientCache.scala index 748ea7400463..240a6d302454 100644 --- a/sdk/cosmos/azure-cosmos-spark_3_2-12/src/main/scala/com/azure/cosmos/spark/CosmosClientCache.scala +++ b/sdk/cosmos/azure-cosmos-spark_3_2-12/src/main/scala/com/azure/cosmos/spark/CosmosClientCache.scala @@ -174,6 +174,15 @@ private[spark] object CosmosClientCache extends BasicLoggingTrait { // scalastyle:off cyclomatic.complexity private[this] def createCosmosAsyncClient(cosmosClientConfiguration: CosmosClientConfiguration, cosmosClientStateHandle: Option[CosmosClientMetadataCachesSnapshot]): CosmosAsyncClient = { + if (cosmosClientConfiguration.enforceNativeTransport && !io.netty.channel.epoll.Epoll.isAvailable) { + throw new IllegalStateException( + "The enforcement of native transport is enabled in your configuration and native transport is not " + + "available. Either ensure `spark.cosmos.enforceNativeTransport` is set to false or make " + + "sure you use a Spark environment supporting native transport.", + io.netty.channel.epoll.Epoll.unavailabilityCause() + ) + } + var builder = new CosmosClientBuilder() .endpoint(cosmosClientConfiguration.endpoint) .userAgentSuffix(cosmosClientConfiguration.applicationName) diff --git a/sdk/cosmos/azure-cosmos-spark_3_2-12/src/main/scala/com/azure/cosmos/spark/CosmosClientConfiguration.scala b/sdk/cosmos/azure-cosmos-spark_3_2-12/src/main/scala/com/azure/cosmos/spark/CosmosClientConfiguration.scala index 61081399bcdc..1cbe4fcc50b8 100644 --- a/sdk/cosmos/azure-cosmos-spark_3_2-12/src/main/scala/com/azure/cosmos/spark/CosmosClientConfiguration.scala +++ b/sdk/cosmos/azure-cosmos-spark_3_2-12/src/main/scala/com/azure/cosmos/spark/CosmosClientConfiguration.scala @@ -14,6 +14,7 @@ private[spark] case class CosmosClientConfiguration ( customApplicationNameSuffix: Option[String], applicationName: String, useGatewayMode: Boolean, + enforceNativeTransport: Boolean, proactiveConnectionInitialization: Option[String], proactiveConnectionInitializationDurationInSeconds: Int, httpConnectionPoolSize: Int, @@ -69,6 +70,7 @@ private[spark] object CosmosClientConfiguration { customApplicationNameSuffix, applicationName, cosmosAccountConfig.useGatewayMode, + cosmosAccountConfig.enforceNativeTransport, cosmosAccountConfig.proactiveConnectionInitialization, cosmosAccountConfig.proactiveConnectionInitializationDurationInSeconds, cosmosAccountConfig.httpConnectionPoolSize, diff --git a/sdk/cosmos/azure-cosmos-spark_3_2-12/src/main/scala/com/azure/cosmos/spark/CosmosConfig.scala b/sdk/cosmos/azure-cosmos-spark_3_2-12/src/main/scala/com/azure/cosmos/spark/CosmosConfig.scala index 43af00d7114b..d5ca9af82784 100644 --- a/sdk/cosmos/azure-cosmos-spark_3_2-12/src/main/scala/com/azure/cosmos/spark/CosmosConfig.scala +++ b/sdk/cosmos/azure-cosmos-spark_3_2-12/src/main/scala/com/azure/cosmos/spark/CosmosConfig.scala @@ -59,6 +59,7 @@ private[spark] object CosmosConfigNames { val DisableTcpConnectionEndpointRediscovery = "spark.cosmos.disableTcpConnectionEndpointRediscovery" val ApplicationName = "spark.cosmos.applicationName" val UseGatewayMode = "spark.cosmos.useGatewayMode" + val EnforceNativeTransport = "spark.cosmos.enforceNativeTransport" val ProactiveConnectionInitialization = "spark.cosmos.proactiveConnectionInitialization" val ProactiveConnectionInitializationDurationInSeconds = "spark.cosmos.proactiveConnectionInitializationDurationInSeconds" val GatewayConnectionPoolSize = "spark.cosmos.http.connectionPoolSize" @@ -151,6 +152,7 @@ private[spark] object CosmosConfigNames { DisableTcpConnectionEndpointRediscovery, ApplicationName, UseGatewayMode, + EnforceNativeTransport, ProactiveConnectionInitialization, ProactiveConnectionInitializationDurationInSeconds, GatewayConnectionPoolSize, @@ -328,6 +330,7 @@ private case class CosmosAccountConfig(endpoint: String, accountName: String, applicationName: Option[String], useGatewayMode: Boolean, + enforceNativeTransport: Boolean, proactiveConnectionInitialization: Option[String], proactiveConnectionInitializationDurationInSeconds: Int, httpConnectionPoolSize: Int, @@ -408,6 +411,12 @@ private object CosmosAccountConfig { parseFromStringFunction = useGatewayMode => useGatewayMode.toBoolean, helpMessage = "Use gateway mode for the client operations") + private val EnforceNativeTransport = CosmosConfigEntry[Boolean](key = CosmosConfigNames.EnforceNativeTransport, + mandatory = false, + defaultValue = Some(false), + parseFromStringFunction = enforceNativeTransport => enforceNativeTransport.toBoolean, + helpMessage = "Flag indicating whether native Netty transport availability should be enforced.") + private val ProactiveConnectionInitialization = CosmosConfigEntry[String](key = CosmosConfigNames.ProactiveConnectionInitialization, mandatory = false, defaultValue = None, @@ -502,6 +511,7 @@ private object CosmosAccountConfig { val accountName = CosmosConfigEntry.parse(cfg, CosmosAccountName) val applicationName = CosmosConfigEntry.parse(cfg, ApplicationName) val useGatewayMode = CosmosConfigEntry.parse(cfg, UseGatewayMode) + val enforceNativeTransport = CosmosConfigEntry.parse(cfg, EnforceNativeTransport) val proactiveConnectionInitialization = CosmosConfigEntry.parse(cfg, ProactiveConnectionInitialization) val proactiveConnectionInitializationDurationInSeconds = CosmosConfigEntry.parse(cfg, ProactiveConnectionInitializationDurationInSeconds) val httpConnectionPoolSize = CosmosConfigEntry.parse(cfg, HttpConnectionPoolSize) @@ -558,6 +568,7 @@ private object CosmosAccountConfig { accountName.get, applicationName, useGatewayMode.get, + enforceNativeTransport.get, proactiveConnectionInitialization, proactiveConnectionInitializationDurationInSeconds.get, httpConnectionPoolSize.get, diff --git a/sdk/cosmos/azure-cosmos-spark_3_2-12/src/test/scala/com/azure/cosmos/spark/CosmosClientCacheITest.scala b/sdk/cosmos/azure-cosmos-spark_3_2-12/src/test/scala/com/azure/cosmos/spark/CosmosClientCacheITest.scala index 060540c927e2..38bce3555f2f 100644 --- a/sdk/cosmos/azure-cosmos-spark_3_2-12/src/test/scala/com/azure/cosmos/spark/CosmosClientCacheITest.scala +++ b/sdk/cosmos/azure-cosmos-spark_3_2-12/src/test/scala/com/azure/cosmos/spark/CosmosClientCacheITest.scala @@ -49,6 +49,7 @@ class CosmosClientCacheITest Some("SampleApplicationName"), "SampleApplicationName", useGatewayMode = true, + enforceNativeTransport = false, proactiveConnectionInitialization = None, proactiveConnectionInitializationDurationInSeconds = 120, httpConnectionPoolSize = 1000, @@ -72,6 +73,7 @@ class CosmosClientCacheITest Some("SampleApplicationName"), "SampleApplicationName", useGatewayMode = true, + enforceNativeTransport = false, proactiveConnectionInitialization = None, proactiveConnectionInitializationDurationInSeconds = 120, httpConnectionPoolSize = 1000, @@ -95,6 +97,7 @@ class CosmosClientCacheITest None, "SampleApplicationName", useGatewayMode = true, + enforceNativeTransport = false, proactiveConnectionInitialization = None, proactiveConnectionInitializationDurationInSeconds = 120, httpConnectionPoolSize = 1000, @@ -118,6 +121,7 @@ class CosmosClientCacheITest None, "SampleApplicationName", useGatewayMode = true, + enforceNativeTransport = false, proactiveConnectionInitialization = None, proactiveConnectionInitializationDurationInSeconds = 120, httpConnectionPoolSize = 1000, @@ -145,6 +149,7 @@ class CosmosClientCacheITest userConfig.customApplicationNameSuffix, userConfig.applicationName, userConfig.useGatewayMode, + userConfig.enforceNativeTransport, userConfig.proactiveConnectionInitialization, userConfig.proactiveConnectionInitializationDurationInSeconds, userConfig.httpConnectionPoolSize, diff --git a/sdk/cosmos/azure-cosmos-spark_3_2-12/src/test/scala/com/azure/cosmos/spark/CosmosPartitionPlannerSpec.scala b/sdk/cosmos/azure-cosmos-spark_3_2-12/src/test/scala/com/azure/cosmos/spark/CosmosPartitionPlannerSpec.scala index ef51932a8794..621aea52b597 100644 --- a/sdk/cosmos/azure-cosmos-spark_3_2-12/src/test/scala/com/azure/cosmos/spark/CosmosPartitionPlannerSpec.scala +++ b/sdk/cosmos/azure-cosmos-spark_3_2-12/src/test/scala/com/azure/cosmos/spark/CosmosPartitionPlannerSpec.scala @@ -22,6 +22,7 @@ class CosmosPartitionPlannerSpec extends UnitSpec { None, UUID.randomUUID().toString, useGatewayMode = false, + enforceNativeTransport = false, proactiveConnectionInitialization = None, proactiveConnectionInitializationDurationInSeconds = 120, httpConnectionPoolSize = 1000, @@ -94,6 +95,7 @@ class CosmosPartitionPlannerSpec extends UnitSpec { None, UUID.randomUUID().toString, useGatewayMode = false, + enforceNativeTransport = false, proactiveConnectionInitialization = None, proactiveConnectionInitializationDurationInSeconds = 120, httpConnectionPoolSize = 1000, @@ -166,6 +168,7 @@ class CosmosPartitionPlannerSpec extends UnitSpec { None, UUID.randomUUID().toString, useGatewayMode = false, + enforceNativeTransport = false, proactiveConnectionInitialization = None, proactiveConnectionInitializationDurationInSeconds = 120, httpConnectionPoolSize = 1000, @@ -238,6 +241,7 @@ class CosmosPartitionPlannerSpec extends UnitSpec { None, UUID.randomUUID().toString, useGatewayMode = false, + enforceNativeTransport = false, proactiveConnectionInitialization = None, proactiveConnectionInitializationDurationInSeconds = 120, httpConnectionPoolSize = 1000, @@ -308,6 +312,7 @@ class CosmosPartitionPlannerSpec extends UnitSpec { None, UUID.randomUUID().toString, useGatewayMode = false, + enforceNativeTransport = false, proactiveConnectionInitialization = None, proactiveConnectionInitializationDurationInSeconds = 120, httpConnectionPoolSize = 1000, @@ -394,6 +399,7 @@ class CosmosPartitionPlannerSpec extends UnitSpec { None, UUID.randomUUID().toString, useGatewayMode = false, + enforceNativeTransport = false, proactiveConnectionInitialization = None, proactiveConnectionInitializationDurationInSeconds = 120, httpConnectionPoolSize = 1000, diff --git a/sdk/cosmos/azure-cosmos-spark_3_2-12/src/test/scala/com/azure/cosmos/spark/PartitionMetadataSpec.scala b/sdk/cosmos/azure-cosmos-spark_3_2-12/src/test/scala/com/azure/cosmos/spark/PartitionMetadataSpec.scala index 7018902c6ea4..a13c5fa2fdc0 100644 --- a/sdk/cosmos/azure-cosmos-spark_3_2-12/src/test/scala/com/azure/cosmos/spark/PartitionMetadataSpec.scala +++ b/sdk/cosmos/azure-cosmos-spark_3_2-12/src/test/scala/com/azure/cosmos/spark/PartitionMetadataSpec.scala @@ -21,6 +21,7 @@ class PartitionMetadataSpec extends UnitSpec { None, UUID.randomUUID().toString, useGatewayMode = false, + enforceNativeTransport = false, proactiveConnectionInitialization = None, proactiveConnectionInitializationDurationInSeconds = 120, httpConnectionPoolSize = 1000, @@ -64,6 +65,7 @@ class PartitionMetadataSpec extends UnitSpec { None, UUID.randomUUID().toString, useGatewayMode = false, + enforceNativeTransport = false, proactiveConnectionInitialization = None, proactiveConnectionInitializationDurationInSeconds = 120, httpConnectionPoolSize = 1000, @@ -146,6 +148,7 @@ class PartitionMetadataSpec extends UnitSpec { None, UUID.randomUUID().toString, useGatewayMode = false, + enforceNativeTransport = false, proactiveConnectionInitialization = None, proactiveConnectionInitializationDurationInSeconds = 120, httpConnectionPoolSize = 1000, @@ -228,6 +231,7 @@ class PartitionMetadataSpec extends UnitSpec { None, UUID.randomUUID().toString, useGatewayMode = false, + enforceNativeTransport = false, proactiveConnectionInitialization = None, proactiveConnectionInitializationDurationInSeconds = 120, httpConnectionPoolSize = 1000, @@ -292,6 +296,7 @@ class PartitionMetadataSpec extends UnitSpec { None, UUID.randomUUID().toString, useGatewayMode = false, + enforceNativeTransport = false, proactiveConnectionInitialization = None, proactiveConnectionInitializationDurationInSeconds = 120, httpConnectionPoolSize = 1000, @@ -351,6 +356,7 @@ class PartitionMetadataSpec extends UnitSpec { None, UUID.randomUUID().toString, useGatewayMode = false, + enforceNativeTransport = false, proactiveConnectionInitialization = None, proactiveConnectionInitializationDurationInSeconds = 120, httpConnectionPoolSize = 1000, @@ -404,6 +410,7 @@ class PartitionMetadataSpec extends UnitSpec { None, UUID.randomUUID().toString, useGatewayMode = false, + enforceNativeTransport = false, proactiveConnectionInitialization = None, proactiveConnectionInitializationDurationInSeconds = 120, httpConnectionPoolSize = 1000, @@ -457,6 +464,7 @@ class PartitionMetadataSpec extends UnitSpec { None, UUID.randomUUID().toString, useGatewayMode = false, + enforceNativeTransport = false, proactiveConnectionInitialization = None, proactiveConnectionInitializationDurationInSeconds = 120, httpConnectionPoolSize = 1000, @@ -510,6 +518,7 @@ class PartitionMetadataSpec extends UnitSpec { None, UUID.randomUUID().toString, useGatewayMode = false, + enforceNativeTransport = false, proactiveConnectionInitialization = None, proactiveConnectionInitializationDurationInSeconds = 120, httpConnectionPoolSize = 1000, @@ -563,6 +572,7 @@ class PartitionMetadataSpec extends UnitSpec { None, UUID.randomUUID().toString, useGatewayMode = false, + enforceNativeTransport = false, proactiveConnectionInitialization = None, proactiveConnectionInitializationDurationInSeconds = 120, httpConnectionPoolSize = 1000, @@ -616,6 +626,7 @@ class PartitionMetadataSpec extends UnitSpec { None, UUID.randomUUID().toString, useGatewayMode = false, + enforceNativeTransport = false, proactiveConnectionInitialization = None, proactiveConnectionInitializationDurationInSeconds = 120, httpConnectionPoolSize = 1000, @@ -686,6 +697,7 @@ class PartitionMetadataSpec extends UnitSpec { None, UUID.randomUUID().toString, useGatewayMode = false, + enforceNativeTransport = false, proactiveConnectionInitialization = None, proactiveConnectionInitializationDurationInSeconds = 120, httpConnectionPoolSize = 1000, diff --git a/sdk/cosmos/azure-cosmos-spark_3_2-12/test-databricks/notebooks/basicScenario.scala b/sdk/cosmos/azure-cosmos-spark_3_2-12/test-databricks/notebooks/basicScenario.scala index 466ef45c07f7..958599a7d4fc 100644 --- a/sdk/cosmos/azure-cosmos-spark_3_2-12/test-databricks/notebooks/basicScenario.scala +++ b/sdk/cosmos/azure-cosmos-spark_3_2-12/test-databricks/notebooks/basicScenario.scala @@ -13,14 +13,16 @@ val cfg = Map("spark.cosmos.accountEndpoint" -> cosmosEndpoint, "spark.cosmos.container" -> cosmosContainerName, "spark.cosmos.preferredRegionsList" -> "West US 2", "spark.cosmos.proactiveConnectionInitialization" -> s"$cosmosDatabaseName/$cosmosContainerName", - "spark.cosmos.proactiveConnectionInitializationDurationInSeconds" -> "10" + "spark.cosmos.proactiveConnectionInitializationDurationInSeconds" -> "10", + "spark.cosmos.enforceNativeTransport" -> "true" ) val cfgWithAutoSchemaInference = Map("spark.cosmos.accountEndpoint" -> cosmosEndpoint, "spark.cosmos.accountKey" -> cosmosMasterKey, "spark.cosmos.database" -> cosmosDatabaseName, "spark.cosmos.container" -> cosmosContainerName, - "spark.cosmos.read.inferSchema.enabled" -> "true" + "spark.cosmos.read.inferSchema.enabled" -> "true", + "spark.cosmos.enforceNativeTransport" -> "true" ) // COMMAND ---------- diff --git a/sdk/cosmos/azure-cosmos-spark_3_2-12/test-databricks/notebooks/basicScenarioAad.scala b/sdk/cosmos/azure-cosmos-spark_3_2-12/test-databricks/notebooks/basicScenarioAad.scala index 970c3b259237..aa19ea68db5e 100644 --- a/sdk/cosmos/azure-cosmos-spark_3_2-12/test-databricks/notebooks/basicScenarioAad.scala +++ b/sdk/cosmos/azure-cosmos-spark_3_2-12/test-databricks/notebooks/basicScenarioAad.scala @@ -24,7 +24,8 @@ val cfg = Map("spark.cosmos.accountEndpoint" -> cosmosEndpoint, "spark.cosmos.auth.aad.clientId" -> clientId, "spark.cosmos.auth.aad.clientSecret" -> clientSecret, "spark.cosmos.database" -> cosmosDatabaseName, - "spark.cosmos.container" -> cosmosContainerName + "spark.cosmos.container" -> cosmosContainerName, + "spark.cosmos.enforceNativeTransport" -> "true" ) val cfgWithAutoSchemaInference = Map("spark.cosmos.accountEndpoint" -> cosmosEndpoint, @@ -36,7 +37,8 @@ val cfgWithAutoSchemaInference = Map("spark.cosmos.accountEndpoint" -> cosmosEnd "spark.cosmos.auth.aad.clientSecret" -> clientSecret, "spark.cosmos.database" -> cosmosDatabaseName, "spark.cosmos.container" -> cosmosContainerName, - "spark.cosmos.read.inferSchema.enabled" -> "true" + "spark.cosmos.read.inferSchema.enabled" -> "true", + "spark.cosmos.enforceNativeTransport" -> "true" ) // COMMAND ---------- diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/CosmosClientBuilder.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/CosmosClientBuilder.java index 1c30988b6112..87a9731c6eff 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/CosmosClientBuilder.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/CosmosClientBuilder.java @@ -1218,7 +1218,7 @@ private void ifThrowIllegalArgException(boolean value, String error) { private void logStartupInfo(StopWatch stopwatch, CosmosAsyncClient client) { stopwatch.stop(); - if (logger.isInfoEnabled()) { + if (logger.isWarnEnabled()) { long time = stopwatch.getTime(); String diagnosticsCfg = ""; String tracingCfg = ""; @@ -1235,11 +1235,14 @@ private void logStartupInfo(StopWatch stopwatch, CosmosAsyncClient client) { logger.warn("Cosmos Client with (Correlation) ID [{}] started up in [{}] ms with the following " + "configuration: serviceEndpoint [{}], preferredRegions [{}], excludedRegions [{}], connectionPolicy [{}], " + "consistencyLevel [{}], contentResponseOnWriteEnabled [{}], sessionCapturingOverride [{}], " + - "connectionSharingAcrossClients [{}], clientTelemetryEnabled [{}], proactiveContainerInit [{}], diagnostics [{}], tracing [{}]", + "connectionSharingAcrossClients [{}], clientTelemetryEnabled [{}], proactiveContainerInit [{}], " + + "diagnostics [{}], tracing [{}], nativeTransport [{}] fastClientOpen [{}]", client.getContextClient().getClientCorrelationId(), time, getEndpoint(), getPreferredRegions(), getExcludedRegions(), getConnectionPolicy(), getConsistencyLevel(), isContentResponseOnWriteEnabled(), isSessionCapturingOverrideEnabled(), isConnectionSharingAcrossClientsEnabled(), - isClientTelemetryEnabled(), getProactiveContainerInitConfig(), diagnosticsCfg, tracingCfg); + isClientTelemetryEnabled(), getProactiveContainerInitConfig(), diagnosticsCfg, + tracingCfg, io.netty.channel.epoll.Epoll.isAvailable(), + io.netty.channel.epoll.Epoll.isTcpFastOpenClientSideAvailable()); } } From c07ed2b2355ff27310e453f4b8f7ccfaa1042209 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Mon, 22 Apr 2024 15:19:40 +0000 Subject: [PATCH 02/10] Update pom.xml --- sdk/cosmos/azure-cosmos-spark_3_2-12/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-spark_3_2-12/pom.xml b/sdk/cosmos/azure-cosmos-spark_3_2-12/pom.xml index 0228ac404b0a..c6a3362c2634 100644 --- a/sdk/cosmos/azure-cosmos-spark_3_2-12/pom.xml +++ b/sdk/cosmos/azure-cosmos-spark_3_2-12/pom.xml @@ -605,7 +605,7 @@ - + org.codehaus.mojo xml-maven-plugin From 7e675363d8925d6936e1629aa90e82c35a732d73 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Mon, 22 Apr 2024 16:25:21 +0000 Subject: [PATCH 03/10] Fix build breaks --- eng/versioning/external_dependencies.txt | 1 + sdk/cosmos/azure-cosmos-spark_3_2-12/pom.xml | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/eng/versioning/external_dependencies.txt b/eng/versioning/external_dependencies.txt index ac898b02c2bc..85b81cb06737 100644 --- a/eng/versioning/external_dependencies.txt +++ b/eng/versioning/external_dependencies.txt @@ -398,6 +398,7 @@ cosmos_org.mockito:mockito-core;4.8.1 cosmos_org.scalatest:scalatest-maven-plugin;2.1.0 cosmos_net.alchim31.maven:scala-maven-plugin;4.8.1 cosmos_org.scalastyle:scalastyle-maven-plugin;1.0.0 +cosmos_com.coderplus.maven.plugins:copy-rename-maven-plugin;1.0.1 ## Cosmos Kafka connector under sdk\cosmos\azure-cosmos-kafka-connect\pom.xml # Cosmos Kafka connector runtime dependencies diff --git a/sdk/cosmos/azure-cosmos-spark_3_2-12/pom.xml b/sdk/cosmos/azure-cosmos-spark_3_2-12/pom.xml index c6a3362c2634..c8a16da11076 100644 --- a/sdk/cosmos/azure-cosmos-spark_3_2-12/pom.xml +++ b/sdk/cosmos/azure-cosmos-spark_3_2-12/pom.xml @@ -227,6 +227,7 @@ org.scala-lang.modules:scala-java8-compat_2.12:[0.8.0] io.projectreactor:reactor-scala-extensions_2.12:[0.8.0] org.scalatest:scalatest_2.12:[3.2.2] + com.coderplus.maven.plugins:copy-rename-maven-plugin:[1.0.1] net.alchim31.maven:scala-maven-plugin:[4.8.1] org.scalastyle:scalastyle-maven-plugin:[1.0.0] com.fasterxml.jackson.datatype:jackson-datatype-jsr310:[2.13.5] @@ -608,7 +609,7 @@ org.apache.maven.plugins maven-antrun-plugin - 3.1.0 + 3.1.0 repack From 52781f3f73ff94afff4613e4ac528fd8987eac40 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Mon, 22 Apr 2024 19:11:27 +0200 Subject: [PATCH 04/10] Update sdk/cosmos/azure-cosmos-spark_3_2-12/docs/configuration-reference.md Co-authored-by: Annie Liang <64233642+xinlian12@users.noreply.github.com> --- .../azure-cosmos-spark_3_2-12/docs/configuration-reference.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/cosmos/azure-cosmos-spark_3_2-12/docs/configuration-reference.md b/sdk/cosmos/azure-cosmos-spark_3_2-12/docs/configuration-reference.md index 5d1c9be3f836..de3cf915cc77 100644 --- a/sdk/cosmos/azure-cosmos-spark_3_2-12/docs/configuration-reference.md +++ b/sdk/cosmos/azure-cosmos-spark_3_2-12/docs/configuration-reference.md @@ -24,7 +24,7 @@ | Config Property Name | Default | Description | |:------------------------------------------------------------------|:--------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `spark.cosmos.useGatewayMode` | `false` | Use gateway mode for the client operations | -| `spark.cosmos.enforceNativeTransport` | `false` | Indicates whether native netty transport should be enforced. If true, on clinet creation an exception will be trown if netty is not able to use native transport. The native transport is more efficient especiall when there is a high number of connections. | +| `spark.cosmos.enforceNativeTransport` | `false` | Indicates whether native netty transport should be enforced. If true, on clinet creation an exception will be thrown if netty is not able to use native transport. The native transport is more efficient especially when there is a high number of connections. | | `spark.cosmos.http.connectionPoolSize` | `1000` | Gateway mode connection pool size | | `spark.cosmos.read.forceEventualConsistency` | `true` | Makes the client use Eventual consistency for read operations instead of using the default account level consistency | | `spark.cosmos.applicationName` | None | Application name | From b40a2330dad40848b62bb828735f97da3121ea26 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Mon, 22 Apr 2024 17:18:09 +0000 Subject: [PATCH 05/10] Fixing build breaks regarding antrun plugin --- eng/versioning/external_dependencies.txt | 1 - sdk/cosmos/azure-cosmos-spark_3_2-12/pom.xml | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/eng/versioning/external_dependencies.txt b/eng/versioning/external_dependencies.txt index 85b81cb06737..ac898b02c2bc 100644 --- a/eng/versioning/external_dependencies.txt +++ b/eng/versioning/external_dependencies.txt @@ -398,7 +398,6 @@ cosmos_org.mockito:mockito-core;4.8.1 cosmos_org.scalatest:scalatest-maven-plugin;2.1.0 cosmos_net.alchim31.maven:scala-maven-plugin;4.8.1 cosmos_org.scalastyle:scalastyle-maven-plugin;1.0.0 -cosmos_com.coderplus.maven.plugins:copy-rename-maven-plugin;1.0.1 ## Cosmos Kafka connector under sdk\cosmos\azure-cosmos-kafka-connect\pom.xml # Cosmos Kafka connector runtime dependencies diff --git a/sdk/cosmos/azure-cosmos-spark_3_2-12/pom.xml b/sdk/cosmos/azure-cosmos-spark_3_2-12/pom.xml index c8a16da11076..0a07ac4193e8 100644 --- a/sdk/cosmos/azure-cosmos-spark_3_2-12/pom.xml +++ b/sdk/cosmos/azure-cosmos-spark_3_2-12/pom.xml @@ -227,7 +227,7 @@ org.scala-lang.modules:scala-java8-compat_2.12:[0.8.0] io.projectreactor:reactor-scala-extensions_2.12:[0.8.0] org.scalatest:scalatest_2.12:[3.2.2] - com.coderplus.maven.plugins:copy-rename-maven-plugin:[1.0.1] + org.apache.maven.plugins:maven-antrun-plugin:[3.1.0] net.alchim31.maven:scala-maven-plugin:[4.8.1] org.scalastyle:scalastyle-maven-plugin:[1.0.0] com.fasterxml.jackson.datatype:jackson-datatype-jsr310:[2.13.5] @@ -609,7 +609,7 @@ org.apache.maven.plugins maven-antrun-plugin - 3.1.0 + 3.1.0 repack From 889158149053c111e2073d2745db8a8fbd202404 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Mon, 22 Apr 2024 21:00:12 +0000 Subject: [PATCH 06/10] Update pom.xml --- sdk/cosmos/azure-cosmos-spark_3_2-12/pom.xml | 70 +++++++++----------- 1 file changed, 31 insertions(+), 39 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-spark_3_2-12/pom.xml b/sdk/cosmos/azure-cosmos-spark_3_2-12/pom.xml index 0a07ac4193e8..57b342539c5b 100644 --- a/sdk/cosmos/azure-cosmos-spark_3_2-12/pom.xml +++ b/sdk/cosmos/azure-cosmos-spark_3_2-12/pom.xml @@ -303,7 +303,7 @@ 3.3.0 - empty-javadoc-jar-with-readme + 01-empty-javadoc-jar-with-readme package jar @@ -315,33 +315,6 @@ - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - copy-readme-to-javadocTemp - prepare-package - - - Deleting existing ${project.basedir}/javadocTemp - - - - Copying ${project.basedir}/README.md to - ${project.basedir}/javadocTemp/README.md - - - - - - run - - - - - net.alchim31.maven scala-maven-plugin @@ -401,7 +374,7 @@ 3.5.2 - shade + 02-shade package shade @@ -612,22 +585,41 @@ 3.1.0 - repack - verify + 01-copy-readme-to-javadocTemp + prepare-package + + + Deleting existing ${project.basedir}/javadocTemp + + + + Copying ${project.basedir}/README.md to + ${project.basedir}/javadocTemp/README.md + + + + + + run + + + + 03-repack + package run - - - - - - - - + + + + + + + + From cbbe74f7b92d0cd955409941831ed7dc358801a8 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Tue, 23 Apr 2024 00:32:13 +0200 Subject: [PATCH 07/10] Update sdk/cosmos/azure-cosmos-spark_3_2-12/docs/configuration-reference.md Co-authored-by: Kushagra Thapar --- .../azure-cosmos-spark_3_2-12/docs/configuration-reference.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/cosmos/azure-cosmos-spark_3_2-12/docs/configuration-reference.md b/sdk/cosmos/azure-cosmos-spark_3_2-12/docs/configuration-reference.md index de3cf915cc77..310df0e97ccb 100644 --- a/sdk/cosmos/azure-cosmos-spark_3_2-12/docs/configuration-reference.md +++ b/sdk/cosmos/azure-cosmos-spark_3_2-12/docs/configuration-reference.md @@ -24,7 +24,7 @@ | Config Property Name | Default | Description | |:------------------------------------------------------------------|:--------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `spark.cosmos.useGatewayMode` | `false` | Use gateway mode for the client operations | -| `spark.cosmos.enforceNativeTransport` | `false` | Indicates whether native netty transport should be enforced. If true, on clinet creation an exception will be thrown if netty is not able to use native transport. The native transport is more efficient especially when there is a high number of connections. | +| `spark.cosmos.enforceNativeTransport` | `false` | Indicates whether native netty transport should be enforced. If true, on client creation an exception will be thrown if netty is not able to use native transport. The native transport is more efficient especially when there is a high number of connections. | | `spark.cosmos.http.connectionPoolSize` | `1000` | Gateway mode connection pool size | | `spark.cosmos.read.forceEventualConsistency` | `true` | Makes the client use Eventual consistency for read operations instead of using the default account level consistency | | `spark.cosmos.applicationName` | None | Application name | From 0a9b3c7b6bfa7ab540b63b80f4343655a0513008 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Mon, 22 Apr 2024 23:22:22 +0000 Subject: [PATCH 08/10] Changelogs --- sdk/cosmos/azure-cosmos-spark_3-1_2-12/CHANGELOG.md | 1 + sdk/cosmos/azure-cosmos-spark_3-2_2-12/CHANGELOG.md | 1 + sdk/cosmos/azure-cosmos-spark_3-3_2-12/CHANGELOG.md | 1 + sdk/cosmos/azure-cosmos-spark_3-4_2-12/CHANGELOG.md | 1 + sdk/cosmos/azure-cosmos-spark_3-5_2-12/CHANGELOG.md | 1 + 5 files changed, 5 insertions(+) diff --git a/sdk/cosmos/azure-cosmos-spark_3-1_2-12/CHANGELOG.md b/sdk/cosmos/azure-cosmos-spark_3-1_2-12/CHANGELOG.md index f7af0a563cc3..cbc45aafc3ba 100644 --- a/sdk/cosmos/azure-cosmos-spark_3-1_2-12/CHANGELOG.md +++ b/sdk/cosmos/azure-cosmos-spark_3-1_2-12/CHANGELOG.md @@ -3,6 +3,7 @@ ### 4.30.0-beta.1 (Unreleased) #### Features Added +* Added capability to use (and enforce) native netty transport. The native transport is more efficient - esepcially when the number of TCP connections being used is high. - See [PR 39834](https://github.com/Azure/azure-sdk-for-java/pull/39834) #### Breaking Changes diff --git a/sdk/cosmos/azure-cosmos-spark_3-2_2-12/CHANGELOG.md b/sdk/cosmos/azure-cosmos-spark_3-2_2-12/CHANGELOG.md index 334abf48752b..dbcfcc5bd793 100644 --- a/sdk/cosmos/azure-cosmos-spark_3-2_2-12/CHANGELOG.md +++ b/sdk/cosmos/azure-cosmos-spark_3-2_2-12/CHANGELOG.md @@ -3,6 +3,7 @@ ### 4.30.0-beta.1 (Unreleased) #### Features Added +* Added capability to use (and enforce) native netty transport. The native transport is more efficient - esepcially when the number of TCP connections being used is high. - See [PR 39834](https://github.com/Azure/azure-sdk-for-java/pull/39834) #### Breaking Changes diff --git a/sdk/cosmos/azure-cosmos-spark_3-3_2-12/CHANGELOG.md b/sdk/cosmos/azure-cosmos-spark_3-3_2-12/CHANGELOG.md index 5c508f97fd96..d8fc7a2b7d6a 100644 --- a/sdk/cosmos/azure-cosmos-spark_3-3_2-12/CHANGELOG.md +++ b/sdk/cosmos/azure-cosmos-spark_3-3_2-12/CHANGELOG.md @@ -3,6 +3,7 @@ ### 4.30.0-beta.1 (Unreleased) #### Features Added +* Added capability to use (and enforce) native netty transport. The native transport is more efficient - esepcially when the number of TCP connections being used is high. - See [PR 39834](https://github.com/Azure/azure-sdk-for-java/pull/39834) #### Breaking Changes diff --git a/sdk/cosmos/azure-cosmos-spark_3-4_2-12/CHANGELOG.md b/sdk/cosmos/azure-cosmos-spark_3-4_2-12/CHANGELOG.md index 2c1df4494966..3e0ce638e8f9 100644 --- a/sdk/cosmos/azure-cosmos-spark_3-4_2-12/CHANGELOG.md +++ b/sdk/cosmos/azure-cosmos-spark_3-4_2-12/CHANGELOG.md @@ -3,6 +3,7 @@ ### 4.30.0-beta.1 (Unreleased) #### Features Added +* Added capability to use (and enforce) native netty transport. The native transport is more efficient - esepcially when the number of TCP connections being used is high. - See [PR 39834](https://github.com/Azure/azure-sdk-for-java/pull/39834) #### Breaking Changes diff --git a/sdk/cosmos/azure-cosmos-spark_3-5_2-12/CHANGELOG.md b/sdk/cosmos/azure-cosmos-spark_3-5_2-12/CHANGELOG.md index b77ad802f00d..a7c102d98bb1 100644 --- a/sdk/cosmos/azure-cosmos-spark_3-5_2-12/CHANGELOG.md +++ b/sdk/cosmos/azure-cosmos-spark_3-5_2-12/CHANGELOG.md @@ -3,6 +3,7 @@ ### 4.30.0-beta.1 (Unreleased) #### Features Added +* Added capability to use (and enforce) native netty transport. The native transport is more efficient - esepcially when the number of TCP connections being used is high. - See [PR 39834](https://github.com/Azure/azure-sdk-for-java/pull/39834) #### Breaking Changes From aeaad1804deda7aa944060f8bdc05a5b040cffe4 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Mon, 22 Apr 2024 23:25:15 +0000 Subject: [PATCH 09/10] Update CHANGELOG.md --- sdk/cosmos/azure-cosmos-spark_3-5_2-12/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/cosmos/azure-cosmos-spark_3-5_2-12/CHANGELOG.md b/sdk/cosmos/azure-cosmos-spark_3-5_2-12/CHANGELOG.md index a7c102d98bb1..0fa0f97f4731 100644 --- a/sdk/cosmos/azure-cosmos-spark_3-5_2-12/CHANGELOG.md +++ b/sdk/cosmos/azure-cosmos-spark_3-5_2-12/CHANGELOG.md @@ -4,7 +4,7 @@ #### Features Added * Added capability to use (and enforce) native netty transport. The native transport is more efficient - esepcially when the number of TCP connections being used is high. - See [PR 39834](https://github.com/Azure/azure-sdk-for-java/pull/39834) - + #### Breaking Changes #### Bugs Fixed From 0fedcea7a4d22513dc26e20ed0b47e437faf74a7 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Mon, 22 Apr 2024 23:25:26 +0000 Subject: [PATCH 10/10] Update CHANGELOG.md --- sdk/cosmos/azure-cosmos-spark_3-5_2-12/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/cosmos/azure-cosmos-spark_3-5_2-12/CHANGELOG.md b/sdk/cosmos/azure-cosmos-spark_3-5_2-12/CHANGELOG.md index 0fa0f97f4731..a7c102d98bb1 100644 --- a/sdk/cosmos/azure-cosmos-spark_3-5_2-12/CHANGELOG.md +++ b/sdk/cosmos/azure-cosmos-spark_3-5_2-12/CHANGELOG.md @@ -4,7 +4,7 @@ #### Features Added * Added capability to use (and enforce) native netty transport. The native transport is more efficient - esepcially when the number of TCP connections being used is high. - See [PR 39834](https://github.com/Azure/azure-sdk-for-java/pull/39834) - + #### Breaking Changes #### Bugs Fixed