diff --git a/sdk/cosmos/azure-cosmos-spark_3-3_2-12/CHANGELOG.md b/sdk/cosmos/azure-cosmos-spark_3-3_2-12/CHANGELOG.md index acaee40c6cd9..0d8f2435a0d0 100644 --- a/sdk/cosmos/azure-cosmos-spark_3-3_2-12/CHANGELOG.md +++ b/sdk/cosmos/azure-cosmos-spark_3-3_2-12/CHANGELOG.md @@ -9,6 +9,7 @@ #### Bugs Fixed * Improved partition planning performance for change feed with large number of feed ranges. - See [PR 49086](https://github.com/Azure/azure-sdk-for-java/pull/49086) * Fixed `UnsupportedOperationException` when using `readManyByPartitionKeys` for empty pages. - See [PR 49311](https://github.com/Azure/azure-sdk-for-java/pull/49311) +* Fixed `OperationCancelledException` ("End-to-end timeout hit") on sparse cross-partition queries by opting into the SDK's `allowEmptyPages` behavior, so the per-page timeout applies per page instead of being exceeded by serial empty-page drains. Note: this surfaces one iterator callback per empty page where previously a single callback could drain many. - See [PR 49276](https://github.com/Azure/azure-sdk-for-java/pull/49276) #### Other Changes diff --git a/sdk/cosmos/azure-cosmos-spark_3-4_2-12/CHANGELOG.md b/sdk/cosmos/azure-cosmos-spark_3-4_2-12/CHANGELOG.md index 09b19a4119c5..573cdec8ef58 100644 --- a/sdk/cosmos/azure-cosmos-spark_3-4_2-12/CHANGELOG.md +++ b/sdk/cosmos/azure-cosmos-spark_3-4_2-12/CHANGELOG.md @@ -9,6 +9,7 @@ #### Bugs Fixed * Improved partition planning performance for change feed with large number of feed ranges. - See [PR 49086](https://github.com/Azure/azure-sdk-for-java/pull/49086) * Fixed `UnsupportedOperationException` when using `readManyByPartitionKeys` for empty pages. - See [PR 49311](https://github.com/Azure/azure-sdk-for-java/pull/49311) +* Fixed `OperationCancelledException` ("End-to-end timeout hit") on sparse cross-partition queries by opting into the SDK's `allowEmptyPages` behavior, so the per-page timeout applies per page instead of being exceeded by serial empty-page drains. Note: this surfaces one iterator callback per empty page where previously a single callback could drain many. - See [PR 49276](https://github.com/Azure/azure-sdk-for-java/pull/49276) #### Other Changes diff --git a/sdk/cosmos/azure-cosmos-spark_3-5_2-12/CHANGELOG.md b/sdk/cosmos/azure-cosmos-spark_3-5_2-12/CHANGELOG.md index a36a8521c71b..1e5a0712ad08 100644 --- a/sdk/cosmos/azure-cosmos-spark_3-5_2-12/CHANGELOG.md +++ b/sdk/cosmos/azure-cosmos-spark_3-5_2-12/CHANGELOG.md @@ -9,6 +9,7 @@ #### Bugs Fixed * Improved partition planning performance for change feed with large number of feed ranges. - See [PR 49086](https://github.com/Azure/azure-sdk-for-java/pull/49086) * Fixed `UnsupportedOperationException` when using `readManyByPartitionKeys` for empty pages. - See [PR 49311](https://github.com/Azure/azure-sdk-for-java/pull/49311) +* Fixed `OperationCancelledException` ("End-to-end timeout hit") on sparse cross-partition queries by opting into the SDK's `allowEmptyPages` behavior, so the per-page timeout applies per page instead of being exceeded by serial empty-page drains. Note: this surfaces one iterator callback per empty page where previously a single callback could drain many. - See [PR 49276](https://github.com/Azure/azure-sdk-for-java/pull/49276) #### Other Changes diff --git a/sdk/cosmos/azure-cosmos-spark_3-5_2-13/CHANGELOG.md b/sdk/cosmos/azure-cosmos-spark_3-5_2-13/CHANGELOG.md index 5d46eb30dde5..09a7cb94ef5a 100644 --- a/sdk/cosmos/azure-cosmos-spark_3-5_2-13/CHANGELOG.md +++ b/sdk/cosmos/azure-cosmos-spark_3-5_2-13/CHANGELOG.md @@ -9,6 +9,7 @@ #### Bugs Fixed * Improved partition planning performance for change feed with large number of feed ranges. - See [PR 49086](https://github.com/Azure/azure-sdk-for-java/pull/49086) * Fixed `UnsupportedOperationException` when using `readManyByPartitionKeys` for empty pages. - See [PR 49311](https://github.com/Azure/azure-sdk-for-java/pull/49311) +* Fixed `OperationCancelledException` ("End-to-end timeout hit") on sparse cross-partition queries by opting into the SDK's `allowEmptyPages` behavior, so the per-page timeout applies per page instead of being exceeded by serial empty-page drains. Note: this surfaces one iterator callback per empty page where previously a single callback could drain many. - See [PR 49276](https://github.com/Azure/azure-sdk-for-java/pull/49276) #### Other Changes diff --git a/sdk/cosmos/azure-cosmos-spark_3/src/main/scala/com/azure/cosmos/spark/ItemsPartitionReader.scala b/sdk/cosmos/azure-cosmos-spark_3/src/main/scala/com/azure/cosmos/spark/ItemsPartitionReader.scala index 44027bafbe7e..7739485bbc91 100644 --- a/sdk/cosmos/azure-cosmos-spark_3/src/main/scala/com/azure/cosmos/spark/ItemsPartitionReader.scala +++ b/sdk/cosmos/azure-cosmos-spark_3/src/main/scala/com/azure/cosmos/spark/ItemsPartitionReader.scala @@ -45,6 +45,14 @@ private case class ItemsPartitionReader .getCosmosQueryRequestOptionsAccessor .disallowQueryPlanRetrieval(new CosmosQueryRequestOptions()) + // Bubble empty pages up to the iterator so the per-page end-to-end timeout + // applies to each individual page rather than being exceeded by serial + // empty-page drains inside ParallelDocumentQueryExecutionContext. + ImplementationBridgeHelpers + .CosmosQueryRequestOptionsHelper + .getCosmosQueryRequestOptionsAccessor + .setAllowEmptyPages(queryOptions, true) + private val readConfig = CosmosReadConfig.parseCosmosReadConfig(config) ThroughputControlHelper.populateThroughputControlGroupName( ImplementationBridgeHelpers diff --git a/sdk/cosmos/azure-cosmos-spark_3/src/test/scala/com/azure/cosmos/spark/TransientIOErrorsRetryingIteratorSpec.scala b/sdk/cosmos/azure-cosmos-spark_3/src/test/scala/com/azure/cosmos/spark/TransientIOErrorsRetryingIteratorSpec.scala index b8400fdd3eff..9185e7529d35 100644 --- a/sdk/cosmos/azure-cosmos-spark_3/src/test/scala/com/azure/cosmos/spark/TransientIOErrorsRetryingIteratorSpec.scala +++ b/sdk/cosmos/azure-cosmos-spark_3/src/test/scala/com/azure/cosmos/spark/TransientIOErrorsRetryingIteratorSpec.scala @@ -180,6 +180,68 @@ class TransientIOErrorsRetryingIteratorSpec extends UnitSpec with BasicLoggingTr factoryCallCount.get shouldEqual 1 } + "TransientIOErrors" should "drain long runs of empty pages without hitting the end-to-end timeout" in { + // Regression test for the empty-page drain scenario: when the SDK is configured with + // emptyPagesAllowed=true the iterator must surface many consecutive empty + // pages without busy-waiting beyond the per-page end-to-end timeout. Even + // with hundreds of empty pages followed by real data, the iterator should + // return all real rows. + val emptyLeadingPages = 200 + val realPages = 5 + val totalPages = emptyLeadingPages + realPages + val iterator = new TransientIOErrorsRetryingIterator( + continuationToken => generateMockedCosmosPagedFluxWithEmptyPrefix( + continuationToken, totalPages, emptyLeadingPages), + pageSize, + 1, + None, + None + ) + iterator.maxRetryIntervalInMs = 5 + + // 2 producers (Left/Right) each emit realPages * pageSize rows + iterator.count(_ => true) shouldEqual (realPages * pageSize * 2) + } + + private def generateMockedCosmosPagedFluxWithEmptyPrefix + ( + continuationToken: String, + initialPageCount: Int, + leadingEmptyPageCount: Int + ) = { + + val leftProducer = generateFeedResponseFluxWithEmptyPrefix( + "Left", initialPageCount, leadingEmptyPageCount, Option.apply(continuationToken)) + val rightProducer = generateFeedResponseFluxWithEmptyPrefix( + "Right", initialPageCount, leadingEmptyPageCount, Option.apply(continuationToken)) + val toBeMerged = Array(leftProducer, rightProducer).toIterable.asJava + val mergedFlux = Flux.mergeSequential(toBeMerged, 1, 2) + UtilBridgeInternal.createCosmosPagedFlux(_ => mergedFlux) + } + + private def generateFeedResponseFluxWithEmptyPrefix + ( + prefix: String, + pageCount: Int, + leadingEmptyPageCount: Int, + requestContinuationToken: Option[String] + ): Flux[FeedResponse[SparkRowItem]] = { + + // generateFeedResponse uses documentStartIndex=-1 as the "emit an empty page" sentinel. + val emptyPageSentinel = -1 + val firstDataPageStartIndex = 1 + + val responses = Array.range(1, pageCount + 1) + .map(i => generateFeedResponse( + prefix, + i, + if (i <= leadingEmptyPageCount) emptyPageSentinel else firstDataPageStartIndex)) + .filter(response => requestContinuationToken.isEmpty || + requestContinuationToken.get < response.getContinuationToken) + + Flux.fromArray(responses) + } + private val objectMapper = new ObjectMapper @throws[JsonProcessingException] diff --git a/sdk/cosmos/azure-cosmos-spark_4-0_2-13/CHANGELOG.md b/sdk/cosmos/azure-cosmos-spark_4-0_2-13/CHANGELOG.md index 9b971d9786c0..6597a56d7fbd 100644 --- a/sdk/cosmos/azure-cosmos-spark_4-0_2-13/CHANGELOG.md +++ b/sdk/cosmos/azure-cosmos-spark_4-0_2-13/CHANGELOG.md @@ -9,6 +9,7 @@ #### Bugs Fixed * Improved partition planning performance for change feed with large number of feed ranges. - See [PR 49086](https://github.com/Azure/azure-sdk-for-java/pull/49086) * Fixed `UnsupportedOperationException` when using `readManyByPartitionKeys` for empty pages. - See [PR 49311](https://github.com/Azure/azure-sdk-for-java/pull/49311) +* Fixed `OperationCancelledException` ("End-to-end timeout hit") on sparse cross-partition queries by opting into the SDK's `allowEmptyPages` behavior, so the per-page timeout applies per page instead of being exceeded by serial empty-page drains. Note: this surfaces one iterator callback per empty page where previously a single callback could drain many. - See [PR 49276](https://github.com/Azure/azure-sdk-for-java/pull/49276) #### Other Changes diff --git a/sdk/cosmos/azure-cosmos-spark_4-1_2-13/CHANGELOG.md b/sdk/cosmos/azure-cosmos-spark_4-1_2-13/CHANGELOG.md index b3c35541850b..40281103598a 100644 --- a/sdk/cosmos/azure-cosmos-spark_4-1_2-13/CHANGELOG.md +++ b/sdk/cosmos/azure-cosmos-spark_4-1_2-13/CHANGELOG.md @@ -9,6 +9,7 @@ #### Bugs Fixed * Improved partition planning performance for change feed with large number of feed ranges. - See [PR 49086](https://github.com/Azure/azure-sdk-for-java/pull/49086) * Fixed `UnsupportedOperationException` when using `readManyByPartitionKeys` for empty pages. - See [PR 49311](https://github.com/Azure/azure-sdk-for-java/pull/49311) +* Fixed `OperationCancelledException` ("End-to-end timeout hit") on sparse cross-partition queries by opting into the SDK's `allowEmptyPages` behavior, so the per-page timeout applies per page instead of being exceeded by serial empty-page drains. Note: this surfaces one iterator callback per empty page where previously a single callback could drain many. - See [PR 49276](https://github.com/Azure/azure-sdk-for-java/pull/49276) #### Other Changes diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosContainerChangeFeedTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosContainerChangeFeedTest.java index 7c8418ed8992..f7bfe4c39356 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosContainerChangeFeedTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosContainerChangeFeedTest.java @@ -332,8 +332,14 @@ public void asyncChangeFeed_fromBeginning_incremental_forLogicalPartition() thro } } - @Test(groups = { "emulator" }, dataProvider = "changeFeedQueryPrefetchingDataProvider", timeOut = TIMEOUT) + @Test(groups = { "emulator" }, dataProvider = "changeFeedQueryPrefetchingDataProvider", + timeOut = TIMEOUT, retryAnalyzer = FlakyTestRetryAnalyzer.class) public void asyncChangeFeedPrefetching(ChangeFeedMode changeFeedMode) throws Exception { + // Note on shape: this test verifies Reactor's prefetch behavior on the change-feed + // byPage stream. The two fire-and-forget `.subscribe()` calls + `Thread.sleep(3000)` + // are intentional — they exercise the prefetch path without backpressure-bounded + // collection. retryAnalyzer = FlakyTestRetryAnalyzer absorbs occasional slow-runner + // jitter (Windows EmulatorTcp Java 8 can take >3s to deliver the first 3 pages). this.createContainer( (cp) -> { if (changeFeedMode.equals(ChangeFeedMode.INCREMENTAL)) { diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/CosmosChangeFeedRequestOptionsWithPagedFluxOptionsTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/CosmosChangeFeedRequestOptionsWithPagedFluxOptionsTest.java new file mode 100644 index 000000000000..412af0ab0167 --- /dev/null +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/CosmosChangeFeedRequestOptionsWithPagedFluxOptionsTest.java @@ -0,0 +1,189 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos.implementation; + +import com.azure.cosmos.CosmosItemSerializer; +import com.azure.cosmos.implementation.changefeed.common.ChangeFeedMode; +import com.azure.cosmos.implementation.changefeed.common.ChangeFeedStartFromInternal; +import com.azure.cosmos.implementation.changefeed.common.ChangeFeedStateV1; +import com.azure.cosmos.implementation.feedranges.FeedRangeEpkImpl; +import com.azure.cosmos.models.CosmosChangeFeedRequestOptions; +import com.azure.cosmos.models.ModelBridgeInternal; +import org.testng.annotations.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Unit tests for the paged-flux pull continuation path on + * {@link CosmosChangeFeedRequestOptions#withCosmosPagedFluxOptions(CosmosPagedFluxOptions)} (package-visible via + * {@link ModelBridgeInternal#getEffectiveChangeFeedRequestOptions(CosmosChangeFeedRequestOptions, CosmosPagedFluxOptions)}). + * + *
That method silently builds a brand-new {@code CosmosChangeFeedRequestOptionsImpl} when the caller supplies a
+ * continuation token via {@link CosmosPagedFluxOptions}, so any field NOT explicitly copied is dropped. These tests
+ * lock in the propagation of fields whose loss would silently break a feature.
+ */
+public class CosmosChangeFeedRequestOptionsWithPagedFluxOptionsTest {
+
+ @Test(groups = { "unit" })
+ public void endLSN_isPropagated_whenContinuationTokenSupplied() {
+ // Locks in the bounded-change-feed contract across a byPage(savedContinuation) round-trip:
+ // a caller who set endLSN=42 must continue to see iteration bounded by LSN 42 after resume.
+ // Before the inheritNonContinuationFieldsFrom fix, endLSN was silently dropped on the rebuild
+ // path, turning a bounded change feed into an unbounded one.
+ CosmosChangeFeedRequestOptions src = CosmosChangeFeedRequestOptions
+ .createForProcessingFromBeginning(FeedRangeEpkImpl.forFullRange());
+ ImplementationBridgeHelpers.CosmosChangeFeedRequestOptionsHelper
+ .getCosmosChangeFeedRequestOptionsAccessor()
+ .setEndLSN(src, 42L);
+
+ CosmosPagedFluxOptions pagedFluxOptions = new CosmosPagedFluxOptions();
+ pagedFluxOptions.setRequestContinuation(buildContinuationToken());
+
+ CosmosChangeFeedRequestOptions effective = ModelBridgeInternal
+ .getEffectiveChangeFeedRequestOptions(src, pagedFluxOptions);
+
+ assertThat(ImplementationBridgeHelpers.CosmosChangeFeedRequestOptionsHelper
+ .getCosmosChangeFeedRequestOptionsAccessor()
+ .getEndLSN(effective))
+ .describedAs("endLSN must survive the paged-flux pull continuation rebuild")
+ .isEqualTo(42L);
+ }
+
+ @Test(groups = { "unit" })
+ public void customSerializer_isPropagated_whenContinuationTokenSupplied() {
+ // Locks in custom-serializer preservation across a byPage(savedContinuation) round-trip:
+ // a caller who registered a custom CosmosItemSerializer must continue to see items
+ // deserialized through that serializer after resume. Before the inheritNonContinuationFieldsFrom
+ // fix, the customSerializer was silently dropped on the rebuild path, falling back to the
+ // SDK's internal default serializer and potentially producing wrong field values.
+ CosmosItemSerializer sentinel = new CosmosItemSerializer() {
+ @Override
+ public The four fields encoded in the continuation token itself ({@code continuationState},
+ * {@code feedRangeInternal}, {@code mode}, {@code startFromInternal}) are intentionally NOT
+ * copied — they describe "where to resume from" and must come from the token, not the caller's
+ * pre-resume options. Every other field IS copied so the caller's configuration (endLSN,
+ * customSerializer, excludeRegions, read-consistency strategy, throughput-control group,
+ * diagnostic thresholds, etc.) survives the {@code byPage(savedContinuation)} round-trip.
+ *
+ * Maintenance contract: when a new field is added to this class, decide whether the
+ * continuation token encodes it. If not (the common case for caller-supplied configuration),
+ * propagate it here.
+ */
+ public void inheritNonContinuationFieldsFrom(CosmosChangeFeedRequestOptionsImpl source) {
+ // continuationState, feedRangeInternal, mode, startFromInternal:
+ // intentionally NOT copied (encoded in the continuation token itself).
+ // collectionRid IS preserved: it lives on the impl but is not embedded in the
+ // continuation token (the token's separate containerRid lives on ChangeFeedStateV1).
+ // The rest-of-SDK clone path (RxDocumentClientImpl.queryDocumentChangeFeedFromPagedFluxInternal
+ // -> accessor.clone -> copy ctor) preserves collectionRid; we match that here.
+ this.maxItemCount = source.maxItemCount;
+ this.maxPrefetchPageCount = source.maxPrefetchPageCount;
+ this.isSplitHandlingDisabled = source.isSplitHandlingDisabled;
+ this.quotaInfoEnabled = source.quotaInfoEnabled;
+ this.throughputControlGroupName = source.throughputControlGroupName;
+ // Merge source's caller-supplied headers into this.customOptions WITHOUT clobbering
+ // headers the token-driven constructor already populated (e.g., FULL_FIDELITY mode
+ // adds CHANGE_FEED_WIRE_FORMAT_VERSION). Overwriting would drop those required
+ // mode-derived headers and produce inconsistent requests (token mode vs. source headers).
+ if (source.customOptions != null) {
+ if (this.customOptions == null) {
+ this.customOptions = new HashMap<>(source.customOptions);
+ } else {
+ for (Map.Entry