Skip to content

PopulateReplicaValidationFlagProperlyOnRequest #32022

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
xinlian12 merged 4 commits into from
Nov 9, 2022
Merged

PopulateReplicaValidationFlagProperlyOnRequest #32022

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
02d1d5a
set requestContext flag properly
Nov 8, 2022
da6511a
add changelog
Nov 8, 2022
4147a11
update changelog
Nov 8, 2022
2cebca2
update changelog
Nov 8, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions sdk/cosmos/azure-cosmos/CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
#### Bugs Fixed
* Fixed a rare race condition for `query plan` cache exceeding the allowed size limit - See [PR 31859](https://github.com/Azure/azure-sdk-for-java/pull/31859)
* Added improvement in `RntbdClientChannelHealthChecker` for detecting continuous transit timeout. - See [PR 31544](https://github.com/Azure/azure-sdk-for-java/pull/31544)
* Fixed an issue in replica validation where addresses may have not sorted properly when replica validation is enabled. - See [PR 32022](https://github.com/Azure/azure-sdk-for-java/pull/32022)

#### Other Changes
* Shaded `MurmurHash3` of apache `commons-codec` to enable removing of the `guava` dependency - CVE-2020-8908 - See [PR 31761](https://github.com/Azure/azure-sdk-for-java/pull/31761)
Expand Down
2 changes: 1 addition & 1 deletion ...e-cosmos/src/main/java/com/azure/cosmos/implementation/DocumentServiceRequestContext.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ public class DocumentServiceRequestContext implements Cloneable {
public volatile CosmosDiagnostics cosmosDiagnostics;
public volatile String resourcePhysicalAddress;
public volatile String throughputControlCycleId;
public volatile boolean replicaAddressValidationEnabled;
public volatile boolean replicaAddressValidationEnabled = Configs.isReplicaAddressValidationEnabled();
private final Set<Uri> failedEndpoints = ConcurrentHashMap.newKeySet();

public DocumentServiceRequestContext() {}
Expand Down
145 changes: 78 additions & 67 deletions .../test/java/com/azure/cosmos/implementation/directconnectivity/AddressEnumeratorTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,106 +29,117 @@ public class AddressEnumeratorTests {

@Test(groups = "unit")
public void replicaAddressValidationEnabledComparatorTests() throws NoSuchMethodException, NoSuchFieldException, IllegalAccessException, InvocationTargetException {
Method sortAddressesMethod = AddressEnumerator.class.getDeclaredMethod("sortAddresses", List.class, RxDocumentServiceRequest.class);
sortAddressesMethod.setAccessible(true);
try {
System.setProperty("COSMOS.REPLICA_ADDRESS_VALIDATION_ENABLED", "true");
Method sortAddressesMethod = AddressEnumerator.class.getDeclaredMethod("sortAddresses", List.class, RxDocumentServiceRequest.class);
sortAddressesMethod.setAccessible(true);

// set a different health status to each endpoint to test the sorting logic
Uri testUri1 = new Uri("https://127.0.0.1:1");
assertThat(testUri1.getHealthStatus()).isEqualTo(Unknown);
// set a different health status to each endpoint to test the sorting logic
Uri testUri1 = new Uri("https://127.0.0.1:1");
assertThat(testUri1.getHealthStatus()).isEqualTo(Unknown);

Uri testUri2 = new Uri("https://127.0.0.1:2");
testUri2.setConnected();
assertThat(testUri2.getHealthStatus()).isEqualTo(Connected);
Uri testUri2 = new Uri("https://127.0.0.1:2");
testUri2.setConnected();
assertThat(testUri2.getHealthStatus()).isEqualTo(Connected);

Uri testUri3 = new Uri("https://127.0.0.1:3");
testUri3.setUnhealthy();
testUri3.setRefreshed();
assertThat(testUri3.getHealthStatus()).isEqualTo(UnhealthyPending);
Uri testUri3 = new Uri("https://127.0.0.1:3");
testUri3.setUnhealthy();
testUri3.setRefreshed();
assertThat(testUri3.getHealthStatus()).isEqualTo(UnhealthyPending);

Uri testUri4 = new Uri("https://127.0.0.1:4");
testUri4.setUnhealthy();
assertThat(testUri4.getHealthStatus()).isEqualTo(Unhealthy);
Uri testUri4 = new Uri("https://127.0.0.1:4");
testUri4.setUnhealthy();
assertThat(testUri4.getHealthStatus()).isEqualTo(Unhealthy);

RxDocumentServiceRequest requestMock = Mockito.mock(RxDocumentServiceRequest.class);
requestMock.requestContext = new DocumentServiceRequestContext();
requestMock.requestContext.replicaAddressValidationEnabled = true;
RxDocumentServiceRequest requestMock = Mockito.mock(RxDocumentServiceRequest.class);
requestMock.requestContext = new DocumentServiceRequestContext();

// when replicaAddressValidation is enabled, we prefer Connected/Unknown > UnhealthyPending > Unhealthy
List<SortAddressesTestScenario> testScenarios = Arrays.asList(
// when replicaAddressValidation is enabled, we prefer Connected/Unknown > UnhealthyPending > Unhealthy
List<SortAddressesTestScenario> testScenarios = Arrays.asList(
new SortAddressesTestScenario(Arrays.asList(testUri1, testUri2), Arrays.asList(testUri1, testUri2)), // unknown, connected -> unknown, connected
new SortAddressesTestScenario(Arrays.asList(testUri2, testUri1), Arrays.asList(testUri2, testUri1)), // connected, unknown -> connected, unknown
new SortAddressesTestScenario(Arrays.asList(testUri3, testUri2), Arrays.asList(testUri2, testUri3)), // unhealthyPending, connected -> connected, unhealthyPending
new SortAddressesTestScenario(Arrays.asList(testUri4, testUri1), Arrays.asList(testUri1, testUri4)), // unhealthy, unknown -> unknown, unhealthy
new SortAddressesTestScenario(Arrays.asList(testUri4, testUri3), Arrays.asList(testUri3, testUri4))); // unhealthy, unhealthyPending -> unhealthyPending, unhealthy

for (SortAddressesTestScenario testScenario : testScenarios) {
System.out.println("Test scenario, comparing " + testScenario.getAddresses());
for (Uri uri : testScenario.getAddresses()) {
this.setTimestamp(uri, Instant.now());
}
List<Uri> sortedAddresses =
for (SortAddressesTestScenario testScenario : testScenarios) {
System.out.println("Test scenario, comparing " + testScenario.getAddresses());
for (Uri uri : testScenario.getAddresses()) {
this.setTimestamp(uri, Instant.now());
}
List<Uri> sortedAddresses =
(List<Uri>) sortAddressesMethod.invoke(null, testScenario.getAddresses(), requestMock);
assertThat(sortedAddresses).containsExactlyElementsOf(testScenario.expectedAddresses);
}
assertThat(sortedAddresses).containsExactlyElementsOf(testScenario.expectedAddresses);
}

System.out.println("Test scenario: comparing when unhealthyPending roll into healthy status after 1 min");
setTimestamp(testUri3, Instant.now().minusMillis(Duration.ofMinutes(2).toMillis()));
List<Uri> sortedAddresses = (List<Uri>) sortAddressesMethod.invoke(null, Arrays.asList(testUri3, testUri2), requestMock);
assertThat(sortedAddresses).containsExactlyElementsOf(Arrays.asList(testUri3, testUri2));
System.out.println("Test scenario: comparing when unhealthyPending roll into healthy status after 1 min");
setTimestamp(testUri3, Instant.now().minusMillis(Duration.ofMinutes(2).toMillis()));
List<Uri> sortedAddresses = (List<Uri>) sortAddressesMethod.invoke(null, Arrays.asList(testUri3, testUri2), requestMock);
assertThat(sortedAddresses).containsExactlyElementsOf(Arrays.asList(testUri3, testUri2));

System.out.println("Test scenario: comparing when there is failedEndpoints marked in request context");
requestMock.requestContext.addToFailedEndpoints(new GoneException("Test"), testUri2);
sortedAddresses = (List<Uri>) sortAddressesMethod.invoke(null, Arrays.asList(testUri4, testUri2), requestMock);
assertThat(sortedAddresses).containsExactlyElementsOf(Arrays.asList(testUri4, testUri2));
} finally {
System.clearProperty("COSMOS.REPLICA_ADDRESS_VALIDATION_ENABLED");
}

System.out.println("Test scenario: comparing when there is failedEndpoints marked in request context");
requestMock.requestContext.addToFailedEndpoints(new GoneException("Test"), testUri2);
sortedAddresses = (List<Uri>) sortAddressesMethod.invoke(null, Arrays.asList(testUri4, testUri2), requestMock);
assertThat(sortedAddresses).containsExactlyElementsOf(Arrays.asList(testUri4, testUri2));
}

@Test(groups = "unit")
public void replicaAddressValidationDisabledComparatorTests() throws NoSuchMethodException, NoSuchFieldException, IllegalAccessException, InvocationTargetException {
Method sortAddressesMethod = AddressEnumerator.class.getDeclaredMethod("sortAddresses", List.class, RxDocumentServiceRequest.class);
sortAddressesMethod.setAccessible(true);
try {
System.setProperty("COSMOS.REPLICA_ADDRESS_VALIDATION_ENABLED", "false");
Method sortAddressesMethod = AddressEnumerator.class.getDeclaredMethod("sortAddresses", List.class, RxDocumentServiceRequest.class);
sortAddressesMethod.setAccessible(true);

// set a different health status to each endpoint to test the sorting logic
Uri testUri1 = new Uri("https://127.0.0.1:1");
assertThat(testUri1.getHealthStatus()).isEqualTo(Unknown);
// set a different health status to each endpoint to test the sorting logic
Uri testUri1 = new Uri("https://127.0.0.1:1");
assertThat(testUri1.getHealthStatus()).isEqualTo(Unknown);

Uri testUri2 = new Uri("https://127.0.0.1:2");
testUri2.setConnected();
assertThat(testUri2.getHealthStatus()).isEqualTo(Connected);
Uri testUri2 = new Uri("https://127.0.0.1:2");
testUri2.setConnected();
assertThat(testUri2.getHealthStatus()).isEqualTo(Connected);

Uri testUri3 = new Uri("https://127.0.0.1:3");
testUri3.setUnhealthy();
testUri3.setRefreshed();
assertThat(testUri3.getHealthStatus()).isEqualTo(UnhealthyPending);
Uri testUri3 = new Uri("https://127.0.0.1:3");
testUri3.setUnhealthy();
testUri3.setRefreshed();
assertThat(testUri3.getHealthStatus()).isEqualTo(UnhealthyPending);

Uri testUri4 = new Uri("https://127.0.0.1:4");
testUri4.setUnhealthy();
assertThat(testUri4.getHealthStatus()).isEqualTo(Unhealthy);
Uri testUri4 = new Uri("https://127.0.0.1:4");
testUri4.setUnhealthy();
assertThat(testUri4.getHealthStatus()).isEqualTo(Unhealthy);

RxDocumentServiceRequest requestMock = Mockito.mock(RxDocumentServiceRequest.class);
requestMock.requestContext = new DocumentServiceRequestContext();
RxDocumentServiceRequest requestMock = Mockito.mock(RxDocumentServiceRequest.class);
requestMock.requestContext = new DocumentServiceRequestContext();

// when replicaAddressValidation is enabled, we prefer Connected/Unknown/UnhealthyPending > Unhealthy
List<SortAddressesTestScenario> testScenarios = Arrays.asList(
// when replicaAddressValidation is enabled, we prefer Connected/Unknown/UnhealthyPending > Unhealthy
List<SortAddressesTestScenario> testScenarios = Arrays.asList(
new SortAddressesTestScenario(Arrays.asList(testUri1, testUri2), Arrays.asList(testUri1, testUri2)), // unknown, connected -> unknown, connected
new SortAddressesTestScenario(Arrays.asList(testUri2, testUri1), Arrays.asList(testUri2, testUri1)), // connected, unknown -> connected, unknown
new SortAddressesTestScenario(Arrays.asList(testUri3, testUri2), Arrays.asList(testUri3, testUri2)), // unhealthyPending, connected -> unhealthyPending, connected
new SortAddressesTestScenario(Arrays.asList(testUri4, testUri1), Arrays.asList(testUri1, testUri4)), // unhealthy, unknown -> unknown, unhealthy
new SortAddressesTestScenario(Arrays.asList(testUri4, testUri3), Arrays.asList(testUri3, testUri4))); // unhealthy, unhealthyPending -> unhealthyPending, unhealthy

for (SortAddressesTestScenario testScenario : testScenarios) {
System.out.println("Test scenario, comparing " + testScenario.getAddresses());
for (Uri uri : testScenario.getAddresses()) {
this.setTimestamp(uri, Instant.now());
}
List<Uri> sortedAddresses =
for (SortAddressesTestScenario testScenario : testScenarios) {
System.out.println("Test scenario, comparing " + testScenario.getAddresses());
for (Uri uri : testScenario.getAddresses()) {
this.setTimestamp(uri, Instant.now());
}
List<Uri> sortedAddresses =
(List<Uri>) sortAddressesMethod.invoke(null, testScenario.getAddresses(), requestMock);
assertThat(sortedAddresses).containsExactlyElementsOf(testScenario.expectedAddresses);
assertThat(sortedAddresses).containsExactlyElementsOf(testScenario.expectedAddresses);
}

System.out.println("Test scenario: comparing when there is failedEndpoints marked in request context");
requestMock.requestContext.addToFailedEndpoints(new GoneException("Test"), testUri2);
List<Uri> sortedAddresses = (List<Uri>) sortAddressesMethod.invoke(null, Arrays.asList(testUri4, testUri2), requestMock);
assertThat(sortedAddresses).containsExactlyElementsOf(Arrays.asList(testUri4, testUri2));
} finally {
System.clearProperty("COSMOS.REPLICA_ADDRESS_VALIDATION_ENABLED");
}

System.out.println("Test scenario: comparing when there is failedEndpoints marked in request context");
requestMock.requestContext.addToFailedEndpoints(new GoneException("Test"), testUri2);
List<Uri> sortedAddresses = (List<Uri>) sortAddressesMethod.invoke(null, Arrays.asList(testUri4, testUri2), requestMock);
assertThat(sortedAddresses).containsExactlyElementsOf(Arrays.asList(testUri4, testUri2));
}

private void setTimestamp(Uri testUri, Instant time) throws NoSuchFieldException, IllegalAccessException {
Expand Down Expand Up @@ -172,4 +183,4 @@ public List<Uri> getExpectedAddresses() {
return expectedAddresses;
}
}
}
}