From 0323bcc785021e429ad6b7ba2065643748c46a49 Mon Sep 17 00:00:00 2001 From: Isabelle Date: Wed, 29 Apr 2026 11:50:59 -0700 Subject: [PATCH 1/5] Add encoder stress tests --- sdk/storage/azure-storage-blob-stress/pom.xml | 20 ++ .../scenarios-matrix.yaml | 315 ++++++++++++++++-- .../com/azure/storage/blob/stress/App.java | 10 + .../storage/blob/stress/CommitBlockList.java | 39 +-- ...ntentValidationAppendBlobOutputStream.java | 88 +++++ .../stress/ContentValidationAppendBlock.java | 76 +++++ ...ontentValidationBlockBlobOutputStream.java | 75 +++++ .../ContentValidationBlockBlobUpload.java | 70 ++++ ...ContentValidationPageBlobOutputStream.java | 105 ++++++ ...entValidationSeekableByteChannelWrite.java | 83 +++++ .../stress/ContentValidationStageBlock.java | 89 +++++ .../ContentValidationStressOptions.java | 26 ++ .../blob/stress/ContentValidationUpload.java | 70 ++++ .../ContentValidationUploadFromFile.java | 138 ++++++++ .../stress/ContentValidationUploadPages.java | 82 +++++ sdk/storage/azure-storage-stress/pom.xml | 20 ++ .../azure/storage/stress/TelemetryHelper.java | 2 +- 17 files changed, 1252 insertions(+), 56 deletions(-) create mode 100644 sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationAppendBlobOutputStream.java create mode 100644 sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationAppendBlock.java create mode 100644 sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationBlockBlobOutputStream.java create mode 100644 sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationBlockBlobUpload.java create mode 100644 sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationPageBlobOutputStream.java create mode 100644 sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationSeekableByteChannelWrite.java create mode 100644 sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationStageBlock.java create mode 100644 sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationStressOptions.java create mode 100644 sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationUpload.java create mode 100644 sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationUploadFromFile.java create mode 100644 sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationUploadPages.java diff --git a/sdk/storage/azure-storage-blob-stress/pom.xml b/sdk/storage/azure-storage-blob-stress/pom.xml index f214138d3f8b..c22d4d2e4096 100644 --- a/sdk/storage/azure-storage-blob-stress/pom.xml +++ b/sdk/storage/azure-storage-blob-stress/pom.xml @@ -19,8 +19,23 @@ - + + + 1.58.0 + + + + io.opentelemetry + opentelemetry-bom + ${opentelemetry.version} + pom + import + + + + ch.qos.logback @@ -64,6 +79,11 @@ opentelemetry-logback-appender-1.0 2.24.0-alpha + + io.opentelemetry + opentelemetry-api-incubator + 1.58.0-alpha + com.azure diff --git a/sdk/storage/azure-storage-blob-stress/scenarios-matrix.yaml b/sdk/storage/azure-storage-blob-stress/scenarios-matrix.yaml index 3a8920a67b35..9e8c1bad70f7 100644 --- a/sdk/storage/azure-storage-blob-stress/scenarios-matrix.yaml +++ b/sdk/storage/azure-storage-blob-stress/scenarios-matrix.yaml @@ -5,7 +5,7 @@ matrix: testScenario: downloadtofile sync: true sizeBytes: 1024 - downloadFaults: true + downloadFaults: false durationMin: 25 imageBuildDir: "../../.." @@ -14,7 +14,7 @@ matrix: testScenario: downloadtofile sync: false sizeBytes: 1024 - downloadFaults: true + downloadFaults: false durationMin: 25 imageBuildDir: "../../.." @@ -23,7 +23,7 @@ matrix: testScenario: downloadtofile sync: true sizeBytes: "52428800" - downloadFaults: true + downloadFaults: false durationMin: 60 imageBuildDir: "../../.." @@ -32,7 +32,7 @@ matrix: testScenario: downloadstream sync: true sizeBytes: 1024 - downloadFaults: true + downloadFaults: false durationMin: 25 imageBuildDir: "../../.." @@ -41,7 +41,7 @@ matrix: testScenario: downloadstream sync: false sizeBytes: 1024 - downloadFaults: true + downloadFaults: false durationMin: 25 imageBuildDir: "../../.." @@ -50,7 +50,7 @@ matrix: testScenario: downloadstream sync: true sizeBytes: "52428800" - downloadFaults: true + downloadFaults: false durationMin: 60 imageBuildDir: "../../.." @@ -59,7 +59,7 @@ matrix: testScenario: downloadcontent sync: true sizeBytes: 1024 - downloadFaults: true + downloadFaults: false durationMin: 25 imageBuildDir: "../../.." @@ -68,7 +68,7 @@ matrix: testScenario: downloadcontent sync: true sizeBytes: "52428800" - downloadFaults: true + downloadFaults: false durationMin: 60 imageBuildDir: "../../.." @@ -77,7 +77,7 @@ matrix: testScenario: openinputstream sync: true sizeBytes: 1024 - downloadFaults: true + downloadFaults: false durationMin: 25 imageBuildDir: "../../.." @@ -86,7 +86,7 @@ matrix: testScenario: openinputstream sync: true sizeBytes: "52428800" - downloadFaults: true + downloadFaults: false durationMin: 60 imageBuildDir: "../../.." @@ -95,7 +95,7 @@ matrix: testScenario: openseekablebytechannelread sync: true sizeBytes: 1024 - downloadFaults: true + downloadFaults: false durationMin: 25 imageBuildDir: "../../.." @@ -104,7 +104,7 @@ matrix: testScenario: openseekablebytechannelread sync: true sizeBytes: "52428800" - downloadFaults: true + downloadFaults: false durationMin: 60 imageBuildDir: "../../.." @@ -113,7 +113,7 @@ matrix: testScenario: appendblock sync: true sizeBytes: 1024 - uploadFaults: true + uploadFaults: false durationMin: 25 imageBuildDir: "../../.." @@ -122,7 +122,7 @@ matrix: testScenario: appendblock sync: false sizeBytes: 1024 - uploadFaults: true + uploadFaults: false durationMin: 25 imageBuildDir: "../../.." @@ -131,7 +131,7 @@ matrix: testScenario: appendblock sync: true sizeBytes: "26214400" - uploadFaults: true + uploadFaults: false durationMin: 30 imageBuildDir: "../../.." @@ -140,7 +140,7 @@ matrix: testScenario: appendbloboutputstream sync: true sizeBytes: 1024 - uploadFaults: true + uploadFaults: false durationMin: 25 imageBuildDir: "../../.." @@ -149,7 +149,7 @@ matrix: testScenario: appendbloboutputstream sync: true sizeBytes: "10240" - uploadFaults: true + uploadFaults: false durationMin: 60 imageBuildDir: "../../.." @@ -158,7 +158,7 @@ matrix: testScenario: blockblobupload sync: true sizeBytes: 1024 - uploadFaults: true + uploadFaults: false durationMin: 25 imageBuildDir: "../../.." @@ -167,7 +167,7 @@ matrix: testScenario: blockblobupload sync: true sizeBytes: "26214400" - uploadFaults: true + uploadFaults: false durationMin: 60 imageBuildDir: "../../.." @@ -176,7 +176,7 @@ matrix: testScenario: blockbloboutputstream sync: true sizeBytes: 1024 - uploadFaults: true + uploadFaults: false durationMin: 25 imageBuildDir: "../../.." @@ -185,7 +185,7 @@ matrix: testScenario: blockbloboutputstream sync: true sizeBytes: "26214400" - uploadFaults: true + uploadFaults: false durationMin: 60 imageBuildDir: "../../.." @@ -194,7 +194,7 @@ matrix: testScenario: commitblocklist sync: true sizeBytes: 1024 - uploadFaults: true + uploadFaults: false durationMin: 25 imageBuildDir: "../../.." @@ -203,7 +203,7 @@ matrix: testScenario: commitblocklist sync: true sizeBytes: "26214400" - uploadFaults: true + uploadFaults: false durationMin: 60 imageBuildDir: "../../.." @@ -212,7 +212,7 @@ matrix: testScenario: openseekablebytechannelwrite sync: true sizeBytes: 1024 - uploadFaults: true + uploadFaults: false durationMin: 10 imageBuildDir: "../../.." @@ -221,7 +221,7 @@ matrix: testScenario: openseekablebytechannelwrite sync: true sizeBytes: "52428800" - uploadFaults: true + uploadFaults: false durationMin: 60 imageBuildDir: "../../.." @@ -230,7 +230,7 @@ matrix: testScenario: stageblock sync: true sizeBytes: 1024 - uploadFaults: true + uploadFaults: false durationMin: 25 imageBuildDir: "../../.." @@ -239,7 +239,7 @@ matrix: testScenario: stageblock sync: true sizeBytes: "26214400" - uploadFaults: true + uploadFaults: false durationMin: 60 imageBuildDir: "../../.." @@ -248,7 +248,7 @@ matrix: testScenario: pagebloboutputstream sync: true sizeBytes: 1024 - uploadFaults: true + uploadFaults: false durationMin: 25 imageBuildDir: "../../.." @@ -257,7 +257,7 @@ matrix: testScenario: pagebloboutputstream sync: true sizeBytes: "10240" - uploadFaults: true + uploadFaults: false durationMin: 60 imageBuildDir: "../../.." @@ -266,7 +266,7 @@ matrix: testScenario: uploadpages sync: true sizeBytes: 1024 - uploadFaults: true + uploadFaults: false durationMin: 25 imageBuildDir: "../../.." @@ -275,7 +275,7 @@ matrix: testScenario: uploadpages sync: true sizeBytes: "4194304" - uploadFaults: true + uploadFaults: false durationMin: 60 imageBuildDir: "../../.." @@ -284,7 +284,7 @@ matrix: testScenario: upload sync: true sizeBytes: 1024 - uploadFaults: true + uploadFaults: false durationMin: 25 imageBuildDir: "../../.." @@ -293,7 +293,7 @@ matrix: testScenario: upload sync: false sizeBytes: 1024 - uploadFaults: true + uploadFaults: false durationMin: 25 imageBuildDir: "../../.." @@ -302,7 +302,7 @@ matrix: testScenario: upload sync: true sizeBytes: "52428800" - uploadFaults: true + uploadFaults: false durationMin: 60 imageBuildDir: "../../.." @@ -311,7 +311,7 @@ matrix: testScenario: uploadfromfile sync: true sizeBytes: 1024 - uploadFaults: true + uploadFaults: false durationMin: 25 imageBuildDir: "../../.." @@ -320,6 +320,249 @@ matrix: testScenario: uploadfromfile sync: true sizeBytes: "52428800" - uploadFaults: true + uploadFaults: false durationMin: 60 imageBuildDir: "../../.." + + # --- Content validation (ContentValidation*; default CRC64). Scenario keys are short: K8s label + # testInstance = "{Scenario}-{BaseName}-{revision}" must be <= 63 chars (see stress-test-job.yaml). + + cv-appendblock-sm: + testScenario: contentvalidationappendblock + sync: true + sizeBytes: 1024 + uploadFaults: false + durationMin: 25 + imageBuildDir: "../../.." + + cv-appendblock-lg: + testScenario: contentvalidationappendblock + sync: true + sizeBytes: "26214400" + uploadFaults: false + durationMin: 30 + imageBuildDir: "../../.." + + cv-appendblock-async-sm: + testScenario: contentvalidationappendblock + sync: false + sizeBytes: 1024 + uploadFaults: false + durationMin: 25 + imageBuildDir: "../../.." + + cv-appendblock-async-lg: + testScenario: contentvalidationappendblock + sync: false + sizeBytes: "26214400" + uploadFaults: false + durationMin: 30 + imageBuildDir: "../../.." + + cv-appendbloboutputstream-sm: + testScenario: contentvalidationappendbloboutputstream + sync: true + sizeBytes: 1024 + uploadFaults: false + durationMin: 25 + imageBuildDir: "../../.." + + cv-appendbloboutputstream-lg: + testScenario: contentvalidationappendbloboutputstream + sync: true + sizeBytes: "10240" + uploadFaults: false + durationMin: 60 + imageBuildDir: "../../.." + + cv-blockblobupload-sm: + testScenario: contentvalidationblockblobupload + sync: true + sizeBytes: 1024 + uploadFaults: false + durationMin: 25 + imageBuildDir: "../../.." + + cv-blockblobupload-lg: + testScenario: contentvalidationblockblobupload + sync: true + sizeBytes: "26214400" + uploadFaults: false + durationMin: 60 + imageBuildDir: "../../.." + + cv-blockbloboutputstream-sm: + testScenario: contentvalidationblockbloboutputstream + sync: true + sizeBytes: 1024 + uploadFaults: false + durationMin: 25 + imageBuildDir: "../../.." + + cv-blockbloboutputstream-lg: + testScenario: contentvalidationblockbloboutputstream + sync: true + sizeBytes: "26214400" + uploadFaults: false + durationMin: 60 + imageBuildDir: "../../.." + + cv-seekablebytechannelwrite-sm: + testScenario: contentvalidationseekablebytechannelwrite + sync: true + sizeBytes: 1024 + uploadFaults: false + durationMin: 10 + imageBuildDir: "../../.." + + cv-seekablebytechannelwrite-lg: + testScenario: contentvalidationseekablebytechannelwrite + sync: true + sizeBytes: "52428800" + uploadFaults: false + durationMin: 60 + imageBuildDir: "../../.." + + cv-stageblock-sm: + testScenario: contentvalidationstageblock + sync: true + sizeBytes: 1024 + uploadFaults: false + durationMin: 25 + imageBuildDir: "../../.." + + cv-stageblock-lg: + testScenario: contentvalidationstageblock + sync: true + sizeBytes: "26214400" + uploadFaults: false + durationMin: 60 + imageBuildDir: "../../.." + + cv-stageblock-async-sm: + testScenario: contentvalidationstageblock + sync: false + sizeBytes: 1024 + uploadFaults: false + durationMin: 25 + imageBuildDir: "../../.." + + cv-stageblock-async-lg: + testScenario: contentvalidationstageblock + sync: false + sizeBytes: "26214400" + uploadFaults: false + durationMin: 60 + imageBuildDir: "../../.." + + cv-pagebloboutputstream-sm: + testScenario: contentvalidationpagebloboutputstream + sync: true + sizeBytes: 1024 + uploadFaults: false + durationMin: 25 + imageBuildDir: "../../.." + + cv-pagebloboutputstream-lg: + testScenario: contentvalidationpagebloboutputstream + sync: true + sizeBytes: "10240" + uploadFaults: false + durationMin: 60 + imageBuildDir: "../../.." + + cv-uploadpages-sm: + testScenario: contentvalidationuploadpages + sync: true + sizeBytes: 1024 + uploadFaults: false + durationMin: 25 + imageBuildDir: "../../.." + + cv-uploadpages-lg: + testScenario: contentvalidationuploadpages + sync: true + sizeBytes: "4194304" + uploadFaults: false + durationMin: 60 + imageBuildDir: "../../.." + + cv-uploadpages-async-sm: + testScenario: contentvalidationuploadpages + sync: false + sizeBytes: 1024 + uploadFaults: false + durationMin: 25 + imageBuildDir: "../../.." + + cv-uploadpages-async-lg: + testScenario: contentvalidationuploadpages + sync: false + sizeBytes: "4194304" + uploadFaults: false + durationMin: 60 + imageBuildDir: "../../.." + + cv-upload-sm: + testScenario: contentvalidationupload + sync: true + sizeBytes: 1024 + uploadFaults: false + durationMin: 25 + imageBuildDir: "../../.." + + cv-upload-lg: + testScenario: contentvalidationupload + sync: true + sizeBytes: "52428800" + uploadFaults: false + durationMin: 60 + imageBuildDir: "../../.." + + cv-upload-async-sm: + testScenario: contentvalidationupload + sync: false + sizeBytes: 1024 + uploadFaults: false + durationMin: 25 + imageBuildDir: "../../.." + + cv-upload-async-lg: + testScenario: contentvalidationupload + sync: false + sizeBytes: "52428800" + uploadFaults: false + durationMin: 60 + imageBuildDir: "../../.." + + cv-uploadfromfile-sm: + testScenario: contentvalidationuploadfromfile + sync: true + sizeBytes: 1024 + uploadFaults: false + durationMin: 25 + imageBuildDir: "../../.." + + cv-uploadfromfile-lg: + testScenario: contentvalidationuploadfromfile + sync: true + sizeBytes: "52428800" + uploadFaults: false + durationMin: 60 + imageBuildDir: "../../.." + + cv-uploadfromfile-async-sm: + testScenario: contentvalidationuploadfromfile + sync: false + sizeBytes: 1024 + uploadFaults: false + durationMin: 25 + imageBuildDir: "../../.." + + cv-uploadfromfile-async-lg: + testScenario: contentvalidationuploadfromfile + sync: false + sizeBytes: "52428800" + uploadFaults: false + durationMin: 60 + imageBuildDir: "../../.." \ No newline at end of file diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/App.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/App.java index e38bd16791ca..3f3219930596 100644 --- a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/App.java +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/App.java @@ -15,6 +15,16 @@ public static void main(String[] args) { BlockBlobOutputStream.class, BlockBlobUpload.class, CommitBlockList.class, + ContentValidationAppendBlobOutputStream.class, + ContentValidationAppendBlock.class, + ContentValidationBlockBlobOutputStream.class, + ContentValidationBlockBlobUpload.class, + ContentValidationPageBlobOutputStream.class, + ContentValidationStageBlock.class, + ContentValidationSeekableByteChannelWrite.class, + ContentValidationUpload.class, + ContentValidationUploadFromFile.class, + ContentValidationUploadPages.class, DownloadToFile.class, DownloadStream.class, DownloadContent.class, diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/CommitBlockList.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/CommitBlockList.java index 310c7618e5d9..5e63058a4ab5 100644 --- a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/CommitBlockList.java +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/CommitBlockList.java @@ -1,6 +1,3 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - package com.azure.storage.blob.stress; import com.azure.core.util.Context; @@ -11,7 +8,6 @@ import com.azure.storage.blob.specialized.BlockBlobAsyncClient; import com.azure.storage.blob.specialized.BlockBlobClient; import com.azure.storage.blob.stress.utils.OriginalContent; -import com.azure.storage.common.Utility; import com.azure.storage.stress.CrcInputStream; import com.azure.storage.stress.StorageStressOptions; import reactor.core.publisher.Flux; @@ -19,10 +15,17 @@ import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; +import java.time.Duration; import java.util.Base64; import java.util.Collections; public class CommitBlockList extends BlobScenarioBase { + // Per-operation ceilings. Must be smaller than the Azure LB idle timeout (~4 min) + // so that a stalled request fails fast rather than hangs the whole test run. + private static final Duration STAGE_TIMEOUT = Duration.ofSeconds(120); + private static final Duration COMMIT_TIMEOUT = Duration.ofSeconds(30); + private static final Duration CHECK_MATCH_TIMEOUT = Duration.ofSeconds(30); + private final OriginalContent originalContent = new OriginalContent(); private final BlobClient syncClient; private final BlobAsyncClient asyncClient; @@ -45,14 +48,11 @@ protected void runInternal(Context span) throws Exception { String blockId = Base64.getEncoder().encodeToString(CoreUtils.randomUuid().toString() .getBytes(StandardCharsets.UTF_8)); try (CrcInputStream inputStream = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize())) { - // First perform non-faulted stage block to send data to the service - blockBlobClientNoFault.stageBlockWithResponse(blockId, inputStream, options.getSize(), null, null, null, - span); - // Then perform faulted commit block list to commit the block + blockBlobClientNoFault.stageBlockWithResponse(blockId, inputStream, options.getSize(), null, null, + STAGE_TIMEOUT, span); blockBlobClient.commitBlockListWithResponse( - new BlockBlobCommitBlockListOptions(Collections.singletonList(blockId)), null, span); - // Confirm the CRC matches for the uploaded input stream - originalContent.checkMatch(inputStream.getContentInfo(), span).block(); + new BlockBlobCommitBlockListOptions(Collections.singletonList(blockId)), COMMIT_TIMEOUT, span); + originalContent.checkMatch(inputStream.getContentInfo(), span).block(CHECK_MATCH_TIMEOUT); } } @@ -64,13 +64,14 @@ protected Mono runInternalAsync(Context span) { .getBytes(StandardCharsets.UTF_8)); Flux byteBufferFlux = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize()) .convertStreamToByteBuffer(); - // First perform non-faulted stage block to send data to the service - return blockBlobAsyncClientNoFault.stageBlockWithResponse(blockId, byteBufferFlux, options.getSize(), null, null) - // Then perform faulted commit block list to commit the block - .then(blockBlobAsyncClient.commitBlockListWithResponse( - new BlockBlobCommitBlockListOptions(Collections.singletonList(blockId)))) - // Confirm the CRC matches for the uploaded byte buffer flux - .then(originalContent.checkMatch(byteBufferFlux, span)); + return blockBlobAsyncClientNoFault + .stageBlockWithResponse(blockId, byteBufferFlux, options.getSize(), null, null) + .timeout(STAGE_TIMEOUT) + .then(blockBlobAsyncClient + .commitBlockListWithResponse( + new BlockBlobCommitBlockListOptions(Collections.singletonList(blockId))) + .timeout(COMMIT_TIMEOUT)) + .then(originalContent.checkMatch(byteBufferFlux, span).timeout(CHECK_MATCH_TIMEOUT)); } @Override @@ -83,4 +84,4 @@ public Mono cleanupAsync() { return asyncNoFaultClient.deleteIfExists() .then(super.cleanupAsync()); } -} +} \ No newline at end of file diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationAppendBlobOutputStream.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationAppendBlobOutputStream.java new file mode 100644 index 000000000000..b2b9d1845fa4 --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationAppendBlobOutputStream.java @@ -0,0 +1,88 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.util.Context; +import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.options.AppendBlobOutputStreamOptions; +import com.azure.storage.blob.specialized.AppendBlobClient; +import com.azure.storage.blob.specialized.BlobOutputStream; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.stress.CrcInputStream; +import reactor.core.publisher.Mono; + +import java.io.IOException; + +import static com.azure.core.util.FluxUtil.monoError; + +/** + * Append blob output stream with {@link AppendBlobOutputStreamOptions#setContentValidationAlgorithm} (sync only). + */ +public class ContentValidationAppendBlobOutputStream extends BlobScenarioBase { + private static final ClientLogger LOGGER = new ClientLogger(ContentValidationAppendBlobOutputStream.class); + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncNoFaultClient; + /** Separate blob used to upload reference content for {@link OriginalContent} checksum (block blob). */ + private final BlobAsyncClient tempSetupBlobClient; + + public ContentValidationAppendBlobOutputStream(ContentValidationStressOptions options) { + super(options); + String blobName = generateBlobName(); + String tempBlobName = generateBlobName(); + + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.tempSetupBlobClient = getAsyncContainerClientNoFault().getBlobAsyncClient(tempBlobName); + } + + @Override + protected void runInternal(Context span) throws IOException { + AppendBlobClient appendBlobClient = syncClient.getAppendBlobClient(); + // Reset the append blob at the start of each iteration. The boolean overload + // getBlobOutputStream(true) does this implicitly via create(true); the options overload + // does not, so we replicate that behavior here. Without this reset, fault-injection + // sequences that commit a block server-side but drop the response leave the cached + // appendPosition stale, causing subsequent retries to fail with 412 AppendPositionConditionNotMet, + // which combined with non-retriable Crc64Mismatch on truncated-body faults collapses the pass rate. + appendBlobClient.create(true); + + AppendBlobOutputStreamOptions streamOptions = new AppendBlobOutputStreamOptions() + .setContentValidationAlgorithm(options.getContentValidationAlgorithm()); + + try (CrcInputStream inputStream = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize()); + BlobOutputStream outputStream = appendBlobClient.getBlobOutputStream(streamOptions)) { + byte[] buffer = new byte[4096]; + int bytesRead; + + while ((bytesRead = inputStream.read(buffer)) != -1) { + outputStream.write(buffer, 0, bytesRead); + } + + outputStream.close(); + originalContent.checkMatch(inputStream.getContentInfo(), span).block(); + } + } + + @Override + protected Mono runInternalAsync(Context span) { + return monoError(LOGGER, new RuntimeException("getBlobOutputStream() does not exist on the async client")); + } + + @Override + public Mono setupAsync() { + return super.setupAsync() + .then(asyncNoFaultClient.getAppendBlobAsyncClient().create()) + .then(originalContent.setupBlob(tempSetupBlobClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.deleteIfExists() + .then(tempSetupBlobClient.deleteIfExists()) + .then(super.cleanupAsync()); + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationAppendBlock.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationAppendBlock.java new file mode 100644 index 000000000000..bcbaf7681742 --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationAppendBlock.java @@ -0,0 +1,76 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.util.Context; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.options.AppendBlobAppendBlockOptions; +import com.azure.storage.blob.specialized.AppendBlobAsyncClient; +import com.azure.storage.blob.specialized.AppendBlobClient; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.stress.CrcInputStream; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.nio.ByteBuffer; + +/** + * Append block with {@link AppendBlobAppendBlockOptions#setContentValidationAlgorithm}. + */ +public class ContentValidationAppendBlock extends BlobScenarioBase { + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncClient; + private final BlobAsyncClient asyncNoFaultClient; + private final BlobAsyncClient tempSetupBlobClient; + + public ContentValidationAppendBlock(ContentValidationStressOptions options) { + super(options); + String blobName = generateBlobName(); + String tempBlobName = generateBlobName(); + + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.asyncClient = getAsyncContainerClient().getBlobAsyncClient(blobName); + this.tempSetupBlobClient = getAsyncContainerClientNoFault().getBlobAsyncClient(tempBlobName); + } + + @Override + protected void runInternal(Context span) { + try (CrcInputStream inputStream = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize())) { + AppendBlobClient appendBlobClient = syncClient.getAppendBlobClient(); + appendBlobClient.appendBlockWithResponse( + new AppendBlobAppendBlockOptions(inputStream, options.getSize()) + .setContentValidationAlgorithm(options.getContentValidationAlgorithm()), + null, span); + originalContent.checkMatch(inputStream.getContentInfo(), span).block(); + } + } + + @Override + protected Mono runInternalAsync(Context span) { + AppendBlobAsyncClient appendBlobAsyncClient = asyncClient.getAppendBlobAsyncClient(); + Flux byteBufferFlux = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize()) + .convertStreamToByteBuffer(); + return appendBlobAsyncClient.appendBlockWithResponse( + new AppendBlobAppendBlockOptions(byteBufferFlux, options.getSize()) + .setContentValidationAlgorithm(options.getContentValidationAlgorithm())) + .then(originalContent.checkMatch(byteBufferFlux, span)); + } + + @Override + public Mono setupAsync() { + return super.setupAsync().then(asyncNoFaultClient.getAppendBlobAsyncClient().create()) + .then(originalContent.setupBlob(tempSetupBlobClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.getAppendBlobAsyncClient().deleteIfExists() + .onErrorResume(e -> Mono.empty()) + .then(tempSetupBlobClient.deleteIfExists()) + .then(super.cleanupAsync()); + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationBlockBlobOutputStream.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationBlockBlobOutputStream.java new file mode 100644 index 000000000000..175b648002a5 --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationBlockBlobOutputStream.java @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.util.Context; +import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.models.ParallelTransferOptions; +import com.azure.storage.blob.options.BlockBlobOutputStreamOptions; +import com.azure.storage.blob.specialized.BlobOutputStream; +import com.azure.storage.blob.specialized.BlockBlobClient; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.stress.CrcInputStream; +import reactor.core.publisher.Mono; + +import java.io.IOException; + +import static com.azure.core.util.FluxUtil.monoError; + +/** + * Block blob output stream with {@link BlockBlobOutputStreamOptions#setContentValidationAlgorithm} (sync only). + */ +public class ContentValidationBlockBlobOutputStream extends BlobScenarioBase { + private static final ClientLogger LOGGER = new ClientLogger(ContentValidationBlockBlobOutputStream.class); + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncNoFaultClient; + private final ParallelTransferOptions parallelTransferOptions; + + public ContentValidationBlockBlobOutputStream(ContentValidationStressOptions options) { + super(options); + String blobName = generateBlobName(); + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.parallelTransferOptions = new ParallelTransferOptions().setMaxConcurrency(options.getMaxConcurrency()); + } + + @Override + protected void runInternal(Context span) throws IOException { + BlockBlobClient blockBlobClient = syncClient.getBlockBlobClient(); + BlockBlobOutputStreamOptions streamOptions = new BlockBlobOutputStreamOptions() + .setParallelTransferOptions(parallelTransferOptions) + .setContentValidationAlgorithm(options.getContentValidationAlgorithm()); + + try (CrcInputStream inputStream = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize()); + BlobOutputStream outputStream = blockBlobClient.getBlobOutputStream(streamOptions, span)) { + byte[] buffer = new byte[4096]; + int bytesRead; + + while ((bytesRead = inputStream.read(buffer)) != -1) { + outputStream.write(buffer, 0, bytesRead); + } + + outputStream.close(); + originalContent.checkMatch(inputStream.getContentInfo(), span).block(); + } + } + + @Override + protected Mono runInternalAsync(Context span) { + return monoError(LOGGER, new RuntimeException("getBlobOutputStream() does not exist on the async client")); + } + + @Override + public Mono setupAsync() { + return super.setupAsync().then(originalContent.setupBlob(asyncNoFaultClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.deleteIfExists().then(super.cleanupAsync()); + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationBlockBlobUpload.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationBlockBlobUpload.java new file mode 100644 index 000000000000..e86a410e1945 --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationBlockBlobUpload.java @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.util.Context; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.options.BlockBlobSimpleUploadOptions; +import com.azure.storage.blob.specialized.BlockBlobAsyncClient; +import com.azure.storage.blob.specialized.BlockBlobClient; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.stress.CrcInputStream; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.nio.ByteBuffer; + +/** + * Single-shot block blob upload with request content validation + * ({@link BlockBlobSimpleUploadOptions#setContentValidationAlgorithm}). + */ +public class ContentValidationBlockBlobUpload extends BlobScenarioBase { + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncClient; + private final BlobAsyncClient asyncNoFaultClient; + + public ContentValidationBlockBlobUpload(ContentValidationStressOptions options) { + super(options); + String blobName = generateBlobName(); + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.asyncClient = getAsyncContainerClient().getBlobAsyncClient(blobName); + } + + @Override + protected void runInternal(Context span) { + BlockBlobClient blockBlobClient = syncClient.getBlockBlobClient(); + try (CrcInputStream inputStream = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize())) { + blockBlobClient.uploadWithResponse( + new BlockBlobSimpleUploadOptions(inputStream, options.getSize()) + .setContentValidationAlgorithm(options.getContentValidationAlgorithm()), + null, span); + originalContent.checkMatch(inputStream.getContentInfo(), span).block(); + } + } + + @Override + protected Mono runInternalAsync(Context span) { + Flux byteBufferFlux = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize()) + .convertStreamToByteBuffer(); + BlockBlobAsyncClient blockBlobAsyncClient = asyncClient.getBlockBlobAsyncClient(); + return blockBlobAsyncClient.uploadWithResponse( + new BlockBlobSimpleUploadOptions(byteBufferFlux, options.getSize()) + .setContentValidationAlgorithm(options.getContentValidationAlgorithm())) + .then(originalContent.checkMatch(byteBufferFlux, span)); + } + + @Override + public Mono setupAsync() { + return super.setupAsync().then(originalContent.setupBlob(asyncNoFaultClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.deleteIfExists() + .then(super.cleanupAsync()); + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationPageBlobOutputStream.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationPageBlobOutputStream.java new file mode 100644 index 000000000000..b49a0305f12f --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationPageBlobOutputStream.java @@ -0,0 +1,105 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.util.Context; +import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.models.PageRange; +import com.azure.storage.blob.options.PageBlobOutputStreamOptions; +import com.azure.storage.blob.specialized.BlobOutputStream; +import com.azure.storage.blob.specialized.PageBlobAsyncClient; +import com.azure.storage.blob.specialized.PageBlobClient; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.stress.CrcInputStream; +import reactor.core.publisher.Mono; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +import static com.azure.core.util.FluxUtil.monoError; + +/** + * Page blob output stream with {@link PageBlobOutputStreamOptions#setContentValidationAlgorithm} (sync only). + */ +public class ContentValidationPageBlobOutputStream extends PageBlobScenarioBase { + private static final ClientLogger LOGGER = new ClientLogger(ContentValidationPageBlobOutputStream.class); + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncNoFaultClient; + /** Page blob used only to seed {@link OriginalContent} (same pattern as {@link PageBlobOutputStream}). */ + private final PageBlobAsyncClient tempSetupPageBlobClient; + + public ContentValidationPageBlobOutputStream(ContentValidationStressOptions options) { + super(options); + String blobName = generateBlobName(); + String tempBlobName = generateBlobName(); + + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + BlobAsyncClient tempSetupBlobClient = getAsyncContainerClientNoFault().getBlobAsyncClient(tempBlobName); + this.tempSetupPageBlobClient = tempSetupBlobClient.getPageBlobAsyncClient(); + } + + @Override + protected void runInternal(Context span) throws IOException { + PageBlobClient pageBlobClient = syncClient.getPageBlobClient(); + PageBlobOutputStreamOptions streamOptions = new PageBlobOutputStreamOptions( + new PageRange().setStart(0).setEnd(options.getSize() - 1)) + .setContentValidationAlgorithm(options.getContentValidationAlgorithm()); + + try (CrcInputStream inputStream = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize()); + BlobOutputStream outputStream = pageBlobClient.getBlobOutputStream(streamOptions)) { + ByteArrayOutputStream bufferStream = new ByteArrayOutputStream(); + byte[] buffer = new byte[512]; + int bytesRead; + + while ((bytesRead = inputStream.read(buffer)) != -1) { + // Always accumulate into bufferStream to avoid dropping or reordering bytes + bufferStream.write(buffer, 0, bytesRead); + // Flush all full 512-byte pages from the accumulator + if (bufferStream.size() >= buffer.length) { + byte[] toWrite = bufferStream.toByteArray(); + int length = toWrite.length - (toWrite.length % buffer.length); + if (length > 0) { + outputStream.write(toWrite, 0, length); + bufferStream.reset(); + // Keep any remaining partial page bytes in the accumulator + bufferStream.write(toWrite, length, toWrite.length - length); + } + } + } + // For page blobs, total content size must be a multiple of 512 bytes. + // Any remaining bytes here indicate misalignment and would result in silent truncation. + if (bufferStream.size() != 0) { + throw new IOException("Remaining bytes in buffer that do not align to 512-byte page size."); + } + + outputStream.close(); + originalContent.checkMatch(inputStream.getContentInfo(), span).block(); + } + } + + @Override + protected Mono runInternalAsync(Context span) { + return monoError(LOGGER, new RuntimeException("getBlobOutputStream() does not exist on the async client")); + } + + @Override + public Mono setupAsync() { + return super.setupAsync() + .then(asyncNoFaultClient.getPageBlobAsyncClient().create(options.getSize())) + .then(tempSetupPageBlobClient.create(options.getSize())) + .then(originalContent.setupPageBlob(tempSetupPageBlobClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.getPageBlobAsyncClient().deleteIfExists() + .onErrorResume(e -> Mono.empty()) + .then(tempSetupPageBlobClient.deleteIfExists()) + .then(super.cleanupAsync()); + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationSeekableByteChannelWrite.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationSeekableByteChannelWrite.java new file mode 100644 index 000000000000..65afda98636f --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationSeekableByteChannelWrite.java @@ -0,0 +1,83 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.util.Context; +import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.options.BlockBlobSeekableByteChannelWriteOptions; +import com.azure.storage.blob.specialized.BlockBlobClient; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.implementation.StorageSeekableByteChannel; +import com.azure.storage.stress.CrcInputStream; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.io.IOException; +import java.nio.ByteBuffer; + +import static com.azure.core.util.FluxUtil.monoError; +import static com.azure.storage.blob.options.BlockBlobSeekableByteChannelWriteOptions.WriteMode.OVERWRITE; + +/** + * Block-blob seekable byte channel write with {@link BlockBlobSeekableByteChannelWriteOptions#setContentValidationAlgorithm}. + * Matches {@link com.azure.storage.blob.BlobContentValidationUploadTests} seekable-channel scenarios (sync only). + */ +public class ContentValidationSeekableByteChannelWrite extends BlobScenarioBase { + private static final ClientLogger LOGGER = new ClientLogger(ContentValidationSeekableByteChannelWrite.class); + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncNoFaultClient; + + public ContentValidationSeekableByteChannelWrite(ContentValidationStressOptions options) { + super(options); + String blobName = generateBlobName(); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + } + + @Override + protected void runInternal(Context span) throws IOException { + BlockBlobClient blockBlobClient = syncClient.getBlockBlobClient(); + BlockBlobSeekableByteChannelWriteOptions writeOptions = new BlockBlobSeekableByteChannelWriteOptions(OVERWRITE) + .setContentValidationAlgorithm(options.getContentValidationAlgorithm()); + + try (CrcInputStream crcInput = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize())) { + Flux byteBufferFlux = crcInput.convertStreamToByteBuffer(); + try (StorageSeekableByteChannel channel = (StorageSeekableByteChannel) blockBlobClient.openSeekableByteChannelWrite( + writeOptions)) { + Mono writeOperation = byteBufferFlux + .doOnNext(buffer -> { + try { + while (buffer.hasRemaining()) { + channel.write(buffer); + } + } catch (IOException e) { + throw LOGGER.logExceptionAsError(new RuntimeException(e)); + } + }).then(); + writeOperation.block(); + channel.getWriteBehavior().commit(options.getSize()); + } + originalContent.checkMatch(byteBufferFlux, span).block(); + } + } + + @Override + protected Mono runInternalAsync(Context span) { + return monoError(LOGGER, new RuntimeException( + "openSeekableByteChannelWrite() does not exist on the async client")); + } + + @Override + public Mono setupAsync() { + return super.setupAsync().then(originalContent.setupBlob(asyncNoFaultClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.deleteIfExists().then(super.cleanupAsync()); + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationStageBlock.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationStageBlock.java new file mode 100644 index 000000000000..c95dd7e9a98a --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationStageBlock.java @@ -0,0 +1,89 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.util.BinaryData; +import com.azure.core.util.Context; +import com.azure.core.util.CoreUtils; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.options.BlockBlobCommitBlockListOptions; +import com.azure.storage.blob.options.BlockBlobStageBlockOptions; +import com.azure.storage.blob.specialized.BlockBlobAsyncClient; +import com.azure.storage.blob.specialized.BlockBlobClient; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.stress.CrcInputStream; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.Collections; + +/** + * Stage block with request content validation on the faulted client, then commit via the non-faulted client. + */ +public class ContentValidationStageBlock extends BlobScenarioBase { + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncClient; + private final BlobClient syncNoFaultClient; + private final BlobAsyncClient asyncNoFaultClient; + + public ContentValidationStageBlock(ContentValidationStressOptions options) { + super(options); + String blobName = generateBlobName(); + this.syncNoFaultClient = getSyncContainerClientNoFault().getBlobClient(blobName); + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.asyncClient = getAsyncContainerClient().getBlobAsyncClient(blobName); + } + + @Override + protected void runInternal(Context span) { + BlockBlobClient blockBlobClient = syncClient.getBlockBlobClient(); + BlockBlobClient blockBlobClientNoFault = syncNoFaultClient.getBlockBlobClient(); + String blockId = Base64.getEncoder().encodeToString(CoreUtils.randomUuid().toString() + .getBytes(StandardCharsets.UTF_8)); + try (CrcInputStream inputStream = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize())) { + BinaryData data = BinaryData.fromStream(inputStream, options.getSize()); + blockBlobClient.stageBlockWithResponse( + new BlockBlobStageBlockOptions(blockId, data) + .setContentValidationAlgorithm(options.getContentValidationAlgorithm()), + null, span); + blockBlobClientNoFault.commitBlockListWithResponse( + new BlockBlobCommitBlockListOptions(Collections.singletonList(blockId)), null, span); + originalContent.checkMatch(inputStream.getContentInfo(), span).block(); + } + } + + @Override + protected Mono runInternalAsync(Context span) { + BlockBlobAsyncClient blockBlobAsyncClient = asyncClient.getBlockBlobAsyncClient(); + BlockBlobAsyncClient blockBlobAsyncClientNoFault = asyncNoFaultClient.getBlockBlobAsyncClient(); + String blockId = Base64.getEncoder().encodeToString(CoreUtils.randomUuid().toString() + .getBytes(StandardCharsets.UTF_8)); + Flux byteBufferFlux = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize()) + .convertStreamToByteBuffer(); + return BinaryData.fromFlux(byteBufferFlux, options.getSize(), false) + .flatMap(binaryData -> blockBlobAsyncClient.stageBlockWithResponse( + new BlockBlobStageBlockOptions(blockId, binaryData) + .setContentValidationAlgorithm(options.getContentValidationAlgorithm()))) + .then(blockBlobAsyncClientNoFault.commitBlockListWithResponse( + new BlockBlobCommitBlockListOptions(Collections.singletonList(blockId)))) + .then(originalContent.checkMatch(byteBufferFlux, span)); + } + + @Override + public Mono setupAsync() { + return super.setupAsync().then(originalContent.setupBlob(asyncNoFaultClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.deleteIfExists() + .then(super.cleanupAsync()); + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationStressOptions.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationStressOptions.java new file mode 100644 index 000000000000..1e92d331d33e --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationStressOptions.java @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.StorageStressOptions; +import com.beust.jcommander.Parameter; + +/** + * Options for stress scenarios that enable transactional request content validation on uploads + * (CRC64 / structured message). See {@link com.azure.storage.blob.BlobContentValidationUploadTests}. + */ +public class ContentValidationStressOptions extends StorageStressOptions { + /** + * Request checksum behavior for upload APIs. Use CRC64 or AUTO to exercise content validation. + * MD5 is not supported for uploadFromFile. NONE disables request validation. + */ + @Parameter(names = { "--requestChecksumAlgorithm" }, + description = "CRC64 (default), AUTO, NONE, or MD5 (not valid for upload-from-file scenarios)") + private ContentValidationAlgorithm requestChecksumAlgorithm = ContentValidationAlgorithm.CRC64; + + public ContentValidationAlgorithm getContentValidationAlgorithm() { + return requestChecksumAlgorithm; + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationUpload.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationUpload.java new file mode 100644 index 000000000000..b2cc61909be3 --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationUpload.java @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.util.Context; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.models.ParallelTransferOptions; +import com.azure.storage.blob.options.BlobParallelUploadOptions; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.stress.CrcInputStream; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.nio.ByteBuffer; + +/** + * Parallel blob upload with {@link com.azure.storage.blob.options.BlobParallelUploadOptions#setContentValidationAlgorithm} + * enabled. Verifies the correctness of the upload request content via CRC (see {@code BlobContentValidationUploadTests}). + */ +public class ContentValidationUpload extends BlobScenarioBase { + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncClient; + private final BlobAsyncClient asyncNoFaultClient; + private final ParallelTransferOptions parallelTransferOptions; + + public ContentValidationUpload(ContentValidationStressOptions options) { + super(options); + String blobName = generateBlobName(); + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.asyncClient = getAsyncContainerClient().getBlobAsyncClient(blobName); + parallelTransferOptions = new ParallelTransferOptions() + .setMaxConcurrency(options.getMaxConcurrency()) + .setMaxSingleUploadSizeLong(4 * 1024 * 1024L); + } + + @Override + protected void runInternal(Context span) { + try (CrcInputStream inputStream = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize())) { + syncClient.uploadWithResponse(new BlobParallelUploadOptions(inputStream) + .setParallelTransferOptions(parallelTransferOptions) + .setContentValidationAlgorithm(options.getContentValidationAlgorithm()), null, span); + originalContent.checkMatch(inputStream.getContentInfo(), span).block(); + } + } + + @Override + protected Mono runInternalAsync(Context span) { + Flux byteBufferFlux = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize()) + .convertStreamToByteBuffer(); + return asyncClient.uploadWithResponse(new BlobParallelUploadOptions(byteBufferFlux) + .setParallelTransferOptions(parallelTransferOptions) + .setContentValidationAlgorithm(options.getContentValidationAlgorithm())) + .then(originalContent.checkMatch(byteBufferFlux, span)); + } + + @Override + public Mono setupAsync() { + return super.setupAsync().then(originalContent.setupBlob(asyncNoFaultClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.deleteIfExists() + .then(super.cleanupAsync()); + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationUploadFromFile.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationUploadFromFile.java new file mode 100644 index 000000000000..a716563c1752 --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationUploadFromFile.java @@ -0,0 +1,138 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.util.BinaryData; +import com.azure.core.util.Context; +import com.azure.core.util.CoreUtils; +import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.models.ParallelTransferOptions; +import com.azure.storage.blob.options.BlobDownloadToFileOptions; +import com.azure.storage.blob.options.BlobUploadFromFileOptions; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.stress.CrcInputStream; +import reactor.core.publisher.Mono; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.UUID; + +/** + * Upload from file with {@link BlobUploadFromFileOptions#setContentValidationAlgorithm}. + */ +public class ContentValidationUploadFromFile extends BlobScenarioBase { + private static final ClientLogger LOGGER = new ClientLogger(ContentValidationUploadFromFile.class); + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncClient; + private final BlobClient syncNoFaultClient; + private final BlobAsyncClient asyncNoFaultClient; + private final ParallelTransferOptions parallelTransferOptions; + + public ContentValidationUploadFromFile(ContentValidationStressOptions options) { + super(options); + String blobName = generateBlobName(); + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + this.syncNoFaultClient = getSyncContainerClientNoFault().getBlobClient(blobName); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.asyncClient = getAsyncContainerClient().getBlobAsyncClient(blobName); + this.parallelTransferOptions = new ParallelTransferOptions() + .setMaxConcurrency(options.getMaxConcurrency()) + .setMaxSingleUploadSizeLong(4 * 1024 * 1024L); + } + + @Override + protected void runInternal(Context span) { + Path downloadPath = getTempPath("test"); + Path uploadFilePath = null; + try (CrcInputStream inputStream = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize())) { + uploadFilePath = generateFile(inputStream); + downloadPath = downloadPath.resolve(CoreUtils.randomUuid() + ".txt"); + syncClient.uploadFromFileWithResponse(new BlobUploadFromFileOptions(uploadFilePath.toString()) + .setParallelTransferOptions(parallelTransferOptions) + .setContentValidationAlgorithm(options.getContentValidationAlgorithm()), + null, span); + syncNoFaultClient.downloadToFileWithResponse( + new BlobDownloadToFileOptions(downloadPath.toString()), null, span); + originalContent.checkMatch(BinaryData.fromFile(downloadPath), span).block(); + } finally { + deleteFile(downloadPath); + deleteFile(uploadFilePath); + } + } + + @Override + protected Mono runInternalAsync(Context span) { + Path downloadPath = getTempPath("test"); + // This is written differently than the other runInternalAsync methods because uploadFromFile requires a file + // path, so we need to generate the temp file. + return Mono.using( + () -> new CrcInputStream(originalContent.getBlobContentHead(), options.getSize()), + inputStream -> uploadAndVerifyAsync(inputStream, downloadPath, span), + CrcInputStream::close); + } + + private Mono uploadAndVerifyAsync(CrcInputStream inputStream, Path downloadDir, Context span) { + Path uploadFilePath = generateFile(inputStream); + Path downloadFilePath = downloadDir.resolve(UUID.randomUUID() + ".txt"); + + return asyncClient.uploadFromFileWithResponse(new BlobUploadFromFileOptions(uploadFilePath.toString()) + .setParallelTransferOptions(parallelTransferOptions) + .setContentValidationAlgorithm(options.getContentValidationAlgorithm())) + .flatMap(ignored -> asyncNoFaultClient.downloadToFileWithResponse( + new BlobDownloadToFileOptions(downloadFilePath.toString()))) + .flatMap(ignored -> originalContent.checkMatch(BinaryData.fromFile(downloadFilePath), span)) + .doFinally(signal -> { + deleteFile(uploadFilePath); + deleteFile(downloadFilePath); + }); + } + + @Override + public Mono setupAsync() { + return super.setupAsync().then(originalContent.setupBlob(asyncNoFaultClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.deleteIfExists() + .then(super.cleanupAsync()); + } + + private Path getTempPath(String prefix) { + try { + return Files.createTempDirectory(prefix); + } catch (IOException e) { + throw LOGGER.logExceptionAsError(new UncheckedIOException(e)); + } + } + + private static void deleteFile(Path path) { + try { + Files.deleteIfExists(path); + } catch (Throwable e) { + LOGGER.atError() + .addKeyValue("path", path) + .log("failed to delete file", e); + } + } + + private static Path generateFile(InputStream inputStream) { + try { + File file = Files.createTempFile(CoreUtils.randomUuid().toString(), ".txt").toFile(); + file.deleteOnExit(); + Files.copy(inputStream, file.toPath(), StandardCopyOption.REPLACE_EXISTING); + return file.toPath(); + } catch (IOException e) { + throw LOGGER.logExceptionAsError(new UncheckedIOException(e)); + } + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationUploadPages.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationUploadPages.java new file mode 100644 index 000000000000..0e3c9603fe1c --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationUploadPages.java @@ -0,0 +1,82 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.util.Context; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.models.PageRange; +import com.azure.storage.blob.options.PageBlobUploadPagesOptions; +import com.azure.storage.blob.specialized.PageBlobAsyncClient; +import com.azure.storage.blob.specialized.PageBlobClient; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.stress.CrcInputStream; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.nio.ByteBuffer; + +/** + * Page blob upload pages with {@link PageBlobUploadPagesOptions#setContentValidationAlgorithm}. + */ +public class ContentValidationUploadPages extends PageBlobScenarioBase { + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncClient; + private final BlobAsyncClient asyncNoFaultClient; + private final PageBlobAsyncClient tempSetupPageBlobClient; + + public ContentValidationUploadPages(ContentValidationStressOptions options) { + super(options); + String blobName = generateBlobName(); + String tempBlobName = generateBlobName(); + + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.asyncClient = getAsyncContainerClient().getBlobAsyncClient(blobName); + BlobAsyncClient tempSetupBlobClient = getAsyncContainerClientNoFault().getBlobAsyncClient(tempBlobName); + this.tempSetupPageBlobClient = tempSetupBlobClient.getPageBlobAsyncClient(); + } + + @Override + protected void runInternal(Context span) { + try (CrcInputStream inputStream = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize())) { + PageBlobClient pageBlobClient = syncClient.getPageBlobClient(); + PageRange range = new PageRange().setStart(0).setEnd(options.getSize() - 1); + pageBlobClient.uploadPagesWithResponse( + new PageBlobUploadPagesOptions(range, inputStream) + .setContentValidationAlgorithm(options.getContentValidationAlgorithm()), + null, span); + originalContent.checkMatch(inputStream.getContentInfo(), span).block(); + } + } + + @Override + protected Mono runInternalAsync(Context span) { + PageBlobAsyncClient pageBlobAsyncClient = asyncClient.getPageBlobAsyncClient(); + Flux byteBufferFlux = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize()) + .convertStreamToByteBuffer(); + PageRange range = new PageRange().setStart(0).setEnd(options.getSize() - 1); + return pageBlobAsyncClient.uploadPagesWithResponse( + new PageBlobUploadPagesOptions(range, byteBufferFlux) + .setContentValidationAlgorithm(options.getContentValidationAlgorithm())) + .then(originalContent.checkMatch(byteBufferFlux, span)); + } + + @Override + public Mono setupAsync() { + return super.setupAsync() + .then(asyncNoFaultClient.getPageBlobAsyncClient().create(options.getSize())) + .then(tempSetupPageBlobClient.create(options.getSize())) + .then(originalContent.setupPageBlob(tempSetupPageBlobClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.getPageBlobAsyncClient().deleteIfExists() + .onErrorResume(e -> Mono.empty()) + .then(tempSetupPageBlobClient.deleteIfExists()) + .then(super.cleanupAsync()); + } +} diff --git a/sdk/storage/azure-storage-stress/pom.xml b/sdk/storage/azure-storage-stress/pom.xml index d8907fe67014..bdf518e233b8 100644 --- a/sdk/storage/azure-storage-stress/pom.xml +++ b/sdk/storage/azure-storage-stress/pom.xml @@ -19,8 +19,23 @@ - + + + 1.58.0 + + + + io.opentelemetry + opentelemetry-bom + ${opentelemetry.version} + pom + import + + + + ch.qos.logback @@ -59,6 +74,11 @@ opentelemetry-logback-appender-1.0 2.24.0-alpha + + io.opentelemetry + opentelemetry-api-incubator + 1.58.0-alpha + com.azure azure-core-metrics-opentelemetry diff --git a/sdk/storage/azure-storage-stress/src/main/java/com/azure/storage/stress/TelemetryHelper.java b/sdk/storage/azure-storage-stress/src/main/java/com/azure/storage/stress/TelemetryHelper.java index 4bf6e523eae1..b633479100b1 100644 --- a/sdk/storage/azure-storage-stress/src/main/java/com/azure/storage/stress/TelemetryHelper.java +++ b/sdk/storage/azure-storage-stress/src/main/java/com/azure/storage/stress/TelemetryHelper.java @@ -124,7 +124,7 @@ public String getDescription() { Cpu.registerObservers(otel); MemoryPools.registerObservers(otel); Threads.registerObservers(otel); - GarbageCollector.registerObservers(otel); + GarbageCollector.registerObservers(otel, true); OpenTelemetryAppender.install(otel); return otel; } From 8c8d793017cf06f95eb662cac06b439630ff9664 Mon Sep 17 00:00:00 2001 From: Isabelle Date: Mon, 1 Jun 2026 09:57:46 -0700 Subject: [PATCH 2/5] enabling fault injector --- .../scenarios-matrix.yaml | 134 +++++++++--------- 1 file changed, 67 insertions(+), 67 deletions(-) diff --git a/sdk/storage/azure-storage-blob-stress/scenarios-matrix.yaml b/sdk/storage/azure-storage-blob-stress/scenarios-matrix.yaml index 9e8c1bad70f7..7b2ef89f330f 100644 --- a/sdk/storage/azure-storage-blob-stress/scenarios-matrix.yaml +++ b/sdk/storage/azure-storage-blob-stress/scenarios-matrix.yaml @@ -5,7 +5,7 @@ matrix: testScenario: downloadtofile sync: true sizeBytes: 1024 - downloadFaults: false + downloadFaults: true durationMin: 25 imageBuildDir: "../../.." @@ -14,7 +14,7 @@ matrix: testScenario: downloadtofile sync: false sizeBytes: 1024 - downloadFaults: false + downloadFaults: true durationMin: 25 imageBuildDir: "../../.." @@ -23,7 +23,7 @@ matrix: testScenario: downloadtofile sync: true sizeBytes: "52428800" - downloadFaults: false + downloadFaults: true durationMin: 60 imageBuildDir: "../../.." @@ -32,7 +32,7 @@ matrix: testScenario: downloadstream sync: true sizeBytes: 1024 - downloadFaults: false + downloadFaults: true durationMin: 25 imageBuildDir: "../../.." @@ -41,7 +41,7 @@ matrix: testScenario: downloadstream sync: false sizeBytes: 1024 - downloadFaults: false + downloadFaults: true durationMin: 25 imageBuildDir: "../../.." @@ -50,7 +50,7 @@ matrix: testScenario: downloadstream sync: true sizeBytes: "52428800" - downloadFaults: false + downloadFaults: true durationMin: 60 imageBuildDir: "../../.." @@ -59,7 +59,7 @@ matrix: testScenario: downloadcontent sync: true sizeBytes: 1024 - downloadFaults: false + downloadFaults: true durationMin: 25 imageBuildDir: "../../.." @@ -68,7 +68,7 @@ matrix: testScenario: downloadcontent sync: true sizeBytes: "52428800" - downloadFaults: false + downloadFaults: true durationMin: 60 imageBuildDir: "../../.." @@ -77,7 +77,7 @@ matrix: testScenario: openinputstream sync: true sizeBytes: 1024 - downloadFaults: false + downloadFaults: true durationMin: 25 imageBuildDir: "../../.." @@ -86,7 +86,7 @@ matrix: testScenario: openinputstream sync: true sizeBytes: "52428800" - downloadFaults: false + downloadFaults: true durationMin: 60 imageBuildDir: "../../.." @@ -95,7 +95,7 @@ matrix: testScenario: openseekablebytechannelread sync: true sizeBytes: 1024 - downloadFaults: false + downloadFaults: true durationMin: 25 imageBuildDir: "../../.." @@ -104,7 +104,7 @@ matrix: testScenario: openseekablebytechannelread sync: true sizeBytes: "52428800" - downloadFaults: false + downloadFaults: true durationMin: 60 imageBuildDir: "../../.." @@ -113,7 +113,7 @@ matrix: testScenario: appendblock sync: true sizeBytes: 1024 - uploadFaults: false + uploadFaults: true durationMin: 25 imageBuildDir: "../../.." @@ -122,7 +122,7 @@ matrix: testScenario: appendblock sync: false sizeBytes: 1024 - uploadFaults: false + uploadFaults: true durationMin: 25 imageBuildDir: "../../.." @@ -131,7 +131,7 @@ matrix: testScenario: appendblock sync: true sizeBytes: "26214400" - uploadFaults: false + uploadFaults: true durationMin: 30 imageBuildDir: "../../.." @@ -140,7 +140,7 @@ matrix: testScenario: appendbloboutputstream sync: true sizeBytes: 1024 - uploadFaults: false + uploadFaults: true durationMin: 25 imageBuildDir: "../../.." @@ -149,7 +149,7 @@ matrix: testScenario: appendbloboutputstream sync: true sizeBytes: "10240" - uploadFaults: false + uploadFaults: true durationMin: 60 imageBuildDir: "../../.." @@ -158,7 +158,7 @@ matrix: testScenario: blockblobupload sync: true sizeBytes: 1024 - uploadFaults: false + uploadFaults: true durationMin: 25 imageBuildDir: "../../.." @@ -167,7 +167,7 @@ matrix: testScenario: blockblobupload sync: true sizeBytes: "26214400" - uploadFaults: false + uploadFaults: true durationMin: 60 imageBuildDir: "../../.." @@ -176,7 +176,7 @@ matrix: testScenario: blockbloboutputstream sync: true sizeBytes: 1024 - uploadFaults: false + uploadFaults: true durationMin: 25 imageBuildDir: "../../.." @@ -185,7 +185,7 @@ matrix: testScenario: blockbloboutputstream sync: true sizeBytes: "26214400" - uploadFaults: false + uploadFaults: true durationMin: 60 imageBuildDir: "../../.." @@ -194,7 +194,7 @@ matrix: testScenario: commitblocklist sync: true sizeBytes: 1024 - uploadFaults: false + uploadFaults: true durationMin: 25 imageBuildDir: "../../.." @@ -203,7 +203,7 @@ matrix: testScenario: commitblocklist sync: true sizeBytes: "26214400" - uploadFaults: false + uploadFaults: true durationMin: 60 imageBuildDir: "../../.." @@ -212,7 +212,7 @@ matrix: testScenario: openseekablebytechannelwrite sync: true sizeBytes: 1024 - uploadFaults: false + uploadFaults: true durationMin: 10 imageBuildDir: "../../.." @@ -221,7 +221,7 @@ matrix: testScenario: openseekablebytechannelwrite sync: true sizeBytes: "52428800" - uploadFaults: false + uploadFaults: true durationMin: 60 imageBuildDir: "../../.." @@ -230,7 +230,7 @@ matrix: testScenario: stageblock sync: true sizeBytes: 1024 - uploadFaults: false + uploadFaults: true durationMin: 25 imageBuildDir: "../../.." @@ -239,7 +239,7 @@ matrix: testScenario: stageblock sync: true sizeBytes: "26214400" - uploadFaults: false + uploadFaults: true durationMin: 60 imageBuildDir: "../../.." @@ -248,7 +248,7 @@ matrix: testScenario: pagebloboutputstream sync: true sizeBytes: 1024 - uploadFaults: false + uploadFaults: true durationMin: 25 imageBuildDir: "../../.." @@ -257,7 +257,7 @@ matrix: testScenario: pagebloboutputstream sync: true sizeBytes: "10240" - uploadFaults: false + uploadFaults: true durationMin: 60 imageBuildDir: "../../.." @@ -266,7 +266,7 @@ matrix: testScenario: uploadpages sync: true sizeBytes: 1024 - uploadFaults: false + uploadFaults: true durationMin: 25 imageBuildDir: "../../.." @@ -275,7 +275,7 @@ matrix: testScenario: uploadpages sync: true sizeBytes: "4194304" - uploadFaults: false + uploadFaults: true durationMin: 60 imageBuildDir: "../../.." @@ -284,7 +284,7 @@ matrix: testScenario: upload sync: true sizeBytes: 1024 - uploadFaults: false + uploadFaults: true durationMin: 25 imageBuildDir: "../../.." @@ -293,7 +293,7 @@ matrix: testScenario: upload sync: false sizeBytes: 1024 - uploadFaults: false + uploadFaults: true durationMin: 25 imageBuildDir: "../../.." @@ -302,7 +302,7 @@ matrix: testScenario: upload sync: true sizeBytes: "52428800" - uploadFaults: false + uploadFaults: true durationMin: 60 imageBuildDir: "../../.." @@ -311,7 +311,7 @@ matrix: testScenario: uploadfromfile sync: true sizeBytes: 1024 - uploadFaults: false + uploadFaults: true durationMin: 25 imageBuildDir: "../../.." @@ -320,7 +320,7 @@ matrix: testScenario: uploadfromfile sync: true sizeBytes: "52428800" - uploadFaults: false + uploadFaults: true durationMin: 60 imageBuildDir: "../../.." @@ -331,7 +331,7 @@ matrix: testScenario: contentvalidationappendblock sync: true sizeBytes: 1024 - uploadFaults: false + uploadFaults: true durationMin: 25 imageBuildDir: "../../.." @@ -339,7 +339,7 @@ matrix: testScenario: contentvalidationappendblock sync: true sizeBytes: "26214400" - uploadFaults: false + uploadFaults: true durationMin: 30 imageBuildDir: "../../.." @@ -347,7 +347,7 @@ matrix: testScenario: contentvalidationappendblock sync: false sizeBytes: 1024 - uploadFaults: false + uploadFaults: true durationMin: 25 imageBuildDir: "../../.." @@ -355,7 +355,7 @@ matrix: testScenario: contentvalidationappendblock sync: false sizeBytes: "26214400" - uploadFaults: false + uploadFaults: true durationMin: 30 imageBuildDir: "../../.." @@ -363,7 +363,7 @@ matrix: testScenario: contentvalidationappendbloboutputstream sync: true sizeBytes: 1024 - uploadFaults: false + uploadFaults: true durationMin: 25 imageBuildDir: "../../.." @@ -371,7 +371,7 @@ matrix: testScenario: contentvalidationappendbloboutputstream sync: true sizeBytes: "10240" - uploadFaults: false + uploadFaults: true durationMin: 60 imageBuildDir: "../../.." @@ -379,7 +379,7 @@ matrix: testScenario: contentvalidationblockblobupload sync: true sizeBytes: 1024 - uploadFaults: false + uploadFaults: true durationMin: 25 imageBuildDir: "../../.." @@ -387,7 +387,7 @@ matrix: testScenario: contentvalidationblockblobupload sync: true sizeBytes: "26214400" - uploadFaults: false + uploadFaults: true durationMin: 60 imageBuildDir: "../../.." @@ -395,7 +395,7 @@ matrix: testScenario: contentvalidationblockbloboutputstream sync: true sizeBytes: 1024 - uploadFaults: false + uploadFaults: true durationMin: 25 imageBuildDir: "../../.." @@ -403,7 +403,7 @@ matrix: testScenario: contentvalidationblockbloboutputstream sync: true sizeBytes: "26214400" - uploadFaults: false + uploadFaults: true durationMin: 60 imageBuildDir: "../../.." @@ -411,7 +411,7 @@ matrix: testScenario: contentvalidationseekablebytechannelwrite sync: true sizeBytes: 1024 - uploadFaults: false + uploadFaults: true durationMin: 10 imageBuildDir: "../../.." @@ -419,7 +419,7 @@ matrix: testScenario: contentvalidationseekablebytechannelwrite sync: true sizeBytes: "52428800" - uploadFaults: false + uploadFaults: true durationMin: 60 imageBuildDir: "../../.." @@ -427,7 +427,7 @@ matrix: testScenario: contentvalidationstageblock sync: true sizeBytes: 1024 - uploadFaults: false + uploadFaults: true durationMin: 25 imageBuildDir: "../../.." @@ -435,7 +435,7 @@ matrix: testScenario: contentvalidationstageblock sync: true sizeBytes: "26214400" - uploadFaults: false + uploadFaults: true durationMin: 60 imageBuildDir: "../../.." @@ -443,7 +443,7 @@ matrix: testScenario: contentvalidationstageblock sync: false sizeBytes: 1024 - uploadFaults: false + uploadFaults: true durationMin: 25 imageBuildDir: "../../.." @@ -451,7 +451,7 @@ matrix: testScenario: contentvalidationstageblock sync: false sizeBytes: "26214400" - uploadFaults: false + uploadFaults: true durationMin: 60 imageBuildDir: "../../.." @@ -459,7 +459,7 @@ matrix: testScenario: contentvalidationpagebloboutputstream sync: true sizeBytes: 1024 - uploadFaults: false + uploadFaults: true durationMin: 25 imageBuildDir: "../../.." @@ -467,7 +467,7 @@ matrix: testScenario: contentvalidationpagebloboutputstream sync: true sizeBytes: "10240" - uploadFaults: false + uploadFaults: true durationMin: 60 imageBuildDir: "../../.." @@ -475,7 +475,7 @@ matrix: testScenario: contentvalidationuploadpages sync: true sizeBytes: 1024 - uploadFaults: false + uploadFaults: true durationMin: 25 imageBuildDir: "../../.." @@ -483,7 +483,7 @@ matrix: testScenario: contentvalidationuploadpages sync: true sizeBytes: "4194304" - uploadFaults: false + uploadFaults: true durationMin: 60 imageBuildDir: "../../.." @@ -491,7 +491,7 @@ matrix: testScenario: contentvalidationuploadpages sync: false sizeBytes: 1024 - uploadFaults: false + uploadFaults: true durationMin: 25 imageBuildDir: "../../.." @@ -499,7 +499,7 @@ matrix: testScenario: contentvalidationuploadpages sync: false sizeBytes: "4194304" - uploadFaults: false + uploadFaults: true durationMin: 60 imageBuildDir: "../../.." @@ -507,7 +507,7 @@ matrix: testScenario: contentvalidationupload sync: true sizeBytes: 1024 - uploadFaults: false + uploadFaults: true durationMin: 25 imageBuildDir: "../../.." @@ -515,7 +515,7 @@ matrix: testScenario: contentvalidationupload sync: true sizeBytes: "52428800" - uploadFaults: false + uploadFaults: true durationMin: 60 imageBuildDir: "../../.." @@ -523,7 +523,7 @@ matrix: testScenario: contentvalidationupload sync: false sizeBytes: 1024 - uploadFaults: false + uploadFaults: true durationMin: 25 imageBuildDir: "../../.." @@ -531,7 +531,7 @@ matrix: testScenario: contentvalidationupload sync: false sizeBytes: "52428800" - uploadFaults: false + uploadFaults: true durationMin: 60 imageBuildDir: "../../.." @@ -539,7 +539,7 @@ matrix: testScenario: contentvalidationuploadfromfile sync: true sizeBytes: 1024 - uploadFaults: false + uploadFaults: true durationMin: 25 imageBuildDir: "../../.." @@ -547,7 +547,7 @@ matrix: testScenario: contentvalidationuploadfromfile sync: true sizeBytes: "52428800" - uploadFaults: false + uploadFaults: true durationMin: 60 imageBuildDir: "../../.." @@ -555,7 +555,7 @@ matrix: testScenario: contentvalidationuploadfromfile sync: false sizeBytes: 1024 - uploadFaults: false + uploadFaults: true durationMin: 25 imageBuildDir: "../../.." @@ -563,6 +563,6 @@ matrix: testScenario: contentvalidationuploadfromfile sync: false sizeBytes: "52428800" - uploadFaults: false + uploadFaults: true durationMin: 60 - imageBuildDir: "../../.." \ No newline at end of file + imageBuildDir: "../../.." From 87e6b936452786e757b7d0d4562a88c756d5728f Mon Sep 17 00:00:00 2001 From: Isabelle Date: Mon, 1 Jun 2026 09:59:15 -0700 Subject: [PATCH 3/5] restoring commitblocklist --- .../storage/blob/stress/CommitBlockList.java | 39 +++++++++---------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/CommitBlockList.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/CommitBlockList.java index 5e63058a4ab5..310c7618e5d9 100644 --- a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/CommitBlockList.java +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/CommitBlockList.java @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + package com.azure.storage.blob.stress; import com.azure.core.util.Context; @@ -8,6 +11,7 @@ import com.azure.storage.blob.specialized.BlockBlobAsyncClient; import com.azure.storage.blob.specialized.BlockBlobClient; import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.Utility; import com.azure.storage.stress.CrcInputStream; import com.azure.storage.stress.StorageStressOptions; import reactor.core.publisher.Flux; @@ -15,17 +19,10 @@ import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; -import java.time.Duration; import java.util.Base64; import java.util.Collections; public class CommitBlockList extends BlobScenarioBase { - // Per-operation ceilings. Must be smaller than the Azure LB idle timeout (~4 min) - // so that a stalled request fails fast rather than hangs the whole test run. - private static final Duration STAGE_TIMEOUT = Duration.ofSeconds(120); - private static final Duration COMMIT_TIMEOUT = Duration.ofSeconds(30); - private static final Duration CHECK_MATCH_TIMEOUT = Duration.ofSeconds(30); - private final OriginalContent originalContent = new OriginalContent(); private final BlobClient syncClient; private final BlobAsyncClient asyncClient; @@ -48,11 +45,14 @@ protected void runInternal(Context span) throws Exception { String blockId = Base64.getEncoder().encodeToString(CoreUtils.randomUuid().toString() .getBytes(StandardCharsets.UTF_8)); try (CrcInputStream inputStream = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize())) { - blockBlobClientNoFault.stageBlockWithResponse(blockId, inputStream, options.getSize(), null, null, - STAGE_TIMEOUT, span); + // First perform non-faulted stage block to send data to the service + blockBlobClientNoFault.stageBlockWithResponse(blockId, inputStream, options.getSize(), null, null, null, + span); + // Then perform faulted commit block list to commit the block blockBlobClient.commitBlockListWithResponse( - new BlockBlobCommitBlockListOptions(Collections.singletonList(blockId)), COMMIT_TIMEOUT, span); - originalContent.checkMatch(inputStream.getContentInfo(), span).block(CHECK_MATCH_TIMEOUT); + new BlockBlobCommitBlockListOptions(Collections.singletonList(blockId)), null, span); + // Confirm the CRC matches for the uploaded input stream + originalContent.checkMatch(inputStream.getContentInfo(), span).block(); } } @@ -64,14 +64,13 @@ protected Mono runInternalAsync(Context span) { .getBytes(StandardCharsets.UTF_8)); Flux byteBufferFlux = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize()) .convertStreamToByteBuffer(); - return blockBlobAsyncClientNoFault - .stageBlockWithResponse(blockId, byteBufferFlux, options.getSize(), null, null) - .timeout(STAGE_TIMEOUT) - .then(blockBlobAsyncClient - .commitBlockListWithResponse( - new BlockBlobCommitBlockListOptions(Collections.singletonList(blockId))) - .timeout(COMMIT_TIMEOUT)) - .then(originalContent.checkMatch(byteBufferFlux, span).timeout(CHECK_MATCH_TIMEOUT)); + // First perform non-faulted stage block to send data to the service + return blockBlobAsyncClientNoFault.stageBlockWithResponse(blockId, byteBufferFlux, options.getSize(), null, null) + // Then perform faulted commit block list to commit the block + .then(blockBlobAsyncClient.commitBlockListWithResponse( + new BlockBlobCommitBlockListOptions(Collections.singletonList(blockId)))) + // Confirm the CRC matches for the uploaded byte buffer flux + .then(originalContent.checkMatch(byteBufferFlux, span)); } @Override @@ -84,4 +83,4 @@ public Mono cleanupAsync() { return asyncNoFaultClient.deleteIfExists() .then(super.cleanupAsync()); } -} \ No newline at end of file +} From 81e9f3bcd3be59a3038bfbe34505a7ea0944f417 Mon Sep 17 00:00:00 2001 From: Isabelle Date: Mon, 1 Jun 2026 14:00:29 -0700 Subject: [PATCH 4/5] renaming files and hardcoding crc64 option --- .../scenarios-matrix.yaml | 123 +++++++++--------- .../com/azure/storage/blob/stress/App.java | 20 +-- ...a => AppendBlobOutputStreamWithCRC64.java} | 28 +++- ...ndBlock.java => AppendBlockWithCRC64.java} | 12 +- ...va => BlockBlobOutputStreamWithCRC64.java} | 12 +- ...oad.java => BlockBlobUploadWithCRC64.java} | 13 +- .../ContentValidationStressOptions.java | 26 ---- ...ava => PageBlobOutputStreamWithCRC64.java} | 12 +- ...=> SeekableByteChannelWriteWithCRC64.java} | 13 +- ...ageBlock.java => StageBlockWithCRC64.java} | 12 +- ...File.java => UploadFromFileWithCRC64.java} | 14 +- ...adPages.java => UploadPagesWithCRC64.java} | 12 +- ...dationUpload.java => UploadWithCRC64.java} | 13 +- 13 files changed, 157 insertions(+), 153 deletions(-) rename sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/{ContentValidationAppendBlobOutputStream.java => AppendBlobOutputStreamWithCRC64.java} (72%) rename sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/{ContentValidationAppendBlock.java => AppendBlockWithCRC64.java} (86%) rename sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/{ContentValidationBlockBlobOutputStream.java => BlockBlobOutputStreamWithCRC64.java} (86%) rename sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/{ContentValidationBlockBlobUpload.java => BlockBlobUploadWithCRC64.java} (85%) delete mode 100644 sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationStressOptions.java rename sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/{ContentValidationPageBlobOutputStream.java => PageBlobOutputStreamWithCRC64.java} (91%) rename sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/{ContentValidationSeekableByteChannelWrite.java => SeekableByteChannelWriteWithCRC64.java} (84%) rename sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/{ContentValidationStageBlock.java => StageBlockWithCRC64.java} (88%) rename sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/{ContentValidationUploadFromFile.java => UploadFromFileWithCRC64.java} (91%) rename sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/{ContentValidationUploadPages.java => UploadPagesWithCRC64.java} (87%) rename sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/{ContentValidationUpload.java => UploadWithCRC64.java} (81%) diff --git a/sdk/storage/azure-storage-blob-stress/scenarios-matrix.yaml b/sdk/storage/azure-storage-blob-stress/scenarios-matrix.yaml index 7b2ef89f330f..3f83b7995432 100644 --- a/sdk/storage/azure-storage-blob-stress/scenarios-matrix.yaml +++ b/sdk/storage/azure-storage-blob-stress/scenarios-matrix.yaml @@ -324,243 +324,240 @@ matrix: durationMin: 60 imageBuildDir: "../../.." - # --- Content validation (ContentValidation*; default CRC64). Scenario keys are short: K8s label - # testInstance = "{Scenario}-{BaseName}-{revision}" must be <= 63 chars (see stress-test-job.yaml). - - cv-appendblock-sm: - testScenario: contentvalidationappendblock + crc64appendblock-sm: + testScenario: appendblockwithcrc64 sync: true sizeBytes: 1024 uploadFaults: true durationMin: 25 imageBuildDir: "../../.." - cv-appendblock-lg: - testScenario: contentvalidationappendblock + crc64appendblock-lg: + testScenario: appendblockwithcrc64 sync: true sizeBytes: "26214400" uploadFaults: true durationMin: 30 imageBuildDir: "../../.." - cv-appendblock-async-sm: - testScenario: contentvalidationappendblock + crc64appendblockasync-sm: + testScenario: appendblockwithcrc64 sync: false sizeBytes: 1024 uploadFaults: true durationMin: 25 imageBuildDir: "../../.." - cv-appendblock-async-lg: - testScenario: contentvalidationappendblock + crc64appendblockasync-lg: + testScenario: appendblockwithcrc64 sync: false sizeBytes: "26214400" uploadFaults: true durationMin: 30 imageBuildDir: "../../.." - cv-appendbloboutputstream-sm: - testScenario: contentvalidationappendbloboutputstream + crc64appendoutputstream-sm: + testScenario: appendbloboutputstreamwithcrc64 sync: true sizeBytes: 1024 uploadFaults: true durationMin: 25 imageBuildDir: "../../.." - cv-appendbloboutputstream-lg: - testScenario: contentvalidationappendbloboutputstream + crc64appendoutputstream-lg: + testScenario: appendbloboutputstreamwithcrc64 sync: true sizeBytes: "10240" uploadFaults: true durationMin: 60 imageBuildDir: "../../.." - cv-blockblobupload-sm: - testScenario: contentvalidationblockblobupload + crc64blockblobupload-sm: + testScenario: blockblobuploadwithcrc64 sync: true sizeBytes: 1024 uploadFaults: true durationMin: 25 imageBuildDir: "../../.." - cv-blockblobupload-lg: - testScenario: contentvalidationblockblobupload + crc64blockblobupload-lg: + testScenario: blockblobuploadwithcrc64 sync: true sizeBytes: "26214400" uploadFaults: true durationMin: 60 imageBuildDir: "../../.." - cv-blockbloboutputstream-sm: - testScenario: contentvalidationblockbloboutputstream + crc64blockoutputstream-sm: + testScenario: blockbloboutputstreamwithcrc64 sync: true sizeBytes: 1024 uploadFaults: true durationMin: 25 imageBuildDir: "../../.." - cv-blockbloboutputstream-lg: - testScenario: contentvalidationblockbloboutputstream + crc64blockoutputstream-lg: + testScenario: blockbloboutputstreamwithcrc64 sync: true sizeBytes: "26214400" uploadFaults: true durationMin: 60 imageBuildDir: "../../.." - cv-seekablebytechannelwrite-sm: - testScenario: contentvalidationseekablebytechannelwrite + crc64bytechannelwrite-sm: + testScenario: seekablebytechannelwritewithcrc64 sync: true sizeBytes: 1024 uploadFaults: true durationMin: 10 imageBuildDir: "../../.." - cv-seekablebytechannelwrite-lg: - testScenario: contentvalidationseekablebytechannelwrite + crc64bytechannelwrite-lg: + testScenario: seekablebytechannelwritewithcrc64 sync: true sizeBytes: "52428800" uploadFaults: true durationMin: 60 imageBuildDir: "../../.." - cv-stageblock-sm: - testScenario: contentvalidationstageblock + crc64stageblock-sm: + testScenario: stageblockwithcrc64 sync: true sizeBytes: 1024 uploadFaults: true durationMin: 25 imageBuildDir: "../../.." - cv-stageblock-lg: - testScenario: contentvalidationstageblock + crc64stageblock-lg: + testScenario: stageblockwithcrc64 sync: true sizeBytes: "26214400" uploadFaults: true durationMin: 60 imageBuildDir: "../../.." - cv-stageblock-async-sm: - testScenario: contentvalidationstageblock + crc64stageblockasync-sm: + testScenario: stageblockwithcrc64 sync: false sizeBytes: 1024 uploadFaults: true durationMin: 25 imageBuildDir: "../../.." - cv-stageblock-async-lg: - testScenario: contentvalidationstageblock + crc64stageblockasync-lg: + testScenario: stageblockwithcrc64 sync: false sizeBytes: "26214400" uploadFaults: true durationMin: 60 imageBuildDir: "../../.." - cv-pagebloboutputstream-sm: - testScenario: contentvalidationpagebloboutputstream + crc64pageoutputstream-sm: + testScenario: pagebloboutputstreamwithcrc64 sync: true sizeBytes: 1024 uploadFaults: true durationMin: 25 imageBuildDir: "../../.." - cv-pagebloboutputstream-lg: - testScenario: contentvalidationpagebloboutputstream + crc64pageoutputstream-lg: + testScenario: pagebloboutputstreamwithcrc64 sync: true sizeBytes: "10240" uploadFaults: true durationMin: 60 imageBuildDir: "../../.." - cv-uploadpages-sm: - testScenario: contentvalidationuploadpages + crc64uploadpages-sm: + testScenario: uploadpageswithcrc64 sync: true sizeBytes: 1024 uploadFaults: true durationMin: 25 imageBuildDir: "../../.." - cv-uploadpages-lg: - testScenario: contentvalidationuploadpages + crc64uploadpages-lg: + testScenario: uploadpageswithcrc64 sync: true sizeBytes: "4194304" uploadFaults: true durationMin: 60 imageBuildDir: "../../.." - cv-uploadpages-async-sm: - testScenario: contentvalidationuploadpages + crc64uploadpagesasync-sm: + testScenario: uploadpageswithcrc64 sync: false sizeBytes: 1024 uploadFaults: true durationMin: 25 imageBuildDir: "../../.." - cv-uploadpages-async-lg: - testScenario: contentvalidationuploadpages + crc64uploadpagesasync-lg: + testScenario: uploadpageswithcrc64 sync: false sizeBytes: "4194304" uploadFaults: true durationMin: 60 imageBuildDir: "../../.." - cv-upload-sm: - testScenario: contentvalidationupload + crc64upload-sm: + testScenario: uploadwithcrc64 sync: true sizeBytes: 1024 uploadFaults: true durationMin: 25 imageBuildDir: "../../.." - cv-upload-lg: - testScenario: contentvalidationupload + crc64upload-lg: + testScenario: uploadwithcrc64 sync: true sizeBytes: "52428800" uploadFaults: true durationMin: 60 imageBuildDir: "../../.." - cv-upload-async-sm: - testScenario: contentvalidationupload + crc64uploadasync-sm: + testScenario: uploadwithcrc64 sync: false sizeBytes: 1024 uploadFaults: true durationMin: 25 imageBuildDir: "../../.." - cv-upload-async-lg: - testScenario: contentvalidationupload + crc64uploadasync-lg: + testScenario: uploadwithcrc64 sync: false sizeBytes: "52428800" uploadFaults: true durationMin: 60 imageBuildDir: "../../.." - cv-uploadfromfile-sm: - testScenario: contentvalidationuploadfromfile + crc64uploadfromfile-sm: + testScenario: uploadfromfilewithcrc64 sync: true sizeBytes: 1024 uploadFaults: true durationMin: 25 imageBuildDir: "../../.." - cv-uploadfromfile-lg: - testScenario: contentvalidationuploadfromfile + crc64uploadfromfile-lg: + testScenario: uploadfromfilewithcrc64 sync: true sizeBytes: "52428800" uploadFaults: true durationMin: 60 imageBuildDir: "../../.." - cv-uploadfromfile-async-sm: - testScenario: contentvalidationuploadfromfile + crc64uploadfromfileasync-sm: + testScenario: uploadfromfilewithcrc64 sync: false sizeBytes: 1024 uploadFaults: true durationMin: 25 imageBuildDir: "../../.." - cv-uploadfromfile-async-lg: - testScenario: contentvalidationuploadfromfile + crc64uploadfromfileasync-lg: + testScenario: uploadfromfilewithcrc64 sync: false sizeBytes: "52428800" uploadFaults: true diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/App.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/App.java index 3f3219930596..6fdf06bec58a 100644 --- a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/App.java +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/App.java @@ -15,16 +15,16 @@ public static void main(String[] args) { BlockBlobOutputStream.class, BlockBlobUpload.class, CommitBlockList.class, - ContentValidationAppendBlobOutputStream.class, - ContentValidationAppendBlock.class, - ContentValidationBlockBlobOutputStream.class, - ContentValidationBlockBlobUpload.class, - ContentValidationPageBlobOutputStream.class, - ContentValidationStageBlock.class, - ContentValidationSeekableByteChannelWrite.class, - ContentValidationUpload.class, - ContentValidationUploadFromFile.class, - ContentValidationUploadPages.class, + AppendBlobOutputStreamWithCRC64.class, + AppendBlockWithCRC64.class, + BlockBlobOutputStreamWithCRC64.class, + BlockBlobUploadWithCRC64.class, + PageBlobOutputStreamWithCRC64.class, + StageBlockWithCRC64.class, + SeekableByteChannelWriteWithCRC64.class, + UploadWithCRC64.class, + UploadFromFileWithCRC64.class, + UploadPagesWithCRC64.class, DownloadToFile.class, DownloadStream.class, DownloadContent.class, diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationAppendBlobOutputStream.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/AppendBlobOutputStreamWithCRC64.java similarity index 72% rename from sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationAppendBlobOutputStream.java rename to sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/AppendBlobOutputStreamWithCRC64.java index b2b9d1845fa4..7ea8a3841f32 100644 --- a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationAppendBlobOutputStream.java +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/AppendBlobOutputStreamWithCRC64.java @@ -3,14 +3,32 @@ package com.azure.storage.blob.stress; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.StorageStressOptions; import com.azure.core.util.Context; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.StorageStressOptions; import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.StorageStressOptions; import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.StorageStressOptions; import com.azure.storage.blob.BlobClient; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.StorageStressOptions; import com.azure.storage.blob.options.AppendBlobOutputStreamOptions; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.StorageStressOptions; import com.azure.storage.blob.specialized.AppendBlobClient; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.StorageStressOptions; import com.azure.storage.blob.specialized.BlobOutputStream; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.StorageStressOptions; import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.StorageStressOptions; import com.azure.storage.stress.CrcInputStream; import reactor.core.publisher.Mono; @@ -19,17 +37,17 @@ import static com.azure.core.util.FluxUtil.monoError; /** - * Append blob output stream with {@link AppendBlobOutputStreamOptions#setContentValidationAlgorithm} (sync only). + * Append blob output stream with CRC64 enabled (sync only). */ -public class ContentValidationAppendBlobOutputStream extends BlobScenarioBase { - private static final ClientLogger LOGGER = new ClientLogger(ContentValidationAppendBlobOutputStream.class); +public class AppendBlobOutputStreamWithCRC64 extends BlobScenarioBase { + private static final ClientLogger LOGGER = new ClientLogger(AppendBlobOutputStreamWithCRC64.class); private final OriginalContent originalContent = new OriginalContent(); private final BlobClient syncClient; private final BlobAsyncClient asyncNoFaultClient; /** Separate blob used to upload reference content for {@link OriginalContent} checksum (block blob). */ private final BlobAsyncClient tempSetupBlobClient; - public ContentValidationAppendBlobOutputStream(ContentValidationStressOptions options) { + public AppendBlobOutputStreamWithCRC64(StorageStressOptions options) { super(options); String blobName = generateBlobName(); String tempBlobName = generateBlobName(); @@ -51,7 +69,7 @@ protected void runInternal(Context span) throws IOException { appendBlobClient.create(true); AppendBlobOutputStreamOptions streamOptions = new AppendBlobOutputStreamOptions() - .setContentValidationAlgorithm(options.getContentValidationAlgorithm()); + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); try (CrcInputStream inputStream = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize()); BlobOutputStream outputStream = appendBlobClient.getBlobOutputStream(streamOptions)) { diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationAppendBlock.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/AppendBlockWithCRC64.java similarity index 86% rename from sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationAppendBlock.java rename to sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/AppendBlockWithCRC64.java index bcbaf7681742..360d19995080 100644 --- a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationAppendBlock.java +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/AppendBlockWithCRC64.java @@ -10,23 +10,25 @@ import com.azure.storage.blob.specialized.AppendBlobAsyncClient; import com.azure.storage.blob.specialized.AppendBlobClient; import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; import com.azure.storage.stress.CrcInputStream; +import com.azure.storage.stress.StorageStressOptions; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import java.nio.ByteBuffer; /** - * Append block with {@link AppendBlobAppendBlockOptions#setContentValidationAlgorithm}. + * Append block with CRC64 enabled. */ -public class ContentValidationAppendBlock extends BlobScenarioBase { +public class AppendBlockWithCRC64 extends BlobScenarioBase { private final OriginalContent originalContent = new OriginalContent(); private final BlobClient syncClient; private final BlobAsyncClient asyncClient; private final BlobAsyncClient asyncNoFaultClient; private final BlobAsyncClient tempSetupBlobClient; - public ContentValidationAppendBlock(ContentValidationStressOptions options) { + public AppendBlockWithCRC64(StorageStressOptions options) { super(options); String blobName = generateBlobName(); String tempBlobName = generateBlobName(); @@ -43,7 +45,7 @@ protected void runInternal(Context span) { AppendBlobClient appendBlobClient = syncClient.getAppendBlobClient(); appendBlobClient.appendBlockWithResponse( new AppendBlobAppendBlockOptions(inputStream, options.getSize()) - .setContentValidationAlgorithm(options.getContentValidationAlgorithm()), + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64), null, span); originalContent.checkMatch(inputStream.getContentInfo(), span).block(); } @@ -56,7 +58,7 @@ protected Mono runInternalAsync(Context span) { .convertStreamToByteBuffer(); return appendBlobAsyncClient.appendBlockWithResponse( new AppendBlobAppendBlockOptions(byteBufferFlux, options.getSize()) - .setContentValidationAlgorithm(options.getContentValidationAlgorithm())) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64)) .then(originalContent.checkMatch(byteBufferFlux, span)); } diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationBlockBlobOutputStream.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/BlockBlobOutputStreamWithCRC64.java similarity index 86% rename from sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationBlockBlobOutputStream.java rename to sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/BlockBlobOutputStreamWithCRC64.java index 175b648002a5..1873ac740d72 100644 --- a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationBlockBlobOutputStream.java +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/BlockBlobOutputStreamWithCRC64.java @@ -12,7 +12,9 @@ import com.azure.storage.blob.specialized.BlobOutputStream; import com.azure.storage.blob.specialized.BlockBlobClient; import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; import com.azure.storage.stress.CrcInputStream; +import com.azure.storage.stress.StorageStressOptions; import reactor.core.publisher.Mono; import java.io.IOException; @@ -20,16 +22,16 @@ import static com.azure.core.util.FluxUtil.monoError; /** - * Block blob output stream with {@link BlockBlobOutputStreamOptions#setContentValidationAlgorithm} (sync only). + * Block blob output stream with CRC64 enabled (sync only). */ -public class ContentValidationBlockBlobOutputStream extends BlobScenarioBase { - private static final ClientLogger LOGGER = new ClientLogger(ContentValidationBlockBlobOutputStream.class); +public class BlockBlobOutputStreamWithCRC64 extends BlobScenarioBase { + private static final ClientLogger LOGGER = new ClientLogger(BlockBlobOutputStreamWithCRC64.class); private final OriginalContent originalContent = new OriginalContent(); private final BlobClient syncClient; private final BlobAsyncClient asyncNoFaultClient; private final ParallelTransferOptions parallelTransferOptions; - public ContentValidationBlockBlobOutputStream(ContentValidationStressOptions options) { + public BlockBlobOutputStreamWithCRC64(StorageStressOptions options) { super(options); String blobName = generateBlobName(); this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); @@ -42,7 +44,7 @@ protected void runInternal(Context span) throws IOException { BlockBlobClient blockBlobClient = syncClient.getBlockBlobClient(); BlockBlobOutputStreamOptions streamOptions = new BlockBlobOutputStreamOptions() .setParallelTransferOptions(parallelTransferOptions) - .setContentValidationAlgorithm(options.getContentValidationAlgorithm()); + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); try (CrcInputStream inputStream = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize()); BlobOutputStream outputStream = blockBlobClient.getBlobOutputStream(streamOptions, span)) { diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationBlockBlobUpload.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/BlockBlobUploadWithCRC64.java similarity index 85% rename from sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationBlockBlobUpload.java rename to sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/BlockBlobUploadWithCRC64.java index e86a410e1945..b88b9b5c808d 100644 --- a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationBlockBlobUpload.java +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/BlockBlobUploadWithCRC64.java @@ -10,23 +10,24 @@ import com.azure.storage.blob.specialized.BlockBlobAsyncClient; import com.azure.storage.blob.specialized.BlockBlobClient; import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; import com.azure.storage.stress.CrcInputStream; +import com.azure.storage.stress.StorageStressOptions; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import java.nio.ByteBuffer; /** - * Single-shot block blob upload with request content validation - * ({@link BlockBlobSimpleUploadOptions#setContentValidationAlgorithm}). + * Single-shot block blob upload with request content validation with CRC64 enabled. */ -public class ContentValidationBlockBlobUpload extends BlobScenarioBase { +public class BlockBlobUploadWithCRC64 extends BlobScenarioBase { private final OriginalContent originalContent = new OriginalContent(); private final BlobClient syncClient; private final BlobAsyncClient asyncClient; private final BlobAsyncClient asyncNoFaultClient; - public ContentValidationBlockBlobUpload(ContentValidationStressOptions options) { + public BlockBlobUploadWithCRC64(StorageStressOptions options) { super(options); String blobName = generateBlobName(); this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); @@ -40,7 +41,7 @@ protected void runInternal(Context span) { try (CrcInputStream inputStream = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize())) { blockBlobClient.uploadWithResponse( new BlockBlobSimpleUploadOptions(inputStream, options.getSize()) - .setContentValidationAlgorithm(options.getContentValidationAlgorithm()), + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64), null, span); originalContent.checkMatch(inputStream.getContentInfo(), span).block(); } @@ -53,7 +54,7 @@ protected Mono runInternalAsync(Context span) { BlockBlobAsyncClient blockBlobAsyncClient = asyncClient.getBlockBlobAsyncClient(); return blockBlobAsyncClient.uploadWithResponse( new BlockBlobSimpleUploadOptions(byteBufferFlux, options.getSize()) - .setContentValidationAlgorithm(options.getContentValidationAlgorithm())) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64)) .then(originalContent.checkMatch(byteBufferFlux, span)); } diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationStressOptions.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationStressOptions.java deleted file mode 100644 index 1e92d331d33e..000000000000 --- a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationStressOptions.java +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.storage.blob.stress; - -import com.azure.storage.common.ContentValidationAlgorithm; -import com.azure.storage.stress.StorageStressOptions; -import com.beust.jcommander.Parameter; - -/** - * Options for stress scenarios that enable transactional request content validation on uploads - * (CRC64 / structured message). See {@link com.azure.storage.blob.BlobContentValidationUploadTests}. - */ -public class ContentValidationStressOptions extends StorageStressOptions { - /** - * Request checksum behavior for upload APIs. Use CRC64 or AUTO to exercise content validation. - * MD5 is not supported for uploadFromFile. NONE disables request validation. - */ - @Parameter(names = { "--requestChecksumAlgorithm" }, - description = "CRC64 (default), AUTO, NONE, or MD5 (not valid for upload-from-file scenarios)") - private ContentValidationAlgorithm requestChecksumAlgorithm = ContentValidationAlgorithm.CRC64; - - public ContentValidationAlgorithm getContentValidationAlgorithm() { - return requestChecksumAlgorithm; - } -} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationPageBlobOutputStream.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/PageBlobOutputStreamWithCRC64.java similarity index 91% rename from sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationPageBlobOutputStream.java rename to sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/PageBlobOutputStreamWithCRC64.java index b49a0305f12f..f4dda3295b18 100644 --- a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationPageBlobOutputStream.java +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/PageBlobOutputStreamWithCRC64.java @@ -13,7 +13,9 @@ import com.azure.storage.blob.specialized.PageBlobAsyncClient; import com.azure.storage.blob.specialized.PageBlobClient; import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; import com.azure.storage.stress.CrcInputStream; +import com.azure.storage.stress.StorageStressOptions; import reactor.core.publisher.Mono; import java.io.ByteArrayOutputStream; @@ -22,17 +24,17 @@ import static com.azure.core.util.FluxUtil.monoError; /** - * Page blob output stream with {@link PageBlobOutputStreamOptions#setContentValidationAlgorithm} (sync only). + * Page blob output stream with CRC64 enabled (sync only). */ -public class ContentValidationPageBlobOutputStream extends PageBlobScenarioBase { - private static final ClientLogger LOGGER = new ClientLogger(ContentValidationPageBlobOutputStream.class); +public class PageBlobOutputStreamWithCRC64 extends PageBlobScenarioBase { + private static final ClientLogger LOGGER = new ClientLogger(PageBlobOutputStreamWithCRC64.class); private final OriginalContent originalContent = new OriginalContent(); private final BlobClient syncClient; private final BlobAsyncClient asyncNoFaultClient; /** Page blob used only to seed {@link OriginalContent} (same pattern as {@link PageBlobOutputStream}). */ private final PageBlobAsyncClient tempSetupPageBlobClient; - public ContentValidationPageBlobOutputStream(ContentValidationStressOptions options) { + public PageBlobOutputStreamWithCRC64(StorageStressOptions options) { super(options); String blobName = generateBlobName(); String tempBlobName = generateBlobName(); @@ -48,7 +50,7 @@ protected void runInternal(Context span) throws IOException { PageBlobClient pageBlobClient = syncClient.getPageBlobClient(); PageBlobOutputStreamOptions streamOptions = new PageBlobOutputStreamOptions( new PageRange().setStart(0).setEnd(options.getSize() - 1)) - .setContentValidationAlgorithm(options.getContentValidationAlgorithm()); + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); try (CrcInputStream inputStream = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize()); BlobOutputStream outputStream = pageBlobClient.getBlobOutputStream(streamOptions)) { diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationSeekableByteChannelWrite.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/SeekableByteChannelWriteWithCRC64.java similarity index 84% rename from sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationSeekableByteChannelWrite.java rename to sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/SeekableByteChannelWriteWithCRC64.java index 65afda98636f..5f345bbfd5f1 100644 --- a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationSeekableByteChannelWrite.java +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/SeekableByteChannelWriteWithCRC64.java @@ -10,8 +10,10 @@ import com.azure.storage.blob.options.BlockBlobSeekableByteChannelWriteOptions; import com.azure.storage.blob.specialized.BlockBlobClient; import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; import com.azure.storage.common.implementation.StorageSeekableByteChannel; import com.azure.storage.stress.CrcInputStream; +import com.azure.storage.stress.StorageStressOptions; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -22,16 +24,15 @@ import static com.azure.storage.blob.options.BlockBlobSeekableByteChannelWriteOptions.WriteMode.OVERWRITE; /** - * Block-blob seekable byte channel write with {@link BlockBlobSeekableByteChannelWriteOptions#setContentValidationAlgorithm}. - * Matches {@link com.azure.storage.blob.BlobContentValidationUploadTests} seekable-channel scenarios (sync only). + * Block-blob seekable byte channel write with CRC64 enabled (sync only). */ -public class ContentValidationSeekableByteChannelWrite extends BlobScenarioBase { - private static final ClientLogger LOGGER = new ClientLogger(ContentValidationSeekableByteChannelWrite.class); +public class SeekableByteChannelWriteWithCRC64 extends BlobScenarioBase { + private static final ClientLogger LOGGER = new ClientLogger(SeekableByteChannelWriteWithCRC64.class); private final OriginalContent originalContent = new OriginalContent(); private final BlobClient syncClient; private final BlobAsyncClient asyncNoFaultClient; - public ContentValidationSeekableByteChannelWrite(ContentValidationStressOptions options) { + public SeekableByteChannelWriteWithCRC64(StorageStressOptions options) { super(options); String blobName = generateBlobName(); this.syncClient = getSyncContainerClient().getBlobClient(blobName); @@ -42,7 +43,7 @@ public ContentValidationSeekableByteChannelWrite(ContentValidationStressOptions protected void runInternal(Context span) throws IOException { BlockBlobClient blockBlobClient = syncClient.getBlockBlobClient(); BlockBlobSeekableByteChannelWriteOptions writeOptions = new BlockBlobSeekableByteChannelWriteOptions(OVERWRITE) - .setContentValidationAlgorithm(options.getContentValidationAlgorithm()); + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); try (CrcInputStream crcInput = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize())) { Flux byteBufferFlux = crcInput.convertStreamToByteBuffer(); diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationStageBlock.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/StageBlockWithCRC64.java similarity index 88% rename from sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationStageBlock.java rename to sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/StageBlockWithCRC64.java index c95dd7e9a98a..c06bbe6a44b6 100644 --- a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationStageBlock.java +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/StageBlockWithCRC64.java @@ -13,7 +13,9 @@ import com.azure.storage.blob.specialized.BlockBlobAsyncClient; import com.azure.storage.blob.specialized.BlockBlobClient; import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; import com.azure.storage.stress.CrcInputStream; +import com.azure.storage.stress.StorageStressOptions; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -23,16 +25,16 @@ import java.util.Collections; /** - * Stage block with request content validation on the faulted client, then commit via the non-faulted client. + * Stage block with CRC64 enabled on the faulted client, then commit via the non-faulted client. */ -public class ContentValidationStageBlock extends BlobScenarioBase { +public class StageBlockWithCRC64 extends BlobScenarioBase { private final OriginalContent originalContent = new OriginalContent(); private final BlobClient syncClient; private final BlobAsyncClient asyncClient; private final BlobClient syncNoFaultClient; private final BlobAsyncClient asyncNoFaultClient; - public ContentValidationStageBlock(ContentValidationStressOptions options) { + public StageBlockWithCRC64(StorageStressOptions options) { super(options); String blobName = generateBlobName(); this.syncNoFaultClient = getSyncContainerClientNoFault().getBlobClient(blobName); @@ -51,7 +53,7 @@ protected void runInternal(Context span) { BinaryData data = BinaryData.fromStream(inputStream, options.getSize()); blockBlobClient.stageBlockWithResponse( new BlockBlobStageBlockOptions(blockId, data) - .setContentValidationAlgorithm(options.getContentValidationAlgorithm()), + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64), null, span); blockBlobClientNoFault.commitBlockListWithResponse( new BlockBlobCommitBlockListOptions(Collections.singletonList(blockId)), null, span); @@ -70,7 +72,7 @@ protected Mono runInternalAsync(Context span) { return BinaryData.fromFlux(byteBufferFlux, options.getSize(), false) .flatMap(binaryData -> blockBlobAsyncClient.stageBlockWithResponse( new BlockBlobStageBlockOptions(blockId, binaryData) - .setContentValidationAlgorithm(options.getContentValidationAlgorithm()))) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64))) .then(blockBlobAsyncClientNoFault.commitBlockListWithResponse( new BlockBlobCommitBlockListOptions(Collections.singletonList(blockId)))) .then(originalContent.checkMatch(byteBufferFlux, span)); diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationUploadFromFile.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/UploadFromFileWithCRC64.java similarity index 91% rename from sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationUploadFromFile.java rename to sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/UploadFromFileWithCRC64.java index a716563c1752..b9594326e6ad 100644 --- a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationUploadFromFile.java +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/UploadFromFileWithCRC64.java @@ -13,7 +13,9 @@ import com.azure.storage.blob.options.BlobDownloadToFileOptions; import com.azure.storage.blob.options.BlobUploadFromFileOptions; import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; import com.azure.storage.stress.CrcInputStream; +import com.azure.storage.stress.StorageStressOptions; import reactor.core.publisher.Mono; import java.io.File; @@ -26,10 +28,10 @@ import java.util.UUID; /** - * Upload from file with {@link BlobUploadFromFileOptions#setContentValidationAlgorithm}. + * Upload from file with CRC64 enabled. */ -public class ContentValidationUploadFromFile extends BlobScenarioBase { - private static final ClientLogger LOGGER = new ClientLogger(ContentValidationUploadFromFile.class); +public class UploadFromFileWithCRC64 extends BlobScenarioBase { + private static final ClientLogger LOGGER = new ClientLogger(UploadFromFileWithCRC64.class); private final OriginalContent originalContent = new OriginalContent(); private final BlobClient syncClient; private final BlobAsyncClient asyncClient; @@ -37,7 +39,7 @@ public class ContentValidationUploadFromFile extends BlobScenarioBase uploadAndVerifyAsync(CrcInputStream inputStream, Path downloa return asyncClient.uploadFromFileWithResponse(new BlobUploadFromFileOptions(uploadFilePath.toString()) .setParallelTransferOptions(parallelTransferOptions) - .setContentValidationAlgorithm(options.getContentValidationAlgorithm())) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64)) .flatMap(ignored -> asyncNoFaultClient.downloadToFileWithResponse( new BlobDownloadToFileOptions(downloadFilePath.toString()))) .flatMap(ignored -> originalContent.checkMatch(BinaryData.fromFile(downloadFilePath), span)) diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationUploadPages.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/UploadPagesWithCRC64.java similarity index 87% rename from sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationUploadPages.java rename to sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/UploadPagesWithCRC64.java index 0e3c9603fe1c..3572871bea9f 100644 --- a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationUploadPages.java +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/UploadPagesWithCRC64.java @@ -11,23 +11,25 @@ import com.azure.storage.blob.specialized.PageBlobAsyncClient; import com.azure.storage.blob.specialized.PageBlobClient; import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; import com.azure.storage.stress.CrcInputStream; +import com.azure.storage.stress.StorageStressOptions; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import java.nio.ByteBuffer; /** - * Page blob upload pages with {@link PageBlobUploadPagesOptions#setContentValidationAlgorithm}. + * Page blob upload pages with CRC64 enabled. */ -public class ContentValidationUploadPages extends PageBlobScenarioBase { +public class UploadPagesWithCRC64 extends PageBlobScenarioBase { private final OriginalContent originalContent = new OriginalContent(); private final BlobClient syncClient; private final BlobAsyncClient asyncClient; private final BlobAsyncClient asyncNoFaultClient; private final PageBlobAsyncClient tempSetupPageBlobClient; - public ContentValidationUploadPages(ContentValidationStressOptions options) { + public UploadPagesWithCRC64(StorageStressOptions options) { super(options); String blobName = generateBlobName(); String tempBlobName = generateBlobName(); @@ -46,7 +48,7 @@ protected void runInternal(Context span) { PageRange range = new PageRange().setStart(0).setEnd(options.getSize() - 1); pageBlobClient.uploadPagesWithResponse( new PageBlobUploadPagesOptions(range, inputStream) - .setContentValidationAlgorithm(options.getContentValidationAlgorithm()), + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64), null, span); originalContent.checkMatch(inputStream.getContentInfo(), span).block(); } @@ -60,7 +62,7 @@ protected Mono runInternalAsync(Context span) { PageRange range = new PageRange().setStart(0).setEnd(options.getSize() - 1); return pageBlobAsyncClient.uploadPagesWithResponse( new PageBlobUploadPagesOptions(range, byteBufferFlux) - .setContentValidationAlgorithm(options.getContentValidationAlgorithm())) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64)) .then(originalContent.checkMatch(byteBufferFlux, span)); } diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationUpload.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/UploadWithCRC64.java similarity index 81% rename from sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationUpload.java rename to sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/UploadWithCRC64.java index b2cc61909be3..55340ba93946 100644 --- a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationUpload.java +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/UploadWithCRC64.java @@ -9,24 +9,25 @@ import com.azure.storage.blob.models.ParallelTransferOptions; import com.azure.storage.blob.options.BlobParallelUploadOptions; import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; import com.azure.storage.stress.CrcInputStream; +import com.azure.storage.stress.StorageStressOptions; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import java.nio.ByteBuffer; /** - * Parallel blob upload with {@link com.azure.storage.blob.options.BlobParallelUploadOptions#setContentValidationAlgorithm} - * enabled. Verifies the correctness of the upload request content via CRC (see {@code BlobContentValidationUploadTests}). + * Parallel blob upload with CRC64 enabled. */ -public class ContentValidationUpload extends BlobScenarioBase { +public class UploadWithCRC64 extends BlobScenarioBase { private final OriginalContent originalContent = new OriginalContent(); private final BlobClient syncClient; private final BlobAsyncClient asyncClient; private final BlobAsyncClient asyncNoFaultClient; private final ParallelTransferOptions parallelTransferOptions; - public ContentValidationUpload(ContentValidationStressOptions options) { + public UploadWithCRC64(StorageStressOptions options) { super(options); String blobName = generateBlobName(); this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); @@ -42,7 +43,7 @@ protected void runInternal(Context span) { try (CrcInputStream inputStream = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize())) { syncClient.uploadWithResponse(new BlobParallelUploadOptions(inputStream) .setParallelTransferOptions(parallelTransferOptions) - .setContentValidationAlgorithm(options.getContentValidationAlgorithm()), null, span); + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64), null, span); originalContent.checkMatch(inputStream.getContentInfo(), span).block(); } } @@ -53,7 +54,7 @@ protected Mono runInternalAsync(Context span) { .convertStreamToByteBuffer(); return asyncClient.uploadWithResponse(new BlobParallelUploadOptions(byteBufferFlux) .setParallelTransferOptions(parallelTransferOptions) - .setContentValidationAlgorithm(options.getContentValidationAlgorithm())) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64)) .then(originalContent.checkMatch(byteBufferFlux, span)); } From 83e0f7ab3648566c3667c52e8b2aa215b55dbb77 Mon Sep 17 00:00:00 2001 From: Isabelle Date: Mon, 1 Jun 2026 19:11:35 -0700 Subject: [PATCH 5/5] adding scenarios back to app.java --- .../main/java/com/azure/storage/blob/stress/App.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/App.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/App.java index 194a290d7a3c..f146945db423 100644 --- a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/App.java +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/App.java @@ -20,6 +20,16 @@ public static void main(String[] args) { DownloadToFileWithCRC64.class, OpenInputStreamWithCRC64.class, OpenSeekableByteChannelReadWithCRC64.class, + AppendBlobOutputStreamWithCRC64.class, + AppendBlockWithCRC64.class, + BlockBlobOutputStreamWithCRC64.class, + BlockBlobUploadWithCRC64.class, + PageBlobOutputStreamWithCRC64.class, + StageBlockWithCRC64.class, + SeekableByteChannelWriteWithCRC64.class, + UploadWithCRC64.class, + UploadFromFileWithCRC64.class, + UploadPagesWithCRC64.class, DownloadToFile.class, DownloadStream.class, DownloadContent.class,