result = new LinkedList<>();
- for (UploadSegmentMetadata segment : metadata.getSegments()) {
- if (segment.getStatus() == SegmentUploadStatus.Pending) {
- result.add(new SegmentQueueItem(segment.getSegmentNumber(), 0));
- }
- }
- return result;
- }
-
- /**
- * Updates the segment metadata status.
- *
- * @param metadata The metadata.
- * @param segmentNumber The segment number.
- * @param newStatus The new status.
- */
- private static void updateSegmentMetadataStatus(UploadMetadata metadata, int segmentNumber, SegmentUploadStatus newStatus) {
- UploadSegmentMetadata[] toSave = metadata.getSegments();
- toSave[segmentNumber].setStatus(newStatus);
- metadata.setSegments(toSave);
- try {
- metadata.save();
- } catch (Exception e) {
- } //no need to crash the program if were unable to save the metadata; it is what's in memory that's important
- }
-
- /**
- * When an object implementing interface Runnable is used
- * to create a thread, starting the thread causes the object's
- * run method to be called in that separately executing
- * thread.
- *
- * The general contract of the method run is that it may
- * take any action whatsoever.
- *
- * In this run, we are allowing each thread to attempt to process all
- * of the remaining segments, which will ultimately result in each thread
- * processing a subset of segments that are still in the queue.
- *
- * @see Thread#run()
- */
- @Override
- public void run() {
- processPendingSegments(pendingSegments, exceptions);
- }
-
- /**
- * Represents a tuple that pairs a segment number with the number of times it was attempted for upload.
- */
- private static class SegmentQueueItem {
- SegmentQueueItem(int segmentNumber, int attemptCount) {
- this.segmentNumber = segmentNumber;
- this.attemptCount = attemptCount;
- }
- public int getSegmentNumber() {
- return segmentNumber;
- }
- private int segmentNumber;
-
- public int getAttemptCount() {
- return attemptCount;
- }
- private int attemptCount;
- }
-}
diff --git a/azure-mgmt-datalake-store-uploader/src/main/java/com/microsoft/azure/management/datalake/store/uploader/SegmentUploadStatus.java b/azure-mgmt-datalake-store-uploader/src/main/java/com/microsoft/azure/management/datalake/store/uploader/SegmentUploadStatus.java
deleted file mode 100644
index 7d5ee00180c6..000000000000
--- a/azure-mgmt-datalake-store-uploader/src/main/java/com/microsoft/azure/management/datalake/store/uploader/SegmentUploadStatus.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/**
- * Copyright (c) Microsoft Corporation. All rights reserved.
- * Licensed under the MIT License. See License.txt in the project root for
- * license information.
- */
-package com.microsoft.azure.management.datalake.store.uploader;
-
-/**
- * Defines various states that a segment upload can have.
- */
-public enum SegmentUploadStatus {
- /**
- * Indicates that the segment is currently scheduled for upload.
- */
- Pending,
-
- /**
- * Indicates that the segment is currently being uploaded.
- */
- InProgress,
-
- /**
- * Indicates that the segment was not uploaded successfully.
- */
- Failed,
-
- /**
- * Indicates that the segment was successfully uploaded.
- */
- Complete
-}
diff --git a/azure-mgmt-datalake-store-uploader/src/main/java/com/microsoft/azure/management/datalake/store/uploader/SingleSegmentUploader.java b/azure-mgmt-datalake-store-uploader/src/main/java/com/microsoft/azure/management/datalake/store/uploader/SingleSegmentUploader.java
deleted file mode 100644
index d0090e5bf3cc..000000000000
--- a/azure-mgmt-datalake-store-uploader/src/main/java/com/microsoft/azure/management/datalake/store/uploader/SingleSegmentUploader.java
+++ /dev/null
@@ -1,312 +0,0 @@
-/**
- * Copyright (c) Microsoft Corporation. All rights reserved.
- * Licensed under the MIT License. See License.txt in the project root for
- * license information.
- */
-package com.microsoft.azure.management.datalake.store.uploader;
-
-import org.apache.commons.lang3.StringUtils;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.RandomAccessFile;
-import java.nio.charset.Charset;
-import java.text.MessageFormat;
-
-/**
- * Represents an uploader for a single segment of a larger file.
- */
-public class SingleSegmentUploader {
-
- /**
- * The length of the buffers to upload (4MB).
- */
- public static final int BUFFER_LENGTH = 4 * 1024 * 1024;
-
- /** 4MB is the maximum length of a single extent. So if one record is longer than this,
- * then we will fast fail, since that record will cross extent boundaries.
- */
- public static final int MAX_RECORD_LENGTH = 4 * 1024 * 1024;
-
- /**
- * During upload retries, this indicates the maximum amount of time, in seconds, that we will wait between retries.
- */
- public static final int MAXIMUM_BACKOFF_WAIT_SECONDS = 32;
-
- /**
- * The maximum number of times to attempt to upload the buffer.
- */
- public static final int MAX_BUFFER_UPLOAD_ATTEMPT_COUNT = 4;
-
- private FrontEndAdapter frontEndAdapter;
- private UploadSegmentMetadata segmentMetadata;
- private UploadMetadata metadata;
-
- /**
- * Creates a new uploader for a single segment.
- *
- * @param segmentNumber The sequence number of the segment.
- * @param uploadMetadata The metadata for the entire upload.
- * @param frontEnd A pointer to the front end.
- */
- public SingleSegmentUploader(int segmentNumber, UploadMetadata uploadMetadata, FrontEndAdapter frontEnd) {
- metadata = uploadMetadata;
- segmentMetadata = uploadMetadata.getSegments()[segmentNumber];
- frontEndAdapter = frontEnd;
- this.useBackOffRetryStrategy = true;
- }
-
- /**
- * Gets or sets a value indicating whether to use a back-off (exponenential) in case of individual block failures.
- * If set to 'false' every retry is handled immediately; otherwise an amount of time is waited between retries, as a function of power of 2.
- */
- private boolean useBackOffRetryStrategy;
-
- /**
- *
- * @return A value indicating whether to use a back-off (exponenential) in case of individual block failures.
- * If set to 'false' every retry is handled immediately; otherwise an amount of time is waited between retries, as a function of power of 2.
- */
- public boolean useBackOffRetryStrategy() {
- return useBackOffRetryStrategy;
- }
-
- /**
- *
- * @param isEnabled A value indicating whether to use a back-off (exponenential) in case of individual block failures.
- * If set to 'false' every retry is handled immediately; otherwise an amount of time is waited between retries, as a function of power of 2.
- */
- public void setUseBackOffRetryStrategy(boolean isEnabled) {
- useBackOffRetryStrategy = isEnabled;
- }
-
- /**
- * Uploads the portion of the InputFilePath to the given TargetStreamPath, starting at the given StartOffset.
- * The segment is further divided into equally-sized blocks which are uploaded in sequence.
- * Each such block is attempted a certain number of times; if after that it still cannot be uploaded, the entire segment is aborted (in which case no cleanup is performed on the server).
- *
- * @throws Exception if there is any failure during the upload
- */
- public void upload() throws Exception {
- File fileInfo = new File(metadata.getInputFilePath());
- if (!(fileInfo.exists())) {
- throw new FileNotFoundException("Unable to locate input file: " + metadata.getInputFilePath());
- }
-
- //open up a reader from the input file, seek to the appropriate offset
- try (RandomAccessFile inputStream = openInputStream()) {
- long endPosition = segmentMetadata.getOffset() + segmentMetadata.getLength();
- if (endPosition > fileInfo.length()) {
- throw new IllegalArgumentException("StartOffset+UploadLength is beyond the end of the input file");
- }
-
- uploadSegmentContents(inputStream, endPosition);
-
- verifyUploadedStream();
- //any exceptions are (re)thrown to be handled by the caller; we do not handle retries or other recovery techniques here
- }
- }
-
- /**
- * Verifies the uploaded stream.
- *
- * @throws Exception if there is any failure validating the stream being uploaded.
- */
- private void verifyUploadedStream() throws Exception {
- //verify that the remote stream has the length we expected.
- int retryCount = 0;
- long remoteLength = -1;
- while (retryCount < MAX_BUFFER_UPLOAD_ATTEMPT_COUNT) {
- retryCount++;
- try {
- remoteLength = frontEndAdapter.getStreamLength(segmentMetadata.getPath());
- break;
- } catch (Exception ex) {
- if (retryCount >= MAX_BUFFER_UPLOAD_ATTEMPT_COUNT) {
- throw ex;
- }
-
- waitForRetry(retryCount, this.useBackOffRetryStrategy);
- }
- }
-
- if (segmentMetadata.getLength() != remoteLength) {
- throw new UploadFailedException(MessageFormat.format("Post-upload stream verification failed: target stream has a length of {0}, expected {1}", remoteLength, segmentMetadata.getLength()));
- }
- }
-
- /**
- * Uploads the segment contents.
- *
- * @param inputStream The input stream.
- * @param endPosition The end position.
- * @throws Exception if there is any failure attempting to upload the contents of a single segment.
- */
- private void uploadSegmentContents(RandomAccessFile inputStream, long endPosition) throws Exception {
- long bytesCopiedSoFar = 0; // we start off with a fresh stream
-
- byte[] buffer = new byte[BUFFER_LENGTH];
- int residualBufferLength = 0; //the number of bytes that remained in the buffer from the last upload (bytes which were not uploaded)
-
- while (inputStream.getFilePointer() < endPosition) {
- //read a block of data, and keep track of how many bytes are actually read
- int bytesRead = readIntoBuffer(inputStream, buffer, residualBufferLength, endPosition);
- int bufferDataLength = residualBufferLength + bytesRead;
-
- //determine the cutoff offset for upload - everything before will be uploaded, everything after is residual; (the position of the last record in this buffer)
- int uploadCutoff = bufferDataLength;
- if (!metadata.isBinary()) {
- uploadCutoff = determineUploadCutoffForTextFile(buffer, bufferDataLength, inputStream);
- }
-
- bytesCopiedSoFar = uploadBuffer(buffer, uploadCutoff, bytesCopiedSoFar);
-
- residualBufferLength = bufferDataLength - uploadCutoff;
- if (residualBufferLength > 0) {
- //move the remainder of the buffer to the front
- System.arraycopy(buffer, uploadCutoff, buffer, 0, residualBufferLength);
- }
- }
-
- //make sure we don't leave anything behind
- if (residualBufferLength > 0) {
- uploadBuffer(buffer, residualBufferLength, bytesCopiedSoFar);
- }
-
- buffer = null;
- }
-
- /**
- * Determines the upload cutoff for text file.
- *
- * @param buffer The buffer.
- * @param bufferDataLength length of the buffer data.
- * @param inputStream The input stream.
- * @return The index within the buffer which indicates a record boundary cutoff for a single append request for a text file.
- * @throws UploadFailedException indicates that the upload failed for the specified reason.
- * @throws IOException indicates the path is inaccessible or does not exist.
- */
- private int determineUploadCutoffForTextFile(byte[] buffer, int bufferDataLength, RandomAccessFile inputStream) throws UploadFailedException, IOException {
- Charset encoding = Charset.forName(metadata.getEncodingName());
- //NOTE: we return an offset, but everywhere else below we treat it as a byte count; in order for that to work, we need to add 1 to the result of FindNewLine.
- int uploadCutoff = StringExtensions.findNewline(buffer, bufferDataLength - 1, bufferDataLength, true, encoding, metadata.getDelimiter()) + 1;
- if (uploadCutoff <= 0 && (metadata.getSegmentCount() > 1 || bufferDataLength >= MAX_RECORD_LENGTH)) {
- throw new UploadFailedException(MessageFormat.format("Found a record that exceeds the maximum allowed record length around offset {0}", inputStream.getFilePointer()));
- }
-
- //a corner case here is when the newline is 2 chars long, and the first of those lands on the last byte of the buffer. If so, let's try to find another
- //newline inside the buffer, because we might be splitting this wrongly.
- if ((metadata.getDelimiter() == null || StringUtils.isEmpty(metadata.getDelimiter())) && uploadCutoff == buffer.length && buffer[buffer.length - 1] == (byte) '\r') {
- int newCutoff = StringExtensions.findNewline(buffer, bufferDataLength - 2, bufferDataLength - 1, true, encoding, metadata.getDelimiter()) + 1;
- if (newCutoff > 0) {
- uploadCutoff = newCutoff;
- }
- }
-
- return uploadCutoff;
- }
-
- /**
- * Uploads the buffer.
- *
- * @param buffer The buffer.
- * @param bytesToCopy The bytes to copy.
- * @param targetStreamOffset The target stream offset.
- * @return The current index within the target stream after uploading the buffer.
- * @throws Exception Thrown if there is a failure uploading the current buffer.
- */
- private long uploadBuffer(byte[] buffer, int bytesToCopy, long targetStreamOffset) throws Exception {
- //append it to the remote stream
- int attemptCount = 0;
- boolean uploadCompleted = false;
- while (!uploadCompleted && attemptCount < MAX_BUFFER_UPLOAD_ATTEMPT_COUNT) {
- attemptCount++;
- try {
- if (targetStreamOffset == 0) {
- frontEndAdapter.createStream(segmentMetadata.getPath(), true, buffer, bytesToCopy);
- } else {
- frontEndAdapter.appendToStream(segmentMetadata.getPath(), buffer, targetStreamOffset, bytesToCopy);
-
- }
-
- uploadCompleted = true;
- targetStreamOffset += bytesToCopy;
- } catch (Exception ex) {
- //if we tried more than the number of times we were allowed to, give up and throw the exception
- if (attemptCount >= MAX_BUFFER_UPLOAD_ATTEMPT_COUNT) {
- throw ex;
- } else {
- waitForRetry(attemptCount, this.useBackOffRetryStrategy);
- }
- }
- }
-
- return targetStreamOffset;
- }
-
- /**
- * Reads the data into the buffer.
- *
- * @param inputStream The stream to read data from.
- * @param buffer The buffer to read data into
- * @param bufferOffset The offset in the buffer to begin pushing data
- * @param streamEndPosition The last point in the stream to read.
- * @return The number of bytes read into the buffer.
- * @throws IOException Thrown if there is an issue accessing the stream or the pointer to the file.
- */
- private int readIntoBuffer(RandomAccessFile inputStream, byte[] buffer, int bufferOffset, long streamEndPosition) throws IOException {
- //read a block of data
- int bytesToRead = buffer.length - bufferOffset;
- if (bytesToRead > streamEndPosition - inputStream.getFilePointer()) {
- //last read may be smaller than previous reads; readjust # of bytes to read accordingly
- bytesToRead = (int) (streamEndPosition - inputStream.getFilePointer());
- }
-
- int remainingBytes = bytesToRead;
-
- while (remainingBytes > 0) {
- //Stream.Read may not read all the bytes we requested, so we need to retry until we filled up the entire buffer
- int bytesRead = inputStream.read(buffer, bufferOffset, remainingBytes);
- bufferOffset += bytesRead;
- remainingBytes = bytesToRead - bufferOffset;
- }
-
- return bytesToRead;
- }
-
- /**
- * Enables use of a back off retry strategy, allowing a caller to wait before attempting an action again.
- *
- * @param attemptCount The number of attempts that have already been done
- * @param useBackOffRetryStrategy whether to use the back off strategy or not.
- * @throws InterruptedException Thrown if there is an interrupt during the sleep.
- */
- public static void waitForRetry(int attemptCount, boolean useBackOffRetryStrategy) throws InterruptedException {
- if (!useBackOffRetryStrategy) {
- //no need to wait
- return;
- }
-
- int intervalSeconds = Math.max(MAXIMUM_BACKOFF_WAIT_SECONDS, (int) Math.pow(2, attemptCount));
- Thread.sleep(intervalSeconds * 1000);
- }
-
- /**
- * Opens the input stream.
- * @return A {@link RandomAccessFile} stream of the file being uploaded.
- * @throws IOException Thrown if the input stream cannot be opened due to file accessibility or existence.
- */
- private RandomAccessFile openInputStream() throws IOException {
- RandomAccessFile stream = new RandomAccessFile(metadata.getInputFilePath(), "r");
-
- if (segmentMetadata.getOffset() >= stream.length()) {
- throw new IllegalArgumentException("StartOffset is beyond the end of the input file");
- }
-
- // always seek from the beginning of the file
- stream.seek(0);
- stream.seek(segmentMetadata.getOffset());
- return stream;
- }
-}
diff --git a/azure-mgmt-datalake-store-uploader/src/main/java/com/microsoft/azure/management/datalake/store/uploader/StringExtensions.java b/azure-mgmt-datalake-store-uploader/src/main/java/com/microsoft/azure/management/datalake/store/uploader/StringExtensions.java
deleted file mode 100644
index eb6d2abd959c..000000000000
--- a/azure-mgmt-datalake-store-uploader/src/main/java/com/microsoft/azure/management/datalake/store/uploader/StringExtensions.java
+++ /dev/null
@@ -1,156 +0,0 @@
-/**
- * Copyright (c) Microsoft Corporation. All rights reserved.
- * Licensed under the MIT License. See License.txt in the project root for
- * license information.
- */
-package com.microsoft.azure.management.datalake.store.uploader;
-
-import org.apache.commons.lang3.StringUtils;
-
-import java.nio.charset.Charset;
-import java.nio.charset.StandardCharsets;
-
-/**
- * A class of helpers to determine the location of record boundaries within byte arrays.
- */
-public final class StringExtensions {
-
- private StringExtensions() {
- // empty constructor that should not be used.
- }
- /**
- * Finds the index in the given buffer of a newline character, either the first or the last (based on the parameters).
- * If a combined newline (\r\n), the index returned is that of the last character in the sequence.
- *
- * @param buffer The buffer to search in.
- * @param startOffset The index of the first byte to start searching at.
- * @param length The number of bytes to search, starting from the given startOffset.
- * @param reverse If true, searches from the startOffset down to the beginning of the buffer. If false, searches upwards.
- * @param encoding Indicates the type of encoding to use for the buffered bytes.
- * @param delimiter Optionally indicates the delimiter to consider as the "new line", which MUST BE a single character. If null, the default is '\\r', '\\n' and '\\r\\n'.
- * @return The index of the closest newline character in the sequence (based on direction) that was found. Returns -1 if not found.
- */
- public static int findNewline(byte[] buffer, int startOffset, int length, boolean reverse, Charset encoding, String delimiter) {
- if (buffer.length == 0 || length == 0) {
- return -1;
- }
-
- // define the bytes per character to use
- int bytesPerChar;
- if (encoding.equals(StandardCharsets.UTF_16) || encoding.equals(StandardCharsets.UTF_16BE) || encoding.equals(StandardCharsets.UTF_16LE)) {
- bytesPerChar = 2;
- } else if (encoding.equals(StandardCharsets.US_ASCII) || encoding.equals(StandardCharsets.UTF_8)) {
- bytesPerChar = 1;
- } else {
- throw new IllegalArgumentException("Only the following encodings are allowed: UTF-8, UTF-16, UTF-16BE, UTF16-LE and ASCII");
- }
-
-
- if (delimiter != null && !StringUtils.isEmpty(delimiter) && delimiter.length() > 1) {
- throw new IllegalArgumentException("The delimiter must only be a single character or unspecified to represent the CRLF delimiter");
- }
-
- if (delimiter != null && !StringUtils.isEmpty(delimiter)) {
- // convert the byte array back to a String
- int startOfSegment = reverse ? startOffset - length + 1 : startOffset;
- String bytesToString = new String(buffer, startOfSegment, length, encoding);
- if (!bytesToString.contains(delimiter)) {
- // didn't find the delimiter.
- return -1;
- }
-
- // the index is returned, which is 0 based, so our loop must include the zero case.
- int numCharsToDelim = reverse ? bytesToString.lastIndexOf(delimiter) : bytesToString.indexOf(delimiter);
- int toReturn = 0;
- for (int i = 0; i <= numCharsToDelim; i++) {
- toReturn += Character.toString(bytesToString.charAt(startOfSegment + i)).getBytes(encoding).length;
- }
-
- // we get the total number of bytes, but we want to return the index (which starts at 0)
- // so we subtract 1 from the total number of bytes to get the final byte index.
- return toReturn - 1;
- }
-
- //endOffset is a 'sentinel' value; we use that to figure out when to stop searching
- int endOffset = reverse ? startOffset - length : startOffset + length;
-
- // if we are starting at the end, we need to move toward the front enough to grab the right number of bytes
- startOffset = reverse ? startOffset - (bytesPerChar - 1) : startOffset;
-
- if (startOffset < 0 || startOffset >= buffer.length) {
- throw new IndexOutOfBoundsException("Given start offset is outside the bounds of the given buffer. In reverse cases, the start offset is modified to ensure we check the full size of the last character");
- }
-
- // make sure that the length we are traversing is at least as long as a single character
- if (length < bytesPerChar) {
- throw new IllegalArgumentException("length must be at least as long as the length, in bytes, of a single character");
- }
-
- if (endOffset < -1 || endOffset > buffer.length) {
- throw new IndexOutOfBoundsException("Given combination of startOffset and length would execute the search outside the bounds of the given buffer.");
- }
-
- int bufferEndOffset = reverse ? startOffset : startOffset + length;
- int result = -1;
- for (int charPos = startOffset; reverse ? charPos != endOffset : charPos + bytesPerChar - 1 < endOffset; charPos = reverse ? charPos - 1 : charPos + 1) {
- char c;
- if (bytesPerChar == 1) {
- c = (char) buffer[charPos];
- }
- else {
- String temp = new String(buffer, charPos, bytesPerChar, encoding);
- if (StringUtils.isEmpty(temp)) {
- continue;
- }
- else {
- c = temp.toCharArray()[0];
- }
- }
-
- if (isNewline(c, delimiter)) {
- result = charPos + bytesPerChar - 1;
- break;
- }
- }
-
- if ((delimiter == null || StringUtils.isEmpty(delimiter)) && !reverse && result < bufferEndOffset - bytesPerChar) {
- char c;
- if (bytesPerChar == 1) {
- c = (char) buffer[result + bytesPerChar];
- }
- else {
- String temp = new String(buffer, result + 1, bytesPerChar, encoding);
- if (StringUtils.isEmpty(temp)) {
- // this can occur if the number of bytes for characters in the string result in an empty string (an invalid code for the given encoding)
- // in this case, that means that we are done for the default delimiter.
- return result;
- }
- else {
- c = temp.toCharArray()[0];
- }
- }
-
- if (isNewline(c, delimiter)) {
- //we originally landed on a \r character; if we have a \r\n character, advance one position to include that
- result += bytesPerChar;
- }
- }
-
- return result;
- }
-
- /**
- * Determines whether the specified character is newline.
- *
- * @param c The character.
- * @param delimiter The delimiter to use. If null or empty CR LF characters are used.
- * @return
- */
- private static boolean isNewline(char c, String delimiter) {
- if ((delimiter == null || StringUtils.isEmpty(delimiter))) {
- return c == '\r' || c == '\n';
- }
-
- return c == delimiter.toCharArray()[0];
- }
-}
diff --git a/azure-mgmt-datalake-store-uploader/src/main/java/com/microsoft/azure/management/datalake/store/uploader/UploadFailedException.java b/azure-mgmt-datalake-store-uploader/src/main/java/com/microsoft/azure/management/datalake/store/uploader/UploadFailedException.java
deleted file mode 100644
index 07e778afe896..000000000000
--- a/azure-mgmt-datalake-store-uploader/src/main/java/com/microsoft/azure/management/datalake/store/uploader/UploadFailedException.java
+++ /dev/null
@@ -1,19 +0,0 @@
-/**
- * Copyright (c) Microsoft Corporation. All rights reserved.
- * Licensed under the MIT License. See License.txt in the project root for
- * license information.
- */
-package com.microsoft.azure.management.datalake.store.uploader;
-
-/**
- * Represents an exception that is thrown when an upload fails.
- */
-public class UploadFailedException extends Exception {
- /**
- * Initializes a new instance of the UploadFailedException exception.
- * @param message The message that describes the error.
- */
- public UploadFailedException(String message) {
- super(message);
- }
-}
diff --git a/azure-mgmt-datalake-store-uploader/src/main/java/com/microsoft/azure/management/datalake/store/uploader/UploadMetadata.java b/azure-mgmt-datalake-store-uploader/src/main/java/com/microsoft/azure/management/datalake/store/uploader/UploadMetadata.java
deleted file mode 100644
index 81b838262ef3..000000000000
--- a/azure-mgmt-datalake-store-uploader/src/main/java/com/microsoft/azure/management/datalake/store/uploader/UploadMetadata.java
+++ /dev/null
@@ -1,444 +0,0 @@
-/**
- * Copyright (c) Microsoft Corporation. All rights reserved.
- * Licensed under the MIT License. See License.txt in the project root for
- * license information.
- */
-package com.microsoft.azure.management.datalake.store.uploader;
-
-import org.apache.commons.lang3.StringUtils;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InvalidObjectException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
-import java.io.Serializable;
-import java.nio.charset.StandardCharsets;
-import java.text.MessageFormat;
-import java.util.BitSet;
-import java.util.UUID;
-
-/**
- * Represents general metadata pertaining to an upload.
- */
-public class UploadMetadata implements Serializable {
- private static Object saveSync = new Object();
-
- /**
- * Constructs a new UploadMetadata from the given parameters.
- *
- * @param metadataFilePath The file path to assign to this metadata file (for saving purposes).
- * @param uploadParameters The parameters to use for constructing this metadata.
- */
- public UploadMetadata(String metadataFilePath, UploadParameters uploadParameters) {
- this.metadataFilePath = metadataFilePath;
-
- this.uploadId = UUID.randomUUID().toString();
- this.inputFilePath = uploadParameters.getInputFilePath();
- this.targetStreamPath = uploadParameters.getTargetStreamPath();
-
-
- String[] streamData = splitTargetStreamPathByName();
- String streamName = streamData[0];
- String streamDirectory = streamData[1];
-
- if (streamDirectory == null || StringUtils.isEmpty(streamDirectory)) {
- // the scenario where the file is being uploaded at the root
- this.segmentStreamDirectory = MessageFormat.format("/{0}.segments.{1}", streamName, UUID.randomUUID());
- } else {
- // the scenario where the file is being uploaded in a sub folder
- this.segmentStreamDirectory = MessageFormat.format("{0}/{1}.segments.{2}",
- streamDirectory,
- streamName, UUID.randomUUID());
- }
-
- this.isBinary = uploadParameters.isBinary();
-
- File fileInfo = new File(uploadParameters.getInputFilePath());
- this.fileLength = fileInfo.length();
-
- this.encodingName = uploadParameters.getFileEncoding().name();
-
- // we are taking the smaller number of segments between segment lengths of 256 and the segment growth logic.
- // this protects us against agressive increase of thread count resulting in far more segments than
- // is reasonable for a given file size. We also ensure that each segment is at least 256mb in size.
- // This is the size that ensures we have the optimal storage creation in the store.
- int preliminarySegmentCount = (int) Math.ceil((double) fileInfo.length() / uploadParameters.getMaxSegementLength());
- this.segmentCount = Math.min(preliminarySegmentCount, UploadSegmentMetadata.calculateSegmentCount(fileInfo.length()));
- this.segmentLength = UploadSegmentMetadata.calculateSegmentLength(fileInfo.length(), this.segmentCount);
-
- this.segments = new UploadSegmentMetadata[this.segmentCount];
- for (int i = 0; i < this.segmentCount; i++) {
- this.segments[i] = new UploadSegmentMetadata(i, this);
- }
- }
-
- /**
- *
- * @return A value indicating the unique identifier associated with this upload.
- */
- public String getUploadId() {
- return uploadId;
- }
-
- /**
- *
- * @return A value indicating the full path to the file to be uploaded.
- */
- public String getInputFilePath() {
- return inputFilePath;
- }
-
- /**
- *
- * @return A value indicating the length (in bytes) of the file to be uploaded.
- */
- public long getFileLength() {
- return fileLength;
- }
-
- /**
- *
- * @return A value indicating the full stream path where the file will be uploaded to.
- */
- public String getTargetStreamPath() {
- return targetStreamPath;
- }
-
- /**
- *
- * @return A value indicating the directory path where intermediate segment streams will be stored.
- */
- public String getSegmentStreamDirectory() {
- return segmentStreamDirectory;
- }
-
- /**
- *
- * @return A value indicating the number of segments this file is split into for purposes of uploading it.
- */
- public int getSegmentCount() {
- return segmentCount;
- }
-
- /**
- *
- * @param segCount Sets the segment count to the specified count.
- */
- public void setSegmentCount(int segCount) {
- segmentCount = segCount;
- }
-
- /**
- *
- * @return A value indicating the length (in bytes) of each segment of the file (except the last one, which may be less).
- */
- public long getSegmentLength() {
- return segmentLength;
- }
-
- /**
- *
- * @param segLength The length to set the segment length to.
- */
- public void setSegmentLength(long segLength) {
- segmentLength = segLength;
- }
- /**
- *
- * @return A pointer to an array of segment metadata. The segments are ordered by their segment number (sequence).
- */
- public UploadSegmentMetadata[] getSegments() {
- return segments;
- }
-
- /**
- *
- * @param segs The value to set the segment array to.
- */
- public void setSegments(UploadSegmentMetadata[] segs) {
- segments = segs;
- }
-
- /**
- *
- * @return A value indicating whether the upload file should be treated as a binary file or not.
- */
- public boolean isBinary() {
- return isBinary;
- }
-
- /**
- *
- * @return The name of the current encoding being used.
- */
- public String getEncodingName() {
- return encodingName;
- }
-
- /**
- *
- * @return A value indicating the record boundary delimiter for the file, if any.
- */
- public String getDelimiter() {
- return delimiter;
- }
-
- /**
- *
- * @return A value indicating the path where this metadata file is located.
- */
- public String getMetadataFilePath() {
- return metadataFilePath;
- }
-
- /**
- *
- * @param metadataFilePath A value indicating the path where this metadata file is located.
- */
- public void setMetadataFilePath(String metadataFilePath) {
- this.metadataFilePath = metadataFilePath;
- }
-
- private transient String metadataFilePath;
-
- /**
- *
- * @param uploadId A value indicating the unique identifier associated with this upload.
- */
- public void setUploadId(String uploadId) {
- this.uploadId = uploadId;
- }
-
- /**
- *
- * @param inputFilePath A value indicating the full path to the file to be uploaded.
- */
- public void setInputFilePath(String inputFilePath) {
- this.inputFilePath = inputFilePath;
- }
-
- /**
- *
- * @param fileLength A value indicating the length (in bytes) of the file to be uploaded.
- */
- public void setFileLength(long fileLength) {
- this.fileLength = fileLength;
- }
-
- /**
- *
- * @param targetStreamPath A value indicating the full stream path where the file will be uploaded to.
- */
- public void setTargetStreamPath(String targetStreamPath) {
- this.targetStreamPath = targetStreamPath;
- }
-
- /**
- *
- * @param segmentStreamDirectory A value indicating the directory path where intermediate segment streams will be stored.
- */
- public void setSegmentStreamDirectory(String segmentStreamDirectory) {
- this.segmentStreamDirectory = segmentStreamDirectory;
- }
-
- /**
- *
- * @param binary A value indicating whether the upload file should be treated as a binary file or not.
- */
- public void setBinary(boolean binary) {
- isBinary = binary;
- }
-
- /**
- *
- * @param encodingName The name of the current encoding being used.
- */
- public void setEncodingName(String encodingName) {
- this.encodingName = encodingName;
- }
-
- /**
- *
- * @param delimiter A value indicating the record boundary delimiter for the file, if any.
- */
- public void setDelimiter(String delimiter) {
- this.delimiter = delimiter;
- }
-
- private String uploadId;
-
- private String inputFilePath;
-
- private long fileLength;
-
- private String targetStreamPath;
-
- private String segmentStreamDirectory;
-
- private int segmentCount;
-
- private long segmentLength;
-
- private UploadSegmentMetadata[] segments;
-
- private boolean isBinary;
-
- private String encodingName;
-
- private String delimiter;
-
- /**
- * Initializes a new instance of the UploadMetadata class for use with unit testing.
- */
- protected UploadMetadata() {
- this.encodingName = StandardCharsets.UTF_8.name();
- }
-
- /**
- * Attempts to load an UploadMetadata object from the given file.
- *
- * @param filePath The full path to the file where to load the metadata from
- * @return A deserialized {@link UploadMetadata} object from the file specified.
- * @throws FileNotFoundException Thrown if the filePath is inaccessible or does not exist
- * @throws InvalidMetadataException Thrown if the metadata is not in the expected format.
- */
- public static UploadMetadata loadFrom(String filePath) throws FileNotFoundException, InvalidMetadataException {
- if (!new File(filePath).exists()) {
- throw new FileNotFoundException("Could not find metadata file: " + filePath);
- }
-
- UploadMetadata result = null;
- try {
- FileInputStream fileIn = new FileInputStream(filePath);
- ObjectInputStream in = new ObjectInputStream(fileIn);
- result = (UploadMetadata) in.readObject();
- in.close();
- fileIn.close();
- result.metadataFilePath = filePath;
- return result;
- } catch (Exception ex) {
- throw new InvalidMetadataException("Unable to parse metadata file", ex);
- }
- }
-
- /**
- * Saves the given metadata to its canonical location. This method is thread-safe.
- *
- * @throws IOException Thrown if the file cannot be saved due to accessibility or there is an error saving the stream to disk.
- * @throws InvalidMetadataException Thrown if the metadata is invalid.
- */
- public void save() throws IOException, InvalidMetadataException {
- if (this.metadataFilePath == null || StringUtils.isEmpty(this.metadataFilePath)) {
- throw new InvalidObjectException("Null or empty metadataFilePath. Cannot save metadata until this property is set.");
- }
-
- //quick check to ensure that the metadata we constructed is sane
- this.validateConsistency();
-
- synchronized (saveSync) {
- File curMetadata = new File(this.metadataFilePath);
- if (curMetadata.exists()) {
- curMetadata.delete();
- }
-
- // always create the full path to the file, since this will not throw if it already exists.
- curMetadata.getParentFile().mkdirs();
- curMetadata.createNewFile();
- try {
- FileOutputStream fileOut =
- new FileOutputStream(this.metadataFilePath);
- ObjectOutputStream out = new ObjectOutputStream(fileOut);
- out.writeObject(this);
- out.close();
- fileOut.close();
- } catch (Exception ex) {
- throw new InvalidMetadataException("Unable to parse metadata object and write it to a file", ex);
- }
- }
- }
-
- /**
- * Deletes the metadata file from disk.
- *
- * @throws InvalidObjectException Thrown if the metadata file path has not yet been set.
- */
- public void deleteFile() throws InvalidObjectException {
- if (this.metadataFilePath == null || StringUtils.isEmpty(this.metadataFilePath)) {
- throw new InvalidObjectException("Null or empty metadataFilePath. Cannot delete metadata until this property is set.");
- }
-
- File curMetadata = new File(this.metadataFilePath);
- if (curMetadata.exists()) {
- curMetadata.delete();
- }
- }
-
- /**
- * Verifies the given metadata for consistency. Checks include:
- * Completeness
- * Existence and consistency with local file
- * Segment data consistency
- *
- * @throws InvalidMetadataException Thrown if the metadata is invalid.
- */
- public void validateConsistency() throws InvalidMetadataException {
- if (this.segments == null || this.segments.length != this.segmentCount) {
- throw new InvalidMetadataException("Inconsistent number of segments");
- }
-
- long sum = 0;
- int lastSegmentNumber = -1;
- BitSet segments = new BitSet(this.segmentCount);
-
- for (UploadSegmentMetadata segment : this.segments) {
- if (segment.getSegmentNumber() < 0 || segment.getSegmentNumber() >= this.segmentCount) {
- throw new InvalidMetadataException(MessageFormat.format("Segment numbers must be at least 0 and less than {0}. Found segment number {1}.", this.segmentCount, segment.getSegmentNumber()));
- }
-
- if (segment.getSegmentNumber() <= lastSegmentNumber) {
- throw new InvalidMetadataException(MessageFormat.format("Segment number {0} appears out of order.", segment.getSegmentNumber()));
- }
-
- if (segments.get(segment.getSegmentNumber())) {
- throw new InvalidMetadataException(MessageFormat.format("Segment number {0} appears twice", segment.getSegmentNumber()));
- }
-
- if (segment.getOffset() != sum) {
- throw new InvalidMetadataException(MessageFormat.format("Segment number {0} has an invalid starting offset ({1}). Expected {2}.", segment.getSegmentNumber(), segment.getOffset(), sum));
- }
-
- segments.set(segment.getSegmentNumber());
- sum += segment.getLength();
- lastSegmentNumber = segment.getSegmentNumber();
- }
-
- if (sum != this.fileLength) {
- throw new InvalidMetadataException("The individual segment lengths do not add up to the input File length");
- }
- }
-
- /**
- * Splits the target stream path, returning the name of the stream and storing the full directory path (if any) in an out variable.
- *
- * @return A string array with the stream name is at index 0 and the stream path (if any) at index 1.
- */
- public String[] splitTargetStreamPathByName() {
- String[] toReturn = new String[2];
- int numFoldersInPath = this.targetStreamPath.split("/").length;
- if (numFoldersInPath - 1 == 0 || (numFoldersInPath - 1 == 1 && this.targetStreamPath.startsWith("/"))) {
- // the scenario where the file is being uploaded at the root
- toReturn[0] = this.targetStreamPath.replaceAll("^[/]", "");
- toReturn[1] = null;
- } else {
- // the scenario where the file is being uploaded in a sub folder
- toReturn[0] = this.targetStreamPath.substring(this.targetStreamPath.lastIndexOf('/') + 1);
- toReturn[1] = this.targetStreamPath.substring(0, this.targetStreamPath.lastIndexOf('/'));
- }
-
- return toReturn;
- }
-}
diff --git a/azure-mgmt-datalake-store-uploader/src/main/java/com/microsoft/azure/management/datalake/store/uploader/UploadMetadataGenerator.java b/azure-mgmt-datalake-store-uploader/src/main/java/com/microsoft/azure/management/datalake/store/uploader/UploadMetadataGenerator.java
deleted file mode 100644
index e2e650f27dcc..000000000000
--- a/azure-mgmt-datalake-store-uploader/src/main/java/com/microsoft/azure/management/datalake/store/uploader/UploadMetadataGenerator.java
+++ /dev/null
@@ -1,252 +0,0 @@
-/**
- * Copyright (c) Microsoft Corporation. All rights reserved.
- * Licensed under the MIT License. See License.txt in the project root for
- * license information.
- */
-package com.microsoft.azure.management.datalake.store.uploader;
-
-import org.apache.commons.lang3.ArrayUtils;
-import org.apache.commons.lang3.StringUtils;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.RandomAccessFile;
-import java.nio.charset.Charset;
-import java.text.MessageFormat;
-
-/**
- * An internally used class for generating the metadata used for upload.
- */
-public class UploadMetadataGenerator {
-
- private UploadParameters parameters;
- private int maxAppendLength;
-
- /**
- * Creates a new instance of the UploadMetadataGenerator with the given parameters and the default maximum append length.
- *
- * @param parameters The parameters.
- */
- public UploadMetadataGenerator(UploadParameters parameters) {
- this(parameters, SingleSegmentUploader.BUFFER_LENGTH);
- }
-
- /**
- * Creates a new instance of the UploadMetadataGenerator with the given parameters and the given maximum append length.
- *
- * @param parameters The parameters
- * @param maxAppendLength The maximum allowed append length when uploading a file.
- */
- public UploadMetadataGenerator(UploadParameters parameters, int maxAppendLength) {
- this.parameters = parameters;
- this.maxAppendLength = maxAppendLength;
- }
-
- /**
- * Attempts to load the metadata from an existing file in its canonical location.
- *
- * @param metadataFilePath The metadata file path.
- * @return The deserialized {@link UploadMetadata} from the specified file path.
- * @throws FileNotFoundException Thrown if the specified metadataFilePath is invalid.
- * @throws InvalidMetadataException Thrown if the metadata itself is invalid.
- */
- public UploadMetadata getExistingMetadata(String metadataFilePath) throws FileNotFoundException, InvalidMetadataException {
- //load from file (based on input parameters)
- UploadMetadata metadata = UploadMetadata.loadFrom(metadataFilePath);
- metadata.validateConsistency();
- return metadata;
- }
-
- /**
- * Creates a new metadata based on the given input parameters, and saves it to its canonical location.
- *
- * @param metadataFilePath Where the serialized metadata will be saved
- * @return A new {@link UploadMetadata} object.
- * @throws IOException Thrown if there is an issue saving the metadata to disk.
- * @throws UploadFailedException Thrown if there is an issue aligning the segment record boundaries
- * @throws InvalidMetadataException Thrown if the metadata is invalid.
- */
- public UploadMetadata createNewMetadata(String metadataFilePath) throws IOException, UploadFailedException, InvalidMetadataException {
- //determine segment count, segment length and upload Id
- //create metadata
- UploadMetadata metadata = new UploadMetadata(metadataFilePath, parameters);
-
- if (!parameters.isBinary() && metadata.getSegmentCount() > 1) {
- this.alignSegmentsToRecordBoundaries(metadata);
- }
-
- //save the initial version
- metadata.save();
-
- return metadata;
- }
-
- /**
- * Aligns segments to match record boundaries (where a record boundary = a new line).
- * If not possible (max record size = 4MB), throws an exception.
- *
- * @param metadata The metadata to realign
- * @throws IOException Thrown if the input file path in the metadata is invalid or inaccessible.
- * @throws UploadFailedException Thrown if the length adjustment cannot be determined.
- */
- private void alignSegmentsToRecordBoundaries(UploadMetadata metadata) throws IOException, UploadFailedException {
- int remainingSegments = 0;
-
- try (RandomAccessFile stream = new RandomAccessFile(metadata.getInputFilePath(), "r")) {
- long offset = 0;
- for (int i = 0; i < metadata.getSegments().length; i++) {
- UploadSegmentMetadata segment = metadata.getSegments()[i];
-
- //updating segment lengths means that both the offset and the length of the next segment needs to be recalculated, to keep the segment lengths somewhat balanced
- long diff = segment.getOffset() - offset;
- segment.setOffset(offset);
- segment.setLength(segment.getLength() + diff);
- if (segment.getOffset() >= metadata.getFileLength()) {
- continue;
- }
-
- if (segment.getSegmentNumber() == metadata.getSegments().length - 1) {
- //last segment picks up the slack
- segment.setLength(metadata.getFileLength() - segment.getOffset());
- } else {
- //figure out how much do we need to adjust the length of the segment so it ends on a record boundary (this can be negative or positive)
- int lengthAdjustment = determineLengthAdjustment(segment, stream, Charset.forName(metadata.getEncodingName()), metadata.getDelimiter()) + 1;
-
- //adjust segment length and offset
- segment.setLength(segment.getLength() + lengthAdjustment);
- }
- offset += segment.getLength();
- remainingSegments++;
- }
- }
-
- //since we adjusted the segment lengths, it's possible that the last segment(s) became of zero length; so remove it
- UploadSegmentMetadata[] segments = metadata.getSegments();
- if (remainingSegments < segments.length) {
- ArrayUtils.subarray(segments, 0, remainingSegments);
- metadata.setSegments(segments);
- metadata.setSegmentCount(segments.length);
- }
-
- //NOTE: we are not validating consistency here; this method is called by createNewMetadata which calls save() after this, which validates consistency anyway.
- }
-
- /**
- * Calculates the value by which we'd need to adjust the length of the given segment, by searching for the nearest newline around it (before and after),
- * and returning the distance to it (which can be positive, if after, or negative, if before).
- *
- * @param segment The segment to do the calculation on.
- * @param stream The full stream used to figure out the adjustment
- * @param encoding The encoding to use to determine where the cutoffs are
- * @param delimiter The delimiter that determines how we adjust. If null then '\\r', \\n' and '\\r\\n' are used.
- * @return How much to adjust the segment length by.
- * @throws UploadFailedException Thrown if proper upload boundaries cannot be determined.
- * @throws IOException Thrown if the stream being used is invalid or inaccessible.
- */
- private int determineLengthAdjustment(UploadSegmentMetadata segment, RandomAccessFile stream, Charset encoding, String delimiter) throws UploadFailedException, IOException {
- long referenceFileOffset = segment.getOffset() + segment.getLength();
- byte[] buffer = new byte[maxAppendLength];
-
- //read 2MB before the segment boundary and 2MB after (for a total of 4MB = max append length)
- int bytesRead = readIntoBufferAroundReference(stream, buffer, referenceFileOffset);
- if (bytesRead > 0) {
- int middlePoint = bytesRead / 2;
- //search for newline in it
- int newLinePosBefore = StringExtensions.findNewline(buffer, middlePoint + 1, middlePoint + 1, true, encoding, delimiter);
-
- //in some cases, we may have a newline that is 2 characters long, and it occurrs exactly on the midpoint, which means we won't be able to find its end.
- //see if that's the case, and then search for a new candidate before it.
- if ((delimiter == null || StringUtils.isEmpty(delimiter)) && newLinePosBefore == middlePoint + 1 && buffer[newLinePosBefore] == (byte) '\r') {
- int newNewLinePosBefore = StringExtensions.findNewline(buffer, middlePoint, middlePoint, true, encoding, delimiter);
- if (newNewLinePosBefore >= 0) {
- newLinePosBefore = newNewLinePosBefore;
- }
- }
-
- int newLinePosAfter = StringExtensions.findNewline(buffer, middlePoint, middlePoint, false, encoding, delimiter);
- if ((delimiter == null || StringUtils.isEmpty(delimiter)) && newLinePosAfter == buffer.length - 1 && buffer[newLinePosAfter] == (byte) '\r' && newLinePosBefore >= 0) {
- newLinePosAfter = -1;
- }
-
- int closestNewLinePos = findClosestToCenter(newLinePosBefore, newLinePosAfter, middlePoint);
-
- //middle point of the buffer corresponds to the reference file offset, so all we need to do is return the difference between the closest newline and the center of the buffer
- if (closestNewLinePos >= 0) {
- return closestNewLinePos - middlePoint;
- }
- }
-
- //if we get this far, we were unable to find a record boundary within our limits => fail the upload
- throw new UploadFailedException(
- MessageFormat.format(
- "Unable to locate a record boundary within {0}MB on either side of segment {1} (offset {2}). This means the record at that offset is larger than {0}MB.",
- maxAppendLength / 1024 / 1024 / 2,
- segment.getSegmentNumber(),
- segment.getOffset(),
- maxAppendLength / 1024 / 1024));
- }
-
- /**
- * Returns the value (of the given two) that is closest in absolute terms to the center value.
- * Values that are negative are ignored (since these are assumed to represent array indices).
- *
- * @param value1 First value to compare
- * @param value2 Second value to compare
- * @param centerValue The center value they are compared against.
- * @return Either value1 or value2 depending on which is closest to the centerValue
- */
- private static int findClosestToCenter(int value1, int value2, int centerValue) {
- if (value1 >= 0) {
- if (value2 >= 0) {
- return Math.abs(value2 - centerValue) > Math.abs(value1 - centerValue) ? value1 : value2;
- } else {
- return value1;
- }
- } else {
- return value2;
- }
- }
-
- /**
- * Reads data from the given file into the given buffer, centered around the given file offset. The first half of the buffer will be
- * filled with data right before the given offset, while the remainder of the buffer will contain data right after it (of course, containing the byte at the given offset).
- * @param stream The stream to read from
- * @param buffer The buffer to read data into
- * @param fileReferenceOffset The offset to start reading from in the stream.
- * @return The number of bytes reads, which could be less than the length of the input buffer if we can't read due to the beginning or the end of the file.
- * @throws IOException Thrown if the stream being used is invalid or inaccessible.
- */
- private static int readIntoBufferAroundReference(RandomAccessFile stream, byte[] buffer, long fileReferenceOffset) throws IOException {
- int length = buffer.length;
- //calculate start offset
- long fileStartOffset = fileReferenceOffset - length / 2;
-
- if (fileStartOffset < 0) {
- //offset is less than zero, adjust it, as well as the length we want to read
- length += (int) fileStartOffset;
- fileStartOffset = 0;
- if (length <= 0) {
- return 0;
- }
- }
-
- if (fileStartOffset + length > stream.length()) {
- //startOffset + length is beyond the end of the stream, adjust the length accordingly
- length = (int) (stream.length() - fileStartOffset);
- if (length <= 0) {
- return 0;
- }
- }
-
- //read the appropriate block of the file into the buffer, using symmetry with respect to its midpoint
- // we always initiate a seek from the origin of the file.
- stream.seek(0);
- stream.seek(fileStartOffset);
- int bufferOffset = 0;
- while (bufferOffset < length) {
- int bytesRead = stream.read(buffer, bufferOffset, length - bufferOffset);
- bufferOffset += bytesRead;
- }
- return length;
- }
-}
diff --git a/azure-mgmt-datalake-store-uploader/src/main/java/com/microsoft/azure/management/datalake/store/uploader/UploadParameters.java b/azure-mgmt-datalake-store-uploader/src/main/java/com/microsoft/azure/management/datalake/store/uploader/UploadParameters.java
deleted file mode 100644
index fc8636e1213d..000000000000
--- a/azure-mgmt-datalake-store-uploader/src/main/java/com/microsoft/azure/management/datalake/store/uploader/UploadParameters.java
+++ /dev/null
@@ -1,371 +0,0 @@
-/**
- * Copyright (c) Microsoft Corporation. All rights reserved.
- * Licensed under the MIT License. See License.txt in the project root for
- * license information.
- */
-package com.microsoft.azure.management.datalake.store.uploader;
-
-import org.apache.commons.lang3.StringUtils;
-
-import java.nio.charset.Charset;
-import java.nio.charset.StandardCharsets;
-
-/**
- * Represents parameters for the DataLake Uploader.
- */
-public class UploadParameters {
-
- /**
- * The default segment length that is used to ensure maximum life cycle performance for uploaded data.
- * 256MB is the default and should not be changed without a good reason.
- */
- private static final long SEGMENT_LENGTH = 256 * 1024 * 1024;
- /**
- * Creates a new set of parameters for the DataLake Uploader with optional values set with their defaults.
- * Defaults are as follows:
- * threadCount = 1
- * isOverwrite = false
- * isResume = false
- * isBinary = true
- * maxSegmentLength = 256mb
- * localMetadataLocation = File.createTempFile()
- *
- * @param inputFilePath The full path to the file to be uploaded.
- * @param targetStreamPath The full stream path where the file will be uploaded to.
- * @param accountName Name of the account to upload to.
- */
- public UploadParameters(String inputFilePath, String targetStreamPath, String accountName) {
- this(inputFilePath, targetStreamPath, accountName, 1, false, false, true, SEGMENT_LENGTH, null);
- }
-
- /**
- * Creates a new set of parameters for the DataLake Uploader with the following optional values set with their defaults.
- * Defaults are as follows:
- * isBinary = true
- * maxSegmentLength = 256mb
- * localMetadataLocation = File.createTempFile().
- *
- * @param inputFilePath The full path to the file to be uploaded.
- * @param targetStreamPath The full stream path where the file will be uploaded to.
- * @param accountName Name of the account to upload to.
- * @param threadCount The maximum number of parallel threads to use for the upload.
- * @param isOverwrite Whether to overwrite the target stream or not.
- * @param isResume Indicates whether to resume a previously interrupted upload.
- */
- public UploadParameters(String inputFilePath, String targetStreamPath, String accountName, int threadCount, boolean isOverwrite, boolean isResume) {
- this(inputFilePath, targetStreamPath, accountName, threadCount, isOverwrite, isResume, true, SEGMENT_LENGTH, null);
- }
-
- /**
- * Creates a new set of parameters for the DataLake Uploader with the following optional values set with their defaults.
- * Defaults are as follows:
- * isBinary = true
- * maxSegmentLength = 256mb
- *
- * @param inputFilePath The full path to the file to be uploaded.
- * @param targetStreamPath The full stream path where the file will be uploaded to.
- * @param accountName Name of the account to upload to.
- * @param threadCount The maximum number of parallel threads to use for the upload.
- * @param isOverwrite Whether to overwrite the target stream or not.
- * @param isResume Indicates whether to resume a previously interrupted upload.
- * @param localMetadataLocation Indicates the directory path where to store the local upload metadata file while the upload is in progress. This location must be writeable from this application. Default location if null: File.createTempFile()
- */
- public UploadParameters(String inputFilePath, String targetStreamPath, String accountName, int threadCount, boolean isOverwrite, boolean isResume, String localMetadataLocation) {
- this(inputFilePath, targetStreamPath, accountName, threadCount, isOverwrite, isResume, true, SEGMENT_LENGTH, localMetadataLocation);
- }
-
- /**
- * Creates a new set of parameters for the DataLake Uploader.
- *
- * @param inputFilePath The full path to the file to be uploaded.
- * @param targetStreamPath The full stream path where the file will be uploaded to.
- * @param accountName Name of the account to upload to.
- * @param threadCount The maximum number of parallel threads to use for the upload.
- * @param isOverwrite Whether to overwrite the target stream or not.
- * @param isResume Indicates whether to resume a previously interrupted upload.
- * @param isBinary Indicates whether to treat the input file as a binary file (true), or whether to align upload blocks to record boundaries (false).
- * @param maxSegmentLength The recommended value is 256mb, which gives optimal performance. Modify at your own risk.
- * @param localMetadataLocation Indicates the directory path where to store the local upload metadata file while the upload is in progress. This location must be writeable from this application. Default location if null: File.createTempFile()
- */
- public UploadParameters(String inputFilePath, String targetStreamPath, String accountName, int threadCount, boolean isOverwrite, boolean isResume, boolean isBinary, long maxSegmentLength, String localMetadataLocation) {
- this.setInputFilePath(inputFilePath);
- this.setTargetStreamPath(targetStreamPath);
- this.setThreadCount(threadCount);
- this.setAccountName(accountName);
- this.setOverwrite(isOverwrite);
- this.setResume(isResume);
- this.setBinary(isBinary);
- this.setMaxSegementLength(maxSegmentLength);
-
- if (localMetadataLocation == null || StringUtils.isEmpty(localMetadataLocation)) {
- localMetadataLocation = System.getProperty("java.io.tmpdir");
- }
-
- this.setLocalMetadataLocation(localMetadataLocation);
-
- this.setUseSegmentBlockBackOffRetryStrategy(true);
-
- // TODO: in the future we will expose these as optional parameters, allowing customers to specify encoding and delimiters.
- this.setFileEncoding(StandardCharsets.UTF_8);
- this.setDelimiter(null);
- }
-
- /**
- * Creates a new set of parameters for the DataLake Uploader used for unit testing.
- *
- * @param inputFilePath The full path to the file to be uploaded.
- * @param targetStreamPath The full stream path where the file will be uploaded to.
- * @param accountName Name of the account to upload to.
- * @param useSegmentBlockBackOffRetryStrategy if set to true [use segment block back off retry strategy].
- * @param threadCount The maximum number of parallel threads to use for the upload.
- * @param isOverwrite Whether to overwrite the target stream or not.
- * @param isResume Indicates whether to resume a previously interrupted upload.
- * @param isBinary Indicates whether to treat the input file as a binary file (true), or whether to align upload blocks to record boundaries (false).
- * @param maxSegmentLength The recommended value is 256mb, which gives optimal performance. Modify at your own risk.
- * @param localMetadataLocation Indicates the directory path where to store the local upload metadata file while the upload is in progress. This location must be writeable from this application. Default location if null: File.createTempFile()
- */
- protected UploadParameters(String inputFilePath, String targetStreamPath, String accountName, boolean useSegmentBlockBackOffRetryStrategy, int threadCount, boolean isOverwrite, boolean isResume, boolean isBinary, long maxSegmentLength, String localMetadataLocation) {
- this(inputFilePath, targetStreamPath, accountName, threadCount, isOverwrite, isResume, isBinary, maxSegmentLength, localMetadataLocation);
- this.setUseSegmentBlockBackOffRetryStrategy(useSegmentBlockBackOffRetryStrategy);
- }
-
- /**
- * Gets a value indicating whether [to use segment block back off retry strategy].
- *
- * @return true if [to use segment block back off retry strategy]; otherwise, false.
- */
- public boolean isUseSegmentBlockBackOffRetryStrategy() {
- return useSegmentBlockBackOffRetryStrategy;
- }
-
- /**
- * Internally sets the value of whether [to use segment block back off retry strategy].
- *
- * @param useSegmentBlockBackOffRetryStrategy
- */
- private void setUseSegmentBlockBackOffRetryStrategy(boolean useSegmentBlockBackOffRetryStrategy) {
- this.useSegmentBlockBackOffRetryStrategy = useSegmentBlockBackOffRetryStrategy;
- }
-
- /**
- * Gets a value indicating the full path to the file to be uploaded.
- *
- * @return The input file path.
- */
- public String getInputFilePath() {
- return inputFilePath;
- }
-
- /**
- * Internally sets the input file path.
- *
- * @param inputFilePath
- */
- private void setInputFilePath(String inputFilePath) {
- this.inputFilePath = inputFilePath;
- }
-
- /**
- * Gets a value indicating the full stream path where the file will be uploaded to.
- *
- * @return The target stream path.
- */
- public String getTargetStreamPath() {
- return targetStreamPath;
- }
-
- /**
- * Internally sets the target stream path.
- *
- * @param targetStreamPath
- */
- private void setTargetStreamPath(String targetStreamPath) {
- this.targetStreamPath = targetStreamPath;
- }
-
- /**
- * Gets a value indicating the name of the account to upload to.
- *
- * @return The name of the account.
- */
- public String getAccountName() {
- return accountName;
- }
-
- /**
- * Internally sets the account name to upload to.
- *
- * @param accountName
- */
- private void setAccountName(String accountName) {
- this.accountName = accountName;
- }
-
- /**
- * Gets a value indicating the maximum number of parallel threads to use for the upload.
- *
- * @return The thread count.
- */
- public int getThreadCount() {
- return threadCount;
- }
-
- /**
- * Internally sets the number of threads that are allowed for the upload.
- *
- * @param threadCount The number of threads to use for the upload.
- */
- protected void setThreadCount(int threadCount) {
- this.threadCount = threadCount;
- }
-
- /**
- * Gets a value indicating whether to overwrite the target stream if it already exists.
- *
- * @return true if this instance is overwrite; otherwise, false.
- */
- public boolean isOverwrite() {
- return overwrite;
- }
-
- /**
- * Internally sets whether the target stream can be overwritten.
- *
- * @param overwrite
- */
- private void setOverwrite(boolean overwrite) {
- this.overwrite = overwrite;
- }
-
- /**
- * Gets a value indicating whether to resume a previously interrupted upload.
- *
- * @return true if this instance is resume; otherwise, false.
- */
- public boolean isResume() {
- return resume;
- }
-
- /**
- * Internally set whether this is a previous upload being resumed.
- *
- * @param resume
- */
- private void setResume(boolean resume) {
- this.resume = resume;
- }
-
- /**
- * Gets a value indicating whether the input file should be treated as a binary (true) or a delimited input (false).
- *
- * @return true if this instance is binary; otherwise, false.
- */
- public boolean isBinary() {
- return binary;
- }
-
- /**
- * Internally set whether the file being uploaded should be binary or delimited input.
- *
- * @param binary
- */
- private void setBinary(boolean binary) {
- this.binary = binary;
- }
-
- /**
- * Gets the maximum length of each segement in bytes.
- *
- * @return The maximum length of each segment in bytes.
- */
- public long getMaxSegementLength() {
- return maxSegementLength;
- }
-
- /**
- * Internally set the maximum length of each segment in bytes.
- *
- * @param maxSegementLength
- */
- private void setMaxSegementLength(long maxSegementLength) {
- this.maxSegementLength = maxSegementLength;
- }
-
- /**
- * Gets a value indicating the directory path where to store the metadata for the upload.
- *
- * @return The local metadata location.
- */
- public String getLocalMetadataLocation() {
- return localMetadataLocation;
- }
-
- /**
- * Internally set the local metadata location.
- *
- * @param localMetadataLocation
- */
- private void setLocalMetadataLocation(String localMetadataLocation) {
- this.localMetadataLocation = localMetadataLocation;
- }
-
- /**
- * Gets a value indicating the encoding of the file being uploaded.
- *
- * @return The file encoding.
- */
- public Charset getFileEncoding() {
- return fileEncoding;
- }
-
- /**
- * Internally sets the value of the file encoding.
- *
- * @param fileEncoding
- */
- private void setFileEncoding(Charset fileEncoding) {
- this.fileEncoding = fileEncoding;
- }
-
- /**
- * Gets a value indicating the record boundary delimiter for the file, if any.
- *
- * @return The record boundary delimiter
- */
- public String getDelimiter() {
- return delimiter;
- }
-
- /**
- * Internally set the value of the record boundary delimiter.
- *
- * @param delimiter
- */
- private void setDelimiter(String delimiter) {
- this.delimiter = delimiter;
- }
-
- private boolean useSegmentBlockBackOffRetryStrategy;
-
- private String inputFilePath;
-
- private String targetStreamPath;
-
- private String accountName;
-
- private int threadCount;
-
- private boolean overwrite;
-
- private boolean resume;
-
- private boolean binary;
-
- private long maxSegementLength;
-
- private String localMetadataLocation;
-
- private Charset fileEncoding;
-
- private String delimiter;
-}
diff --git a/azure-mgmt-datalake-store-uploader/src/main/java/com/microsoft/azure/management/datalake/store/uploader/UploadSegmentMetadata.java b/azure-mgmt-datalake-store-uploader/src/main/java/com/microsoft/azure/management/datalake/store/uploader/UploadSegmentMetadata.java
deleted file mode 100644
index 07aadb6af73a..000000000000
--- a/azure-mgmt-datalake-store-uploader/src/main/java/com/microsoft/azure/management/datalake/store/uploader/UploadSegmentMetadata.java
+++ /dev/null
@@ -1,261 +0,0 @@
-/**
- * Copyright (c) Microsoft Corporation. All rights reserved.
- * Licensed under the MIT License. See License.txt in the project root for
- * license information.
- */
-package com.microsoft.azure.management.datalake.store.uploader;
-
-import java.io.Serializable;
-import java.text.MessageFormat;
-
-/**
- * Represents metadata for a particular file segment.
- */
-public class UploadSegmentMetadata implements Serializable {
-
- /**
- * Initializes a new instance of the UploadSegmentMetadata for use with unit tests.
- */
- protected UploadSegmentMetadata() {
- // does nothing, used for unit tests
- }
-
- /**
- * Creates a new UploadSegmentMetadata with the given segment number.
- *
- * @param segmentNumber The segment number for this instance.
- * @param metadata The full metadata associated with this segment.
- */
- public UploadSegmentMetadata(int segmentNumber, UploadMetadata metadata) {
- this.segmentNumber = segmentNumber;
- this.status = SegmentUploadStatus.Pending;
-
- String targetStreamName = metadata.splitTargetStreamPathByName()[0];
- this.path = MessageFormat.format("{0}/{1}.{2}.segment{3}", metadata.getSegmentStreamDirectory(), targetStreamName, metadata.getUploadId(), this.segmentNumber);
- this.offset = this.segmentNumber * metadata.getSegmentLength(); // segment number is zero-based
- this.length = calculateSegmentLength(this.segmentNumber, metadata);
- }
-
- /**
- * Calculates the length of a typical (non-terminal) segment for a file of the given length that is split into the given number of segments.
- *
- * @param fileLength The length of the file, in bytes.
- * @param segmentCount The number of segments to split the file into.
- * @return The length of this segment, in bytes.
- */
- public static long calculateSegmentLength(long fileLength, int segmentCount) {
- if (segmentCount < 0) {
- throw new IllegalArgumentException("Number of segments must be a positive integer");
- }
-
- if (segmentCount == 0) {
- // In this case, we are attempting to upload an empty file,
- // in which case the uploader should just return
- return 0;
- }
-
- long segmentLength = fileLength / segmentCount;
-
- //if the file cannot be split into even segments, we need to increment the typical segment length by 1
- //in order to have the last segment in the file be smaller than the other ones.
- if (fileLength % segmentCount != 0) {
- //BUT we can only do this IF this wouldn't cause the last segment to have a negative length
- if (fileLength - (segmentCount - 1) * (segmentLength + 1) > 0) {
- segmentLength++;
- }
- }
-
- return segmentLength;
- }
-
- /**
- * Calculates the length of the segment with given number for a file with given length that is split into the given number of segments.
- * @param segmentNumber The segment number.
- * @param metadata The metadata for the current upload.
- * @return The length of this segment, in bytes.
- */
- public static long calculateSegmentLength(int segmentNumber, UploadMetadata metadata) {
- if (segmentNumber < 0 || segmentNumber >= metadata.getSegmentCount()) {
- throw new IndexOutOfBoundsException("Segment Number must be at least zero and less than the total number of segments");
- }
-
- if (metadata.getFileLength() < 0) {
- throw new IllegalArgumentException("Cannot have a negative file length");
- }
-
- //verify if the last segment would have a positive value
- long lastSegmentLength = metadata.getFileLength() - (metadata.getSegmentCount() - 1) * metadata.getSegmentLength();
- if (lastSegmentLength < 0) {
- throw new IllegalArgumentException("The given values for segmentCount and segmentLength cannot possibly be used to split a file with the given fileLength (the last segment would have a negative length)");
- } else if (lastSegmentLength > metadata.getSegmentLength()) {
- //verify if the given segmentCount and segmentLength combination would produce an even split
- if (metadata.getFileLength() - (metadata.getSegmentCount() - 1) * (metadata.getSegmentLength() + 1) > 0) {
- throw new IllegalArgumentException("The given values for segmentCount and segmentLength would not produce an even split of a file with given fileLength");
- }
- }
-
- if (metadata.getFileLength() == 0) {
- return 0;
- }
-
- //all segments except the last one have the same length;
- //the last one only has the 'full' length if by some miracle the file length is a perfect multiple of the Segment length
- if (segmentNumber < metadata.getSegmentCount() - 1) {
- return metadata.getSegmentLength();
- } else {
- return lastSegmentLength;
- }
- }
-
- /**
- * Used to calculate the total number of segments that we should create.
- */
- private static final int BASE_MULTIPLIER = 50;
-
- /**
- * The Multiplier is the number of times the segment count is inflated when the length of the file increases by a factor of 'Reducer'.
- */
- private static final int SEGMENT_COUNT_MULTIPLIER = 2;
-
- /**
- * The minimum number of bytes in a segment. For best performance, should be sync-ed with the upload buffer length.
- */
- public static final int MINIMUM_SEGMENT_SIZE = SingleSegmentUploader.BUFFER_LENGTH;
-
- /**
- * Calculates the number of segments a file of the given length should be split into.
- * The method to calculate this is based on some empirical measurements that allows both the number of segments and the length of each segment to grow as the input file size grows.
- * They both grow on a logarithmic pattern as the file length increases.
- * The formula is roughly this:
- * Multiplier = Min(100, 50 * 2 ^ Log10(FileLengthInGB))
- * SegmentCount = Max(1, Multiplier * 2 ^ Log10(FileLengthInGB)
- * Essentially we quadruple the number of segments for each tenfold increase in the file length, with certain caps. The formula is designed to support both small files and
- * extremely large files (and not cause very small segment lengths or very large number of segments).
- *
- * @param fileLength The length of the file, in bytes.
- * @return The number of segments to split the file into. Returns 0 if fileLength is 0.
- */
- public static int calculateSegmentCount(long fileLength) {
- if (fileLength < 0) {
- throw new IllegalArgumentException("File length cannot be negative");
- }
-
- if (fileLength == 0) {
- //empty file => no segments
- return 0;
- }
-
- int minNumberOfSegments = (int) Math.max(1, fileLength / MINIMUM_SEGMENT_SIZE);
-
- //convert the file length into GB
- double lengthInGb = fileLength / 1024.0 / 1024 / 1024;
-
- //apply the formula described in the class description and return the result
- double baseMultiplier = calculateBaseMultiplier(lengthInGb);
- int segmentCount = (int) (baseMultiplier * Math.pow(SEGMENT_COUNT_MULTIPLIER, Math.log10(lengthInGb)));
- if (segmentCount > minNumberOfSegments) {
- segmentCount = minNumberOfSegments;
- }
-
- if (segmentCount < 1) {
- segmentCount = 1;
- }
-
- return segmentCount;
- }
-
- private static double calculateBaseMultiplier(double lengthInGb) {
- double value = BASE_MULTIPLIER * Math.pow(2, Math.log10(lengthInGb));
- return Math.min(100, value);
- }
-
- private int segmentNumber;
-
- private long offset;
-
- private long length;
-
- private SegmentUploadStatus status;
-
- private String path;
-
- /**
- *
- * @return A value indicating the stream path assigned to this segment.
- */
- public String getPath() {
- return path;
- }
-
- /**
- *
- * @param path A value indicating the stream path assigned to this segment.
- */
- public void setPath(String path) {
- this.path = path;
- }
-
- /**
- *
- * @return A value indicating the number (sequence) of the segment in the file.
- */
- public int getSegmentNumber() {
- return segmentNumber;
- }
-
- /**
- *
- * @param segmentNumber A value indicating the number (sequence) of the segment in the file.
- */
- public void setSegmentNumber(int segmentNumber) {
- this.segmentNumber = segmentNumber;
- }
-
- /**
- *
- * @return A value indicating the starting offset of the segment in the file.
- */
- public long getOffset() {
- return offset;
- }
-
- /**
- *
- * @param offset A value indicating the starting offset of the segment in the file.
- */
- public void setOffset(long offset) {
- this.offset = offset;
- }
-
- /**
- *
- * @return A value indicating the size of the segment (in bytes).
- */
- public long getLength() {
- return length;
- }
-
- /**
- *
- * @param length A value indicating the size of the segment (in bytes).
- */
- public void setLength(long length) {
- this.length = length;
- }
-
- /**
- *
- * @return A value indicating the current upload status for this segment.
- */
- public SegmentUploadStatus getStatus() {
- return status;
- }
-
- /**
- *
- * @param status A value indicating the current upload status for this segment.
- */
- public void setStatus(SegmentUploadStatus status) {
- this.status = status;
- }
-}
diff --git a/azure-mgmt-datalake-store-uploader/src/main/java/com/microsoft/azure/management/datalake/store/uploader/package-info.java b/azure-mgmt-datalake-store-uploader/src/main/java/com/microsoft/azure/management/datalake/store/uploader/package-info.java
deleted file mode 100644
index d0384e3d1d10..000000000000
--- a/azure-mgmt-datalake-store-uploader/src/main/java/com/microsoft/azure/management/datalake/store/uploader/package-info.java
+++ /dev/null
@@ -1,9 +0,0 @@
-// Copyright (c) Microsoft Corporation. All rights reserved.
-// Licensed under the MIT License. See License.txt in the project root for
-// license information.
-
-/**
- * This package contains the classes for DataLakeStoreUploader.
- * The client used to efficiently and rapidly upload files into an Azure Data Lake Store account.
- */
-package com.microsoft.azure.management.datalake.store.uploader;
\ No newline at end of file
diff --git a/azure-mgmt-datalake-store-uploader/src/test/java/com/microsoft/azure/management/datalake/store/uploader/DataLakeUploaderTestBase.java b/azure-mgmt-datalake-store-uploader/src/test/java/com/microsoft/azure/management/datalake/store/uploader/DataLakeUploaderTestBase.java
deleted file mode 100644
index aaa1a0870cdc..000000000000
--- a/azure-mgmt-datalake-store-uploader/src/test/java/com/microsoft/azure/management/datalake/store/uploader/DataLakeUploaderTestBase.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/**
- * Copyright (c) Microsoft Corporation. All rights reserved.
- * Licensed under the MIT License. See License.txt in the project root for
- * license information.
- */
-package com.microsoft.azure.management.datalake.store.uploader;
-
-import com.microsoft.azure.AzureEnvironment;
-import com.microsoft.azure.RestClient;
-import com.microsoft.azure.credentials.UserTokenCredentials;
-import com.microsoft.azure.management.datalake.store.implementation.DataLakeStoreAccountManagementClientImpl;
-import com.microsoft.azure.management.datalake.store.implementation.DataLakeStoreFileSystemManagementClientImpl;
-import com.microsoft.azure.management.resources.implementation.ResourceManagementClientImpl;
-import okhttp3.OkHttpClient;
-import okhttp3.logging.HttpLoggingInterceptor;
-import retrofit2.Retrofit;
-
-import java.util.concurrent.TimeUnit;
-
-public abstract class DataLakeUploaderTestBase {
- protected static ResourceManagementClientImpl resourceManagementClient;
- protected static DataLakeStoreAccountManagementClientImpl dataLakeStoreAccountManagementClient;
- protected static DataLakeStoreFileSystemManagementClientImpl dataLakeStoreFileSystemManagementClient;
-
- public static void createClients() {
- UserTokenCredentials credentials = new UserTokenCredentials(
- System.getenv("arm.clientid"),
- System.getenv("arm.domain"),
- System.getenv("arm.username"),
- System.getenv("arm.password"),
- AzureEnvironment.AZURE);
-
- RestClient restClient = new RestClient.Builder()
- .withBaseUrl(AzureEnvironment.AZURE, AzureEnvironment.Endpoint.RESOURCE_MANAGER)
- .withCredentials(credentials)
- .withLogLevel(HttpLoggingInterceptor.Level.BODY)
- .build();
-
- resourceManagementClient = new ResourceManagementClientImpl(restClient).withSubscriptionId(System.getenv("arm.subscriptionid"));
- dataLakeStoreAccountManagementClient = new DataLakeStoreAccountManagementClientImpl(restClient).withSubscriptionId(System.getenv("arm.subscriptionid"));
-
- RestClient dataPlaneClient = new RestClient.Builder(new OkHttpClient.Builder().connectTimeout(100, TimeUnit.SECONDS), new Retrofit.Builder())
- .withBaseUrl("https://{accountName}.{adlsFileSystemDnsSuffix}")
- .withCredentials(credentials)
- .withLogLevel(HttpLoggingInterceptor.Level.NONE) // No logging for this client because we are executing a lot of requests.
- .build();
-
- dataLakeStoreFileSystemManagementClient = new DataLakeStoreFileSystemManagementClientImpl(dataPlaneClient);
- }
-
- public static String generateName(String prefix) {
- int randomSuffix = (int) (Math.random() * 1000);
- return prefix + randomSuffix;
- }
-}
diff --git a/azure-mgmt-datalake-store-uploader/src/test/java/com/microsoft/azure/management/datalake/store/uploader/DataLakeUploaderTests.java b/azure-mgmt-datalake-store-uploader/src/test/java/com/microsoft/azure/management/datalake/store/uploader/DataLakeUploaderTests.java
deleted file mode 100644
index ab255c5824af..000000000000
--- a/azure-mgmt-datalake-store-uploader/src/test/java/com/microsoft/azure/management/datalake/store/uploader/DataLakeUploaderTests.java
+++ /dev/null
@@ -1,366 +0,0 @@
-/**
- * Copyright (c) Microsoft Corporation. All rights reserved.
- * Licensed under the MIT License. See License.txt in the project root for
- * license information.
- */
-package com.microsoft.azure.management.datalake.store.uploader;
-
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import javax.management.OperationsException;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-
-/**
- * Unit tests for the uploader.
- */
-public class DataLakeUploaderTests {
- private static final int LargeFileLength = 50 * 1024 * 1024; // 50mb
- private static byte[] _largeFileData = new byte[LargeFileLength];
- private static String _largeFilePath;
- private static final int SmallFileLength = 128;
- private static byte[] _smallFileData = new byte[SmallFileLength];
- private static String _smallFilePath;
- private static final int ThreadCount = 1;
- private static final String TargetStreamPath = "1";
-
- private static String curMetadataPath;
-
- @BeforeClass
- public static void Setup() throws IOException {
- _largeFilePath = TestHelpers.GenerateFileData(_largeFileData);
- _smallFilePath = TestHelpers.GenerateFileData(_smallFileData);
- }
-
- @AfterClass
- public static void Teardown()
- {
- File large = new File(_largeFilePath);
- File small = new File(_smallFilePath);
- if (large.exists())
- {
- large.delete();
- }
-
- if (small.exists())
- {
- small.delete();
- }
- }
-
- /**
- * Tests the case when invalid parameters are being passed to the uploader.
- *
- * @throws Exception
- */
- @Test
- public void DataLakeUploader_InvalidParameters() throws Exception
- {
- //invalid file path
- File invalidFilePath = File.createTempFile("adlsUploader", "noneexistent");
- invalidFilePath.delete();
- Assert.assertFalse("Unit test error: generated temp file actually exists", invalidFilePath.exists());
-
- try {
- new DataLakeStoreUploader(new UploadParameters(invalidFilePath.toString(), "1", "foo", 1, false, false, true, 4 * 1024 * 1024, null),new InMemoryFrontEnd());
- Assert.assertTrue("Expected a file not found exception but no exception was thrown!", false);
- }
- catch (FileNotFoundException e) {
- // do nothing this is expected
- }
-
- //no target stream
- try {
- new DataLakeStoreUploader(new UploadParameters(_largeFilePath, null, "foo", 1, false, false, true, 4 * 1024 * 1024, null),new InMemoryFrontEnd());
- Assert.assertTrue("Expected a file not found exception but no exception was thrown!", false);
- }
- catch (IllegalArgumentException e) {
- // do nothing this is expected
- }
-
- //target stream ends in '/'
- try {
- new DataLakeStoreUploader(new UploadParameters(_largeFilePath, "1/", "foo", 1, false, false, true, 4 * 1024 * 1024, null),new InMemoryFrontEnd());
- Assert.assertTrue("Expected exception for invalid target stream but no exception was thrown!", false);
- }
- catch (IllegalArgumentException e) {
- // do nothing this is expected
- }
-
- //no account name
- try {
- new DataLakeStoreUploader(new UploadParameters(_largeFilePath, "1", null, 1, false, false, true, 4 * 1024 * 1024, null),new InMemoryFrontEnd());
- Assert.assertTrue("Expected exception for null account name but no exception was thrown!", false);
- }
- catch (IllegalArgumentException e) {
- // do nothing this is expected
- }
-
- //bad thread count
- try {
- new DataLakeStoreUploader(new UploadParameters(_largeFilePath, "1", "foo", 0, false, false, true, 4 * 1024 * 1024, null),new InMemoryFrontEnd());
- Assert.assertTrue("Expected an exception for invalid thread count but no exception was thrown!", false);
- }
- catch (IllegalArgumentException e) {
- // do nothing this is expected
- }
-
- try {
- new DataLakeStoreUploader(new UploadParameters(_largeFilePath, "1", "foo", DataLakeStoreUploader.MAX_ALLOWED_THREADS + 1, false, false, true, 4 * 1024 * 1024, null),new InMemoryFrontEnd());
- Assert.assertTrue("Expected an exception for invalid thread count but no exception was thrown!", false);
- }
- catch (IllegalArgumentException e) {
- // do nothing this is expected
- }
- }
-
- /**
- * Tests the case when the target stream exists and we haven't set the overwrite flag.
- *
- * @throws Exception
- */
- @Test
- public void DataLakeUploader_TargetExistsNoOverwrite() throws Exception {
- InMemoryFrontEnd frontEnd = new InMemoryFrontEnd();
- frontEnd.createStream(TargetStreamPath, true, null, 0);
-
- //no resume, no overwrite
- UploadParameters up = CreateParameters(false, false, _smallFilePath, true);
- DataLakeStoreUploader uploader = new DataLakeStoreUploader(up, frontEnd);
- try {
- uploader.execute();
- Assert.assertTrue("Expected an exception for no overwrite when file exists but no exception was thrown!", false);
- }
- catch (OperationsException e) {
- // expected
- }
-
- //resume, no overwrite
- up = CreateParameters(true, false, _smallFilePath, false);
- uploader = new DataLakeStoreUploader(up, frontEnd);
- try {
- uploader.execute();
- Assert.assertTrue("Expected an exception for no overwrite when file exists but no exception was thrown!", false);
- }
- catch (OperationsException e) {
- // expected
- }
-
- //resume, overwrite
- up = CreateParameters(true, true, _smallFilePath, false);
- uploader = new DataLakeStoreUploader(up, frontEnd);
- uploader.execute();
-
-
- //no resume, overwrite
- up = CreateParameters(false, true, _smallFilePath, true);
- uploader = new DataLakeStoreUploader(up, frontEnd);
- uploader.execute();
- }
-
- /**
- * Tests the case of a fresh upload with multiple segments.\
- *
- * @throws Exception
- */
- @Test
- public void DataLakeUploader_FreshUpload() throws Exception {
- InMemoryFrontEnd frontEnd = new InMemoryFrontEnd();
- UploadParameters up = CreateParameters(false, false, null, true);
- DataLakeStoreUploader uploader = new DataLakeStoreUploader(up, frontEnd);
-
- uploader.execute();
-
- VerifyFileUploadedSuccessfully(up, frontEnd);
- }
-
- /**
- * Tests the resume upload when the metadata indicates all files are uploaded but no files exist on the server.
- *
- * @throws Exception
- */
- @Test
- public void DataLakeUploader_ResumeUploadWithAllMissingFiles() throws Exception {
- //this scenario is achieved by refusing to execute the concat command on the front end for the initial upload (which will interrupt it)
- //and then resuming the upload against a fresh front-end (which obviously has no files there)
-
- InMemoryFrontEnd backingFrontEnd1 = new InMemoryFrontEnd();
- UploaderFrontEndMock frontEnd1 = new UploaderFrontEndMock(backingFrontEnd1, true, false);
-
- //attempt full upload
- UploadParameters up = CreateParameters(false, false, null, true);
- DataLakeStoreUploader uploader = new DataLakeStoreUploader(up, frontEnd1);
- uploader.deleteMetadataFile();
-
- try {
- uploader.execute();
- Assert.assertTrue("Expected an intentional exception during concat but none was thrown!", false);
- }
- catch (IntentionalException e) {
- // expected
- }
-
- Assert.assertFalse("Target stream should not have been created", frontEnd1.streamExists(up.getTargetStreamPath()));
- Assert.assertTrue("No temporary streams seem to have been created", 0 < backingFrontEnd1.getStreamCount());
-
- //attempt to resume the upload
- InMemoryFrontEnd frontEnd2 = new InMemoryFrontEnd();
- up = CreateParameters(true, false, null, false);
- uploader = new DataLakeStoreUploader(up, frontEnd2);
-
- //at this point the metadata exists locally but there are no target files in frontEnd2
- try
- {
- uploader.execute();
- }
- finally
- {
- uploader.deleteMetadataFile();
- }
-
- VerifyFileUploadedSuccessfully(up, frontEnd2);
- }
-
- /**
- * Tests the resume upload when only some segments were uploaded previously
- *
- * @throws Exception
- */
- @Test
- public void DataLakeUploader_ResumePartialUpload() throws Exception {
- //attempt to load the file fully, but only allow creating 1 target stream
- InMemoryFrontEnd backingFrontEnd = new InMemoryFrontEnd();
- UploaderFrontEndMock frontEnd = new UploaderFrontEndMock(backingFrontEnd, false, true);
-
- UploadParameters up = CreateParameters(false, false, null, true);
- DataLakeStoreUploader uploader = new DataLakeStoreUploader(up, frontEnd);
- uploader.deleteMetadataFile();
-
- try {
- uploader.execute();
- Assert.assertTrue("Expected an aggregate exception during upload due to failing out creating more than one stream but no exception was thrown!", false);
- }
- catch (AggregateUploadException e) {
- // expected
- }
-
- Assert.assertFalse("Target stream should not have been created", frontEnd.streamExists(up.getTargetStreamPath()));
- Assert.assertEquals(1, backingFrontEnd.getStreamCount());
-
- //resume the upload but point it to the real back-end, which doesn't throw exceptions
- up = CreateParameters(true, false, null, false);
- uploader = new DataLakeStoreUploader(up, backingFrontEnd);
-
- try
- {
- uploader.execute();
- }
- finally
- {
- uploader.deleteMetadataFile();
- }
-
- VerifyFileUploadedSuccessfully(up, backingFrontEnd);
- }
-
- /**
- * Tests the upload case with only 1 segment (since that is an optimization of the broader case).
- *
- * @throws Exception
- */
- @Test
- public void DataLakeUploader_UploadSingleSegment() throws Exception {
- InMemoryFrontEnd frontEnd = new InMemoryFrontEnd();
- File fileToFolder = File.createTempFile("adlsUploader", "segmentTest");
- fileToFolder.delete();
- fileToFolder.mkdirs();
- UploadParameters up = new UploadParameters(
- _smallFilePath,
- "1",
- "foo",
- ThreadCount,
- false,
- false,
- true,
- 4 * 1024 * 1024,
- fileToFolder.getAbsolutePath());
-
- FileOutputStream writer = new FileOutputStream(_smallFilePath);
- writer.write(_smallFileData);
- writer.flush();
- writer.close();
-
- DataLakeStoreUploader uploader = new DataLakeStoreUploader(up, frontEnd);
- uploader.execute();
-
- VerifyFileUploadedSuccessfully(up, frontEnd, _smallFileData);
- }
-
- /**
- * Creates a parameter object.
- *
- * @param isResume Whether to resume.
- * @param isOverwrite Whether to enable overwrite.
- * @param filePath The file path.
- * @param createNewFolder indicates that we should create a new folder location where the data should be placed.
- * @return A {@link UploadParameters} object.
- * @throws IOException
- */
- private UploadParameters CreateParameters(boolean isResume, boolean isOverwrite, String filePath, boolean createNewFolder) throws IOException {
- if (filePath == null)
- {
- filePath = _largeFilePath;
- }
-
- File fileToFolder = File.createTempFile("adlsUploader", "metadata");
- if(createNewFolder) {
- fileToFolder.delete();
- fileToFolder.mkdirs();
- curMetadataPath = fileToFolder.getAbsolutePath();
- }
-
- return new UploadParameters(
- filePath,
- "1",
- "foo",
- false,
- ThreadCount,
- isOverwrite,
- isResume,
- true,
- 4 * 1024 * 1024,
- curMetadataPath);
- }
-
- /**
- * Verifies the file was successfully uploaded.
- *
- * @param up The upload parameters.
- * @param frontEnd The front end to use.
- * @throws Exception
- */
- private void VerifyFileUploadedSuccessfully(UploadParameters up, InMemoryFrontEnd frontEnd) throws Exception {
- VerifyFileUploadedSuccessfully(up, frontEnd, _largeFileData);
- }
-
- /**
- * Verifies the file was successfully uploaded.
- * @param up The upload parameters.
- * @param frontEnd The front end to use.
- * @param fileContents The file contents.
- * @throws Exception
- */
- private void VerifyFileUploadedSuccessfully(UploadParameters up, InMemoryFrontEnd frontEnd, byte[] fileContents) throws Exception {
- Assert.assertTrue("Uploaded stream does not exist", frontEnd.streamExists(up.getTargetStreamPath()));
- Assert.assertEquals(1, frontEnd.getStreamCount());
- Assert.assertEquals(fileContents.length, frontEnd.getStreamLength(up.getTargetStreamPath()));
-
- byte[] uploadedData = frontEnd.GetStreamContents(up.getTargetStreamPath());
- Assert.assertArrayEquals("Uploaded stream is not binary identical to input file", fileContents, uploadedData);
- }
-}
diff --git a/azure-mgmt-datalake-store-uploader/src/test/java/com/microsoft/azure/management/datalake/store/uploader/InMemoryFrontEnd.java b/azure-mgmt-datalake-store-uploader/src/test/java/com/microsoft/azure/management/datalake/store/uploader/InMemoryFrontEnd.java
deleted file mode 100644
index d41715203254..000000000000
--- a/azure-mgmt-datalake-store-uploader/src/test/java/com/microsoft/azure/management/datalake/store/uploader/InMemoryFrontEnd.java
+++ /dev/null
@@ -1,276 +0,0 @@
-/**
- * Copyright (c) Microsoft Corporation. All rights reserved.
- * Licensed under the MIT License. See License.txt in the project root for
- * license information.
- */
-package com.microsoft.azure.management.datalake.store.uploader;
-
-import com.microsoft.azure.CloudException;
-
-import java.util.Hashtable;
-import java.util.LinkedList;
-
-/**
- * Test front-end, fully in-memory.
- */
-public class InMemoryFrontEnd implements FrontEndAdapter {
- private Hashtable _streams = new Hashtable<>();
-
- /**
- *
- * @param streamPath The relative path to the stream.
- * @param overwrite Whether to overwrite an existing stream.
- * @param data
- * @param byteCount
- * @Throws CloudException
- */
- public void createStream(String streamPath, boolean overwrite, byte[] data, int byteCount) throws CloudException {
- if (overwrite)
- {
- _streams.put(streamPath, new StreamData(streamPath));
- }
- else
- {
- if (streamExists(streamPath))
- {
- throw new CloudException("stream exists");
- }
-
- _streams.put(streamPath, new StreamData(streamPath));
- }
-
- // if there is data passed in, we should do the same operation as in append
- if (data != null)
- {
- if (byteCount > data.length)
- {
- throw new CloudException("invalid byteCount");
- }
-
- StreamData stream = _streams.get(streamPath);
-
- //always make a copy of the original buffer since it is reused
- byte[] toAppend = new byte[byteCount];
- System.arraycopy(data, 0, toAppend, 0, byteCount);
-
- stream.Append(toAppend);
- }
- }
-
- /**
- *
- * @param streamPath The relative path to the stream.
- * @param recurse if set to true recursively delete. This is used for folder streams only.
- * @Throws CloudException
- */
- public void deleteStream(String streamPath, boolean recurse) throws CloudException {
- if (!streamExists(streamPath))
- {
- throw new CloudException("stream does not exist");
- }
- _streams.remove(streamPath);
- }
-
- /**
- *
- * @param streamPath The relative path to the stream.
- * @param data An array of bytes to be appended to the stream.
- * @param offset The offset at which to append to the stream.
- * @param byteCount
- * @Throws CloudException
- */
- public void appendToStream(String streamPath, byte[] data, long offset, int byteCount) throws CloudException {
- if (!streamExists(streamPath))
- {
- throw new CloudException("stream does not exist");
- }
-
- if (byteCount > data.length)
- {
- throw new CloudException("invalid byteCount");
- }
-
- StreamData stream = _streams.get(streamPath);
- if (stream.Length != offset)
- {
- throw new CloudException("offset != stream.length");
- }
-
- //always make a copy of the original buffer since it is reused
- byte[] toAppend = new byte[byteCount];
- System.arraycopy(data, 0, toAppend, 0, byteCount);
-
- stream.Append(toAppend);
- }
-
- /**
- *
- * @param streamPath The relative path to the stream.
- * @return True or false if the stream exists
- */
- public boolean streamExists(String streamPath)
- {
- return _streams.containsKey(streamPath);
- }
-
- /**
- *
- * @param streamPath The relative path to the stream.
- * @return
- * @Throws CloudException
- */
- public long getStreamLength(String streamPath) throws CloudException {
- if (!streamExists(streamPath))
- {
- throw new CloudException("stream does not exist");
- }
-
- return _streams.get(streamPath).Length;
- }
-
- /**
- *
- * @param targetStreamPath The relative path to the target stream.
- * @param inputStreamPaths An ordered array of paths to the input streams.
- * @Throws CloudException
- */
- public void concatenate(String targetStreamPath, String[] inputStreamPaths) throws CloudException {
- if (streamExists(targetStreamPath))
- {
- throw new CloudException("target stream exists");
- }
-
- final int bufferSize = 4 * 1024 * 1024;
- byte[] buffer = new byte[bufferSize];
-
- try
- {
- createStream(targetStreamPath, true, null, 0);
- StreamData targetStream = _streams.get(targetStreamPath);
-
- for (String inputStreamPath: inputStreamPaths)
- {
- if (!streamExists(inputStreamPath))
- {
- throw new CloudException("input stream does not exist");
- }
-
- StreamData stream = _streams.get(inputStreamPath);
- for (byte[] chunk: stream.GetDataChunks())
- {
- targetStream.Append(chunk);
- }
- }
- }
- catch (CloudException e)
- {
- if (streamExists(targetStreamPath))
- {
- deleteStream(targetStreamPath, false);
- }
- throw e;
- }
-
- for (String inputStreamPath: inputStreamPaths)
- {
- deleteStream(inputStreamPath, false);
- }
- }
-
- /**
- *
- * @param streamPath
- * @return
- * @Throws CloudException
- */
- public Iterable GetAppendBlocks(String streamPath) throws CloudException {
- if (!streamExists(streamPath))
- {
- throw new CloudException("stream does not exist");
- }
-
- StreamData sd = _streams.get(streamPath);
- return sd.GetDataChunks();
- }
-
- /**
- *
- * @param streamPath
- * @return
- * @Throws CloudException
- */
- public byte[] GetStreamContents(String streamPath) throws CloudException {
- if (!streamExists(streamPath))
- {
- throw new CloudException("stream does not exist");
- }
-
- StreamData sd = _streams.get(streamPath);
-
- if (sd.Length > Integer.MAX_VALUE)
- {
- throw new OutOfMemoryError("Stream has too much data and cannot be fit into a single array");
- }
-
- byte[] result = new byte[(int)sd.Length];
- int position = 0;
- for (byte[] chunk: sd.GetDataChunks())
- {
- System.arraycopy(chunk, 0, result, position, chunk.length);
- position += chunk.length;
- }
-
- return result;
- }
-
- /**
- * Returns the number of "streams" that have been created by this adapter.
- *
- * @return the number of streams.
- */
- public int getStreamCount()
- {
- return _streams.size();
- }
-
- /**
- * Represents stream data for unit testing purposes.
- */
- private class StreamData
- {
- private LinkedList _data;
-
- /**
- * Initializes new stream data with the given name.
- * @param name
- */
- public StreamData(String name)
- {
- _data = new LinkedList();
- this.Name = name;
- this.Length = 0;
- }
-
- public String getName() {
- return Name;
- }
-
- public void setName(String name) {
- Name = name;
- }
-
- public String Name;
- public long Length;
-
- public void Append(byte[] data)
- {
- _data.addLast(data);
- this.Length += data.length;
- }
-
- public Iterable GetDataChunks()
- {
- return _data;
- }
- }
-}
diff --git a/azure-mgmt-datalake-store-uploader/src/test/java/com/microsoft/azure/management/datalake/store/uploader/IntentionalException.java b/azure-mgmt-datalake-store-uploader/src/test/java/com/microsoft/azure/management/datalake/store/uploader/IntentionalException.java
deleted file mode 100644
index f7b52cd4fd50..000000000000
--- a/azure-mgmt-datalake-store-uploader/src/test/java/com/microsoft/azure/management/datalake/store/uploader/IntentionalException.java
+++ /dev/null
@@ -1,13 +0,0 @@
-/**
- * Copyright (c) Microsoft Corporation. All rights reserved.
- * Licensed under the MIT License. See License.txt in the project root for
- * license information.
- */
-package com.microsoft.azure.management.datalake.store.uploader;
-
-import com.microsoft.azure.CloudException;
-
-/**
- * An exception that we want our mocks to throw sometimes to test out various code paths.
- */
-public class IntentionalException extends CloudException { }
\ No newline at end of file
diff --git a/azure-mgmt-datalake-store-uploader/src/test/java/com/microsoft/azure/management/datalake/store/uploader/MsuMockFrontEnd.java b/azure-mgmt-datalake-store-uploader/src/test/java/com/microsoft/azure/management/datalake/store/uploader/MsuMockFrontEnd.java
deleted file mode 100644
index 2286692d55f3..000000000000
--- a/azure-mgmt-datalake-store-uploader/src/test/java/com/microsoft/azure/management/datalake/store/uploader/MsuMockFrontEnd.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/**
- * Copyright (c) Microsoft Corporation. All rights reserved.
- * Licensed under the MIT License. See License.txt in the project root for
- * license information.
- */
-package com.microsoft.azure.management.datalake.store.uploader;
-
-import com.microsoft.rest.RestException;
-import org.junit.Assert;
-
-import java.io.IOException;
-
-/**
- * A mocked front end for testing out the code paths of the {@link MultipleSegmentUploader}
- */
-public class MsuMockFrontEnd implements FrontEndAdapter {
-
- private FrontEndAdapter BaseAdapter;
- private boolean TestRetry;
- private int FailCount;
- private int CallCount;
-
- /**
- * Constructor with base front end.
- * @param baseAdapter The "real" front end to use for non-mocked calls
- * @param testRetry Indicates if it should mock out retry logic.
- * @param failCount Required if mocking retry logic, indicates the number of failures to allow.
- */
- public MsuMockFrontEnd(FrontEndAdapter baseAdapter, boolean testRetry, int failCount)
- {
- TestRetry = testRetry;
- BaseAdapter = baseAdapter;
- FailCount = failCount;
- CallCount = 0;
- }
-
- public void createStream(String streamPath, boolean overwrite, byte[] data, int byteCount) throws RestException, IOException {
- if (TestRetry) {
- CallCount++;
- if (CallCount <= FailCount)
- {
- throw new IntentionalException();
- }
- }
-
- BaseAdapter.createStream(streamPath, overwrite, data, byteCount);
- }
-
- public void deleteStream(String streamPath, boolean recurse) throws IOException, RestException {
- BaseAdapter.deleteStream(streamPath, recurse);
- }
-
- public void appendToStream(String streamPath, byte[] data, long offset, int byteCount) throws IOException, RestException {
- if (TestRetry) {
- CallCount++;
- if (CallCount <= FailCount)
- {
- throw new IntentionalException();
- }
- }
-
- BaseAdapter.appendToStream(streamPath, data, offset, byteCount);
- }
-
- public boolean streamExists(String streamPath) throws IOException, RestException {
- return BaseAdapter.streamExists(streamPath);
- }
-
- public long getStreamLength(String streamPath) throws IOException, RestException {
- return BaseAdapter.getStreamLength(streamPath);
- }
-
- public void concatenate(String targetStreamPath, String[] inputStreamPaths) throws IOException, RestException {
- Assert.assertTrue("concatenate should not be called when using 1 segment", false);
- BaseAdapter.concatenate(targetStreamPath, inputStreamPaths);
- }
-}
diff --git a/azure-mgmt-datalake-store-uploader/src/test/java/com/microsoft/azure/management/datalake/store/uploader/MultipleSegmentUploaderTests.java b/azure-mgmt-datalake-store-uploader/src/test/java/com/microsoft/azure/management/datalake/store/uploader/MultipleSegmentUploaderTests.java
deleted file mode 100644
index ebf652f976ce..000000000000
--- a/azure-mgmt-datalake-store-uploader/src/test/java/com/microsoft/azure/management/datalake/store/uploader/MultipleSegmentUploaderTests.java
+++ /dev/null
@@ -1,305 +0,0 @@
-/**
- * Copyright (c) Microsoft Corporation. All rights reserved.
- * Licensed under the MIT License. See License.txt in the project root for
- * license information.
- */
-package com.microsoft.azure.management.datalake.store.uploader;
-
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.text.MessageFormat;
-import java.util.Random;
-
-/**
- * Represents a class of unit tests targeting the {@link MultipleSegmentUploader}
- */
-public class MultipleSegmentUploaderTests {
- private static byte[] _smallFileContents = new byte[10 * 1024]; //10KB file
- private static String _smallFilePath;
-
- @BeforeClass
- public static void Setup() throws IOException {
- _smallFilePath = GenerateFileData(_smallFileContents);
- }
-
- private static String GenerateFileData(byte[] contents) throws IOException {
- File tempFile = File.createTempFile("adlmsu", ".data");
-
- Random rnd = new Random(0);
- rnd.nextBytes(contents);
- Assert.assertTrue("The temp file at the following path was not created: " + tempFile.getAbsolutePath(), tempFile.exists());
-
- try (FileOutputStream stream = new FileOutputStream(tempFile)) {
- stream.write(contents);
- }
-
- return tempFile.getAbsolutePath();
- }
-
- @AfterClass
- public static void Teardown()
- {
- File smallFile = new File(_smallFilePath);
- if (smallFile.exists())
- {
- smallFile.delete();
- }
- }
-
- /**
- * Tests an uneventful upload from scratch made of 1 segment.
- *
- * @throws Exception
- */
- @Test
- public void MultipleSegmentUploader_OneSegment() throws Exception {
- InMemoryFrontEnd fe = new InMemoryFrontEnd();
- UploadMetadata metadata = CreateMetadata(1);
- try
- {
- MultipleSegmentUploader msu = new MultipleSegmentUploader(metadata, 1, fe);
- msu.setUseSegmentBlockBackOffRetryStrategy(false);
- msu.upload();
- VerifyTargetStreamsAreComplete(metadata, fe);
- }
- finally
- {
- metadata.deleteFile();
- }
- }
-
- /**
- * Tests an uneventful upload from scratch made of several segments
- *
- * @throws Exception
- */
- @Test
- public void MultipleSegmentUploader_MultipleSegments() throws Exception
- {
- InMemoryFrontEnd fe = new InMemoryFrontEnd();
- UploadMetadata metadata = CreateMetadata(10);
- try
- {
- MultipleSegmentUploader msu = new MultipleSegmentUploader(metadata, 1, fe);
- msu.setUseSegmentBlockBackOffRetryStrategy(false);
- msu.upload();
- VerifyTargetStreamsAreComplete(metadata, fe);
- }
- finally
- {
- metadata.deleteFile();
- }
- }
-
- /**
- * Tests an uneventful upload from scratch made of several segments
- *
- * @throws Exception
- */
- @Test
- public void MultipleSegmentUploader_MultipleSegmentsAndMultipleThreads() throws Exception
- {
- InMemoryFrontEnd fe = new InMemoryFrontEnd();
- UploadMetadata metadata = CreateMetadata(10);
- int threadCount = metadata.getSegmentCount() * 10; //intentionally setting this higher than the # of segments
- try
- {
- MultipleSegmentUploader msu = new MultipleSegmentUploader(metadata, threadCount, fe);
- msu.setUseSegmentBlockBackOffRetryStrategy(false);
- msu.upload();
- VerifyTargetStreamsAreComplete(metadata, fe);
- }
- finally
- {
- metadata.deleteFile();
- }
- }
-
- /**
- * Tests an uneventful upload from resume made of several segments
- *
- * @throws Exception
- */
- @Test
- public void MultipleSegmentUploader_ResumedUploadWithMultipleSegments() throws Exception
- {
- //the strategy here is to upload everything, then delete a set of the segments, and verify that a resume will pick up the slack
-
- InMemoryFrontEnd fe = new InMemoryFrontEnd();
- UploadMetadata metadata = CreateMetadata(10);
-
- try
- {
- MultipleSegmentUploader msu = new MultipleSegmentUploader(metadata, 1, fe);
- msu.setUseSegmentBlockBackOffRetryStrategy(false);
- msu.upload();
- VerifyTargetStreamsAreComplete(metadata, fe);
-
- //delete about 50% of segments
- for (int i = 0; i < metadata.getSegmentCount(); i++)
- {
- UploadSegmentMetadata currentSegment = metadata.getSegments()[i];
- if (i % 2 == 0)
- {
- currentSegment.setStatus(SegmentUploadStatus.Pending);
- fe.deleteStream(currentSegment.getPath(), false);
- }
- }
-
- //re-upload everything
- msu = new MultipleSegmentUploader(metadata, 1, fe);
- msu.upload();
- VerifyTargetStreamsAreComplete(metadata, fe);
- }
- finally
- {
- metadata.deleteFile();
- }
- }
-
- /**
- * Tests an upload made of several segments, where
- * some fail a couple of times => upload can finish.
- * some fail too many times => upload will not finish
- *
- * @throws Exception
- */
- @Test
- public void MultipleSegmentUploader_SegmentInstability() throws Exception
- {
- TestRetry(0);
- TestRetry(1);
- TestRetry(2);
- TestRetry(3);
- TestRetry(4);
- TestRetry(5);
- }
-
- private void TestRetry(int segmentFailCount) throws Exception
- {
- //we only have access to the underlying FrontEnd, so we need to simulate many exceptions in order to force a segment to fail the upload (multiply by SingleSegmentUploader.MaxBufferUploadAttemptAccount)
- //this only works because we have a small file, which we know will fit in only one buffer (for a larger file, more complex operations are necessary)
- int actualfailCount = segmentFailCount * SingleSegmentUploader.MAX_BUFFER_UPLOAD_ATTEMPT_COUNT;
- boolean expectSuccess = segmentFailCount < MultipleSegmentUploader.MAX_UPLOAD_ATTEMPT_COUNT;
-
- int callCount = 0;
-
- //create a mock front end sitting on top of a working front end that simulates some erros for some time
- InMemoryFrontEnd workingFrontEnd = new InMemoryFrontEnd();
- MsuMockFrontEnd fe = new MsuMockFrontEnd(workingFrontEnd, true, actualfailCount);
-
- UploadMetadata metadata = CreateMetadata(1);
- try
- {
- MultipleSegmentUploader msu = new MultipleSegmentUploader(metadata, 1, fe);
- msu.setUseSegmentBlockBackOffRetryStrategy(false);
-
- if (expectSuccess)
- {
- //the upload method should not throw any exceptions in this case
- msu.upload();
-
- //if we are expecting success, verify that both the metadata and the target streams are complete
- VerifyTargetStreamsAreComplete(metadata, workingFrontEnd);
- }
- else
- {
- //the upload method should throw an aggregate exception in this case
- try {
- msu.upload();
- Assert.assertTrue("An aggregate upload exception was expected but no exception was thrown.", false);
- }
- catch (AggregateUploadException ex) {
- // do nothing, expected
- }
-
- //if we do not expect success, verify that at least 1 segment was marked as Failed
- boolean foundFailedSegment = false;
- for (UploadSegmentMetadata s: metadata.getSegments()) {
- if(s.getStatus() == SegmentUploadStatus.Failed) {
- foundFailedSegment = true;
- break;
- }
- }
- Assert.assertTrue("Could not find any failed segments", foundFailedSegment);
-
- //for every other segment, verify it was completed OK
- for (UploadSegmentMetadata segment: metadata.getSegments())
- {
- if( segment.getStatus() != SegmentUploadStatus.Failed) {
- VerifyTargetStreamIsComplete(segment, metadata, workingFrontEnd);
- }
- }
- }
- }
- finally
- {
- metadata.deleteFile();
- }
- }
-
- private void VerifyTargetStreamsAreComplete(UploadMetadata metadata, InMemoryFrontEnd fe) throws Exception {
- for (UploadSegmentMetadata segment: metadata.getSegments())
- {
- VerifyTargetStreamIsComplete(segment, metadata, fe);
- }
- }
-
- private void VerifyTargetStreamIsComplete(UploadSegmentMetadata segmentMetadata, UploadMetadata metadata, InMemoryFrontEnd frontEnd) throws Exception {
- Assert.assertEquals(SegmentUploadStatus.Complete, segmentMetadata.getStatus());
- Assert.assertTrue(MessageFormat.format("Segment {0} was not uploaded", segmentMetadata.getSegmentNumber()), frontEnd.streamExists(segmentMetadata.getPath()));
- Assert.assertEquals(segmentMetadata.getLength(), frontEnd.getStreamLength(segmentMetadata.getPath()));
-
- byte[] actualContents = frontEnd.GetStreamContents(segmentMetadata.getPath());
- byte[] expectedContents = GetExpectedContents(segmentMetadata, metadata);
- Assert.assertArrayEquals(MessageFormat.format("Segment {0} has unexpected contents", segmentMetadata.getSegmentNumber()), expectedContents, actualContents);
- }
-
-
- private byte[] GetExpectedContents(UploadSegmentMetadata segment, UploadMetadata metadata)
- {
- byte[] result = new byte[(int)segment.getLength()];
- System.arraycopy(_smallFileContents, (int) (segment.getSegmentNumber() * metadata.getSegmentLength()), result, 0, (int)segment.getLength());
- return result;
- }
-
- private UploadMetadata CreateMetadata(int segmentCount) throws IOException {
- File path = File.createTempFile("adlsmsumetadata", ".xml");
- UploadMetadata metadata = new UploadMetadata();
-
- metadata.setMetadataFilePath(path.getAbsolutePath());
- metadata.setInputFilePath(_smallFilePath);
- metadata.setFileLength(_smallFileContents.length);
- metadata.setSegmentCount(segmentCount);
- metadata.setSegmentLength(UploadSegmentMetadata.calculateSegmentLength(_smallFileContents.length, segmentCount));
-
- metadata.setTargetStreamPath("abc");
- metadata.setUploadId("123");
- metadata.setBinary(true);
-
- UploadSegmentMetadata[] toSet = new UploadSegmentMetadata[segmentCount];
- long offset = 0;
- for (int i = 0; i < segmentCount; i++)
- {
- long length = UploadSegmentMetadata.calculateSegmentLength(i, metadata);
- toSet[i] = new UploadSegmentMetadata();
-
- toSet[i].setSegmentNumber(i);
- toSet[i].setOffset(offset);
- toSet[i].setStatus(SegmentUploadStatus.Pending);
- toSet[i].setLength(length);
- toSet[i].setPath(MessageFormat.format("{0}.{1}.segment{2}", metadata.getTargetStreamPath(), metadata.getUploadId(), i));
-
- offset += length;
- }
-
- metadata.setSegments(toSet);
- return metadata;
- }
-}
diff --git a/azure-mgmt-datalake-store-uploader/src/test/java/com/microsoft/azure/management/datalake/store/uploader/PerformanceUploadTests.java b/azure-mgmt-datalake-store-uploader/src/test/java/com/microsoft/azure/management/datalake/store/uploader/PerformanceUploadTests.java
deleted file mode 100644
index eb1bece357c3..000000000000
--- a/azure-mgmt-datalake-store-uploader/src/test/java/com/microsoft/azure/management/datalake/store/uploader/PerformanceUploadTests.java
+++ /dev/null
@@ -1,167 +0,0 @@
-/**
- * Copyright (c) Microsoft Corporation. All rights reserved.
- * Licensed under the MIT License. See License.txt in the project root for
- * license information.
- */
-package com.microsoft.azure.management.datalake.store.uploader;
-
-import com.google.common.base.Stopwatch;
-import com.microsoft.azure.management.datalake.store.models.DataLakeStoreAccount;
-import com.microsoft.azure.management.datalake.store.implementation.DataLakeStoreFileSystemManagementClientImpl;
-import com.microsoft.azure.management.resources.implementation.ResourceGroupInner;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.RandomAccessFile;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.concurrent.TimeUnit;
-
-public class PerformanceUploadTests extends DataLakeUploaderTestBase {
-
- private static String rgName = generateName("javaadlsrg");
- private static String adlsAcct = generateName("javaadlsacct");
-
- private static final String location = "East US 2";
- private static String destFolder = generateName("performanceTest");
-
- private static final String Local10GbFileName = "C:\\data\\FixedBlockPerfData.txt"; // 10GB perf test binary file.
- private static final String localLargeFileName = "C:\\data\\MicrosoftTelemetry.tsv"; // 2.5GB perf test binary file
- private static final String localFileName = "C:\\data\\2mbfile.txt"; // 2mb perf test binary file
-
- @BeforeClass
- public static void Setup() throws Exception {
- createClients();
-
- ResourceGroupInner group = new ResourceGroupInner();
- String location = "eastus2";
- group.withLocation(location);
- resourceManagementClient.resourceGroups().createOrUpdate(rgName, group);
-
- // create storage and ADLS accounts, setting the accessKey
- DataLakeStoreAccount adlsAccount = new DataLakeStoreAccount();
- adlsAccount.withLocation(location);
- adlsAccount.withName(adlsAcct);
- dataLakeStoreAccountManagementClient.accounts().create(rgName, adlsAcct, adlsAccount);
-
- File smallFile = new File(localFileName);
- if (!smallFile.exists()) {
- smallFile.createNewFile();
- try (FileOutputStream stream = new FileOutputStream(smallFile)) {
- byte[] contents = new byte[4 * 1024 * 1024];
- Arrays.fill(contents, (byte) 'a');
- stream.write(contents);
- }
- }
-
- File largeFile = new File(localLargeFileName);
- if (!largeFile.exists()) {
- try (RandomAccessFile stream = new RandomAccessFile(largeFile, "rw")) {
- stream.setLength((long)(2.5* 1024 * 1024 * 1024) - 10); // 2.5GB minus 10 bytes
- byte[] content = new byte[10];
- Arrays.fill(content, (byte)'a');
- stream.write(content);
- }
- }
-
- File tenGBFile = new File(Local10GbFileName);
- if (!tenGBFile.exists()) {
- try (RandomAccessFile stream = new RandomAccessFile(tenGBFile, "rw")) {
- stream.setLength((long)(10* 1024 * 1024 * 1024) - 10); // 10GB minus 10 bytes
- byte[] content = new byte[10];
- Arrays.fill(content, (byte)'a');
- stream.write(content);
- }
- }
- }
-
- @AfterClass
- public static void cleanup() throws Exception {
- try {
- resourceManagementClient.resourceGroups().delete(rgName);
- }
- catch (Exception e) {
- // ignore failures during cleanup, as it is best effort
- }
- }
-
- @Test
- public void Test2mbFileUpload() throws Exception {
- String folder = "begoldsm";
- ArrayList perfMetrics = new ArrayList();
-
- // upload Rentrak data.
- boolean force = true; //Set this to true if you want to overwrite existing data
- System.out.println("Uploading 2mb data...");
- for (int i = 0; i < 10; ++i) {
- String destLocation = destFolder + "/" + folder + "2mbFile.txt";
- Stopwatch watch = Stopwatch.createStarted();
- UploadFile(dataLakeStoreFileSystemManagementClient, adlsAcct, localFileName, destLocation, force);
- watch.stop();
- long elapsedMs = watch.elapsed(TimeUnit.MILLISECONDS);
- System.out.println("File Uploaded : " + i);
- perfMetrics.add(elapsedMs);
- }
-
- for( long perf: perfMetrics){
- System.out.println(perf);
- }
- }
-
- @Test
- public void Test2_5gbFileUpload() throws Exception {
- String folder = "begoldsm";
- ArrayList perfMetrics = new ArrayList();
-
- // upload Rentrak data.
- boolean force = true; //Set this to true if you want to overwrite existing data
- System.out.println("Uploading 2.5GB data...");
- for (int i = 0; i < 5; ++i) {
- String destLocation = destFolder + "/" + folder + "2_5gbFile.txt";
- Stopwatch watch = Stopwatch.createStarted();
- UploadFile(dataLakeStoreFileSystemManagementClient, adlsAcct, localLargeFileName, destLocation, force);
- watch.stop();
- long elapsedMs = watch.elapsed(TimeUnit.MILLISECONDS);
- System.out.println("File Uploaded : " + i);
- perfMetrics.add(elapsedMs);
- }
-
- for( long perf: perfMetrics){
- System.out.println(perf);
- }
- }
-
- @Test
- public void Test10gbFileUpload() throws Exception {
- String folder = "begoldsm";
- ArrayList perfMetrics = new ArrayList();
-
- // upload Rentrak data.
- boolean force = true; //Set this to true if you want to overwrite existing data
- System.out.println("Uploading 10GB data...");
- for (int i = 0; i < 3; ++i) {
- String destLocation = destFolder + "/" + folder + "10gbFile.txt";
- Stopwatch watch = Stopwatch.createStarted();
- UploadFile(dataLakeStoreFileSystemManagementClient, adlsAcct, Local10GbFileName, destLocation, force);
- watch.stop();
- long elapsedMs = watch.elapsed(TimeUnit.MILLISECONDS);
- System.out.println("File Uploaded : " + i);
- perfMetrics.add(elapsedMs);
- }
-
- for( long perf: perfMetrics){
- System.out.println(perf);
- }
- }
-
- public static boolean UploadFile(DataLakeStoreFileSystemManagementClientImpl dataLakeStoreFileSystemClient, String dlAccountName, String srcPath, String destPath, boolean force) throws Exception {
- UploadParameters parameters = new UploadParameters(srcPath, destPath, dlAccountName, 40, force, false);
- FrontEndAdapter frontend = new DataLakeStoreFrontEndAdapterImpl(dlAccountName, dataLakeStoreFileSystemClient);
- DataLakeStoreUploader uploader = new DataLakeStoreUploader(parameters, frontend);
- uploader.execute();
- return true;
- }
-}
diff --git a/azure-mgmt-datalake-store-uploader/src/test/java/com/microsoft/azure/management/datalake/store/uploader/SingleSegmentUploaderTests.java b/azure-mgmt-datalake-store-uploader/src/test/java/com/microsoft/azure/management/datalake/store/uploader/SingleSegmentUploaderTests.java
deleted file mode 100644
index 0a78383d0d34..000000000000
--- a/azure-mgmt-datalake-store-uploader/src/test/java/com/microsoft/azure/management/datalake/store/uploader/SingleSegmentUploaderTests.java
+++ /dev/null
@@ -1,296 +0,0 @@
-/**
- * Copyright (c) Microsoft Corporation. All rights reserved.
- * Licensed under the MIT License. See License.txt in the project root for
- * license information.
- */
-package com.microsoft.azure.management.datalake.store.uploader;
-
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.File;
-import java.io.IOException;
-
-public class SingleSegmentUploaderTests {
- private static byte[] _smallFileContents = new byte[10 * 1024]; //10KB file
- private static String _smallFilePath;
-
- private static byte[] _largeFileContents = new byte[10 * 1024 * 1024]; //10MB file
- private static String _largeFilePath;
-
- private static byte[] _textFileContents = new byte[20 * 1024 * 1024]; //20MB file
- private static String _textFilePath;
-
- private static byte[] _badTextFileContents = new byte[10 * 1024 * 1024]; //10MB file
- private static String _badTextFilePath;
-
- private static final String StreamPath = "abc";
-
- @BeforeClass
- public static void Setup() throws IOException {
- _smallFilePath = TestHelpers.GenerateFileData(_smallFileContents);
- _largeFilePath = TestHelpers.GenerateFileData(_largeFileContents);
- _textFilePath = TestHelpers.GenerateTextFileData(_textFileContents, 1, SingleSegmentUploader.BUFFER_LENGTH);
- _badTextFilePath = TestHelpers.GenerateTextFileData(_badTextFileContents, SingleSegmentUploader.BUFFER_LENGTH + 1, SingleSegmentUploader.BUFFER_LENGTH + 2);
- }
-
- @AfterClass
- public static void Teardown()
- {
- File largeFile = new File(_largeFilePath);
- File smallFile = new File(_smallFilePath);
- File textFile = new File(_textFilePath);
- File badFile = new File(_badTextFilePath);
- if (largeFile.exists())
- {
- largeFile.delete();
- }
-
- if (smallFile.exists())
- {
- smallFile.delete();
- }
-
- if (textFile.exists())
- {
- textFile.delete();
- }
-
- if (badFile.exists())
- {
- badFile.delete();
- }
- }
-
- /**
- * Tests a simple upload consisting of a single block (the file is small enough to be uploaded without splitting into smaller buffers)
- *
- * @throws Exception
- */
- @Test
- public void SingleSegmentUploader_UploadSingleBlockStream() throws Exception {
- InMemoryFrontEnd fe = new InMemoryFrontEnd();
-
- UploadMetadata metadata = CreateMetadata(_smallFilePath, _smallFileContents.length);
- SingleSegmentUploader ssu = new SingleSegmentUploader(0, metadata, fe);
- ssu.setUseBackOffRetryStrategy(false);
- ssu.upload();
-
- byte[] actualContents = fe.GetStreamContents(StreamPath);
- Assert.assertArrayEquals("Unexpected uploaded stream contents.", _smallFileContents, actualContents);
- }
-
- /**
- * Tests an uploading consisting of a larger file, which will need to be uploaded in sequential buffers.
- *
- * @throws Exception
- */
- @Test
- public void SingleSegmentUploader_UploadMultiBlockStream() throws Exception {
- InMemoryFrontEnd fe = new InMemoryFrontEnd();
-
- UploadMetadata metadata = CreateMetadata(_largeFilePath, _largeFileContents.length);
-
- SingleSegmentUploader ssu = new SingleSegmentUploader(0, metadata, fe);
- ssu.setUseBackOffRetryStrategy(false);
- ssu.upload();
-
- byte[] actualContents = fe.GetStreamContents(StreamPath);
- Assert.assertArrayEquals("Unexpected uploaded stream contents.", _largeFileContents, actualContents);
- }
-
- /**
- * Tests the case when only a part of the file is to be uploaded (i.e., all other cases feed in the entire file)
- *
- * @throws Exception
- */
- @Test
- public void SingleSegmentUploader_UploadFileRange() throws Exception {
- int length = _smallFileContents.length / 3;
-
- InMemoryFrontEnd fe = new InMemoryFrontEnd();
-
- UploadMetadata metadata = CreateMetadata(_smallFilePath, length);
-
- SingleSegmentUploader ssu = new SingleSegmentUploader(0, metadata, fe);
- ssu.setUseBackOffRetryStrategy(false);
- ssu.upload();
-
- byte[] actualContents = fe.GetStreamContents(StreamPath);
- byte[] expectedContents = new byte[length];
- System.arraycopy(_smallFileContents, 0, expectedContents, 0, length);
- Assert.assertArrayEquals("Unexpected uploaded stream contents.", expectedContents, actualContents);
-
- }
-
- /**
- * Tests the case when an existing stream with the same name already exists on the server. That stream needs to be fully replaced with the new data.
- *
- * @throws Exception
- */
- @Test
- public void SingleSegmentUploader_TargetStreamExists() throws Exception {
- InMemoryFrontEnd fe = new InMemoryFrontEnd();
-
- //load up an existing stream
- fe.createStream(StreamPath, true, null, 0);
- byte[] data = "random".getBytes();
- fe.appendToStream(StreamPath, data, 0, data.length);
-
- //force a re-upload of the stream
- UploadMetadata metadata = CreateMetadata(_smallFilePath, _smallFileContents.length);
- SingleSegmentUploader ssu = new SingleSegmentUploader(0, metadata, fe);
- ssu.setUseBackOffRetryStrategy(false);
- ssu.upload();
-
- byte[] actualContents = fe.GetStreamContents(StreamPath);
- Assert.assertArrayEquals("Unexpected uploaded stream contents.", _smallFileContents, actualContents);
- }
-
- /**
- * Tests the case when the upload did "succeed", but the server reports back a different stream length than expected.
- *
- * @throws Exception
- */
- @Test
- public void SingleSegmentUploader_VerifyUploadStreamFails() throws Exception {
- //create a mock front end which doesn't do anything
- SsuMockFrontEnd fe = new SsuMockFrontEnd(new InMemoryFrontEnd(), true, false , -1);
-
- //upload some data
- UploadMetadata metadata = CreateMetadata(_smallFilePath, _smallFileContents.length);
- SingleSegmentUploader ssu = new SingleSegmentUploader(0, metadata, fe);
- ssu.setUseBackOffRetryStrategy(false);
-
- //the upload method should fail if it cannot verify that the stream was uploaded after the upload (i.e., it will get a length of 0 at the end)
- try {
- ssu.upload();
- Assert.assertTrue("the upload method should fail if it cannot verify that the stream was uploaded, but it succeeded!", false);
- }
- catch (UploadFailedException ex) {
- // do nothing, expected
- }
- }
-
- /**
- * Tests the case when the SingleSegmentUploader should upload a non-binary file (i.e., split on record boundaries).
- *
- * @throws Exception
- */
- @Test
- public void SingleSegmentUploader_UploadNonBinaryFile() throws Exception {
- InMemoryFrontEnd fe = new InMemoryFrontEnd();
-
- UploadMetadata metadata = CreateMetadata(_textFilePath, _textFileContents.length);
- metadata.setBinary(false);
-
- SingleSegmentUploader ssu = new SingleSegmentUploader(0, metadata, fe);
- ssu.setUseBackOffRetryStrategy(false);
- ssu.upload();
-
- //verify the entire file is identical to the source file
- byte[] actualContents = fe.GetStreamContents(StreamPath);
- Assert.assertArrayEquals("Unexpected uploaded stream contents.", _textFileContents, actualContents);
-
- //verify the append blocks start/end on record boundaries
- Iterable appendBlocks = fe.GetAppendBlocks(StreamPath);
- int lengthSoFar = 0;
- for (byte[] append: appendBlocks)
- {
- lengthSoFar += append.length;
- if (lengthSoFar < actualContents.length)
- {
- Assert.assertEquals('\n', (char)append[append.length - 1]);
- }
- }
- }
-
- /**
- * Tests the case when the SingleSegmentUploader tries upload a non-binary file (i.e., split on record boundaries), but at least one record is larger than the max allowed size.
- *
- * @throws Exception
- */
- @Test
- public void SingleSegmentUploader_UploadNonBinaryFileTooLargeRecord() throws Exception {
- InMemoryFrontEnd fe = new InMemoryFrontEnd();
-
- UploadMetadata metadata = CreateMetadata(_badTextFilePath, _badTextFileContents.length);
- metadata.setBinary(false);
-
- SingleSegmentUploader ssu = new SingleSegmentUploader(0, metadata, fe);
- ssu.setUseBackOffRetryStrategy(false);
-
- try {
- ssu.upload();
- Assert.assertTrue("Should fail when a record is too large to fit within a single record boundary when splitting on boundaries, but didn't!", false);
- }
- catch (UploadFailedException ex) {
- // do nothing, expected
- }
- }
-
- /**
- * Tests various scenarios where the upload will fail repeatedly; verifies that the uploader will retry a certain number of times before finally giving up
- *
- * @throws Exception
- */
- @Test
- public void SingleSegmentUploader_RetryBlock() throws Exception {
- TestRetryBlock(0);
- TestRetryBlock(1);
- TestRetryBlock(2);
- TestRetryBlock(3);
- TestRetryBlock(4);
- TestRetryBlock(5);
- }
-
- public void TestRetryBlock(int failCount) throws Exception {
- boolean expectSuccess = failCount < SingleSegmentUploader.MAX_BUFFER_UPLOAD_ATTEMPT_COUNT;
-
- int callCount = 0;
-
- InMemoryFrontEnd workingFrontEnd = new InMemoryFrontEnd();
- SsuMockFrontEnd fe = new SsuMockFrontEnd(workingFrontEnd, false, true, failCount);
-
- UploadMetadata metadata = CreateMetadata(_smallFilePath, _smallFileContents.length);
-
- SingleSegmentUploader ssu = new SingleSegmentUploader(0, metadata, fe);
- ssu.setUseBackOffRetryStrategy(false);
-
- if (expectSuccess)
- {
- ssu.upload();
- byte[] actualContents = workingFrontEnd.GetStreamContents(StreamPath);
- Assert.assertArrayEquals("Unexpected uploaded stream contents.", _smallFileContents, actualContents);
- }
- else
- {
- try {
- ssu.upload();
- Assert.assertTrue("upload should have failed due to too many retries but didn't!", false);
- }
- catch (Exception ex) {
- Assert.assertTrue("Expected an intentional exception and got: " + ex, IntentionalException.class.isInstance(ex));
- }
- }
- }
-
- private UploadMetadata CreateMetadata(String filePath, long filelength)
- {
- UploadMetadata metadata = new UploadMetadata();
- metadata.setInputFilePath(filePath);
- metadata.setFileLength(filelength);
- metadata.setTargetStreamPath(StreamPath);
- metadata.setSegmentCount(1);
- metadata.setSegmentLength(UploadSegmentMetadata.calculateSegmentLength(filelength, 1));
- metadata.setBinary(true);
-
- UploadSegmentMetadata[] toSet = new UploadSegmentMetadata[1];
- toSet[0] = new UploadSegmentMetadata(0, metadata);
- toSet[0].setPath(metadata.getTargetStreamPath());
- metadata.setSegments(toSet);
- return metadata;
- }
-}
diff --git a/azure-mgmt-datalake-store-uploader/src/test/java/com/microsoft/azure/management/datalake/store/uploader/SsuMockFrontEnd.java b/azure-mgmt-datalake-store-uploader/src/test/java/com/microsoft/azure/management/datalake/store/uploader/SsuMockFrontEnd.java
deleted file mode 100644
index 0eea59395e07..000000000000
--- a/azure-mgmt-datalake-store-uploader/src/test/java/com/microsoft/azure/management/datalake/store/uploader/SsuMockFrontEnd.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/**
- * Copyright (c) Microsoft Corporation. All rights reserved.
- * Licensed under the MIT License. See License.txt in the project root for
- * license information.
- */
-package com.microsoft.azure.management.datalake.store.uploader;
-
-import com.microsoft.rest.RestException;
-
-import java.io.IOException;
-
-/**
- * Represents a mocked front end for testing the {@link SingleSegmentUploader}
- */
-public class SsuMockFrontEnd implements FrontEndAdapter {
-
- private FrontEndAdapter BaseAdapter;
-
- private boolean DoNothing;
-
- private boolean TestRetry;
-
- private int CallCount;
-
- private int FailCount;
-
- /**
- * Constructor with base front end.
- * @param baseAdapter The base adapter to use for non-mocked methods
- * @param doNothing If true, indicates that all methods should perform no actions and return default values.
- * @param testRetry If true, indicates that method implementations should test for the retry code paths. Cannot be true if doNothing is true.
- * @param failCount This is required when testRetry is true. It indicates the number of failures to allow for retries.
- */
- public SsuMockFrontEnd(FrontEndAdapter baseAdapter, boolean doNothing, boolean testRetry, int failCount)
- {
- BaseAdapter = baseAdapter;
- DoNothing = doNothing;
- TestRetry = testRetry;
- CallCount = 0;
- FailCount = failCount;
- }
-
- public void createStream(String streamPath, boolean overwrite, byte[] data, int byteCount) throws RestException, IOException {
- if (!DoNothing && !TestRetry) {
- BaseAdapter.createStream(streamPath, overwrite, data, byteCount);
- }
- else if(TestRetry) {
- CallCount++;
- if (CallCount <= FailCount)
- {
- throw new IntentionalException();
- }
- BaseAdapter.createStream(streamPath, overwrite, data, byteCount);
- }
- }
-
- public void deleteStream(String streamPath, boolean recurse) throws RestException, IOException {
- if (!DoNothing) {
- BaseAdapter.deleteStream(streamPath, recurse);
- }
- }
-
- public void appendToStream(String streamPath, byte[] data, long offset, int byteCount) throws RestException, IOException {
- if (!DoNothing && !TestRetry) {
- BaseAdapter.appendToStream(streamPath, data, offset, byteCount);
- }
- else if(TestRetry) {
- CallCount++;
- if (CallCount <= FailCount)
- {
- throw new IntentionalException();
- }
- BaseAdapter.appendToStream(streamPath, data, offset, byteCount);
- }
- }
-
- public boolean streamExists(String streamPath) throws RestException, IOException {
- if (!DoNothing) {
- return BaseAdapter.streamExists(streamPath);
- }
-
- return true;
- }
-
- public long getStreamLength(String streamPath) throws RestException, IOException {
- if (!DoNothing) {
- return BaseAdapter.getStreamLength(streamPath);
- }
-
- return 0;
- }
-
- public void concatenate(String targetStreamPath, String[] inputStreamPaths) throws RestException, IOException {
- if (!DoNothing) {
- BaseAdapter.concatenate(targetStreamPath, inputStreamPaths);
- }
- }
-}
diff --git a/azure-mgmt-datalake-store-uploader/src/test/java/com/microsoft/azure/management/datalake/store/uploader/StringExtensionsTests.java b/azure-mgmt-datalake-store-uploader/src/test/java/com/microsoft/azure/management/datalake/store/uploader/StringExtensionsTests.java
deleted file mode 100644
index 5eb8bfe10281..000000000000
--- a/azure-mgmt-datalake-store-uploader/src/test/java/com/microsoft/azure/management/datalake/store/uploader/StringExtensionsTests.java
+++ /dev/null
@@ -1,299 +0,0 @@
-/**
- * Copyright (c) Microsoft Corporation. All rights reserved.
- * Licensed under the MIT License. See License.txt in the project root for
- * license information.
- */
-package com.microsoft.azure.management.datalake.store.uploader;
-
-import org.apache.commons.lang3.tuple.ImmutableTriple;
-import org.apache.commons.lang3.tuple.Triple;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-
-public class StringExtensionsTests {
- private static final String customDelim = ";";
-
- private static ArrayList> TestDataUTF8 = new ArrayList<>();
-
- private static ArrayList> TestDataUTF8CustomDelim = new ArrayList<>();
-
- private static ArrayList> TestDataUTF16 = new ArrayList<>();
-
- private static ArrayList> TestDataUTF16CustomDelim = new ArrayList<>();
-
- private static ArrayList> TestDataUTF32 = new ArrayList<>();
-
- private static ArrayList> TestDataUTF32CustomDelim = new ArrayList<>();
-
- @BeforeClass
- public static void setup() throws Exception {
-
- TestDataUTF8.add(new ImmutableTriple<>("", -1, -1));
- TestDataUTF8.add(new ImmutableTriple<>("a", -1, -1));
- TestDataUTF8.add(new ImmutableTriple<>("a b", -1, -1));
- TestDataUTF8.add(new ImmutableTriple<>("\r", 0, 0));
- TestDataUTF8.add(new ImmutableTriple<>("\n", 0, 0));
- TestDataUTF8.add(new ImmutableTriple<>("\r\n", 1, 1));
- TestDataUTF8.add(new ImmutableTriple<>("\n\r", 1, 1));
- TestDataUTF8.add(new ImmutableTriple<>("\r\nabcde", 1, 1));
- TestDataUTF8.add(new ImmutableTriple<>("abcde\r", 5, 5));
- TestDataUTF8.add(new ImmutableTriple<>("abcde\n", 5, 5));
- TestDataUTF8.add(new ImmutableTriple<>("abcde\r\n", 6, 6));
- TestDataUTF8.add(new ImmutableTriple<>("abcde\rabcde", 5, 5));
- TestDataUTF8.add(new ImmutableTriple<>("abcde\nabcde", 5, 5));
- TestDataUTF8.add(new ImmutableTriple<>("abcde\r\nabcde", 6, 6));
- TestDataUTF8.add(new ImmutableTriple<>("a\rb\na\r\n", 1, 6));
- TestDataUTF8.add(new ImmutableTriple<>("\rb\na\r\n", 0, 5));
-
- TestDataUTF8CustomDelim.add(new ImmutableTriple<>("", -1, -1));
- TestDataUTF8CustomDelim.add(new ImmutableTriple<>("a", -1, -1));
- TestDataUTF8CustomDelim.add(new ImmutableTriple<>("a b", -1, -1));
- TestDataUTF8CustomDelim.add(new ImmutableTriple<>(";", 0, 0));
- TestDataUTF8CustomDelim.add(new ImmutableTriple<>("a;", 1, 1));
- TestDataUTF8CustomDelim.add(new ImmutableTriple<>("b;", 1, 1));
- TestDataUTF8CustomDelim.add(new ImmutableTriple<>("a;abcde", 1, 1));
- TestDataUTF8CustomDelim.add(new ImmutableTriple<>("abcde;", 5, 5));
- TestDataUTF8CustomDelim.add(new ImmutableTriple<>("abcde\r;", 6, 6));
- TestDataUTF8CustomDelim.add(new ImmutableTriple<>("abcde;abcde", 5, 5));
- TestDataUTF8CustomDelim.add(new ImmutableTriple<>("abcde;abcde", 5, 5));
- TestDataUTF8CustomDelim.add(new ImmutableTriple<>("abcde\r;abcde", 6, 6));
- TestDataUTF8CustomDelim.add(new ImmutableTriple<>("a;b\na\r;", 1, 6));
- TestDataUTF8CustomDelim.add(new ImmutableTriple<>(";b\na\r;", 0, 5));
-
- TestDataUTF16.add(new ImmutableTriple("", -1, -1));
- TestDataUTF16.add(new ImmutableTriple("a", -1, -1));
- TestDataUTF16.add(new ImmutableTriple("a b", -1, -1));
- TestDataUTF16.add(new ImmutableTriple("\r", 1, 1));
- TestDataUTF16.add(new ImmutableTriple("\n", 1, 1));
- TestDataUTF16.add(new ImmutableTriple("\r\n", 3, 3));
- TestDataUTF16.add(new ImmutableTriple("\n\r", 3, 3));
- TestDataUTF16.add(new ImmutableTriple("\r\nabcde", 3, 3));
- TestDataUTF16.add(new ImmutableTriple("abcde\r", 11, 11));
- TestDataUTF16.add(new ImmutableTriple("abcde\n", 11, 11));
- TestDataUTF16.add(new ImmutableTriple("abcde\r\n", 13, 13));
- TestDataUTF16.add(new ImmutableTriple("abcde\rabcde", 11, 11));
- TestDataUTF16.add(new ImmutableTriple("abcde\nabcde", 11, 11));
- TestDataUTF16.add(new ImmutableTriple("abcde\r\nabcde", 13, 13));
- TestDataUTF16.add(new ImmutableTriple("a\rb\na\r\n", 3, 13));
- TestDataUTF16.add(new ImmutableTriple("\rb\na\r\n", 1, 11));
-
- TestDataUTF16CustomDelim.add(new ImmutableTriple("", -1, -1));
- TestDataUTF16CustomDelim.add(new ImmutableTriple("a", -1, -1));
- TestDataUTF16CustomDelim.add(new ImmutableTriple("a b", -1, -1));
- TestDataUTF16CustomDelim.add(new ImmutableTriple(";", 1, 1));
- TestDataUTF16CustomDelim.add(new ImmutableTriple("a;", 3, 3));
- TestDataUTF16CustomDelim.add(new ImmutableTriple("b;", 3, 3));
- TestDataUTF16CustomDelim.add(new ImmutableTriple("a;abcde", 3, 3));
- TestDataUTF16CustomDelim.add(new ImmutableTriple("abcde;", 11, 11));
- TestDataUTF16CustomDelim.add(new ImmutableTriple("abcde\r;", 13, 13));
- TestDataUTF16CustomDelim.add(new ImmutableTriple("abcde;abcde", 11, 11));
- TestDataUTF16CustomDelim.add(new ImmutableTriple("abcde;abcde", 11, 11));
- TestDataUTF16CustomDelim.add(new ImmutableTriple("abcde\r;abcde", 13, 13));
- TestDataUTF16CustomDelim.add(new ImmutableTriple("a;b\na\r;", 3, 13));
- TestDataUTF16CustomDelim.add(new ImmutableTriple(";b\na\r;", 1, 11));
-
- TestDataUTF32.add(new ImmutableTriple("", -1, -1));
- TestDataUTF32.add(new ImmutableTriple("a", -1, -1));
- TestDataUTF32.add(new ImmutableTriple("a b", -1, -1));
- TestDataUTF32.add(new ImmutableTriple("\r", 3, 3));
- TestDataUTF32.add(new ImmutableTriple("\n", 3, 3));
- TestDataUTF32.add(new ImmutableTriple("\r\n", 7, 7));
- TestDataUTF32.add(new ImmutableTriple("\n\r", 7, 7));
- TestDataUTF32.add(new ImmutableTriple("\r\nabcde", 7, 7));
- TestDataUTF32.add(new ImmutableTriple("abcde\r", 23, 23));
- TestDataUTF32.add(new ImmutableTriple("abcde\n", 23, 23));
- TestDataUTF32.add(new ImmutableTriple("abcde\r\n", 27, 27));
- TestDataUTF32.add(new ImmutableTriple("abcde\rabcde", 23, 23));
- TestDataUTF32.add(new ImmutableTriple("abcde\nabcde", 23, 23));
- TestDataUTF32.add(new ImmutableTriple("abcde\r\nabcde", 27, 27));
- TestDataUTF32.add(new ImmutableTriple("a\rb\na\r\n", 7, 27));
- TestDataUTF32.add(new ImmutableTriple("\rb\na\r\n", 3, 23));
-
- TestDataUTF32CustomDelim.add(new ImmutableTriple("", -1, -1));
- TestDataUTF32CustomDelim.add(new ImmutableTriple("a", -1, -1));
- TestDataUTF32CustomDelim.add(new ImmutableTriple("a b", -1, -1));
- TestDataUTF32CustomDelim.add(new ImmutableTriple(";", 3, 3));
- TestDataUTF32CustomDelim.add(new ImmutableTriple("a;", 7, 7));
- TestDataUTF32CustomDelim.add(new ImmutableTriple("b;", 7, 7));
- TestDataUTF32CustomDelim.add(new ImmutableTriple("a;abcde", 7, 7));
- TestDataUTF32CustomDelim.add(new ImmutableTriple("abcde;", 23, 23));
- TestDataUTF32CustomDelim.add(new ImmutableTriple("abcde\r;", 27, 27));
- TestDataUTF32CustomDelim.add(new ImmutableTriple("abcde;abcde", 23, 23));
- TestDataUTF32CustomDelim.add(new ImmutableTriple("abcde;abcde", 23, 23));
- TestDataUTF32CustomDelim.add(new ImmutableTriple("abcde\r;abcde", 27, 27));
- TestDataUTF32CustomDelim.add(new ImmutableTriple("a;b\na\r;", 7, 27));
- TestDataUTF32CustomDelim.add(new ImmutableTriple(";b\na\r;", 3, 23));
- }
-
- @Test
- public void StringExtensions_FindNewLine_UTF8()
- {
- for (Triple t: TestDataUTF8)
- {
- byte[] exactBuffer = t.getLeft().getBytes(StandardCharsets.UTF_8);
- byte[] largerBuffer = new byte[exactBuffer.length + 100];
- System.arraycopy(exactBuffer, 0, largerBuffer, 0, exactBuffer.length);
-
- int forwardInExactBuffer = StringExtensions.findNewline(exactBuffer, 0, exactBuffer.length, false, StandardCharsets.UTF_8, null);
- Assert.assertEquals(t.getMiddle().intValue(), forwardInExactBuffer);
-
- int forwardInLargeBuffer = StringExtensions.findNewline(largerBuffer, 0, exactBuffer.length, false, StandardCharsets.UTF_8, null);
- Assert.assertEquals(t.getMiddle().intValue(), forwardInLargeBuffer);
-
- int reverseInExactBuffer = StringExtensions.findNewline(exactBuffer, Math.max(0, exactBuffer.length - 1), exactBuffer.length, true, StandardCharsets.UTF_8, null);
- Assert.assertEquals(t.getRight().intValue(), reverseInExactBuffer);
-
- int reverseInLargeBuffer = StringExtensions.findNewline(largerBuffer, Math.max(0, exactBuffer.length - 1), exactBuffer.length, true, StandardCharsets.UTF_8, null);
- Assert.assertEquals(t.getRight().intValue(), reverseInLargeBuffer);
- }
-
- for (Triple t: TestDataUTF8CustomDelim)
- {
- byte[] exactBuffer = t.getLeft().getBytes(StandardCharsets.UTF_8);
- byte[] largerBuffer = new byte[exactBuffer.length + 100];
- System.arraycopy(exactBuffer, 0, largerBuffer, 0, exactBuffer.length);
-
- int forwardInExactBuffer = StringExtensions.findNewline(exactBuffer, 0, exactBuffer.length, false, StandardCharsets.UTF_8, customDelim);
- Assert.assertEquals(t.getMiddle().intValue(), forwardInExactBuffer);
-
- int forwardInLargeBuffer = StringExtensions.findNewline(largerBuffer, 0, exactBuffer.length, false, StandardCharsets.UTF_8, customDelim);
- Assert.assertEquals(t.getMiddle().intValue(), forwardInLargeBuffer);
-
- int reverseInExactBuffer = StringExtensions.findNewline(exactBuffer, Math.max(0, exactBuffer.length - 1), exactBuffer.length, true, StandardCharsets.UTF_8, customDelim);
- Assert.assertEquals(t.getRight().intValue(), reverseInExactBuffer);
-
- int reverseInLargeBuffer = StringExtensions.findNewline(largerBuffer, Math.max(0, exactBuffer.length - 1), exactBuffer.length, true, StandardCharsets.UTF_8, customDelim);
- Assert.assertEquals(t.getRight().intValue(), reverseInLargeBuffer);
- }
- }
-
- @Test
- public void StringExtensions_FindNewLine_UTF16()
- {
- for (Triple t: TestDataUTF16)
- {
- byte[] exactBuffer = t.getLeft().getBytes(StandardCharsets.UTF_16LE);
- byte[] largerBuffer = new byte[exactBuffer.length + 100];
- System.arraycopy(exactBuffer, 0, largerBuffer, 0, exactBuffer.length);
-
- int forwardInExactBuffer = StringExtensions.findNewline(exactBuffer, 0, exactBuffer.length, false, StandardCharsets.UTF_16LE, null);
- Assert.assertEquals(t.getMiddle().intValue(), forwardInExactBuffer);
-
- int forwardInLargeBuffer = StringExtensions.findNewline(largerBuffer, 0, exactBuffer.length, false, StandardCharsets.UTF_16LE, null);
- Assert.assertEquals(t.getMiddle().intValue(), forwardInLargeBuffer);
-
- int reverseInExactBuffer = StringExtensions.findNewline(exactBuffer, Math.max(0, exactBuffer.length - 1), exactBuffer.length, true, StandardCharsets.UTF_16LE, null);
- Assert.assertEquals(t.getRight().intValue(), reverseInExactBuffer);
-
- int reverseInLargeBuffer = StringExtensions.findNewline(largerBuffer, Math.max(0, exactBuffer.length - 1), exactBuffer.length, true, StandardCharsets.UTF_16LE, null);
- Assert.assertEquals(t.getRight().intValue(), reverseInLargeBuffer);
- }
-
- for (Triple t: TestDataUTF16CustomDelim)
- {
- byte[] exactBuffer = t.getLeft().getBytes(StandardCharsets.UTF_16LE);
- byte[] largerBuffer = new byte[exactBuffer.length + 100];
- System.arraycopy(exactBuffer, 0, largerBuffer, 0, exactBuffer.length);
-
- int forwardInExactBuffer = StringExtensions.findNewline(exactBuffer, 0, exactBuffer.length, false, StandardCharsets.UTF_16LE, customDelim);
- Assert.assertEquals(t.getMiddle().intValue(), forwardInExactBuffer);
-
- int forwardInLargeBuffer = StringExtensions.findNewline(largerBuffer, 0, exactBuffer.length, false, StandardCharsets.UTF_16LE, customDelim);
- Assert.assertEquals(t.getMiddle().intValue(), forwardInLargeBuffer);
-
- int reverseInExactBuffer = StringExtensions.findNewline(exactBuffer, Math.max(0, exactBuffer.length - 1), exactBuffer.length, true, StandardCharsets.UTF_16LE, customDelim);
- Assert.assertEquals(t.getRight().intValue(), reverseInExactBuffer);
-
- int reverseInLargeBuffer = StringExtensions.findNewline(largerBuffer, Math.max(0, exactBuffer.length - 1), exactBuffer.length, true, StandardCharsets.UTF_16LE, customDelim);
- Assert.assertEquals(t.getRight().intValue(), reverseInLargeBuffer);
- }
- }
-
- @Test
- public void StringExtensions_FindNewLine_UTF16BigEndian()
- {
- for (Triple t: TestDataUTF16)
- {
- byte[] exactBuffer = t.getLeft().getBytes(StandardCharsets.UTF_16BE);
- byte[] largerBuffer = new byte[exactBuffer.length + 100];
- System.arraycopy(exactBuffer, 0, largerBuffer, 0, exactBuffer.length);
-
- int forwardInExactBuffer = StringExtensions.findNewline(exactBuffer, 0, exactBuffer.length, false, StandardCharsets.UTF_16BE, null);
- Assert.assertEquals(t.getMiddle().intValue(), forwardInExactBuffer);
-
- int forwardInLargeBuffer = StringExtensions.findNewline(largerBuffer, 0, exactBuffer.length, false, StandardCharsets.UTF_16BE, null);
- Assert.assertEquals(t.getMiddle().intValue(), forwardInLargeBuffer);
-
- int reverseInExactBuffer = StringExtensions.findNewline(exactBuffer, Math.max(0, exactBuffer.length - 1), exactBuffer.length, true, StandardCharsets.UTF_16BE, null);
- Assert.assertEquals(t.getRight().intValue(), reverseInExactBuffer);
-
- int reverseInLargeBuffer = StringExtensions.findNewline(largerBuffer, Math.max(0, exactBuffer.length - 1), exactBuffer.length, true, StandardCharsets.UTF_16BE, null);
- Assert.assertEquals(t.getRight().intValue(), reverseInLargeBuffer);
- }
-
- for (Triple t: TestDataUTF16CustomDelim)
- {
- byte[] exactBuffer = t.getLeft().getBytes(StandardCharsets.UTF_16BE);
- byte[] largerBuffer = new byte[exactBuffer.length + 100];
- System.arraycopy(exactBuffer, 0, largerBuffer, 0, exactBuffer.length);
-
- int forwardInExactBuffer = StringExtensions.findNewline(exactBuffer, 0, exactBuffer.length, false, StandardCharsets.UTF_16BE, customDelim);
- Assert.assertEquals(t.getMiddle().intValue(), forwardInExactBuffer);
-
- int forwardInLargeBuffer = StringExtensions.findNewline(largerBuffer, 0, exactBuffer.length, false, StandardCharsets.UTF_16BE, customDelim);
- Assert.assertEquals(t.getMiddle().intValue(), forwardInLargeBuffer);
-
- int reverseInExactBuffer = StringExtensions.findNewline(exactBuffer, Math.max(0, exactBuffer.length - 1), exactBuffer.length, true, StandardCharsets.UTF_16BE, customDelim);
- Assert.assertEquals(t.getRight().intValue(), reverseInExactBuffer);
-
- int reverseInLargeBuffer = StringExtensions.findNewline(largerBuffer, Math.max(0, exactBuffer.length - 1), exactBuffer.length, true, StandardCharsets.UTF_16BE, customDelim);
- Assert.assertEquals(t.getRight().intValue(), reverseInLargeBuffer);
- }
- }
-
- @Test
- public void StringExtensions_FindNewLine_ASCII()
- {
- for (Triple t: TestDataUTF8)
- {
- byte[] exactBuffer = t.getLeft().getBytes(StandardCharsets.US_ASCII);
- byte[] largerBuffer = new byte[exactBuffer.length + 100];
- System.arraycopy(exactBuffer, 0, largerBuffer, 0, exactBuffer.length);
-
- int forwardInExactBuffer = StringExtensions.findNewline(exactBuffer, 0, exactBuffer.length, false, StandardCharsets.US_ASCII, null);
- Assert.assertEquals(t.getMiddle().intValue(), forwardInExactBuffer);
-
- int forwardInLargeBuffer = StringExtensions.findNewline(largerBuffer, 0, exactBuffer.length, false, StandardCharsets.US_ASCII, null);
- Assert.assertEquals(t.getMiddle().intValue(), forwardInLargeBuffer);
-
- int reverseInExactBuffer = StringExtensions.findNewline(exactBuffer, Math.max(0, exactBuffer.length - 1), exactBuffer.length, true, StandardCharsets.US_ASCII, null);
- Assert.assertEquals(t.getRight().intValue(), reverseInExactBuffer);
-
- int reverseInLargeBuffer = StringExtensions.findNewline(largerBuffer, Math.max(0, exactBuffer.length - 1), exactBuffer.length, true, StandardCharsets.US_ASCII, null);
- Assert.assertEquals(t.getRight().intValue(), reverseInLargeBuffer);
- }
-
- for (Triple t: TestDataUTF8CustomDelim)
- {
- byte[] exactBuffer = t.getLeft().getBytes(StandardCharsets.US_ASCII);
- byte[] largerBuffer = new byte[exactBuffer.length + 100];
- System.arraycopy(exactBuffer, 0, largerBuffer, 0, exactBuffer.length);
-
- int forwardInExactBuffer = StringExtensions.findNewline(exactBuffer, 0, exactBuffer.length, false, StandardCharsets.US_ASCII, customDelim);
- Assert.assertEquals(t.getMiddle().intValue(), forwardInExactBuffer);
-
- int forwardInLargeBuffer = StringExtensions.findNewline(largerBuffer, 0, exactBuffer.length, false, StandardCharsets.US_ASCII, customDelim);
- Assert.assertEquals(t.getMiddle().intValue(), forwardInLargeBuffer);
-
- int reverseInExactBuffer = StringExtensions.findNewline(exactBuffer, Math.max(0, exactBuffer.length - 1), exactBuffer.length, true, StandardCharsets.US_ASCII, customDelim);
- Assert.assertEquals(t.getRight().intValue(), reverseInExactBuffer);
-
- int reverseInLargeBuffer = StringExtensions.findNewline(largerBuffer, Math.max(0, exactBuffer.length - 1), exactBuffer.length, true, StandardCharsets.US_ASCII, customDelim);
- Assert.assertEquals(t.getRight().intValue(), reverseInLargeBuffer);
- }
- }
-}
diff --git a/azure-mgmt-datalake-store-uploader/src/test/java/com/microsoft/azure/management/datalake/store/uploader/TestHelpers.java b/azure-mgmt-datalake-store-uploader/src/test/java/com/microsoft/azure/management/datalake/store/uploader/TestHelpers.java
deleted file mode 100644
index c7a8b0bab25f..000000000000
--- a/azure-mgmt-datalake-store-uploader/src/test/java/com/microsoft/azure/management/datalake/store/uploader/TestHelpers.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/**
- * Copyright (c) Microsoft Corporation. All rights reserved.
- * Licensed under the MIT License. See License.txt in the project root for
- * license information.
- */
-package com.microsoft.azure.management.datalake.store.uploader;
-
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.util.Random;
-
-public class TestHelpers {
- /**
- * Generates some random data and writes it out to a temp file and to an in-memory array
- *
- * @param contents The array to write random data to (the length of this array will be the size of the file).
- * @return The path of the file that will be created.
- * @throws IOException
- */
- static String GenerateFileData(byte[] contents) throws IOException {
- File filePath = File.createTempFile("adlUploader", "test.data");
-
- Random rnd = new Random(0);
- rnd.nextBytes(contents);
- if (filePath.exists())
- {
- filePath.delete();
- }
-
- FileOutputStream writer = new FileOutputStream(filePath);
- writer.write(contents);
- writer.flush();
- writer.close();
- return filePath.toString();
- }
-
- /**
- * Generates some random data and writes it out to a temp file and to an in-memory array
- *
- * @param contents The array to write random data to (the length of this array will be the size of the file).
- * @param minRecordLength The minimum amount of data to write (inclusive)
- * @param maxRecordLength The maximum amount of data to write (exclusive)
- * @return The path of the file that will be created.
- * @throws IOException
- */
- static String GenerateTextFileData(byte[] contents, int minRecordLength, int maxRecordLength) throws IOException {
- File filePath = File.createTempFile("adlUploader", "test.data");
- int offset = 0;
- while (offset < contents.length)
- {
- int recordLength = minRecordLength + (int)(Math.random()*((maxRecordLength - minRecordLength) + 1));
- recordLength = Math.min(recordLength, contents.length - offset - 2);
-
- int recordEndPos = offset + recordLength;
- while (offset < recordEndPos)
- {
- contents[offset] = (byte)((int)'a' + (int)(Math.random()*(((int)'z' - (int)'a') + 1)));
- offset++;
- }
- contents[offset++] = (byte)'\r';
- contents[offset++] = (byte)'\n';
- }
- if (filePath.exists())
- {
- filePath.delete();
- }
-
- FileOutputStream writer = new FileOutputStream(filePath);
- writer.write(contents);
- writer.flush();
- writer.close();
- return filePath.toString();
- }
-}
diff --git a/azure-mgmt-datalake-store-uploader/src/test/java/com/microsoft/azure/management/datalake/store/uploader/UploadMetadataGeneratorTests.java b/azure-mgmt-datalake-store-uploader/src/test/java/com/microsoft/azure/management/datalake/store/uploader/UploadMetadataGeneratorTests.java
deleted file mode 100644
index e06d56a22a59..000000000000
--- a/azure-mgmt-datalake-store-uploader/src/test/java/com/microsoft/azure/management/datalake/store/uploader/UploadMetadataGeneratorTests.java
+++ /dev/null
@@ -1,183 +0,0 @@
-/**
- * Copyright (c) Microsoft Corporation. All rights reserved.
- * Licensed under the MIT License. See License.txt in the project root for
- * license information.
- */
-package com.microsoft.azure.management.datalake.store.uploader;
-
-import com.google.common.io.CountingOutputStream;
-import org.apache.commons.io.FileUtils;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.RandomAccessFile;
-import java.text.MessageFormat;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Random;
-
-/**
- * Unit tests that target the {@link UploadMetadataGenerator} class
- */
-public class UploadMetadataGeneratorTests {
- private static final int MaxAppendLength = 4 * 1024 * 1024;
- private static final byte[] NewLine = "\r\n".getBytes();
- private static final List FileLengthsMB = Arrays.asList(2, 4, 10, 14.123456, 20.123456, 23.456789, 30.987654, 37.897643, 50.546213, 53.456789, 123.456789 );
-
- @Test
- public void UploadMetadataGenerator_AlignSegmentsToRecordBoundaries() throws IOException, UploadFailedException, InvalidMetadataException {
- //We keep creating a file, by appending a number of bytes to it (taken from FileLengthsInMB).
- //At each iteration, we append a new blob of data, and then run the whole test on the entire file
- Random rnd = new Random(0);
- File folderPath = new File(MessageFormat.format("{0}\\uploadtest", new File(".").getAbsoluteFile()));
- File filePath = new File(folderPath, "verifymetadata.txt");
- try
- {
- if (!folderPath.exists())
- {
- folderPath.mkdirs();
- }
-
- if (filePath.exists())
- {
- filePath.delete();
- }
-
- for (Number lengthMB: FileLengthsMB)
- {
- int appendLength = (int)(lengthMB.doubleValue()*1024*1024);
- AppendToFile(filePath.getAbsolutePath(), appendLength, rnd, 0, MaxAppendLength);
- String metadataFilePath = filePath + ".metadata.txt";
-
- UploadParameters up = new UploadParameters(filePath.getAbsolutePath(), filePath.getAbsolutePath(), null, 1, false, false, false, 4*1024*1024, null);
- UploadMetadataGenerator mg = new UploadMetadataGenerator(up, MaxAppendLength);
- UploadMetadata metadata = mg.createNewMetadata(metadataFilePath);
-
- VerifySegmentsAreOnRecordBoundaries(metadata, filePath.getAbsolutePath());
- }
- }
- finally
- {
- if (folderPath.exists())
- {
- FileUtils.deleteQuietly(folderPath);
- }
- }
- }
-
- @Test
- public void UploadMetadataGenerator_AlignSegmentsToRecordBoundariesTooLargeRecord() throws IOException {
- //We keep creating a file, by appending a number of bytes to it (taken from FileLengthsInMB).
- //At each iteration, we append a new blob of data, and then run the whole test on the entire file
- Random rnd = new Random(0);
- File folderPath = new File(MessageFormat.format("{0}\\uploadtest", new File(".").getAbsolutePath()));
- File filePath = new File(folderPath, "verifymetadata.txt");
- try
- {
- if (!folderPath.exists())
- {
- folderPath.mkdirs();
- }
-
- if (filePath.exists())
- {
- filePath.delete();
- }
- for (Number lengthMB: FileLengthsMB)
- {
- if(lengthMB.intValue() > MaxAppendLength) {
- int length = lengthMB.intValue() * 1024 * 1024;
- AppendToFile(filePath.getAbsolutePath(), length, rnd, MaxAppendLength + 1, MaxAppendLength + 10);
- String metadataFilePath = filePath + ".metadata.txt";
-
- UploadParameters up = new UploadParameters(filePath.getAbsolutePath(), filePath.getAbsolutePath(), null, 1, false, false, false, 4 * 1024 * 1024, null);
- UploadMetadataGenerator mg = new UploadMetadataGenerator(up, MaxAppendLength);
-
- try {
- mg.createNewMetadata(metadataFilePath);
- Assert.assertTrue("Method createNewMetadata should fail due to record boundaries being being too large for the record, but didn't", false);
- }
- catch(Exception e) {
- // do nothing, expected
- }
- }
- }
- }
- finally
- {
- if (folderPath.exists())
- {
- FileUtils.deleteQuietly(folderPath);
- }
- }
- }
-
- private void VerifySegmentsAreOnRecordBoundaries(UploadMetadata metadata, String filePath) throws IOException {
- try(RandomAccessFile stream = new RandomAccessFile(filePath, "r"))
- {
- for (UploadSegmentMetadata segment: metadata.getSegments())
- {
- if (segment.getSegmentNumber() > 0)
- {
- //verify that each segment starts with a non-newline and that the 2 previous characters before that offset are newline characters
-
- //2 characters behind: newline
- // always seek from the file origin
- stream.seek(0);
- stream.seek(segment.getOffset() - 2);
- char c1 = (char)stream.read();
- Assert.assertTrue(MessageFormat.format("Expecting a newline at offset {0}", segment.getOffset() - 2), IsNewline(c1));
-
- //1 character behind: newline
- char c2 = (char)stream.read();
- Assert.assertTrue(MessageFormat.format("Expecting a newline at offset {0}", segment.getOffset() - 2), IsNewline(c2));
-
- //by test design, we never have two consecutive newlines that are the same; we'd always have \r\n, but never \r\r or \r\n
- char c3 = (char)stream.read();
- Assert.assertNotEquals(c2, c3);
- }
- }
- }
- }
-
- private boolean IsNewline(char c)
- {
- return c == '\r' || c == '\n';
- }
-
- private String AppendToFile(String filePath, int length, Random random, int minRecordLength, int maxRecordLength) throws IOException {
- try (CountingOutputStream stream = new CountingOutputStream(new FileOutputStream(filePath)))
- {
- int newLength = (int) (new File(filePath).length() + length);
- while (true)
- {
- int recordLength = minRecordLength + random.nextInt(maxRecordLength - minRecordLength);
- if (stream.getCount() + recordLength + NewLine.length > newLength)
- {
- recordLength = newLength - NewLine.length - (int)stream.getCount();
- if (recordLength < 0)
- {
- stream.write(NewLine, 0, NewLine.length);
- break;
- }
- }
- WriteRecord(stream, recordLength);
- stream.write(NewLine, 0, NewLine.length);
- }
- }
-
- return filePath;
- }
-
- private void WriteRecord(CountingOutputStream stream, int count) throws IOException {
- byte[] record = new byte[count];
- for (int i = 0; i < count; i++)
- {
- record[i] = (byte)('a' + i % 25);
- }
- stream.write(record, 0, record.length);
- }
-}
diff --git a/azure-mgmt-datalake-store-uploader/src/test/java/com/microsoft/azure/management/datalake/store/uploader/UploadSegmentMetadataTests.java b/azure-mgmt-datalake-store-uploader/src/test/java/com/microsoft/azure/management/datalake/store/uploader/UploadSegmentMetadataTests.java
deleted file mode 100644
index c17f05479f56..000000000000
--- a/azure-mgmt-datalake-store-uploader/src/test/java/com/microsoft/azure/management/datalake/store/uploader/UploadSegmentMetadataTests.java
+++ /dev/null
@@ -1,176 +0,0 @@
-/**
- * Copyright (c) Microsoft Corporation. All rights reserved.
- * Licensed under the MIT License. See License.txt in the project root for
- * license information.
- */
-package com.microsoft.azure.management.datalake.store.uploader;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-/**
- * Unit tests that target the {@link UploadSegmentMetadata} class.
- */
-public class UploadSegmentMetadataTests {
- /**
- * Tests that segment count calculation works (it's hard to verify correctness without having access to the data that the base class has,
- * so we'll just check the boundary conditions, that it's monotonically increasing and that it doesn't throw exceptions for various inputs.
- */
- @Test
- public void UploadMetadata_CalculateSegmentCount()
- {
- try {
- UploadSegmentMetadata.calculateSegmentCount(-1);
- Assert.assertTrue("calculateSegmentCount should have failed for invalid count but it succeeded!", false);
- }
- catch (IllegalArgumentException ex) {
- // do nothing, this is expected
- }
-
-
- Assert.assertEquals(0, UploadSegmentMetadata.calculateSegmentCount(0));
-
- long maxLength = 100 * (long)Math.pow(2, 40);//100 TB
- long increment = 10 * (long)Math.pow(2, 30); //10GB
- int lastValue = 0;
- for (long length = (long)Math.pow(2, 20); length < maxLength; length += increment)
- {
- int value = UploadSegmentMetadata.calculateSegmentCount(length);
- Assert.assertTrue("Function is not monotonically increasing", lastValue <= value);
- lastValue = value;
- }
- }
-
- /**
- * Tests the correct calculation for a typical segment length.
- */
- @Test
- public void UploadSegmentMetadata_CalculateTypicalSegmentLength()
- {
- try {
- UploadSegmentMetadata.calculateSegmentLength(1000, -1);
- Assert.assertTrue("calculateSegmentLength should have failed for invalid length but it succeeded!", false);
- }
- catch (IllegalArgumentException ex) {
- // do nothing, expected
- }
-
- int maxSegmentCount = 16536;
- long fileLength = (long)Math.pow(2, 30); // see comment below about actually making this larger than Int32.MaxValue
- long segmentLength;
-
- for (int segmentCount = 1; segmentCount < maxSegmentCount; segmentCount++)
- {
- segmentLength = UploadSegmentMetadata.calculateSegmentLength(fileLength, segmentCount);
-
- //the next two asserts verify that the value calculated will split the input file into a balanced set of segments;
- //all the segments should have the same length, except the last one which may have less than that (but never more).
- //a quick heuristic to verify this is: (SegmentLength-1)*SegmentCount < FileLength <= SegmentLength*SegmentCount
- Assert.assertTrue("SegmentLength * SegmentCount must be at least the length of the input file", segmentLength * segmentCount >= fileLength);
- Assert.assertTrue("(SegmentLength - 1) * SegmentCount must be smaller than the length of the input file", (segmentLength - 1) * segmentCount < fileLength);
- }
-
- // test segmentCount == fileLength;
- segmentLength = UploadSegmentMetadata.calculateSegmentLength(fileLength, (int)fileLength); //for this to work, FileLength must be less than In32.MaxValue
- Assert.assertEquals(1, segmentLength);
-
- // test that if segment count = 0 then the return value is 0.
- Assert.assertEquals(
- 0,
- UploadSegmentMetadata.calculateSegmentLength(fileLength, 0));
- }
-
- /**
- * Tests the correct calculation for a particular segment length (ending vs non-ending).
- */
- @Test
- public void UploadSegmentMetadata_CalculateParticularSegmentLength()
- {
-
- UploadMetadata lengthOf10 = new UploadMetadata();
- lengthOf10.setFileLength(10);
- lengthOf10.setSegmentCount(5);
- lengthOf10.setSegmentLength(2);
-
- UploadMetadata lengthOfNegative10 = new UploadMetadata();
- lengthOfNegative10.setFileLength(-10);
- lengthOfNegative10.setSegmentCount(5);
- lengthOfNegative10.setSegmentLength(2);
-
- UploadMetadata lengthOf100 = new UploadMetadata();
- lengthOf100.setFileLength(100);
- lengthOf100.setSegmentCount(2);
- lengthOf100.setSegmentLength(2);
-
- UploadMetadata lengthOf100SegmentCount5 = new UploadMetadata();
- lengthOf100SegmentCount5.setFileLength(100);
- lengthOf100SegmentCount5.setSegmentCount(5);
- lengthOf100SegmentCount5.setSegmentLength(26);
- //verify bad inputs
- try {
- UploadSegmentMetadata.calculateSegmentLength(-1, lengthOf10);
- Assert.assertTrue("calculateSegmentLength should have failed for invalid segment number but it succeeded!", false);
- }
- catch (IndexOutOfBoundsException ex) {
- // do nothing, expected
- }
-
- try {
- UploadSegmentMetadata.calculateSegmentLength(100, lengthOf10);
- Assert.assertTrue("calculateSegmentLength should have failed for invalid segment number but it succeeded!", false);
- }
- catch (IndexOutOfBoundsException ex) {
- // do nothing, expected
- }
-
- try {
- UploadSegmentMetadata.calculateSegmentLength(1, lengthOfNegative10);
- Assert.assertTrue("calculateSegmentLength should have failed for invalid segment number but it succeeded!", false);
- }
- catch (IllegalArgumentException ex) {
- // do nothing, expected
- }
-
- try {
- UploadSegmentMetadata.calculateSegmentLength(1, lengthOf100);
- Assert.assertTrue("calculateSegmentLength should have failed for invalid segment number but it succeeded!", false);
- }
- catch (IllegalArgumentException ex) {
- // do nothing, expected
- }
-
- try {
- UploadSegmentMetadata.calculateSegmentLength(1, lengthOf100SegmentCount5);
- Assert.assertTrue("calculateSegmentLength should have failed for invalid segment number but it succeeded!", false);
- }
- catch (IllegalArgumentException ex) {
- // do nothing, expected
- }
-
- //test various scenarios with a fixed file length, and varying the segment count from 1 to the FileLength
-
- int FileLength = 16 * (int)Math.pow(2, 20);//16MB
-
- for (int segmentCount = 1; segmentCount <= FileLength; segmentCount += 1024)
- {
- long typicalSegmentLength = UploadSegmentMetadata.calculateSegmentLength(FileLength, segmentCount);
-
- UploadMetadata uploadMetadata = new UploadMetadata();
- uploadMetadata.setFileLength(FileLength);
- uploadMetadata.setSegmentCount(segmentCount);
- uploadMetadata.setSegmentLength(typicalSegmentLength);
-
- long firstSegmentLength = UploadSegmentMetadata.calculateSegmentLength(0, uploadMetadata);
- long lastSegmentLength = UploadSegmentMetadata.calculateSegmentLength(segmentCount - 1, uploadMetadata);
-
- Assert.assertEquals(typicalSegmentLength, firstSegmentLength);
- if (segmentCount == 1)
- {
- Assert.assertEquals(firstSegmentLength, lastSegmentLength);
- }
-
- long reconstructedFileLength = typicalSegmentLength * (segmentCount - 1) + lastSegmentLength;
- Assert.assertEquals(FileLength, reconstructedFileLength);
- }
- }
-}
diff --git a/azure-mgmt-datalake-store-uploader/src/test/java/com/microsoft/azure/management/datalake/store/uploader/UploaderFrontEndMock.java b/azure-mgmt-datalake-store-uploader/src/test/java/com/microsoft/azure/management/datalake/store/uploader/UploaderFrontEndMock.java
deleted file mode 100644
index 3e42cf307518..000000000000
--- a/azure-mgmt-datalake-store-uploader/src/test/java/com/microsoft/azure/management/datalake/store/uploader/UploaderFrontEndMock.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/**
- * Copyright (c) Microsoft Corporation. All rights reserved.
- * Licensed under the MIT License. See License.txt in the project root for
- * license information.
- */
-package com.microsoft.azure.management.datalake.store.uploader;
-
-import com.microsoft.rest.RestException;
-import org.junit.Assert;
-
-import java.io.IOException;
-
-/**
- * A front end mock used for unit testing {@link DataLakeStoreUploader}
- */
-public class UploaderFrontEndMock implements FrontEndAdapter {
-
- private FrontEndAdapter BaseAdapter;
- private int createStreamCount;
- private boolean ThrowInConcat;
- private boolean ThrowInCreate;
-
- /**
- * Constructor with base front end.
- * @param baseAdapter The real front end to use when methods are not mocked.
- * @param throwInConcat If true, indicates that concatenation implementation should throw instead of doing work.
- * @param throwInCreate If true, indicates that the create implementation should throw instead of doing work.
- */
- public UploaderFrontEndMock(FrontEndAdapter baseAdapter, boolean throwInConcat, boolean throwInCreate)
- {
- createStreamCount = 0;
- ThrowInConcat = throwInConcat;
- ThrowInCreate = throwInCreate;
- BaseAdapter = baseAdapter;
- }
-
- public void createStream(String streamPath, boolean overwrite, byte[] data, int byteCount) throws RestException, IOException {
-
- if(ThrowInCreate) {
- createStreamCount++;
- if (createStreamCount > 1) {
- //we only allow 1 file to be created
- throw new IntentionalException();
- }
- }
-
- BaseAdapter.createStream(streamPath, overwrite, data, byteCount);
- }
-
- public void deleteStream(String streamPath, boolean recurse) throws RestException, IOException {
- BaseAdapter.deleteStream(streamPath, recurse);
- }
-
- public void appendToStream(String streamPath, byte[] data, long offset, int byteCount) throws RestException, IOException {
- BaseAdapter.appendToStream(streamPath, data, offset, byteCount);
- }
-
- public boolean streamExists(String streamPath) throws RestException, IOException {
- return BaseAdapter.streamExists(streamPath);
- }
-
- public long getStreamLength(String streamPath) throws RestException, IOException {
- return BaseAdapter.getStreamLength(streamPath);
- }
-
- public void concatenate(String targetStreamPath, String[] inputStreamPaths) throws RestException, IOException {
- if(ThrowInConcat) {
- throw new IntentionalException();
- }
-
- Assert.assertTrue("concatenate should not be called when using 1 segment", false);
- BaseAdapter.concatenate(targetStreamPath, inputStreamPaths);
- }
-}
diff --git a/azure-mgmt-datalake-store/src/main/java/com/microsoft/azure/management/datalake/store/Accounts.java b/azure-mgmt-datalake-store/src/main/java/com/microsoft/azure/management/datalake/store/Accounts.java
index cea3374d8145..741724dd2728 100644
--- a/azure-mgmt-datalake-store/src/main/java/com/microsoft/azure/management/datalake/store/Accounts.java
+++ b/azure-mgmt-datalake-store/src/main/java/com/microsoft/azure/management/datalake/store/Accounts.java
@@ -8,7 +8,6 @@
package com.microsoft.azure.management.datalake.store;
-import com.microsoft.azure.CloudException;
import com.microsoft.azure.ListOperationCallback;
import com.microsoft.azure.management.datalake.store.models.DataLakeStoreAccount;
import com.microsoft.azure.management.datalake.store.models.FirewallRule;
@@ -17,7 +16,6 @@
import com.microsoft.rest.ServiceCall;
import com.microsoft.rest.ServiceCallback;
import com.microsoft.rest.ServiceResponse;
-import java.io.IOException;
import java.util.List;
import rx.Observable;
@@ -32,12 +30,8 @@ public interface Accounts {
* @param resourceGroupName The name of the Azure resource group that contains the Data Lake Store account.
* @param accountName The name of the Data Lake Store account from which to delete the firewall rule.
* @param firewallRuleName The name of the firewall rule to delete.
- * @throws CloudException exception thrown from REST call
- * @throws IOException exception thrown from serialization/deserialization
- * @throws IllegalArgumentException exception thrown from invalid parameters
- * @return the {@link ServiceResponse} object if successful.
*/
- ServiceResponse deleteFirewallRule(String resourceGroupName, String accountName, String firewallRuleName) throws CloudException, IOException, IllegalArgumentException;
+ void deleteFirewallRule(String resourceGroupName, String accountName, String firewallRuleName);
/**
* Deletes the specified firewall rule from the specified Data Lake Store account.
@@ -58,7 +52,17 @@ public interface Accounts {
* @param firewallRuleName The name of the firewall rule to delete.
* @return the {@link ServiceResponse} object if successful.
*/
- Observable> deleteFirewallRuleAsync(String resourceGroupName, String accountName, String firewallRuleName);
+ Observable deleteFirewallRuleAsync(String resourceGroupName, String accountName, String firewallRuleName);
+
+ /**
+ * Deletes the specified firewall rule from the specified Data Lake Store account.
+ *
+ * @param resourceGroupName The name of the Azure resource group that contains the Data Lake Store account.
+ * @param accountName The name of the Data Lake Store account from which to delete the firewall rule.
+ * @param firewallRuleName The name of the firewall rule to delete.
+ * @return the {@link ServiceResponse} object if successful.
+ */
+ Observable> deleteFirewallRuleWithServiceResponseAsync(String resourceGroupName, String accountName, String firewallRuleName);
/**
* Gets the specified Data Lake Store firewall rule.
@@ -66,12 +70,9 @@ public interface Accounts {
* @param resourceGroupName The name of the Azure resource group that contains the Data Lake Store account.
* @param accountName The name of the Data Lake Store account from which to get the firewall rule.
* @param firewallRuleName The name of the firewall rule to retrieve.
- * @throws CloudException exception thrown from REST call
- * @throws IOException exception thrown from serialization/deserialization
- * @throws IllegalArgumentException exception thrown from invalid parameters
- * @return the FirewallRule object wrapped in {@link ServiceResponse} if successful.
+ * @return the FirewallRule object if successful.
*/
- ServiceResponse getFirewallRule(String resourceGroupName, String accountName, String firewallRuleName) throws CloudException, IOException, IllegalArgumentException;
+ FirewallRule getFirewallRule(String resourceGroupName, String accountName, String firewallRuleName);
/**
* Gets the specified Data Lake Store firewall rule.
@@ -92,19 +93,26 @@ public interface Accounts {
* @param firewallRuleName The name of the firewall rule to retrieve.
* @return the observable to the FirewallRule object
*/
- Observable> getFirewallRuleAsync(String resourceGroupName, String accountName, String firewallRuleName);
+ Observable getFirewallRuleAsync(String resourceGroupName, String accountName, String firewallRuleName);
+
+ /**
+ * Gets the specified Data Lake Store firewall rule.
+ *
+ * @param resourceGroupName The name of the Azure resource group that contains the Data Lake Store account.
+ * @param accountName The name of the Data Lake Store account from which to get the firewall rule.
+ * @param firewallRuleName The name of the firewall rule to retrieve.
+ * @return the observable to the FirewallRule object
+ */
+ Observable> getFirewallRuleWithServiceResponseAsync(String resourceGroupName, String accountName, String firewallRuleName);
/**
* Lists the Data Lake Store firewall rules within the specified Data Lake Store account.
*
* @param resourceGroupName The name of the Azure resource group that contains the Data Lake Store account.
* @param accountName The name of the Data Lake Store account from which to get the firewall rules.
- * @throws CloudException exception thrown from REST call
- * @throws IOException exception thrown from serialization/deserialization
- * @throws IllegalArgumentException exception thrown from invalid parameters
- * @return the List<FirewallRule> object wrapped in {@link ServiceResponse} if successful.
+ * @return the PagedList<FirewallRule> object if successful.
*/
- ServiceResponse> listFirewallRules(final String resourceGroupName, final String accountName) throws CloudException, IOException, IllegalArgumentException;
+ PagedList listFirewallRules(final String resourceGroupName, final String accountName);
/**
* Lists the Data Lake Store firewall rules within the specified Data Lake Store account.
@@ -121,9 +129,18 @@ public interface Accounts {
*
* @param resourceGroupName The name of the Azure resource group that contains the Data Lake Store account.
* @param accountName The name of the Data Lake Store account from which to get the firewall rules.
- * @return the observable to the List<FirewallRule> object
+ * @return the observable to the PagedList<FirewallRule> object
+ */
+ Observable> listFirewallRulesAsync(final String resourceGroupName, final String accountName);
+
+ /**
+ * Lists the Data Lake Store firewall rules within the specified Data Lake Store account.
+ *
+ * @param resourceGroupName The name of the Azure resource group that contains the Data Lake Store account.
+ * @param accountName The name of the Data Lake Store account from which to get the firewall rules.
+ * @return the observable to the PagedList<FirewallRule> object
*/
- Observable>> listFirewallRulesAsync(final String resourceGroupName, final String accountName);
+ Observable>> listFirewallRulesWithServiceResponseAsync(final String resourceGroupName, final String accountName);
/**
* Creates or updates the specified firewall rule.
@@ -132,12 +149,9 @@ public interface Accounts {
* @param accountName The name of the Data Lake Store account to which to add the firewall rule.
* @param name The name of the firewall rule to create or update.
* @param parameters Parameters supplied to create the create firewall rule.
- * @throws CloudException exception thrown from REST call
- * @throws IOException exception thrown from serialization/deserialization
- * @throws IllegalArgumentException exception thrown from invalid parameters
- * @return the FirewallRule object wrapped in {@link ServiceResponse} if successful.
+ * @return the FirewallRule object if successful.
*/
- ServiceResponse createOrUpdateFirewallRule(String resourceGroupName, String accountName, String name, FirewallRule parameters) throws CloudException, IOException, IllegalArgumentException;
+ FirewallRule createOrUpdateFirewallRule(String resourceGroupName, String accountName, String name, FirewallRule parameters);
/**
* Creates or updates the specified firewall rule.
@@ -160,7 +174,18 @@ public interface Accounts {
* @param parameters Parameters supplied to create the create firewall rule.
* @return the observable to the FirewallRule object
*/
- Observable> createOrUpdateFirewallRuleAsync(String resourceGroupName, String accountName, String name, FirewallRule parameters);
+ Observable createOrUpdateFirewallRuleAsync(String resourceGroupName, String accountName, String name, FirewallRule parameters);
+
+ /**
+ * Creates or updates the specified firewall rule.
+ *
+ * @param resourceGroupName The name of the Azure resource group that contains the Data Lake Store account.
+ * @param accountName The name of the Data Lake Store account to which to add the firewall rule.
+ * @param name The name of the firewall rule to create or update.
+ * @param parameters Parameters supplied to create the create firewall rule.
+ * @return the observable to the FirewallRule object
+ */
+ Observable> createOrUpdateFirewallRuleWithServiceResponseAsync(String resourceGroupName, String accountName, String name, FirewallRule parameters);
/**
* Creates the specified Data Lake Store account.
@@ -168,13 +193,9 @@ public interface Accounts {
* @param resourceGroupName The name of the Azure resource group that contains the Data Lake Store account.
* @param name The name of the Data Lake Store account to create.
* @param parameters Parameters supplied to create the Data Lake Store account.
- * @throws CloudException exception thrown from REST call
- * @throws IOException exception thrown from serialization/deserialization
- * @throws IllegalArgumentException exception thrown from invalid parameters
- * @throws InterruptedException exception thrown when long running operation is interrupted
- * @return the DataLakeStoreAccount object wrapped in {@link ServiceResponse} if successful.
+ * @return the DataLakeStoreAccount object if successful.
*/
- ServiceResponse create(String resourceGroupName, String name, DataLakeStoreAccount parameters) throws CloudException, IOException, IllegalArgumentException, InterruptedException;
+ DataLakeStoreAccount create(String resourceGroupName, String name, DataLakeStoreAccount parameters);
/**
* Creates the specified Data Lake Store account.
@@ -195,7 +216,7 @@ public interface Accounts {
* @param parameters Parameters supplied to create the Data Lake Store account.
* @return the observable to the DataLakeStoreAccount object
*/
- Observable> createAsync(String resourceGroupName, String name, DataLakeStoreAccount parameters);
+ Observable createAsync(String resourceGroupName, String name, DataLakeStoreAccount parameters);
/**
* Creates the specified Data Lake Store account.
@@ -203,12 +224,19 @@ public interface Accounts {
* @param resourceGroupName The name of the Azure resource group that contains the Data Lake Store account.
* @param name The name of the Data Lake Store account to create.
* @param parameters Parameters supplied to create the Data Lake Store account.
- * @throws CloudException exception thrown from REST call
- * @throws IOException exception thrown from serialization/deserialization
- * @throws IllegalArgumentException exception thrown from invalid parameters
- * @return the DataLakeStoreAccount object wrapped in {@link ServiceResponse} if successful.
+ * @return the observable to the DataLakeStoreAccount object
+ */
+ Observable> createWithServiceResponseAsync(String resourceGroupName, String name, DataLakeStoreAccount parameters);
+
+ /**
+ * Creates the specified Data Lake Store account.
+ *
+ * @param resourceGroupName The name of the Azure resource group that contains the Data Lake Store account.
+ * @param name The name of the Data Lake Store account to create.
+ * @param parameters Parameters supplied to create the Data Lake Store account.
+ * @return the DataLakeStoreAccount object if successful.
*/
- ServiceResponse beginCreate(String resourceGroupName, String name, DataLakeStoreAccount parameters) throws CloudException, IOException, IllegalArgumentException;
+ DataLakeStoreAccount beginCreate(String resourceGroupName, String name, DataLakeStoreAccount parameters);
/**
* Creates the specified Data Lake Store account.
@@ -229,7 +257,17 @@ public interface Accounts {
* @param parameters Parameters supplied to create the Data Lake Store account.
* @return the observable to the DataLakeStoreAccount object
*/
- Observable> beginCreateAsync(String resourceGroupName, String name, DataLakeStoreAccount parameters);
+ Observable beginCreateAsync(String resourceGroupName, String name, DataLakeStoreAccount parameters);
+
+ /**
+ * Creates the specified Data Lake Store account.
+ *
+ * @param resourceGroupName The name of the Azure resource group that contains the Data Lake Store account.
+ * @param name The name of the Data Lake Store account to create.
+ * @param parameters Parameters supplied to create the Data Lake Store account.
+ * @return the observable to the DataLakeStoreAccount object
+ */
+ Observable> beginCreateWithServiceResponseAsync(String resourceGroupName, String name, DataLakeStoreAccount parameters);
/**
* Updates the specified Data Lake Store account information.
@@ -237,13 +275,9 @@ public interface Accounts {
* @param resourceGroupName The name of the Azure resource group that contains the Data Lake Store account.
* @param name The name of the Data Lake Store account to update.
* @param parameters Parameters supplied to update the Data Lake Store account.
- * @throws CloudException exception thrown from REST call
- * @throws IOException exception thrown from serialization/deserialization
- * @throws IllegalArgumentException exception thrown from invalid parameters
- * @throws InterruptedException exception thrown when long running operation is interrupted
- * @return the DataLakeStoreAccount object wrapped in {@link ServiceResponse} if successful.
+ * @return the DataLakeStoreAccount object if successful.
*/
- ServiceResponse update(String resourceGroupName, String name, DataLakeStoreAccount parameters) throws CloudException, IOException, IllegalArgumentException, InterruptedException;
+ DataLakeStoreAccount update(String resourceGroupName, String name, DataLakeStoreAccount parameters);
/**
* Updates the specified Data Lake Store account information.
@@ -264,7 +298,17 @@ public interface Accounts {
* @param parameters Parameters supplied to update the Data Lake Store account.
* @return the observable to the DataLakeStoreAccount object
*/
- Observable> updateAsync(String resourceGroupName, String name, DataLakeStoreAccount parameters);
+ Observable updateAsync(String resourceGroupName, String name, DataLakeStoreAccount parameters);
+
+ /**
+ * Updates the specified Data Lake Store account information.
+ *
+ * @param resourceGroupName The name of the Azure resource group that contains the Data Lake Store account.
+ * @param name The name of the Data Lake Store account to update.
+ * @param parameters Parameters supplied to update the Data Lake Store account.
+ * @return the observable to the DataLakeStoreAccount object
+ */
+ Observable> updateWithServiceResponseAsync(String resourceGroupName, String name, DataLakeStoreAccount parameters);
/**
* Updates the specified Data Lake Store account information.
@@ -272,12 +316,9 @@ public interface Accounts {
* @param resourceGroupName The name of the Azure resource group that contains the Data Lake Store account.
* @param name The name of the Data Lake Store account to update.
* @param parameters Parameters supplied to update the Data Lake Store account.
- * @throws CloudException exception thrown from REST call
- * @throws IOException exception thrown from serialization/deserialization
- * @throws IllegalArgumentException exception thrown from invalid parameters
- * @return the DataLakeStoreAccount object wrapped in {@link ServiceResponse} if successful.
+ * @return the DataLakeStoreAccount object if successful.
*/
- ServiceResponse beginUpdate(String resourceGroupName, String name, DataLakeStoreAccount parameters) throws CloudException, IOException, IllegalArgumentException;
+ DataLakeStoreAccount beginUpdate(String resourceGroupName, String name, DataLakeStoreAccount parameters);
/**
* Updates the specified Data Lake Store account information.
@@ -298,20 +339,25 @@ public interface Accounts {
* @param parameters Parameters supplied to update the Data Lake Store account.
* @return the observable to the DataLakeStoreAccount object
*/
- Observable> beginUpdateAsync(String resourceGroupName, String name, DataLakeStoreAccount parameters);
+ Observable beginUpdateAsync(String resourceGroupName, String name, DataLakeStoreAccount parameters);
+
+ /**
+ * Updates the specified Data Lake Store account information.
+ *
+ * @param resourceGroupName The name of the Azure resource group that contains the Data Lake Store account.
+ * @param name The name of the Data Lake Store account to update.
+ * @param parameters Parameters supplied to update the Data Lake Store account.
+ * @return the observable to the DataLakeStoreAccount object
+ */
+ Observable> beginUpdateWithServiceResponseAsync(String resourceGroupName, String name, DataLakeStoreAccount parameters);
/**
* Deletes the specified Data Lake Store account.
*
* @param resourceGroupName The name of the Azure resource group that contains the Data Lake Store account.
* @param accountName The name of the Data Lake Store account to delete.
- * @throws CloudException exception thrown from REST call
- * @throws IOException exception thrown from serialization/deserialization
- * @throws IllegalArgumentException exception thrown from invalid parameters
- * @throws InterruptedException exception thrown when long running operation is interrupted
- * @return the {@link ServiceResponse} object if successful.
*/
- ServiceResponse delete(String resourceGroupName, String accountName) throws CloudException, IOException, IllegalArgumentException, InterruptedException;
+ void delete(String resourceGroupName, String accountName);
/**
* Deletes the specified Data Lake Store account.
@@ -330,19 +376,24 @@ public interface Accounts {
* @param accountName The name of the Data Lake Store account to delete.
* @return the {@link ServiceResponse} object if successful.
*/
- Observable> deleteAsync(String resourceGroupName, String accountName);
+ Observable deleteAsync(String resourceGroupName, String accountName);
/**
* Deletes the specified Data Lake Store account.
*
* @param resourceGroupName The name of the Azure resource group that contains the Data Lake Store account.
* @param accountName The name of the Data Lake Store account to delete.
- * @throws CloudException exception thrown from REST call
- * @throws IOException exception thrown from serialization/deserialization
- * @throws IllegalArgumentException exception thrown from invalid parameters
* @return the {@link ServiceResponse} object if successful.
*/
- ServiceResponse beginDelete(String resourceGroupName, String accountName) throws CloudException, IOException, IllegalArgumentException;
+ Observable> deleteWithServiceResponseAsync(String resourceGroupName, String accountName);
+
+ /**
+ * Deletes the specified Data Lake Store account.
+ *
+ * @param resourceGroupName The name of the Azure resource group that contains the Data Lake Store account.
+ * @param accountName The name of the Data Lake Store account to delete.
+ */
+ void beginDelete(String resourceGroupName, String accountName);
/**
* Deletes the specified Data Lake Store account.
@@ -361,19 +412,25 @@ public interface Accounts {
* @param accountName The name of the Data Lake Store account to delete.
* @return the {@link ServiceResponse} object if successful.
*/
- Observable> beginDeleteAsync(String resourceGroupName, String accountName);
+ Observable beginDeleteAsync(String resourceGroupName, String accountName);
+
+ /**
+ * Deletes the specified Data Lake Store account.
+ *
+ * @param resourceGroupName The name of the Azure resource group that contains the Data Lake Store account.
+ * @param accountName The name of the Data Lake Store account to delete.
+ * @return the {@link ServiceResponse} object if successful.
+ */
+ Observable> beginDeleteWithServiceResponseAsync(String resourceGroupName, String accountName);
/**
* Gets the specified Data Lake Store account.
*
* @param resourceGroupName The name of the Azure resource group that contains the Data Lake Store account.
* @param accountName The name of the Data Lake Store account to retrieve.
- * @throws CloudException exception thrown from REST call
- * @throws IOException exception thrown from serialization/deserialization
- * @throws IllegalArgumentException exception thrown from invalid parameters
- * @return the DataLakeStoreAccount object wrapped in {@link ServiceResponse} if successful.
+ * @return the DataLakeStoreAccount object if successful.
*/
- ServiceResponse get(String resourceGroupName, String accountName) throws CloudException, IOException, IllegalArgumentException;
+ DataLakeStoreAccount get(String resourceGroupName, String accountName);
/**
* Gets the specified Data Lake Store account.
@@ -392,18 +449,60 @@ public interface Accounts {
* @param accountName The name of the Data Lake Store account to retrieve.
* @return the observable to the DataLakeStoreAccount object
*/
- Observable> getAsync(String resourceGroupName, String accountName);
+ Observable getAsync(String resourceGroupName, String accountName);
+
+ /**
+ * Gets the specified Data Lake Store account.
+ *
+ * @param resourceGroupName The name of the Azure resource group that contains the Data Lake Store account.
+ * @param accountName The name of the Data Lake Store account to retrieve.
+ * @return the observable to the DataLakeStoreAccount object
+ */
+ Observable> getWithServiceResponseAsync(String resourceGroupName, String accountName);
+
+ /**
+ * Attempts to enable a user managed key vault for encryption of the specified Data Lake Store account.
+ *
+ * @param resourceGroupName The name of the Azure resource group that contains the Data Lake Store account.
+ * @param accountName The name of the Data Lake Store account to attempt to enable the Key Vault for.
+ */
+ void enableKeyVault(String resourceGroupName, String accountName);
+
+ /**
+ * Attempts to enable a user managed key vault for encryption of the specified Data Lake Store account.
+ *
+ * @param resourceGroupName The name of the Azure resource group that contains the Data Lake Store account.
+ * @param accountName The name of the Data Lake Store account to attempt to enable the Key Vault for.
+ * @param serviceCallback the async ServiceCallback to handle successful and failed responses.
+ * @return the {@link ServiceCall} object
+ */
+ ServiceCall enableKeyVaultAsync(String resourceGroupName, String accountName, final ServiceCallback serviceCallback);
+
+ /**
+ * Attempts to enable a user managed key vault for encryption of the specified Data Lake Store account.
+ *
+ * @param resourceGroupName The name of the Azure resource group that contains the Data Lake Store account.
+ * @param accountName The name of the Data Lake Store account to attempt to enable the Key Vault for.
+ * @return the {@link ServiceResponse} object if successful.
+ */
+ Observable enableKeyVaultAsync(String resourceGroupName, String accountName);
+
+ /**
+ * Attempts to enable a user managed key vault for encryption of the specified Data Lake Store account.
+ *
+ * @param resourceGroupName The name of the Azure resource group that contains the Data Lake Store account.
+ * @param accountName The name of the Data Lake Store account to attempt to enable the Key Vault for.
+ * @return the {@link ServiceResponse} object if successful.
+ */
+ Observable> enableKeyVaultWithServiceResponseAsync(String resourceGroupName, String accountName);
/**
* Lists the Data Lake Store accounts within a specific resource group. The response includes a link to the next page of results, if any.
*
* @param resourceGroupName The name of the Azure resource group that contains the Data Lake Store account(s).
- * @throws CloudException exception thrown from REST call
- * @throws IOException exception thrown from serialization/deserialization
- * @throws IllegalArgumentException exception thrown from invalid parameters
- * @return the List<DataLakeStoreAccount> object wrapped in {@link ServiceResponse} if successful.
+ * @return the PagedList<DataLakeStoreAccount> object if successful.
*/
- ServiceResponse> listByResourceGroup(final String resourceGroupName) throws CloudException, IOException, IllegalArgumentException;
+ PagedList listByResourceGroup(final String resourceGroupName);
/**
* Lists the Data Lake Store accounts within a specific resource group. The response includes a link to the next page of results, if any.
@@ -413,6 +512,22 @@ public interface Accounts {
* @return the {@link ServiceCall} object
*/
ServiceCall> listByResourceGroupAsync(final String resourceGroupName, final ListOperationCallback serviceCallback);
+
+ /**
+ * Lists the Data Lake Store accounts within a specific resource group. The response includes a link to the next page of results, if any.
+ *
+ * @param resourceGroupName The name of the Azure resource group that contains the Data Lake Store account(s).
+ * @return the observable to the PagedList<DataLakeStoreAccount> object
+ */
+ Observable> listByResourceGroupAsync(final String resourceGroupName);
+
+ /**
+ * Lists the Data Lake Store accounts within a specific resource group. The response includes a link to the next page of results, if any.
+ *
+ * @param resourceGroupName The name of the Azure resource group that contains the Data Lake Store account(s).
+ * @return the observable to the PagedList<DataLakeStoreAccount> object
+ */
+ Observable>> listByResourceGroupWithServiceResponseAsync(final String resourceGroupName);
/**
* Lists the Data Lake Store accounts within a specific resource group. The response includes a link to the next page of results, if any.
*
@@ -426,12 +541,9 @@ public interface Accounts {
* @param count A Boolean value of true or false to request a count of the matching resources included with the resources in the response, e.g. Categories?$count=true. Optional.
* @param search A free form search. A free-text search expression to match for whether a particular entry should be included in the feed, e.g. Categories?$search=blue OR green. Optional.
* @param format The desired return format. Return the response in particular formatxii without access to request headers for standard content-type negotiation (e.g Orders?$format=json). Optional.
- * @throws CloudException exception thrown from REST call
- * @throws IOException exception thrown from serialization/deserialization
- * @throws IllegalArgumentException exception thrown from invalid parameters
- * @return the List<DataLakeStoreAccount> object wrapped in {@link ServiceResponse} if successful.
+ * @return the PagedList<DataLakeStoreAccount> object if successful.
*/
- ServiceResponse> listByResourceGroup(final String resourceGroupName, final String filter, final Integer top, final Integer skip, final String expand, final String select, final String orderby, final Boolean count, final String search, final String format) throws CloudException, IOException, IllegalArgumentException;
+ PagedList listByResourceGroup(final String resourceGroupName, final String filter, final Integer top, final Integer skip, final String expand, final String select, final String orderby, final Boolean count, final String search, final String format);
/**
* Lists the Data Lake Store accounts within a specific resource group. The response includes a link to the next page of results, if any.
@@ -464,19 +576,33 @@ public interface Accounts {
* @param count A Boolean value of true or false to request a count of the matching resources included with the resources in the response, e.g. Categories?$count=true. Optional.
* @param search A free form search. A free-text search expression to match for whether a particular entry should be included in the feed, e.g. Categories?$search=blue OR green. Optional.
* @param format The desired return format. Return the response in particular formatxii without access to request headers for standard content-type negotiation (e.g Orders?$format=json). Optional.
- * @return the observable to the List<DataLakeStoreAccount> object
+ * @return the observable to the PagedList<DataLakeStoreAccount> object
+ */
+ Observable> listByResourceGroupAsync(final String resourceGroupName, final String filter, final Integer top, final Integer skip, final String expand, final String select, final String orderby, final Boolean count, final String search, final String format);
+
+ /**
+ * Lists the Data Lake Store accounts within a specific resource group. The response includes a link to the next page of results, if any.
+ *
+ * @param resourceGroupName The name of the Azure resource group that contains the Data Lake Store account(s).
+ * @param filter OData filter. Optional.
+ * @param top The number of items to return. Optional.
+ * @param skip The number of items to skip over before returning elements. Optional.
+ * @param expand OData expansion. Expand related resources in line with the retrieved resources, e.g. Categories/$expand=Products would expand Product data in line with each Category entry. Optional.
+ * @param select OData Select statement. Limits the properties on each entry to just those requested, e.g. Categories?$select=CategoryName,Description. Optional.
+ * @param orderby OrderBy clause. One or more comma-separated expressions with an optional "asc" (the default) or "desc" depending on the order you'd like the values sorted, e.g. Categories?$orderby=CategoryName desc. Optional.
+ * @param count A Boolean value of true or false to request a count of the matching resources included with the resources in the response, e.g. Categories?$count=true. Optional.
+ * @param search A free form search. A free-text search expression to match for whether a particular entry should be included in the feed, e.g. Categories?$search=blue OR green. Optional.
+ * @param format The desired return format. Return the response in particular formatxii without access to request headers for standard content-type negotiation (e.g Orders?$format=json). Optional.
+ * @return the observable to the PagedList<DataLakeStoreAccount> object
*/
- Observable>> listByResourceGroupAsync(final String resourceGroupName, final String filter, final Integer top, final Integer skip, final String expand, final String select, final String orderby, final Boolean count, final String search, final String format);
+ Observable>> listByResourceGroupWithServiceResponseAsync(final String resourceGroupName, final String filter, final Integer top, final Integer skip, final String expand, final String select, final String orderby, final Boolean count, final String search, final String format);
/**
* Lists the Data Lake Store accounts within the subscription. The response includes a link to the next page of results, if any.
*
- * @throws CloudException exception thrown from REST call
- * @throws IOException exception thrown from serialization/deserialization
- * @throws IllegalArgumentException exception thrown from invalid parameters
- * @return the List<DataLakeStoreAccount> object wrapped in {@link ServiceResponse} if successful.
+ * @return the PagedList<DataLakeStoreAccount> object if successful.
*/
- ServiceResponse> list() throws CloudException, IOException, IllegalArgumentException;
+ PagedList list();
/**
* Lists the Data Lake Store accounts within the subscription. The response includes a link to the next page of results, if any.
@@ -485,6 +611,20 @@ public interface Accounts {
* @return the {@link ServiceCall} object
*/
ServiceCall> listAsync(final ListOperationCallback serviceCallback);
+
+ /**
+ * Lists the Data Lake Store accounts within the subscription. The response includes a link to the next page of results, if any.
+ *
+ * @return the observable to the PagedList<DataLakeStoreAccount> object
+ */
+ Observable> listAsync();
+
+ /**
+ * Lists the Data Lake Store accounts within the subscription. The response includes a link to the next page of results, if any.
+ *
+ * @return the observable to the PagedList<DataLakeStoreAccount> object
+ */
+ Observable>> listWithServiceResponseAsync();
/**
* Lists the Data Lake Store accounts within the subscription. The response includes a link to the next page of results, if any.
*
@@ -497,12 +637,9 @@ public interface Accounts {
* @param count The Boolean value of true or false to request a count of the matching resources included with the resources in the response, e.g. Categories?$count=true. Optional.
* @param search A free form search. A free-text search expression to match for whether a particular entry should be included in the feed, e.g. Categories?$search=blue OR green. Optional.
* @param format The desired return format. Return the response in particular formatxii without access to request headers for standard content-type negotiation (e.g Orders?$format=json). Optional.
- * @throws CloudException exception thrown from REST call
- * @throws IOException exception thrown from serialization/deserialization
- * @throws IllegalArgumentException exception thrown from invalid parameters
- * @return the List<DataLakeStoreAccount> object wrapped in {@link ServiceResponse} if successful.
+ * @return the PagedList<DataLakeStoreAccount> object if successful.
*/
- ServiceResponse> list(final String filter, final Integer top, final Integer skip, final String expand, final String select, final String orderby, final Boolean count, final String search, final String format) throws CloudException, IOException, IllegalArgumentException;
+ PagedList list(final String filter, final Integer top, final Integer skip, final String expand, final String select, final String orderby, final Boolean count, final String search, final String format);
/**
* Lists the Data Lake Store accounts within the subscription. The response includes a link to the next page of results, if any.
@@ -533,20 +670,33 @@ public interface Accounts {
* @param count The Boolean value of true or false to request a count of the matching resources included with the resources in the response, e.g. Categories?$count=true. Optional.
* @param search A free form search. A free-text search expression to match for whether a particular entry should be included in the feed, e.g. Categories?$search=blue OR green. Optional.
* @param format The desired return format. Return the response in particular formatxii without access to request headers for standard content-type negotiation (e.g Orders?$format=json). Optional.
- * @return the observable to the List<DataLakeStoreAccount> object
+ * @return the observable to the PagedList<DataLakeStoreAccount> object
*/
- Observable>> listAsync(final String filter, final Integer top, final Integer skip, final String expand, final String select, final String orderby, final Boolean count, final String search, final String format);
+ Observable> listAsync(final String filter, final Integer top, final Integer skip, final String expand, final String select, final String orderby, final Boolean count, final String search, final String format);
+
+ /**
+ * Lists the Data Lake Store accounts within the subscription. The response includes a link to the next page of results, if any.
+ *
+ * @param filter OData filter. Optional.
+ * @param top The number of items to return. Optional.
+ * @param skip The number of items to skip over before returning elements. Optional.
+ * @param expand OData expansion. Expand related resources in line with the retrieved resources, e.g. Categories/$expand=Products would expand Product data in line with each Category entry. Optional.
+ * @param select OData Select statement. Limits the properties on each entry to just those requested, e.g. Categories?$select=CategoryName,Description. Optional.
+ * @param orderby OrderBy clause. One or more comma-separated expressions with an optional "asc" (the default) or "desc" depending on the order you'd like the values sorted, e.g. Categories?$orderby=CategoryName desc. Optional.
+ * @param count The Boolean value of true or false to request a count of the matching resources included with the resources in the response, e.g. Categories?$count=true. Optional.
+ * @param search A free form search. A free-text search expression to match for whether a particular entry should be included in the feed, e.g. Categories?$search=blue OR green. Optional.
+ * @param format The desired return format. Return the response in particular formatxii without access to request headers for standard content-type negotiation (e.g Orders?$format=json). Optional.
+ * @return the observable to the PagedList<DataLakeStoreAccount> object
+ */
+ Observable>> listWithServiceResponseAsync(final String filter, final Integer top, final Integer skip, final String expand, final String select, final String orderby, final Boolean count, final String search, final String format);
/**
* Lists the Data Lake Store firewall rules within the specified Data Lake Store account.
*
* @param nextPageLink The NextLink from the previous successful call to List operation.
- * @throws CloudException exception thrown from REST call
- * @throws IOException exception thrown from serialization/deserialization
- * @throws IllegalArgumentException exception thrown from invalid parameters
- * @return the List<FirewallRule> object wrapped in {@link ServiceResponse} if successful.
+ * @return the PagedList<FirewallRule> object if successful.
*/
- ServiceResponse> listFirewallRulesNext(final String nextPageLink) throws CloudException, IOException, IllegalArgumentException;
+ PagedList listFirewallRulesNext(final String nextPageLink);
/**
* Lists the Data Lake Store firewall rules within the specified Data Lake Store account.
@@ -562,20 +712,25 @@ public interface Accounts {
* Lists the Data Lake Store firewall rules within the specified Data Lake Store account.
*
* @param nextPageLink The NextLink from the previous successful call to List operation.
- * @return the observable to the List<FirewallRule> object
+ * @return the observable to the PagedList<FirewallRule> object
+ */
+ Observable> listFirewallRulesNextAsync(final String nextPageLink);
+
+ /**
+ * Lists the Data Lake Store firewall rules within the specified Data Lake Store account.
+ *
+ * @param nextPageLink The NextLink from the previous successful call to List operation.
+ * @return the observable to the PagedList<FirewallRule> object
*/
- Observable>> listFirewallRulesNextAsync(final String nextPageLink);
+ Observable>> listFirewallRulesNextWithServiceResponseAsync(final String nextPageLink);
/**
* Lists the Data Lake Store accounts within a specific resource group. The response includes a link to the next page of results, if any.
*
* @param nextPageLink The NextLink from the previous successful call to List operation.
- * @throws CloudException exception thrown from REST call
- * @throws IOException exception thrown from serialization/deserialization
- * @throws IllegalArgumentException exception thrown from invalid parameters
- * @return the List<DataLakeStoreAccount> object wrapped in {@link ServiceResponse} if successful.
+ * @return the PagedList<DataLakeStoreAccount> object if successful.
*/
- ServiceResponse> listByResourceGroupNext(final String nextPageLink) throws CloudException, IOException, IllegalArgumentException;
+ PagedList listByResourceGroupNext(final String nextPageLink);
/**
* Lists the Data Lake Store accounts within a specific resource group. The response includes a link to the next page of results, if any.
@@ -591,20 +746,25 @@ public interface Accounts {
* Lists the Data Lake Store accounts within a specific resource group. The response includes a link to the next page of results, if any.
*
* @param nextPageLink The NextLink from the previous successful call to List operation.
- * @return the observable to the List<DataLakeStoreAccount> object
+ * @return the observable to the PagedList<DataLakeStoreAccount> object
*/
- Observable>> listByResourceGroupNextAsync(final String nextPageLink);
+ Observable> listByResourceGroupNextAsync(final String nextPageLink);
+
+ /**
+ * Lists the Data Lake Store accounts within a specific resource group. The response includes a link to the next page of results, if any.
+ *
+ * @param nextPageLink The NextLink from the previous successful call to List operation.
+ * @return the observable to the PagedList<DataLakeStoreAccount> object
+ */
+ Observable>> listByResourceGroupNextWithServiceResponseAsync(final String nextPageLink);
/**
* Lists the Data Lake Store accounts within the subscription. The response includes a link to the next page of results, if any.
*
* @param nextPageLink The NextLink from the previous successful call to List operation.
- * @throws CloudException exception thrown from REST call
- * @throws IOException exception thrown from serialization/deserialization
- * @throws IllegalArgumentException exception thrown from invalid parameters
- * @return the List<DataLakeStoreAccount> object wrapped in {@link ServiceResponse} if successful.
+ * @return the PagedList<DataLakeStoreAccount> object if successful.
*/
- ServiceResponse> listNext(final String nextPageLink) throws CloudException, IOException, IllegalArgumentException;
+ PagedList listNext(final String nextPageLink);
/**
* Lists the Data Lake Store accounts within the subscription. The response includes a link to the next page of results, if any.
@@ -620,8 +780,16 @@ public interface Accounts {
* Lists the Data Lake Store accounts within the subscription. The response includes a link to the next page of results, if any.
*
* @param nextPageLink The NextLink from the previous successful call to List operation.
- * @return the observable to the List<DataLakeStoreAccount> object
+ * @return the observable to the PagedList<DataLakeStoreAccount> object
+ */
+ Observable> listNextAsync(final String nextPageLink);
+
+ /**
+ * Lists the Data Lake Store accounts within the subscription. The response includes a link to the next page of results, if any.
+ *
+ * @param nextPageLink The NextLink from the previous successful call to List operation.
+ * @return the observable to the PagedList<DataLakeStoreAccount> object
*/
- Observable>> listNextAsync(final String nextPageLink);
+ Observable>> listNextWithServiceResponseAsync(final String nextPageLink);
}
diff --git a/azure-mgmt-datalake-store/src/main/java/com/microsoft/azure/management/datalake/store/DataLakeStoreFileSystemManagementClient.java b/azure-mgmt-datalake-store/src/main/java/com/microsoft/azure/management/datalake/store/DataLakeStoreFileSystemManagementClient.java
deleted file mode 100644
index 66175c9c0e65..000000000000
--- a/azure-mgmt-datalake-store/src/main/java/com/microsoft/azure/management/datalake/store/DataLakeStoreFileSystemManagementClient.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/**
- * Copyright (c) Microsoft Corporation. All rights reserved.
- * Licensed under the MIT License. See License.txt in the project root for
- * license information.
- *
- * Code generated by Microsoft (R) AutoRest Code Generator.
- */
-
-package com.microsoft.azure.management.datalake.store;
-
-import com.microsoft.azure.AzureClient;
-import com.microsoft.azure.RestClient;
-
-/**
- * The interface for DataLakeStoreFileSystemManagementClient class.
- */
-public interface DataLakeStoreFileSystemManagementClient {
- /**
- * Gets the REST client.
- *
- * @return the {@link RestClient} object.
- */
- RestClient restClient();
-
- /**
- * Gets the {@link AzureClient} used for long running operations.
- * @return the azure client;
- */
- AzureClient getAzureClient();
-
- /**
- * Gets the User-Agent header for the client.
- *
- * @return the user agent string.
- */
- String userAgent();
-
- /**
- * Gets Client Api Version..
- *
- * @return the apiVersion value.
- */
- String apiVersion();
-
- /**
- * Gets Gets the URI used as the base for all cloud service requests..
- *
- * @return the adlsFileSystemDnsSuffix value.
- */
- String adlsFileSystemDnsSuffix();
-
- /**
- * Sets Gets the URI used as the base for all cloud service requests..
- *
- * @param adlsFileSystemDnsSuffix the adlsFileSystemDnsSuffix value.
- * @return the service client itself
- */
- DataLakeStoreFileSystemManagementClient withAdlsFileSystemDnsSuffix(String adlsFileSystemDnsSuffix);
-
- /**
- * Gets Gets or sets the preferred language for the response..
- *
- * @return the acceptLanguage value.
- */
- String acceptLanguage();
-
- /**
- * Sets Gets or sets the preferred language for the response..
- *
- * @param acceptLanguage the acceptLanguage value.
- * @return the service client itself
- */
- DataLakeStoreFileSystemManagementClient withAcceptLanguage(String acceptLanguage);
-
- /**
- * Gets Gets or sets the retry timeout in seconds for Long Running Operations. Default value is 30..
- *
- * @return the longRunningOperationRetryTimeout value.
- */
- int longRunningOperationRetryTimeout();
-
- /**
- * Sets Gets or sets the retry timeout in seconds for Long Running Operations. Default value is 30..
- *
- * @param longRunningOperationRetryTimeout the longRunningOperationRetryTimeout value.
- * @return the service client itself
- */
- DataLakeStoreFileSystemManagementClient withLongRunningOperationRetryTimeout(int longRunningOperationRetryTimeout);
-
- /**
- * Gets When set to true a unique x-ms-client-request-id value is generated and included in each request. Default is true..
- *
- * @return the generateClientRequestId value.
- */
- boolean generateClientRequestId();
-
- /**
- * Sets When set to true a unique x-ms-client-request-id value is generated and included in each request. Default is true..
- *
- * @param generateClientRequestId the generateClientRequestId value.
- * @return the service client itself
- */
- DataLakeStoreFileSystemManagementClient withGenerateClientRequestId(boolean generateClientRequestId);
-
- /**
- * Gets the FileSystems object to access its operations.
- * @return the FileSystems object.
- */
- FileSystems fileSystems();
-
-}
diff --git a/azure-mgmt-datalake-store/src/main/java/com/microsoft/azure/management/datalake/store/FileSystems.java b/azure-mgmt-datalake-store/src/main/java/com/microsoft/azure/management/datalake/store/FileSystems.java
deleted file mode 100644
index 58a692a55b6a..000000000000
--- a/azure-mgmt-datalake-store/src/main/java/com/microsoft/azure/management/datalake/store/FileSystems.java
+++ /dev/null
@@ -1,905 +0,0 @@
-/**
- * Copyright (c) Microsoft Corporation. All rights reserved.
- * Licensed under the MIT License. See License.txt in the project root for
- * license information.
- *
- * Code generated by Microsoft (R) AutoRest Code Generator.
- */
-
-package com.microsoft.azure.management.datalake.store;
-
-import com.microsoft.azure.management.datalake.store.models.AclStatusResult;
-import com.microsoft.azure.management.datalake.store.models.AdlsErrorException;
-import com.microsoft.azure.management.datalake.store.models.AppendModeType;
-import com.microsoft.azure.management.datalake.store.models.ContentSummaryResult;
-import com.microsoft.azure.management.datalake.store.models.FileOperationResult;
-import com.microsoft.azure.management.datalake.store.models.FileStatusesResult;
-import com.microsoft.azure.management.datalake.store.models.FileStatusResult;
-import com.microsoft.rest.ServiceCall;
-import com.microsoft.rest.ServiceCallback;
-import com.microsoft.rest.ServiceResponse;
-import java.io.InputStream;
-import java.io.IOException;
-import java.util.List;
-import rx.Observable;
-
-/**
- * An instance of this class provides access to all the operations defined
- * in FileSystems.
- */
-public interface FileSystems {
- /**
- * Appends to the specified file. This method supports multiple concurrent appends to the file. NOTE: Concurrent append and normal (serial) append CANNOT be used interchangeably. Once a file has been appended to using either append option, it can only be appended to using that append option.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param filePath The Data Lake Store path (starting with '/') of the file to which to append using concurrent append.
- * @param streamContents The file contents to include when appending to the file.
- * @throws AdlsErrorException exception thrown from REST call
- * @throws IOException exception thrown from serialization/deserialization
- * @throws IllegalArgumentException exception thrown from invalid parameters
- * @return the {@link ServiceResponse} object if successful.
- */
- ServiceResponse concurrentAppend(String accountName, String filePath, byte[] streamContents) throws AdlsErrorException, IOException, IllegalArgumentException;
-
- /**
- * Appends to the specified file. This method supports multiple concurrent appends to the file. NOTE: Concurrent append and normal (serial) append CANNOT be used interchangeably. Once a file has been appended to using either append option, it can only be appended to using that append option.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param filePath The Data Lake Store path (starting with '/') of the file to which to append using concurrent append.
- * @param streamContents The file contents to include when appending to the file.
- * @param serviceCallback the async ServiceCallback to handle successful and failed responses.
- * @return the {@link ServiceCall} object
- */
- ServiceCall concurrentAppendAsync(String accountName, String filePath, byte[] streamContents, final ServiceCallback serviceCallback);
- /**
- * Appends to the specified file. This method supports multiple concurrent appends to the file. NOTE: Concurrent append and normal (serial) append CANNOT be used interchangeably. Once a file has been appended to using either append option, it can only be appended to using that append option.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param filePath The Data Lake Store path (starting with '/') of the file to which to append using concurrent append.
- * @param streamContents The file contents to include when appending to the file.
- * @param appendMode Indicates the concurrent append call should create the file if it doesn't exist or just open the existing file for append. Possible values include: 'autocreate'
- * @throws AdlsErrorException exception thrown from REST call
- * @throws IOException exception thrown from serialization/deserialization
- * @throws IllegalArgumentException exception thrown from invalid parameters
- * @return the {@link ServiceResponse} object if successful.
- */
- ServiceResponse concurrentAppend(String accountName, String filePath, byte[] streamContents, AppendModeType appendMode) throws AdlsErrorException, IOException, IllegalArgumentException;
-
- /**
- * Appends to the specified file. This method supports multiple concurrent appends to the file. NOTE: Concurrent append and normal (serial) append CANNOT be used interchangeably. Once a file has been appended to using either append option, it can only be appended to using that append option.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param filePath The Data Lake Store path (starting with '/') of the file to which to append using concurrent append.
- * @param streamContents The file contents to include when appending to the file.
- * @param appendMode Indicates the concurrent append call should create the file if it doesn't exist or just open the existing file for append. Possible values include: 'autocreate'
- * @param serviceCallback the async ServiceCallback to handle successful and failed responses.
- * @return the {@link ServiceCall} object
- */
- ServiceCall concurrentAppendAsync(String accountName, String filePath, byte[] streamContents, AppendModeType appendMode, final ServiceCallback serviceCallback);
-
- /**
- * Appends to the specified file. This method supports multiple concurrent appends to the file. NOTE: Concurrent append and normal (serial) append CANNOT be used interchangeably. Once a file has been appended to using either append option, it can only be appended to using that append option.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param filePath The Data Lake Store path (starting with '/') of the file to which to append using concurrent append.
- * @param streamContents The file contents to include when appending to the file.
- * @param appendMode Indicates the concurrent append call should create the file if it doesn't exist or just open the existing file for append. Possible values include: 'autocreate'
- * @return the {@link ServiceResponse} object if successful.
- */
- Observable> concurrentAppendAsync(String accountName, String filePath, byte[] streamContents, AppendModeType appendMode);
-
- /**
- * Checks if the specified access is available at the given path.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param path The Data Lake Store path (starting with '/') of the file or directory for which to check access.
- * @throws AdlsErrorException exception thrown from REST call
- * @throws IOException exception thrown from serialization/deserialization
- * @throws IllegalArgumentException exception thrown from invalid parameters
- * @return the {@link ServiceResponse} object if successful.
- */
- ServiceResponse checkAccess(String accountName, String path) throws AdlsErrorException, IOException, IllegalArgumentException;
-
- /**
- * Checks if the specified access is available at the given path.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param path The Data Lake Store path (starting with '/') of the file or directory for which to check access.
- * @param serviceCallback the async ServiceCallback to handle successful and failed responses.
- * @return the {@link ServiceCall} object
- */
- ServiceCall checkAccessAsync(String accountName, String path, final ServiceCallback serviceCallback);
- /**
- * Checks if the specified access is available at the given path.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param path The Data Lake Store path (starting with '/') of the file or directory for which to check access.
- * @param fsaction File system operation read/write/execute in string form, matching regex pattern '[rwx-]{3}'
- * @throws AdlsErrorException exception thrown from REST call
- * @throws IOException exception thrown from serialization/deserialization
- * @throws IllegalArgumentException exception thrown from invalid parameters
- * @return the {@link ServiceResponse} object if successful.
- */
- ServiceResponse checkAccess(String accountName, String path, String fsaction) throws AdlsErrorException, IOException, IllegalArgumentException;
-
- /**
- * Checks if the specified access is available at the given path.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param path The Data Lake Store path (starting with '/') of the file or directory for which to check access.
- * @param fsaction File system operation read/write/execute in string form, matching regex pattern '[rwx-]{3}'
- * @param serviceCallback the async ServiceCallback to handle successful and failed responses.
- * @return the {@link ServiceCall} object
- */
- ServiceCall checkAccessAsync(String accountName, String path, String fsaction, final ServiceCallback serviceCallback);
-
- /**
- * Checks if the specified access is available at the given path.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param path The Data Lake Store path (starting with '/') of the file or directory for which to check access.
- * @param fsaction File system operation read/write/execute in string form, matching regex pattern '[rwx-]{3}'
- * @return the {@link ServiceResponse} object if successful.
- */
- Observable> checkAccessAsync(String accountName, String path, String fsaction);
-
- /**
- * Creates a directory.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param path The Data Lake Store path (starting with '/') of the directory to create.
- * @throws AdlsErrorException exception thrown from REST call
- * @throws IOException exception thrown from serialization/deserialization
- * @throws IllegalArgumentException exception thrown from invalid parameters
- * @return the FileOperationResult object wrapped in {@link ServiceResponse} if successful.
- */
- ServiceResponse mkdirs(String accountName, String path) throws AdlsErrorException, IOException, IllegalArgumentException;
-
- /**
- * Creates a directory.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param path The Data Lake Store path (starting with '/') of the directory to create.
- * @param serviceCallback the async ServiceCallback to handle successful and failed responses.
- * @return the {@link ServiceCall} object
- */
- ServiceCall mkdirsAsync(String accountName, String path, final ServiceCallback serviceCallback);
-
- /**
- * Creates a directory.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param path The Data Lake Store path (starting with '/') of the directory to create.
- * @return the observable to the FileOperationResult object
- */
- Observable> mkdirsAsync(String accountName, String path);
-
- /**
- * Concatenates the list of source files into the destination file, removing all source files upon success.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param destinationPath The Data Lake Store path (starting with '/') of the destination file resulting from the concatenation.
- * @param sources A list of comma seperated Data Lake Store paths (starting with '/') of the files to concatenate, in the order in which they should be concatenated.
- * @throws AdlsErrorException exception thrown from REST call
- * @throws IOException exception thrown from serialization/deserialization
- * @throws IllegalArgumentException exception thrown from invalid parameters
- * @return the {@link ServiceResponse} object if successful.
- */
- ServiceResponse concat(String accountName, String destinationPath, List sources) throws AdlsErrorException, IOException, IllegalArgumentException;
-
- /**
- * Concatenates the list of source files into the destination file, removing all source files upon success.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param destinationPath The Data Lake Store path (starting with '/') of the destination file resulting from the concatenation.
- * @param sources A list of comma seperated Data Lake Store paths (starting with '/') of the files to concatenate, in the order in which they should be concatenated.
- * @param serviceCallback the async ServiceCallback to handle successful and failed responses.
- * @return the {@link ServiceCall} object
- */
- ServiceCall concatAsync(String accountName, String destinationPath, List sources, final ServiceCallback serviceCallback);
-
- /**
- * Concatenates the list of source files into the destination file, removing all source files upon success.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param destinationPath The Data Lake Store path (starting with '/') of the destination file resulting from the concatenation.
- * @param sources A list of comma seperated Data Lake Store paths (starting with '/') of the files to concatenate, in the order in which they should be concatenated.
- * @return the {@link ServiceResponse} object if successful.
- */
- Observable> concatAsync(String accountName, String destinationPath, List sources);
-
- /**
- * Concatenates the list of source files into the destination file, deleting all source files upon success. This method accepts more source file paths than the Concat method. This method and the parameters it accepts are subject to change for usability in an upcoming version.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param msConcatDestinationPath The Data Lake Store path (starting with '/') of the destination file resulting from the concatenation.
- * @param streamContents A list of Data Lake Store paths (starting with '/') of the source files. Must be in the format: sources=<comma separated list>
- * @throws AdlsErrorException exception thrown from REST call
- * @throws IOException exception thrown from serialization/deserialization
- * @throws IllegalArgumentException exception thrown from invalid parameters
- * @return the {@link ServiceResponse} object if successful.
- */
- ServiceResponse msConcat(String accountName, String msConcatDestinationPath, byte[] streamContents) throws AdlsErrorException, IOException, IllegalArgumentException;
-
- /**
- * Concatenates the list of source files into the destination file, deleting all source files upon success. This method accepts more source file paths than the Concat method. This method and the parameters it accepts are subject to change for usability in an upcoming version.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param msConcatDestinationPath The Data Lake Store path (starting with '/') of the destination file resulting from the concatenation.
- * @param streamContents A list of Data Lake Store paths (starting with '/') of the source files. Must be in the format: sources=<comma separated list>
- * @param serviceCallback the async ServiceCallback to handle successful and failed responses.
- * @return the {@link ServiceCall} object
- */
- ServiceCall msConcatAsync(String accountName, String msConcatDestinationPath, byte[] streamContents, final ServiceCallback serviceCallback);
- /**
- * Concatenates the list of source files into the destination file, deleting all source files upon success. This method accepts more source file paths than the Concat method. This method and the parameters it accepts are subject to change for usability in an upcoming version.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param msConcatDestinationPath The Data Lake Store path (starting with '/') of the destination file resulting from the concatenation.
- * @param streamContents A list of Data Lake Store paths (starting with '/') of the source files. Must be in the format: sources=<comma separated list>
- * @param deleteSourceDirectory Indicates that as an optimization instead of deleting each individual source stream, delete the source stream folder if all streams are in the same folder instead. This results in a substantial performance improvement when the only streams in the folder are part of the concatenation operation. WARNING: This includes the deletion of any other files that are not source files. Only set this to true when source files are the only files in the source directory.
- * @throws AdlsErrorException exception thrown from REST call
- * @throws IOException exception thrown from serialization/deserialization
- * @throws IllegalArgumentException exception thrown from invalid parameters
- * @return the {@link ServiceResponse} object if successful.
- */
- ServiceResponse msConcat(String accountName, String msConcatDestinationPath, byte[] streamContents, Boolean deleteSourceDirectory) throws AdlsErrorException, IOException, IllegalArgumentException;
-
- /**
- * Concatenates the list of source files into the destination file, deleting all source files upon success. This method accepts more source file paths than the Concat method. This method and the parameters it accepts are subject to change for usability in an upcoming version.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param msConcatDestinationPath The Data Lake Store path (starting with '/') of the destination file resulting from the concatenation.
- * @param streamContents A list of Data Lake Store paths (starting with '/') of the source files. Must be in the format: sources=<comma separated list>
- * @param deleteSourceDirectory Indicates that as an optimization instead of deleting each individual source stream, delete the source stream folder if all streams are in the same folder instead. This results in a substantial performance improvement when the only streams in the folder are part of the concatenation operation. WARNING: This includes the deletion of any other files that are not source files. Only set this to true when source files are the only files in the source directory.
- * @param serviceCallback the async ServiceCallback to handle successful and failed responses.
- * @return the {@link ServiceCall} object
- */
- ServiceCall msConcatAsync(String accountName, String msConcatDestinationPath, byte[] streamContents, Boolean deleteSourceDirectory, final ServiceCallback serviceCallback);
-
- /**
- * Concatenates the list of source files into the destination file, deleting all source files upon success. This method accepts more source file paths than the Concat method. This method and the parameters it accepts are subject to change for usability in an upcoming version.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param msConcatDestinationPath The Data Lake Store path (starting with '/') of the destination file resulting from the concatenation.
- * @param streamContents A list of Data Lake Store paths (starting with '/') of the source files. Must be in the format: sources=<comma separated list>
- * @param deleteSourceDirectory Indicates that as an optimization instead of deleting each individual source stream, delete the source stream folder if all streams are in the same folder instead. This results in a substantial performance improvement when the only streams in the folder are part of the concatenation operation. WARNING: This includes the deletion of any other files that are not source files. Only set this to true when source files are the only files in the source directory.
- * @return the {@link ServiceResponse} object if successful.
- */
- Observable> msConcatAsync(String accountName, String msConcatDestinationPath, byte[] streamContents, Boolean deleteSourceDirectory);
-
- /**
- * Get the list of file status objects specified by the file path, with optional pagination parameters.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param listFilePath The Data Lake Store path (starting with '/') of the directory to list.
- * @throws AdlsErrorException exception thrown from REST call
- * @throws IOException exception thrown from serialization/deserialization
- * @throws IllegalArgumentException exception thrown from invalid parameters
- * @return the FileStatusesResult object wrapped in {@link ServiceResponse} if successful.
- */
- ServiceResponse listFileStatus(String accountName, String listFilePath) throws AdlsErrorException, IOException, IllegalArgumentException;
-
- /**
- * Get the list of file status objects specified by the file path, with optional pagination parameters.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param listFilePath The Data Lake Store path (starting with '/') of the directory to list.
- * @param serviceCallback the async ServiceCallback to handle successful and failed responses.
- * @return the {@link ServiceCall} object
- */
- ServiceCall listFileStatusAsync(String accountName, String listFilePath, final ServiceCallback serviceCallback);
- /**
- * Get the list of file status objects specified by the file path, with optional pagination parameters.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param listFilePath The Data Lake Store path (starting with '/') of the directory to list.
- * @param listSize Gets or sets the number of items to return. Optional.
- * @param listAfter Gets or sets the item or lexographical index after which to begin returning results. For example, a file list of 'a','b','d' and listAfter='b' will return 'd', and a listAfter='c' will also return 'd'. Optional.
- * @param listBefore Gets or sets the item or lexographical index before which to begin returning results. For example, a file list of 'a','b','d' and listBefore='d' will return 'a','b', and a listBefore='c' will also return 'a','b'. Optional.
- * @throws AdlsErrorException exception thrown from REST call
- * @throws IOException exception thrown from serialization/deserialization
- * @throws IllegalArgumentException exception thrown from invalid parameters
- * @return the FileStatusesResult object wrapped in {@link ServiceResponse} if successful.
- */
- ServiceResponse listFileStatus(String accountName, String listFilePath, Integer listSize, String listAfter, String listBefore) throws AdlsErrorException, IOException, IllegalArgumentException;
-
- /**
- * Get the list of file status objects specified by the file path, with optional pagination parameters.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param listFilePath The Data Lake Store path (starting with '/') of the directory to list.
- * @param listSize Gets or sets the number of items to return. Optional.
- * @param listAfter Gets or sets the item or lexographical index after which to begin returning results. For example, a file list of 'a','b','d' and listAfter='b' will return 'd', and a listAfter='c' will also return 'd'. Optional.
- * @param listBefore Gets or sets the item or lexographical index before which to begin returning results. For example, a file list of 'a','b','d' and listBefore='d' will return 'a','b', and a listBefore='c' will also return 'a','b'. Optional.
- * @param serviceCallback the async ServiceCallback to handle successful and failed responses.
- * @return the {@link ServiceCall} object
- */
- ServiceCall listFileStatusAsync(String accountName, String listFilePath, Integer listSize, String listAfter, String listBefore, final ServiceCallback serviceCallback);
-
- /**
- * Get the list of file status objects specified by the file path, with optional pagination parameters.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param listFilePath The Data Lake Store path (starting with '/') of the directory to list.
- * @param listSize Gets or sets the number of items to return. Optional.
- * @param listAfter Gets or sets the item or lexographical index after which to begin returning results. For example, a file list of 'a','b','d' and listAfter='b' will return 'd', and a listAfter='c' will also return 'd'. Optional.
- * @param listBefore Gets or sets the item or lexographical index before which to begin returning results. For example, a file list of 'a','b','d' and listBefore='d' will return 'a','b', and a listBefore='c' will also return 'a','b'. Optional.
- * @return the observable to the FileStatusesResult object
- */
- Observable> listFileStatusAsync(String accountName, String listFilePath, Integer listSize, String listAfter, String listBefore);
-
- /**
- * Gets the file content summary object specified by the file path.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param getContentSummaryFilePath The Data Lake Store path (starting with '/') of the file for which to retrieve the summary.
- * @throws AdlsErrorException exception thrown from REST call
- * @throws IOException exception thrown from serialization/deserialization
- * @throws IllegalArgumentException exception thrown from invalid parameters
- * @return the ContentSummaryResult object wrapped in {@link ServiceResponse} if successful.
- */
- ServiceResponse getContentSummary(String accountName, String getContentSummaryFilePath) throws AdlsErrorException, IOException, IllegalArgumentException;
-
- /**
- * Gets the file content summary object specified by the file path.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param getContentSummaryFilePath The Data Lake Store path (starting with '/') of the file for which to retrieve the summary.
- * @param serviceCallback the async ServiceCallback to handle successful and failed responses.
- * @return the {@link ServiceCall} object
- */
- ServiceCall getContentSummaryAsync(String accountName, String getContentSummaryFilePath, final ServiceCallback serviceCallback);
-
- /**
- * Gets the file content summary object specified by the file path.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param getContentSummaryFilePath The Data Lake Store path (starting with '/') of the file for which to retrieve the summary.
- * @return the observable to the ContentSummaryResult object
- */
- Observable> getContentSummaryAsync(String accountName, String getContentSummaryFilePath);
-
- /**
- * Get the file status object specified by the file path.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param getFilePath The Data Lake Store path (starting with '/') of the file or directory for which to retrieve the status.
- * @throws AdlsErrorException exception thrown from REST call
- * @throws IOException exception thrown from serialization/deserialization
- * @throws IllegalArgumentException exception thrown from invalid parameters
- * @return the FileStatusResult object wrapped in {@link ServiceResponse} if successful.
- */
- ServiceResponse getFileStatus(String accountName, String getFilePath) throws AdlsErrorException, IOException, IllegalArgumentException;
-
- /**
- * Get the file status object specified by the file path.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param getFilePath The Data Lake Store path (starting with '/') of the file or directory for which to retrieve the status.
- * @param serviceCallback the async ServiceCallback to handle successful and failed responses.
- * @return the {@link ServiceCall} object
- */
- ServiceCall getFileStatusAsync(String accountName, String getFilePath, final ServiceCallback serviceCallback);
-
- /**
- * Get the file status object specified by the file path.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param getFilePath The Data Lake Store path (starting with '/') of the file or directory for which to retrieve the status.
- * @return the observable to the FileStatusResult object
- */
- Observable> getFileStatusAsync(String accountName, String getFilePath);
-
- /**
- * Appends to the specified file. This method does not support multiple concurrent appends to the file. NOTE: Concurrent append and normal (serial) append CANNOT be used interchangeably. Once a file has been appended to using either append option, it can only be appended to using that append option. Use the ConcurrentAppend option if you would like support for concurrent appends.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param directFilePath The Data Lake Store path (starting with '/') of the file to which to append.
- * @param streamContents The file contents to include when appending to the file.
- * @throws AdlsErrorException exception thrown from REST call
- * @throws IOException exception thrown from serialization/deserialization
- * @throws IllegalArgumentException exception thrown from invalid parameters
- * @return the {@link ServiceResponse} object if successful.
- */
- ServiceResponse append(String accountName, String directFilePath, byte[] streamContents) throws AdlsErrorException, IOException, IllegalArgumentException;
-
- /**
- * Appends to the specified file. This method does not support multiple concurrent appends to the file. NOTE: Concurrent append and normal (serial) append CANNOT be used interchangeably. Once a file has been appended to using either append option, it can only be appended to using that append option. Use the ConcurrentAppend option if you would like support for concurrent appends.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param directFilePath The Data Lake Store path (starting with '/') of the file to which to append.
- * @param streamContents The file contents to include when appending to the file.
- * @param serviceCallback the async ServiceCallback to handle successful and failed responses.
- * @return the {@link ServiceCall} object
- */
- ServiceCall appendAsync(String accountName, String directFilePath, byte[] streamContents, final ServiceCallback serviceCallback);
- /**
- * Appends to the specified file. This method does not support multiple concurrent appends to the file. NOTE: Concurrent append and normal (serial) append CANNOT be used interchangeably. Once a file has been appended to using either append option, it can only be appended to using that append option. Use the ConcurrentAppend option if you would like support for concurrent appends.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param directFilePath The Data Lake Store path (starting with '/') of the file to which to append.
- * @param streamContents The file contents to include when appending to the file.
- * @param offset The optional offset in the stream to begin the append operation. Default is to append at the end of the stream.
- * @throws AdlsErrorException exception thrown from REST call
- * @throws IOException exception thrown from serialization/deserialization
- * @throws IllegalArgumentException exception thrown from invalid parameters
- * @return the {@link ServiceResponse} object if successful.
- */
- ServiceResponse append(String accountName, String directFilePath, byte[] streamContents, Long offset) throws AdlsErrorException, IOException, IllegalArgumentException;
-
- /**
- * Appends to the specified file. This method does not support multiple concurrent appends to the file. NOTE: Concurrent append and normal (serial) append CANNOT be used interchangeably. Once a file has been appended to using either append option, it can only be appended to using that append option. Use the ConcurrentAppend option if you would like support for concurrent appends.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param directFilePath The Data Lake Store path (starting with '/') of the file to which to append.
- * @param streamContents The file contents to include when appending to the file.
- * @param offset The optional offset in the stream to begin the append operation. Default is to append at the end of the stream.
- * @param serviceCallback the async ServiceCallback to handle successful and failed responses.
- * @return the {@link ServiceCall} object
- */
- ServiceCall appendAsync(String accountName, String directFilePath, byte[] streamContents, Long offset, final ServiceCallback serviceCallback);
-
- /**
- * Appends to the specified file. This method does not support multiple concurrent appends to the file. NOTE: Concurrent append and normal (serial) append CANNOT be used interchangeably. Once a file has been appended to using either append option, it can only be appended to using that append option. Use the ConcurrentAppend option if you would like support for concurrent appends.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param directFilePath The Data Lake Store path (starting with '/') of the file to which to append.
- * @param streamContents The file contents to include when appending to the file.
- * @param offset The optional offset in the stream to begin the append operation. Default is to append at the end of the stream.
- * @return the {@link ServiceResponse} object if successful.
- */
- Observable> appendAsync(String accountName, String directFilePath, byte[] streamContents, Long offset);
-
- /**
- * Creates a file with optionally specified content.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param directFilePath The Data Lake Store path (starting with '/') of the file to create.
- * @throws AdlsErrorException exception thrown from REST call
- * @throws IOException exception thrown from serialization/deserialization
- * @throws IllegalArgumentException exception thrown from invalid parameters
- * @return the {@link ServiceResponse} object if successful.
- */
- ServiceResponse create(String accountName, String directFilePath) throws AdlsErrorException, IOException, IllegalArgumentException;
-
- /**
- * Creates a file with optionally specified content.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param directFilePath The Data Lake Store path (starting with '/') of the file to create.
- * @param serviceCallback the async ServiceCallback to handle successful and failed responses.
- * @return the {@link ServiceCall} object
- */
- ServiceCall createAsync(String accountName, String directFilePath, final ServiceCallback serviceCallback);
- /**
- * Creates a file with optionally specified content.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param directFilePath The Data Lake Store path (starting with '/') of the file to create.
- * @param streamContents The file contents to include when creating the file. This parameter is optional, resulting in an empty file if not specified.
- * @param overwrite The indication of if the file should be overwritten.
- * @throws AdlsErrorException exception thrown from REST call
- * @throws IOException exception thrown from serialization/deserialization
- * @throws IllegalArgumentException exception thrown from invalid parameters
- * @return the {@link ServiceResponse} object if successful.
- */
- ServiceResponse create(String accountName, String directFilePath, byte[] streamContents, Boolean overwrite) throws AdlsErrorException, IOException, IllegalArgumentException;
-
- /**
- * Creates a file with optionally specified content.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param directFilePath The Data Lake Store path (starting with '/') of the file to create.
- * @param streamContents The file contents to include when creating the file. This parameter is optional, resulting in an empty file if not specified.
- * @param overwrite The indication of if the file should be overwritten.
- * @param serviceCallback the async ServiceCallback to handle successful and failed responses.
- * @return the {@link ServiceCall} object
- */
- ServiceCall createAsync(String accountName, String directFilePath, byte[] streamContents, Boolean overwrite, final ServiceCallback serviceCallback);
-
- /**
- * Creates a file with optionally specified content.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param directFilePath The Data Lake Store path (starting with '/') of the file to create.
- * @param streamContents The file contents to include when creating the file. This parameter is optional, resulting in an empty file if not specified.
- * @param overwrite The indication of if the file should be overwritten.
- * @return the {@link ServiceResponse} object if successful.
- */
- Observable> createAsync(String accountName, String directFilePath, byte[] streamContents, Boolean overwrite);
-
- /**
- * Opens and reads from the specified file.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param directFilePath The Data Lake Store path (starting with '/') of the file to open.
- * @throws AdlsErrorException exception thrown from REST call
- * @throws IOException exception thrown from serialization/deserialization
- * @throws IllegalArgumentException exception thrown from invalid parameters
- * @return the InputStream object wrapped in {@link ServiceResponse} if successful.
- */
- ServiceResponse open(String accountName, String directFilePath) throws AdlsErrorException, IOException, IllegalArgumentException;
-
- /**
- * Opens and reads from the specified file.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param directFilePath The Data Lake Store path (starting with '/') of the file to open.
- * @param serviceCallback the async ServiceCallback to handle successful and failed responses.
- * @return the {@link ServiceCall} object
- */
- ServiceCall openAsync(String accountName, String directFilePath, final ServiceCallback serviceCallback);
- /**
- * Opens and reads from the specified file.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param directFilePath The Data Lake Store path (starting with '/') of the file to open.
- * @param length the Long value
- * @param offset the Long value
- * @throws AdlsErrorException exception thrown from REST call
- * @throws IOException exception thrown from serialization/deserialization
- * @throws IllegalArgumentException exception thrown from invalid parameters
- * @return the InputStream object wrapped in {@link ServiceResponse} if successful.
- */
- ServiceResponse open(String accountName, String directFilePath, Long length, Long offset) throws AdlsErrorException, IOException, IllegalArgumentException;
-
- /**
- * Opens and reads from the specified file.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param directFilePath The Data Lake Store path (starting with '/') of the file to open.
- * @param length the Long value
- * @param offset the Long value
- * @param serviceCallback the async ServiceCallback to handle successful and failed responses.
- * @return the {@link ServiceCall} object
- */
- ServiceCall openAsync(String accountName, String directFilePath, Long length, Long offset, final ServiceCallback serviceCallback);
-
- /**
- * Opens and reads from the specified file.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param directFilePath The Data Lake Store path (starting with '/') of the file to open.
- * @param length the Long value
- * @param offset the Long value
- * @return the observable to the InputStream object
- */
- Observable> openAsync(String accountName, String directFilePath, Long length, Long offset);
-
- /**
- * Sets the Access Control List (ACL) for a file or folder.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param setAclFilePath The Data Lake Store path (starting with '/') of the file or directory on which to set the ACL.
- * @param aclspec The ACL spec included in ACL creation operations in the format '[default:]user|group|other::r|-w|-x|-'
- * @throws AdlsErrorException exception thrown from REST call
- * @throws IOException exception thrown from serialization/deserialization
- * @throws IllegalArgumentException exception thrown from invalid parameters
- * @return the {@link ServiceResponse} object if successful.
- */
- ServiceResponse setAcl(String accountName, String setAclFilePath, String aclspec) throws AdlsErrorException, IOException, IllegalArgumentException;
-
- /**
- * Sets the Access Control List (ACL) for a file or folder.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param setAclFilePath The Data Lake Store path (starting with '/') of the file or directory on which to set the ACL.
- * @param aclspec The ACL spec included in ACL creation operations in the format '[default:]user|group|other::r|-w|-x|-'
- * @param serviceCallback the async ServiceCallback to handle successful and failed responses.
- * @return the {@link ServiceCall} object
- */
- ServiceCall setAclAsync(String accountName, String setAclFilePath, String aclspec, final ServiceCallback serviceCallback);
-
- /**
- * Sets the Access Control List (ACL) for a file or folder.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param setAclFilePath The Data Lake Store path (starting with '/') of the file or directory on which to set the ACL.
- * @param aclspec The ACL spec included in ACL creation operations in the format '[default:]user|group|other::r|-w|-x|-'
- * @return the {@link ServiceResponse} object if successful.
- */
- Observable> setAclAsync(String accountName, String setAclFilePath, String aclspec);
-
- /**
- * Modifies existing Access Control List (ACL) entries on a file or folder.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param modifyAclFilePath The Data Lake Store path (starting with '/') of the file or directory with the ACL being modified.
- * @param aclspec The ACL specification included in ACL modification operations in the format '[default:]user|group|other::r|-w|-x|-'
- * @throws AdlsErrorException exception thrown from REST call
- * @throws IOException exception thrown from serialization/deserialization
- * @throws IllegalArgumentException exception thrown from invalid parameters
- * @return the {@link ServiceResponse} object if successful.
- */
- ServiceResponse modifyAclEntries(String accountName, String modifyAclFilePath, String aclspec) throws AdlsErrorException, IOException, IllegalArgumentException;
-
- /**
- * Modifies existing Access Control List (ACL) entries on a file or folder.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param modifyAclFilePath The Data Lake Store path (starting with '/') of the file or directory with the ACL being modified.
- * @param aclspec The ACL specification included in ACL modification operations in the format '[default:]user|group|other::r|-w|-x|-'
- * @param serviceCallback the async ServiceCallback to handle successful and failed responses.
- * @return the {@link ServiceCall} object
- */
- ServiceCall modifyAclEntriesAsync(String accountName, String modifyAclFilePath, String aclspec, final ServiceCallback serviceCallback);
-
- /**
- * Modifies existing Access Control List (ACL) entries on a file or folder.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param modifyAclFilePath The Data Lake Store path (starting with '/') of the file or directory with the ACL being modified.
- * @param aclspec The ACL specification included in ACL modification operations in the format '[default:]user|group|other::r|-w|-x|-'
- * @return the {@link ServiceResponse} object if successful.
- */
- Observable> modifyAclEntriesAsync(String accountName, String modifyAclFilePath, String aclspec);
-
- /**
- * Removes existing Access Control List (ACL) entries for a file or folder.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param removeAclFilePath The Data Lake Store path (starting with '/') of the file or directory with the ACL being removed.
- * @param aclspec The ACL spec included in ACL removal operations in the format '[default:]user|group|other'
- * @throws AdlsErrorException exception thrown from REST call
- * @throws IOException exception thrown from serialization/deserialization
- * @throws IllegalArgumentException exception thrown from invalid parameters
- * @return the {@link ServiceResponse} object if successful.
- */
- ServiceResponse removeAclEntries(String accountName, String removeAclFilePath, String aclspec) throws AdlsErrorException, IOException, IllegalArgumentException;
-
- /**
- * Removes existing Access Control List (ACL) entries for a file or folder.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param removeAclFilePath The Data Lake Store path (starting with '/') of the file or directory with the ACL being removed.
- * @param aclspec The ACL spec included in ACL removal operations in the format '[default:]user|group|other'
- * @param serviceCallback the async ServiceCallback to handle successful and failed responses.
- * @return the {@link ServiceCall} object
- */
- ServiceCall removeAclEntriesAsync(String accountName, String removeAclFilePath, String aclspec, final ServiceCallback serviceCallback);
-
- /**
- * Removes existing Access Control List (ACL) entries for a file or folder.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param removeAclFilePath The Data Lake Store path (starting with '/') of the file or directory with the ACL being removed.
- * @param aclspec The ACL spec included in ACL removal operations in the format '[default:]user|group|other'
- * @return the {@link ServiceResponse} object if successful.
- */
- Observable> removeAclEntriesAsync(String accountName, String removeAclFilePath, String aclspec);
-
- /**
- * Gets Access Control List (ACL) entries for the specified file or directory.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param aclFilePath The Data Lake Store path (starting with '/') of the file or directory for which to get the ACL.
- * @throws AdlsErrorException exception thrown from REST call
- * @throws IOException exception thrown from serialization/deserialization
- * @throws IllegalArgumentException exception thrown from invalid parameters
- * @return the AclStatusResult object wrapped in {@link ServiceResponse} if successful.
- */
- ServiceResponse getAclStatus(String accountName, String aclFilePath) throws AdlsErrorException, IOException, IllegalArgumentException;
-
- /**
- * Gets Access Control List (ACL) entries for the specified file or directory.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param aclFilePath The Data Lake Store path (starting with '/') of the file or directory for which to get the ACL.
- * @param serviceCallback the async ServiceCallback to handle successful and failed responses.
- * @return the {@link ServiceCall} object
- */
- ServiceCall getAclStatusAsync(String accountName, String aclFilePath, final ServiceCallback serviceCallback);
-
- /**
- * Gets Access Control List (ACL) entries for the specified file or directory.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param aclFilePath The Data Lake Store path (starting with '/') of the file or directory for which to get the ACL.
- * @return the observable to the AclStatusResult object
- */
- Observable> getAclStatusAsync(String accountName, String aclFilePath);
-
- /**
- * Deletes the requested file or directory, optionally recursively.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param filePath The Data Lake Store path (starting with '/') of the file or directory to delete.
- * @throws AdlsErrorException exception thrown from REST call
- * @throws IOException exception thrown from serialization/deserialization
- * @throws IllegalArgumentException exception thrown from invalid parameters
- * @return the FileOperationResult object wrapped in {@link ServiceResponse} if successful.
- */
- ServiceResponse delete(String accountName, String filePath) throws AdlsErrorException, IOException, IllegalArgumentException;
-
- /**
- * Deletes the requested file or directory, optionally recursively.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param filePath The Data Lake Store path (starting with '/') of the file or directory to delete.
- * @param serviceCallback the async ServiceCallback to handle successful and failed responses.
- * @return the {@link ServiceCall} object
- */
- ServiceCall deleteAsync(String accountName, String filePath, final ServiceCallback serviceCallback);
- /**
- * Deletes the requested file or directory, optionally recursively.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param filePath The Data Lake Store path (starting with '/') of the file or directory to delete.
- * @param recursive The optional switch indicating if the delete should be recursive
- * @throws AdlsErrorException exception thrown from REST call
- * @throws IOException exception thrown from serialization/deserialization
- * @throws IllegalArgumentException exception thrown from invalid parameters
- * @return the FileOperationResult object wrapped in {@link ServiceResponse} if successful.
- */
- ServiceResponse delete(String accountName, String filePath, Boolean recursive) throws AdlsErrorException, IOException, IllegalArgumentException;
-
- /**
- * Deletes the requested file or directory, optionally recursively.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param filePath The Data Lake Store path (starting with '/') of the file or directory to delete.
- * @param recursive The optional switch indicating if the delete should be recursive
- * @param serviceCallback the async ServiceCallback to handle successful and failed responses.
- * @return the {@link ServiceCall} object
- */
- ServiceCall deleteAsync(String accountName, String filePath, Boolean recursive, final ServiceCallback serviceCallback);
-
- /**
- * Deletes the requested file or directory, optionally recursively.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param filePath The Data Lake Store path (starting with '/') of the file or directory to delete.
- * @param recursive The optional switch indicating if the delete should be recursive
- * @return the observable to the FileOperationResult object
- */
- Observable> deleteAsync(String accountName, String filePath, Boolean recursive);
-
- /**
- * Rename a file or directory.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param renameFilePath The Data Lake Store path (starting with '/') of the file or directory to move/rename.
- * @param destination The path to move/rename the file or folder to
- * @throws AdlsErrorException exception thrown from REST call
- * @throws IOException exception thrown from serialization/deserialization
- * @throws IllegalArgumentException exception thrown from invalid parameters
- * @return the FileOperationResult object wrapped in {@link ServiceResponse} if successful.
- */
- ServiceResponse rename(String accountName, String renameFilePath, String destination) throws AdlsErrorException, IOException, IllegalArgumentException;
-
- /**
- * Rename a file or directory.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param renameFilePath The Data Lake Store path (starting with '/') of the file or directory to move/rename.
- * @param destination The path to move/rename the file or folder to
- * @param serviceCallback the async ServiceCallback to handle successful and failed responses.
- * @return the {@link ServiceCall} object
- */
- ServiceCall renameAsync(String accountName, String renameFilePath, String destination, final ServiceCallback serviceCallback);
-
- /**
- * Rename a file or directory.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param renameFilePath The Data Lake Store path (starting with '/') of the file or directory to move/rename.
- * @param destination The path to move/rename the file or folder to
- * @return the observable to the FileOperationResult object
- */
- Observable> renameAsync(String accountName, String renameFilePath, String destination);
-
- /**
- * Sets the owner of a file or directory.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param setOwnerFilePath The Data Lake Store path (starting with '/') of the file or directory for which to set the owner.
- * @throws AdlsErrorException exception thrown from REST call
- * @throws IOException exception thrown from serialization/deserialization
- * @throws IllegalArgumentException exception thrown from invalid parameters
- * @return the {@link ServiceResponse} object if successful.
- */
- ServiceResponse setOwner(String accountName, String setOwnerFilePath) throws AdlsErrorException, IOException, IllegalArgumentException;
-
- /**
- * Sets the owner of a file or directory.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param setOwnerFilePath The Data Lake Store path (starting with '/') of the file or directory for which to set the owner.
- * @param serviceCallback the async ServiceCallback to handle successful and failed responses.
- * @return the {@link ServiceCall} object
- */
- ServiceCall setOwnerAsync(String accountName, String setOwnerFilePath, final ServiceCallback serviceCallback);
- /**
- * Sets the owner of a file or directory.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param setOwnerFilePath The Data Lake Store path (starting with '/') of the file or directory for which to set the owner.
- * @param owner The AAD Object ID of the user owner of the file or directory. If empty, the property will remain unchanged.
- * @param group The AAD Object ID of the group owner of the file or directory. If empty, the property will remain unchanged.
- * @throws AdlsErrorException exception thrown from REST call
- * @throws IOException exception thrown from serialization/deserialization
- * @throws IllegalArgumentException exception thrown from invalid parameters
- * @return the {@link ServiceResponse} object if successful.
- */
- ServiceResponse setOwner(String accountName, String setOwnerFilePath, String owner, String group) throws AdlsErrorException, IOException, IllegalArgumentException;
-
- /**
- * Sets the owner of a file or directory.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param setOwnerFilePath The Data Lake Store path (starting with '/') of the file or directory for which to set the owner.
- * @param owner The AAD Object ID of the user owner of the file or directory. If empty, the property will remain unchanged.
- * @param group The AAD Object ID of the group owner of the file or directory. If empty, the property will remain unchanged.
- * @param serviceCallback the async ServiceCallback to handle successful and failed responses.
- * @return the {@link ServiceCall} object
- */
- ServiceCall setOwnerAsync(String accountName, String setOwnerFilePath, String owner, String group, final ServiceCallback serviceCallback);
-
- /**
- * Sets the owner of a file or directory.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param setOwnerFilePath The Data Lake Store path (starting with '/') of the file or directory for which to set the owner.
- * @param owner The AAD Object ID of the user owner of the file or directory. If empty, the property will remain unchanged.
- * @param group The AAD Object ID of the group owner of the file or directory. If empty, the property will remain unchanged.
- * @return the {@link ServiceResponse} object if successful.
- */
- Observable> setOwnerAsync(String accountName, String setOwnerFilePath, String owner, String group);
-
- /**
- * Sets the permission of the file or folder.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param setPermissionFilePath The Data Lake Store path (starting with '/') of the file or directory for which to set the permission.
- * @throws AdlsErrorException exception thrown from REST call
- * @throws IOException exception thrown from serialization/deserialization
- * @throws IllegalArgumentException exception thrown from invalid parameters
- * @return the {@link ServiceResponse} object if successful.
- */
- ServiceResponse setPermission(String accountName, String setPermissionFilePath) throws AdlsErrorException, IOException, IllegalArgumentException;
-
- /**
- * Sets the permission of the file or folder.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param setPermissionFilePath The Data Lake Store path (starting with '/') of the file or directory for which to set the permission.
- * @param serviceCallback the async ServiceCallback to handle successful and failed responses.
- * @return the {@link ServiceCall} object
- */
- ServiceCall setPermissionAsync(String accountName, String setPermissionFilePath, final ServiceCallback serviceCallback);
- /**
- * Sets the permission of the file or folder.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param setPermissionFilePath The Data Lake Store path (starting with '/') of the file or directory for which to set the permission.
- * @param permission A string representation of the permission (i.e 'rwx'). If empty, this property remains unchanged.
- * @throws AdlsErrorException exception thrown from REST call
- * @throws IOException exception thrown from serialization/deserialization
- * @throws IllegalArgumentException exception thrown from invalid parameters
- * @return the {@link ServiceResponse} object if successful.
- */
- ServiceResponse setPermission(String accountName, String setPermissionFilePath, String permission) throws AdlsErrorException, IOException, IllegalArgumentException;
-
- /**
- * Sets the permission of the file or folder.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param setPermissionFilePath The Data Lake Store path (starting with '/') of the file or directory for which to set the permission.
- * @param permission A string representation of the permission (i.e 'rwx'). If empty, this property remains unchanged.
- * @param serviceCallback the async ServiceCallback to handle successful and failed responses.
- * @return the {@link ServiceCall} object
- */
- ServiceCall setPermissionAsync(String accountName, String setPermissionFilePath, String permission, final ServiceCallback serviceCallback);
-
- /**
- * Sets the permission of the file or folder.
- *
- * @param accountName The Azure Data Lake Store account to execute filesystem operations on.
- * @param setPermissionFilePath The Data Lake Store path (starting with '/') of the file or directory for which to set the permission.
- * @param permission A string representation of the permission (i.e 'rwx'). If empty, this property remains unchanged.
- * @return the {@link ServiceResponse} object if successful.
- */
- Observable> setPermissionAsync(String accountName, String setPermissionFilePath, String permission);
-
-}
diff --git a/azure-mgmt-datalake-store/src/main/java/com/microsoft/azure/management/datalake/store/implementation/AccountsImpl.java b/azure-mgmt-datalake-store/src/main/java/com/microsoft/azure/management/datalake/store/implementation/AccountsImpl.java
index 61d11db012a4..15aaca8a9440 100644
--- a/azure-mgmt-datalake-store/src/main/java/com/microsoft/azure/management/datalake/store/implementation/AccountsImpl.java
+++ b/azure-mgmt-datalake-store/src/main/java/com/microsoft/azure/management/datalake/store/implementation/AccountsImpl.java
@@ -20,7 +20,6 @@
import com.microsoft.azure.management.datalake.store.models.PageImpl;
import com.microsoft.azure.Page;
import com.microsoft.azure.PagedList;
-import com.microsoft.rest.RestException;
import com.microsoft.rest.ServiceCall;
import com.microsoft.rest.ServiceCallback;
import com.microsoft.rest.ServiceResponse;
@@ -35,6 +34,7 @@
import retrofit2.http.HTTP;
import retrofit2.http.PATCH;
import retrofit2.http.Path;
+import retrofit2.http.POST;
import retrofit2.http.PUT;
import retrofit2.http.Query;
import retrofit2.Response;
@@ -111,6 +111,10 @@ interface AccountsService {
@GET("subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.DataLakeStore/accounts/{accountName}")
Observable> get(@Path("resourceGroupName") String resourceGroupName, @Path("accountName") String accountName, @Path("subscriptionId") String subscriptionId, @Query("api-version") String apiVersion, @Header("accept-language") String acceptLanguage, @Header("User-Agent") String userAgent);
+ @Headers("Content-Type: application/json; charset=utf-8")
+ @POST("subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.DataLakeStore/accounts/{accountName}/enableKeyVault")
+ Observable> enableKeyVault(@Path("resourceGroupName") String resourceGroupName, @Path("accountName") String accountName, @Path("subscriptionId") String subscriptionId, @Query("api-version") String apiVersion, @Header("accept-language") String acceptLanguage, @Header("User-Agent") String userAgent);
+
@Headers("Content-Type: application/json; charset=utf-8")
@GET("subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.DataLakeStore/accounts")
Observable> listByResourceGroup(@Path("resourceGroupName") String resourceGroupName, @Path("subscriptionId") String subscriptionId, @Query("$filter") String filter, @Query("$top") Integer top, @Query("$skip") Integer skip, @Query("$expand") String expand, @Query("$select") String select, @Query("$orderby") String orderby, @Query("$count") Boolean count, @Query("$search") String search, @Query("$format") String format, @Query("api-version") String apiVersion, @Header("accept-language") String acceptLanguage, @Header("User-Agent") String userAgent);
@@ -139,13 +143,9 @@ interface AccountsService {
* @param resourceGroupName The name of the Azure resource group that contains the Data Lake Store account.
* @param accountName The name of the Data Lake Store account from which to delete the firewall rule.
* @param firewallRuleName The name of the firewall rule to delete.
- * @throws CloudException exception thrown from REST call
- * @throws IOException exception thrown from serialization/deserialization
- * @throws IllegalArgumentException exception thrown from invalid parameters
- * @return the {@link ServiceResponse} object if successful.
*/
- public ServiceResponse deleteFirewallRule(String resourceGroupName, String accountName, String firewallRuleName) throws CloudException, IOException, IllegalArgumentException {
- return deleteFirewallRuleAsync(resourceGroupName, accountName, firewallRuleName).toBlocking().single();
+ public void deleteFirewallRule(String resourceGroupName, String accountName, String firewallRuleName) {
+ deleteFirewallRuleWithServiceResponseAsync(resourceGroupName, accountName, firewallRuleName).toBlocking().single().getBody();
}
/**
@@ -158,7 +158,7 @@ public ServiceResponse deleteFirewallRule(String resourceGroupName, String
* @return the {@link ServiceCall} object
*/
public ServiceCall deleteFirewallRuleAsync(String resourceGroupName, String accountName, String firewallRuleName, final ServiceCallback serviceCallback) {
- return ServiceCall.create(deleteFirewallRuleAsync(resourceGroupName, accountName, firewallRuleName), serviceCallback);
+ return ServiceCall.create(deleteFirewallRuleWithServiceResponseAsync(resourceGroupName, accountName, firewallRuleName), serviceCallback);
}
/**
@@ -169,7 +169,24 @@ public ServiceCall deleteFirewallRuleAsync(String resourceGroupName, Strin
* @param firewallRuleName The name of the firewall rule to delete.
* @return the {@link ServiceResponse} object if successful.
*/
- public Observable> deleteFirewallRuleAsync(String resourceGroupName, String accountName, String firewallRuleName) {
+ public Observable deleteFirewallRuleAsync(String resourceGroupName, String accountName, String firewallRuleName) {
+ return deleteFirewallRuleWithServiceResponseAsync(resourceGroupName, accountName, firewallRuleName).map(new Func1, Void>() {
+ @Override
+ public Void call(ServiceResponse response) {
+ return response.getBody();
+ }
+ });
+ }
+
+ /**
+ * Deletes the specified firewall rule from the specified Data Lake Store account.
+ *
+ * @param resourceGroupName The name of the Azure resource group that contains the Data Lake Store account.
+ * @param accountName The name of the Data Lake Store account from which to delete the firewall rule.
+ * @param firewallRuleName The name of the firewall rule to delete.
+ * @return the {@link ServiceResponse} object if successful.
+ */
+ public Observable> deleteFirewallRuleWithServiceResponseAsync(String resourceGroupName, String accountName, String firewallRuleName) {
if (resourceGroupName == null) {
throw new IllegalArgumentException("Parameter resourceGroupName is required and cannot be null.");
}
@@ -212,13 +229,10 @@ private ServiceResponse deleteFirewallRuleDelegate(Response
* @param resourceGroupName The name of the Azure resource group that contains the Data Lake Store account.
* @param accountName The name of the Data Lake Store account from which to get the firewall rule.
* @param firewallRuleName The name of the firewall rule to retrieve.
- * @throws CloudException exception thrown from REST call
- * @throws IOException exception thrown from serialization/deserialization
- * @throws IllegalArgumentException exception thrown from invalid parameters
- * @return the FirewallRule object wrapped in {@link ServiceResponse} if successful.
+ * @return the FirewallRule object if successful.
*/
- public ServiceResponse getFirewallRule(String resourceGroupName, String accountName, String firewallRuleName) throws CloudException, IOException, IllegalArgumentException {
- return getFirewallRuleAsync(resourceGroupName, accountName, firewallRuleName).toBlocking().single();
+ public FirewallRule getFirewallRule(String resourceGroupName, String accountName, String firewallRuleName) {
+ return getFirewallRuleWithServiceResponseAsync(resourceGroupName, accountName, firewallRuleName).toBlocking().single().getBody();
}
/**
@@ -231,7 +245,7 @@ public ServiceResponse getFirewallRule(String resourceGroupName, S
* @return the {@link ServiceCall} object
*/
public ServiceCall getFirewallRuleAsync(String resourceGroupName, String accountName, String firewallRuleName, final ServiceCallback serviceCallback) {
- return ServiceCall.create(getFirewallRuleAsync(resourceGroupName, accountName, firewallRuleName), serviceCallback);
+ return ServiceCall.create(getFirewallRuleWithServiceResponseAsync(resourceGroupName, accountName, firewallRuleName), serviceCallback);
}
/**
@@ -242,7 +256,24 @@ public ServiceCall getFirewallRuleAsync(String resourceGroupName,
* @param firewallRuleName The name of the firewall rule to retrieve.
* @return the observable to the FirewallRule object
*/
- public Observable> getFirewallRuleAsync(String resourceGroupName, String accountName, String firewallRuleName) {
+ public Observable getFirewallRuleAsync(String resourceGroupName, String accountName, String firewallRuleName) {
+ return getFirewallRuleWithServiceResponseAsync(resourceGroupName, accountName, firewallRuleName).map(new Func1, FirewallRule>() {
+ @Override
+ public FirewallRule call(ServiceResponse response) {
+ return response.getBody();
+ }
+ });
+ }
+
+ /**
+ * Gets the specified Data Lake Store firewall rule.
+ *
+ * @param resourceGroupName The name of the Azure resource group that contains the Data Lake Store account.
+ * @param accountName The name of the Data Lake Store account from which to get the firewall rule.
+ * @param firewallRuleName The name of the firewall rule to retrieve.
+ * @return the observable to the FirewallRule object
+ */
+ public Observable> getFirewallRuleWithServiceResponseAsync(String resourceGroupName, String accountName, String firewallRuleName) {
if (resourceGroupName == null) {
throw new IllegalArgumentException("Parameter resourceGroupName is required and cannot be null.");
}
@@ -284,20 +315,16 @@ private ServiceResponse