[Storage] Download stream refactor

sdk/storage/azure-storage-blob/HISTORY.md

@@ -5,7 +5,20 @@
 **Breaking changes**
 
 - `set_container_access_policy` has required parameter `signed_identifiers`.
-- NoRetry policy has been removed. Use keyword argument `retry_total=0` for no retries. 
+- `NoRetry` policy has been removed. Use keyword argument `retry_total=0` for no retries. 
+- `StorageStreamDownloader` is no longer iterable. To iterate over the blob data stream, use `StorageStreamDownloader.chunks`.
+- The public attributes of `StorageStreamDownloader` have been limited to:
+  - `name` (str): The name of the blob.
+  - `container` (str): The container the blob is being downloaded from.
+  - `properties` (`BlobProperties`): The properties of the blob.
+  - `size` (int): The size of the download. Either the total blob size, or the length of a subsection if sepcified. Previously called `download_size`.
+- `StorageStreamDownloader` now has new functions:
+  - `readall()`: Reads the complete download stream, returning bytes. This replaces the functions `content_as_bytes` and `content_as_text` which have been deprecated.
+  - `readinto(stream)`: Download the complete stream into the supplied writable stream, returning the number of bytes written. This replaces the function `download_to_stream` which has been deprecated.
+- Module level functions `upload_blob_to_url` and `download_blob_from_url` functions options are now keyword only:
+  - `overwrite`
+  - `max_concurrency`
+  - `encoding`
 - Removed types that were accidentally exposed from two modules. Only `BlobServiceClient`, `ContainerClient`, 
 `BlobClient` and `LeaseClient` should be imported from azure.storage.blob.aio
 - `Logging` has been renamed to `BlobAnalyticsLogging`.
@@ -14,6 +27,7 @@
 
 **New features**
 
+- Added async module-level `upload_blob_to_url` and `download_blob_from_url` functions.
 - `ResourceTypes`, and `Services` now have method `from_string` which takes parameters as a string.
 
 ## Version 12.0.0b4:

sdk/storage/azure-storage-blob/README.md

@@ -153,7 +153,7 @@ blob = BlobClient.from_connection_string("my_connection_string", container="myco
 
 with open("./BlockDestination.txt", "wb") as my_blob:
     blob_data = blob.download_blob()
-    my_blob.writelines(blob_data.content_as_bytes())
+    my_blob.writelines(blob_data.readall())
 ```
 
 Download a blob asynchronously.
@@ -165,7 +165,7 @@ blob = BlobClient.from_connection_string("my_connection_string", container="myco
 
 with open("./BlockDestination.txt", "wb") as my_blob:
     stream = await blob.download_blob()
-    data = await stream.content_as_bytes()
+    data = await stream.readall()
     my_blob.write(data)
 ```
 

sdk/storage/azure-storage-blob/azure/storage/blob/__init__.py

@@ -39,7 +39,6 @@
     CorsRule,
     ContainerProperties,
     BlobProperties,
-    BlobPrefix,
     LeaseProperties,
     ContentSettings,
     CopyProperties,
@@ -54,54 +53,9 @@
 __version__ = VERSION
 
 
-__all__ = [
-    'BlobServiceClient',
-    'ContainerClient',
-    'BlobClient',
-    'BlobType',
-    'LeaseClient',
-    'StorageErrorCode',
-    'UserDelegationKey',
-    'ExponentialRetry',
-    'LinearRetry',
-    'LocationMode',
-    'BlockState',
-    'StandardBlobTier',
-    'PremiumPageBlobTier',
-    'SequenceNumberAction',
-    'PublicAccess',
-    'BlobAnalyticsLogging',
-    'Metrics',
-    'RetentionPolicy',
-    'StaticWebsite',
-    'CorsRule',
-    'ContainerProperties',
-    'BlobProperties',
-    'BlobPrefix',
-    'LeaseProperties',
-    'ContentSettings',
-    'CopyProperties',
-    'BlobBlock',
-    'PageRange',
-    'AccessPolicy',
-    'ContainerSasPermissions',
-    'BlobSasPermissions',
-    'ResourceTypes',
-    'AccountSasPermissions',
-    'StorageStreamDownloader',
-    'CustomerProvidedEncryptionKey',
-    'RehydratePriority',
-    'generate_account_sas',
-    'generate_container_sas',
-    'generate_blob_sas'
-]
-
-
 def upload_blob_to_url(
         blob_url,  # type: str
         data,  # type: Union[Iterable[AnyStr], IO[AnyStr]]
-        max_concurrency=1,  # type: int
-        encoding='UTF-8', # type: str
         credential=None,  # type: Any
         **kwargs):
     # type: (...) -> dict[str, Any]
@@ -114,38 +68,50 @@ def upload_blob_to_url(
     :param data:
         The data to upload. This can be bytes, text, an iterable or a file-like object.
     :type data: bytes or str or Iterable
-    :param bool overwrite:
-        Whether the blob to be uploaded should overwrite the current data.
-        If True, upload_blob_to_url will overwrite any existing data. If set to False, the
-        operation will fail with a ResourceExistsError.
     :param credential:
         The credentials with which to authenticate. This is optional if the
         blob URL already has a SAS token. The value can be a SAS token string, an account
         shared access key, or an instance of a TokenCredentials class from azure.identity.
         If the URL already has a SAS token, specifying an explicit credential will take priority.
+    :keyword bool overwrite:
+        Whether the blob to be uploaded should overwrite the current data.
+        If True, upload_blob_to_url will overwrite any existing data. If set to False, the
+        operation will fail with a ResourceExistsError.
+    :keyword int max_concurrency:
+        The number of parallel connections with which to download.
+    :keyword int length:
+        Number of bytes to read from the stream. This is optional, but
+        should be supplied for optimal performance.
+    :keyword metadata:
+        Name-value pairs associated with the blob as metadata.
+    :type metadata: dict(str, str)
+    :keyword bool validate_content:
+        If true, calculates an MD5 hash for each chunk of the blob. The storage
+        service checks the hash of the content that has arrived with the hash
+        that was sent. This is primarily valuable for detecting bitflips on
+        the wire if using http instead of https as https (the default) will
+        already validate. Note that this MD5 hash is not stored with the
+        blob. Also note that if enabled, the memory-efficient upload algorithm
+        will not be used, because computing the MD5 hash requires buffering
+        entire blocks, and doing so defeats the purpose of the memory-efficient algorithm.
+    :keyword str encoding:
+        Encoding to use if text is supplied as input. Defaults to UTF-8.
     :returns: Blob-updated property dict (Etag and last modified)
     :rtype: dict(str, Any)
     """
     with BlobClient.from_blob_url(blob_url, credential=credential) as client:
-        return client.upload_blob(
-            data=data,
-            blob_type=BlobType.BlockBlob,
-            max_concurrency=max_concurrency,
-            encoding=encoding,
-            **kwargs)
+        return client.upload_blob(data=data, blob_type=BlobType.BlockBlob, **kwargs)
 
 
-def _download_to_stream(client, handle, max_concurrency, **kwargs):
+def _download_to_stream(client, handle, **kwargs):
     """Download data to specified open file-handle."""
     stream = client.download_blob(**kwargs)
-    stream.download_to_stream(handle, max_concurrency=max_concurrency)
+    stream.readinto(handle)
 
 
 def download_blob_from_url(
         blob_url,  # type: str
         output,  # type: str
-        overwrite=False,  # type: bool
-        max_concurrency=1,  # type: int
         credential=None,  # type: Any
         **kwargs):
     # type: (...) -> None
@@ -157,23 +123,85 @@ def download_blob_from_url(
         Where the data should be downloaded to. This could be either a file path to write to,
         or an open IO handle to write to.
     :type output: str or writable stream.
-    :param bool overwrite:
-        Whether the local file should be overwritten if it already exists. The default value is
-        `False` - in which case a ValueError will be raised if the file already exists. If set to
-        `True`, an attempt will be made to write to the existing file. If a stream handle is passed
-        in, this value is ignored.
     :param credential:
         The credentials with which to authenticate. This is optional if the
         blob URL already has a SAS token or the blob is public. The value can be a SAS token string,
         an account shared access key, or an instance of a TokenCredentials class from azure.identity.
         If the URL already has a SAS token, specifying an explicit credential will take priority.
+    :keyword bool overwrite:
+        Whether the local file should be overwritten if it already exists. The default value is
+        `False` - in which case a ValueError will be raised if the file already exists. If set to
+        `True`, an attempt will be made to write to the existing file. If a stream handle is passed
+        in, this value is ignored.
+    :keyword int max_concurrency:
+        The number of parallel connections with which to download.
+    :keyword int offset:
+        Start of byte range to use for downloading a section of the blob.
+        Must be set if length is provided.
+    :keyword int length:
+        Number of bytes to read from the stream. This is optional, but
+        should be supplied for optimal performance.
+    :keyword bool validate_content:
+        If true, calculates an MD5 hash for each chunk of the blob. The storage
+        service checks the hash of the content that has arrived with the hash
+        that was sent. This is primarily valuable for detecting bitflips on
+        the wire if using http instead of https as https (the default) will
+        already validate. Note that this MD5 hash is not stored with the
+        blob. Also note that if enabled, the memory-efficient upload algorithm
+        will not be used, because computing the MD5 hash requires buffering
+        entire blocks, and doing so defeats the purpose of the memory-efficient algorithm.
     :rtype: None
     """
+    overwrite = kwargs.pop('overwrite', False)
     with BlobClient.from_blob_url(blob_url, credential=credential) as client:
         if hasattr(output, 'write'):
-            _download_to_stream(client, output, max_concurrency, **kwargs)
+            _download_to_stream(client, output, **kwargs)
         else:
             if not overwrite and os.path.isfile(output):
                 raise ValueError("The file '{}' already exists.".format(output))
             with open(output, 'wb') as file_handle:
-                _download_to_stream(client, file_handle, max_concurrency, **kwargs)
+                _download_to_stream(client, file_handle, **kwargs)
+
+
+__all__ = [
+    'upload_blob_to_url',
+    'download_blob_from_url',
+    'BlobServiceClient',
+    'ContainerClient',
+    'BlobClient',
+    'BlobType',
+    'LeaseClient',
+    'StorageErrorCode',
+    'UserDelegationKey',
+    'ExponentialRetry',
+    'LinearRetry',
+    'LocationMode',
+    'BlockState',
+    'StandardBlobTier',
+    'PremiumPageBlobTier',
+    'SequenceNumberAction',
+    'PublicAccess',
+    'BlobAnalyticsLogging',
+    'Metrics',
+    'RetentionPolicy',
+    'StaticWebsite',
+    'CorsRule',
+    'ContainerProperties',
+    'BlobProperties',
+    'LeaseProperties',
+    'ContentSettings',
+    'CopyProperties',
+    'BlobBlock',
+    'PageRange',
+    'AccessPolicy',
+    'ContainerSasPermissions',
+    'BlobSasPermissions',
+    'ResourceTypes',
+    'AccountSasPermissions',
+    'StorageStreamDownloader',
+    'CustomerProvidedEncryptionKey',
+    'RehydratePriority',
+    'generate_account_sas',
+    'generate_container_sas',
+    'generate_blob_sas'
+]