Feature/storage preview5

eng/tox/import_all.py

@@ -16,13 +16,16 @@
 # keyvault has dependency issue when loading private module _BearerTokenCredentialPolicyBase from azure.core.pipeline.policies
 # azure.core.tracing.opencensus and azure.eventhub.checkpointstoreblob.aio are skipped due to a known issue in loading azure.core.tracing.opencensus
 excluded_packages = [
-    "azure.core.tracing.opencensus", 
+    "azure.core.tracing.opencensus",
     "azure.eventhub.checkpointstoreblob.aio",
     "azure.identity",
     "azure.keyvault.certificates", # Github issue 7879
     "azure.keyvault.keys", # Github issue 7879
     "azure.keyvault.secrets", # Github issue 7879
     "azure.appconfiguration", # Github issue 7879. revisit and close after azure-core POST b4 is released.
+    "azure.storage.blob", # Github issue 7879.
+    "azure.storage.file", # Github issue 7879.
+    "azure.storage.queue", # Github issue 7879.
     "azure",
     "azure-mgmt"]
 
@@ -59,4 +62,4 @@ def should_run_import_all(package_name):
         logging.info("Verified module dependency, no issues found")
     else:
         pass
-        logging.error("Package {} is excluded from dependency check".format(package_name)) 
+        logging.error("Package {} is excluded from dependency check".format(package_name))

sdk/storage/azure-storage-blob/HISTORY.md

@@ -1,5 +1,36 @@
 # Change Log azure-storage-blob
 
+## Version 12.0.0b5:
+
+**Breaking changes**
+
+- `set_container_access_policy` has required parameter `signed_identifiers`.
+- `NoRetry` policy has been removed. Use keyword argument `retry_total=0` for no retries. 
+- `StorageStreamDownloader` is no longer iterable. To iterate over the blob data stream, use `StorageStreamDownloader.chunks`.
+- The public attributes of `StorageStreamDownloader` have been limited to:
+  - `name` (str): The name of the blob.
+  - `container` (str): The container the blob is being downloaded from.
+  - `properties` (`BlobProperties`): The properties of the blob.
+  - `size` (int): The size of the download. Either the total blob size, or the length of a subsection if sepcified. Previously called `download_size`.
+- `StorageStreamDownloader` now has new functions:
+  - `readall()`: Reads the complete download stream, returning bytes. This replaces the functions `content_as_bytes` and `content_as_text` which have been deprecated.
+  - `readinto(stream)`: Download the complete stream into the supplied writable stream, returning the number of bytes written. This replaces the function `download_to_stream` which has been deprecated.
+- Module level functions `upload_blob_to_url` and `download_blob_from_url` functions options are now keyword only:
+  - `overwrite`
+  - `max_concurrency`
+  - `encoding`
+- Removed types that were accidentally exposed from two modules. Only `BlobServiceClient`, `ContainerClient`, 
+`BlobClient` and `LeaseClient` should be imported from azure.storage.blob.aio
+- `Logging` has been renamed to `BlobAnalyticsLogging`.
+- Client and model files have been made internal. Users should import from the top level modules `azure.storage.blob` and `azure.storage.blob.aio` only.
+- All operations that take Etag conditional parameters (`if_match` and `if_none_match`) now take explicit `etag` and `match_condition` parameters, where `etag` is the Etag value, and `match_condition` is an instance of `azure.core.MatchConditions`.
+- The `generate_shared_access_signature` methods on each of `BlobServiceClient`, `ContainerClient` and `BlobClient` have been replaced by module level functions `generate_account_sas`, `generate_container_sas` and `generate_blob_sas`.
+- The batch APIs now have an additional keyword only argument `raise_on_any_failure` which defaults to True. This will raise an error even if there's a partial batch failure.
+
+**New features**
+
+- Added async module-level `upload_blob_to_url` and `download_blob_from_url` functions.
+- `ResourceTypes`, and `Services` now have method `from_string` which takes parameters as a string.
 
 ## Version 12.0.0b4:
 
@@ -37,6 +68,8 @@ changed include:
 - Add support for set_premium_page_blob_tier_blobs to ContainerClient (Python 3 only)
 - Added support to set rehydrate blob priority for Block Blob, including Set Standard Blob Tier/Copy Blob APIs
 - Added blob tier support for Block Blob, including Upload Blob/Commit Block List/Copy Blob APIs.
+- `AccountSasPermissions`, `BlobSasPermissions`, `ContainerSasPermissions` now have method `from_string` 
+which takes parameters as a string.
 
 **Fixes and improvements**
 - Downloading page blobs now take advantage of their sparseness.

sdk/storage/azure-storage-blob/README.md

@@ -153,7 +153,7 @@ blob = BlobClient.from_connection_string("my_connection_string", container="myco
 
 with open("./BlockDestination.txt", "wb") as my_blob:
     blob_data = blob.download_blob()
-    my_blob.writelines(blob_data.content_as_bytes())
+    my_blob.writelines(blob_data.readall())
 ```
 
 Download a blob asynchronously.
@@ -165,7 +165,7 @@ blob = BlobClient.from_connection_string("my_connection_string", container="myco
 
 with open("./BlockDestination.txt", "wb") as my_blob:
     stream = await blob.download_blob()
-    data = await stream.content_as_bytes()
+    data = await stream.readall()
     my_blob.write(data)
 ```
 

sdk/storage/azure-storage-blob/azure/storage/blob/__init__.py

@@ -6,14 +6,16 @@
 
 import os
 
-from typing import Union, Iterable, AnyStr, IO, Any # pylint: disable=unused-import
-from .version import VERSION
-from .blob_client import BlobClient
-from .container_client import ContainerClient
-from .blob_service_client import BlobServiceClient
-from .lease import LeaseClient
-from .download import StorageStreamDownloader
-from ._shared.policies import ExponentialRetry, LinearRetry, NoRetry
+from typing import Union, Iterable, AnyStr, IO, Any, Dict  # pylint: disable=unused-import
+from ._version import VERSION
+from ._blob_client import BlobClient
+from ._container_client import ContainerClient
+from ._blob_service_client import BlobServiceClient
+from ._lease import LeaseClient
+from ._download import StorageStreamDownloader
+from ._shared_access_signature import generate_account_sas, generate_container_sas, generate_blob_sas
+from ._shared.policies import ExponentialRetry, LinearRetry
+from ._shared.response_handlers import PartialBatchErrorException
 from ._shared.models import(
     LocationMode,
     ResourceTypes,
@@ -24,23 +26,20 @@
 from ._generated.models import (
     RehydratePriority
 )
-from .models import (
+from ._models import (
     BlobType,
     BlockState,
     StandardBlobTier,
     PremiumPageBlobTier,
     SequenceNumberAction,
     PublicAccess,
-    Logging,
+    BlobAnalyticsLogging,
     Metrics,
     RetentionPolicy,
     StaticWebsite,
     CorsRule,
     ContainerProperties,
-    ContainerPropertiesPaged,
     BlobProperties,
-    BlobPropertiesPaged,
-    BlobPrefix,
     LeaseProperties,
     ContentSettings,
     CopyProperties,
@@ -55,57 +54,12 @@
 __version__ = VERSION
 
 
-__all__ = [
-    'BlobServiceClient',
-    'ContainerClient',
-    'BlobClient',
-    'BlobType',
-    'LeaseClient',
-    'StorageErrorCode',
-    'UserDelegationKey',
-    'ExponentialRetry',
-    'LinearRetry',
-    'NoRetry',
-    'LocationMode',
-    'BlockState',
-    'StandardBlobTier',
-    'PremiumPageBlobTier',
-    'SequenceNumberAction',
-    'PublicAccess',
-    'Logging',
-    'Metrics',
-    'RetentionPolicy',
-    'StaticWebsite',
-    'CorsRule',
-    'ContainerProperties',
-    'ContainerPropertiesPaged',
-    'BlobProperties',
-    'BlobPropertiesPaged',
-    'BlobPrefix',
-    'LeaseProperties',
-    'ContentSettings',
-    'CopyProperties',
-    'BlobBlock',
-    'PageRange',
-    'AccessPolicy',
-    'ContainerSasPermissions',
-    'BlobSasPermissions',
-    'ResourceTypes',
-    'AccountSasPermissions',
-    'StorageStreamDownloader',
-    'CustomerProvidedEncryptionKey',
-    'RehydratePriority'
-]
-
-
 def upload_blob_to_url(
         blob_url,  # type: str
         data,  # type: Union[Iterable[AnyStr], IO[AnyStr]]
-        max_concurrency=1,  # type: int
-        encoding='UTF-8', # type: str
         credential=None,  # type: Any
         **kwargs):
-    # type: (...) -> dict[str, Any]
+    # type: (...) -> Dict[str, Any]
     """Upload data to a given URL
 
     The data will be uploaded as a block blob.
@@ -115,38 +69,50 @@ def upload_blob_to_url(
     :param data:
         The data to upload. This can be bytes, text, an iterable or a file-like object.
     :type data: bytes or str or Iterable
-    :param bool overwrite:
-        Whether the blob to be uploaded should overwrite the current data.
-        If True, upload_blob_to_url will overwrite any existing data. If set to False, the
-        operation will fail with a ResourceExistsError.
     :param credential:
         The credentials with which to authenticate. This is optional if the
         blob URL already has a SAS token. The value can be a SAS token string, an account
         shared access key, or an instance of a TokenCredentials class from azure.identity.
         If the URL already has a SAS token, specifying an explicit credential will take priority.
+    :keyword bool overwrite:
+        Whether the blob to be uploaded should overwrite the current data.
+        If True, upload_blob_to_url will overwrite any existing data. If set to False, the
+        operation will fail with a ResourceExistsError.
+    :keyword int max_concurrency:
+        The number of parallel connections with which to download.
+    :keyword int length:
+        Number of bytes to read from the stream. This is optional, but
+        should be supplied for optimal performance.
+    :keyword metadata:
+        Name-value pairs associated with the blob as metadata.
+    :type metadata: dict(str, str)
+    :keyword bool validate_content:
+        If true, calculates an MD5 hash for each chunk of the blob. The storage
+        service checks the hash of the content that has arrived with the hash
+        that was sent. This is primarily valuable for detecting bitflips on
+        the wire if using http instead of https as https (the default) will
+        already validate. Note that this MD5 hash is not stored with the
+        blob. Also note that if enabled, the memory-efficient upload algorithm
+        will not be used, because computing the MD5 hash requires buffering
+        entire blocks, and doing so defeats the purpose of the memory-efficient algorithm.
+    :keyword str encoding:
+        Encoding to use if text is supplied as input. Defaults to UTF-8.
     :returns: Blob-updated property dict (Etag and last modified)
     :rtype: dict(str, Any)
     """
     with BlobClient.from_blob_url(blob_url, credential=credential) as client:
-        return client.upload_blob(
-            data=data,
-            blob_type=BlobType.BlockBlob,
-            max_concurrency=max_concurrency,
-            encoding=encoding,
-            **kwargs)
+        return client.upload_blob(data=data, blob_type=BlobType.BlockBlob, **kwargs)
 
 
-def _download_to_stream(client, handle, max_concurrency, **kwargs):
+def _download_to_stream(client, handle, **kwargs):
     """Download data to specified open file-handle."""
     stream = client.download_blob(**kwargs)
-    stream.download_to_stream(handle, max_concurrency=max_concurrency)
+    stream.readinto(handle)
 
 
 def download_blob_from_url(
         blob_url,  # type: str
         output,  # type: str
-        overwrite=False,  # type: bool
-        max_concurrency=1,  # type: int
         credential=None,  # type: Any
         **kwargs):
     # type: (...) -> None
@@ -158,23 +124,86 @@ def download_blob_from_url(
         Where the data should be downloaded to. This could be either a file path to write to,
         or an open IO handle to write to.
     :type output: str or writable stream.
-    :param bool overwrite:
-        Whether the local file should be overwritten if it already exists. The default value is
-        `False` - in which case a ValueError will be raised if the file already exists. If set to
-        `True`, an attempt will be made to write to the existing file. If a stream handle is passed
-        in, this value is ignored.
     :param credential:
         The credentials with which to authenticate. This is optional if the
         blob URL already has a SAS token or the blob is public. The value can be a SAS token string,
         an account shared access key, or an instance of a TokenCredentials class from azure.identity.
         If the URL already has a SAS token, specifying an explicit credential will take priority.
+    :keyword bool overwrite:
+        Whether the local file should be overwritten if it already exists. The default value is
+        `False` - in which case a ValueError will be raised if the file already exists. If set to
+        `True`, an attempt will be made to write to the existing file. If a stream handle is passed
+        in, this value is ignored.
+    :keyword int max_concurrency:
+        The number of parallel connections with which to download.
+    :keyword int offset:
+        Start of byte range to use for downloading a section of the blob.
+        Must be set if length is provided.
+    :keyword int length:
+        Number of bytes to read from the stream. This is optional, but
+        should be supplied for optimal performance.
+    :keyword bool validate_content:
+        If true, calculates an MD5 hash for each chunk of the blob. The storage
+        service checks the hash of the content that has arrived with the hash
+        that was sent. This is primarily valuable for detecting bitflips on
+        the wire if using http instead of https as https (the default) will
+        already validate. Note that this MD5 hash is not stored with the
+        blob. Also note that if enabled, the memory-efficient upload algorithm
+        will not be used, because computing the MD5 hash requires buffering
+        entire blocks, and doing so defeats the purpose of the memory-efficient algorithm.
     :rtype: None
     """
+    overwrite = kwargs.pop('overwrite', False)
     with BlobClient.from_blob_url(blob_url, credential=credential) as client:
         if hasattr(output, 'write'):
-            _download_to_stream(client, output, max_concurrency, **kwargs)
+            _download_to_stream(client, output, **kwargs)
         else:
             if not overwrite and os.path.isfile(output):
                 raise ValueError("The file '{}' already exists.".format(output))
             with open(output, 'wb') as file_handle:
-                _download_to_stream(client, file_handle, max_concurrency, **kwargs)
+                _download_to_stream(client, file_handle, **kwargs)
+
+
+__all__ = [
+    'upload_blob_to_url',
+    'download_blob_from_url',
+    'BlobServiceClient',
+    'ContainerClient',
+    'BlobClient',
+    'BlobType',
+    'LeaseClient',
+    'StorageErrorCode',
+    'UserDelegationKey',
+    'ExponentialRetry',
+    'LinearRetry',
+    'LocationMode',
+    'BlockState',
+    'StandardBlobTier',
+    'PremiumPageBlobTier',
+    'SequenceNumberAction',
+    'PublicAccess',
+    'BlobAnalyticsLogging',
+    'Metrics',
+    'RetentionPolicy',
+    'StaticWebsite',
+    'CorsRule',
+    'ContainerProperties',
+    'BlobProperties',
+    'LeaseProperties',
+    'ContentSettings',
+    'CopyProperties',
+    'BlobBlock',
+    'PageRange',
+    'AccessPolicy',
+    'ContainerSasPermissions',
+    'BlobSasPermissions',
+    'ResourceTypes',
+    'AccountSasPermissions',
+    'StorageStreamDownloader',
+    'CustomerProvidedEncryptionKey',
+    'RehydratePriority',
+    'generate_account_sas',
+    'generate_container_sas',
+    'generate_blob_sas',
+    'PartialBatchErrorException'
+]