From 79096e082b32d047ee669e553992ee09f12f8617 Mon Sep 17 00:00:00 2001 From: antisch Date: Mon, 21 Oct 2019 13:13:24 -0700 Subject: [PATCH 1/7] Bumped version --- sdk/storage/azure-storage-blob/azure/storage/blob/version.py | 2 +- sdk/storage/azure-storage-file/HISTORY.md | 2 +- sdk/storage/azure-storage-file/azure/storage/file/version.py | 2 +- sdk/storage/azure-storage-queue/azure/storage/queue/version.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/version.py b/sdk/storage/azure-storage-blob/azure/storage/blob/version.py index d8737a7b65ef..63dfa66aca46 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/version.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/version.py @@ -4,4 +4,4 @@ # license information. # -------------------------------------------------------------------------- -VERSION = "12.0.0b4" +VERSION = "12.0.0" diff --git a/sdk/storage/azure-storage-file/HISTORY.md b/sdk/storage/azure-storage-file/HISTORY.md index 2db50ac13e99..f565ff287ccd 100644 --- a/sdk/storage/azure-storage-file/HISTORY.md +++ b/sdk/storage/azure-storage-file/HISTORY.md @@ -1,6 +1,6 @@ # Change Log azure-storage-file -## Version 12.0.0: +## Version 12.0.0b5: **Breaking changes** diff --git a/sdk/storage/azure-storage-file/azure/storage/file/version.py b/sdk/storage/azure-storage-file/azure/storage/file/version.py index 5cd1ed1b1cef..ce4ecebdf250 100644 --- a/sdk/storage/azure-storage-file/azure/storage/file/version.py +++ b/sdk/storage/azure-storage-file/azure/storage/file/version.py @@ -4,4 +4,4 @@ # license information. # -------------------------------------------------------------------------- -VERSION = '12.0.0b4' +VERSION = '12.0.0b5' diff --git a/sdk/storage/azure-storage-queue/azure/storage/queue/version.py b/sdk/storage/azure-storage-queue/azure/storage/queue/version.py index cc32a533d2ce..41140c9c724b 100644 --- a/sdk/storage/azure-storage-queue/azure/storage/queue/version.py +++ b/sdk/storage/azure-storage-queue/azure/storage/queue/version.py @@ -9,4 +9,4 @@ # regenerated. # -------------------------------------------------------------------------- -VERSION = "12.0.0b4" +VERSION = "12.0.0" From f01f971eb3304f7e2f567e2b7ba20ee9a83846e1 Mon Sep 17 00:00:00 2001 From: antisch Date: Mon, 21 Oct 2019 14:02:32 -0700 Subject: [PATCH 2/7] Docs tweak --- .../azure/storage/blob/container_client.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/container_client.py b/sdk/storage/azure-storage-blob/azure/storage/blob/container_client.py index cecaf7f3466e..8181d4467782 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/container_client.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/container_client.py @@ -692,7 +692,7 @@ def upload_blob( :param metadata: Name-value pairs associated with the blob as metadata. :type metadata: dict(str, str) - :param str encoding: + :keyword str encoding: Defaults to UTF-8. :keyword bool overwrite: Whether the blob to be uploaded should overwrite the current data. If True, upload_blob will silently overwrite the existing data. If set to False, the @@ -807,10 +807,6 @@ def delete_blob( :param blob: The blob with which to interact. If specified, this value will override a blob value specified in the blob URL. :type blob: str or ~azure.storage.blob.BlobProperties - :param str delete_snapshots: - Required if the blob has associated snapshots. Values include: - - "only": Deletes only the blobs snapshots. - - "include": Deletes the blob along with all snapshots. :param str delete_snapshots: Required if the blob has associated snapshots. Values include: - "only": Deletes only the blobs snapshots. From baf7e7fa8106ca4a36d910ab2d51e6dc63068f49 Mon Sep 17 00:00:00 2001 From: antisch Date: Mon, 21 Oct 2019 14:23:16 -0700 Subject: [PATCH 3/7] Re-bumped version --- sdk/storage/azure-storage-blob/HISTORY.md | 2 +- sdk/storage/azure-storage-blob/azure/storage/blob/version.py | 2 +- sdk/storage/azure-storage-queue/HISTORY.md | 2 +- sdk/storage/azure-storage-queue/azure/storage/queue/version.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sdk/storage/azure-storage-blob/HISTORY.md b/sdk/storage/azure-storage-blob/HISTORY.md index b4567c2f91b4..fdc9fe23eb9d 100644 --- a/sdk/storage/azure-storage-blob/HISTORY.md +++ b/sdk/storage/azure-storage-blob/HISTORY.md @@ -1,6 +1,6 @@ # Change Log azure-storage-blob -## Version 12.0.0: +## Version 12.0.0b5: **Breaking changes** diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/version.py b/sdk/storage/azure-storage-blob/azure/storage/blob/version.py index 63dfa66aca46..58517ac93b1a 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/version.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/version.py @@ -4,4 +4,4 @@ # license information. # -------------------------------------------------------------------------- -VERSION = "12.0.0" +VERSION = "12.0.0b5" diff --git a/sdk/storage/azure-storage-queue/HISTORY.md b/sdk/storage/azure-storage-queue/HISTORY.md index 5ff6703625bb..b84cc4ff5fbe 100644 --- a/sdk/storage/azure-storage-queue/HISTORY.md +++ b/sdk/storage/azure-storage-queue/HISTORY.md @@ -1,6 +1,6 @@ # Change Log azure-storage-queue -## Version 12.0.0: +## Version 12.0.0b5: **Breaking changes** diff --git a/sdk/storage/azure-storage-queue/azure/storage/queue/version.py b/sdk/storage/azure-storage-queue/azure/storage/queue/version.py index 41140c9c724b..6ba1d06c4c45 100644 --- a/sdk/storage/azure-storage-queue/azure/storage/queue/version.py +++ b/sdk/storage/azure-storage-queue/azure/storage/queue/version.py @@ -9,4 +9,4 @@ # regenerated. # -------------------------------------------------------------------------- -VERSION = "12.0.0" +VERSION = "12.0.0b5" From c71cd5a4377ab1e96c880da42d09780d45b42f4d Mon Sep 17 00:00:00 2001 From: antisch Date: Mon, 21 Oct 2019 14:24:45 -0700 Subject: [PATCH 4/7] Added 3.8 classifier --- sdk/storage/azure-storage-blob/setup.py | 1 + sdk/storage/azure-storage-file/setup.py | 1 + sdk/storage/azure-storage-queue/setup.py | 1 + 3 files changed, 3 insertions(+) diff --git a/sdk/storage/azure-storage-blob/setup.py b/sdk/storage/azure-storage-blob/setup.py index 63becf5ff97b..2bd738fb9d93 100644 --- a/sdk/storage/azure-storage-blob/setup.py +++ b/sdk/storage/azure-storage-blob/setup.py @@ -79,6 +79,7 @@ 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', 'License :: OSI Approved :: MIT License', ], zip_safe=False, diff --git a/sdk/storage/azure-storage-file/setup.py b/sdk/storage/azure-storage-file/setup.py index 24ce7e90317c..636d78eb0f55 100644 --- a/sdk/storage/azure-storage-file/setup.py +++ b/sdk/storage/azure-storage-file/setup.py @@ -67,6 +67,7 @@ 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', 'License :: OSI Approved :: MIT License', ], zip_safe=False, diff --git a/sdk/storage/azure-storage-queue/setup.py b/sdk/storage/azure-storage-queue/setup.py index 0a71c9ad7af8..eda59e2a67c3 100644 --- a/sdk/storage/azure-storage-queue/setup.py +++ b/sdk/storage/azure-storage-queue/setup.py @@ -67,6 +67,7 @@ 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', 'License :: OSI Approved :: MIT License', ], zip_safe=False, From 1856a5312113667036f1b635ac4916493ba5c7dc Mon Sep 17 00:00:00 2001 From: antisch Date: Mon, 21 Oct 2019 14:52:21 -0700 Subject: [PATCH 5/7] Added proxy policy to pipeline --- .../azure-storage-blob/azure/storage/blob/_shared/base_client.py | 1 + .../azure/storage/blob/_shared/base_client_async.py | 1 + 2 files changed, 2 insertions(+) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/base_client.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/base_client.py index de86215853d1..5999a495b97f 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/base_client.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/base_client.py @@ -178,6 +178,7 @@ def _create_pipeline(self, credential, **kwargs): policies = [ QueueMessagePolicy(), config.headers_policy, + config.proxy_policy, config.user_agent_policy, StorageContentValidation(), StorageRequestHook(**kwargs), diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/base_client_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/base_client_async.py index 8d2265ec1181..a64531d415ac 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/base_client_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/base_client_async.py @@ -82,6 +82,7 @@ def _create_pipeline(self, credential, **kwargs): policies = [ QueueMessagePolicy(), config.headers_policy, + config.proxy_policy, config.user_agent_policy, StorageContentValidation(), StorageRequestHook(**kwargs), From fede71c642a98969786d2619f1ebbe4cb1ea6cbd Mon Sep 17 00:00:00 2001 From: antisch Date: Mon, 21 Oct 2019 14:52:59 -0700 Subject: [PATCH 6/7] Synced base clients --- .../azure/storage/file/_shared/base_client.py | 19 +++++++++++++--- .../storage/file/_shared/base_client_async.py | 22 +++++++++++++++---- .../storage/queue/_shared/base_client.py | 19 +++++++++++++--- .../queue/_shared/base_client_async.py | 22 +++++++++++++++---- 4 files changed, 68 insertions(+), 14 deletions(-) diff --git a/sdk/storage/azure-storage-file/azure/storage/file/_shared/base_client.py b/sdk/storage/azure-storage-file/azure/storage/file/_shared/base_client.py index e26ba3fda37f..5999a495b97f 100644 --- a/sdk/storage/azure-storage-file/azure/storage/file/_shared/base_client.py +++ b/sdk/storage/azure-storage-file/azure/storage/file/_shared/base_client.py @@ -28,7 +28,7 @@ from azure.core.configuration import Configuration from azure.core.exceptions import HttpResponseError from azure.core.pipeline import Pipeline -from azure.core.pipeline.transport import RequestsTransport, HttpTransport # pylint: disable=unused-import +from azure.core.pipeline.transport import RequestsTransport, HttpTransport from azure.core.pipeline.policies import ( RedirectPolicy, ContentDecodePolicy, @@ -54,7 +54,7 @@ ExponentialRetry, ) from .._generated.models import StorageErrorException -from .response_handlers import process_storage_error +from .response_handlers import process_storage_error, PartialBatchErrorException _LOGGER = logging.getLogger(__name__) @@ -178,6 +178,7 @@ def _create_pipeline(self, credential, **kwargs): policies = [ QueueMessagePolicy(), config.headers_policy, + config.proxy_policy, config.user_agent_policy, StorageContentValidation(), StorageRequestHook(**kwargs), @@ -199,6 +200,8 @@ def _batch_send( ): """Given a series of request, do a Storage batch call. """ + # Pop it here, so requests doesn't feel bad about additional kwarg + raise_on_any_failure = kwargs.pop("raise_on_any_failure", True) request = self._client._client.post( # pylint: disable=protected-access url='https://{}/?comp=batch'.format(self.primary_hostname), headers={ @@ -222,7 +225,17 @@ def _batch_send( try: if response.status_code not in [202]: raise HttpResponseError(response=response) - return response.parts() + parts = response.parts() + if raise_on_any_failure: + parts = list(response.parts()) + if any(p for p in parts if not 200 <= p.status_code < 300): + error = PartialBatchErrorException( + message="There is a partial failure in the batch operation.", + response=response, parts=parts + ) + raise error + return iter(parts) + return parts except StorageErrorException as error: process_storage_error(error) diff --git a/sdk/storage/azure-storage-file/azure/storage/file/_shared/base_client_async.py b/sdk/storage/azure-storage-file/azure/storage/file/_shared/base_client_async.py index 4d3fb4c19286..a64531d415ac 100644 --- a/sdk/storage/azure-storage-file/azure/storage/file/_shared/base_client_async.py +++ b/sdk/storage/azure-storage-file/azure/storage/file/_shared/base_client_async.py @@ -9,8 +9,8 @@ TYPE_CHECKING ) import logging - from azure.core.pipeline import AsyncPipeline +from azure.core.async_paging import AsyncList from azure.core.exceptions import HttpResponseError from azure.core.pipeline.policies import ( ContentDecodePolicy, @@ -34,13 +34,12 @@ from .policies_async import AsyncStorageResponseHook from .._generated.models import StorageErrorException -from .response_handlers import process_storage_error +from .response_handlers import process_storage_error, PartialBatchErrorException if TYPE_CHECKING: from azure.core.pipeline import Pipeline from azure.core.pipeline.transport import HttpRequest from azure.core.configuration import Configuration - _LOGGER = logging.getLogger(__name__) @@ -83,6 +82,7 @@ def _create_pipeline(self, credential, **kwargs): policies = [ QueueMessagePolicy(), config.headers_policy, + config.proxy_policy, config.user_agent_policy, StorageContentValidation(), StorageRequestHook(**kwargs), @@ -104,6 +104,8 @@ async def _batch_send( ): """Given a series of request, do a Storage batch call. """ + # Pop it here, so requests doesn't feel bad about additional kwarg + raise_on_any_failure = kwargs.pop("raise_on_any_failure", True) request = self._client._client.post( # pylint: disable=protected-access url='https://{}/?comp=batch'.format(self.primary_hostname), headers={ @@ -127,7 +129,19 @@ async def _batch_send( try: if response.status_code not in [202]: raise HttpResponseError(response=response) - return response.parts() # Return an AsyncIterator + parts = response.parts() # Return an AsyncIterator + if raise_on_any_failure: + parts_list = [] + async for part in parts: + parts_list.append(part) + if any(p for p in parts_list if not 200 <= p.status_code < 300): + error = PartialBatchErrorException( + message="There is a partial failure in the batch operation.", + response=response, parts=parts_list + ) + raise error + return AsyncList(parts_list) + return parts except StorageErrorException as error: process_storage_error(error) diff --git a/sdk/storage/azure-storage-queue/azure/storage/queue/_shared/base_client.py b/sdk/storage/azure-storage-queue/azure/storage/queue/_shared/base_client.py index e26ba3fda37f..5999a495b97f 100644 --- a/sdk/storage/azure-storage-queue/azure/storage/queue/_shared/base_client.py +++ b/sdk/storage/azure-storage-queue/azure/storage/queue/_shared/base_client.py @@ -28,7 +28,7 @@ from azure.core.configuration import Configuration from azure.core.exceptions import HttpResponseError from azure.core.pipeline import Pipeline -from azure.core.pipeline.transport import RequestsTransport, HttpTransport # pylint: disable=unused-import +from azure.core.pipeline.transport import RequestsTransport, HttpTransport from azure.core.pipeline.policies import ( RedirectPolicy, ContentDecodePolicy, @@ -54,7 +54,7 @@ ExponentialRetry, ) from .._generated.models import StorageErrorException -from .response_handlers import process_storage_error +from .response_handlers import process_storage_error, PartialBatchErrorException _LOGGER = logging.getLogger(__name__) @@ -178,6 +178,7 @@ def _create_pipeline(self, credential, **kwargs): policies = [ QueueMessagePolicy(), config.headers_policy, + config.proxy_policy, config.user_agent_policy, StorageContentValidation(), StorageRequestHook(**kwargs), @@ -199,6 +200,8 @@ def _batch_send( ): """Given a series of request, do a Storage batch call. """ + # Pop it here, so requests doesn't feel bad about additional kwarg + raise_on_any_failure = kwargs.pop("raise_on_any_failure", True) request = self._client._client.post( # pylint: disable=protected-access url='https://{}/?comp=batch'.format(self.primary_hostname), headers={ @@ -222,7 +225,17 @@ def _batch_send( try: if response.status_code not in [202]: raise HttpResponseError(response=response) - return response.parts() + parts = response.parts() + if raise_on_any_failure: + parts = list(response.parts()) + if any(p for p in parts if not 200 <= p.status_code < 300): + error = PartialBatchErrorException( + message="There is a partial failure in the batch operation.", + response=response, parts=parts + ) + raise error + return iter(parts) + return parts except StorageErrorException as error: process_storage_error(error) diff --git a/sdk/storage/azure-storage-queue/azure/storage/queue/_shared/base_client_async.py b/sdk/storage/azure-storage-queue/azure/storage/queue/_shared/base_client_async.py index 4d3fb4c19286..a64531d415ac 100644 --- a/sdk/storage/azure-storage-queue/azure/storage/queue/_shared/base_client_async.py +++ b/sdk/storage/azure-storage-queue/azure/storage/queue/_shared/base_client_async.py @@ -9,8 +9,8 @@ TYPE_CHECKING ) import logging - from azure.core.pipeline import AsyncPipeline +from azure.core.async_paging import AsyncList from azure.core.exceptions import HttpResponseError from azure.core.pipeline.policies import ( ContentDecodePolicy, @@ -34,13 +34,12 @@ from .policies_async import AsyncStorageResponseHook from .._generated.models import StorageErrorException -from .response_handlers import process_storage_error +from .response_handlers import process_storage_error, PartialBatchErrorException if TYPE_CHECKING: from azure.core.pipeline import Pipeline from azure.core.pipeline.transport import HttpRequest from azure.core.configuration import Configuration - _LOGGER = logging.getLogger(__name__) @@ -83,6 +82,7 @@ def _create_pipeline(self, credential, **kwargs): policies = [ QueueMessagePolicy(), config.headers_policy, + config.proxy_policy, config.user_agent_policy, StorageContentValidation(), StorageRequestHook(**kwargs), @@ -104,6 +104,8 @@ async def _batch_send( ): """Given a series of request, do a Storage batch call. """ + # Pop it here, so requests doesn't feel bad about additional kwarg + raise_on_any_failure = kwargs.pop("raise_on_any_failure", True) request = self._client._client.post( # pylint: disable=protected-access url='https://{}/?comp=batch'.format(self.primary_hostname), headers={ @@ -127,7 +129,19 @@ async def _batch_send( try: if response.status_code not in [202]: raise HttpResponseError(response=response) - return response.parts() # Return an AsyncIterator + parts = response.parts() # Return an AsyncIterator + if raise_on_any_failure: + parts_list = [] + async for part in parts: + parts_list.append(part) + if any(p for p in parts_list if not 200 <= p.status_code < 300): + error = PartialBatchErrorException( + message="There is a partial failure in the batch operation.", + response=response, parts=parts_list + ) + raise error + return AsyncList(parts_list) + return parts except StorageErrorException as error: process_storage_error(error) From a5eedc214f3cdd509f2f6a9eff94ea383d910f38 Mon Sep 17 00:00:00 2001 From: antisch Date: Mon, 21 Oct 2019 15:20:28 -0700 Subject: [PATCH 7/7] Added batch exception --- .../azure/storage/file/_shared/response_handlers.py | 13 +++++++++++++ .../storage/queue/_shared/response_handlers.py | 13 +++++++++++++ 2 files changed, 26 insertions(+) diff --git a/sdk/storage/azure-storage-file/azure/storage/file/_shared/response_handlers.py b/sdk/storage/azure-storage-file/azure/storage/file/_shared/response_handlers.py index fbf9889d762c..ce625d589504 100644 --- a/sdk/storage/azure-storage-file/azure/storage/file/_shared/response_handlers.py +++ b/sdk/storage/azure-storage-file/azure/storage/file/_shared/response_handlers.py @@ -31,6 +31,19 @@ _LOGGER = logging.getLogger(__name__) +class PartialBatchErrorException(HttpResponseError): + """There is a partial failure in batch operations. + + :param message: The message of the exception. + :param response: Server response to be deserialized. + :param parts: A list of the parts in multipart response. + """ + + def __init__(self, message, response, parts): + self.parts = parts + super(PartialBatchErrorException, self).__init__(message=message, response=response) + + def parse_length_from_content_range(content_range): ''' Parses the blob length from the content range header: bytes 1-3/65537 diff --git a/sdk/storage/azure-storage-queue/azure/storage/queue/_shared/response_handlers.py b/sdk/storage/azure-storage-queue/azure/storage/queue/_shared/response_handlers.py index fbf9889d762c..ce625d589504 100644 --- a/sdk/storage/azure-storage-queue/azure/storage/queue/_shared/response_handlers.py +++ b/sdk/storage/azure-storage-queue/azure/storage/queue/_shared/response_handlers.py @@ -31,6 +31,19 @@ _LOGGER = logging.getLogger(__name__) +class PartialBatchErrorException(HttpResponseError): + """There is a partial failure in batch operations. + + :param message: The message of the exception. + :param response: Server response to be deserialized. + :param parts: A list of the parts in multipart response. + """ + + def __init__(self, message, response, parts): + self.parts = parts + super(PartialBatchErrorException, self).__init__(message=message, response=response) + + def parse_length_from_content_range(content_range): ''' Parses the blob length from the content range header: bytes 1-3/65537