From c22ac51ea6b5bae799283c156332d581561d90aa Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Mon, 21 Oct 2019 10:19:06 -0700 Subject: [PATCH 1/4] Plug HttpLoggingPolicy to Storage --- .../azure/storage/blob/_shared/base_client.py | 6 ++++-- .../storage/blob/_shared/base_client_async.py | 6 ++++-- .../azure/storage/file/_shared/base_client.py | 7 ++++--- .../storage/file/_shared/base_client_async.py | 17 +++++++++++------ .../azure/storage/queue/_shared/base_client.py | 7 ++++--- .../storage/queue/_shared/base_client_async.py | 17 +++++++++++------ 6 files changed, 38 insertions(+), 22 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/base_client.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/base_client.py index 93f83c839ea4..72a4a5e11d94 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/base_client.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/base_client.py @@ -34,7 +34,8 @@ ContentDecodePolicy, BearerTokenCredentialPolicy, ProxyPolicy, - DistributedTracingPolicy + DistributedTracingPolicy, + HttpLoggingPolicy, ) from .constants import STORAGE_OAUTH_SCOPE, SERVICE_HOST_BASE, DEFAULT_SOCKET_TIMEOUT @@ -187,7 +188,8 @@ def _create_pipeline(self, credential, **kwargs): config.retry_policy, config.logging_policy, StorageResponseHook(**kwargs), - DistributedTracingPolicy(), + DistributedTracingPolicy(**kwargs), + HttpLoggingPolicy(**kwargs) ] return config, Pipeline(config.transport, policies=policies) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/base_client_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/base_client_async.py index 215292a64f3e..3a8ca9f2860f 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/base_client_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/base_client_async.py @@ -16,7 +16,8 @@ ContentDecodePolicy, AsyncBearerTokenCredentialPolicy, AsyncRedirectPolicy, - DistributedTracingPolicy + DistributedTracingPolicy, + HttpLoggingPolicy, ) from azure.core.pipeline.transport import AsyncHttpTransport @@ -91,7 +92,8 @@ def _create_pipeline(self, credential, **kwargs): config.retry_policy, config.logging_policy, AsyncStorageResponseHook(**kwargs), - DistributedTracingPolicy(), + DistributedTracingPolicy(**kwargs), + HttpLoggingPolicy(**kwargs), ] return config, AsyncPipeline(config.transport, policies=policies) diff --git a/sdk/storage/azure-storage-file/azure/storage/file/_shared/base_client.py b/sdk/storage/azure-storage-file/azure/storage/file/_shared/base_client.py index 2b9fade672ea..72a4a5e11d94 100644 --- a/sdk/storage/azure-storage-file/azure/storage/file/_shared/base_client.py +++ b/sdk/storage/azure-storage-file/azure/storage/file/_shared/base_client.py @@ -34,7 +34,8 @@ ContentDecodePolicy, BearerTokenCredentialPolicy, ProxyPolicy, - DistributedTracingPolicy + DistributedTracingPolicy, + HttpLoggingPolicy, ) from .constants import STORAGE_OAUTH_SCOPE, SERVICE_HOST_BASE, DEFAULT_SOCKET_TIMEOUT @@ -187,7 +188,8 @@ def _create_pipeline(self, credential, **kwargs): config.retry_policy, config.logging_policy, StorageResponseHook(**kwargs), - DistributedTracingPolicy(), + DistributedTracingPolicy(**kwargs), + HttpLoggingPolicy(**kwargs) ] return config, Pipeline(config.transport, policies=policies) @@ -224,7 +226,6 @@ def _batch_send( except StorageErrorException as error: process_storage_error(error) - class TransportWrapper(HttpTransport): """Wrapper class that ensures that an inner client created by a `get_client` method does not close the outer transport for the parent diff --git a/sdk/storage/azure-storage-file/azure/storage/file/_shared/base_client_async.py b/sdk/storage/azure-storage-file/azure/storage/file/_shared/base_client_async.py index 77f6e48ef5ab..3a8ca9f2860f 100644 --- a/sdk/storage/azure-storage-file/azure/storage/file/_shared/base_client_async.py +++ b/sdk/storage/azure-storage-file/azure/storage/file/_shared/base_client_async.py @@ -16,10 +16,11 @@ ContentDecodePolicy, AsyncBearerTokenCredentialPolicy, AsyncRedirectPolicy, - DistributedTracingPolicy + DistributedTracingPolicy, + HttpLoggingPolicy, ) - from azure.core.pipeline.transport import AsyncHttpTransport + from .constants import STORAGE_OAUTH_SCOPE, DEFAULT_SOCKET_TIMEOUT from .authentication import SharedKeyCredentialPolicy from .base_client import create_configuration @@ -28,7 +29,8 @@ StorageRequestHook, StorageHosts, StorageHeadersPolicy, - QueueMessagePolicy) + QueueMessagePolicy +) from .policies_async import AsyncStorageResponseHook from .._generated.models import StorageErrorException @@ -36,6 +38,7 @@ if TYPE_CHECKING: from azure.core.pipeline import Pipeline + from azure.core.pipeline.transport import HttpRequest from azure.core.configuration import Configuration _LOGGER = logging.getLogger(__name__) @@ -89,12 +92,14 @@ def _create_pipeline(self, credential, **kwargs): config.retry_policy, config.logging_policy, AsyncStorageResponseHook(**kwargs), - DistributedTracingPolicy(), + DistributedTracingPolicy(**kwargs), + HttpLoggingPolicy(**kwargs), ] return config, AsyncPipeline(config.transport, policies=policies) async def _batch_send( - self, *reqs # type: HttpRequest + self, *reqs: 'HttpRequest', + **kwargs ): """Given a series of request, do a Storage batch call. """ @@ -114,7 +119,7 @@ async def _batch_send( ) pipeline_response = await self._pipeline.run( - request, + request, **kwargs ) response = pipeline_response.http_response diff --git a/sdk/storage/azure-storage-queue/azure/storage/queue/_shared/base_client.py b/sdk/storage/azure-storage-queue/azure/storage/queue/_shared/base_client.py index 2b9fade672ea..72a4a5e11d94 100644 --- a/sdk/storage/azure-storage-queue/azure/storage/queue/_shared/base_client.py +++ b/sdk/storage/azure-storage-queue/azure/storage/queue/_shared/base_client.py @@ -34,7 +34,8 @@ ContentDecodePolicy, BearerTokenCredentialPolicy, ProxyPolicy, - DistributedTracingPolicy + DistributedTracingPolicy, + HttpLoggingPolicy, ) from .constants import STORAGE_OAUTH_SCOPE, SERVICE_HOST_BASE, DEFAULT_SOCKET_TIMEOUT @@ -187,7 +188,8 @@ def _create_pipeline(self, credential, **kwargs): config.retry_policy, config.logging_policy, StorageResponseHook(**kwargs), - DistributedTracingPolicy(), + DistributedTracingPolicy(**kwargs), + HttpLoggingPolicy(**kwargs) ] return config, Pipeline(config.transport, policies=policies) @@ -224,7 +226,6 @@ def _batch_send( except StorageErrorException as error: process_storage_error(error) - class TransportWrapper(HttpTransport): """Wrapper class that ensures that an inner client created by a `get_client` method does not close the outer transport for the parent diff --git a/sdk/storage/azure-storage-queue/azure/storage/queue/_shared/base_client_async.py b/sdk/storage/azure-storage-queue/azure/storage/queue/_shared/base_client_async.py index 77f6e48ef5ab..3a8ca9f2860f 100644 --- a/sdk/storage/azure-storage-queue/azure/storage/queue/_shared/base_client_async.py +++ b/sdk/storage/azure-storage-queue/azure/storage/queue/_shared/base_client_async.py @@ -16,10 +16,11 @@ ContentDecodePolicy, AsyncBearerTokenCredentialPolicy, AsyncRedirectPolicy, - DistributedTracingPolicy + DistributedTracingPolicy, + HttpLoggingPolicy, ) - from azure.core.pipeline.transport import AsyncHttpTransport + from .constants import STORAGE_OAUTH_SCOPE, DEFAULT_SOCKET_TIMEOUT from .authentication import SharedKeyCredentialPolicy from .base_client import create_configuration @@ -28,7 +29,8 @@ StorageRequestHook, StorageHosts, StorageHeadersPolicy, - QueueMessagePolicy) + QueueMessagePolicy +) from .policies_async import AsyncStorageResponseHook from .._generated.models import StorageErrorException @@ -36,6 +38,7 @@ if TYPE_CHECKING: from azure.core.pipeline import Pipeline + from azure.core.pipeline.transport import HttpRequest from azure.core.configuration import Configuration _LOGGER = logging.getLogger(__name__) @@ -89,12 +92,14 @@ def _create_pipeline(self, credential, **kwargs): config.retry_policy, config.logging_policy, AsyncStorageResponseHook(**kwargs), - DistributedTracingPolicy(), + DistributedTracingPolicy(**kwargs), + HttpLoggingPolicy(**kwargs), ] return config, AsyncPipeline(config.transport, policies=policies) async def _batch_send( - self, *reqs # type: HttpRequest + self, *reqs: 'HttpRequest', + **kwargs ): """Given a series of request, do a Storage batch call. """ @@ -114,7 +119,7 @@ async def _batch_send( ) pipeline_response = await self._pipeline.run( - request, + request, **kwargs ) response = pipeline_response.http_response From 5a87671750cb7ca2e31a8bf1275bc783b37e8281 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Mon, 21 Oct 2019 12:05:35 -0700 Subject: [PATCH 2/4] Update dependencies --- sdk/storage/azure-storage-blob/setup.py | 2 +- sdk/storage/azure-storage-file/setup.py | 2 +- sdk/storage/azure-storage-queue/setup.py | 2 +- shared_requirements.txt | 5 +---- 4 files changed, 4 insertions(+), 7 deletions(-) diff --git a/sdk/storage/azure-storage-blob/setup.py b/sdk/storage/azure-storage-blob/setup.py index 63becf5ff97b..2510e7a7b08f 100644 --- a/sdk/storage/azure-storage-blob/setup.py +++ b/sdk/storage/azure-storage-blob/setup.py @@ -91,7 +91,7 @@ 'tests.common' ]), install_requires=[ - "azure-core<2.0.0,>=1.0.0b4", + "azure-core<2.0.0,>=1.0.0b5", "msrest>=0.6.10", "cryptography>=2.1.4" ], diff --git a/sdk/storage/azure-storage-file/setup.py b/sdk/storage/azure-storage-file/setup.py index 24ce7e90317c..78115d900ddd 100644 --- a/sdk/storage/azure-storage-file/setup.py +++ b/sdk/storage/azure-storage-file/setup.py @@ -79,7 +79,7 @@ 'tests.common' ]), install_requires=[ - "azure-core<2.0.0,>=1.0.0b4", + "azure-core<2.0.0,>=1.0.0b5", "msrest>=0.6.10", "cryptography>=2.1.4" ], diff --git a/sdk/storage/azure-storage-queue/setup.py b/sdk/storage/azure-storage-queue/setup.py index 0a71c9ad7af8..0077dd703492 100644 --- a/sdk/storage/azure-storage-queue/setup.py +++ b/sdk/storage/azure-storage-queue/setup.py @@ -79,7 +79,7 @@ 'tests.common' ]), install_requires=[ - "azure-core<2.0.0,>=1.0.0b4", + "azure-core<2.0.0,>=1.0.0b5", "msrest>=0.6.10", "cryptography>=2.1.4" ], diff --git a/shared_requirements.txt b/shared_requirements.txt index ded20019e56c..1685344cf2a6 100644 --- a/shared_requirements.txt +++ b/shared_requirements.txt @@ -6,7 +6,7 @@ azure-cognitiveservices-language-nspkg azure-cognitiveservices-search-nspkg azure-cognitiveservices-vision-nspkg azure-common~=1.1 -azure-core<2.0.0,>=1.0.0b4 +azure-core<2.0.0,>=1.0.0b5 azure-cosmosdb-table~=1.0 azure-datalake-store~=0.0.18 azure-eventhub<6.0.0,>=5.0.0b3 @@ -103,9 +103,6 @@ aiohttp>=3.0 aiodns>=2.0 python-dateutil>=2.8.0 six>=1.6 -#override azure-storage-blob azure-core<2.0.0,>=1.0.0b4 -#override azure-storage-queue azure-core<2.0.0,>=1.0.0b4 -#override azure-storage-file azure-core<2.0.0,>=1.0.0b4 #override azure-eventhub azure-core<2.0.0,>=1.0.0b4 #override azure-storage-blob msrest>=0.6.10 #override azure-storage-queue msrest>=0.6.10 From 1f902312803688c22e10468d5ab878888a7638c9 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Mon, 21 Oct 2019 12:21:09 -0700 Subject: [PATCH 3/4] Skip depends job for Storage --- eng/tox/import_all.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/eng/tox/import_all.py b/eng/tox/import_all.py index e3e19129650c..c96b22e4b176 100644 --- a/eng/tox/import_all.py +++ b/eng/tox/import_all.py @@ -16,12 +16,15 @@ # keyvault has dependency issue when loading private module _BearerTokenCredentialPolicyBase from azure.core.pipeline.policies # azure.core.tracing.opencensus and azure.eventhub.checkpointstoreblob.aio are skipped due to a known issue in loading azure.core.tracing.opencensus excluded_packages = [ - "azure.core.tracing.opencensus", + "azure.core.tracing.opencensus", "azure.eventhub.checkpointstoreblob.aio", "azure.keyvault.certificates", # Github issue 7879 "azure.keyvault.keys", # Github issue 7879 "azure.keyvault.secrets", # Github issue 7879 "azure.appconfiguration", # Github issue 7879. revisit and close after azure-core POST b4 is released. + "azure.storage.blob", # Github issue 7879. + "azure.storage.file", # Github issue 7879. + "azure.storage.queue", # Github issue 7879. "azure", "azure-mgmt"] @@ -58,4 +61,4 @@ def should_run_import_all(package_name): logging.info("Verified module dependency, no issues found") else: pass - logging.error("Package {} is excluded from dependency check".format(package_name)) + logging.error("Package {} is excluded from dependency check".format(package_name)) From f47127cd038467df922c16ff025d9593d1263823 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Mon, 21 Oct 2019 12:27:30 -0700 Subject: [PATCH 4/4] Fix shared req --- shared_requirements.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/shared_requirements.txt b/shared_requirements.txt index 1685344cf2a6..8443f655abf7 100644 --- a/shared_requirements.txt +++ b/shared_requirements.txt @@ -119,4 +119,6 @@ opencensus-ext-azure>=0.3.1 #override azure-eventhub-checkpointstoreblob-aio aiohttp<4.0,>=3.0 #override azure-eventhub uamqp<2.0,>=1.2.3 #override azure-appconfiguration azure-core<2.0.0,>=1.0.0b4 -#override azure-appconfiguration msrest>=0.6.10 \ No newline at end of file +#override azure-appconfiguration msrest>=0.6.10 +#override azure-cosmos azure-core<2.0.0,>=1.0.0b4 +#override azure-core-tracing-opencensus azure-core<2.0.0,>=1.0.0b4 \ No newline at end of file