Pull latest master

.github/CODEOWNERS

@@ -1,5 +1,5 @@
 # Instructions for CODEOWNERS file format and automatic build failure notifications:
-# https://github.com/Azure/azure-sdk/blob/master/docs/engineering-system/codeowners.md
+# https://github.com/Azure/azure-sdk/blob/master/docs/policies/opensource.md#codeowners
 
 ###########
 # SDK

eng/pipelines/templates/jobs/archetype-sdk-client.yml

@@ -21,7 +21,7 @@ jobs:
         BeforePublishSteps: ${{ parameters.BeforePublishSteps }}
 
   - job: 'Analyze'
-    condition: ne(variables['Skip.Analyze'], 'true')
+    condition: and(succeededOrFailed(), ne(variables['Skip.Analyze'], 'true'))
     variables:
     - template: ../variables/globals.yml
 
@@ -35,7 +35,7 @@ jobs:
     - template: ../steps/analyze.yml
 
   - job: 'Test'
-    condition: ne(variables['Skip.Test'], 'true')
+    condition: and(succeededOrFailed(), ne(variables['Skip.Test'], 'true'))
     timeoutInMinutes: ${{ parameters.TestTimeoutInMinutes }}
     variables:
     - template: ../variables/globals.yml

sdk/core/azure-core/HISTORY.md

@@ -10,6 +10,7 @@
 - Tracing: DistributedTracingPolicy now accepts kwargs network_span_namer to change network span name  #7773
 - Tracing: Implementation of AbstractSpan can now use the mixin HttpSpanMixin to get HTTP span update automatically  #7773
 - Tracing: AbstractSpan contract "change_context" introduced  #7773
+- Introduce new policy HttpLoggingPolicy  #7988
 
 ### Bug fixes
 

sdk/core/azure-core/MANIFEST.in

@@ -1,5 +1,5 @@
 recursive-include tests *.py *.yaml
 include *.md
 include azure/__init__.py
-recursive-include examples *.py
+recursive-include samples *.py
 

sdk/core/azure-core/azure/core/_match_conditions.py

@@ -27,6 +27,7 @@
 from enum import Enum
 
 class MatchConditions(Enum):
+    """An enum to describe match conditions. """
     Unconditionally = 1
     IfNotModified = 2
     IfModified = 3

sdk/core/azure-core/azure/core/_pipeline_client.py

@@ -68,7 +68,7 @@ class PipelineClient(PipelineClientBase):
 
     .. admonition:: Example:
 
-        .. literalinclude:: ../examples/test_example_sync.py
+        .. literalinclude:: ../samples/test_example_sync.py
             :start-after: [START build_pipeline_client]
             :end-before: [END build_pipeline_client]
             :language: python

sdk/core/azure-core/azure/core/_pipeline_client_async.py

@@ -67,7 +67,7 @@ class AsyncPipelineClient(PipelineClientBase):
 
     .. admonition:: Example:
 
-        .. literalinclude:: ../examples/test_example_async.py
+        .. literalinclude:: ../samples/test_example_async.py
             :start-after: [START build_async_pipeline_client]
             :end-before: [END build_async_pipeline_client]
             :language: python

sdk/core/azure-core/azure/core/configuration.py

@@ -48,7 +48,7 @@ class Configuration(object):
 
     .. admonition:: Example:
 
-        .. literalinclude:: ../examples/test_example_config.py
+        .. literalinclude:: ../samples/test_example_config.py
             :start-after: [START configuration]
             :end-before: [END configuration]
             :language: python
@@ -99,7 +99,7 @@ class ConnectionConfiguration(object):
 
     .. admonition:: Example:
 
-        .. literalinclude:: ../examples/test_example_config.py
+        .. literalinclude:: ../samples/test_example_config.py
             :start-after: [START connection_configuration]
             :end-before: [END connection_configuration]
             :language: python

sdk/core/azure-core/azure/core/pipeline/_base.py

@@ -125,7 +125,7 @@ class Pipeline(AbstractContextManager, Generic[HTTPRequestType, HTTPResponseType
 
     .. admonition:: Example:
 
-        .. literalinclude:: ../examples/test_example_sync.py
+        .. literalinclude:: ../samples/test_example_sync.py
             :start-after: [START build_pipeline]
             :end-before: [END build_pipeline]
             :language: python

sdk/core/azure-core/azure/core/pipeline/_base_async.py

@@ -139,7 +139,7 @@ class AsyncPipeline(
 
     .. admonition:: Example:
 
-        .. literalinclude:: ../examples/test_example_async.py
+        .. literalinclude:: ../samples/test_example_async.py
             :start-after: [START build_async_pipeline]
             :end-before: [END build_async_pipeline]
             :language: python

sdk/core/azure-core/azure/core/pipeline/policies/__init__.py

@@ -35,7 +35,8 @@
     UserAgentPolicy,
     NetworkTraceLoggingPolicy,
     ContentDecodePolicy,
-    ProxyPolicy
+    ProxyPolicy,
+    HttpLoggingPolicy,
 )
 
 __all__ = [
@@ -52,6 +53,7 @@
     'CustomHookPolicy',
     'DistributedTracingPolicy',
     'RequestHistory',
+    'HttpLoggingPolicy',
 ]
 
 #pylint: disable=unused-import

sdk/core/azure-core/azure/core/pipeline/policies/_base.py

@@ -125,7 +125,7 @@ def on_exception(self, request):  # pylint: disable=unused-argument
 
         .. admonition:: Example:
 
-            .. literalinclude:: ../examples/test_example_sansio.py
+            .. literalinclude:: ../samples/test_example_sansio.py
                 :start-after: [START on_exception]
                 :end-before: [END on_exception]
                 :language: python

sdk/core/azure-core/azure/core/pipeline/policies/_redirect.py

@@ -52,7 +52,7 @@ class RedirectPolicy(HTTPPolicy):
 
     .. admonition:: Example:
 
-        .. literalinclude:: ../examples/test_example_sync.py
+        .. literalinclude:: ../samples/test_example_sync.py
             :start-after: [START redirect_policy]
             :end-before: [END redirect_policy]
             :language: python

sdk/core/azure-core/azure/core/pipeline/policies/_redirect_async.py

@@ -40,7 +40,7 @@ class AsyncRedirectPolicy(RedirectPolicy, AsyncHTTPPolicy):  # type: ignore
 
     .. admonition:: Example:
 
-        .. literalinclude:: ../examples/test_example_async.py
+        .. literalinclude:: ../samples/test_example_async.py
             :start-after: [START async_redirect_policy]
             :end-before: [END async_redirect_policy]
             :language: python

sdk/core/azure-core/azure/core/pipeline/policies/_retry.py

@@ -74,7 +74,7 @@ class RetryPolicy(HTTPPolicy):
 
     .. admonition:: Example:
 
-        .. literalinclude:: ../examples/test_example_sync.py
+        .. literalinclude:: ../samples/test_example_sync.py
             :start-after: [START retry_policy]
             :end-before: [END retry_policy]
             :language: python

sdk/core/azure-core/azure/core/pipeline/policies/_retry_async.py

@@ -67,7 +67,7 @@ class AsyncRetryPolicy(RetryPolicy, AsyncHTTPPolicy):  # type: ignore
 
     .. admonition:: Example:
 
-        .. literalinclude:: ../examples/test_example_async.py
+        .. literalinclude:: ../samples/test_example_async.py
             :start-after: [START async_retry_policy]
             :end-before: [END async_retry_policy]
             :language: python

sdk/core/azure-core/azure/core/pipeline/policies/_universal.py

@@ -36,6 +36,7 @@
 import re
 from typing import (Mapping, IO, TypeVar, TYPE_CHECKING, Type, cast, List, Callable, Iterator, # pylint: disable=unused-import
                     Any, Union, Dict, Optional, AnyStr)
+from six.moves import urllib
 
 from azure.core import __version__  as azcore_version
 from azure.core.exceptions import (
@@ -66,7 +67,7 @@ class HeadersPolicy(SansIOHTTPPolicy):
 
     .. admonition:: Example:
 
-        .. literalinclude:: ../examples/test_example_sansio.py
+        .. literalinclude:: ../samples/test_example_sansio.py
             :start-after: [START headers_policy]
             :end-before: [END headers_policy]
             :language: python
@@ -114,7 +115,7 @@ class UserAgentPolicy(SansIOHTTPPolicy):
 
     .. admonition:: Example:
 
-        .. literalinclude:: ../examples/test_example_sansio.py
+        .. literalinclude:: ../samples/test_example_sansio.py
             :start-after: [START user_agent_policy]
             :end-before: [END user_agent_policy]
             :language: python
@@ -185,7 +186,7 @@ class NetworkTraceLoggingPolicy(SansIOHTTPPolicy):
 
     .. admonition:: Example:
 
-        .. literalinclude:: ../examples/test_example_sansio.py
+        .. literalinclude:: ../samples/test_example_sansio.py
             :start-after: [START network_trace_logging_policy]
             :end-before: [END network_trace_logging_policy]
             :language: python
@@ -267,6 +268,106 @@ def on_response(self, request, response):
                 _LOGGER.debug("Failed to log response: %s", repr(err))
 
 
+class HttpLoggingPolicy(SansIOHTTPPolicy):
+    """The Pipeline policy that handles logging of HTTP requests and responses.
+    """
+
+    DEFAULT_HEADERS_WHITELIST = set([
+        "x-ms-client-request-id",
+        "x-ms-return-client-request-id",
+        "traceparent",
+        "Accept",
+        "Cache-Control",
+        "Connection",
+        "Content-Length",
+        "Content-Type",
+        "Date",
+        "ETag",
+        "Expires",
+        "If-Match",
+        "If-Modified-Since",
+        "If-None-Match",
+        "If-Unmodified-Since",
+        "Last-Modified",
+        "Pragma",
+        "Request-Id",
+        "Retry-After",
+        "Server",
+        "Transfer-Encoding",
+        "User-Agent"
+    ])
+    REDACTED_PLACEHOLDER = "REDACTED"
+
+    def __init__(self, logger=None, **kwargs):  # pylint: disable=unused-argument
+        self.logger = logger or logging.getLogger(
+            "azure.core.pipeline.policies.http_logging_policy"
+        )
+        self.allowed_query_params = set()
+        self.allowed_header_names = set(HttpLoggingPolicy.DEFAULT_HEADERS_WHITELIST)
+
+    def _redact_query_param(self, key, value):
+        lower_case_allowed_query_params = [
+            param.lower() for param in self.allowed_query_params
+        ]
+        return value if key.lower() in lower_case_allowed_query_params else HttpLoggingPolicy.REDACTED_PLACEHOLDER
+
+    def _redact_header(self, key, value):
+        lower_case_allowed_header_names = [
+            header.lower() for header in self.allowed_header_names
+        ]
+        return value if key.lower() in lower_case_allowed_header_names else HttpLoggingPolicy.REDACTED_PLACEHOLDER
+
+    def on_request(self, request):
+        # type: (PipelineRequest) -> None
+        """Logs HTTP method, url and headers.
+        :param request: The PipelineRequest object.
+        :type request: ~azure.core.pipeline.PipelineRequest
+        """
+        http_request = request.http_request
+        options = request.context.options
+        # Get logger in my context first (request has been retried)
+        # then read from kwargs (pop if that's the case)
+        # then use my instance logger
+        logger = request.context.setdefault("logger", options.pop("logger", self.logger))
+
+        if not logger.isEnabledFor(logging.INFO):
+            return
+
+        try:
+            parsed_url = list(urllib.parse.urlparse(http_request.url))
+            parsed_qp = urllib.parse.parse_qsl(parsed_url[4], keep_blank_values=True)
+            filtered_qp = [(key, self._redact_query_param(key, value)) for key, value in parsed_qp]
+            # 4 is query
+            parsed_url[4] = "&".join(["=".join(part) for part in filtered_qp])
+            redacted_url = urllib.parse.urlunparse(parsed_url)
+
+            logger.info("Request URL: %r", redacted_url)
+            logger.info("Request method: %r", http_request.method)
+            logger.info("Request headers:")
+            for header, value in http_request.headers.items():
+                value = self._redact_header(header, value)
+                logger.info("    %r: %r", header, value)
+        except Exception as err:  # pylint: disable=broad-except
+            logger.warning("Failed to log request: %s", repr(err))
+
+    def on_response(self, request, response):
+        # type: (PipelineRequest, PipelineResponse) -> None
+        http_response = response.http_response
+        logger = response.context.get("logger")
+
+        if not logger.isEnabledFor(logging.INFO):
+            return
+
+        try:
+            logger.info("Response status: %r", http_response.status_code)
+            logger.info("Response headers:")
+            for res_header, value in http_response.headers.items():
+                value = self._redact_header(res_header, value)
+                logger.info("    %r: %r", res_header, value)
+        except Exception as err:  # pylint: disable=broad-except
+            logger.warning("Failed to log response: %s", repr(err))
+
+
 class ContentDecodePolicy(SansIOHTTPPolicy):
     """Policy for decoding unstreamed response content.
     """
@@ -410,7 +511,7 @@ class ProxyPolicy(SansIOHTTPPolicy):
 
     .. admonition:: Example:
 
-        .. literalinclude:: ../examples/test_example_sansio.py
+        .. literalinclude:: ../samples/test_example_sansio.py
             :start-after: [START proxy_policy]
             :end-before: [END proxy_policy]
             :language: python

sdk/core/azure-core/azure/core/pipeline/transport/_aiohttp.py

@@ -62,7 +62,7 @@ class AioHttpTransport(AsyncHttpTransport):
 
     .. admonition:: Example:
 
-        .. literalinclude:: ../examples/test_example_async.py
+        .. literalinclude:: ../samples/test_example_async.py
             :start-after: [START aiohttp]
             :end-before: [END aiohttp]
             :language: python

sdk/core/azure-core/azure/core/pipeline/transport/_requests_asyncio.py

@@ -66,7 +66,7 @@ class AsyncioRequestsTransport(RequestsTransport, AsyncHttpTransport):  # type:
 
     .. admonition:: Example:
 
-        .. literalinclude:: ../examples/test_example_async.py
+        .. literalinclude:: ../samples/test_example_async.py
             :start-after: [START asyncio]
             :end-before: [END asyncio]
             :language: python

sdk/core/azure-core/azure/core/pipeline/transport/_requests_basic.py

@@ -159,7 +159,7 @@ class RequestsTransport(HttpTransport):
 
     .. admonition:: Example:
 
-        .. literalinclude:: ../examples/test_example_sync.py
+        .. literalinclude:: ../samples/test_example_sync.py
             :start-after: [START requests]
             :end-before: [END requests]
             :language: python

sdk/core/azure-core/azure/core/pipeline/transport/_requests_trio.py

@@ -137,7 +137,7 @@ class TrioRequestsTransport(RequestsTransport, AsyncHttpTransport):  # type: ign
 
     .. admonition:: Example:
 
-        .. literalinclude:: ../examples/test_example_async.py
+        .. literalinclude:: ../samples/test_example_async.py
             :start-after: [START trio]
             :end-before: [END trio]
             :language: python