From cf68d0e83c591b35879ac418c88908e3be1b9bb3 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Thu, 5 Sep 2019 14:38:52 -0700 Subject: [PATCH 01/33] Serialize HTTPRequest --- .../azure/core/pipeline/transport/base.py | 36 ++++++++++++ .../azure-core/tests/test_basic_transport.py | 57 +++++++++++++++++++ 2 files changed, 93 insertions(+) create mode 100644 sdk/core/azure-core/tests/test_basic_transport.py diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/base.py b/sdk/core/azure-core/azure/core/pipeline/transport/base.py index 4fb45d3b9b0d..a53e99285237 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/base.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/base.py @@ -40,6 +40,8 @@ from typing import (TYPE_CHECKING, Generic, TypeVar, cast, IO, List, Union, Any, Mapping, Dict, # pylint: disable=unused-import Optional, Tuple, Callable, Iterator) +from six.moves.http_client import HTTPConnection + # This file is NOT using any "requests" HTTP implementation # However, the CaseInsensitiveDict is handy. # If one day we reach the point where "requests" can be skip totally, @@ -81,6 +83,37 @@ def _urljoin(base_url, stub_url): return parsed.geturl() +class _HTTPSerializer(HTTPConnection): + """Hacking the stdlib HTTPConnection to serialize HTTP request as strings. + """ + def __init__(self, *args, **kwargs): + self.skip_host = kwargs.pop("skip_host", True) + self.skip_accept_encoding = kwargs.pop("skip_accept_encoding", True) + self.buffer = b'' + kwargs.setdefault("host", "fakehost") + super(_HTTPSerializer, self).__init__(*args, **kwargs) + + def putheader(self, header, *values): + if self.skip_host and header == "Host": + return + if self.skip_accept_encoding and header == "Accept-Encoding": + return + super(_HTTPSerializer, self).putheader(header, *values) + + def send(self, data): + self.buffer += data + +def _serialize_request(http_request): + serializer = _HTTPSerializer() + serializer.request( + method=http_request.method, + url=http_request.url, + body=http_request.body, + headers=http_request.headers + ) + return serializer.buffer + + class HttpTransport(AbstractContextManager, ABC, Generic[HTTPRequestType, HTTPResponseType]): # type: ignore """An http sender ABC. """ @@ -245,6 +278,9 @@ def set_bytes_body(self, data): self.data = data self.files = None + def serialize(self): + return _serialize_request(self) + class _HttpResponseBase(object): """Represent a HTTP response. diff --git a/sdk/core/azure-core/tests/test_basic_transport.py b/sdk/core/azure-core/tests/test_basic_transport.py new file mode 100644 index 000000000000..d64c0d209242 --- /dev/null +++ b/sdk/core/azure-core/tests/test_basic_transport.py @@ -0,0 +1,57 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE.txt in the project root for +# license information. +# ------------------------------------------------------------------------- +import time + +from azure.core.pipeline.transport import HttpRequest + +def test_http_request_serialization(): + # Method + Url + request = HttpRequest("DELETE", "/container0/blob0") + serialized = request.serialize() + + expected = b'DELETE /container0/blob0 HTTP/1.1\r\n\r\n' + assert serialized == expected + + # Method + Url + Headers + request = HttpRequest( + "DELETE", + "/container0/blob0", + headers={ + "x-ms-date": "Thu, 14 Jun 2018 16:46:54 GMT", + "Authorization": "SharedKey account:G4jjBXA7LI/RnWKIOQ8i9xH4p76pAQ+4Fs4R1VxasaE=", + "Content-Length": "0", + } + ) + serialized = request.serialize() + + expected = ( + b'DELETE /container0/blob0 HTTP/1.1\r\n' + b'x-ms-date: Thu, 14 Jun 2018 16:46:54 GMT\r\n' + b'Authorization: SharedKey account:G4jjBXA7LI/RnWKIOQ8i9xH4p76pAQ+4Fs4R1VxasaE=\r\n' + b'Content-Length: 0\r\n\r\n' + ) + assert serialized == expected + + + # Method + Url + Headers + Body + request = HttpRequest( + "DELETE", + "/container0/blob0", + headers={ + "x-ms-date": "Thu, 14 Jun 2018 16:46:54 GMT", + }, + ) + request.set_bytes_body(b"I am groot") + serialized = request.serialize() + + expected = ( + b'DELETE /container0/blob0 HTTP/1.1\r\n' + b'x-ms-date: Thu, 14 Jun 2018 16:46:54 GMT\r\n' + b'Content-Length: 10\r\n' + b'\r\n' + b'I am groot' + ) + assert serialized == expected From b236e90515204ed6a6e4e990e9d22d78174f0163 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Thu, 5 Sep 2019 15:54:45 -0700 Subject: [PATCH 02/33] Basic deserialize Http Response --- .../azure/core/pipeline/transport/base.py | 33 ++++++++++++- .../azure-core/tests/test_basic_transport.py | 48 +++++++++++++++++++ 2 files changed, 80 insertions(+), 1 deletion(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/base.py b/sdk/core/azure-core/azure/core/pipeline/transport/base.py index a53e99285237..e8c41d47b654 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/base.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/base.py @@ -40,7 +40,8 @@ from typing import (TYPE_CHECKING, Generic, TypeVar, cast, IO, List, Union, Any, Mapping, Dict, # pylint: disable=unused-import Optional, Tuple, Callable, Iterator) -from six.moves.http_client import HTTPConnection +from six.moves.http_client import HTTPConnection, HTTPResponse as _HTTPResponse +from io import BytesIO # This file is NOT using any "requests" HTTP implementation # However, the CaseInsensitiveDict is handy. @@ -333,6 +334,36 @@ def stream_download(self, pipeline): is supported. """ +class BytesIOSocket(object): + """Mocking the "makefile" of socket for HTTPResponse + """ + def __init__(self, bytes_data): + self.bytes_data = bytes_data + + def makefile(self, mode): + return BytesIO(self.bytes_data) + +class StdlibHttpResponse(HttpResponse): + def __init__(self, request, stdlib_response): + super(StdlibHttpResponse, self).__init__(request, stdlib_response) + self.status_code = stdlib_response.status + self.headers = dict(stdlib_response.getheaders()) + self.reason = stdlib_response.reason + self.content_type = self.headers.get('Content-Type') + self.data = stdlib_response.read() + + def body(self): + return self.data + + +def _deserialize_response(http_response_as_bytes, http_request): + local_socket = BytesIOSocket(http_response_as_bytes) + response = _HTTPResponse( + local_socket, + method=http_request.method + ) + response.begin() + return StdlibHttpResponse(http_request, response) class PipelineClientBase(object): """Base class for pipeline clients. diff --git a/sdk/core/azure-core/tests/test_basic_transport.py b/sdk/core/azure-core/tests/test_basic_transport.py index d64c0d209242..897aa48aa0d3 100644 --- a/sdk/core/azure-core/tests/test_basic_transport.py +++ b/sdk/core/azure-core/tests/test_basic_transport.py @@ -6,6 +6,8 @@ import time from azure.core.pipeline.transport import HttpRequest +from azure.core.pipeline.transport.base import _deserialize_response + def test_http_request_serialization(): # Method + Url @@ -55,3 +57,49 @@ def test_http_request_serialization(): b'I am groot' ) assert serialized == expected + +def test_response_deserialization(): + + # Method + Url + request = HttpRequest("DELETE", "/container0/blob0") + body = ( + b'HTTP/1.1 202 Accepted\r\n' + b'x-ms-request-id: 778fdc83-801e-0000-62ff-0334671e284f\r\n' + b'x-ms-version: 2018-11-09\r\n' + ) + + response = _deserialize_response(body, request) + + assert response.status_code == 202 + assert response.reason == "Accepted" + assert response.headers == { + 'x-ms-request-id': '778fdc83-801e-0000-62ff-0334671e284f', + 'x-ms-version': '2018-11-09' + } + + # Method + Url + Headers + Body + request = HttpRequest( + "DELETE", + "/container0/blob0", + headers={ + "x-ms-date": "Thu, 14 Jun 2018 16:46:54 GMT", + }, + ) + request.set_bytes_body(b"I am groot") + body = ( + b'HTTP/1.1 200 OK\r\n' + b'x-ms-request-id: 778fdc83-801e-0000-62ff-0334671e284f\r\n' + b'x-ms-version: 2018-11-09\r\n' + b'\r\n' + b'I am groot' + ) + + response = _deserialize_response(body, request) + + assert response.status_code == 200 + assert response.reason == "OK" + assert response.headers == { + 'x-ms-request-id': '778fdc83-801e-0000-62ff-0334671e284f', + 'x-ms-version': '2018-11-09' + } + assert response.text() == "I am groot" \ No newline at end of file From 4985fa63b0948e205bd46290a61136d631004448 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Thu, 5 Sep 2019 16:25:23 -0700 Subject: [PATCH 03/33] http.client response transport --- .../azure/core/pipeline/transport/base.py | 43 +++++++++++++------ .../azure-core/tests/test_basic_transport.py | 23 +++++++++- 2 files changed, 51 insertions(+), 15 deletions(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/base.py b/sdk/core/azure-core/azure/core/pipeline/transport/base.py index e8c41d47b654..be69855ae418 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/base.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/base.py @@ -334,8 +334,34 @@ def stream_download(self, pipeline): is supported. """ + +class HttpClientTransportResponse(HttpResponse): + """Create a HTTPResponse from an http.client response. + + Body will NOT be read by the constructor. Call "body()" to load the body in memory if necessary. + + :param HttpRequest request: The request. + :param httpclient_response: The object returned from an HTTP(S)Connection from http.client + """ + def __init__(self, request, httpclient_response): + super(HttpClientTransportResponse, self).__init__(request, httpclient_response) + self.status_code = httpclient_response.status + self.headers = dict(httpclient_response.getheaders()) + self.reason = httpclient_response.reason + self.content_type = self.headers.get('Content-Type') + self.data = None + + def body(self): + if self.data is None: + self.data = self.internal_response.read() + return self.data + + class BytesIOSocket(object): - """Mocking the "makefile" of socket for HTTPResponse + """Mocking the "makefile" of socket for HTTPResponse. + + This can be used to create a http.client.HTTPResponse object + based on bytes and not a real socket. """ def __init__(self, bytes_data): self.bytes_data = bytes_data @@ -343,18 +369,6 @@ def __init__(self, bytes_data): def makefile(self, mode): return BytesIO(self.bytes_data) -class StdlibHttpResponse(HttpResponse): - def __init__(self, request, stdlib_response): - super(StdlibHttpResponse, self).__init__(request, stdlib_response) - self.status_code = stdlib_response.status - self.headers = dict(stdlib_response.getheaders()) - self.reason = stdlib_response.reason - self.content_type = self.headers.get('Content-Type') - self.data = stdlib_response.read() - - def body(self): - return self.data - def _deserialize_response(http_response_as_bytes, http_request): local_socket = BytesIOSocket(http_response_as_bytes) @@ -363,7 +377,8 @@ def _deserialize_response(http_response_as_bytes, http_request): method=http_request.method ) response.begin() - return StdlibHttpResponse(http_request, response) + return HttpClientTransportResponse(http_request, response) + class PipelineClientBase(object): """Base class for pipeline clients. diff --git a/sdk/core/azure-core/tests/test_basic_transport.py b/sdk/core/azure-core/tests/test_basic_transport.py index 897aa48aa0d3..6f1c2b5d578d 100644 --- a/sdk/core/azure-core/tests/test_basic_transport.py +++ b/sdk/core/azure-core/tests/test_basic_transport.py @@ -3,10 +3,11 @@ # Licensed under the MIT License. See LICENSE.txt in the project root for # license information. # ------------------------------------------------------------------------- +from six.moves.http_client import HTTPConnection import time from azure.core.pipeline.transport import HttpRequest -from azure.core.pipeline.transport.base import _deserialize_response +from azure.core.pipeline.transport.base import HttpClientTransportResponse, _deserialize_response def test_http_request_serialization(): @@ -58,6 +59,26 @@ def test_http_request_serialization(): ) assert serialized == expected + +def test_http_client_response(): + # Create a core request + request = HttpRequest("GET", "www.httpbin.org") + + # Fake a transport based on http.client + conn = HTTPConnection("www.httpbin.org") + conn.request("GET", "/get") + r1 = conn.getresponse() + + response = HttpClientTransportResponse(request, r1) + + # Don't assume too much in those assert, since we reach a real server + assert response.internal_response is r1 + assert response.reason is not None + assert response.status_code == 200 + assert len(response.headers.keys()) != 0 + assert len(response.text()) != 0 + + def test_response_deserialization(): # Method + Url From 79078a653a8e95a2f6010e3c8454898afdbefd70 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Fri, 6 Sep 2019 13:59:23 -0700 Subject: [PATCH 04/33] Multipart helper - base version --- .../azure/core/pipeline/transport/base.py | 84 ++++++++++++- .../azure-core/tests/test_basic_transport.py | 118 +++++++++++++++++- 2 files changed, 198 insertions(+), 4 deletions(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/base.py b/sdk/core/azure-core/azure/core/pipeline/transport/base.py index be69855ae418..2814c5940a32 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/base.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/base.py @@ -48,7 +48,7 @@ # If one day we reach the point where "requests" can be skip totally, # might provide our own implementation from requests.structures import CaseInsensitiveDict -from azure.core.pipeline import ABC, AbstractContextManager, PipelineRequest, PipelineResponse +from azure.core.pipeline import ABC, AbstractContextManager, PipelineRequest, PipelineResponse, PipelineContext HTTPResponseType = TypeVar("HTTPResponseType") @@ -583,3 +583,85 @@ def merge(self, url, params=None, headers=None, content=None, form_content=None) """ request = self._request('MERGE', url, params, headers, content, form_content, None) return request + + +from email.message import Message +from email import message_from_string + + +class MultiPartHelper(object): + def __init__( + self, + main_request, # type: HttpRequest + ): + self.main_request = main_request + self.requests = [] # type: List[HttpRequest] + self.policies = [] # type List[SansIOHTTPPolicy] + + def prepare_request(self): + # Apply on_requests concurrently to all requests + import concurrent.futures + + def prepare_requests(request): + context = PipelineContext(None) + pipeline_request = PipelineRequest(request, context) + for policy in self.policies: + policy.on_request(pipeline_request) + + with concurrent.futures.ThreadPoolExecutor() as executor: + # List comprehension to raise exceptions if happened + [_ for _ in executor.map(prepare_requests, self.requests)] + + # Update the main request with the body + main_message = Message() + main_message.add_header("Content-Type", "multipart/mixed") + for req in self.requests: + part_message = Message() + part_message.add_header('content-type', 'application/http') + part_message.set_payload(req.serialize()) + main_message.attach(part_message) + + full_message = main_message.as_bytes() + headers, _, body = full_message.split(b'\n', maxsplit=2) + self.main_request.set_bytes_body(body) + self.main_request.headers['Content-Type'] = 'multipart/mixed; boundary='+main_message.get_boundary() + + def parse_response(self, response): + body_as_str = response.text() + # In order to use email.message parser, I need full HTTP bytes. Faking something to make the parser happy + http_body = ( + 'Content-Type: ' + + response.content_type + + '\n\n' + + body_as_str + ) + + message = message_from_string(http_body) # type: Message + + # Rebuild an HTTP response from pure string + responses = [] + for request, raw_reponse in zip(self.requests, message.get_payload()): + responses.append(_deserialize_response(raw_reponse.get_payload().encode('ascii'), request)) + + # Apply on_response concurrently to all requests + import concurrent.futures + + def parse_responses(response): + http_request = response.request + context = PipelineContext(None) + pipeline_request = PipelineRequest(http_request, context) + pipeline_response = PipelineResponse( + http_request, + response, + context=context + ) + + for policy in self.policies: + policy.on_response(pipeline_request, pipeline_response) + + with concurrent.futures.ThreadPoolExecutor() as executor: + # List comprehension to raise exceptions if happened + [_ for _ in executor.map(parse_responses, responses)] + + return responses + diff --git a/sdk/core/azure-core/tests/test_basic_transport.py b/sdk/core/azure-core/tests/test_basic_transport.py index 6f1c2b5d578d..4270bbb8b68e 100644 --- a/sdk/core/azure-core/tests/test_basic_transport.py +++ b/sdk/core/azure-core/tests/test_basic_transport.py @@ -6,8 +6,9 @@ from six.moves.http_client import HTTPConnection import time -from azure.core.pipeline.transport import HttpRequest -from azure.core.pipeline.transport.base import HttpClientTransportResponse, _deserialize_response +from azure.core.pipeline.transport import HttpRequest, HttpResponse +from azure.core.pipeline.transport.base import HttpClientTransportResponse, _deserialize_response, MultiPartHelper +from azure.core.pipeline.policies import HeadersPolicy def test_http_request_serialization(): @@ -123,4 +124,115 @@ def test_response_deserialization(): 'x-ms-request-id': '778fdc83-801e-0000-62ff-0334671e284f', 'x-ms-version': '2018-11-09' } - assert response.text() == "I am groot" \ No newline at end of file + assert response.text() == "I am groot" + + +def test_multipart_send(): + + header_policy = HeadersPolicy({ + 'x-ms-date': 'Thu, 14 Jun 2018 16:46:54 GMT' + }) + + request = HttpRequest("POST", "http://account.blob.core.windows.net/?comp=batch") + + req0 = HttpRequest("DELETE", "/container0/blob0") + req1 = HttpRequest("DELETE", "/container1/blob1") + + helper = MultiPartHelper(request) + helper.requests = [ + req0, req1 + ] + helper.policies = [ + header_policy + ] + + helper.prepare_request() + + assert request.body == ( + b'--===============6566992931842418154==\n' + b'content-type: application/http\n' + b'\n' + b'DELETE /container0/blob0 HTTP/1.1\n' + b'x-ms-date: Thu, 14 Jun 2018 16:46:54 GMT\n\n\n' + b'--===============6566992931842418154==\n' + b'content-type: application/http\n' + b'\n' + b'DELETE /container1/blob1 HTTP/1.1\n' + b'x-ms-date: Thu, 14 Jun 2018 16:46:54 GMT\n\n\n' + b'--===============6566992931842418154==--\n' + ) + + print(request.body) + print(request.headers) + +def test_multipart_receive(): + + class MockResponse(HttpResponse): + def __init__(self, body, content_type): + super(MockResponse, self).__init__(None, None) + self._body = body + self.content_type = content_type + + def body(self): + return self._body + + class ResponsePolicy(object): + def on_response(self, request, response): + # type: (PipelineRequest, PipelineResponse) -> None + response.http_response.headers['x-ms-fun'] = 'true' + + body_as_str = ( + "--batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed\n" + "Content-Type: application/http\n" + "Content-ID: 0\n" + "\n" + "HTTP/1.1 202 Accepted\n" + "x-ms-request-id: 778fdc83-801e-0000-62ff-0334671e284f\n" + "x-ms-version: 2018-11-09\n" + "\n" + "--batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed\n" + "Content-Type: application/http\n" + "Content-ID: 2\n" + "\n" + "HTTP/1.1 404 The specified blob does not exist.\n" + "x-ms-error-code: BlobNotFound\n" + "x-ms-request-id: 778fdc83-801e-0000-62ff-0334671e2852\n" + "x-ms-version: 2018-11-09\n" + "Content-Length: 216\n" + "Content-Type: application/xml\n" + "\n" + '\n' + "BlobNotFoundThe specified blob does not exist.\n" + "RequestId:778fdc83-801e-0000-62ff-0334671e2852\n" + "Time:2018-06-14T16:46:54.6040685Z\n" + "--batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed--" + ) + + response = MockResponse( + body_as_str.encode('ascii'), + "multipart/mixed; boundary=batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed" + ) + + + request = HttpRequest("POST", "http://account.blob.core.windows.net/?comp=batch") + + req0 = HttpRequest("DELETE", "/container0/blob0") + req1 = HttpRequest("DELETE", "/container1/blob1") + + helper = MultiPartHelper(request) + helper.requests = [ + req0, req1 + ] + helper.policies = [ + ResponsePolicy() + ] + + response = helper.parse_response(response) + + res0 = response[0] + assert res0.status_code == 202 + assert res0.headers['x-ms-fun'] == 'true' + + res1 = response[1] + assert res1.status_code == 404 + assert res1.headers['x-ms-fun'] == 'true' From 5f81289df94435f88f10a83c20cb530e93ca2fc9 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Fri, 6 Sep 2019 16:06:17 -0700 Subject: [PATCH 05/33] Working full mock scenario --- .../azure-core/azure/core/pipeline/base.py | 14 ++- .../azure/core/pipeline/transport/base.py | 11 +- .../azure-core/tests/test_basic_transport.py | 101 ++++++++++++++---- 3 files changed, 101 insertions(+), 25 deletions(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/base.py b/sdk/core/azure-core/azure/core/pipeline/base.py index 19df8b939ebb..3eea92ab0bdb 100644 --- a/sdk/core/azure-core/azure/core/pipeline/base.py +++ b/sdk/core/azure-core/azure/core/pipeline/base.py @@ -147,7 +147,19 @@ def run(self, request, **kwargs): :return: The PipelineResponse object :rtype: ~azure.core.pipeline.PipelineResponse """ + multipart_helper = None + if request.multipart_mixed_info: + from .transport.base import MultiPartHelper + multipart_helper = MultiPartHelper(request) + multipart_helper.prepare_request() + context = PipelineContext(self._transport, **kwargs) pipeline_request = PipelineRequest(request, context) # type: PipelineRequest first_node = self._impl_policies[0] if self._impl_policies else _TransportRunner(self._transport) - return first_node.send(pipeline_request) # type: ignore + response = first_node.send(pipeline_request) # type: ignore + + if multipart_helper: + part_responses = multipart_helper.parse_response(response.http_response) + response.context['MULTIPART_RESPONSE'] = part_responses + + return response diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/base.py b/sdk/core/azure-core/azure/core/pipeline/transport/base.py index 2814c5940a32..d16a957bbb0f 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/base.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/base.py @@ -156,6 +156,7 @@ def __init__(self, method, url, headers=None, files=None, data=None): self.headers = CaseInsensitiveDict(headers) self.files = files self.data = data + self.multipart_mixed_info = None def __repr__(self): return '' % (self.method) @@ -279,6 +280,12 @@ def set_bytes_body(self, data): self.data = data self.files = None + def set_multipart_mixed(self, *requests, **kwargs): + self.multipart_mixed_info = ( + requests, + kwargs.pop("policies", []) + ) + def serialize(self): return _serialize_request(self) @@ -595,8 +602,8 @@ def __init__( main_request, # type: HttpRequest ): self.main_request = main_request - self.requests = [] # type: List[HttpRequest] - self.policies = [] # type List[SansIOHTTPPolicy] + self.requests = self.main_request.multipart_mixed_info[0] # type: List[HttpRequest] + self.policies = self.main_request.multipart_mixed_info[1] # type: List[SansIOHTTPPolicy] def prepare_request(self): # Apply on_requests concurrently to all requests diff --git a/sdk/core/azure-core/tests/test_basic_transport.py b/sdk/core/azure-core/tests/test_basic_transport.py index 4270bbb8b68e..da649cc73c38 100644 --- a/sdk/core/azure-core/tests/test_basic_transport.py +++ b/sdk/core/azure-core/tests/test_basic_transport.py @@ -6,9 +6,10 @@ from six.moves.http_client import HTTPConnection import time -from azure.core.pipeline.transport import HttpRequest, HttpResponse +from azure.core.pipeline.transport import HttpRequest, HttpResponse, RequestsTransport from azure.core.pipeline.transport.base import HttpClientTransportResponse, _deserialize_response, MultiPartHelper from azure.core.pipeline.policies import HeadersPolicy +from azure.core.pipeline import Pipeline def test_http_request_serialization(): @@ -133,21 +134,20 @@ def test_multipart_send(): 'x-ms-date': 'Thu, 14 Jun 2018 16:46:54 GMT' }) - request = HttpRequest("POST", "http://account.blob.core.windows.net/?comp=batch") - req0 = HttpRequest("DELETE", "/container0/blob0") req1 = HttpRequest("DELETE", "/container1/blob1") - helper = MultiPartHelper(request) - helper.requests = [ - req0, req1 - ] - helper.policies = [ - header_policy - ] + request = HttpRequest("POST", "http://account.blob.core.windows.net/?comp=batch") + request.set_multipart_mixed( + req0, + req1, + policies=[header_policy] + ) + helper = MultiPartHelper(request) helper.prepare_request() + # FIXME Boundary is random, so need to improve this test with a regexp or something assert request.body == ( b'--===============6566992931842418154==\n' b'content-type: application/http\n' @@ -162,9 +162,6 @@ def test_multipart_send(): b'--===============6566992931842418154==--\n' ) - print(request.body) - print(request.headers) - def test_multipart_receive(): class MockResponse(HttpResponse): @@ -213,19 +210,17 @@ def on_response(self, request, response): "multipart/mixed; boundary=batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed" ) - - request = HttpRequest("POST", "http://account.blob.core.windows.net/?comp=batch") - req0 = HttpRequest("DELETE", "/container0/blob0") req1 = HttpRequest("DELETE", "/container1/blob1") + request = HttpRequest("POST", "http://account.blob.core.windows.net/?comp=batch") + request.set_multipart_mixed( + req0, + req1, + policies=[ResponsePolicy()] + ) + helper = MultiPartHelper(request) - helper.requests = [ - req0, req1 - ] - helper.policies = [ - ResponsePolicy() - ] response = helper.parse_response(response) @@ -236,3 +231,65 @@ def on_response(self, request, response): res1 = response[1] assert res1.status_code == 404 assert res1.headers['x-ms-fun'] == 'true' + +def test_pipeline_full_scenario(): + + # To get this test to zork, create a mocky.io: + # Content type: multipart/mixed; boundary=batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed + # Body: + """ +--batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed +Content-Type: application/http +Content-ID: 0 + +HTTP/1.1 202 Accepted +x-ms-request-id: 778fdc83-801e-0000-62ff-0334671e284f +x-ms-version: 2018-11-09 + +--batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed +Content-Type: application/http +Content-ID: 1 + +HTTP/1.1 202 Accepted +x-ms-request-id: 778fdc83-801e-0000-62ff-0334671e2851 +x-ms-version: 2018-11-09 + +--batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed +Content-Type: application/http +Content-ID: 2 + +HTTP/1.1 404 The specified blob does not exist. +x-ms-error-code: BlobNotFound +x-ms-request-id: 778fdc83-801e-0000-62ff-0334671e2852 +x-ms-version: 2018-11-09 +Content-Length: 216 +Content-Type: application/xml + + +BlobNotFoundThe specified blob does not exist. +RequestId:778fdc83-801e-0000-62ff-0334671e2852 +Time:2018-06-14T16:46:54.6040685Z +--batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed-- +""" + + header_policy = HeadersPolicy({ + 'x-ms-date': 'Thu, 14 Jun 2018 16:46:54 GMT' + }) + + req0 = HttpRequest("DELETE", "/container0/blob0") + req1 = HttpRequest("DELETE", "/container1/blob1") + req2 = HttpRequest("DELETE", "/container2/blob2") + + request = HttpRequest("GET", "http://www.mocky.io/v2/5d72e4892f0000ce9c7d4ec1") + request.set_multipart_mixed( + req0, + req1, + req2, + policies=[header_policy] + ) + + with Pipeline(RequestsTransport(), policies=[]) as pipeline: + response = pipeline.run(request) + + assert len(response.context['MULTIPART_RESPONSE']) == 3 + assert response.context['MULTIPART_RESPONSE'][2].status_code == 404 From 61be567a25efef90305867d59f661d2b7aba7c4c Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Tue, 17 Sep 2019 10:56:43 -0700 Subject: [PATCH 06/33] Parse multipart response based on content-type --- sdk/core/azure-core/azure/core/pipeline/base.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/base.py b/sdk/core/azure-core/azure/core/pipeline/base.py index 3eea92ab0bdb..dfe5c2c21c6a 100644 --- a/sdk/core/azure-core/azure/core/pipeline/base.py +++ b/sdk/core/azure-core/azure/core/pipeline/base.py @@ -158,7 +158,9 @@ def run(self, request, **kwargs): first_node = self._impl_policies[0] if self._impl_policies else _TransportRunner(self._transport) response = first_node.send(pipeline_request) # type: ignore - if multipart_helper: + if (response.http_response.content_type or '').startswith("multipart/mixed"): + from .transport.base import MultiPartHelper + multipart_helper = MultiPartHelper(request) part_responses = multipart_helper.parse_response(response.http_response) response.context['MULTIPART_RESPONSE'] = part_responses From 9bb1c95e89e81d37cbea851e87521dd38036b683 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Tue, 17 Sep 2019 17:05:35 -0700 Subject: [PATCH 07/33] Adding BOM tests --- .../azure/core/pipeline/transport/base.py | 22 ++++--- .../azure-core/tests/test_basic_transport.py | 66 +++++++++++++++++++ 2 files changed, 80 insertions(+), 8 deletions(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/base.py b/sdk/core/azure-core/azure/core/pipeline/transport/base.py index d16a957bbb0f..46789346fed4 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/base.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/base.py @@ -593,7 +593,10 @@ def merge(self, url, params=None, headers=None, content=None, form_content=None) from email.message import Message -from email import message_from_string +try: + from email import message_from_bytes as message_parser +except ImportError: # 2.7 + from email import message_from_string as message_parser class MultiPartHelper(object): @@ -634,21 +637,24 @@ def prepare_requests(request): self.main_request.headers['Content-Type'] = 'multipart/mixed; boundary='+main_message.get_boundary() def parse_response(self, response): - body_as_str = response.text() + body_as_bytes = response.body() # In order to use email.message parser, I need full HTTP bytes. Faking something to make the parser happy http_body = ( - 'Content-Type: ' + - response.content_type + - '\n\n' + - body_as_str + b'Content-Type: ' + + response.content_type.encode('ascii') + + b'\r\n\r\n' + + body_as_bytes ) - message = message_from_string(http_body) # type: Message + message = message_parser(http_body) # type: Message # Rebuild an HTTP response from pure string responses = [] for request, raw_reponse in zip(self.requests, message.get_payload()): - responses.append(_deserialize_response(raw_reponse.get_payload().encode('ascii'), request)) + if raw_reponse.get_content_type() == "application/http": + responses.append(_deserialize_response(raw_reponse.get_payload(decode=True), request)) + else: + raise ValueError("Multipart doesn't support part other than application/http for now") # Apply on_response concurrently to all requests import concurrent.futures diff --git a/sdk/core/azure-core/tests/test_basic_transport.py b/sdk/core/azure-core/tests/test_basic_transport.py index da649cc73c38..81aad1614f6e 100644 --- a/sdk/core/azure-core/tests/test_basic_transport.py +++ b/sdk/core/azure-core/tests/test_basic_transport.py @@ -127,6 +127,23 @@ def test_response_deserialization(): } assert response.text() == "I am groot" +def test_response_deserialization_utf8_bom(): + + request = HttpRequest("DELETE", "/container0/blob0") + body = ( + b'HTTP/1.1 400 One of the request inputs is not valid.\r\n' + b'x-ms-error-code: InvalidInput\r\n' + b'x-ms-request-id: 5f3f9f2f-e01e-00cc-6eb1-6d00b5000000\r\n' + b'x-ms-version: 2019-02-02\r\n' + b'Content-Length: 220\r\n' + b'Content-Type: application/xml\r\n' + b'Server: Windows-Azure-Blob/1.0\r\n' + b'\r\n' + b'\xef\xbb\xbf\nInvalidInputOne' + b'of the request inputs is not valid.\nRequestId:5f3f9f2f-e01e-00cc-6eb1-6d00b5000000\nTime:2019-09-17T23:44:07.4671860Z' + ) + response = _deserialize_response(body, request) + assert response.body().startswith(b'\xef\xbb\xbf') def test_multipart_send(): @@ -224,6 +241,8 @@ def on_response(self, request, response): response = helper.parse_response(response) + assert len(response) == 2 + res0 = response[0] assert res0.status_code == 202 assert res0.headers['x-ms-fun'] == 'true' @@ -232,6 +251,53 @@ def on_response(self, request, response): assert res1.status_code == 404 assert res1.headers['x-ms-fun'] == 'true' +def test_multipart_receive_with_bom(): + + class MockResponse(HttpResponse): + def __init__(self, body, content_type): + super(MockResponse, self).__init__(None, None) + self._body = body + self.content_type = content_type + + def body(self): + return self._body + + body_as_bytes = ( + b"--batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed\n" + b"Content-Type: application/http\n" + b"Content-Transfer-Encoding: binary\n" + b"Content-ID: 0\n" + b'\r\n' + b'HTTP/1.1 400 One of the request inputs is not valid.\r\n' + b'Content-Length: 220\r\n' + b'Content-Type: application/xml\r\n' + b'Server: Windows-Azure-Blob/1.0\r\n' + b'\r\n' + b'\xef\xbb\xbf\nInvalidInputOne' + b'of the request inputs is not valid.\nRequestId:5f3f9f2f-e01e-00cc-6eb1-6d00b5000000\nTime:2019-09-17T23:44:07.4671860Z\n' + b"--batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed--" + ) + + response = MockResponse( + body_as_bytes, + "multipart/mixed; boundary=batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed" + ) + + req0 = HttpRequest("DELETE", "/container0/blob0") + + request = HttpRequest("POST", "http://account.blob.core.windows.net/?comp=batch") + request.set_multipart_mixed(req0) + + helper = MultiPartHelper(request) + + response = helper.parse_response(response) + assert len(response) == 1 + + res0 = response[0] + assert res0.status_code == 400 + assert res0.body().startswith(b'\xef\xbb\xbf') + + def test_pipeline_full_scenario(): # To get this test to zork, create a mocky.io: From edf8ac232fb8a436527c8dae81bc00f5ad5cebc2 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Wed, 18 Sep 2019 15:57:40 -0700 Subject: [PATCH 08/33] Improve a bit multipart generation --- .../azure-core/azure/core/pipeline/transport/base.py | 11 +++++++---- sdk/core/azure-core/tests/test_basic_transport.py | 9 +++++++-- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/base.py b/sdk/core/azure-core/azure/core/pipeline/transport/base.py index 46789346fed4..d3ec451698c1 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/base.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/base.py @@ -593,6 +593,7 @@ def merge(self, url, params=None, headers=None, content=None, form_content=None) from email.message import Message +from email.policy import HTTP try: from email import message_from_bytes as message_parser except ImportError: # 2.7 @@ -625,14 +626,16 @@ def prepare_requests(request): # Update the main request with the body main_message = Message() main_message.add_header("Content-Type", "multipart/mixed") - for req in self.requests: + for i, req in enumerate(self.requests): part_message = Message() - part_message.add_header('content-type', 'application/http') + part_message.add_header('Content-Type', 'application/http') + part_message.add_header('Content-Transfer-Encoding', 'binary') + part_message.add_header('Content-ID', str(i)) part_message.set_payload(req.serialize()) main_message.attach(part_message) - full_message = main_message.as_bytes() - headers, _, body = full_message.split(b'\n', maxsplit=2) + full_message = main_message.as_bytes(policy=HTTP) + headers, _, body = full_message.split(b'\r\n', maxsplit=2) self.main_request.set_bytes_body(body) self.main_request.headers['Content-Type'] = 'multipart/mixed; boundary='+main_message.get_boundary() diff --git a/sdk/core/azure-core/tests/test_basic_transport.py b/sdk/core/azure-core/tests/test_basic_transport.py index 81aad1614f6e..1933f22fa0d2 100644 --- a/sdk/core/azure-core/tests/test_basic_transport.py +++ b/sdk/core/azure-core/tests/test_basic_transport.py @@ -17,7 +17,11 @@ def test_http_request_serialization(): request = HttpRequest("DELETE", "/container0/blob0") serialized = request.serialize() - expected = b'DELETE /container0/blob0 HTTP/1.1\r\n\r\n' + expected = ( + b'DELETE /container0/blob0 HTTP/1.1\r\n' + # No headers + b'\r\n' + ) assert serialized == expected # Method + Url + Headers @@ -36,7 +40,8 @@ def test_http_request_serialization(): b'DELETE /container0/blob0 HTTP/1.1\r\n' b'x-ms-date: Thu, 14 Jun 2018 16:46:54 GMT\r\n' b'Authorization: SharedKey account:G4jjBXA7LI/RnWKIOQ8i9xH4p76pAQ+4Fs4R1VxasaE=\r\n' - b'Content-Length: 0\r\n\r\n' + b'Content-Length: 0\r\n' + b'\r\n' ) assert serialized == expected From 3e0cf8dcb09a887f250a4c31400023e5714a443e Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Thu, 19 Sep 2019 14:09:33 -0700 Subject: [PATCH 09/33] Refactor prepare_multipart_mixed --- .../azure-core/azure/core/pipeline/base.py | 7 +----- .../azure/core/pipeline/base_async.py | 1 + .../azure/core/pipeline/transport/base.py | 23 +++++++++++++++++++ 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/base.py b/sdk/core/azure-core/azure/core/pipeline/base.py index dfe5c2c21c6a..991a92a9dff0 100644 --- a/sdk/core/azure-core/azure/core/pipeline/base.py +++ b/sdk/core/azure-core/azure/core/pipeline/base.py @@ -147,12 +147,7 @@ def run(self, request, **kwargs): :return: The PipelineResponse object :rtype: ~azure.core.pipeline.PipelineResponse """ - multipart_helper = None - if request.multipart_mixed_info: - from .transport.base import MultiPartHelper - multipart_helper = MultiPartHelper(request) - multipart_helper.prepare_request() - + request.prepare_multipart_mixed() context = PipelineContext(self._transport, **kwargs) pipeline_request = PipelineRequest(request, context) # type: PipelineRequest first_node = self._impl_policies[0] if self._impl_policies else _TransportRunner(self._transport) diff --git a/sdk/core/azure-core/azure/core/pipeline/base_async.py b/sdk/core/azure-core/azure/core/pipeline/base_async.py index 9a44d8441ca8..a40c7c2ae694 100644 --- a/sdk/core/azure-core/azure/core/pipeline/base_async.py +++ b/sdk/core/azure-core/azure/core/pipeline/base_async.py @@ -161,6 +161,7 @@ async def run(self, request: PipelineRequest, **kwargs: Any): :return: The PipelineResponse object. :rtype: ~azure.core.pipeline.PipelineResponse """ + request.prepare_multipart_mixed() context = PipelineContext(self._transport, **kwargs) pipeline_request = PipelineRequest(request, context) first_node = self._impl_policies[0] if self._impl_policies else _AsyncTransportRunner(self._transport) diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/base.py b/sdk/core/azure-core/azure/core/pipeline/transport/base.py index d3ec451698c1..cc72d4576821 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/base.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/base.py @@ -281,12 +281,35 @@ def set_bytes_body(self, data): self.files = None def set_multipart_mixed(self, *requests, **kwargs): + # type: (HttpRequest, Any) -> None + """Set the part of a multipart/mixed. + + Only support args for now are HttpRequest objects. + + kwargs: + - policies: SansIOPolicy to apply at preparation time + + :param requests: HttpRequests object + """ self.multipart_mixed_info = ( requests, kwargs.pop("policies", []) ) + def prepare_multipart_mixed(self): + # type: () -> None + """Will prepare the body of this request according to the multipart information. + + Does nothing if "set_multipart_mixed" was never called. + """ + if self.multipart_mixed_info: + multipart_helper = MultiPartHelper(self) + multipart_helper.prepare_request() + def serialize(self): + # type: () -> bytes + """Serialize this request using application/http spec. + """ return _serialize_request(self) From f12856c4a0263b36a6ef406cb16473d75b0020e2 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Fri, 20 Sep 2019 10:52:45 -0700 Subject: [PATCH 10/33] Core cleaning --- sdk/core/azure-core/azure/core/pipeline/base.py | 10 +--------- .../azure/core/pipeline/transport/base.py | 13 +++++++++++++ 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/base.py b/sdk/core/azure-core/azure/core/pipeline/base.py index 991a92a9dff0..fdb4dd641af9 100644 --- a/sdk/core/azure-core/azure/core/pipeline/base.py +++ b/sdk/core/azure-core/azure/core/pipeline/base.py @@ -151,12 +151,4 @@ def run(self, request, **kwargs): context = PipelineContext(self._transport, **kwargs) pipeline_request = PipelineRequest(request, context) # type: PipelineRequest first_node = self._impl_policies[0] if self._impl_policies else _TransportRunner(self._transport) - response = first_node.send(pipeline_request) # type: ignore - - if (response.http_response.content_type or '').startswith("multipart/mixed"): - from .transport.base import MultiPartHelper - multipart_helper = MultiPartHelper(request) - part_responses = multipart_helper.parse_response(response.http_response) - response.context['MULTIPART_RESPONSE'] = part_responses - - return response + return first_node.send(pipeline_request) # type: ignore diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/base.py b/sdk/core/azure-core/azure/core/pipeline/transport/base.py index cc72d4576821..a48e7ded6cca 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/base.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/base.py @@ -354,6 +354,19 @@ def text(self, encoding=None): """ return self.body().decode(encoding or "utf-8") + def parts(self): + # type: () -> Iterable + """Assuming the content-type is multipart/mixed, will return the parts as an iterable. + + :rtype: list + :raises ValueError: If the content is not multipart/mixed + """ + if not self.content_type or not self.content_type.startswith("multipart/mixed"): + raise ValueError("You can't get parts if the response is nit multipart/mixed") + + multipart_helper = MultiPartHelper(self.request) + return multipart_helper.parse_response(self) + class HttpResponse(_HttpResponseBase): def stream_download(self, pipeline): From d765053c9bad233a4679173f75a3938794c45c7f Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Fri, 20 Sep 2019 14:09:29 -0700 Subject: [PATCH 11/33] Fix tests --- .../azure/core/pipeline/transport/base.py | 13 +++ .../azure-core/tests/test_basic_transport.py | 88 +++++++++++-------- 2 files changed, 63 insertions(+), 38 deletions(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/base.py b/sdk/core/azure-core/azure/core/pipeline/transport/base.py index a48e7ded6cca..3e59c9fe8d0d 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/base.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/base.py @@ -640,10 +640,21 @@ class MultiPartHelper(object): def __init__( self, main_request, # type: HttpRequest + boundary=None, # type: str ): + """Create a multipart helper to serialize and parse multipart/mixed payload. + + boundary is optional, and one will be generate if you don't provide one. + Note that no verification are made on the boundary, this is considered advanced + enough so you know how to respect RFC1341 7.2.1 and provide a correct boundary. + + :param HttpRequest main_request: The request. + :param str boundary: Optional boundary + """ self.main_request = main_request self.requests = self.main_request.multipart_mixed_info[0] # type: List[HttpRequest] self.policies = self.main_request.multipart_mixed_info[1] # type: List[SansIOHTTPPolicy] + self._boundary = boundary def prepare_request(self): # Apply on_requests concurrently to all requests @@ -662,6 +673,8 @@ def prepare_requests(request): # Update the main request with the body main_message = Message() main_message.add_header("Content-Type", "multipart/mixed") + if self._boundary: + main_message.set_boundary(self._boundary) for i, req in enumerate(self.requests): part_message = Message() part_message.add_header('Content-Type', 'application/http') diff --git a/sdk/core/azure-core/tests/test_basic_transport.py b/sdk/core/azure-core/tests/test_basic_transport.py index 1933f22fa0d2..9de9712e19f0 100644 --- a/sdk/core/azure-core/tests/test_basic_transport.py +++ b/sdk/core/azure-core/tests/test_basic_transport.py @@ -166,24 +166,35 @@ def test_multipart_send(): policies=[header_policy] ) - helper = MultiPartHelper(request) + helper = MultiPartHelper( + request, + boundary="batch_357de4f7-6d0b-4e02-8cd2-6361411a9525" # Fix it so test are deterministic + ) helper.prepare_request() - # FIXME Boundary is random, so need to improve this test with a regexp or something assert request.body == ( - b'--===============6566992931842418154==\n' - b'content-type: application/http\n' - b'\n' - b'DELETE /container0/blob0 HTTP/1.1\n' - b'x-ms-date: Thu, 14 Jun 2018 16:46:54 GMT\n\n\n' - b'--===============6566992931842418154==\n' - b'content-type: application/http\n' - b'\n' - b'DELETE /container1/blob1 HTTP/1.1\n' - b'x-ms-date: Thu, 14 Jun 2018 16:46:54 GMT\n\n\n' - b'--===============6566992931842418154==--\n' + b'--batch_357de4f7-6d0b-4e02-8cd2-6361411a9525\r\n' + b'Content-Type: application/http\r\n' + b'Content-Transfer-Encoding: binary\r\n' + b'Content-ID: 0\r\n' + b'\r\n' + b'DELETE /container0/blob0 HTTP/1.1\r\n' + b'x-ms-date: Thu, 14 Jun 2018 16:46:54 GMT\r\n' + b'\r\n' + b'\r\n' + b'--batch_357de4f7-6d0b-4e02-8cd2-6361411a9525\r\n' + b'Content-Type: application/http\r\n' + b'Content-Transfer-Encoding: binary\r\n' + b'Content-ID: 1\r\n' + b'\r\n' + b'DELETE /container1/blob1 HTTP/1.1\r\n' + b'x-ms-date: Thu, 14 Jun 2018 16:46:54 GMT\r\n' + b'\r\n' + b'\r\n' + b'--batch_357de4f7-6d0b-4e02-8cd2-6361411a9525--\r\n' ) + def test_multipart_receive(): class MockResponse(HttpResponse): @@ -201,29 +212,29 @@ def on_response(self, request, response): response.http_response.headers['x-ms-fun'] = 'true' body_as_str = ( - "--batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed\n" - "Content-Type: application/http\n" - "Content-ID: 0\n" - "\n" - "HTTP/1.1 202 Accepted\n" - "x-ms-request-id: 778fdc83-801e-0000-62ff-0334671e284f\n" - "x-ms-version: 2018-11-09\n" - "\n" - "--batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed\n" - "Content-Type: application/http\n" - "Content-ID: 2\n" - "\n" - "HTTP/1.1 404 The specified blob does not exist.\n" - "x-ms-error-code: BlobNotFound\n" - "x-ms-request-id: 778fdc83-801e-0000-62ff-0334671e2852\n" - "x-ms-version: 2018-11-09\n" - "Content-Length: 216\n" - "Content-Type: application/xml\n" - "\n" - '\n' - "BlobNotFoundThe specified blob does not exist.\n" - "RequestId:778fdc83-801e-0000-62ff-0334671e2852\n" - "Time:2018-06-14T16:46:54.6040685Z\n" + "--batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed\r\n" + "Content-Type: application/http\r\n" + "Content-ID: 0\r\n" + "\r\n" + "HTTP/1.1 202 Accepted\r\n" + "x-ms-request-id: 778fdc83-801e-0000-62ff-0334671e284f\r\n" + "x-ms-version: 2018-11-09\r\n" + "\r\n" + "--batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed\r\n" + "Content-Type: application/http\r\n" + "Content-ID: 2\r\n" + "\r\n" + "HTTP/1.1 404 The specified blob does not exist.\r\n" + "x-ms-error-code: BlobNotFound\r\n" + "x-ms-request-id: 778fdc83-801e-0000-62ff-0334671e2852\r\n" + "x-ms-version: 2018-11-09\r\n" + "Content-Length: 216\r\n" + "Content-Type: application/xml\r\n" + "\r\n" + '\r\n' + "BlobNotFoundThe specified blob does not exist.\r\n" + "RequestId:778fdc83-801e-0000-62ff-0334671e2852\r\n" + "Time:2018-06-14T16:46:54.6040685Z\r\n" "--batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed--" ) @@ -362,5 +373,6 @@ def test_pipeline_full_scenario(): with Pipeline(RequestsTransport(), policies=[]) as pipeline: response = pipeline.run(request) - assert len(response.context['MULTIPART_RESPONSE']) == 3 - assert response.context['MULTIPART_RESPONSE'][2].status_code == 404 + parts = response.http_response.parts() + assert len(parts) == 3 + assert parts[2].status_code == 404 From 8fc8ae4bb597b3544ce950356d8edfa3ae215f23 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Fri, 20 Sep 2019 14:32:43 -0700 Subject: [PATCH 12/33] Case insensitive dict fix --- .../azure/core/pipeline/transport/base.py | 33 +++++++++++++++---- .../azure-core/tests/test_basic_transport.py | 2 ++ 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/base.py b/sdk/core/azure-core/azure/core/pipeline/transport/base.py index 3e59c9fe8d0d..bedfa99a176c 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/base.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/base.py @@ -43,11 +43,6 @@ from six.moves.http_client import HTTPConnection, HTTPResponse as _HTTPResponse from io import BytesIO -# This file is NOT using any "requests" HTTP implementation -# However, the CaseInsensitiveDict is handy. -# If one day we reach the point where "requests" can be skip totally, -# might provide our own implementation -from requests.structures import CaseInsensitiveDict from azure.core.pipeline import ABC, AbstractContextManager, PipelineRequest, PipelineResponse, PipelineContext @@ -58,6 +53,30 @@ _LOGGER = logging.getLogger(__name__) +def _case_insensitive_dict(*args, **kwargs): + """Return a case-insensitive dict from a structure that a dict would have accepted. + + Rational is I don't want to re-implement this, but I don't want + to assume "requests" or "aiohttp" are installed either. + So I use the one from "requests" or the one from "aiohttp" ("multidict") + If one day this library is used in an HTTP context without "requests" nor "aiohttp" installed, + we can add "multidict" as a dependency or re-implement our own. + """ + try: + from requests.structures import CaseInsensitiveDict + return CaseInsensitiveDict(*args, **kwargs) + except ImportError: + pass + try: + # multidict is installed by aiohttp + from multidict import CIMultiDict + return CIMultiDict(*args, **kwargs) + except ImportError: + raise ValueError( + "Neither 'requests' or 'multidict' are installed and no case-insensitive dict impl have been found" + ) + + def _format_url_section(template, **kwargs): components = template.split("/") while components: @@ -153,7 +172,7 @@ def __init__(self, method, url, headers=None, files=None, data=None): # type: (str, str, Mapping[str, str], Any, Any) -> None self.method = method self.url = url - self.headers = CaseInsensitiveDict(headers) + self.headers = _case_insensitive_dict(headers) self.files = files self.data = data self.multipart_mixed_info = None @@ -389,7 +408,7 @@ class HttpClientTransportResponse(HttpResponse): def __init__(self, request, httpclient_response): super(HttpClientTransportResponse, self).__init__(request, httpclient_response) self.status_code = httpclient_response.status - self.headers = dict(httpclient_response.getheaders()) + self.headers = _case_insensitive_dict(httpclient_response.getheaders()) self.reason = httpclient_response.reason self.content_type = self.headers.get('Content-Type') self.data = None diff --git a/sdk/core/azure-core/tests/test_basic_transport.py b/sdk/core/azure-core/tests/test_basic_transport.py index 9de9712e19f0..c40a5f6e2308 100644 --- a/sdk/core/azure-core/tests/test_basic_transport.py +++ b/sdk/core/azure-core/tests/test_basic_transport.py @@ -84,6 +84,8 @@ def test_http_client_response(): assert response.status_code == 200 assert len(response.headers.keys()) != 0 assert len(response.text()) != 0 + assert "content-type" in response.headers + assert "Content-Type" in response.headers def test_response_deserialization(): From 358208de536fbe97b413c0e3d119c66319b09389 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Fri, 20 Sep 2019 15:55:09 -0700 Subject: [PATCH 13/33] Simplify code --- sdk/core/azure-core/azure/core/pipeline/transport/base.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/base.py b/sdk/core/azure-core/azure/core/pipeline/transport/base.py index bedfa99a176c..b0b2bab2488e 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/base.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/base.py @@ -107,16 +107,12 @@ class _HTTPSerializer(HTTPConnection): """Hacking the stdlib HTTPConnection to serialize HTTP request as strings. """ def __init__(self, *args, **kwargs): - self.skip_host = kwargs.pop("skip_host", True) - self.skip_accept_encoding = kwargs.pop("skip_accept_encoding", True) self.buffer = b'' kwargs.setdefault("host", "fakehost") super(_HTTPSerializer, self).__init__(*args, **kwargs) def putheader(self, header, *values): - if self.skip_host and header == "Host": - return - if self.skip_accept_encoding and header == "Accept-Encoding": + if header in ["Host", "Accept-Encoding"]: return super(_HTTPSerializer, self).putheader(header, *values) From 6eb0f12fbd6075a59e43cf5d1121424678b8ff64 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Fri, 20 Sep 2019 16:17:48 -0700 Subject: [PATCH 14/33] pylint --- .../azure/core/pipeline/transport/base.py | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/base.py b/sdk/core/azure-core/azure/core/pipeline/transport/base.py index b0b2bab2488e..087a05b985d1 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/base.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/base.py @@ -25,6 +25,13 @@ # -------------------------------------------------------------------------- from __future__ import absolute_import import abc +from email.message import Message +from email.policy import HTTP +try: + from email import message_from_bytes as message_parser +except ImportError: # 2.7 + from email import message_from_string as message_parser +from io import BytesIO import json import logging import os @@ -41,7 +48,6 @@ Optional, Tuple, Callable, Iterator) from six.moves.http_client import HTTPConnection, HTTPResponse as _HTTPResponse -from io import BytesIO from azure.core.pipeline import ABC, AbstractContextManager, PipelineRequest, PipelineResponse, PipelineContext @@ -424,7 +430,7 @@ class BytesIOSocket(object): def __init__(self, bytes_data): self.bytes_data = bytes_data - def makefile(self, mode): + def makefile(self, _): return BytesIO(self.bytes_data) @@ -643,14 +649,6 @@ def merge(self, url, params=None, headers=None, content=None, form_content=None) return request -from email.message import Message -from email.policy import HTTP -try: - from email import message_from_bytes as message_parser -except ImportError: # 2.7 - from email import message_from_string as message_parser - - class MultiPartHelper(object): def __init__( self, @@ -683,7 +681,7 @@ def prepare_requests(request): with concurrent.futures.ThreadPoolExecutor() as executor: # List comprehension to raise exceptions if happened - [_ for _ in executor.map(prepare_requests, self.requests)] + [_ for _ in executor.map(prepare_requests, self.requests)] # pylint: disable=expression-not-assigned # Update the main request with the body main_message = Message() @@ -699,7 +697,7 @@ def prepare_requests(request): main_message.attach(part_message) full_message = main_message.as_bytes(policy=HTTP) - headers, _, body = full_message.split(b'\r\n', maxsplit=2) + _, _, body = full_message.split(b'\r\n', maxsplit=2) self.main_request.set_bytes_body(body) self.main_request.headers['Content-Type'] = 'multipart/mixed; boundary='+main_message.get_boundary() @@ -741,7 +739,6 @@ def parse_responses(response): with concurrent.futures.ThreadPoolExecutor() as executor: # List comprehension to raise exceptions if happened - [_ for _ in executor.map(parse_responses, responses)] + [_ for _ in executor.map(parse_responses, responses)] # pylint: disable=expression-not-assigned return responses - From 1b35a9e975f99845bd6ed75bda0989a54b53f17a Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Fri, 20 Sep 2019 16:24:48 -0700 Subject: [PATCH 15/33] mypy --- sdk/core/azure-core/azure/core/pipeline/base_async.py | 2 +- .../azure-core/azure/core/pipeline/transport/base.py | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/base_async.py b/sdk/core/azure-core/azure/core/pipeline/base_async.py index a40c7c2ae694..fa97ba314e95 100644 --- a/sdk/core/azure-core/azure/core/pipeline/base_async.py +++ b/sdk/core/azure-core/azure/core/pipeline/base_async.py @@ -153,7 +153,7 @@ async def __aenter__(self) -> 'AsyncPipeline': async def __aexit__(self, *exc_details): # pylint: disable=arguments-differ await self._transport.__aexit__(*exc_details) - async def run(self, request: PipelineRequest, **kwargs: Any): + async def run(self, request: HTTPRequestType, **kwargs: Any): """Runs the HTTP Request through the chained policies. :param request: The HTTP request object. diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/base.py b/sdk/core/azure-core/azure/core/pipeline/transport/base.py index 087a05b985d1..d46798491573 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/base.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/base.py @@ -30,7 +30,7 @@ try: from email import message_from_bytes as message_parser except ImportError: # 2.7 - from email import message_from_string as message_parser + from email import message_from_string as message_parser # type: ignore from io import BytesIO import json import logging @@ -45,13 +45,16 @@ import xml.etree.ElementTree as ET from typing import (TYPE_CHECKING, Generic, TypeVar, cast, IO, List, Union, Any, Mapping, Dict, # pylint: disable=unused-import - Optional, Tuple, Callable, Iterator) + Optional, Tuple, Callable, Iterator, Iterable) from six.moves.http_client import HTTPConnection, HTTPResponse as _HTTPResponse from azure.core.pipeline import ABC, AbstractContextManager, PipelineRequest, PipelineResponse, PipelineContext +if TYPE_CHECKING: + from ..policies import SansIOHTTPPolicy + HTTPResponseType = TypeVar("HTTPResponseType") HTTPRequestType = TypeVar("HTTPRequestType") PipelineType = TypeVar("PipelineType") @@ -177,7 +180,7 @@ def __init__(self, method, url, headers=None, files=None, data=None): self.headers = _case_insensitive_dict(headers) self.files = files self.data = data - self.multipart_mixed_info = None + self.multipart_mixed_info = None # type: Optional[Tuple] def __repr__(self): return '' % (self.method) @@ -665,6 +668,8 @@ def __init__( :param str boundary: Optional boundary """ self.main_request = main_request + if self.main_request.multipart_mixed_info is None: + raise ValueError("This request doesn't have multipart information available") self.requests = self.main_request.multipart_mixed_info[0] # type: List[HttpRequest] self.policies = self.main_request.multipart_mixed_info[1] # type: List[SansIOHTTPPolicy] self._boundary = boundary From 1a6fe94b38a270608961148fa7302e5a73f16f86 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Mon, 23 Sep 2019 16:27:08 -0700 Subject: [PATCH 16/33] Python 2.7 improvement --- .../azure/core/pipeline/transport/base.py | 13 ++++++++----- sdk/core/azure-core/tests/test_basic_transport.py | 13 ++++++++++--- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/base.py b/sdk/core/azure-core/azure/core/pipeline/transport/base.py index d46798491573..47fcbeae765d 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/base.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/base.py @@ -26,7 +26,6 @@ from __future__ import absolute_import import abc from email.message import Message -from email.policy import HTTP try: from email import message_from_bytes as message_parser except ImportError: # 2.7 @@ -112,7 +111,7 @@ def _urljoin(base_url, stub_url): return parsed.geturl() -class _HTTPSerializer(HTTPConnection): +class _HTTPSerializer(HTTPConnection, object): """Hacking the stdlib HTTPConnection to serialize HTTP request as strings. """ def __init__(self, *args, **kwargs): @@ -433,7 +432,7 @@ class BytesIOSocket(object): def __init__(self, bytes_data): self.bytes_data = bytes_data - def makefile(self, _): + def makefile(self, *args): return BytesIO(self.bytes_data) @@ -701,8 +700,12 @@ def prepare_requests(request): part_message.set_payload(req.serialize()) main_message.attach(part_message) - full_message = main_message.as_bytes(policy=HTTP) - _, _, body = full_message.split(b'\r\n', maxsplit=2) + try: + from email.policy import HTTP + full_message = main_message.as_bytes(policy=HTTP) + except ImportError: # Python 2.7 + full_message = main_message.as_string() + _, _, body = full_message.split(b'\r\n', 2) self.main_request.set_bytes_body(body) self.main_request.headers['Content-Type'] = 'multipart/mixed; boundary='+main_message.get_boundary() diff --git a/sdk/core/azure-core/tests/test_basic_transport.py b/sdk/core/azure-core/tests/test_basic_transport.py index c40a5f6e2308..a86d43e07a35 100644 --- a/sdk/core/azure-core/tests/test_basic_transport.py +++ b/sdk/core/azure-core/tests/test_basic_transport.py @@ -36,14 +36,21 @@ def test_http_request_serialization(): ) serialized = request.serialize() - expected = ( + # For some reason Python 2.7 rotate the headers. Both are correct. + expected = [( b'DELETE /container0/blob0 HTTP/1.1\r\n' b'x-ms-date: Thu, 14 Jun 2018 16:46:54 GMT\r\n' b'Authorization: SharedKey account:G4jjBXA7LI/RnWKIOQ8i9xH4p76pAQ+4Fs4R1VxasaE=\r\n' b'Content-Length: 0\r\n' b'\r\n' - ) - assert serialized == expected + ),( + b'DELETE /container0/blob0 HTTP/1.1\r\n' + b'Content-Length: 0\r\n' + b'Authorization: SharedKey account:G4jjBXA7LI/RnWKIOQ8i9xH4p76pAQ+4Fs4R1VxasaE=\r\n' + b'x-ms-date: Thu, 14 Jun 2018 16:46:54 GMT\r\n' + b'\r\n' + )] + assert serialized in expected # Method + Url + Headers + Body From 1c024d37179e2723097c3db45838f642c78a0381 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Mon, 23 Sep 2019 17:19:04 -0700 Subject: [PATCH 17/33] Python 2.7 improvement, part2 --- sdk/core/azure-core/azure/core/pipeline/transport/base.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/base.py b/sdk/core/azure-core/azure/core/pipeline/transport/base.py index 47fcbeae765d..76e9dcaacbc1 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/base.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/base.py @@ -703,9 +703,11 @@ def prepare_requests(request): try: from email.policy import HTTP full_message = main_message.as_bytes(policy=HTTP) + eol = b'\r\n' except ImportError: # Python 2.7 full_message = main_message.as_string() - _, _, body = full_message.split(b'\r\n', 2) + eol = b'\n' + _, _, body = full_message.split(eol, 2) self.main_request.set_bytes_body(body) self.main_request.headers['Content-Type'] = 'multipart/mixed; boundary='+main_message.get_boundary() From 98e744db6a264dde2beb0dc088f6310078f0cdca Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Tue, 24 Sep 2019 14:48:37 -0700 Subject: [PATCH 18/33] Remove unecessary test --- .../azure-core/tests/test_basic_transport.py | 64 ------------------- 1 file changed, 64 deletions(-) diff --git a/sdk/core/azure-core/tests/test_basic_transport.py b/sdk/core/azure-core/tests/test_basic_transport.py index a86d43e07a35..27021c1f2a62 100644 --- a/sdk/core/azure-core/tests/test_basic_transport.py +++ b/sdk/core/azure-core/tests/test_basic_transport.py @@ -321,67 +321,3 @@ def body(self): res0 = response[0] assert res0.status_code == 400 assert res0.body().startswith(b'\xef\xbb\xbf') - - -def test_pipeline_full_scenario(): - - # To get this test to zork, create a mocky.io: - # Content type: multipart/mixed; boundary=batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed - # Body: - """ ---batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed -Content-Type: application/http -Content-ID: 0 - -HTTP/1.1 202 Accepted -x-ms-request-id: 778fdc83-801e-0000-62ff-0334671e284f -x-ms-version: 2018-11-09 - ---batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed -Content-Type: application/http -Content-ID: 1 - -HTTP/1.1 202 Accepted -x-ms-request-id: 778fdc83-801e-0000-62ff-0334671e2851 -x-ms-version: 2018-11-09 - ---batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed -Content-Type: application/http -Content-ID: 2 - -HTTP/1.1 404 The specified blob does not exist. -x-ms-error-code: BlobNotFound -x-ms-request-id: 778fdc83-801e-0000-62ff-0334671e2852 -x-ms-version: 2018-11-09 -Content-Length: 216 -Content-Type: application/xml - - -BlobNotFoundThe specified blob does not exist. -RequestId:778fdc83-801e-0000-62ff-0334671e2852 -Time:2018-06-14T16:46:54.6040685Z ---batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed-- -""" - - header_policy = HeadersPolicy({ - 'x-ms-date': 'Thu, 14 Jun 2018 16:46:54 GMT' - }) - - req0 = HttpRequest("DELETE", "/container0/blob0") - req1 = HttpRequest("DELETE", "/container1/blob1") - req2 = HttpRequest("DELETE", "/container2/blob2") - - request = HttpRequest("GET", "http://www.mocky.io/v2/5d72e4892f0000ce9c7d4ec1") - request.set_multipart_mixed( - req0, - req1, - req2, - policies=[header_policy] - ) - - with Pipeline(RequestsTransport(), policies=[]) as pipeline: - response = pipeline.run(request) - - parts = response.http_response.parts() - assert len(parts) == 3 - assert parts[2].status_code == 404 From 4abc62f77049f7f78dda764805cf0361e3fd03d9 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Mon, 30 Sep 2019 13:51:09 -0700 Subject: [PATCH 19/33] Refactor parts --- .../azure/core/pipeline/transport/base.py | 13 ++ .../core/pipeline/transport/base_async.py | 36 ++++- .../test_basic_transport.py | 137 ++++++++++++++++++ .../azure-core/tests/test_basic_transport.py | 48 +++--- 4 files changed, 204 insertions(+), 30 deletions(-) create mode 100644 sdk/core/azure-core/tests/azure_core_asynctests/test_basic_transport.py diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/base.py b/sdk/core/azure-core/azure/core/pipeline/transport/base.py index 76e9dcaacbc1..c8663d6b0cf0 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/base.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/base.py @@ -400,6 +400,19 @@ def stream_download(self, pipeline): is supported. """ + def parts(self): + # type: () -> Iterator + """Assuming the content-type is multipart/mixed, will return the parts as an iterator. + + :rtype: iterator + :raises ValueError: If the content is not multipart/mixed + """ + if not self.content_type or not self.content_type.startswith("multipart/mixed"): + raise ValueError("You can't get parts if the response is not multipart/mixed") + + multipart_helper = MultiPartHelper(self.request) + return multipart_helper.parse_response(self) + class HttpClientTransportResponse(HttpResponse): """Create a HTTPResponse from an http.client response. diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/base_async.py b/sdk/core/azure-core/azure/core/pipeline/transport/base_async.py index 8a3ed3a522f6..74ea5932d4aa 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/base_async.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/base_async.py @@ -26,11 +26,10 @@ import asyncio import abc +from collections.abc import AsyncIterator -from typing import Any, List, Union, Callable, AsyncIterator, Optional, Generic, TypeVar -from azure.core.pipeline import PipelineRequest, PipelineResponse, Pipeline -from azure.core.pipeline.policies import SansIOHTTPPolicy -from .base import _HttpResponseBase +from typing import AsyncIterator as AsyncIteratorType, Iterator, Generic, TypeVar +from .base import _HttpResponseBase, MultiPartHelper try: from contextlib import AbstractAsyncContextManager # type: ignore @@ -65,12 +64,27 @@ def _iterate_response_content(iterator): raise _ResponseStopIteration() +class _PartGenerator(AsyncIterator): + """Until parts is a real async iterator, wrap the sync call. + + :param parts: An iterable of parts + """ + def __init__(self, parts: Iterator) -> None: + self.parts = iter(parts) + + async def __anext__(self): + try: + return next(self.parts) + except StopIteration: + raise StopAsyncIteration() + + class AsyncHttpResponse(_HttpResponseBase): """An AsyncHttpResponse ABC. Allows for the asynchronous streaming of data from the response. """ - def stream_download(self, pipeline) -> AsyncIterator[bytes]: + def stream_download(self, pipeline) -> AsyncIteratorType[bytes]: """Generator for streaming response body data. Should be implemented by sub-classes if streaming download @@ -80,6 +94,18 @@ def stream_download(self, pipeline) -> AsyncIterator[bytes]: :type pipeline: azure.core.pipeline """ + def parts(self) -> AsyncIterator: + """Assuming the content-type is multipart/mixed, will return the parts as an async iterator. + + :rtype: AsyncIterator + :raises ValueError: If the content is not multipart/mixed + """ + if not self.content_type or not self.content_type.startswith("multipart/mixed"): + raise ValueError("You can't get parts if the response is not multipart/mixed") + + multipart_helper = MultiPartHelper(self.request) + return _PartGenerator(multipart_helper.parse_response(self)) + class AsyncHttpTransport(AbstractAsyncContextManager, abc.ABC, Generic[HTTPRequestType, AsyncHTTPResponseType]): """An http sender ABC. diff --git a/sdk/core/azure-core/tests/azure_core_asynctests/test_basic_transport.py b/sdk/core/azure-core/tests/azure_core_asynctests/test_basic_transport.py new file mode 100644 index 000000000000..5cb1b6fc7913 --- /dev/null +++ b/sdk/core/azure-core/tests/azure_core_asynctests/test_basic_transport.py @@ -0,0 +1,137 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE.txt in the project root for +# license information. +# ------------------------------------------------------------------------- +from six.moves.http_client import HTTPConnection +import time + +from azure.core.pipeline.transport import HttpRequest, AsyncHttpResponse +from azure.core.pipeline.policies import HeadersPolicy +from azure.core.pipeline import AsyncPipeline + +import pytest + + +@pytest.mark.asyncio +async def test_multipart_receive(): + + class MockResponse(AsyncHttpResponse): + def __init__(self, request, body, content_type): + super(MockResponse, self).__init__(request, None) + self._body = body + self.content_type = content_type + + def body(self): + return self._body + + class ResponsePolicy(object): + def on_response(self, request, response): + # type: (PipelineRequest, PipelineResponse) -> None + response.http_response.headers['x-ms-fun'] = 'true' + + req0 = HttpRequest("DELETE", "/container0/blob0") + req1 = HttpRequest("DELETE", "/container1/blob1") + + request = HttpRequest("POST", "http://account.blob.core.windows.net/?comp=batch") + request.set_multipart_mixed( + req0, + req1, + policies=[ResponsePolicy()] + ) + + body_as_str = ( + "--batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed\r\n" + "Content-Type: application/http\r\n" + "Content-ID: 0\r\n" + "\r\n" + "HTTP/1.1 202 Accepted\r\n" + "x-ms-request-id: 778fdc83-801e-0000-62ff-0334671e284f\r\n" + "x-ms-version: 2018-11-09\r\n" + "\r\n" + "--batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed\r\n" + "Content-Type: application/http\r\n" + "Content-ID: 2\r\n" + "\r\n" + "HTTP/1.1 404 The specified blob does not exist.\r\n" + "x-ms-error-code: BlobNotFound\r\n" + "x-ms-request-id: 778fdc83-801e-0000-62ff-0334671e2852\r\n" + "x-ms-version: 2018-11-09\r\n" + "Content-Length: 216\r\n" + "Content-Type: application/xml\r\n" + "\r\n" + '\r\n' + "BlobNotFoundThe specified blob does not exist.\r\n" + "RequestId:778fdc83-801e-0000-62ff-0334671e2852\r\n" + "Time:2018-06-14T16:46:54.6040685Z\r\n" + "--batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed--" + ) + + response = MockResponse( + request, + body_as_str.encode('ascii'), + "multipart/mixed; boundary=batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed" + ) + + parts = [] + async for part in response.parts(): + parts.append(part) + + assert len(parts) == 2 + + res0 = parts[0] + assert res0.status_code == 202 + assert res0.headers['x-ms-fun'] == 'true' + + res1 = parts[1] + assert res1.status_code == 404 + assert res1.headers['x-ms-fun'] == 'true' + + +@pytest.mark.asyncio +async def test_multipart_receive_with_bom(): + + req0 = HttpRequest("DELETE", "/container0/blob0") + + request = HttpRequest("POST", "http://account.blob.core.windows.net/?comp=batch") + request.set_multipart_mixed(req0) + + class MockResponse(AsyncHttpResponse): + def __init__(self, request, body, content_type): + super(MockResponse, self).__init__(request, None) + self._body = body + self.content_type = content_type + + def body(self): + return self._body + + body_as_bytes = ( + b"--batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed\n" + b"Content-Type: application/http\n" + b"Content-Transfer-Encoding: binary\n" + b"Content-ID: 0\n" + b'\r\n' + b'HTTP/1.1 400 One of the request inputs is not valid.\r\n' + b'Content-Length: 220\r\n' + b'Content-Type: application/xml\r\n' + b'Server: Windows-Azure-Blob/1.0\r\n' + b'\r\n' + b'\xef\xbb\xbf\nInvalidInputOne' + b'of the request inputs is not valid.\nRequestId:5f3f9f2f-e01e-00cc-6eb1-6d00b5000000\nTime:2019-09-17T23:44:07.4671860Z\n' + b"--batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed--" + ) + + response = MockResponse( + request, + body_as_bytes, + "multipart/mixed; boundary=batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed" + ) + + parts = [] + async for part in response.parts(): + parts.append(part) + assert len(parts) == 1 + + res0 = parts[0] + assert res0.status_code == 400 + assert res0.body().startswith(b'\xef\xbb\xbf') diff --git a/sdk/core/azure-core/tests/test_basic_transport.py b/sdk/core/azure-core/tests/test_basic_transport.py index 27021c1f2a62..b336f38d557d 100644 --- a/sdk/core/azure-core/tests/test_basic_transport.py +++ b/sdk/core/azure-core/tests/test_basic_transport.py @@ -207,8 +207,8 @@ def test_multipart_send(): def test_multipart_receive(): class MockResponse(HttpResponse): - def __init__(self, body, content_type): - super(MockResponse, self).__init__(None, None) + def __init__(self, request, body, content_type): + super(MockResponse, self).__init__(request, None) self._body = body self.content_type = content_type @@ -220,6 +220,16 @@ def on_response(self, request, response): # type: (PipelineRequest, PipelineResponse) -> None response.http_response.headers['x-ms-fun'] = 'true' + req0 = HttpRequest("DELETE", "/container0/blob0") + req1 = HttpRequest("DELETE", "/container1/blob1") + + request = HttpRequest("POST", "http://account.blob.core.windows.net/?comp=batch") + request.set_multipart_mixed( + req0, + req1, + policies=[ResponsePolicy()] + ) + body_as_str = ( "--batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed\r\n" "Content-Type: application/http\r\n" @@ -248,23 +258,12 @@ def on_response(self, request, response): ) response = MockResponse( + request, body_as_str.encode('ascii'), "multipart/mixed; boundary=batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed" ) - req0 = HttpRequest("DELETE", "/container0/blob0") - req1 = HttpRequest("DELETE", "/container1/blob1") - - request = HttpRequest("POST", "http://account.blob.core.windows.net/?comp=batch") - request.set_multipart_mixed( - req0, - req1, - policies=[ResponsePolicy()] - ) - - helper = MultiPartHelper(request) - - response = helper.parse_response(response) + response = response.parts() assert len(response) == 2 @@ -278,9 +277,14 @@ def on_response(self, request, response): def test_multipart_receive_with_bom(): + req0 = HttpRequest("DELETE", "/container0/blob0") + + request = HttpRequest("POST", "http://account.blob.core.windows.net/?comp=batch") + request.set_multipart_mixed(req0) + class MockResponse(HttpResponse): - def __init__(self, body, content_type): - super(MockResponse, self).__init__(None, None) + def __init__(self, request, body, content_type): + super(MockResponse, self).__init__(request, None) self._body = body self.content_type = content_type @@ -304,18 +308,12 @@ def body(self): ) response = MockResponse( + request, body_as_bytes, "multipart/mixed; boundary=batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed" ) - req0 = HttpRequest("DELETE", "/container0/blob0") - - request = HttpRequest("POST", "http://account.blob.core.windows.net/?comp=batch") - request.set_multipart_mixed(req0) - - helper = MultiPartHelper(request) - - response = helper.parse_response(response) + response = response.parts() assert len(response) == 1 res0 = response[0] From c7e1788770abe6509897c1e5a0761b0019c60f48 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Mon, 30 Sep 2019 14:39:55 -0700 Subject: [PATCH 20/33] Recursive multipart --- .../azure/core/pipeline/transport/base.py | 26 ++++++-- .../core/pipeline/transport/base_async.py | 14 +++- .../test_basic_transport.py | 66 +++++++++++++++++++ .../azure-core/tests/test_basic_transport.py | 60 +++++++++++++++++ 4 files changed, 159 insertions(+), 7 deletions(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/base.py b/sdk/core/azure-core/azure/core/pipeline/transport/base.py index c8663d6b0cf0..1a9577f920e7 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/base.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/base.py @@ -414,7 +414,7 @@ def parts(self): return multipart_helper.parse_response(self) -class HttpClientTransportResponse(HttpResponse): +class _HttpClientTransportResponse(_HttpResponseBase): """Create a HTTPResponse from an http.client response. Body will NOT be read by the constructor. Call "body()" to load the body in memory if necessary. @@ -423,7 +423,7 @@ class HttpClientTransportResponse(HttpResponse): :param httpclient_response: The object returned from an HTTP(S)Connection from http.client """ def __init__(self, request, httpclient_response): - super(HttpClientTransportResponse, self).__init__(request, httpclient_response) + super(_HttpClientTransportResponse, self).__init__(request, httpclient_response) self.status_code = httpclient_response.status self.headers = _case_insensitive_dict(httpclient_response.getheaders()) self.reason = httpclient_response.reason @@ -436,6 +436,16 @@ def body(self): return self.data +class HttpClientTransportResponse(_HttpClientTransportResponse, HttpResponse): + """Create a HTTPResponse from an http.client response. + + Body will NOT be read by the constructor. Call "body()" to load the body in memory if necessary. + + :param HttpRequest request: The request. + :param httpclient_response: The object returned from an HTTP(S)Connection from http.client + """ + + class BytesIOSocket(object): """Mocking the "makefile" of socket for HTTPResponse. @@ -449,14 +459,14 @@ def makefile(self, *args): return BytesIO(self.bytes_data) -def _deserialize_response(http_response_as_bytes, http_request): +def _deserialize_response(http_response_as_bytes, http_request, http_response_type=HttpClientTransportResponse): local_socket = BytesIOSocket(http_response_as_bytes) response = _HTTPResponse( local_socket, method=http_request.method ) response.begin() - return HttpClientTransportResponse(http_request, response) + return http_response_type(http_request, response) class PipelineClientBase(object): @@ -669,6 +679,7 @@ def __init__( self, main_request, # type: HttpRequest boundary=None, # type: str + http_response_type=HttpClientTransportResponse, # type: Type[_HttpResponseBase] ): """Create a multipart helper to serialize and parse multipart/mixed payload. @@ -685,6 +696,7 @@ def __init__( self.requests = self.main_request.multipart_mixed_info[0] # type: List[HttpRequest] self.policies = self.main_request.multipart_mixed_info[1] # type: List[SansIOHTTPPolicy] self._boundary = boundary + self._http_response_type = http_response_type def prepare_request(self): # Apply on_requests concurrently to all requests @@ -740,7 +752,11 @@ def parse_response(self, response): responses = [] for request, raw_reponse in zip(self.requests, message.get_payload()): if raw_reponse.get_content_type() == "application/http": - responses.append(_deserialize_response(raw_reponse.get_payload(decode=True), request)) + responses.append(_deserialize_response( + raw_reponse.get_payload(decode=True), + request, + http_response_type=self._http_response_type + )) else: raise ValueError("Multipart doesn't support part other than application/http for now") diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/base_async.py b/sdk/core/azure-core/azure/core/pipeline/transport/base_async.py index 74ea5932d4aa..860f6eb5111b 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/base_async.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/base_async.py @@ -29,7 +29,7 @@ from collections.abc import AsyncIterator from typing import AsyncIterator as AsyncIteratorType, Iterator, Generic, TypeVar -from .base import _HttpResponseBase, MultiPartHelper +from .base import _HttpResponseBase, _HttpClientTransportResponse, MultiPartHelper try: from contextlib import AbstractAsyncContextManager # type: ignore @@ -103,10 +103,20 @@ def parts(self) -> AsyncIterator: if not self.content_type or not self.content_type.startswith("multipart/mixed"): raise ValueError("You can't get parts if the response is not multipart/mixed") - multipart_helper = MultiPartHelper(self.request) + multipart_helper = MultiPartHelper(self.request, http_response_type=AsyncHttpClientTransportResponse) return _PartGenerator(multipart_helper.parse_response(self)) +class AsyncHttpClientTransportResponse(_HttpClientTransportResponse, AsyncHttpResponse): + """Create a HTTPResponse from an http.client response. + + Body will NOT be read by the constructor. Call "body()" to load the body in memory if necessary. + + :param HttpRequest request: The request. + :param httpclient_response: The object returned from an HTTP(S)Connection from http.client + """ + + class AsyncHttpTransport(AbstractAsyncContextManager, abc.ABC, Generic[HTTPRequestType, AsyncHTTPResponseType]): """An http sender ABC. """ diff --git a/sdk/core/azure-core/tests/azure_core_asynctests/test_basic_transport.py b/sdk/core/azure-core/tests/azure_core_asynctests/test_basic_transport.py index 5cb1b6fc7913..301fcbf59ed2 100644 --- a/sdk/core/azure-core/tests/azure_core_asynctests/test_basic_transport.py +++ b/sdk/core/azure-core/tests/azure_core_asynctests/test_basic_transport.py @@ -135,3 +135,69 @@ def body(self): res0 = parts[0] assert res0.status_code == 400 assert res0.body().startswith(b'\xef\xbb\xbf') + + +@pytest.mark.asyncio +async def test_recursive_multipart_receive(): + req0 = HttpRequest("DELETE", "/container0/blob0") + internal_req0 = HttpRequest("DELETE", "/container0/blob0") + req0.set_multipart_mixed(internal_req0) + + request = HttpRequest("POST", "http://account.blob.core.windows.net/?comp=batch") + request.set_multipart_mixed(req0) + + class MockResponse(AsyncHttpResponse): + def __init__(self, request, body, content_type): + super(MockResponse, self).__init__(request, None) + self._body = body + self.content_type = content_type + + def body(self): + return self._body + + internal_body_as_str = ( + "--batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed\r\n" + "Content-Type: application/http\r\n" + "Content-ID: 0\r\n" + "\r\n" + "HTTP/1.1 400 Accepted\r\n" + "x-ms-request-id: 778fdc83-801e-0000-62ff-0334671e284f\r\n" + "x-ms-version: 2018-11-09\r\n" + "\r\n" + "--batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed--" + ) + + body_as_str = ( + "--batchresponse_8d5f5bcd-2cb5-44bb-91b5-e9a722e68cb6\r\n" + "Content-Type: application/http\r\n" + "Content-ID: 0\r\n" + "\r\n" + "HTTP/1.1 202 Accepted\r\n" + "Content-Type: multipart/mixed; boundary=batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed\r\n" + "\r\n" + "{}" + "--batchresponse_8d5f5bcd-2cb5-44bb-91b5-e9a722e68cb6--" + ).format(internal_body_as_str) + + response = MockResponse( + request, + body_as_str.encode('ascii'), + "multipart/mixed; boundary=batchresponse_8d5f5bcd-2cb5-44bb-91b5-e9a722e68cb6" + ) + + parts = [] + async for part in response.parts(): + parts.append(part) + + assert len(parts) == 1 + + res0 = parts[0] + assert res0.status_code == 202 + + internal_parts = [] + async for part in res0.parts(): + internal_parts.append(part) + assert len(internal_parts) == 1 + + internal_response0 = internal_parts[0] + assert internal_response0.status_code == 400 diff --git a/sdk/core/azure-core/tests/test_basic_transport.py b/sdk/core/azure-core/tests/test_basic_transport.py index b336f38d557d..7678c0eece68 100644 --- a/sdk/core/azure-core/tests/test_basic_transport.py +++ b/sdk/core/azure-core/tests/test_basic_transport.py @@ -319,3 +319,63 @@ def body(self): res0 = response[0] assert res0.status_code == 400 assert res0.body().startswith(b'\xef\xbb\xbf') + + +def test_recursive_multipart_receive(): + req0 = HttpRequest("DELETE", "/container0/blob0") + internal_req0 = HttpRequest("DELETE", "/container0/blob0") + req0.set_multipart_mixed(internal_req0) + + request = HttpRequest("POST", "http://account.blob.core.windows.net/?comp=batch") + request.set_multipart_mixed(req0) + + class MockResponse(HttpResponse): + def __init__(self, request, body, content_type): + super(MockResponse, self).__init__(request, None) + self._body = body + self.content_type = content_type + + def body(self): + return self._body + + internal_body_as_str = ( + "--batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed\r\n" + "Content-Type: application/http\r\n" + "Content-ID: 0\r\n" + "\r\n" + "HTTP/1.1 400 Accepted\r\n" + "x-ms-request-id: 778fdc83-801e-0000-62ff-0334671e284f\r\n" + "x-ms-version: 2018-11-09\r\n" + "\r\n" + "--batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed--" + ) + + body_as_str = ( + "--batchresponse_8d5f5bcd-2cb5-44bb-91b5-e9a722e68cb6\r\n" + "Content-Type: application/http\r\n" + "Content-ID: 0\r\n" + "\r\n" + "HTTP/1.1 202 Accepted\r\n" + "Content-Type: multipart/mixed; boundary=batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed\r\n" + "\r\n" + "{}" + "--batchresponse_8d5f5bcd-2cb5-44bb-91b5-e9a722e68cb6--" + ).format(internal_body_as_str) + + response = MockResponse( + request, + body_as_str.encode('ascii'), + "multipart/mixed; boundary=batchresponse_8d5f5bcd-2cb5-44bb-91b5-e9a722e68cb6" + ) + + response = response.parts() + assert len(response) == 1 + + res0 = response[0] + assert res0.status_code == 202 + + internal_response = res0.parts() + assert len(internal_response) == 1 + + internal_response0 = internal_response[0] + assert internal_response0.status_code == 400 From 605d9870823ee2f23df8e7227e33cf8ce5974c57 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Mon, 30 Sep 2019 14:47:22 -0700 Subject: [PATCH 21/33] No multipart serialization on 2.7 --- .../azure/core/pipeline/transport/base.py | 7 +++++-- .../azure-core/tests/test_basic_transport.py | 19 +++++++++---------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/base.py b/sdk/core/azure-core/azure/core/pipeline/transport/base.py index 1a9577f920e7..660b0dd66246 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/base.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/base.py @@ -730,8 +730,11 @@ def prepare_requests(request): full_message = main_message.as_bytes(policy=HTTP) eol = b'\r\n' except ImportError: # Python 2.7 - full_message = main_message.as_string() - eol = b'\n' + # Right now we decide to not support Python 2.7 on serialization, since + # it doesn't serialize a valid HTTP request (and our main scenario Storage refuses it) + raise NotImplementedError("Multipart request are not supported on Python 2.7") + # full_message = main_message.as_string() + # eol = b'\n' _, _, body = full_message.split(eol, 2) self.main_request.set_bytes_body(body) self.main_request.headers['Content-Type'] = 'multipart/mixed; boundary='+main_message.get_boundary() diff --git a/sdk/core/azure-core/tests/test_basic_transport.py b/sdk/core/azure-core/tests/test_basic_transport.py index 7678c0eece68..06188c215310 100644 --- a/sdk/core/azure-core/tests/test_basic_transport.py +++ b/sdk/core/azure-core/tests/test_basic_transport.py @@ -5,13 +5,17 @@ # ------------------------------------------------------------------------- from six.moves.http_client import HTTPConnection import time +import sys from azure.core.pipeline.transport import HttpRequest, HttpResponse, RequestsTransport from azure.core.pipeline.transport.base import HttpClientTransportResponse, _deserialize_response, MultiPartHelper from azure.core.pipeline.policies import HeadersPolicy from azure.core.pipeline import Pipeline +import pytest + +@pytest.mark.skipif(sys.version_info < (3, 0), reason="Multipart serialization not supported on 2.7") def test_http_request_serialization(): # Method + Url request = HttpRequest("DELETE", "/container0/blob0") @@ -36,21 +40,14 @@ def test_http_request_serialization(): ) serialized = request.serialize() - # For some reason Python 2.7 rotate the headers. Both are correct. - expected = [( + expected = ( b'DELETE /container0/blob0 HTTP/1.1\r\n' b'x-ms-date: Thu, 14 Jun 2018 16:46:54 GMT\r\n' b'Authorization: SharedKey account:G4jjBXA7LI/RnWKIOQ8i9xH4p76pAQ+4Fs4R1VxasaE=\r\n' b'Content-Length: 0\r\n' b'\r\n' - ),( - b'DELETE /container0/blob0 HTTP/1.1\r\n' - b'Content-Length: 0\r\n' - b'Authorization: SharedKey account:G4jjBXA7LI/RnWKIOQ8i9xH4p76pAQ+4Fs4R1VxasaE=\r\n' - b'x-ms-date: Thu, 14 Jun 2018 16:46:54 GMT\r\n' - b'\r\n' - )] - assert serialized in expected + ) + assert serialized == expected # Method + Url + Headers + Body @@ -159,6 +156,8 @@ def test_response_deserialization_utf8_bom(): response = _deserialize_response(body, request) assert response.body().startswith(b'\xef\xbb\xbf') + +@pytest.mark.skipif(sys.version_info < (3, 0), reason="Multipart serialization not supported on 2.7") def test_multipart_send(): header_policy = HeadersPolicy({ From d82e737e3e15c261df5f632287fbf361ff08db8b Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Mon, 30 Sep 2019 15:26:40 -0700 Subject: [PATCH 22/33] pylint --- sdk/core/azure-core/azure/core/pipeline/transport/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/base.py b/sdk/core/azure-core/azure/core/pipeline/transport/base.py index 660b0dd66246..a9fe46661185 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/base.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/base.py @@ -455,7 +455,7 @@ class BytesIOSocket(object): def __init__(self, bytes_data): self.bytes_data = bytes_data - def makefile(self, *args): + def makefile(self, *_): return BytesIO(self.bytes_data) From 2b7c844a195d80be3060a6a01d390f2c1c7bcd64 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Mon, 30 Sep 2019 15:32:53 -0700 Subject: [PATCH 23/33] mypy --- sdk/core/azure-core/azure/core/pipeline/base.py | 2 +- .../azure-core/azure/core/pipeline/base_async.py | 2 +- .../azure/core/pipeline/transport/base.py | 15 +-------------- 3 files changed, 3 insertions(+), 16 deletions(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/base.py b/sdk/core/azure-core/azure/core/pipeline/base.py index f50d4030f72f..1965a0e33f58 100644 --- a/sdk/core/azure-core/azure/core/pipeline/base.py +++ b/sdk/core/azure-core/azure/core/pipeline/base.py @@ -147,7 +147,7 @@ def run(self, request, **kwargs): :return: The PipelineResponse object :rtype: ~azure.core.pipeline.PipelineResponse """ - request.prepare_multipart_mixed() + request.prepare_multipart_mixed() # type: ignore context = PipelineContext(self._transport, **kwargs) pipeline_request = PipelineRequest(request, context) # type: PipelineRequest[HTTPRequestType] first_node = self._impl_policies[0] if self._impl_policies else _TransportRunner(self._transport) diff --git a/sdk/core/azure-core/azure/core/pipeline/base_async.py b/sdk/core/azure-core/azure/core/pipeline/base_async.py index fa97ba314e95..6832f21475ca 100644 --- a/sdk/core/azure-core/azure/core/pipeline/base_async.py +++ b/sdk/core/azure-core/azure/core/pipeline/base_async.py @@ -161,7 +161,7 @@ async def run(self, request: HTTPRequestType, **kwargs: Any): :return: The PipelineResponse object. :rtype: ~azure.core.pipeline.PipelineResponse """ - request.prepare_multipart_mixed() + request.prepare_multipart_mixed() # type: ignore context = PipelineContext(self._transport, **kwargs) pipeline_request = PipelineRequest(request, context) first_node = self._impl_policies[0] if self._impl_policies else _AsyncTransportRunner(self._transport) diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/base.py b/sdk/core/azure-core/azure/core/pipeline/transport/base.py index a9fe46661185..2ee7538bb3a1 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/base.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/base.py @@ -44,7 +44,7 @@ import xml.etree.ElementTree as ET from typing import (TYPE_CHECKING, Generic, TypeVar, cast, IO, List, Union, Any, Mapping, Dict, # pylint: disable=unused-import - Optional, Tuple, Callable, Iterator, Iterable) + Optional, Tuple, Callable, Iterator, Iterable, Type) from six.moves.http_client import HTTPConnection, HTTPResponse as _HTTPResponse @@ -377,19 +377,6 @@ def text(self, encoding=None): """ return self.body().decode(encoding or "utf-8") - def parts(self): - # type: () -> Iterable - """Assuming the content-type is multipart/mixed, will return the parts as an iterable. - - :rtype: list - :raises ValueError: If the content is not multipart/mixed - """ - if not self.content_type or not self.content_type.startswith("multipart/mixed"): - raise ValueError("You can't get parts if the response is nit multipart/mixed") - - multipart_helper = MultiPartHelper(self.request) - return multipart_helper.parse_response(self) - class HttpResponse(_HttpResponseBase): def stream_download(self, pipeline): From 9364cd5282b1b5c52c156533a01240646d0fa7cf Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Mon, 30 Sep 2019 15:35:06 -0700 Subject: [PATCH 24/33] Fix test for 3.5 --- sdk/core/azure-core/tests/test_basic_transport.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sdk/core/azure-core/tests/test_basic_transport.py b/sdk/core/azure-core/tests/test_basic_transport.py index 06188c215310..852470a9168e 100644 --- a/sdk/core/azure-core/tests/test_basic_transport.py +++ b/sdk/core/azure-core/tests/test_basic_transport.py @@ -4,6 +4,7 @@ # license information. # ------------------------------------------------------------------------- from six.moves.http_client import HTTPConnection +from collections import OrderedDict import time import sys @@ -32,11 +33,12 @@ def test_http_request_serialization(): request = HttpRequest( "DELETE", "/container0/blob0", - headers={ + # Use OrderedDict to get consistent test result on 3.5 where order is not guaranted + headers=OrderedDict({ "x-ms-date": "Thu, 14 Jun 2018 16:46:54 GMT", "Authorization": "SharedKey account:G4jjBXA7LI/RnWKIOQ8i9xH4p76pAQ+4Fs4R1VxasaE=", "Content-Length": "0", - } + }) ) serialized = request.serialize() From d0d4e5b149e73238ccbe3402e3baace6a8f8dc4e Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Mon, 30 Sep 2019 15:36:39 -0700 Subject: [PATCH 25/33] Adding six as dep for azure-core --- sdk/core/azure-core/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/core/azure-core/setup.py b/sdk/core/azure-core/setup.py index bdab06be195c..c8af32a69659 100644 --- a/sdk/core/azure-core/setup.py +++ b/sdk/core/azure-core/setup.py @@ -62,6 +62,7 @@ ]), install_requires=[ 'requests>=2.18.4', + 'six', ], extras_require={ ":python_version<'3.0'": ['azure-nspkg'], From 2a8cabdae63cc2f3b43a2ad6dfdf3db725eded08 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Mon, 30 Sep 2019 16:05:01 -0700 Subject: [PATCH 26/33] Make analyze job happy --- sdk/core/azure-core/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core/setup.py b/sdk/core/azure-core/setup.py index c8af32a69659..d53d718637a2 100644 --- a/sdk/core/azure-core/setup.py +++ b/sdk/core/azure-core/setup.py @@ -62,7 +62,7 @@ ]), install_requires=[ 'requests>=2.18.4', - 'six', + 'six>=1.6', ], extras_require={ ":python_version<'3.0'": ['azure-nspkg'], From f0d54c327fc450be08c138d337ba2357229f986d Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Tue, 1 Oct 2019 08:49:11 -0700 Subject: [PATCH 27/33] Skip test that assumes dict ordering for consistency in 3.5 --- sdk/core/azure-core/tests/test_basic_transport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core/tests/test_basic_transport.py b/sdk/core/azure-core/tests/test_basic_transport.py index 852470a9168e..c3a2b56a6229 100644 --- a/sdk/core/azure-core/tests/test_basic_transport.py +++ b/sdk/core/azure-core/tests/test_basic_transport.py @@ -16,7 +16,7 @@ import pytest -@pytest.mark.skipif(sys.version_info < (3, 0), reason="Multipart serialization not supported on 2.7") +@pytest.mark.skipif(sys.version_info < (3, 6), reason="Multipart serialization not supported on 2.7 + dict order not deterministic on 3.5") def test_http_request_serialization(): # Method + Url request = HttpRequest("DELETE", "/container0/blob0") From 4be0c711cca3bfdf81c96e98d1f1dde69d482772 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Tue, 1 Oct 2019 16:56:55 -0700 Subject: [PATCH 28/33] Accept async on_response on async scenarios --- .../azure/core/pipeline/transport/base.py | 14 ++- .../core/pipeline/transport/base_async.py | 85 ++++++++++++++++--- .../test_basic_transport.py | 9 +- 3 files changed, 93 insertions(+), 15 deletions(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/base.py b/sdk/core/azure-core/azure/core/pipeline/transport/base.py index 2ee7538bb3a1..bc1ce14ab8b8 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/base.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/base.py @@ -49,6 +49,7 @@ from six.moves.http_client import HTTPConnection, HTTPResponse as _HTTPResponse from azure.core.pipeline import ABC, AbstractContextManager, PipelineRequest, PipelineResponse, PipelineContext +from ..base import _await_result if TYPE_CHECKING: @@ -693,12 +694,15 @@ def prepare_requests(request): context = PipelineContext(None) pipeline_request = PipelineRequest(request, context) for policy in self.policies: - policy.on_request(pipeline_request) + _await_result(policy.on_request, pipeline_request) with concurrent.futures.ThreadPoolExecutor() as executor: # List comprehension to raise exceptions if happened [_ for _ in executor.map(prepare_requests, self.requests)] # pylint: disable=expression-not-assigned + self._build_final_request() + + def _build_final_request(self): # Update the main request with the body main_message = Message() main_message.add_header("Content-Type", "multipart/mixed") @@ -726,7 +730,7 @@ def prepare_requests(request): self.main_request.set_bytes_body(body) self.main_request.headers['Content-Type'] = 'multipart/mixed; boundary='+main_message.get_boundary() - def parse_response(self, response): + def _get_raw_parts(self, response): body_as_bytes = response.body() # In order to use email.message parser, I need full HTTP bytes. Faking something to make the parser happy http_body = ( @@ -749,6 +753,10 @@ def parse_response(self, response): )) else: raise ValueError("Multipart doesn't support part other than application/http for now") + return responses + + def parse_response(self, response): + responses = self._get_raw_parts(response) # Apply on_response concurrently to all requests import concurrent.futures @@ -764,7 +772,7 @@ def parse_responses(response): ) for policy in self.policies: - policy.on_response(pipeline_request, pipeline_response) + _await_result(policy.on_response, pipeline_request, pipeline_response) with concurrent.futures.ThreadPoolExecutor() as executor: # List comprehension to raise exceptions if happened diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/base_async.py b/sdk/core/azure-core/azure/core/pipeline/transport/base_async.py index 860f6eb5111b..c4f7d51ea245 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/base_async.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/base_async.py @@ -28,12 +28,21 @@ import abc from collections.abc import AsyncIterator -from typing import AsyncIterator as AsyncIteratorType, Iterator, Generic, TypeVar -from .base import _HttpResponseBase, _HttpClientTransportResponse, MultiPartHelper +from typing import AsyncIterator as AsyncIteratorType, Generic, TypeVar +from .base import ( + _HttpResponseBase, + _HttpClientTransportResponse, + MultiPartHelper, + PipelineContext, + PipelineRequest, + PipelineResponse, +) +from ..base_async import _await_result try: from contextlib import AbstractAsyncContextManager # type: ignore -except ImportError: # Python <= 3.7 +except ImportError: # Python <= 3.7 + class AbstractAsyncContextManager(object): # type: ignore async def __aenter__(self): """Return `self` upon entering the runtime context.""" @@ -69,12 +78,24 @@ class _PartGenerator(AsyncIterator): :param parts: An iterable of parts """ - def __init__(self, parts: Iterator) -> None: - self.parts = iter(parts) + + def __init__(self, response) -> None: + self._response = response + self._multipart_helper = None + self._parts = None async def __anext__(self): + if not self._multipart_helper: + self._multipart_helper = AsyncMultipartHelper( + self._response.request, + http_response_type=AsyncHttpClientTransportResponse, + ) + self._parts = iter( + await self._multipart_helper.parse_response(self._response) + ) + try: - return next(self.parts) + return next(self._parts) except StopIteration: raise StopAsyncIteration() @@ -84,6 +105,7 @@ class AsyncHttpResponse(_HttpResponseBase): Allows for the asynchronous streaming of data from the response. """ + def stream_download(self, pipeline) -> AsyncIteratorType[bytes]: """Generator for streaming response body data. @@ -101,10 +123,10 @@ def parts(self) -> AsyncIterator: :raises ValueError: If the content is not multipart/mixed """ if not self.content_type or not self.content_type.startswith("multipart/mixed"): - raise ValueError("You can't get parts if the response is not multipart/mixed") - - multipart_helper = MultiPartHelper(self.request, http_response_type=AsyncHttpClientTransportResponse) - return _PartGenerator(multipart_helper.parse_response(self)) + raise ValueError( + "You can't get parts if the response is not multipart/mixed" + ) + return _PartGenerator(self) class AsyncHttpClientTransportResponse(_HttpClientTransportResponse, AsyncHttpResponse): @@ -117,7 +139,11 @@ class AsyncHttpClientTransportResponse(_HttpClientTransportResponse, AsyncHttpRe """ -class AsyncHttpTransport(AbstractAsyncContextManager, abc.ABC, Generic[HTTPRequestType, AsyncHTTPResponseType]): +class AsyncHttpTransport( + AbstractAsyncContextManager, + abc.ABC, + Generic[HTTPRequestType, AsyncHTTPResponseType], +): """An http sender ABC. """ @@ -143,3 +169,40 @@ def __enter__(self): def __exit__(self, exc_type, exc_val, exc_tb): # __exit__ should exist in pair with __enter__ but never executed pass # pragma: no cover + + +class AsyncMultipartHelper(MultiPartHelper): + async def prepare_request(self): + async def prepare_requests(request): + context = PipelineContext(None) + pipeline_request = PipelineRequest(request, context) + for policy in self.policies: + await _await_result(policy.on_request, pipeline_request) + + # Not happy to make this code asyncio specific, but that's multipart only for now + # If we need trio and multipart, let's reinvesitgate that later + await asyncio.gather(*[prepare_requests(req) for req in self.requests]) + + self._build_final_request() + + async def parse_response(self, response): + responses = self._get_raw_parts(response) + + async def parse_responses(response): + http_request = response.request + context = PipelineContext(None) + pipeline_request = PipelineRequest(http_request, context) + pipeline_response = PipelineResponse( + http_request, response, context=context + ) + + for policy in self.policies: + await _await_result( + policy.on_response, pipeline_request, pipeline_response + ) + + # Not happy to make this code asyncio specific, but that's multipart only for now + # If we need trio and multipart, let's reinvesitgate that later + await asyncio.gather(*[parse_responses(res) for res in responses]) + + return responses diff --git a/sdk/core/azure-core/tests/azure_core_asynctests/test_basic_transport.py b/sdk/core/azure-core/tests/azure_core_asynctests/test_basic_transport.py index 301fcbf59ed2..81815da4896a 100644 --- a/sdk/core/azure-core/tests/azure_core_asynctests/test_basic_transport.py +++ b/sdk/core/azure-core/tests/azure_core_asynctests/test_basic_transport.py @@ -30,6 +30,11 @@ def on_response(self, request, response): # type: (PipelineRequest, PipelineResponse) -> None response.http_response.headers['x-ms-fun'] = 'true' + class AsyncResponsePolicy(object): + async def on_response(self, request, response): + # type: (PipelineRequest, PipelineResponse) -> None + response.http_response.headers['x-ms-async-fun'] = 'true' + req0 = HttpRequest("DELETE", "/container0/blob0") req1 = HttpRequest("DELETE", "/container1/blob1") @@ -37,7 +42,7 @@ def on_response(self, request, response): request.set_multipart_mixed( req0, req1, - policies=[ResponsePolicy()] + policies=[ResponsePolicy(), AsyncResponsePolicy()] ) body_as_str = ( @@ -82,10 +87,12 @@ def on_response(self, request, response): res0 = parts[0] assert res0.status_code == 202 assert res0.headers['x-ms-fun'] == 'true' + assert res0.headers['x-ms-async-fun'] == 'true' res1 = parts[1] assert res1.status_code == 404 assert res1.headers['x-ms-fun'] == 'true' + assert res1.headers['x-ms-async-fun'] == 'true' @pytest.mark.asyncio From 50b68ef225824ab0c22aded2bb00fe116d7982a4 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Tue, 1 Oct 2019 17:01:27 -0700 Subject: [PATCH 29/33] Async on_request support --- .../test_basic_transport.py | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/sdk/core/azure-core/tests/azure_core_asynctests/test_basic_transport.py b/sdk/core/azure-core/tests/azure_core_asynctests/test_basic_transport.py index 81815da4896a..2ca9e637c4a7 100644 --- a/sdk/core/azure-core/tests/azure_core_asynctests/test_basic_transport.py +++ b/sdk/core/azure-core/tests/azure_core_asynctests/test_basic_transport.py @@ -7,12 +7,60 @@ import time from azure.core.pipeline.transport import HttpRequest, AsyncHttpResponse +from azure.core.pipeline.transport.base_async import AsyncMultipartHelper from azure.core.pipeline.policies import HeadersPolicy from azure.core.pipeline import AsyncPipeline import pytest +@pytest.mark.asyncio +async def test_multipart_send(): + + class RequestPolicy(object): + async def on_request(self, request): + # type: (PipelineRequest) -> None + request.http_request.headers['x-ms-date'] = 'Thu, 14 Jun 2018 16:46:54 GMT' + + req0 = HttpRequest("DELETE", "/container0/blob0") + req1 = HttpRequest("DELETE", "/container1/blob1") + + request = HttpRequest("POST", "http://account.blob.core.windows.net/?comp=batch") + request.set_multipart_mixed( + req0, + req1, + policies=[RequestPolicy()] + ) + + helper = AsyncMultipartHelper( + request, + boundary="batch_357de4f7-6d0b-4e02-8cd2-6361411a9525" # Fix it so test are deterministic + ) + await helper.prepare_request() + + assert request.body == ( + b'--batch_357de4f7-6d0b-4e02-8cd2-6361411a9525\r\n' + b'Content-Type: application/http\r\n' + b'Content-Transfer-Encoding: binary\r\n' + b'Content-ID: 0\r\n' + b'\r\n' + b'DELETE /container0/blob0 HTTP/1.1\r\n' + b'x-ms-date: Thu, 14 Jun 2018 16:46:54 GMT\r\n' + b'\r\n' + b'\r\n' + b'--batch_357de4f7-6d0b-4e02-8cd2-6361411a9525\r\n' + b'Content-Type: application/http\r\n' + b'Content-Transfer-Encoding: binary\r\n' + b'Content-ID: 1\r\n' + b'\r\n' + b'DELETE /container1/blob1 HTTP/1.1\r\n' + b'x-ms-date: Thu, 14 Jun 2018 16:46:54 GMT\r\n' + b'\r\n' + b'\r\n' + b'--batch_357de4f7-6d0b-4e02-8cd2-6361411a9525--\r\n' + ) + + @pytest.mark.asyncio async def test_multipart_receive(): From b1046f3c1531362148e90acee66d245ef65745fc Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Wed, 2 Oct 2019 10:43:34 -0700 Subject: [PATCH 30/33] Complete support of async sansio in multipart with pipeline tests --- .../azure/core/pipeline/__init__.py | 40 +- .../azure-core/azure/core/pipeline/base.py | 73 ++- .../azure/core/pipeline/base_async.py | 68 ++- .../azure/core/pipeline/transport/base.py | 480 ++++++++++-------- .../core/pipeline/transport/base_async.py | 82 ++- .../test_basic_transport.py | 31 +- .../azure-core/tests/test_basic_transport.py | 19 +- 7 files changed, 464 insertions(+), 329 deletions(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/__init__.py b/sdk/core/azure-core/azure/core/pipeline/__init__.py index b7b06deacbf6..a5458c4c450f 100644 --- a/sdk/core/azure-core/azure/core/pipeline/__init__.py +++ b/sdk/core/azure-core/azure/core/pipeline/__init__.py @@ -25,19 +25,22 @@ # -------------------------------------------------------------------------- import abc -from typing import (TypeVar, Any, Dict, Optional, Generic) +from typing import TypeVar, Generic try: ABC = abc.ABC -except AttributeError: # Python 2.7, abc exists, but not ABC - ABC = abc.ABCMeta('ABC', (object,), {'__slots__': ()}) # type: ignore +except AttributeError: # Python 2.7, abc exists, but not ABC + ABC = abc.ABCMeta("ABC", (object,), {"__slots__": ()}) # type: ignore HTTPResponseType = TypeVar("HTTPResponseType") HTTPRequestType = TypeVar("HTTPRequestType") try: - from contextlib import AbstractContextManager # type: ignore #pylint: disable=unused-import -except ImportError: # Python <= 3.5 + from contextlib import ( # pylint: disable=unused-import + AbstractContextManager, + ) # type: ignore +except ImportError: # Python <= 3.5 + class AbstractContextManager(object): # type: ignore def __enter__(self): """Return `self` upon entering the runtime context.""" @@ -60,19 +63,20 @@ class PipelineContext(dict): :param transport: The HTTP transport type. :param kwargs: Developer-defined keyword arguments. """ - def __init__(self, transport, **kwargs): #pylint: disable=super-init-not-called + + def __init__(self, transport, **kwargs): # pylint: disable=super-init-not-called self.transport = transport self.options = kwargs - self._protected = ['transport', 'options'] + self._protected = ["transport", "options"] def __setitem__(self, key, item): if key in self._protected: - raise ValueError('Context value {} cannot be overwritten.'.format(key)) + raise ValueError("Context value {} cannot be overwritten.".format(key)) return super(PipelineContext, self).__setitem__(key, item) def __delitem__(self, key): if key in self._protected: - raise ValueError('Context value {} cannot be deleted.'.format(key)) + raise ValueError("Context value {} cannot be deleted.".format(key)) return super(PipelineContext, self).__delitem__(key) def clear(self): @@ -93,7 +97,7 @@ def pop(self, *args): """Removes specified key and returns the value. """ if args and args[0] in self._protected: - raise ValueError('Context value {} cannot be popped.'.format(args[0])) + raise ValueError("Context value {} cannot be popped.".format(args[0])) return super(PipelineContext, self).pop(*args) @@ -108,6 +112,7 @@ class PipelineRequest(Generic[HTTPRequestType]): :param context: Contains the context - data persisted between pipeline requests. :type context: ~azure.core.pipeline.PipelineContext """ + def __init__(self, http_request, context): # type: (HTTPRequestType, PipelineContext) -> None self.http_request = http_request @@ -131,6 +136,7 @@ class PipelineResponse(Generic[HTTPRequestType, HTTPResponseType]): :param context: Contains the context - data persisted between pipeline requests. :type context: ~azure.core.pipeline.PipelineContext """ + def __init__(self, http_request, http_response, context): # type: (HTTPRequestType, HTTPResponseType, PipelineContext) -> None self.http_request = http_request @@ -138,17 +144,13 @@ def __init__(self, http_request, http_response, context): self.context = context -from .base import Pipeline #pylint: disable=wrong-import-position +from .base import Pipeline # pylint: disable=wrong-import-position -__all__ = [ - 'Pipeline', - 'PipelineRequest', - 'PipelineResponse', - 'PipelineContext' -] +__all__ = ["Pipeline", "PipelineRequest", "PipelineResponse", "PipelineContext"] try: - from .base_async import AsyncPipeline #pylint: disable=unused-import - __all__.append('AsyncPipeline') + from .base_async import AsyncPipeline # pylint: disable=unused-import + + __all__.append("AsyncPipeline") except (SyntaxError, ImportError): pass # Asynchronous pipelines not supported. diff --git a/sdk/core/azure-core/azure/core/pipeline/base.py b/sdk/core/azure-core/azure/core/pipeline/base.py index a4450eb59fe6..109827c818e4 100644 --- a/sdk/core/azure-core/azure/core/pipeline/base.py +++ b/sdk/core/azure-core/azure/core/pipeline/base.py @@ -25,10 +25,15 @@ # -------------------------------------------------------------------------- import logging -from typing import (TYPE_CHECKING, Generic, TypeVar, cast, IO, List, Union, Any, Mapping, Dict, Optional, # pylint: disable=unused-import - Tuple, Callable, Iterator) -from azure.core.pipeline import AbstractContextManager, PipelineRequest, PipelineResponse, PipelineContext +from typing import Generic, TypeVar, List, Union, Any +from azure.core.pipeline import ( + AbstractContextManager, + PipelineRequest, + PipelineResponse, + PipelineContext, +) from azure.core.pipeline.policies import HTTPPolicy, SansIOHTTPPolicy + HTTPResponseType = TypeVar("HTTPResponseType") HTTPRequestType = TypeVar("HTTPRequestType") HttpTransportType = TypeVar("HttpTransportType") @@ -40,8 +45,10 @@ def _await_result(func, *args, **kwargs): """If func returns an awaitable, raise that this runner can't handle it.""" result = func(*args, **kwargs) - if hasattr(result, '__await__'): - raise TypeError("Policy {} returned awaitable object in non-async pipeline.".format(func)) + if hasattr(result, "__await__"): + raise TypeError( + "Policy {} returned awaitable object in non-async pipeline.".format(func) + ) return result @@ -71,7 +78,7 @@ def send(self, request): _await_result(self._policy.on_request, request) try: response = self.next.send(request) - except Exception: #pylint: disable=broad-except + except Exception: # pylint: disable=broad-except if not _await_result(self._policy.on_exception, request): raise else: @@ -86,6 +93,7 @@ class _TransportRunner(HTTPPolicy): :param sender: The Http Transport instance. """ + def __init__(self, sender): # type: (HttpTransportType) -> None super(_TransportRunner, self).__init__() @@ -102,7 +110,7 @@ def send(self, request): return PipelineResponse( request.http_request, self._sender.send(request.http_request, **request.context.options), - context=request.context + context=request.context, ) @@ -123,29 +131,59 @@ class Pipeline(AbstractContextManager, Generic[HTTPRequestType, HTTPResponseType :dedent: 4 :caption: Builds the pipeline for synchronous transport. """ + def __init__(self, transport, policies=None): # type: (HttpTransportType, PoliciesType) -> None self._impl_policies = [] # type: List[HTTPPolicy] self._transport = transport # type: ignore - for policy in (policies or []): + for policy in policies or []: if isinstance(policy, SansIOHTTPPolicy): self._impl_policies.append(_SansIOHTTPPolicyRunner(policy)) elif policy: self._impl_policies.append(policy) - for index in range(len(self._impl_policies)-1): - self._impl_policies[index].next = self._impl_policies[index+1] + for index in range(len(self._impl_policies) - 1): + self._impl_policies[index].next = self._impl_policies[index + 1] if self._impl_policies: self._impl_policies[-1].next = _TransportRunner(self._transport) def __enter__(self): # type: () -> Pipeline - self._transport.__enter__() # type: ignore + self._transport.__enter__() # type: ignore return self def __exit__(self, *exc_details): # pylint: disable=arguments-differ self._transport.__exit__(*exc_details) + @staticmethod + def _prepare_multipart_mixed_request(request): + # type: (HTTPRequestType) -> None + """Will execute the multipart policies. + + Does nothing if "set_multipart_mixed" was never called. + """ + multipart_mixed_info = request.multipart_mixed_info # type: ignore + if not multipart_mixed_info: + return + + requests = multipart_mixed_info[0] # type: List[HTTPRequestType] + policies = multipart_mixed_info[1] # type: List[SansIOHTTPPolicy] + + # Apply on_requests concurrently to all requests + import concurrent.futures + + def prepare_requests(req): + context = PipelineContext(None) + pipeline_request = PipelineRequest(req, context) + for policy in policies: + _await_result(policy.on_request, pipeline_request) + + with concurrent.futures.ThreadPoolExecutor() as executor: + # List comprehension to raise exceptions if happened + [ # pylint: disable=expression-not-assigned + _ for _ in executor.map(prepare_requests, requests) + ] + def run(self, request, **kwargs): # type: (HTTPRequestType, Any) -> PipelineResponse """Runs the HTTP Request through the chained policies. @@ -155,8 +193,15 @@ def run(self, request, **kwargs): :return: The PipelineResponse object :rtype: ~azure.core.pipeline.PipelineResponse """ - request.prepare_multipart_mixed() # type: ignore + self._prepare_multipart_mixed_request(request) + request.prepare_multipart_body() # type: ignore context = PipelineContext(self._transport, **kwargs) - pipeline_request = PipelineRequest(request, context) # type: PipelineRequest[HTTPRequestType] - first_node = self._impl_policies[0] if self._impl_policies else _TransportRunner(self._transport) + pipeline_request = PipelineRequest( + request, context + ) # type: PipelineRequest[HTTPRequestType] + first_node = ( + self._impl_policies[0] + if self._impl_policies + else _TransportRunner(self._transport) + ) return first_node.send(pipeline_request) # type: ignore diff --git a/sdk/core/azure-core/azure/core/pipeline/base_async.py b/sdk/core/azure-core/azure/core/pipeline/base_async.py index c806c4111225..d772e634b6b9 100644 --- a/sdk/core/azure-core/azure/core/pipeline/base_async.py +++ b/sdk/core/azure-core/azure/core/pipeline/base_async.py @@ -32,12 +32,15 @@ AsyncHTTPResponseType = TypeVar("AsyncHTTPResponseType") HTTPRequestType = TypeVar("HTTPRequestType") -ImplPoliciesType = List[AsyncHTTPPolicy[HTTPRequestType, AsyncHTTPResponseType]] #pylint: disable=unsubscriptable-object +ImplPoliciesType = List[ + AsyncHTTPPolicy[HTTPRequestType, AsyncHTTPResponseType] +] # pylint: disable=unsubscriptable-object AsyncPoliciesType = List[Union[AsyncHTTPPolicy, SansIOHTTPPolicy]] try: from contextlib import AbstractAsyncContextManager # type: ignore -except ImportError: # Python <= 3.7 +except ImportError: # Python <= 3.7 + class AbstractAsyncContextManager(object): # type: ignore async def __aenter__(self): """Return `self` upon entering the runtime context.""" @@ -52,13 +55,15 @@ async def __aexit__(self, exc_type, exc_value, traceback): async def _await_result(func, *args, **kwargs): """If func returns an awaitable, await it.""" result = func(*args, **kwargs) - if hasattr(result, '__await__'): + if hasattr(result, "__await__"): # type ignore on await: https://github.com/python/mypy/issues/7587 return await result # type: ignore return result -class _SansIOAsyncHTTPPolicyRunner(AsyncHTTPPolicy[HTTPRequestType, AsyncHTTPResponseType]): #pylint: disable=unsubscriptable-object +class _SansIOAsyncHTTPPolicyRunner( + AsyncHTTPPolicy[HTTPRequestType, AsyncHTTPResponseType] +): # pylint: disable=unsubscriptable-object """Async implementation of the SansIO policy. Modifies the request and sends to the next policy in the chain. @@ -82,7 +87,7 @@ async def send(self, request: PipelineRequest) -> PipelineResponse: await _await_result(self._policy.on_request, request) try: response = await self.next.send(request) # type: ignore - except Exception: #pylint: disable=broad-except + except Exception: # pylint: disable=broad-except if not await _await_result(self._policy.on_exception, request): raise else: @@ -90,13 +95,16 @@ async def send(self, request: PipelineRequest) -> PipelineResponse: return response -class _AsyncTransportRunner(AsyncHTTPPolicy[HTTPRequestType, AsyncHTTPResponseType]): #pylint: disable=unsubscriptable-object +class _AsyncTransportRunner( + AsyncHTTPPolicy[HTTPRequestType, AsyncHTTPResponseType] +): # pylint: disable=unsubscriptable-object """Async Transport runner. Uses specified HTTP transport type to send request and returns response. :param sender: The async Http Transport instance. """ + def __init__(self, sender) -> None: super(_AsyncTransportRunner, self).__init__() self._sender = sender @@ -112,11 +120,13 @@ async def send(self, request): return PipelineResponse( request.http_request, await self._sender.send(request.http_request, **request.context.options), - request.context + request.context, ) -class AsyncPipeline(AbstractAsyncContextManager, Generic[HTTPRequestType, AsyncHTTPResponseType]): +class AsyncPipeline( + AbstractAsyncContextManager, Generic[HTTPRequestType, AsyncHTTPResponseType] +): """Async pipeline implementation. This is implemented as a context manager, that will activate the context @@ -138,13 +148,13 @@ def __init__(self, transport, policies: AsyncPoliciesType = None) -> None: self._impl_policies = [] # type: ImplPoliciesType self._transport = transport - for policy in (policies or []): + for policy in policies or []: if isinstance(policy, SansIOHTTPPolicy): self._impl_policies.append(_SansIOAsyncHTTPPolicyRunner(policy)) elif policy: self._impl_policies.append(policy) - for index in range(len(self._impl_policies)-1): - self._impl_policies[index].next = self._impl_policies[index+1] + for index in range(len(self._impl_policies) - 1): + self._impl_policies[index].next = self._impl_policies[index + 1] if self._impl_policies: self._impl_policies[-1].next = _AsyncTransportRunner(self._transport) @@ -155,13 +165,38 @@ def __exit__(self, exc_type, exc_val, exc_tb): # __exit__ should exist in pair with __enter__ but never executed pass # pragma: no cover - async def __aenter__(self) -> 'AsyncPipeline': + async def __aenter__(self) -> "AsyncPipeline": await self._transport.__aenter__() return self async def __aexit__(self, *exc_details): # pylint: disable=arguments-differ await self._transport.__aexit__(*exc_details) + async def _prepare_multipart_mixed_request(self, request): + # type: (HTTPRequestType) -> None + """Will execute the multipart policies. + + Does nothing if "set_multipart_mixed" was never called. + """ + multipart_mixed_info = request.multipart_mixed_info # type: ignore + if not multipart_mixed_info: + return + + requests = multipart_mixed_info[0] # type: List[HTTPRequestType] + policies = multipart_mixed_info[1] # type: List[SansIOHTTPPolicy] + + async def prepare_requests(req): + context = PipelineContext(None) + pipeline_request = PipelineRequest(req, context) + for policy in policies: + await _await_result(policy.on_request, pipeline_request) + + # Not happy to make this code asyncio specific, but that's multipart only for now + # If we need trio and multipart, let's reinvesitgate that later + import asyncio + + await asyncio.gather(*[prepare_requests(req) for req in requests]) + async def run(self, request: HTTPRequestType, **kwargs: Any): """Runs the HTTP Request through the chained policies. @@ -170,8 +205,13 @@ async def run(self, request: HTTPRequestType, **kwargs: Any): :return: The PipelineResponse object. :rtype: ~azure.core.pipeline.PipelineResponse """ - request.prepare_multipart_mixed() # type: ignore + await self._prepare_multipart_mixed_request(request) + request.prepare_multipart_body() # type: ignore context = PipelineContext(self._transport, **kwargs) pipeline_request = PipelineRequest(request, context) - first_node = self._impl_policies[0] if self._impl_policies else _AsyncTransportRunner(self._transport) + first_node = ( + self._impl_policies[0] + if self._impl_policies + else _AsyncTransportRunner(self._transport) + ) return await first_node.send(pipeline_request) # type: ignore diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/base.py b/sdk/core/azure-core/azure/core/pipeline/transport/base.py index bc1ce14ab8b8..ac4951cf7d0b 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/base.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/base.py @@ -26,29 +26,50 @@ from __future__ import absolute_import import abc from email.message import Message + try: from email import message_from_bytes as message_parser -except ImportError: # 2.7 +except ImportError: # 2.7 from email import message_from_string as message_parser # type: ignore from io import BytesIO import json import logging import os import time + try: binary_type = str - from urlparse import urlparse # type: ignore + from urlparse import urlparse # type: ignore except ImportError: - binary_type = bytes # type: ignore + binary_type = bytes # type: ignore from urllib.parse import urlparse import xml.etree.ElementTree as ET -from typing import (TYPE_CHECKING, Generic, TypeVar, cast, IO, List, Union, Any, Mapping, Dict, # pylint: disable=unused-import - Optional, Tuple, Callable, Iterator, Iterable, Type) +from typing import ( + TYPE_CHECKING, + Generic, + TypeVar, + cast, + IO, + List, + Union, + Any, + Mapping, + Dict, + Optional, + Tuple, + Iterator, +) from six.moves.http_client import HTTPConnection, HTTPResponse as _HTTPResponse -from azure.core.pipeline import ABC, AbstractContextManager, PipelineRequest, PipelineResponse, PipelineContext +from azure.core.pipeline import ( + ABC, + AbstractContextManager, + PipelineRequest, + PipelineResponse, + PipelineContext, +) from ..base import _await_result @@ -73,12 +94,14 @@ def _case_insensitive_dict(*args, **kwargs): """ try: from requests.structures import CaseInsensitiveDict + return CaseInsensitiveDict(*args, **kwargs) except ImportError: pass try: # multidict is installed by aiohttp from multidict import CIMultiDict + return CIMultiDict(*args, **kwargs) except ImportError: raise ValueError( @@ -93,7 +116,9 @@ def _format_url_section(template, **kwargs): return template.format(**kwargs) except KeyError as key: formatted_components = template.split("/") - components = [c for c in formatted_components if "{{{}}}".format(key.args[0]) not in c] + components = [ + c for c in formatted_components if "{{{}}}".format(key.args[0]) not in c + ] template = "/".join(components) # No URL sections left - returning None @@ -108,15 +133,16 @@ def _urljoin(base_url, stub_url): :rtype: str """ parsed = urlparse(base_url) - parsed = parsed._replace(path=parsed.path + '/' + stub_url) + parsed = parsed._replace(path=parsed.path + "/" + stub_url) return parsed.geturl() class _HTTPSerializer(HTTPConnection, object): """Hacking the stdlib HTTPConnection to serialize HTTP request as strings. """ + def __init__(self, *args, **kwargs): - self.buffer = b'' + self.buffer = b"" kwargs.setdefault("host", "fakehost") super(_HTTPSerializer, self).__init__(*args, **kwargs) @@ -128,18 +154,21 @@ def putheader(self, header, *values): def send(self, data): self.buffer += data + def _serialize_request(http_request): serializer = _HTTPSerializer() serializer.request( method=http_request.method, url=http_request.url, body=http_request.body, - headers=http_request.headers + headers=http_request.headers, ) return serializer.buffer -class HttpTransport(AbstractContextManager, ABC, Generic[HTTPRequestType, HTTPResponseType]): # type: ignore +class HttpTransport( + AbstractContextManager, ABC, Generic[HTTPRequestType, HTTPResponseType] +): # type: ignore """An http sender ABC. """ @@ -157,7 +186,7 @@ def open(self): def close(self): """Close the session if it is not externally owned.""" - def sleep(self, duration): #pylint: disable=no-self-use + def sleep(self, duration): # pylint: disable=no-self-use time.sleep(duration) @@ -173,6 +202,7 @@ class HttpRequest(object): :param data: Body to be sent. :type data: bytes or str. """ + def __init__(self, method, url, headers=None, files=None, data=None): # type: (str, str, Mapping[str, str], Any, Any) -> None self.method = method @@ -183,14 +213,14 @@ def __init__(self, method, url, headers=None, files=None, data=None): self.multipart_mixed_info = None # type: Optional[Tuple] def __repr__(self): - return '' % (self.method) + return "" % (self.method) @property def query(self): """The query parameters of the request as a dict.""" query = urlparse(self.url).query if query: - return {p[0]: p[-1] for p in [p.partition('=') for p in query.split('&')]} + return {p[0]: p[-1] for p in [p.partition("=") for p in query.split("&")]} return {} @property @@ -211,11 +241,11 @@ def _format_data(data): :param data: The request field data. :type data: str or file-like object. """ - if hasattr(data, 'read'): + if hasattr(data, "read"): data = cast(IO, data) data_name = None try: - if data.name[0] != '<' and data.name[-1] != '>': + if data.name[0] != "<" and data.name[-1] != ">": data_name = os.path.basename(data.name) except (AttributeError, TypeError): pass @@ -232,14 +262,13 @@ def format_parameters(self, params): """ query = urlparse(self.url).query if query: - self.url = self.url.partition('?')[0] + self.url = self.url.partition("?")[0] existing_params = { - p[0]: p[-1] - for p in [p.partition('=') for p in query.split('&')] + p[0]: p[-1] for p in [p.partition("=") for p in query.split("&")] } params.update(existing_params) query_params = ["{}={}".format(k, v) for k, v in params.items()] - query = '?' + '&'.join(query_params) + query = "?" + "&".join(query_params) self.url = self.url + query def set_streamed_data_body(self, data): @@ -247,9 +276,12 @@ def set_streamed_data_body(self, data): :param data: The request field data. """ - if not isinstance(data, binary_type) and \ - not any(hasattr(data, attr) for attr in ["read", "__iter__", "__aiter__"]): - raise TypeError("A streamable data source must be an open file-like object or iterable.") + if not isinstance(data, binary_type) and not any( + hasattr(data, attr) for attr in ["read", "__iter__", "__aiter__"] + ): + raise TypeError( + "A streamable data source must be an open file-like object or iterable." + ) self.data = data self.files = None @@ -263,7 +295,7 @@ def set_xml_body(self, data): else: bytes_data = ET.tostring(data, encoding="utf8") self.data = bytes_data.replace(b"encoding='utf8'", b"encoding='utf-8'") - self.headers['Content-Length'] = str(len(self.data)) + self.headers["Content-Length"] = str(len(self.data)) self.files = None def set_json_body(self, data): @@ -275,7 +307,7 @@ def set_json_body(self, data): self.data = None else: self.data = json.dumps(data) - self.headers['Content-Length'] = str(len(self.data)) + self.headers["Content-Length"] = str(len(self.data)) self.files = None def set_formdata_body(self, data=None): @@ -285,13 +317,15 @@ def set_formdata_body(self, data=None): """ if data is None: data = {} - content_type = self.headers.pop('Content-Type', None) if self.headers else None + content_type = self.headers.pop("Content-Type", None) if self.headers else None - if content_type and content_type.lower() == 'application/x-www-form-urlencoded': + if content_type and content_type.lower() == "application/x-www-form-urlencoded": self.data = {f: d for f, d in data.items() if d is not None} self.files = None - else: # Assume "multipart/form-data" - self.files = {f: self._format_data(d) for f, d in data.items() if d is not None} + else: # Assume "multipart/form-data" + self.files = { + f: self._format_data(d) for f, d in data.items() if d is not None + } self.data = None def set_bytes_body(self, data): @@ -300,7 +334,7 @@ def set_bytes_body(self, data): :param data: The request field data. """ if data: - self.headers['Content-Length'] = str(len(data)) + self.headers["Content-Length"] = str(len(data)) self.data = data self.files = None @@ -310,25 +344,68 @@ def set_multipart_mixed(self, *requests, **kwargs): Only support args for now are HttpRequest objects. + boundary is optional, and one will be generated if you don't provide one. + Note that no verification are made on the boundary, this is considered advanced + enough so you know how to respect RFC1341 7.2.1 and provide a correct boundary. + kwargs: - policies: SansIOPolicy to apply at preparation time + - boundary: Optional boundary :param requests: HttpRequests object """ self.multipart_mixed_info = ( requests, - kwargs.pop("policies", []) + kwargs.pop("policies", []), + kwargs.pop("boundary", []), ) - def prepare_multipart_mixed(self): + def prepare_multipart_body(self): # type: () -> None """Will prepare the body of this request according to the multipart information. + This call assumes the on_request policies have been applied already in their + correct context (sync/async) + Does nothing if "set_multipart_mixed" was never called. """ - if self.multipart_mixed_info: - multipart_helper = MultiPartHelper(self) - multipart_helper.prepare_request() + if not self.multipart_mixed_info: + return + + requests = self.multipart_mixed_info[0] # type: List[HttpRequest] + boundary = self.multipart_mixed_info[2] # type: Optional[str] + + # Update the main request with the body + main_message = Message() + main_message.add_header("Content-Type", "multipart/mixed") + if boundary: + main_message.set_boundary(boundary) + for i, req in enumerate(requests): + part_message = Message() + part_message.add_header("Content-Type", "application/http") + part_message.add_header("Content-Transfer-Encoding", "binary") + part_message.add_header("Content-ID", str(i)) + part_message.set_payload(req.serialize()) + main_message.attach(part_message) + + try: + from email.policy import HTTP + + full_message = main_message.as_bytes(policy=HTTP) + eol = b"\r\n" + except ImportError: # Python 2.7 + # Right now we decide to not support Python 2.7 on serialization, since + # it doesn't serialize a valid HTTP request (and our main scenario Storage refuses it) + raise NotImplementedError( + "Multipart request are not supported on Python 2.7" + ) + # full_message = main_message.as_string() + # eol = b'\n' + _, _, body = full_message.split(eol, 2) + self.set_bytes_body(body) + self.headers["Content-Type"] = ( + "multipart/mixed; boundary=" + main_message.get_boundary() + ) def serialize(self): # type: () -> bytes @@ -353,6 +430,7 @@ class _HttpResponseBase(object): :param str content_type: The content type. :param int block_size: Defaults to 4096 bytes. """ + def __init__(self, request, internal_response, block_size=None): # type: (HttpRequest, Any, Optional[int]) -> None self.request = request @@ -363,11 +441,11 @@ def __init__(self, request, internal_response, block_size=None): self.content_type = None # type: Optional[str] self.block_size = block_size or 4096 # Default to same as Requests - def body(self): # type: () -> bytes """Return the whole body as bytes in memory. """ + raise NotImplementedError() def text(self, encoding=None): # type: (str) -> str @@ -378,8 +456,47 @@ def text(self, encoding=None): """ return self.body().decode(encoding or "utf-8") + def _get_raw_parts(self, http_response_type=None): + # type (Optional[Type[_HttpResponseBase]]) -> Iterator[HttpResponse] + """Assuming this body is multipart, return the iterator or parts. -class HttpResponse(_HttpResponseBase): + If parts are application/http use http_response_type or HttpClientTransportResponse + as enveloppe. + """ + if http_response_type is None: + http_response_type = HttpClientTransportResponse + + body_as_bytes = self.body() + # In order to use email.message parser, I need full HTTP bytes. Faking something to make the parser happy + http_body = ( + b"Content-Type: " + + self.content_type.encode("ascii") + + b"\r\n\r\n" + + body_as_bytes + ) + + message = message_parser(http_body) # type: Message + + # Rebuild an HTTP response from pure string + requests = self.request.multipart_mixed_info[0] # type: List[HttpRequest] + responses = [] + for request, raw_reponse in zip(requests, message.get_payload()): + if raw_reponse.get_content_type() == "application/http": + responses.append( + _deserialize_response( + raw_reponse.get_payload(decode=True), + request, + http_response_type=http_response_type, + ) + ) + else: + raise ValueError( + "Multipart doesn't support part other than application/http for now" + ) + return responses + + +class HttpResponse(_HttpResponseBase): # pylint: disable=abstract-method def stream_download(self, pipeline): # type: (PipelineType) -> Iterator[bytes] """Generator for streaming request body data. @@ -389,17 +506,42 @@ def stream_download(self, pipeline): """ def parts(self): - # type: () -> Iterator + # type: () -> Iterator[HttpResponse] """Assuming the content-type is multipart/mixed, will return the parts as an iterator. :rtype: iterator :raises ValueError: If the content is not multipart/mixed """ if not self.content_type or not self.content_type.startswith("multipart/mixed"): - raise ValueError("You can't get parts if the response is not multipart/mixed") + raise ValueError( + "You can't get parts if the response is not multipart/mixed" + ) + + responses = self._get_raw_parts() + if self.request.multipart_mixed_info: + policies = self.request.multipart_mixed_info[1] # type: List[SansIOHTTPPolicy] + + # Apply on_response concurrently to all requests + import concurrent.futures + + def parse_responses(response): + http_request = response.request + context = PipelineContext(None) + pipeline_request = PipelineRequest(http_request, context) + pipeline_response = PipelineResponse( + http_request, response, context=context + ) + + for policy in policies: + _await_result(policy.on_response, pipeline_request, pipeline_response) + + with concurrent.futures.ThreadPoolExecutor() as executor: + # List comprehension to raise exceptions if happened + [ # pylint: disable=expression-not-assigned + _ for _ in executor.map(parse_responses, responses) + ] - multipart_helper = MultiPartHelper(self.request) - return multipart_helper.parse_response(self) + return responses class _HttpClientTransportResponse(_HttpResponseBase): @@ -410,12 +552,13 @@ class _HttpClientTransportResponse(_HttpResponseBase): :param HttpRequest request: The request. :param httpclient_response: The object returned from an HTTP(S)Connection from http.client """ + def __init__(self, request, httpclient_response): super(_HttpClientTransportResponse, self).__init__(request, httpclient_response) self.status_code = httpclient_response.status self.headers = _case_insensitive_dict(httpclient_response.getheaders()) self.reason = httpclient_response.reason - self.content_type = self.headers.get('Content-Type') + self.content_type = self.headers.get("Content-Type") self.data = None def body(self): @@ -440,6 +583,7 @@ class BytesIOSocket(object): This can be used to create a http.client.HTTPResponse object based on bytes and not a real socket. """ + def __init__(self, bytes_data): self.bytes_data = bytes_data @@ -447,12 +591,11 @@ def makefile(self, *_): return BytesIO(self.bytes_data) -def _deserialize_response(http_response_as_bytes, http_request, http_response_type=HttpClientTransportResponse): +def _deserialize_response( + http_response_as_bytes, http_request, http_response_type=HttpClientTransportResponse +): local_socket = BytesIOSocket(http_response_as_bytes) - response = _HTTPResponse( - local_socket, - method=http_request.method - ) + response = _HTTPResponse(local_socket, method=http_request.method) response.begin() return http_response_type(http_request, response) @@ -467,14 +610,15 @@ def __init__(self, base_url): self._base_url = base_url def _request( - self, method, # type: str - url, # type: str - params, # type: Optional[Dict[str, str]] - headers, # type: Optional[Dict[str, str]] - content, # type: Any - form_content, # type: Optional[Dict[str, Any]] - stream_content, # type: Any - ): + self, + method, # type: str + url, # type: str + params, # type: Optional[Dict[str, str]] + headers, # type: Optional[Dict[str, str]] + content, # type: Any + form_content, # type: Optional[Dict[str, Any]] + stream_content, # type: Any + ): # type: (...) -> HttpRequest """Create HttpRequest object. @@ -521,20 +665,21 @@ def format_url(self, url_template, **kwargs): if url: parsed = urlparse(url) if not parsed.scheme or not parsed.netloc: - url = url.lstrip('/') - base = self._base_url.format(**kwargs).rstrip('/') + url = url.lstrip("/") + base = self._base_url.format(**kwargs).rstrip("/") url = _urljoin(base, url) else: url = self._base_url.format(**kwargs) return url def get( - self, url, # type: str - params=None, # type: Optional[Dict[str, str]] - headers=None, # type: Optional[Dict[str, str]] - content=None, # type: Any - form_content=None # type: Optional[Dict[str, Any]] - ): + self, + url, # type: str + params=None, # type: Optional[Dict[str, str]] + headers=None, # type: Optional[Dict[str, str]] + content=None, # type: Any + form_content=None, # type: Optional[Dict[str, Any]] + ): # type: (...) -> HttpRequest """Create a GET request object. @@ -545,18 +690,21 @@ def get( :return: An HttpRequest object :rtype: ~azure.core.pipeline.transport.HttpRequest """ - request = self._request('GET', url, params, headers, content, form_content, None) - request.method = 'GET' + request = self._request( + "GET", url, params, headers, content, form_content, None + ) + request.method = "GET" return request def put( - self, url, # type: str - params=None, # type: Optional[Dict[str, str]] - headers=None, # type: Optional[Dict[str, str]] - content=None, # type: Any - form_content=None, # type: Optional[Dict[str, Any]] - stream_content=None # type: Any - ): + self, + url, # type: str + params=None, # type: Optional[Dict[str, str]] + headers=None, # type: Optional[Dict[str, str]] + content=None, # type: Any + form_content=None, # type: Optional[Dict[str, Any]] + stream_content=None, # type: Any + ): # type: (...) -> HttpRequest """Create a PUT request object. @@ -567,17 +715,20 @@ def put( :return: An HttpRequest object :rtype: ~azure.core.pipeline.transport.HttpRequest """ - request = self._request('PUT', url, params, headers, content, form_content, stream_content) + request = self._request( + "PUT", url, params, headers, content, form_content, stream_content + ) return request def post( - self, url, # type: str - params=None, # type: Optional[Dict[str, str]] - headers=None, # type: Optional[Dict[str, str]] - content=None, # type: Any - form_content=None, # type: Optional[Dict[str, Any]] - stream_content=None # type: Any - ): + self, + url, # type: str + params=None, # type: Optional[Dict[str, str]] + headers=None, # type: Optional[Dict[str, str]] + content=None, # type: Any + form_content=None, # type: Optional[Dict[str, Any]] + stream_content=None, # type: Any + ): # type: (...) -> HttpRequest """Create a POST request object. @@ -588,17 +739,20 @@ def post( :return: An HttpRequest object :rtype: ~azure.core.pipeline.transport.HttpRequest """ - request = self._request('POST', url, params, headers, content, form_content, stream_content) + request = self._request( + "POST", url, params, headers, content, form_content, stream_content + ) return request def head( - self, url, # type: str - params=None, # type: Optional[Dict[str, str]] - headers=None, # type: Optional[Dict[str, str]] - content=None, # type: Any - form_content=None, # type: Optional[Dict[str, Any]] - stream_content=None # type: Any - ): + self, + url, # type: str + params=None, # type: Optional[Dict[str, str]] + headers=None, # type: Optional[Dict[str, str]] + content=None, # type: Any + form_content=None, # type: Optional[Dict[str, Any]] + stream_content=None, # type: Any + ): # type: (...) -> HttpRequest """Create a HEAD request object. @@ -609,17 +763,20 @@ def head( :return: An HttpRequest object :rtype: ~azure.core.pipeline.transport.HttpRequest """ - request = self._request('HEAD', url, params, headers, content, form_content, stream_content) + request = self._request( + "HEAD", url, params, headers, content, form_content, stream_content + ) return request def patch( - self, url, # type: str - params=None, # type: Optional[Dict[str, str]] - headers=None, # type: Optional[Dict[str, str]] - content=None, # type: Any - form_content=None, # type: Optional[Dict[str, Any]] - stream_content=None # type: Any - ): + self, + url, # type: str + params=None, # type: Optional[Dict[str, str]] + headers=None, # type: Optional[Dict[str, str]] + content=None, # type: Any + form_content=None, # type: Optional[Dict[str, Any]] + stream_content=None, # type: Any + ): # type: (...) -> HttpRequest """Create a PATCH request object. @@ -630,7 +787,9 @@ def patch( :return: An HttpRequest object :rtype: ~azure.core.pipeline.transport.HttpRequest """ - request = self._request('PATCH', url, params, headers, content, form_content, stream_content) + request = self._request( + "PATCH", url, params, headers, content, form_content, stream_content + ) return request def delete(self, url, params=None, headers=None, content=None, form_content=None): @@ -644,7 +803,9 @@ def delete(self, url, params=None, headers=None, content=None, form_content=None :return: An HttpRequest object :rtype: ~azure.core.pipeline.transport.HttpRequest """ - request = self._request('DELETE', url, params, headers, content, form_content, None) + request = self._request( + "DELETE", url, params, headers, content, form_content, None + ) return request def merge(self, url, params=None, headers=None, content=None, form_content=None): @@ -658,124 +819,7 @@ def merge(self, url, params=None, headers=None, content=None, form_content=None) :return: An HttpRequest object :rtype: ~azure.core.pipeline.transport.HttpRequest """ - request = self._request('MERGE', url, params, headers, content, form_content, None) - return request - - -class MultiPartHelper(object): - def __init__( - self, - main_request, # type: HttpRequest - boundary=None, # type: str - http_response_type=HttpClientTransportResponse, # type: Type[_HttpResponseBase] - ): - """Create a multipart helper to serialize and parse multipart/mixed payload. - - boundary is optional, and one will be generate if you don't provide one. - Note that no verification are made on the boundary, this is considered advanced - enough so you know how to respect RFC1341 7.2.1 and provide a correct boundary. - - :param HttpRequest main_request: The request. - :param str boundary: Optional boundary - """ - self.main_request = main_request - if self.main_request.multipart_mixed_info is None: - raise ValueError("This request doesn't have multipart information available") - self.requests = self.main_request.multipart_mixed_info[0] # type: List[HttpRequest] - self.policies = self.main_request.multipart_mixed_info[1] # type: List[SansIOHTTPPolicy] - self._boundary = boundary - self._http_response_type = http_response_type - - def prepare_request(self): - # Apply on_requests concurrently to all requests - import concurrent.futures - - def prepare_requests(request): - context = PipelineContext(None) - pipeline_request = PipelineRequest(request, context) - for policy in self.policies: - _await_result(policy.on_request, pipeline_request) - - with concurrent.futures.ThreadPoolExecutor() as executor: - # List comprehension to raise exceptions if happened - [_ for _ in executor.map(prepare_requests, self.requests)] # pylint: disable=expression-not-assigned - - self._build_final_request() - - def _build_final_request(self): - # Update the main request with the body - main_message = Message() - main_message.add_header("Content-Type", "multipart/mixed") - if self._boundary: - main_message.set_boundary(self._boundary) - for i, req in enumerate(self.requests): - part_message = Message() - part_message.add_header('Content-Type', 'application/http') - part_message.add_header('Content-Transfer-Encoding', 'binary') - part_message.add_header('Content-ID', str(i)) - part_message.set_payload(req.serialize()) - main_message.attach(part_message) - - try: - from email.policy import HTTP - full_message = main_message.as_bytes(policy=HTTP) - eol = b'\r\n' - except ImportError: # Python 2.7 - # Right now we decide to not support Python 2.7 on serialization, since - # it doesn't serialize a valid HTTP request (and our main scenario Storage refuses it) - raise NotImplementedError("Multipart request are not supported on Python 2.7") - # full_message = main_message.as_string() - # eol = b'\n' - _, _, body = full_message.split(eol, 2) - self.main_request.set_bytes_body(body) - self.main_request.headers['Content-Type'] = 'multipart/mixed; boundary='+main_message.get_boundary() - - def _get_raw_parts(self, response): - body_as_bytes = response.body() - # In order to use email.message parser, I need full HTTP bytes. Faking something to make the parser happy - http_body = ( - b'Content-Type: ' + - response.content_type.encode('ascii') + - b'\r\n\r\n' + - body_as_bytes + request = self._request( + "MERGE", url, params, headers, content, form_content, None ) - - message = message_parser(http_body) # type: Message - - # Rebuild an HTTP response from pure string - responses = [] - for request, raw_reponse in zip(self.requests, message.get_payload()): - if raw_reponse.get_content_type() == "application/http": - responses.append(_deserialize_response( - raw_reponse.get_payload(decode=True), - request, - http_response_type=self._http_response_type - )) - else: - raise ValueError("Multipart doesn't support part other than application/http for now") - return responses - - def parse_response(self, response): - responses = self._get_raw_parts(response) - - # Apply on_response concurrently to all requests - import concurrent.futures - - def parse_responses(response): - http_request = response.request - context = PipelineContext(None) - pipeline_request = PipelineRequest(http_request, context) - pipeline_response = PipelineResponse( - http_request, - response, - context=context - ) - - for policy in self.policies: - _await_result(policy.on_response, pipeline_request, pipeline_response) - - with concurrent.futures.ThreadPoolExecutor() as executor: - # List comprehension to raise exceptions if happened - [_ for _ in executor.map(parse_responses, responses)] # pylint: disable=expression-not-assigned - - return responses + return request diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/base_async.py b/sdk/core/azure-core/azure/core/pipeline/transport/base_async.py index c4f7d51ea245..aa69ea619187 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/base_async.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/base_async.py @@ -32,7 +32,6 @@ from .base import ( _HttpResponseBase, _HttpClientTransportResponse, - MultiPartHelper, PipelineContext, PipelineRequest, PipelineResponse, @@ -79,20 +78,41 @@ class _PartGenerator(AsyncIterator): :param parts: An iterable of parts """ - def __init__(self, response) -> None: + def __init__(self, response: "AsyncHttpResponse") -> None: self._response = response - self._multipart_helper = None self._parts = None + async def _parse_response(self): + responses = self._response._get_raw_parts( # pylint: disable=protected-access + http_response_type=AsyncHttpClientTransportResponse + ) + if self._response.request.multipart_mixed_info: + policies = self._response.request.multipart_mixed_info[ + 1 + ] # type: List[SansIOHTTPPolicy] + + async def parse_responses(response): + http_request = response.request + context = PipelineContext(None) + pipeline_request = PipelineRequest(http_request, context) + pipeline_response = PipelineResponse( + http_request, response, context=context + ) + + for policy in policies: + await _await_result( + policy.on_response, pipeline_request, pipeline_response + ) + + # Not happy to make this code asyncio specific, but that's multipart only for now + # If we need trio and multipart, let's reinvesitgate that later + await asyncio.gather(*[parse_responses(res) for res in responses]) + + return responses + async def __anext__(self): - if not self._multipart_helper: - self._multipart_helper = AsyncMultipartHelper( - self._response.request, - http_response_type=AsyncHttpClientTransportResponse, - ) - self._parts = iter( - await self._multipart_helper.parse_response(self._response) - ) + if not self._parts: + self._parts = iter(await self._parse_response()) try: return next(self._parts) @@ -100,7 +120,7 @@ async def __anext__(self): raise StopAsyncIteration() -class AsyncHttpResponse(_HttpResponseBase): +class AsyncHttpResponse(_HttpResponseBase): # pylint: disable=abstract-method """An AsyncHttpResponse ABC. Allows for the asynchronous streaming of data from the response. @@ -126,6 +146,7 @@ def parts(self) -> AsyncIterator: raise ValueError( "You can't get parts if the response is not multipart/mixed" ) + return _PartGenerator(self) @@ -169,40 +190,3 @@ def __enter__(self): def __exit__(self, exc_type, exc_val, exc_tb): # __exit__ should exist in pair with __enter__ but never executed pass # pragma: no cover - - -class AsyncMultipartHelper(MultiPartHelper): - async def prepare_request(self): - async def prepare_requests(request): - context = PipelineContext(None) - pipeline_request = PipelineRequest(request, context) - for policy in self.policies: - await _await_result(policy.on_request, pipeline_request) - - # Not happy to make this code asyncio specific, but that's multipart only for now - # If we need trio and multipart, let's reinvesitgate that later - await asyncio.gather(*[prepare_requests(req) for req in self.requests]) - - self._build_final_request() - - async def parse_response(self, response): - responses = self._get_raw_parts(response) - - async def parse_responses(response): - http_request = response.request - context = PipelineContext(None) - pipeline_request = PipelineRequest(http_request, context) - pipeline_response = PipelineResponse( - http_request, response, context=context - ) - - for policy in self.policies: - await _await_result( - policy.on_response, pipeline_request, pipeline_response - ) - - # Not happy to make this code asyncio specific, but that's multipart only for now - # If we need trio and multipart, let's reinvesitgate that later - await asyncio.gather(*[parse_responses(res) for res in responses]) - - return responses diff --git a/sdk/core/azure-core/tests/azure_core_asynctests/test_basic_transport.py b/sdk/core/azure-core/tests/azure_core_asynctests/test_basic_transport.py index 2ca9e637c4a7..4b2b600de985 100644 --- a/sdk/core/azure-core/tests/azure_core_asynctests/test_basic_transport.py +++ b/sdk/core/azure-core/tests/azure_core_asynctests/test_basic_transport.py @@ -6,8 +6,12 @@ from six.moves.http_client import HTTPConnection import time -from azure.core.pipeline.transport import HttpRequest, AsyncHttpResponse -from azure.core.pipeline.transport.base_async import AsyncMultipartHelper +try: + from unittest import mock +except ImportError: + import mock + +from azure.core.pipeline.transport import HttpRequest, AsyncHttpResponse, AsyncHttpTransport from azure.core.pipeline.policies import HeadersPolicy from azure.core.pipeline import AsyncPipeline @@ -17,6 +21,19 @@ @pytest.mark.asyncio async def test_multipart_send(): + # transport = mock.MagicMock(spec=AsyncHttpTransport) + # MagicMock support async cxt manager only after 3.8 + # https://github.com/python/cpython/pull/9296 + + class MockAsyncHttpTransport(AsyncHttpTransport): + async def aenter(self): return self + async def aexit(self, *args): pass + async def open(self): pass + async def close(self): pass + async def send(self, request, **kwargs): pass + + transport = MockAsyncHttpTransport() + class RequestPolicy(object): async def on_request(self, request): # type: (PipelineRequest) -> None @@ -29,14 +46,12 @@ async def on_request(self, request): request.set_multipart_mixed( req0, req1, - policies=[RequestPolicy()] - ) - - helper = AsyncMultipartHelper( - request, + policies=[RequestPolicy()], boundary="batch_357de4f7-6d0b-4e02-8cd2-6361411a9525" # Fix it so test are deterministic ) - await helper.prepare_request() + + async with AsyncPipeline(transport) as pipeline: + await pipeline.run(request) assert request.body == ( b'--batch_357de4f7-6d0b-4e02-8cd2-6361411a9525\r\n' diff --git a/sdk/core/azure-core/tests/test_basic_transport.py b/sdk/core/azure-core/tests/test_basic_transport.py index c3a2b56a6229..797c0589049d 100644 --- a/sdk/core/azure-core/tests/test_basic_transport.py +++ b/sdk/core/azure-core/tests/test_basic_transport.py @@ -8,8 +8,13 @@ import time import sys +try: + from unittest import mock +except ImportError: + import mock + from azure.core.pipeline.transport import HttpRequest, HttpResponse, RequestsTransport -from azure.core.pipeline.transport.base import HttpClientTransportResponse, _deserialize_response, MultiPartHelper +from azure.core.pipeline.transport.base import HttpClientTransportResponse, HttpTransport, _deserialize_response from azure.core.pipeline.policies import HeadersPolicy from azure.core.pipeline import Pipeline @@ -162,6 +167,8 @@ def test_response_deserialization_utf8_bom(): @pytest.mark.skipif(sys.version_info < (3, 0), reason="Multipart serialization not supported on 2.7") def test_multipart_send(): + transport = mock.MagicMock(spec=HttpTransport) + header_policy = HeadersPolicy({ 'x-ms-date': 'Thu, 14 Jun 2018 16:46:54 GMT' }) @@ -173,14 +180,12 @@ def test_multipart_send(): request.set_multipart_mixed( req0, req1, - policies=[header_policy] - ) - - helper = MultiPartHelper( - request, + policies=[header_policy], boundary="batch_357de4f7-6d0b-4e02-8cd2-6361411a9525" # Fix it so test are deterministic ) - helper.prepare_request() + + with Pipeline(transport) as pipeline: + pipeline.run(request) assert request.body == ( b'--batch_357de4f7-6d0b-4e02-8cd2-6361411a9525\r\n' From b22f6e1c923ef4c869e7599b4b8c0aa68df2a758 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Wed, 2 Oct 2019 10:58:17 -0700 Subject: [PATCH 31/33] Fix mock naming --- .../tests/azure_core_asynctests/test_basic_transport.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/core/azure-core/tests/azure_core_asynctests/test_basic_transport.py b/sdk/core/azure-core/tests/azure_core_asynctests/test_basic_transport.py index 4b2b600de985..8557f0b679cb 100644 --- a/sdk/core/azure-core/tests/azure_core_asynctests/test_basic_transport.py +++ b/sdk/core/azure-core/tests/azure_core_asynctests/test_basic_transport.py @@ -26,8 +26,8 @@ async def test_multipart_send(): # https://github.com/python/cpython/pull/9296 class MockAsyncHttpTransport(AsyncHttpTransport): - async def aenter(self): return self - async def aexit(self, *args): pass + async def __aenter__(self): return self + async def __aexit__(self, *args): pass async def open(self): pass async def close(self): pass async def send(self, request, **kwargs): pass From 0d20e4d32e09973ddbfcb1cb69bc87ccfddce4a6 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Wed, 2 Oct 2019 11:55:54 -0700 Subject: [PATCH 32/33] pylint --- sdk/core/azure-core/azure/core/pipeline/base_async.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/base_async.py b/sdk/core/azure-core/azure/core/pipeline/base_async.py index d772e634b6b9..87ffa563cabd 100644 --- a/sdk/core/azure-core/azure/core/pipeline/base_async.py +++ b/sdk/core/azure-core/azure/core/pipeline/base_async.py @@ -33,8 +33,10 @@ AsyncHTTPResponseType = TypeVar("AsyncHTTPResponseType") HTTPRequestType = TypeVar("HTTPRequestType") ImplPoliciesType = List[ - AsyncHTTPPolicy[HTTPRequestType, AsyncHTTPResponseType] -] # pylint: disable=unsubscriptable-object + AsyncHTTPPolicy[ # pylint: disable=unsubscriptable-object + HTTPRequestType, AsyncHTTPResponseType + ] +] AsyncPoliciesType = List[Union[AsyncHTTPPolicy, SansIOHTTPPolicy]] try: From a97ce1b088093feb34e35eae9f1713efec200e24 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Wed, 2 Oct 2019 12:55:15 -0700 Subject: [PATCH 33/33] ChangeLog / Readme --- sdk/core/azure-core/HISTORY.md | 8 ++++++++ sdk/core/azure-core/README.md | 13 +++++++++++++ 2 files changed, 21 insertions(+) diff --git a/sdk/core/azure-core/HISTORY.md b/sdk/core/azure-core/HISTORY.md index 2a96cb826c44..d855d0b93c9d 100644 --- a/sdk/core/azure-core/HISTORY.md +++ b/sdk/core/azure-core/HISTORY.md @@ -9,6 +9,14 @@ - Tracing: network span context is available with the TRACING_CONTEXT in pipeline response #7252 - Tracing: Span contract now has `kind`, `traceparent` and is a context manager #7252 +- SansIOHTTPPolicy methods can now be coroutines #7497 +- Add multipart/mixed support #7083: + + - HttpRequest now has a "set_multipart_mixed" method to set the parts of this request + - HttpRequest now has a "prepare_multipart_body" method to build final body. + - HttpResponse now has a "parts" method to return an iterator of parts + - AsyncHttpResponse now has a "parts" methods to return an async iterator of parts + - Note that multipart/MIXED is a Python 3.x only feature ### Bug fixes diff --git a/sdk/core/azure-core/README.md b/sdk/core/azure-core/README.md index c0f2ff5ca18b..100580d4ec6d 100644 --- a/sdk/core/azure-core/README.md +++ b/sdk/core/azure-core/README.md @@ -241,6 +241,11 @@ class HttpRequest(object): def set_bytes_body(self, data): """Set generic bytes as the body of the request.""" + + def set_multipart_mixed(self, *requests, **kwargs): + """Set requests for a multipart/mixed body. + Optionally apply "policies" in kwargs to each request. + """ ``` The HttpResponse object on the other hand will generally have a transport-specific derivative. @@ -285,6 +290,12 @@ class HttpResponse(object): and asynchronous generator. """ + def parts(self): + """An iterator of parts if content-type is multipart/mixed. + For the AsyncHttpResponse object this function will return + and asynchronous iterator. + """ + ``` ### PipelineRequest and PipelineResponse @@ -344,6 +355,8 @@ def on_exception(self, request): """ ``` +SansIOHTTPPolicy methods can be declared as coroutines, but then they can only be used with a AsyncPipeline. + Current provided sans IO policies include: ```python from azure.core.pipeline.policies import (