From edbaec3b91599aa41e785de979a099c16a56b7e7 Mon Sep 17 00:00:00 2001 From: Steven Jin Date: Mon, 11 Jul 2022 18:50:49 -0700 Subject: [PATCH 01/32] Add Pyodide transport file --- .../azure/core/pipeline/transport/_pyodide.py | 98 +++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py b/sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py new file mode 100644 index 000000000000..d792d8d38ab8 --- /dev/null +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py @@ -0,0 +1,98 @@ +from collections.abc import AsyncIterator + +from io import BytesIO +from azure.core.exceptions import HttpResponseError +from ._requests_asyncio import AsyncioRequestsTransport +from azure.core.rest._http_response_impl_async import AsyncHttpResponseImpl +from pyodide.http import pyfetch # pylint: disable=import-error +from pyodide import JsException # pylint: disable=import-error +from requests.structures import CaseInsensitiveDict # FIXME + +class PyodideTransport(AsyncioRequestsTransport): + """Implements a basic HTTP sender using the pyodide javascript fetch api.""" + + async def send(self, request, **kwargs): # type: ignore + """Send request object according to configuration. + + :param request: The request object to be sent. + :type request: ~azure.core.pipeline.transport.HttpRequest + :return: An HTTPResponse object. + :rtype: PyodideResponseTransport + """ + endpoint = request.url + request_headers = dict(request.headers) + init = { + "method": request.method, + "headers": request_headers, + "body": request.data, + "files": request.files, + "verify": kwargs.pop("connection_verify", self.connection_config.verify), + "cert": kwargs.pop("connection_cert", self.connection_config.cert), + "allow_redirects": False, + **kwargs, + } + + try: + response = await pyfetch(endpoint, **init) + except JsException as error: + raise HttpResponseError(error, error=error) + + headers = CaseInsensitiveDict(response.js_response.headers) + transport_response = PyodideTransportResponse( + request=request, + internal_response=response, + block_size=self.connection_config.data_block_size, + status_code=response.status, + reason=response.status_text, + content_type=headers.get("content-type"), + headers=headers, + stream_download_generator=PyodideStreamDownloadGenerator, + ) + await transport_response.read() + return transport_response + + +class PyodideTransportResponse(AsyncHttpResponseImpl): + """Async response object for the pyodide transport.""" + + async def close(self): + """This is kinda weird but AsyncHttpResponseImpl assumed that + the internal response is a `requests.Reponse` object (I think). + + Also, you can't really close connections at will using pyfetch. + """ + self._is_closed = True + + async def load_body(self): + """Load the body of the response.""" + if self._content is None: + # This line can only be called once. Subsequent calls will raise an `OSError`. + self._content = await self.internal_response.bytes() + + +class PyodideStreamDownloadGenerator(AsyncIterator): + """Simple stream download generator that returns the contents of + a request. + """ + + def __init__(self, pipeline, response: PyodideTransportResponse, **__) -> None: + self.pipeline = pipeline + self.block_size = response.block_size + self.response = response + self.stream = None + self.done = False + + + async def __anext__(self): + """Assume that all the data we need is in `_internal_response`.""" + if self.stream is None: + await self.response.load_body() + self.stream = BytesIO(self.response.content) + self.stream.seek(0) + + chunk = self.stream.read(self.block_size) + if not chunk: + self.done = True + raise StopAsyncIteration() + else: + return chunk From 67efe58e192fb873a97856b75b13d68ab099a6d0 Mon Sep 17 00:00:00 2001 From: Steven Jin Date: Tue, 12 Jul 2022 08:41:22 -0700 Subject: [PATCH 02/32] pyodide tests --- .../tests/test_pyodide_transport.py | 119 ++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 sdk/core/azure-core/tests/test_pyodide_transport.py diff --git a/sdk/core/azure-core/tests/test_pyodide_transport.py b/sdk/core/azure-core/tests/test_pyodide_transport.py new file mode 100644 index 000000000000..bfd0a3b02e0e --- /dev/null +++ b/sdk/core/azure-core/tests/test_pyodide_transport.py @@ -0,0 +1,119 @@ +"""Tests that mock the browser layer.""" +import sys +from unittest import mock + +import pytest +from azure.core.exceptions import HttpResponseError +from azure.core.pipeline._base_async import AsyncPipeline +from azure.core.pipeline.policies._retry_async import AsyncRetryPolicy +from azure.core.rest import HttpRequest + +PLACEHOLDER_ENDPOINT = "https://my-resource-group.cognitiveservices.azure.com/" + + +class TestPyodideTransportClass: + """Unittest for the Pyodide transport.""" + + @pytest.fixture(scope="class", autouse=True) + def mock_pyodide_module(self): + """Create a mock for the Pyodide module.""" + mock_pyodide_module = mock.Mock() + mock_pyodide_module.http.pyfetch = mock.AsyncMock() + mock_pyodide_module.JsException = type("JsException", (Exception,), {}) + return mock_pyodide_module + + @pytest.fixture(scope="class", autouse=True) + def transport(self, mock_pyodide_module): + """Add the mock Pyodide module to `sys.modules` and import our transport.""" + # Use patch so we don't clutter up the `sys.modules` namespace. + patch_dict = ( + ("pyodide", mock_pyodide_module), + ("pyodide.http", mock_pyodide_module.http), + ) + with mock.patch.dict(sys.modules, patch_dict): + # weird stuff is hppenig here, have to do full import. + # if I do `from azure.core.pipline.transport import _pyodide` + # I end up with `_pyodide = 'pyodide'` ????????=-p + import azure.core.pipeline.transport._pyodide + + yield azure.core.pipeline.transport._pyodide + + @pytest.fixture(scope="class", autouse=True) + def pipeline(self, transport): + """Create a pipeline to test.""" + return AsyncPipeline(transport.PyodideTransport(), [AsyncRetryPolicy()]) + + @pytest.fixture(scope="class", autouse=True) + def mock_pyfetch(self, mock_pyodide_module): + """Utility fixture for less typing.""" + return mock_pyodide_module.http.pyfetch + + def create_mock_response( + self, body: bytes, headers: dict, status: int, status_text: str + ) -> mock.Mock: + """Create a mock response object that mimics `pyodide.http.FetchResponse`""" + mock_response = mock.Mock() + if isinstance(body, str): + body = bytes(body, encoding="utf-8") + mock_response.body = body + mock_response.js_response.headers = headers + mock_response.status = status + mock_response.status_text = status_text + mock_response.bytes = mock.AsyncMock(return_value=body) + return mock_response + + @pytest.mark.asyncio + async def test_successful_send(self, mock_pyfetch, mock_pyodide_module, pipeline): + """Test that a successful send returns the correct values.""" + # setup data + mock_pyfetch.reset_mock() + method = "POST" + headers = {"key": "value"} + data = b"data" + request = HttpRequest( + method=method, url=PLACEHOLDER_ENDPOINT, headers=headers, data=data + ) + response_body = b"0123" + response_headers = {"header": "value"} + response_status = 200 + response_text = "OK" + mock_response = self.create_mock_response( + body=response_body, + headers=response_headers, + status=response_status, + status_text=response_text, + ) + mock_pyodide_module.http.pyfetch.return_value = mock_response + response = (await pipeline.run(request=request)).http_response + # Check that the pipeline processed the data correctly. + assert response.body() == response_body + assert response.status_code == response_status + assert response.headers == response_headers + assert response.reason == response_text + + # Check that the call had the correct arguments. + mock_pyfetch.assert_called_once() + args = mock_pyfetch.call_args.args + kwargs = mock_pyfetch.call_args.kwargs + assert len(args) == 1 + assert args[0] == PLACEHOLDER_ENDPOINT + assert kwargs["method"] == method + assert kwargs["body"] == data + assert not kwargs["allow_redirects"] + assert kwargs["headers"]["key"] == "value" + assert kwargs["headers"]["Content-Length"] == str(len(data)) + assert kwargs["verify"] + assert kwargs["cert"] is None + assert not kwargs["files"] + + @pytest.mark.asyncio + async def test_unsuccessful_send(self, mock_pyfetch, mock_pyodide_module, pipeline): + """Test that the pipeline is failing correctly.""" + mock_pyfetch.reset_mock() + mock_pyfetch.side_effect = mock_pyodide_module.JsException + retry_total = 3 + request = HttpRequest(method="GET", url=PLACEHOLDER_ENDPOINT) + with pytest.raises(HttpResponseError): + await pipeline.run(request) + # 3 retries plus the original request. + assert mock_pyfetch.call_count == retry_total + 1 From ad9c4f63183823e05009c2c07f46f1c49044742e Mon Sep 17 00:00:00 2001 From: Steven Jin Date: Wed, 13 Jul 2022 10:50:14 -0700 Subject: [PATCH 03/32] license and some typing --- .../azure/core/pipeline/transport/_pyodide.py | 40 +++++++++++++++++-- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py b/sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py index d792d8d38ab8..4f14a98447ec 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py @@ -1,13 +1,42 @@ -from collections.abc import AsyncIterator +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- +from collections.abc import AsyncIterator from io import BytesIO + from azure.core.exceptions import HttpResponseError -from ._requests_asyncio import AsyncioRequestsTransport +from azure.core.pipeline import Pipeline from azure.core.rest._http_response_impl_async import AsyncHttpResponseImpl -from pyodide.http import pyfetch # pylint: disable=import-error from pyodide import JsException # pylint: disable=import-error +from pyodide.http import pyfetch # pylint: disable=import-error from requests.structures import CaseInsensitiveDict # FIXME +from ._requests_asyncio import AsyncioRequestsTransport + + class PyodideTransport(AsyncioRequestsTransport): """Implements a basic HTTP sender using the pyodide javascript fetch api.""" @@ -73,9 +102,12 @@ async def load_body(self): class PyodideStreamDownloadGenerator(AsyncIterator): """Simple stream download generator that returns the contents of a request. + + :param pipeline: The pipeline object + :param response: The response object. """ - def __init__(self, pipeline, response: PyodideTransportResponse, **__) -> None: + def __init__(self, pipeline: Pipeline, response: PyodideTransportResponse, **__) -> None: self.pipeline = pipeline self.block_size = response.block_size self.response = response From 5a5282620426cfa3e431522f97653a141eb6904b Mon Sep 17 00:00:00 2001 From: Steven Jin Date: Fri, 15 Jul 2022 11:08:29 -0700 Subject: [PATCH 04/32] Add proper streaming --- sdk/core/azure-core/CHANGELOG.md | 2 + .../azure/core/pipeline/transport/_pyodide.py | 75 +++++++++++-------- .../tests/test_pyodide_transport.py | 22 +++++- 3 files changed, 68 insertions(+), 31 deletions(-) diff --git a/sdk/core/azure-core/CHANGELOG.md b/sdk/core/azure-core/CHANGELOG.md index d213c6e69615..bd73da49fb62 100644 --- a/sdk/core/azure-core/CHANGELOG.md +++ b/sdk/core/azure-core/CHANGELOG.md @@ -4,6 +4,8 @@ ### Features Added +- Added Pyodide-compatible transport. + ### Breaking Changes ### Bugs Fixed diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py b/sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py index 4f14a98447ec..786545949a3f 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py @@ -27,16 +27,15 @@ from collections.abc import AsyncIterator from io import BytesIO -from azure.core.exceptions import HttpResponseError -from azure.core.pipeline import Pipeline -from azure.core.rest._http_response_impl_async import AsyncHttpResponseImpl from pyodide import JsException # pylint: disable=import-error from pyodide.http import pyfetch # pylint: disable=import-error from requests.structures import CaseInsensitiveDict # FIXME +from azure.core.exceptions import HttpResponseError +from azure.core.pipeline import Pipeline +from azure.core.rest._http_response_impl_async import AsyncHttpResponseImpl from ._requests_asyncio import AsyncioRequestsTransport - class PyodideTransport(AsyncioRequestsTransport): """Implements a basic HTTP sender using the pyodide javascript fetch api.""" @@ -48,6 +47,7 @@ async def send(self, request, **kwargs): # type: ignore :return: An HTTPResponse object. :rtype: PyodideResponseTransport """ + stream_response = kwargs.pop("stream_response", False) endpoint = request.url request_headers = dict(request.headers) init = { @@ -77,54 +77,69 @@ async def send(self, request, **kwargs): # type: ignore headers=headers, stream_download_generator=PyodideStreamDownloadGenerator, ) - await transport_response.read() + if not stream_response: + await transport_response.load_body() + return transport_response class PyodideTransportResponse(AsyncHttpResponseImpl): - """Async response object for the pyodide transport.""" + """Async response object for the pyodide transport. + """ - async def close(self): - """This is kinda weird but AsyncHttpResponseImpl assumed that - the internal response is a `requests.Reponse` object (I think). + def __init__(self, **kwargs): + super(PyodideTransportResponse, self).__init__(**kwargs) + # clone to avoid reading from the same `FetchResponse` a second time in `load_body`. + self.reader = self.internal_response.clone().js_response.body.getReader() - Also, you can't really close connections at will using pyfetch. + async def close(self): + """We don't actually have control over closing connections in the browser, so we just pretend + to close. """ self._is_closed = True async def load_body(self): """Load the body of the response.""" if self._content is None: - # This line can only be called once. Subsequent calls will raise an `OSError`. - self._content = await self.internal_response.bytes() + self._content = await self._internal_response.bytes() class PyodideStreamDownloadGenerator(AsyncIterator): """Simple stream download generator that returns the contents of a request. - - :param pipeline: The pipeline object - :param response: The response object. """ - def __init__(self, pipeline: Pipeline, response: PyodideTransportResponse, **__) -> None: - self.pipeline = pipeline + def __init__(self, response: PyodideTransportResponse, **__) -> None: self.block_size = response.block_size self.response = response - self.stream = None + # use this to efficiently store bytes. + self.stream = BytesIO() + self.closed = False + # We cannot control how many bytes we get from `response.reader`. `self.buffer_left` + # indicates how many unread bytes there are in `self.stream` + self.buffer_left = 0 self.done = False - + + def __aiter__(self): + return self async def __anext__(self): - """Assume that all the data we need is in `_internal_response`.""" - if self.stream is None: - await self.response.load_body() - self.stream = BytesIO(self.response.content) - self.stream.seek(0) - - chunk = self.stream.read(self.block_size) - if not chunk: - self.done = True + if self.closed: raise StopAsyncIteration() - else: - return chunk + + # remember the initial stream position + start_pos = self.stream.tell() + # move stream position to the end + self.stream.read() + # read from reader until there is no more data or we have `self.block_size` unread bytes. + while self.buffer_left < self.block_size: + read = await self.response.reader.read() + if read.done: + self.closed = True + break + self.buffer_left += self.stream.write(bytes(read.value)) + + # move the stream position back to where we started + self.stream.seek(start_pos) + self.buffer_left -= self.block_size + return self.stream.read(self.block_size) diff --git a/sdk/core/azure-core/tests/test_pyodide_transport.py b/sdk/core/azure-core/tests/test_pyodide_transport.py index bfd0a3b02e0e..ef1d517890a6 100644 --- a/sdk/core/azure-core/tests/test_pyodide_transport.py +++ b/sdk/core/azure-core/tests/test_pyodide_transport.py @@ -1,5 +1,6 @@ """Tests that mock the browser layer.""" import sys +from typing import NamedTuple from unittest import mock import pytest @@ -33,7 +34,7 @@ def transport(self, mock_pyodide_module): with mock.patch.dict(sys.modules, patch_dict): # weird stuff is hppenig here, have to do full import. # if I do `from azure.core.pipline.transport import _pyodide` - # I end up with `_pyodide = 'pyodide'` ????????=-p + # I end up with `_pyodide = 'pyodide'` ??????? import azure.core.pipeline.transport._pyodide yield azure.core.pipeline.transport._pyodide @@ -117,3 +118,22 @@ async def test_unsuccessful_send(self, mock_pyfetch, mock_pyodide_module, pipeli await pipeline.run(request) # 3 retries plus the original request. assert mock_pyfetch.call_count == retry_total + 1 + + @pytest.mark.asyncio + async def test_download_generator(self, transport): + """Test that the download generator is working correctly.""" + class ReaderReturn(NamedTuple): + value: bytes + done: bool + + response_mock = mock.Mock() + response_mock.block_size = 5 + response_mock.reader.read = mock.AsyncMock() + response_mock.reader.read.return_value = ReaderReturn(value=b"01", done=False) + generator = transport.PyodideStreamDownloadGenerator(response=response_mock) + + assert len(await generator.__anext__()) == response_mock.block_size + assert response_mock.reader.read.call_count == 3 + assert len(await generator.__anext__()) == response_mock.block_size + # 5 becuase there is a leftover byte from the previous `__anext__` call. + assert response_mock.reader.read.call_count == 5 From e35e8f5803edaec7e4bfd39a63f6d07d918a4c07 Mon Sep 17 00:00:00 2001 From: Steven Jin Date: Fri, 15 Jul 2022 11:22:28 -0700 Subject: [PATCH 05/32] typo --- sdk/core/azure-core/tests/test_pyodide_transport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core/tests/test_pyodide_transport.py b/sdk/core/azure-core/tests/test_pyodide_transport.py index ef1d517890a6..198c51a5839e 100644 --- a/sdk/core/azure-core/tests/test_pyodide_transport.py +++ b/sdk/core/azure-core/tests/test_pyodide_transport.py @@ -135,5 +135,5 @@ class ReaderReturn(NamedTuple): assert len(await generator.__anext__()) == response_mock.block_size assert response_mock.reader.read.call_count == 3 assert len(await generator.__anext__()) == response_mock.block_size - # 5 becuase there is a leftover byte from the previous `__anext__` call. + # 5 because there is a leftover byte from the previous `__anext__` call. assert response_mock.reader.read.call_count == 5 From f0a18a7415f5aacc99bad938bcfc8a5a1e8b75c3 Mon Sep 17 00:00:00 2001 From: Steven Jin Date: Fri, 15 Jul 2022 13:15:55 -0700 Subject: [PATCH 06/32] docs --- .../azure/core/pipeline/transport/_pyodide.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py b/sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py index 786545949a3f..3456a191d962 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py @@ -32,12 +32,16 @@ from requests.structures import CaseInsensitiveDict # FIXME from azure.core.exceptions import HttpResponseError -from azure.core.pipeline import Pipeline from azure.core.rest._http_response_impl_async import AsyncHttpResponseImpl from ._requests_asyncio import AsyncioRequestsTransport class PyodideTransport(AsyncioRequestsTransport): - """Implements a basic HTTP sender using the pyodide javascript fetch api.""" + """Implements a basic HTTP sender using the Pyodide Javascript Fetch API. + + WARNING: Pyodide is still an alpha technology. As such, this transport + is highly experimental and subject to breaking changes. This transport was + built around Pyodide version 0.20.0. + """ async def send(self, request, **kwargs): # type: ignore """Send request object according to configuration. @@ -84,8 +88,7 @@ async def send(self, request, **kwargs): # type: ignore class PyodideTransportResponse(AsyncHttpResponseImpl): - """Async response object for the pyodide transport. - """ + """Async response object for the `PyodideTransport`.""" def __init__(self, **kwargs): super(PyodideTransportResponse, self).__init__(**kwargs) @@ -120,9 +123,6 @@ def __init__(self, response: PyodideTransportResponse, **__) -> None: self.buffer_left = 0 self.done = False - def __aiter__(self): - return self - async def __anext__(self): if self.closed: raise StopAsyncIteration() From d772153806cca60ae16d96ad3dcbf8ae609e6160 Mon Sep 17 00:00:00 2001 From: Steven Jin Date: Fri, 15 Jul 2022 13:59:32 -0700 Subject: [PATCH 07/32] fix merge --- .../azure-core/azure/core/pipeline/transport/_pyodide.py | 2 +- sdk/core/azure-core/tests/test_pyodide_transport.py | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py b/sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py index 3456a191d962..a8a318e85281 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py @@ -29,8 +29,8 @@ from pyodide import JsException # pylint: disable=import-error from pyodide.http import pyfetch # pylint: disable=import-error -from requests.structures import CaseInsensitiveDict # FIXME +from azure.core.utils import CaseInsensitiveDict from azure.core.exceptions import HttpResponseError from azure.core.rest._http_response_impl_async import AsyncHttpResponseImpl from ._requests_asyncio import AsyncioRequestsTransport diff --git a/sdk/core/azure-core/tests/test_pyodide_transport.py b/sdk/core/azure-core/tests/test_pyodide_transport.py index 198c51a5839e..fc15cbe2825f 100644 --- a/sdk/core/azure-core/tests/test_pyodide_transport.py +++ b/sdk/core/azure-core/tests/test_pyodide_transport.py @@ -92,6 +92,10 @@ async def test_successful_send(self, mock_pyfetch, mock_pyodide_module, pipeline assert response.headers == response_headers assert response.reason == response_text + assert not response._is_closed + response.close() + assert response._is_closed + # Check that the call had the correct arguments. mock_pyfetch.assert_called_once() args = mock_pyfetch.call_args.args @@ -137,3 +141,8 @@ class ReaderReturn(NamedTuple): assert len(await generator.__anext__()) == response_mock.block_size # 5 because there is a leftover byte from the previous `__anext__` call. assert response_mock.reader.read.call_count == 5 + + response_mock.reader.read.return_value = ReaderReturn(value=None, done=True) + await generator.__anext__() + with pytest.raises(StopAsyncIteration): + await generator.__anext__() \ No newline at end of file From 05034da9a843b745df949ca96a3a0766c3902418 Mon Sep 17 00:00:00 2001 From: Steven Jin Date: Mon, 18 Jul 2022 11:30:30 -0700 Subject: [PATCH 08/32] some readability --- .../samples/pyodide/async_testing.py | 27 ++++++ .../samples/pyodide/browser_test.py | 80 +++++++++++++++++ .../azure-core/samples/pyodide/example-env | 4 + .../azure-core/samples/pyodide/index.html | 89 +++++++++++++++++++ sdk/core/azure-core/samples/pyodide/readme.md | 36 ++++++++ .../samples/pyodide/requirements.txt | 6 ++ 6 files changed, 242 insertions(+) create mode 100644 sdk/core/azure-core/samples/pyodide/async_testing.py create mode 100644 sdk/core/azure-core/samples/pyodide/browser_test.py create mode 100644 sdk/core/azure-core/samples/pyodide/example-env create mode 100644 sdk/core/azure-core/samples/pyodide/index.html create mode 100644 sdk/core/azure-core/samples/pyodide/readme.md create mode 100644 sdk/core/azure-core/samples/pyodide/requirements.txt diff --git a/sdk/core/azure-core/samples/pyodide/async_testing.py b/sdk/core/azure-core/samples/pyodide/async_testing.py new file mode 100644 index 000000000000..1e3eff2f9901 --- /dev/null +++ b/sdk/core/azure-core/samples/pyodide/async_testing.py @@ -0,0 +1,27 @@ +import traceback +import sys + + +class AsyncTestSuite: + """Async test cases + Test must be asynchronous and follow the pattern `test*`. + """ + + async def run(self): + """Run the tests an print the results.""" + print("".join(("-" * 8, type(self).__name__, "-" * 8))) + for method_name in dir(self): + if not method_name.startswith("test"): + continue + print(method_name, end="... ") + try: + await getattr(self, method_name)() + except AssertionError: + print("FAIL") + traceback.print_exception(*sys.exc_info()) + except Exception: # pylint: disable=broad-except + print("ERROR") + traceback.print_exception(*sys.exc_info()) + else: + print("PASS") + print() diff --git a/sdk/core/azure-core/samples/pyodide/browser_test.py b/sdk/core/azure-core/samples/pyodide/browser_test.py new file mode 100644 index 000000000000..e6c2e0475fc8 --- /dev/null +++ b/sdk/core/azure-core/samples/pyodide/browser_test.py @@ -0,0 +1,80 @@ +from unittest.mock import _patch_dict, patch +from uuid import uuid4 + +from azure.ai.textanalytics.aio import TextAnalyticsClient +from azure.core.credentials import AzureKeyCredential +from azure.core.pipeline.transport._pyodide import PyodideTransport +from azure.storage.blob.aio import BlobClient, BlobServiceClient + +# pylint: disable=import-error +from async_testing import AsyncTestSuite + + +class PyodideTransportIntegrationTestSuite(AsyncTestSuite): + """Integration tests for the Pyodide transport.""" + + text_analytics_client: TextAnalyticsClient + blob_service_client: BlobServiceClient + + def __init__( + self, + text_analytics_key: str, + text_analytics_endpoint: str, + blob_service_key: str, + blob_service_endpoint: str, + ): + self.text_analytics_client = TextAnalyticsClient( + endpoint=text_analytics_endpoint, + credential=AzureKeyCredential(text_analytics_key), + transport=PyodideTransport(), + ) + self.blob_service_client = BlobServiceClient( + blob_service_endpoint, blob_service_key, transport=PyodideTransport() + ) + + async def test_sentiment_analysis(self): + """Test that sentiment analysis works.""" + results = await self.text_analytics_client.analyze_sentiment( + ["good great amazing"] + ) + assert len(results) == 1 + result = results[0] + assert result.sentiment == "positive" + assert result.confidence_scores.positive > 0.98 + assert result.confidence_scores.neutral < 0.02 + assert result.confidence_scores.negative < 0.02 + + async def test_storage(self): + """Test that we can upload and download from blob storage""" + account_name = uuid4().hex + container_client = await self.blob_service_client.create_container(account_name) + blob_name = uuid4().hex + blob_data = b"012345" + try: + assert await container_client.exists() + blob_client = container_client.get_blob_client(blob_name) + await blob_client.upload_blob(blob_data) + + # make a new client so we don't have cached data + blob_client = BlobClient( + account_url=self.blob_service_client.url, + container_name=container_client.container_name, + blob_name=blob_name, + credential=container_client.credential, + max_single_get_size=1, + max_chunk_get_size=1, + transport=PyodideTransport(), + ) + assert await blob_client.exists() + downloader = await blob_client.download_blob() + i = 0 + async for chunk in downloader.chunks(): + assert chunk == bytes(str(i), "utf-8") + i += 1 + assert i == len(blob_data) + + except Exception: + await container_client.delete_container() + raise + else: + await container_client.delete_container() diff --git a/sdk/core/azure-core/samples/pyodide/example-env b/sdk/core/azure-core/samples/pyodide/example-env new file mode 100644 index 000000000000..57f925f76d42 --- /dev/null +++ b/sdk/core/azure-core/samples/pyodide/example-env @@ -0,0 +1,4 @@ +TEXT_ANALYTICS_KEY= +TEXT_ANALYTICS_ENDPOINT= +BLOB_SERVICE_KEY= +BLOB_SERVICE_ENDPOINT=https://.blob.core.windows.net diff --git a/sdk/core/azure-core/samples/pyodide/index.html b/sdk/core/azure-core/samples/pyodide/index.html new file mode 100644 index 000000000000..34a921b498d8 --- /dev/null +++ b/sdk/core/azure-core/samples/pyodide/index.html @@ -0,0 +1,89 @@ + + + + + + + + + + diff --git a/sdk/core/azure-core/samples/pyodide/readme.md b/sdk/core/azure-core/samples/pyodide/readme.md new file mode 100644 index 000000000000..3febb530a1eb --- /dev/null +++ b/sdk/core/azure-core/samples/pyodide/readme.md @@ -0,0 +1,36 @@ +# Integration Testing + +## Running + +Once you have set up your Azure Resources and your `.env` filem, from this directory, run + +```python +python -h http.server 8000 +``` + +(You can use any other port). Then, from a Chromium-based browser such as Edge, go to [`http://localhost:8000/test/integration`](http://localhost:8000/test/integration) and the tests will be run in the browser. Dev tip: keep your browser's devtools open. + +## Adding tests + +Add tests in `browser_test.py`. I couldn't get `pytest` or `unittest` to cooperate with me, so I made my own little async testing framework (`async_test.py`). If you are creating new files to test or new packages, update the `TEST_FILES` and `PACKAGES` variables in `index.html`, import the test case, and run it. + +## Sensitive values + +To run the tests, you need a `.env` folder in this directory with your sensitive values. +see `example-env`. You can then access the values as environment variables using `os.getenv`. + +## Dependencies + +All all packages listed in `requirements.txt` will be available in the testing environment. + +## Azure Resources + +You need your own Text Analytics and Blob Storage accounts to run these tests. Blob storage requirest some additional configuration to work. To set up Blob Storage, navigate to your storage client homepage and go to the `Resource Sharing (CORS)` tab. Create a rule with the following values + +| Allowed origins | Allowed methods | Allowed headers | Exposed headers | Max age | +|-----------------|-----------------|-----------------|-----------------|---------| +| `\*` | All | `\*` | See below | `3600` | + +For exposed headers, put + +> Server,Content-Range,ETag,Last-Modified,Accept-Ranges,x-ms-* diff --git a/sdk/core/azure-core/samples/pyodide/requirements.txt b/sdk/core/azure-core/samples/pyodide/requirements.txt new file mode 100644 index 000000000000..ebf1a28ec837 --- /dev/null +++ b/sdk/core/azure-core/samples/pyodide/requirements.txt @@ -0,0 +1,6 @@ +python-dotenv +# todo: delete the following line +https://tsjinxuanstorage2.blob.core.windows.net/pyodide/azure_core-1.24.3-py3-none-any.whl +azure-ai-textanalytics +azure-storage-blob +azure-ai-formrecognizer From e6596cfc2c30776a603d0ba521fae94f016d6f35 Mon Sep 17 00:00:00 2001 From: Steven Jin Date: Mon, 18 Jul 2022 13:50:18 -0700 Subject: [PATCH 09/32] fix download generator init --- sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py b/sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py index a8a318e85281..fab5ab84d3af 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py @@ -112,7 +112,7 @@ class PyodideStreamDownloadGenerator(AsyncIterator): a request. """ - def __init__(self, response: PyodideTransportResponse, **__) -> None: + def __init__(self, response: PyodideTransportResponse, *_, **__) -> None: self.block_size = response.block_size self.response = response # use this to efficiently store bytes. From bb981f639132d9ead7f04d56274dd991c170b063 Mon Sep 17 00:00:00 2001 From: Steven Jin Date: Mon, 18 Jul 2022 16:36:07 -0700 Subject: [PATCH 10/32] typing --- .../azure/core/pipeline/transport/_pyodide.py | 117 +++++++++--------- 1 file changed, 60 insertions(+), 57 deletions(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py b/sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py index fab5ab84d3af..4cb1689bca29 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py @@ -33,59 +33,9 @@ from azure.core.utils import CaseInsensitiveDict from azure.core.exceptions import HttpResponseError from azure.core.rest._http_response_impl_async import AsyncHttpResponseImpl -from ._requests_asyncio import AsyncioRequestsTransport - -class PyodideTransport(AsyncioRequestsTransport): - """Implements a basic HTTP sender using the Pyodide Javascript Fetch API. - - WARNING: Pyodide is still an alpha technology. As such, this transport - is highly experimental and subject to breaking changes. This transport was - built around Pyodide version 0.20.0. - """ - - async def send(self, request, **kwargs): # type: ignore - """Send request object according to configuration. - - :param request: The request object to be sent. - :type request: ~azure.core.pipeline.transport.HttpRequest - :return: An HTTPResponse object. - :rtype: PyodideResponseTransport - """ - stream_response = kwargs.pop("stream_response", False) - endpoint = request.url - request_headers = dict(request.headers) - init = { - "method": request.method, - "headers": request_headers, - "body": request.data, - "files": request.files, - "verify": kwargs.pop("connection_verify", self.connection_config.verify), - "cert": kwargs.pop("connection_cert", self.connection_config.cert), - "allow_redirects": False, - **kwargs, - } - - try: - response = await pyfetch(endpoint, **init) - except JsException as error: - raise HttpResponseError(error, error=error) - - headers = CaseInsensitiveDict(response.js_response.headers) - transport_response = PyodideTransportResponse( - request=request, - internal_response=response, - block_size=self.connection_config.data_block_size, - status_code=response.status, - reason=response.status_text, - content_type=headers.get("content-type"), - headers=headers, - stream_download_generator=PyodideStreamDownloadGenerator, - ) - if not stream_response: - await transport_response.load_body() - - return transport_response +from ._requests_asyncio import AsyncioRequestsTransport +from . import HttpRequest class PyodideTransportResponse(AsyncHttpResponseImpl): """Async response object for the `PyodideTransport`.""" @@ -95,24 +45,24 @@ def __init__(self, **kwargs): # clone to avoid reading from the same `FetchResponse` a second time in `load_body`. self.reader = self.internal_response.clone().js_response.body.getReader() - async def close(self): + async def close(self) -> None: """We don't actually have control over closing connections in the browser, so we just pretend to close. """ self._is_closed = True - async def load_body(self): + async def load_body(self) -> None: """Load the body of the response.""" if self._content is None: self._content = await self._internal_response.bytes() -class PyodideStreamDownloadGenerator(AsyncIterator): +class PyodideStreamDownloadGenerator(AsyncIterator[bytes]): """Simple stream download generator that returns the contents of a request. """ - def __init__(self, response: PyodideTransportResponse, *_, **__) -> None: + def __init__(self, response: PyodideTransportResponse, *_, **__): self.block_size = response.block_size self.response = response # use this to efficiently store bytes. @@ -123,7 +73,8 @@ def __init__(self, response: PyodideTransportResponse, *_, **__) -> None: self.buffer_left = 0 self.done = False - async def __anext__(self): + async def __anext__(self) -> bytes: + """Get the next block of bytes.""" if self.closed: raise StopAsyncIteration() @@ -143,3 +94,55 @@ async def __anext__(self): self.stream.seek(start_pos) self.buffer_left -= self.block_size return self.stream.read(self.block_size) + +class PyodideTransport(AsyncioRequestsTransport): + """Implements a basic HTTP sender using the Pyodide Javascript Fetch API. + + WARNING: Pyodide is still an alpha technology. As such, this transport + is highly experimental and subject to breaking changes. This transport was + built around Pyodide version 0.20.0. + """ + + async def send(self, request: HttpRequest, **kwargs) -> PyodideTransportResponse: + """Send request object according to configuration. + + :param request: The request object to be sent. + :type request: ~azure.core.pipeline.transport.HttpRequest + :return: An HTTPResponse object. + :rtype: PyodideResponseTransport + """ + stream_response = kwargs.pop("stream_response", False) + endpoint = request.url + request_headers = dict(request.headers) + init = { + "method": request.method, + "headers": request_headers, + "body": request.data, + "files": request.files, + "verify": kwargs.pop("connection_verify", self.connection_config.verify), + "cert": kwargs.pop("connection_cert", self.connection_config.cert), + "allow_redirects": False, + **kwargs, + } + + try: + response = await pyfetch(endpoint, **init) + except JsException as error: + raise HttpResponseError(error, error=error) + + headers = CaseInsensitiveDict(response.js_response.headers) + transport_response = PyodideTransportResponse( + request=request, + internal_response=response, + block_size=self.connection_config.data_block_size, + status_code=response.status, + reason=response.status_text, + content_type=headers.get("content-type"), + headers=headers, + stream_download_generator=PyodideStreamDownloadGenerator, + ) + if not stream_response: + await transport_response.load_body() + + return transport_response + From c4f4bf8b5b59878e9744d0c3801d4f64f8665978 Mon Sep 17 00:00:00 2001 From: Steven Jin Date: Mon, 18 Jul 2022 16:43:56 -0700 Subject: [PATCH 11/32] fix failing test and async warning --- .../azure-core/azure/core/pipeline/transport/_pyodide.py | 5 ++++- sdk/core/azure-core/tests/test_pyodide_transport.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py b/sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py index 4cb1689bca29..b80e866af564 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py @@ -55,7 +55,10 @@ async def load_body(self) -> None: """Load the body of the response.""" if self._content is None: self._content = await self._internal_response.bytes() - + + def body(self) -> bytes: + """The body is just the content.""" + return self.content class PyodideStreamDownloadGenerator(AsyncIterator[bytes]): """Simple stream download generator that returns the contents of diff --git a/sdk/core/azure-core/tests/test_pyodide_transport.py b/sdk/core/azure-core/tests/test_pyodide_transport.py index fc15cbe2825f..5212adb1105b 100644 --- a/sdk/core/azure-core/tests/test_pyodide_transport.py +++ b/sdk/core/azure-core/tests/test_pyodide_transport.py @@ -93,7 +93,7 @@ async def test_successful_send(self, mock_pyfetch, mock_pyodide_module, pipeline assert response.reason == response_text assert not response._is_closed - response.close() + await response.close() assert response._is_closed # Check that the call had the correct arguments. From dbe5a85ae57a0d0f3eb2f1ea545f38b57109c823 Mon Sep 17 00:00:00 2001 From: Steven Jin Date: Mon, 18 Jul 2022 16:49:10 -0700 Subject: [PATCH 12/32] lint --- sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py b/sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py index b80e866af564..b756cb24bfd1 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py @@ -55,7 +55,7 @@ async def load_body(self) -> None: """Load the body of the response.""" if self._content is None: self._content = await self._internal_response.bytes() - + def body(self) -> bytes: """The body is just the content.""" return self.content @@ -148,4 +148,3 @@ async def send(self, request: HttpRequest, **kwargs) -> PyodideTransportResponse await transport_response.load_body() return transport_response - From c69a1c31fb2de403f71362bd7498e967f610444b Mon Sep 17 00:00:00 2001 From: Steven Jin Date: Tue, 19 Jul 2022 08:43:31 -0700 Subject: [PATCH 13/32] make Pyodide classes public --- .../azure-core/azure/core/pipeline/transport/__init__.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/__init__.py b/sdk/core/azure-core/azure/core/pipeline/transport/__init__.py index 03d5b0d19c88..ae0a309518ca 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/__init__.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/__init__.py @@ -158,3 +158,11 @@ def __getattr__(name): pass # Aiohttp not installed except (ImportError, SyntaxError): pass # Asynchronous pipelines not supported. + +# see https://pyodide.org/en/stable/usage/faq.html#how-to-detect-that-code-is-run-with-pyodide +if "pyodide" in sys.modules: + from ._pyodide import PyodideTransport, PyodideTransportResponse + __all__.extend([ + 'PyodideTransport', + 'PyodideTransportResponse', + ]) \ No newline at end of file From 96d95cd7382a444f99c2868d281f835b52d3cc3f Mon Sep 17 00:00:00 2001 From: Steven Jin Date: Wed, 20 Jul 2022 09:06:42 -0700 Subject: [PATCH 14/32] decompress --- .../azure/core/pipeline/transport/_pyodide.py | 39 +++++++++++-------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py b/sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py index b756cb24bfd1..7c9fd114ad17 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py @@ -29,13 +29,15 @@ from pyodide import JsException # pylint: disable=import-error from pyodide.http import pyfetch # pylint: disable=import-error +import js -from azure.core.utils import CaseInsensitiveDict from azure.core.exceptions import HttpResponseError -from azure.core.rest._http_response_impl_async import AsyncHttpResponseImpl +from azure.core.utils import CaseInsensitiveDict -from ._requests_asyncio import AsyncioRequestsTransport +from ...rest._http_response_impl_async import AsyncHttpResponseImpl from . import HttpRequest +from ._requests_asyncio import AsyncioRequestsTransport + class PyodideTransportResponse(AsyncHttpResponseImpl): """Async response object for the `PyodideTransport`.""" @@ -43,7 +45,7 @@ class PyodideTransportResponse(AsyncHttpResponseImpl): def __init__(self, **kwargs): super(PyodideTransportResponse, self).__init__(**kwargs) # clone to avoid reading from the same `FetchResponse` a second time in `load_body`. - self.reader = self.internal_response.clone().js_response.body.getReader() + self._reader = self.internal_response.clone().js_response.body.getReader() async def close(self) -> None: """We don't actually have control over closing connections in the browser, so we just pretend @@ -60,17 +62,22 @@ def body(self) -> bytes: """The body is just the content.""" return self.content -class PyodideStreamDownloadGenerator(AsyncIterator[bytes]): +class PyodideStreamDownloadGenerator(AsyncIterator): """Simple stream download generator that returns the contents of a request. """ - def __init__(self, response: PyodideTransportResponse, *_, **__): + def __init__(self, response: PyodideTransportResponse, *_, **kwargs): self.block_size = response.block_size self.response = response # use this to efficiently store bytes. - self.stream = BytesIO() - self.closed = False + if kwargs.pop("decompress", False): + self._reader = response._reader.pipeThrough(js.DecompressStream.new("gzip")) + else: + self._reader = response._reader + self._stream = BytesIO() + + self._closed = False # We cannot control how many bytes we get from `response.reader`. `self.buffer_left` # indicates how many unread bytes there are in `self.stream` self.buffer_left = 0 @@ -78,25 +85,25 @@ def __init__(self, response: PyodideTransportResponse, *_, **__): async def __anext__(self) -> bytes: """Get the next block of bytes.""" - if self.closed: + if self._closed: raise StopAsyncIteration() # remember the initial stream position - start_pos = self.stream.tell() + start_pos = self._stream.tell() # move stream position to the end - self.stream.read() + self._stream.read() # read from reader until there is no more data or we have `self.block_size` unread bytes. while self.buffer_left < self.block_size: - read = await self.response.reader.read() + read = await self._reader.read() if read.done: - self.closed = True + self._closed = True break - self.buffer_left += self.stream.write(bytes(read.value)) + self.buffer_left += self._stream.write(bytes(read.value)) # move the stream position back to where we started - self.stream.seek(start_pos) + self._stream.seek(start_pos) self.buffer_left -= self.block_size - return self.stream.read(self.block_size) + return self._stream.read(self.block_size) class PyodideTransport(AsyncioRequestsTransport): """Implements a basic HTTP sender using the Pyodide Javascript Fetch API. From 142a364607477321ae7b5cc6ecb6390cfd204b2a Mon Sep 17 00:00:00 2001 From: Steven Jin Date: Wed, 20 Jul 2022 09:07:00 -0700 Subject: [PATCH 15/32] 3.7-compatible tests --- .../tests/test_pyodide_transport.py | 83 +++++++++++++++---- 1 file changed, 66 insertions(+), 17 deletions(-) diff --git a/sdk/core/azure-core/tests/test_pyodide_transport.py b/sdk/core/azure-core/tests/test_pyodide_transport.py index 5212adb1105b..4135dc6f4928 100644 --- a/sdk/core/azure-core/tests/test_pyodide_transport.py +++ b/sdk/core/azure-core/tests/test_pyodide_transport.py @@ -1,4 +1,30 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +# -------------------------------------------------------------------------- """Tests that mock the browser layer.""" +import asyncio import sys from typing import NamedTuple from unittest import mock @@ -15,21 +41,27 @@ class TestPyodideTransportClass: """Unittest for the Pyodide transport.""" - @pytest.fixture(scope="class", autouse=True) + @pytest.fixture() def mock_pyodide_module(self): """Create a mock for the Pyodide module.""" mock_pyodide_module = mock.Mock() - mock_pyodide_module.http.pyfetch = mock.AsyncMock() + mock_pyodide_module.http.pyfetch = mock.Mock() mock_pyodide_module.JsException = type("JsException", (Exception,), {}) return mock_pyodide_module - - @pytest.fixture(scope="class", autouse=True) - def transport(self, mock_pyodide_module): + + @pytest.fixture() + def mock_js_module(self): + """Mock the `js` module""" + return mock.Mock() + + @pytest.fixture() + def transport(self, mock_pyodide_module, mock_js_module): """Add the mock Pyodide module to `sys.modules` and import our transport.""" # Use patch so we don't clutter up the `sys.modules` namespace. patch_dict = ( ("pyodide", mock_pyodide_module), ("pyodide.http", mock_pyodide_module.http), + ("js", mock_js_module), ) with mock.patch.dict(sys.modules, patch_dict): # weird stuff is hppenig here, have to do full import. @@ -39,12 +71,12 @@ def transport(self, mock_pyodide_module): yield azure.core.pipeline.transport._pyodide - @pytest.fixture(scope="class", autouse=True) + @pytest.fixture() def pipeline(self, transport): """Create a pipeline to test.""" return AsyncPipeline(transport.PyodideTransport(), [AsyncRetryPolicy()]) - @pytest.fixture(scope="class", autouse=True) + @pytest.fixture() def mock_pyfetch(self, mock_pyodide_module): """Utility fixture for less typing.""" return mock_pyodide_module.http.pyfetch @@ -60,8 +92,14 @@ def create_mock_response( mock_response.js_response.headers = headers mock_response.status = status mock_response.status_text = status_text - mock_response.bytes = mock.AsyncMock(return_value=body) - return mock_response + bytes_promise = asyncio.Future() + bytes_promise.set_result(body) + mock_response.bytes = mock.Mock() + mock_response.bytes.return_value = bytes_promise + + response_promise = asyncio.Future() + response_promise.set_result(mock_response) + return response_promise @pytest.mark.asyncio async def test_successful_send(self, mock_pyfetch, mock_pyodide_module, pipeline): @@ -98,8 +136,8 @@ async def test_successful_send(self, mock_pyfetch, mock_pyodide_module, pipeline # Check that the call had the correct arguments. mock_pyfetch.assert_called_once() - args = mock_pyfetch.call_args.args - kwargs = mock_pyfetch.call_args.kwargs + args = mock_pyfetch.call_args[0] + kwargs = mock_pyfetch.call_args[1] assert len(args) == 1 assert args[0] == PLACEHOLDER_ENDPOINT assert kwargs["method"] == method @@ -132,17 +170,28 @@ class ReaderReturn(NamedTuple): response_mock = mock.Mock() response_mock.block_size = 5 - response_mock.reader.read = mock.AsyncMock() - response_mock.reader.read.return_value = ReaderReturn(value=b"01", done=False) + response_mock._reader.read = mock.Mock() + read_promise = asyncio.Future() + read_promise.set_result(ReaderReturn(value=b"01", done=False)) + response_mock._reader.read.return_value = read_promise generator = transport.PyodideStreamDownloadGenerator(response=response_mock) assert len(await generator.__anext__()) == response_mock.block_size - assert response_mock.reader.read.call_count == 3 + assert response_mock._reader.read.call_count == 3 assert len(await generator.__anext__()) == response_mock.block_size # 5 because there is a leftover byte from the previous `__anext__` call. - assert response_mock.reader.read.call_count == 5 + assert response_mock._reader.read.call_count == 5 - response_mock.reader.read.return_value = ReaderReturn(value=None, done=True) + read_promise = asyncio.Future() + read_promise.set_result(ReaderReturn(value=None, done=True)) + response_mock._reader.read.return_value = read_promise await generator.__anext__() with pytest.raises(StopAsyncIteration): - await generator.__anext__() \ No newline at end of file + await generator.__anext__() + + @pytest.mark.asyncio + async def test_download_generator_compress(self, transport, mock_js_module): + """Test that we are attempting to decompress data when passing the `decompress`.""" + transport.PyodideStreamDownloadGenerator(response=mock.Mock(), decompress=True) + mock_js_module.DecompressStream.new.assert_called_once_with("gzip") + \ No newline at end of file From c7c7d09442c500b274656fd32b8a4f22c282a997 Mon Sep 17 00:00:00 2001 From: Steven Jin Date: Wed, 20 Jul 2022 11:07:56 -0700 Subject: [PATCH 16/32] fix decompression --- .../azure/core/pipeline/transport/_pyodide.py | 9 +++++---- .../azure-core/tests/test_pyodide_transport.py | 14 +++++++------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py b/sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py index 7c9fd114ad17..fdf3617843d7 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py @@ -45,7 +45,8 @@ class PyodideTransportResponse(AsyncHttpResponseImpl): def __init__(self, **kwargs): super(PyodideTransportResponse, self).__init__(**kwargs) # clone to avoid reading from the same `FetchResponse` a second time in `load_body`. - self._reader = self.internal_response.clone().js_response.body.getReader() + self._js_stream = self.internal_response.clone().js_response.body + self._js_reader = None async def close(self) -> None: """We don't actually have control over closing connections in the browser, so we just pretend @@ -72,9 +73,9 @@ def __init__(self, response: PyodideTransportResponse, *_, **kwargs): self.response = response # use this to efficiently store bytes. if kwargs.pop("decompress", False): - self._reader = response._reader.pipeThrough(js.DecompressStream.new("gzip")) + self._js_reader = response._js_stream.pipeThrough(js.DecompressStream.new("gzip")).getReader() else: - self._reader = response._reader + self._js_reader = response._js_stream.getReader() self._stream = BytesIO() self._closed = False @@ -94,7 +95,7 @@ async def __anext__(self) -> bytes: self._stream.read() # read from reader until there is no more data or we have `self.block_size` unread bytes. while self.buffer_left < self.block_size: - read = await self._reader.read() + read = await self._js_reader.read() if read.done: self._closed = True break diff --git a/sdk/core/azure-core/tests/test_pyodide_transport.py b/sdk/core/azure-core/tests/test_pyodide_transport.py index 4135dc6f4928..d1f8b8c907ec 100644 --- a/sdk/core/azure-core/tests/test_pyodide_transport.py +++ b/sdk/core/azure-core/tests/test_pyodide_transport.py @@ -86,8 +86,6 @@ def create_mock_response( ) -> mock.Mock: """Create a mock response object that mimics `pyodide.http.FetchResponse`""" mock_response = mock.Mock() - if isinstance(body, str): - body = bytes(body, encoding="utf-8") mock_response.body = body mock_response.js_response.headers = headers mock_response.status = status @@ -170,21 +168,23 @@ class ReaderReturn(NamedTuple): response_mock = mock.Mock() response_mock.block_size = 5 - response_mock._reader.read = mock.Mock() + response_mock._js_reader.read = mock.Mock() read_promise = asyncio.Future() read_promise.set_result(ReaderReturn(value=b"01", done=False)) - response_mock._reader.read.return_value = read_promise + reader = mock.Mock() + reader.read.return_value = read_promise + response_mock._js_stream.getReader.return_value = reader generator = transport.PyodideStreamDownloadGenerator(response=response_mock) assert len(await generator.__anext__()) == response_mock.block_size - assert response_mock._reader.read.call_count == 3 + assert reader.read.call_count == 3 assert len(await generator.__anext__()) == response_mock.block_size # 5 because there is a leftover byte from the previous `__anext__` call. - assert response_mock._reader.read.call_count == 5 + assert reader.read.call_count == 5 read_promise = asyncio.Future() read_promise.set_result(ReaderReturn(value=None, done=True)) - response_mock._reader.read.return_value = read_promise + reader.read.return_value = read_promise await generator.__anext__() with pytest.raises(StopAsyncIteration): await generator.__anext__() From a667c12a793a362db4f8ed96fdb0ffe007181383 Mon Sep 17 00:00:00 2001 From: Steven Jin Date: Wed, 20 Jul 2022 11:30:05 -0700 Subject: [PATCH 17/32] decompress integration tests --- .../azure/core/pipeline/transport/_pyodide.py | 2 +- .../azure-core/samples/pyodide/browser_test.py | 12 +++++++++++- .../azure-core/samples/pyodide/data/hello-world.gz | Bin 0 -> 44 bytes .../azure-core/samples/pyodide/requirements.txt | 2 +- 4 files changed, 13 insertions(+), 3 deletions(-) create mode 100644 sdk/core/azure-core/samples/pyodide/data/hello-world.gz diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py b/sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py index fdf3617843d7..5128952b0263 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py @@ -73,7 +73,7 @@ def __init__(self, response: PyodideTransportResponse, *_, **kwargs): self.response = response # use this to efficiently store bytes. if kwargs.pop("decompress", False): - self._js_reader = response._js_stream.pipeThrough(js.DecompressStream.new("gzip")).getReader() + self._js_reader = response._js_stream.pipeThrough(js.DecompressionStream.new("gzip")).getReader() else: self._js_reader = response._js_stream.getReader() self._stream = BytesIO() diff --git a/sdk/core/azure-core/samples/pyodide/browser_test.py b/sdk/core/azure-core/samples/pyodide/browser_test.py index e6c2e0475fc8..1fd7535032e5 100644 --- a/sdk/core/azure-core/samples/pyodide/browser_test.py +++ b/sdk/core/azure-core/samples/pyodide/browser_test.py @@ -3,7 +3,8 @@ from azure.ai.textanalytics.aio import TextAnalyticsClient from azure.core.credentials import AzureKeyCredential -from azure.core.pipeline.transport._pyodide import PyodideTransport +from azure.core.pipeline.transport import PyodideTransport, HttpRequest +from azure.core.pipeline.transport._pyodide import PyodideStreamDownloadGenerator from azure.storage.blob.aio import BlobClient, BlobServiceClient # pylint: disable=import-error @@ -32,6 +33,15 @@ def __init__( blob_service_endpoint, blob_service_key, transport=PyodideTransport() ) + async def test_decompress_generator(self): + """Test that we can decompress streams properly.""" + url = "data/hello-world.gz" + request = HttpRequest(method="GET", url=url) + transport = PyodideTransport() + response = await transport.send(request, stream_response=True) + data = b"".join([x async for x in response.iter_bytes()]) + assert data == b"hello world!\n" + async def test_sentiment_analysis(self): """Test that sentiment analysis works.""" results = await self.text_analytics_client.analyze_sentiment( diff --git a/sdk/core/azure-core/samples/pyodide/data/hello-world.gz b/sdk/core/azure-core/samples/pyodide/data/hello-world.gz new file mode 100644 index 0000000000000000000000000000000000000000..34336ab5dd953ee01a608522d648b83797414fc9 GIT binary patch literal 44 zcmb2|=HR&IdLxN}IU_YECttTbzbGe#;k3uulPAM9&g-A@e8|M0Yt6Bqhk=0s0CRN@ Ai~s-t literal 0 HcmV?d00001 diff --git a/sdk/core/azure-core/samples/pyodide/requirements.txt b/sdk/core/azure-core/samples/pyodide/requirements.txt index ebf1a28ec837..3e0a7d91664b 100644 --- a/sdk/core/azure-core/samples/pyodide/requirements.txt +++ b/sdk/core/azure-core/samples/pyodide/requirements.txt @@ -1,6 +1,6 @@ python-dotenv # todo: delete the following line -https://tsjinxuanstorage2.blob.core.windows.net/pyodide/azure_core-1.24.3-py3-none-any.whl +https://tsjinxuanstorage2.blob.core.windows.net/pyodide/azure_core-1.25.0-py3-none-any.whl azure-ai-textanalytics azure-storage-blob azure-ai-formrecognizer From 8b6227e407ae4a1029cb2316892676c595278aad Mon Sep 17 00:00:00 2001 From: Steven Jin Date: Wed, 20 Jul 2022 12:32:58 -0700 Subject: [PATCH 18/32] decompression typo --- sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py | 2 +- sdk/core/azure-core/tests/test_pyodide_transport.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py b/sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py index fdf3617843d7..5128952b0263 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py @@ -73,7 +73,7 @@ def __init__(self, response: PyodideTransportResponse, *_, **kwargs): self.response = response # use this to efficiently store bytes. if kwargs.pop("decompress", False): - self._js_reader = response._js_stream.pipeThrough(js.DecompressStream.new("gzip")).getReader() + self._js_reader = response._js_stream.pipeThrough(js.DecompressionStream.new("gzip")).getReader() else: self._js_reader = response._js_stream.getReader() self._stream = BytesIO() diff --git a/sdk/core/azure-core/tests/test_pyodide_transport.py b/sdk/core/azure-core/tests/test_pyodide_transport.py index d1f8b8c907ec..6d5ddcaebe3f 100644 --- a/sdk/core/azure-core/tests/test_pyodide_transport.py +++ b/sdk/core/azure-core/tests/test_pyodide_transport.py @@ -193,5 +193,5 @@ class ReaderReturn(NamedTuple): async def test_download_generator_compress(self, transport, mock_js_module): """Test that we are attempting to decompress data when passing the `decompress`.""" transport.PyodideStreamDownloadGenerator(response=mock.Mock(), decompress=True) - mock_js_module.DecompressStream.new.assert_called_once_with("gzip") + mock_js_module.DecompressionStream.new.assert_called_once_with("gzip") \ No newline at end of file From 9001e1c90b624af26b24808122090af297cd772a Mon Sep 17 00:00:00 2001 From: Steven Jin Date: Wed, 20 Jul 2022 13:06:09 -0700 Subject: [PATCH 19/32] remove pyodide import from init --- .../azure-core/azure/core/pipeline/transport/__init__.py | 8 -------- .../core/pipeline/transport/{_pyodide.py => pyodide.py} | 2 +- sdk/core/azure-core/tests/test_pyodide_transport.py | 7 ++----- 3 files changed, 3 insertions(+), 14 deletions(-) rename sdk/core/azure-core/azure/core/pipeline/transport/{_pyodide.py => pyodide.py} (99%) diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/__init__.py b/sdk/core/azure-core/azure/core/pipeline/transport/__init__.py index ae0a309518ca..03d5b0d19c88 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/__init__.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/__init__.py @@ -158,11 +158,3 @@ def __getattr__(name): pass # Aiohttp not installed except (ImportError, SyntaxError): pass # Asynchronous pipelines not supported. - -# see https://pyodide.org/en/stable/usage/faq.html#how-to-detect-that-code-is-run-with-pyodide -if "pyodide" in sys.modules: - from ._pyodide import PyodideTransport, PyodideTransportResponse - __all__.extend([ - 'PyodideTransport', - 'PyodideTransportResponse', - ]) \ No newline at end of file diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py b/sdk/core/azure-core/azure/core/pipeline/transport/pyodide.py similarity index 99% rename from sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py rename to sdk/core/azure-core/azure/core/pipeline/transport/pyodide.py index 5128952b0263..761eba37be4d 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_pyodide.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/pyodide.py @@ -27,9 +27,9 @@ from collections.abc import AsyncIterator from io import BytesIO +import js # pylint: disable=import-error from pyodide import JsException # pylint: disable=import-error from pyodide.http import pyfetch # pylint: disable=import-error -import js from azure.core.exceptions import HttpResponseError from azure.core.utils import CaseInsensitiveDict diff --git a/sdk/core/azure-core/tests/test_pyodide_transport.py b/sdk/core/azure-core/tests/test_pyodide_transport.py index 6d5ddcaebe3f..161717e44448 100644 --- a/sdk/core/azure-core/tests/test_pyodide_transport.py +++ b/sdk/core/azure-core/tests/test_pyodide_transport.py @@ -64,12 +64,9 @@ def transport(self, mock_pyodide_module, mock_js_module): ("js", mock_js_module), ) with mock.patch.dict(sys.modules, patch_dict): - # weird stuff is hppenig here, have to do full import. - # if I do `from azure.core.pipline.transport import _pyodide` - # I end up with `_pyodide = 'pyodide'` ??????? - import azure.core.pipeline.transport._pyodide + import azure.core.pipeline.transport.pyodide - yield azure.core.pipeline.transport._pyodide + yield azure.core.pipeline.transport.pyodide @pytest.fixture() def pipeline(self, transport): From 375014397c6ca8531c50146c1ee68ad2caedb6aa Mon Sep 17 00:00:00 2001 From: Steven Jin Date: Wed, 20 Jul 2022 13:44:02 -0700 Subject: [PATCH 20/32] Add pipeline arg to download generator --- .../azure/core/pipeline/transport/pyodide.py | 13 +++++++------ sdk/core/azure-core/samples/pyodide/browser_test.py | 11 ++++++++--- sdk/core/azure-core/tests/test_pyodide_transport.py | 10 +++++----- 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/pyodide.py b/sdk/core/azure-core/azure/core/pipeline/transport/pyodide.py index 761eba37be4d..01ac2ffc1fbe 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/pyodide.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/pyodide.py @@ -32,6 +32,7 @@ from pyodide.http import pyfetch # pylint: disable=import-error from azure.core.exceptions import HttpResponseError +from azure.core.pipeline import Pipeline from azure.core.utils import CaseInsensitiveDict from ...rest._http_response_impl_async import AsyncHttpResponseImpl @@ -68,8 +69,8 @@ class PyodideStreamDownloadGenerator(AsyncIterator): a request. """ - def __init__(self, response: PyodideTransportResponse, *_, **kwargs): - self.block_size = response.block_size + def __init__(self, pipeline: Pipeline, response: PyodideTransportResponse, *_, **kwargs): + self._block_size = response._block_size self.response = response # use this to efficiently store bytes. if kwargs.pop("decompress", False): @@ -93,8 +94,8 @@ async def __anext__(self) -> bytes: start_pos = self._stream.tell() # move stream position to the end self._stream.read() - # read from reader until there is no more data or we have `self.block_size` unread bytes. - while self.buffer_left < self.block_size: + # read from reader until there is no more data or we have `self._block_size` unread bytes. + while self.buffer_left < self._block_size: read = await self._js_reader.read() if read.done: self._closed = True @@ -103,8 +104,8 @@ async def __anext__(self) -> bytes: # move the stream position back to where we started self._stream.seek(start_pos) - self.buffer_left -= self.block_size - return self._stream.read(self.block_size) + self.buffer_left -= self._block_size + return self._stream.read(self._block_size) class PyodideTransport(AsyncioRequestsTransport): """Implements a basic HTTP sender using the Pyodide Javascript Fetch API. diff --git a/sdk/core/azure-core/samples/pyodide/browser_test.py b/sdk/core/azure-core/samples/pyodide/browser_test.py index 1fd7535032e5..6f47d4698899 100644 --- a/sdk/core/azure-core/samples/pyodide/browser_test.py +++ b/sdk/core/azure-core/samples/pyodide/browser_test.py @@ -3,8 +3,8 @@ from azure.ai.textanalytics.aio import TextAnalyticsClient from azure.core.credentials import AzureKeyCredential -from azure.core.pipeline.transport import PyodideTransport, HttpRequest -from azure.core.pipeline.transport._pyodide import PyodideStreamDownloadGenerator +from azure.core.pipeline.transport import HttpRequest +from azure.core.pipeline.transport.pyodide import PyodideTransport from azure.storage.blob.aio import BlobClient, BlobServiceClient # pylint: disable=import-error @@ -40,7 +40,12 @@ async def test_decompress_generator(self): transport = PyodideTransport() response = await transport.send(request, stream_response=True) data = b"".join([x async for x in response.iter_bytes()]) - assert data == b"hello world!\n" + assert data == b"hello world\n" + + response = await transport.send(request, stream_response=True) + data = b"".join([x async for x in response.iter_raw()]) + assert data != b"hello world\n" + async def test_sentiment_analysis(self): """Test that sentiment analysis works.""" diff --git a/sdk/core/azure-core/tests/test_pyodide_transport.py b/sdk/core/azure-core/tests/test_pyodide_transport.py index 161717e44448..53f5c14defda 100644 --- a/sdk/core/azure-core/tests/test_pyodide_transport.py +++ b/sdk/core/azure-core/tests/test_pyodide_transport.py @@ -164,18 +164,18 @@ class ReaderReturn(NamedTuple): done: bool response_mock = mock.Mock() - response_mock.block_size = 5 + response_mock._block_size = 5 response_mock._js_reader.read = mock.Mock() read_promise = asyncio.Future() read_promise.set_result(ReaderReturn(value=b"01", done=False)) reader = mock.Mock() reader.read.return_value = read_promise response_mock._js_stream.getReader.return_value = reader - generator = transport.PyodideStreamDownloadGenerator(response=response_mock) + generator = transport.PyodideStreamDownloadGenerator(pipeline=None, response=response_mock) - assert len(await generator.__anext__()) == response_mock.block_size + assert len(await generator.__anext__()) == response_mock._block_size assert reader.read.call_count == 3 - assert len(await generator.__anext__()) == response_mock.block_size + assert len(await generator.__anext__()) == response_mock._block_size # 5 because there is a leftover byte from the previous `__anext__` call. assert reader.read.call_count == 5 @@ -189,6 +189,6 @@ class ReaderReturn(NamedTuple): @pytest.mark.asyncio async def test_download_generator_compress(self, transport, mock_js_module): """Test that we are attempting to decompress data when passing the `decompress`.""" - transport.PyodideStreamDownloadGenerator(response=mock.Mock(), decompress=True) + transport.PyodideStreamDownloadGenerator(pipeline=None, response=mock.Mock(), decompress=True) mock_js_module.DecompressionStream.new.assert_called_once_with("gzip") \ No newline at end of file From 533672c2fb0ef8e47345a4a20ffe515cef867eeb Mon Sep 17 00:00:00 2001 From: Steven Jin Date: Wed, 20 Jul 2022 13:47:37 -0700 Subject: [PATCH 21/32] add pipeline arg to download generator --- .../azure/core/pipeline/transport/pyodide.py | 14 ++++++++------ .../azure-core/tests/test_pyodide_transport.py | 10 +++++----- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/pyodide.py b/sdk/core/azure-core/azure/core/pipeline/transport/pyodide.py index 761eba37be4d..b721fe7e296e 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/pyodide.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/pyodide.py @@ -32,6 +32,7 @@ from pyodide.http import pyfetch # pylint: disable=import-error from azure.core.exceptions import HttpResponseError +from azure.core.pipeline import Pipeline from azure.core.utils import CaseInsensitiveDict from ...rest._http_response_impl_async import AsyncHttpResponseImpl @@ -68,8 +69,9 @@ class PyodideStreamDownloadGenerator(AsyncIterator): a request. """ - def __init__(self, response: PyodideTransportResponse, *_, **kwargs): - self.block_size = response.block_size + # pylint: disable=unused-argument + def __init__(self, pipeline: Pipeline, response: PyodideTransportResponse, *_, **kwargs): + self._block_size = response._block_size self.response = response # use this to efficiently store bytes. if kwargs.pop("decompress", False): @@ -93,8 +95,8 @@ async def __anext__(self) -> bytes: start_pos = self._stream.tell() # move stream position to the end self._stream.read() - # read from reader until there is no more data or we have `self.block_size` unread bytes. - while self.buffer_left < self.block_size: + # read from reader until there is no more data or we have `self._block_size` unread bytes. + while self.buffer_left < self._block_size: read = await self._js_reader.read() if read.done: self._closed = True @@ -103,8 +105,8 @@ async def __anext__(self) -> bytes: # move the stream position back to where we started self._stream.seek(start_pos) - self.buffer_left -= self.block_size - return self._stream.read(self.block_size) + self.buffer_left -= self._block_size + return self._stream.read(self._block_size) class PyodideTransport(AsyncioRequestsTransport): """Implements a basic HTTP sender using the Pyodide Javascript Fetch API. diff --git a/sdk/core/azure-core/tests/test_pyodide_transport.py b/sdk/core/azure-core/tests/test_pyodide_transport.py index 161717e44448..53f5c14defda 100644 --- a/sdk/core/azure-core/tests/test_pyodide_transport.py +++ b/sdk/core/azure-core/tests/test_pyodide_transport.py @@ -164,18 +164,18 @@ class ReaderReturn(NamedTuple): done: bool response_mock = mock.Mock() - response_mock.block_size = 5 + response_mock._block_size = 5 response_mock._js_reader.read = mock.Mock() read_promise = asyncio.Future() read_promise.set_result(ReaderReturn(value=b"01", done=False)) reader = mock.Mock() reader.read.return_value = read_promise response_mock._js_stream.getReader.return_value = reader - generator = transport.PyodideStreamDownloadGenerator(response=response_mock) + generator = transport.PyodideStreamDownloadGenerator(pipeline=None, response=response_mock) - assert len(await generator.__anext__()) == response_mock.block_size + assert len(await generator.__anext__()) == response_mock._block_size assert reader.read.call_count == 3 - assert len(await generator.__anext__()) == response_mock.block_size + assert len(await generator.__anext__()) == response_mock._block_size # 5 because there is a leftover byte from the previous `__anext__` call. assert reader.read.call_count == 5 @@ -189,6 +189,6 @@ class ReaderReturn(NamedTuple): @pytest.mark.asyncio async def test_download_generator_compress(self, transport, mock_js_module): """Test that we are attempting to decompress data when passing the `decompress`.""" - transport.PyodideStreamDownloadGenerator(response=mock.Mock(), decompress=True) + transport.PyodideStreamDownloadGenerator(pipeline=None, response=mock.Mock(), decompress=True) mock_js_module.DecompressionStream.new.assert_called_once_with("gzip") \ No newline at end of file From 06fdc8dbc842cd32dcb961553ed39df79ca49c02 Mon Sep 17 00:00:00 2001 From: Steven Jin Date: Wed, 20 Jul 2022 13:54:08 -0700 Subject: [PATCH 22/32] some renaming --- .../async_testing.py | 0 .../browser_test.py | 0 .../data/hello-world.gz | Bin .../example-env | 0 .../index.html | 0 .../{pyodide => test_pyodide_integration}/readme.md | 9 +++++---- .../requirements.txt | 0 7 files changed, 5 insertions(+), 4 deletions(-) rename sdk/core/azure-core/samples/{pyodide => test_pyodide_integration}/async_testing.py (100%) rename sdk/core/azure-core/samples/{pyodide => test_pyodide_integration}/browser_test.py (100%) rename sdk/core/azure-core/samples/{pyodide => test_pyodide_integration}/data/hello-world.gz (100%) rename sdk/core/azure-core/samples/{pyodide => test_pyodide_integration}/example-env (100%) rename sdk/core/azure-core/samples/{pyodide => test_pyodide_integration}/index.html (100%) rename sdk/core/azure-core/samples/{pyodide => test_pyodide_integration}/readme.md (75%) rename sdk/core/azure-core/samples/{pyodide => test_pyodide_integration}/requirements.txt (100%) diff --git a/sdk/core/azure-core/samples/pyodide/async_testing.py b/sdk/core/azure-core/samples/test_pyodide_integration/async_testing.py similarity index 100% rename from sdk/core/azure-core/samples/pyodide/async_testing.py rename to sdk/core/azure-core/samples/test_pyodide_integration/async_testing.py diff --git a/sdk/core/azure-core/samples/pyodide/browser_test.py b/sdk/core/azure-core/samples/test_pyodide_integration/browser_test.py similarity index 100% rename from sdk/core/azure-core/samples/pyodide/browser_test.py rename to sdk/core/azure-core/samples/test_pyodide_integration/browser_test.py diff --git a/sdk/core/azure-core/samples/pyodide/data/hello-world.gz b/sdk/core/azure-core/samples/test_pyodide_integration/data/hello-world.gz similarity index 100% rename from sdk/core/azure-core/samples/pyodide/data/hello-world.gz rename to sdk/core/azure-core/samples/test_pyodide_integration/data/hello-world.gz diff --git a/sdk/core/azure-core/samples/pyodide/example-env b/sdk/core/azure-core/samples/test_pyodide_integration/example-env similarity index 100% rename from sdk/core/azure-core/samples/pyodide/example-env rename to sdk/core/azure-core/samples/test_pyodide_integration/example-env diff --git a/sdk/core/azure-core/samples/pyodide/index.html b/sdk/core/azure-core/samples/test_pyodide_integration/index.html similarity index 100% rename from sdk/core/azure-core/samples/pyodide/index.html rename to sdk/core/azure-core/samples/test_pyodide_integration/index.html diff --git a/sdk/core/azure-core/samples/pyodide/readme.md b/sdk/core/azure-core/samples/test_pyodide_integration/readme.md similarity index 75% rename from sdk/core/azure-core/samples/pyodide/readme.md rename to sdk/core/azure-core/samples/test_pyodide_integration/readme.md index 3febb530a1eb..b77429080bd1 100644 --- a/sdk/core/azure-core/samples/pyodide/readme.md +++ b/sdk/core/azure-core/samples/test_pyodide_integration/readme.md @@ -2,13 +2,13 @@ ## Running -Once you have set up your Azure Resources and your `.env` filem, from this directory, run +Once you have set up your Azure Resources and your `.env` file, from this directory, run ```python python -h http.server 8000 ``` -(You can use any other port). Then, from a Chromium-based browser such as Edge, go to [`http://localhost:8000/test/integration`](http://localhost:8000/test/integration) and the tests will be run in the browser. Dev tip: keep your browser's devtools open. +(You can use any other port). Then, from a Chromium-based browser such as Edge, go to [`http://localhost:8000/`](http://localhost:8000/) and the tests will be run in the browser. Dev tip: keep your browser's devtools open. ## Adding tests @@ -17,7 +17,8 @@ Add tests in `browser_test.py`. I couldn't get `pytest` or `unittest` to coopera ## Sensitive values To run the tests, you need a `.env` folder in this directory with your sensitive values. -see `example-env`. You can then access the values as environment variables using `os.getenv`. +see `example-env`. You can then access the values as environment variables using `os.getenv`. You will +need to have your `textanalytics` key and endpoint as well as your Blob Storage key and url. ## Dependencies @@ -29,7 +30,7 @@ You need your own Text Analytics and Blob Storage accounts to run these tests. B | Allowed origins | Allowed methods | Allowed headers | Exposed headers | Max age | |-----------------|-----------------|-----------------|-----------------|---------| -| `\*` | All | `\*` | See below | `3600` | +| `*` | All | `*` | See below | `3600` | For exposed headers, put diff --git a/sdk/core/azure-core/samples/pyodide/requirements.txt b/sdk/core/azure-core/samples/test_pyodide_integration/requirements.txt similarity index 100% rename from sdk/core/azure-core/samples/pyodide/requirements.txt rename to sdk/core/azure-core/samples/test_pyodide_integration/requirements.txt From e2f8ec8228285dd7de69c7629c4095c0db5e856f Mon Sep 17 00:00:00 2001 From: Steven Jin Date: Wed, 20 Jul 2022 16:34:17 -0700 Subject: [PATCH 23/32] izzy pr --- .../azure/core/pipeline/transport/pyodide.py | 21 +++++++------------ 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/pyodide.py b/sdk/core/azure-core/azure/core/pipeline/transport/pyodide.py index b721fe7e296e..61f9a725798e 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/pyodide.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/pyodide.py @@ -44,8 +44,8 @@ class PyodideTransportResponse(AsyncHttpResponseImpl): """Async response object for the `PyodideTransport`.""" def __init__(self, **kwargs): - super(PyodideTransportResponse, self).__init__(**kwargs) - # clone to avoid reading from the same `FetchResponse` a second time in `load_body`. + super().__init__(**kwargs) + # clone to avoid reading from the same `FetchResponse` a second time in `read`. self._js_stream = self.internal_response.clone().js_response.body self._js_reader = None @@ -55,11 +55,6 @@ async def close(self) -> None: """ self._is_closed = True - async def load_body(self) -> None: - """Load the body of the response.""" - if self._content is None: - self._content = await self._internal_response.bytes() - def body(self) -> bytes: """The body is just the content.""" return self.content @@ -81,9 +76,9 @@ def __init__(self, pipeline: Pipeline, response: PyodideTransportResponse, *_, * self._stream = BytesIO() self._closed = False - # We cannot control how many bytes we get from `response.reader`. `self.buffer_left` + # We cannot control how many bytes we get from `response.reader`. `self._buffer_left` # indicates how many unread bytes there are in `self.stream` - self.buffer_left = 0 + self._buffer_left = 0 self.done = False async def __anext__(self) -> bytes: @@ -96,16 +91,16 @@ async def __anext__(self) -> bytes: # move stream position to the end self._stream.read() # read from reader until there is no more data or we have `self._block_size` unread bytes. - while self.buffer_left < self._block_size: + while self._buffer_left < self._block_size: read = await self._js_reader.read() if read.done: self._closed = True break - self.buffer_left += self._stream.write(bytes(read.value)) + self._buffer_left += self._stream.write(bytes(read.value)) # move the stream position back to where we started self._stream.seek(start_pos) - self.buffer_left -= self._block_size + self._buffer_left -= self._block_size return self._stream.read(self._block_size) class PyodideTransport(AsyncioRequestsTransport): @@ -155,6 +150,6 @@ async def send(self, request: HttpRequest, **kwargs) -> PyodideTransportResponse stream_download_generator=PyodideStreamDownloadGenerator, ) if not stream_response: - await transport_response.load_body() + await transport_response.read() return transport_response From 8a436ccdc85084258944fd33f06080c18f7be57d Mon Sep 17 00:00:00 2001 From: Steven Jin Date: Thu, 21 Jul 2022 10:21:01 -0700 Subject: [PATCH 24/32] fix decompression --- .../azure/core/pipeline/transport/pyodide.py | 22 +++++++++++-------- .../test_pyodide_integration/browser_test.py | 10 +++++++++ .../tests/test_pyodide_transport.py | 14 ++++++++++-- 3 files changed, 35 insertions(+), 11 deletions(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/pyodide.py b/sdk/core/azure-core/azure/core/pipeline/transport/pyodide.py index 61f9a725798e..14f1780099df 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/pyodide.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/pyodide.py @@ -43,11 +43,10 @@ class PyodideTransportResponse(AsyncHttpResponseImpl): """Async response object for the `PyodideTransport`.""" - def __init__(self, **kwargs): - super().__init__(**kwargs) - # clone to avoid reading from the same `FetchResponse` a second time in `read`. - self._js_stream = self.internal_response.clone().js_response.body - self._js_reader = None + @property + def js_stream(self): + """So we get a fresh stream every time.""" + return self._internal_response.clone().js_response.body async def close(self) -> None: """We don't actually have control over closing connections in the browser, so we just pretend @@ -58,6 +57,11 @@ async def close(self) -> None: def body(self) -> bytes: """The body is just the content.""" return self.content + + async def load_body(self) -> None: + """Backcompat""" + if self._content is None: + self._content = await self._internal_response.clone().bytes() class PyodideStreamDownloadGenerator(AsyncIterator): """Simple stream download generator that returns the contents of @@ -69,10 +73,10 @@ def __init__(self, pipeline: Pipeline, response: PyodideTransportResponse, *_, * self._block_size = response._block_size self.response = response # use this to efficiently store bytes. - if kwargs.pop("decompress", False): - self._js_reader = response._js_stream.pipeThrough(js.DecompressionStream.new("gzip")).getReader() + if kwargs.pop("decompress", False) and self.response.headers.get("enc", None) in ("gzip", "deflate"): + self._js_reader = response.js_stream.pipeThrough(js.DecompressionStream.new("gzip")).getReader() else: - self._js_reader = response._js_stream.getReader() + self._js_reader = response.js_stream.getReader() self._stream = BytesIO() self._closed = False @@ -150,6 +154,6 @@ async def send(self, request: HttpRequest, **kwargs) -> PyodideTransportResponse stream_download_generator=PyodideStreamDownloadGenerator, ) if not stream_response: - await transport_response.read() + await transport_response.load_body() return transport_response diff --git a/sdk/core/azure-core/samples/test_pyodide_integration/browser_test.py b/sdk/core/azure-core/samples/test_pyodide_integration/browser_test.py index 6f47d4698899..5e10689bd6cd 100644 --- a/sdk/core/azure-core/samples/test_pyodide_integration/browser_test.py +++ b/sdk/core/azure-core/samples/test_pyodide_integration/browser_test.py @@ -39,6 +39,7 @@ async def test_decompress_generator(self): request = HttpRequest(method="GET", url=url) transport = PyodideTransport() response = await transport.send(request, stream_response=True) + response.headers["enc"] = "deflate" data = b"".join([x async for x in response.iter_bytes()]) assert data == b"hello world\n" @@ -46,6 +47,15 @@ async def test_decompress_generator(self): data = b"".join([x async for x in response.iter_raw()]) assert data != b"hello world\n" + response = await transport.send(request, stream_response=True) + data = b"".join([x async for x in response.iter_bytes()]) + assert data != b"hello world\n" + + response = await transport.send(request, stream_response=True) + response.headers["enc"] = "deflate" + data = b"".join([x async for x in response.iter_bytes()]) + assert data == b"hello world\n" + async def test_sentiment_analysis(self): """Test that sentiment analysis works.""" diff --git a/sdk/core/azure-core/tests/test_pyodide_transport.py b/sdk/core/azure-core/tests/test_pyodide_transport.py index 53f5c14defda..246510b7766d 100644 --- a/sdk/core/azure-core/tests/test_pyodide_transport.py +++ b/sdk/core/azure-core/tests/test_pyodide_transport.py @@ -91,6 +91,7 @@ def create_mock_response( bytes_promise.set_result(body) mock_response.bytes = mock.Mock() mock_response.bytes.return_value = bytes_promise + mock_response.clone.return_value = mock_response response_promise = asyncio.Future() response_promise.set_result(mock_response) @@ -120,6 +121,7 @@ async def test_successful_send(self, mock_pyfetch, mock_pyodide_module, pipeline mock_pyodide_module.http.pyfetch.return_value = mock_response response = (await pipeline.run(request=request)).http_response # Check that the pipeline processed the data correctly. + await response.load_body() assert response.body() == response_body assert response.status_code == response_status assert response.headers == response_headers @@ -144,6 +146,12 @@ async def test_successful_send(self, mock_pyfetch, mock_pyodide_module, pipeline assert kwargs["cert"] is None assert not kwargs["files"] + # check that the js_stream property is cloning + num_clones = mock_pyfetch.clone.call_count + response.js_stream + assert mock_pyfetch.call_count == num_clones + 1 + + @pytest.mark.asyncio async def test_unsuccessful_send(self, mock_pyfetch, mock_pyodide_module, pipeline): """Test that the pipeline is failing correctly.""" @@ -170,7 +178,7 @@ class ReaderReturn(NamedTuple): read_promise.set_result(ReaderReturn(value=b"01", done=False)) reader = mock.Mock() reader.read.return_value = read_promise - response_mock._js_stream.getReader.return_value = reader + response_mock.js_stream.getReader.return_value = reader generator = transport.PyodideStreamDownloadGenerator(pipeline=None, response=response_mock) assert len(await generator.__anext__()) == response_mock._block_size @@ -189,6 +197,8 @@ class ReaderReturn(NamedTuple): @pytest.mark.asyncio async def test_download_generator_compress(self, transport, mock_js_module): """Test that we are attempting to decompress data when passing the `decompress`.""" - transport.PyodideStreamDownloadGenerator(pipeline=None, response=mock.Mock(), decompress=True) + response = mock.Mock() + response.headers = {"enc": "deflate"} + transport.PyodideStreamDownloadGenerator(pipeline=None, response=response, decompress=True) mock_js_module.DecompressionStream.new.assert_called_once_with("gzip") \ No newline at end of file From ceae0e22e331115aa11140c0f65c9228e7a00118 Mon Sep 17 00:00:00 2001 From: Steven Jin Date: Thu, 21 Jul 2022 10:29:41 -0700 Subject: [PATCH 25/32] better integration tests --- .../samples/test_pyodide_integration/readme.md | 14 ++++++++++---- .../test_pyodide_integration/requirements.txt | 3 +-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/sdk/core/azure-core/samples/test_pyodide_integration/readme.md b/sdk/core/azure-core/samples/test_pyodide_integration/readme.md index b77429080bd1..bb6f848ae6a0 100644 --- a/sdk/core/azure-core/samples/test_pyodide_integration/readme.md +++ b/sdk/core/azure-core/samples/test_pyodide_integration/readme.md @@ -2,13 +2,19 @@ ## Running -Once you have set up your Azure Resources and your `.env` file, from this directory, run +Once you have set up your Azure Resources and your `.env` file, navigate to the `/sdk/core/azure-core`. Make a wheel of core by running + +```bash +python setup.py bdist-wheel +``` + +and run ```python -python -h http.server 8000 +python -h http.server ``` -(You can use any other port). Then, from a Chromium-based browser such as Edge, go to [`http://localhost:8000/`](http://localhost:8000/) and the tests will be run in the browser. Dev tip: keep your browser's devtools open. +to set up the test server. Then, from a Chromium-based browser such as Edge, go to [`http://localhost:8000/samples/test_pyodide_integration`](http://localhost:8000/samples/test_pyodide_integration) and the tests will be run in the browser. Dev tip: keep your browser's devtools open. If you make a change to the source code, remember to rebuild your wheel. Note that you might have to update `requirements.txt` depending on the version of `azure-core`. Just make sure the url to `azure-core` in `requirements.txt` matches that of the wheel in `../../dist/`. ## Adding tests @@ -34,4 +40,4 @@ You need your own Text Analytics and Blob Storage accounts to run these tests. B For exposed headers, put -> Server,Content-Range,ETag,Last-Modified,Accept-Ranges,x-ms-* +> Server,Content-Range,ETag,Last-Modified,Accept-Ranges,x-ms-*,enc diff --git a/sdk/core/azure-core/samples/test_pyodide_integration/requirements.txt b/sdk/core/azure-core/samples/test_pyodide_integration/requirements.txt index 3e0a7d91664b..97135659a4df 100644 --- a/sdk/core/azure-core/samples/test_pyodide_integration/requirements.txt +++ b/sdk/core/azure-core/samples/test_pyodide_integration/requirements.txt @@ -1,6 +1,5 @@ python-dotenv -# todo: delete the following line -https://tsjinxuanstorage2.blob.core.windows.net/pyodide/azure_core-1.25.0-py3-none-any.whl +http://localhost:8000/dist/azure_core-1.25.0-py3-none-any.whl azure-ai-textanalytics azure-storage-blob azure-ai-formrecognizer From b602484695657306a049ef12c5281b3c78a5a5ec Mon Sep 17 00:00:00 2001 From: Steven Jin Date: Thu, 21 Jul 2022 12:03:01 -0700 Subject: [PATCH 26/32] lint --- sdk/core/azure-core/azure/core/pipeline/transport/pyodide.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/pyodide.py b/sdk/core/azure-core/azure/core/pipeline/transport/pyodide.py index 14f1780099df..81755a1cb7b3 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/pyodide.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/pyodide.py @@ -57,7 +57,7 @@ async def close(self) -> None: def body(self) -> bytes: """The body is just the content.""" return self.content - + async def load_body(self) -> None: """Backcompat""" if self._content is None: From c38b35a20f18548e396fb52cf9cad75d35e8fa4b Mon Sep 17 00:00:00 2001 From: Steven Jin Date: Thu, 21 Jul 2022 12:11:03 -0700 Subject: [PATCH 27/32] emphasize experimental --- sdk/core/azure-core/CHANGELOG.md | 2 +- .../azure-core/azure/core/pipeline/transport/pyodide.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/sdk/core/azure-core/CHANGELOG.md b/sdk/core/azure-core/CHANGELOG.md index 1289b62606d2..6e3f30c8d502 100644 --- a/sdk/core/azure-core/CHANGELOG.md +++ b/sdk/core/azure-core/CHANGELOG.md @@ -6,7 +6,7 @@ Azure-core is supported on Python 3.7 or later. For more details, please read ou ### Features Added -- Added Pyodide-compatible transport. +- Added **experimental** Pyodide-compatible transport. - Added `CaseInsensitiveDict` implementation in `azure.core.utils` removing dependency on `requests` and `aiohttp` ### Breaking Changes diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/pyodide.py b/sdk/core/azure-core/azure/core/pipeline/transport/pyodide.py index 81755a1cb7b3..cab40298f68b 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/pyodide.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/pyodide.py @@ -108,11 +108,11 @@ async def __anext__(self) -> bytes: return self._stream.read(self._block_size) class PyodideTransport(AsyncioRequestsTransport): - """Implements a basic HTTP sender using the Pyodide Javascript Fetch API. + """**This object is experimental**, meaning it may be changed in a future release + or might break with a future Pyodide release. This transport was built with Pyodide + version 0.20.0. - WARNING: Pyodide is still an alpha technology. As such, this transport - is highly experimental and subject to breaking changes. This transport was - built around Pyodide version 0.20.0. + Implements a basic HTTP sender using the Pyodide Javascript Fetch API. """ async def send(self, request: HttpRequest, **kwargs) -> PyodideTransportResponse: From 3d0b702dac8d5ceac7a859de3ab863e500d41e0b Mon Sep 17 00:00:00 2001 From: Steven Jin Date: Thu, 4 Aug 2022 09:23:09 -0700 Subject: [PATCH 28/32] use new __init__ file --- .../azure/core/pipeline/transport/__init__.py | 12 ++ .../azure/core/pipeline/transport/pyodide.py | 159 ------------------ 2 files changed, 12 insertions(+), 159 deletions(-) delete mode 100644 sdk/core/azure-core/azure/core/pipeline/transport/pyodide.py diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/__init__.py b/sdk/core/azure-core/azure/core/pipeline/transport/__init__.py index 7daa8fe4c959..87afe33482af 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/__init__.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/__init__.py @@ -102,6 +102,18 @@ def __getattr__(name): transport = TrioRequestsTransportResponse except ImportError: raise ImportError("trio package is not installed") + if name == 'PyodideTransport': + try: + from ._pyodide import PyodideTransport + transport = PyodideTransport + except ImportError: + raise ImportError("pyodide package is not installed") + if name == 'PyodideTransportResponse': + try: + from ._pyodide import PyodideTransportResponse + transport = PyodideTransportResponse + except ImportError: + raise ImportError("pyodide package is not installed") if transport: return transport raise AttributeError(f"module 'azure.core.pipeline.transport' has no attribute {name}") diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/pyodide.py b/sdk/core/azure-core/azure/core/pipeline/transport/pyodide.py deleted file mode 100644 index cab40298f68b..000000000000 --- a/sdk/core/azure-core/azure/core/pipeline/transport/pyodide.py +++ /dev/null @@ -1,159 +0,0 @@ -# -------------------------------------------------------------------------- -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# The MIT License (MIT) -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to -# deal in the Software without restriction, including without limitation the -# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -# sell copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# -# -------------------------------------------------------------------------- - -from collections.abc import AsyncIterator -from io import BytesIO - -import js # pylint: disable=import-error -from pyodide import JsException # pylint: disable=import-error -from pyodide.http import pyfetch # pylint: disable=import-error - -from azure.core.exceptions import HttpResponseError -from azure.core.pipeline import Pipeline -from azure.core.utils import CaseInsensitiveDict - -from ...rest._http_response_impl_async import AsyncHttpResponseImpl -from . import HttpRequest -from ._requests_asyncio import AsyncioRequestsTransport - - -class PyodideTransportResponse(AsyncHttpResponseImpl): - """Async response object for the `PyodideTransport`.""" - - @property - def js_stream(self): - """So we get a fresh stream every time.""" - return self._internal_response.clone().js_response.body - - async def close(self) -> None: - """We don't actually have control over closing connections in the browser, so we just pretend - to close. - """ - self._is_closed = True - - def body(self) -> bytes: - """The body is just the content.""" - return self.content - - async def load_body(self) -> None: - """Backcompat""" - if self._content is None: - self._content = await self._internal_response.clone().bytes() - -class PyodideStreamDownloadGenerator(AsyncIterator): - """Simple stream download generator that returns the contents of - a request. - """ - - # pylint: disable=unused-argument - def __init__(self, pipeline: Pipeline, response: PyodideTransportResponse, *_, **kwargs): - self._block_size = response._block_size - self.response = response - # use this to efficiently store bytes. - if kwargs.pop("decompress", False) and self.response.headers.get("enc", None) in ("gzip", "deflate"): - self._js_reader = response.js_stream.pipeThrough(js.DecompressionStream.new("gzip")).getReader() - else: - self._js_reader = response.js_stream.getReader() - self._stream = BytesIO() - - self._closed = False - # We cannot control how many bytes we get from `response.reader`. `self._buffer_left` - # indicates how many unread bytes there are in `self.stream` - self._buffer_left = 0 - self.done = False - - async def __anext__(self) -> bytes: - """Get the next block of bytes.""" - if self._closed: - raise StopAsyncIteration() - - # remember the initial stream position - start_pos = self._stream.tell() - # move stream position to the end - self._stream.read() - # read from reader until there is no more data or we have `self._block_size` unread bytes. - while self._buffer_left < self._block_size: - read = await self._js_reader.read() - if read.done: - self._closed = True - break - self._buffer_left += self._stream.write(bytes(read.value)) - - # move the stream position back to where we started - self._stream.seek(start_pos) - self._buffer_left -= self._block_size - return self._stream.read(self._block_size) - -class PyodideTransport(AsyncioRequestsTransport): - """**This object is experimental**, meaning it may be changed in a future release - or might break with a future Pyodide release. This transport was built with Pyodide - version 0.20.0. - - Implements a basic HTTP sender using the Pyodide Javascript Fetch API. - """ - - async def send(self, request: HttpRequest, **kwargs) -> PyodideTransportResponse: - """Send request object according to configuration. - - :param request: The request object to be sent. - :type request: ~azure.core.pipeline.transport.HttpRequest - :return: An HTTPResponse object. - :rtype: PyodideResponseTransport - """ - stream_response = kwargs.pop("stream_response", False) - endpoint = request.url - request_headers = dict(request.headers) - init = { - "method": request.method, - "headers": request_headers, - "body": request.data, - "files": request.files, - "verify": kwargs.pop("connection_verify", self.connection_config.verify), - "cert": kwargs.pop("connection_cert", self.connection_config.cert), - "allow_redirects": False, - **kwargs, - } - - try: - response = await pyfetch(endpoint, **init) - except JsException as error: - raise HttpResponseError(error, error=error) - - headers = CaseInsensitiveDict(response.js_response.headers) - transport_response = PyodideTransportResponse( - request=request, - internal_response=response, - block_size=self.connection_config.data_block_size, - status_code=response.status, - reason=response.status_text, - content_type=headers.get("content-type"), - headers=headers, - stream_download_generator=PyodideStreamDownloadGenerator, - ) - if not stream_response: - await transport_response.load_body() - - return transport_response From 47657355b3a8671c901d0f709728a66e7acbbc72 Mon Sep 17 00:00:00 2001 From: Steven Jin Date: Thu, 4 Aug 2022 09:37:09 -0700 Subject: [PATCH 29/32] duplicate integration tests --- .../test_pyodide_integration/async_testing.py | 27 ----- .../test_pyodide_integration/browser_test.py | 105 ------------------ .../data/hello-world.gz | Bin 44 -> 0 bytes .../test_pyodide_integration/example-env | 4 - .../test_pyodide_integration/index.html | 89 --------------- .../test_pyodide_integration/readme.md | 43 ------- .../test_pyodide_integration/requirements.txt | 5 - 7 files changed, 273 deletions(-) delete mode 100644 sdk/core/azure-core/samples/test_pyodide_integration/async_testing.py delete mode 100644 sdk/core/azure-core/samples/test_pyodide_integration/browser_test.py delete mode 100644 sdk/core/azure-core/samples/test_pyodide_integration/data/hello-world.gz delete mode 100644 sdk/core/azure-core/samples/test_pyodide_integration/example-env delete mode 100644 sdk/core/azure-core/samples/test_pyodide_integration/index.html delete mode 100644 sdk/core/azure-core/samples/test_pyodide_integration/readme.md delete mode 100644 sdk/core/azure-core/samples/test_pyodide_integration/requirements.txt diff --git a/sdk/core/azure-core/samples/test_pyodide_integration/async_testing.py b/sdk/core/azure-core/samples/test_pyodide_integration/async_testing.py deleted file mode 100644 index 1e3eff2f9901..000000000000 --- a/sdk/core/azure-core/samples/test_pyodide_integration/async_testing.py +++ /dev/null @@ -1,27 +0,0 @@ -import traceback -import sys - - -class AsyncTestSuite: - """Async test cases - Test must be asynchronous and follow the pattern `test*`. - """ - - async def run(self): - """Run the tests an print the results.""" - print("".join(("-" * 8, type(self).__name__, "-" * 8))) - for method_name in dir(self): - if not method_name.startswith("test"): - continue - print(method_name, end="... ") - try: - await getattr(self, method_name)() - except AssertionError: - print("FAIL") - traceback.print_exception(*sys.exc_info()) - except Exception: # pylint: disable=broad-except - print("ERROR") - traceback.print_exception(*sys.exc_info()) - else: - print("PASS") - print() diff --git a/sdk/core/azure-core/samples/test_pyodide_integration/browser_test.py b/sdk/core/azure-core/samples/test_pyodide_integration/browser_test.py deleted file mode 100644 index 5e10689bd6cd..000000000000 --- a/sdk/core/azure-core/samples/test_pyodide_integration/browser_test.py +++ /dev/null @@ -1,105 +0,0 @@ -from unittest.mock import _patch_dict, patch -from uuid import uuid4 - -from azure.ai.textanalytics.aio import TextAnalyticsClient -from azure.core.credentials import AzureKeyCredential -from azure.core.pipeline.transport import HttpRequest -from azure.core.pipeline.transport.pyodide import PyodideTransport -from azure.storage.blob.aio import BlobClient, BlobServiceClient - -# pylint: disable=import-error -from async_testing import AsyncTestSuite - - -class PyodideTransportIntegrationTestSuite(AsyncTestSuite): - """Integration tests for the Pyodide transport.""" - - text_analytics_client: TextAnalyticsClient - blob_service_client: BlobServiceClient - - def __init__( - self, - text_analytics_key: str, - text_analytics_endpoint: str, - blob_service_key: str, - blob_service_endpoint: str, - ): - self.text_analytics_client = TextAnalyticsClient( - endpoint=text_analytics_endpoint, - credential=AzureKeyCredential(text_analytics_key), - transport=PyodideTransport(), - ) - self.blob_service_client = BlobServiceClient( - blob_service_endpoint, blob_service_key, transport=PyodideTransport() - ) - - async def test_decompress_generator(self): - """Test that we can decompress streams properly.""" - url = "data/hello-world.gz" - request = HttpRequest(method="GET", url=url) - transport = PyodideTransport() - response = await transport.send(request, stream_response=True) - response.headers["enc"] = "deflate" - data = b"".join([x async for x in response.iter_bytes()]) - assert data == b"hello world\n" - - response = await transport.send(request, stream_response=True) - data = b"".join([x async for x in response.iter_raw()]) - assert data != b"hello world\n" - - response = await transport.send(request, stream_response=True) - data = b"".join([x async for x in response.iter_bytes()]) - assert data != b"hello world\n" - - response = await transport.send(request, stream_response=True) - response.headers["enc"] = "deflate" - data = b"".join([x async for x in response.iter_bytes()]) - assert data == b"hello world\n" - - - async def test_sentiment_analysis(self): - """Test that sentiment analysis works.""" - results = await self.text_analytics_client.analyze_sentiment( - ["good great amazing"] - ) - assert len(results) == 1 - result = results[0] - assert result.sentiment == "positive" - assert result.confidence_scores.positive > 0.98 - assert result.confidence_scores.neutral < 0.02 - assert result.confidence_scores.negative < 0.02 - - async def test_storage(self): - """Test that we can upload and download from blob storage""" - account_name = uuid4().hex - container_client = await self.blob_service_client.create_container(account_name) - blob_name = uuid4().hex - blob_data = b"012345" - try: - assert await container_client.exists() - blob_client = container_client.get_blob_client(blob_name) - await blob_client.upload_blob(blob_data) - - # make a new client so we don't have cached data - blob_client = BlobClient( - account_url=self.blob_service_client.url, - container_name=container_client.container_name, - blob_name=blob_name, - credential=container_client.credential, - max_single_get_size=1, - max_chunk_get_size=1, - transport=PyodideTransport(), - ) - assert await blob_client.exists() - downloader = await blob_client.download_blob() - i = 0 - async for chunk in downloader.chunks(): - assert chunk == bytes(str(i), "utf-8") - i += 1 - assert i == len(blob_data) - - except Exception: - await container_client.delete_container() - raise - else: - await container_client.delete_container() diff --git a/sdk/core/azure-core/samples/test_pyodide_integration/data/hello-world.gz b/sdk/core/azure-core/samples/test_pyodide_integration/data/hello-world.gz deleted file mode 100644 index 34336ab5dd953ee01a608522d648b83797414fc9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 44 zcmb2|=HR&IdLxN}IU_YECttTbzbGe#;k3uulPAM9&g-A@e8|M0Yt6Bqhk=0s0CRN@ Ai~s-t diff --git a/sdk/core/azure-core/samples/test_pyodide_integration/example-env b/sdk/core/azure-core/samples/test_pyodide_integration/example-env deleted file mode 100644 index 57f925f76d42..000000000000 --- a/sdk/core/azure-core/samples/test_pyodide_integration/example-env +++ /dev/null @@ -1,4 +0,0 @@ -TEXT_ANALYTICS_KEY= -TEXT_ANALYTICS_ENDPOINT= -BLOB_SERVICE_KEY= -BLOB_SERVICE_ENDPOINT=https://.blob.core.windows.net diff --git a/sdk/core/azure-core/samples/test_pyodide_integration/index.html b/sdk/core/azure-core/samples/test_pyodide_integration/index.html deleted file mode 100644 index 34a921b498d8..000000000000 --- a/sdk/core/azure-core/samples/test_pyodide_integration/index.html +++ /dev/null @@ -1,89 +0,0 @@ - - - - - - - - - - diff --git a/sdk/core/azure-core/samples/test_pyodide_integration/readme.md b/sdk/core/azure-core/samples/test_pyodide_integration/readme.md deleted file mode 100644 index bb6f848ae6a0..000000000000 --- a/sdk/core/azure-core/samples/test_pyodide_integration/readme.md +++ /dev/null @@ -1,43 +0,0 @@ -# Integration Testing - -## Running - -Once you have set up your Azure Resources and your `.env` file, navigate to the `/sdk/core/azure-core`. Make a wheel of core by running - -```bash -python setup.py bdist-wheel -``` - -and run - -```python -python -h http.server -``` - -to set up the test server. Then, from a Chromium-based browser such as Edge, go to [`http://localhost:8000/samples/test_pyodide_integration`](http://localhost:8000/samples/test_pyodide_integration) and the tests will be run in the browser. Dev tip: keep your browser's devtools open. If you make a change to the source code, remember to rebuild your wheel. Note that you might have to update `requirements.txt` depending on the version of `azure-core`. Just make sure the url to `azure-core` in `requirements.txt` matches that of the wheel in `../../dist/`. - -## Adding tests - -Add tests in `browser_test.py`. I couldn't get `pytest` or `unittest` to cooperate with me, so I made my own little async testing framework (`async_test.py`). If you are creating new files to test or new packages, update the `TEST_FILES` and `PACKAGES` variables in `index.html`, import the test case, and run it. - -## Sensitive values - -To run the tests, you need a `.env` folder in this directory with your sensitive values. -see `example-env`. You can then access the values as environment variables using `os.getenv`. You will -need to have your `textanalytics` key and endpoint as well as your Blob Storage key and url. - -## Dependencies - -All all packages listed in `requirements.txt` will be available in the testing environment. - -## Azure Resources - -You need your own Text Analytics and Blob Storage accounts to run these tests. Blob storage requirest some additional configuration to work. To set up Blob Storage, navigate to your storage client homepage and go to the `Resource Sharing (CORS)` tab. Create a rule with the following values - -| Allowed origins | Allowed methods | Allowed headers | Exposed headers | Max age | -|-----------------|-----------------|-----------------|-----------------|---------| -| `*` | All | `*` | See below | `3600` | - -For exposed headers, put - -> Server,Content-Range,ETag,Last-Modified,Accept-Ranges,x-ms-*,enc diff --git a/sdk/core/azure-core/samples/test_pyodide_integration/requirements.txt b/sdk/core/azure-core/samples/test_pyodide_integration/requirements.txt deleted file mode 100644 index 97135659a4df..000000000000 --- a/sdk/core/azure-core/samples/test_pyodide_integration/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -python-dotenv -http://localhost:8000/dist/azure_core-1.25.0-py3-none-any.whl -azure-ai-textanalytics -azure-storage-blob -azure-ai-formrecognizer From c59b06af795417be6ff0d88d058e8f64daa3327e Mon Sep 17 00:00:00 2001 From: Steven Jin Date: Thu, 4 Aug 2022 09:37:26 -0700 Subject: [PATCH 30/32] use new transport __init__ --- .../azure/core/pipeline/transport/__init__.py | 2 ++ .../tests/test_pyodide_transport.py | 24 ++++++++++++++++--- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/__init__.py b/sdk/core/azure-core/azure/core/pipeline/transport/__init__.py index 87afe33482af..9c7cfb4f7809 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/__init__.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/__init__.py @@ -43,6 +43,8 @@ 'TrioRequestsTransportResponse', 'AioHttpTransport', 'AioHttpTransportResponse', + 'PyodideTransport', + 'PyodideTransportResponse', ] # pylint: disable=unused-import, redefined-outer-name, no-member, too-many-statements, too-many-branches diff --git a/sdk/core/azure-core/tests/test_pyodide_transport.py b/sdk/core/azure-core/tests/test_pyodide_transport.py index 246510b7766d..3424be4cdbf3 100644 --- a/sdk/core/azure-core/tests/test_pyodide_transport.py +++ b/sdk/core/azure-core/tests/test_pyodide_transport.py @@ -64,9 +64,9 @@ def transport(self, mock_pyodide_module, mock_js_module): ("js", mock_js_module), ) with mock.patch.dict(sys.modules, patch_dict): - import azure.core.pipeline.transport.pyodide + import azure.core.pipeline.transport._pyodide - yield azure.core.pipeline.transport.pyodide + yield azure.core.pipeline.transport._pyodide @pytest.fixture() def pipeline(self, transport): @@ -201,4 +201,22 @@ async def test_download_generator_compress(self, transport, mock_js_module): response.headers = {"enc": "deflate"} transport.PyodideStreamDownloadGenerator(pipeline=None, response=response, decompress=True) mock_js_module.DecompressionStream.new.assert_called_once_with("gzip") - \ No newline at end of file + + def test_valid_import(self, transport): + """Test that we can import Pyodide classes from `azure.core.pipeline.transport` + Adding the transport fixture will mock the Pyodide modules in `sys.modules`. + """ + # Use patch so we don't clutter up the `sys.modules` namespace. + import azure.core.pipeline.transport as transport + assert transport.PyodideTransport + assert transport.PyodideTransportResponse + + def test_invalid_import(self): + """Test that correct errors are thrown when importing Pyodide class in the wrong + context. + """ + import azure.core.pipeline.transport as transport + with pytest.raises(ImportError): + transport.PyodideTransport + with pytest.raises(ImportError): + transport.PyodideTransportResponse From 6ae9f251d58cc54e0feb1d3443d173e44eadece8 Mon Sep 17 00:00:00 2001 From: Steven Jin Date: Thu, 4 Aug 2022 09:46:13 -0700 Subject: [PATCH 31/32] integration tests disappeared --- .../pyodide_integration/async_testing.py | 27 +++++ .../samples/pyodide_integration/browser.py | 105 ++++++++++++++++++ .../pyodide_integration/data/hello-world.gz | Bin 0 -> 44 bytes .../samples/pyodide_integration/example-env | 4 + .../samples/pyodide_integration/index.html | 89 +++++++++++++++ .../samples/pyodide_integration/readme.md | 43 +++++++ .../pyodide_integration/requirements.txt | 5 + 7 files changed, 273 insertions(+) create mode 100644 sdk/core/azure-core/samples/pyodide_integration/async_testing.py create mode 100644 sdk/core/azure-core/samples/pyodide_integration/browser.py create mode 100644 sdk/core/azure-core/samples/pyodide_integration/data/hello-world.gz create mode 100644 sdk/core/azure-core/samples/pyodide_integration/example-env create mode 100644 sdk/core/azure-core/samples/pyodide_integration/index.html create mode 100644 sdk/core/azure-core/samples/pyodide_integration/readme.md create mode 100644 sdk/core/azure-core/samples/pyodide_integration/requirements.txt diff --git a/sdk/core/azure-core/samples/pyodide_integration/async_testing.py b/sdk/core/azure-core/samples/pyodide_integration/async_testing.py new file mode 100644 index 000000000000..1e3eff2f9901 --- /dev/null +++ b/sdk/core/azure-core/samples/pyodide_integration/async_testing.py @@ -0,0 +1,27 @@ +import traceback +import sys + + +class AsyncTestSuite: + """Async test cases + Test must be asynchronous and follow the pattern `test*`. + """ + + async def run(self): + """Run the tests an print the results.""" + print("".join(("-" * 8, type(self).__name__, "-" * 8))) + for method_name in dir(self): + if not method_name.startswith("test"): + continue + print(method_name, end="... ") + try: + await getattr(self, method_name)() + except AssertionError: + print("FAIL") + traceback.print_exception(*sys.exc_info()) + except Exception: # pylint: disable=broad-except + print("ERROR") + traceback.print_exception(*sys.exc_info()) + else: + print("PASS") + print() diff --git a/sdk/core/azure-core/samples/pyodide_integration/browser.py b/sdk/core/azure-core/samples/pyodide_integration/browser.py new file mode 100644 index 000000000000..5e10689bd6cd --- /dev/null +++ b/sdk/core/azure-core/samples/pyodide_integration/browser.py @@ -0,0 +1,105 @@ +from unittest.mock import _patch_dict, patch +from uuid import uuid4 + +from azure.ai.textanalytics.aio import TextAnalyticsClient +from azure.core.credentials import AzureKeyCredential +from azure.core.pipeline.transport import HttpRequest +from azure.core.pipeline.transport.pyodide import PyodideTransport +from azure.storage.blob.aio import BlobClient, BlobServiceClient + +# pylint: disable=import-error +from async_testing import AsyncTestSuite + + +class PyodideTransportIntegrationTestSuite(AsyncTestSuite): + """Integration tests for the Pyodide transport.""" + + text_analytics_client: TextAnalyticsClient + blob_service_client: BlobServiceClient + + def __init__( + self, + text_analytics_key: str, + text_analytics_endpoint: str, + blob_service_key: str, + blob_service_endpoint: str, + ): + self.text_analytics_client = TextAnalyticsClient( + endpoint=text_analytics_endpoint, + credential=AzureKeyCredential(text_analytics_key), + transport=PyodideTransport(), + ) + self.blob_service_client = BlobServiceClient( + blob_service_endpoint, blob_service_key, transport=PyodideTransport() + ) + + async def test_decompress_generator(self): + """Test that we can decompress streams properly.""" + url = "data/hello-world.gz" + request = HttpRequest(method="GET", url=url) + transport = PyodideTransport() + response = await transport.send(request, stream_response=True) + response.headers["enc"] = "deflate" + data = b"".join([x async for x in response.iter_bytes()]) + assert data == b"hello world\n" + + response = await transport.send(request, stream_response=True) + data = b"".join([x async for x in response.iter_raw()]) + assert data != b"hello world\n" + + response = await transport.send(request, stream_response=True) + data = b"".join([x async for x in response.iter_bytes()]) + assert data != b"hello world\n" + + response = await transport.send(request, stream_response=True) + response.headers["enc"] = "deflate" + data = b"".join([x async for x in response.iter_bytes()]) + assert data == b"hello world\n" + + + async def test_sentiment_analysis(self): + """Test that sentiment analysis works.""" + results = await self.text_analytics_client.analyze_sentiment( + ["good great amazing"] + ) + assert len(results) == 1 + result = results[0] + assert result.sentiment == "positive" + assert result.confidence_scores.positive > 0.98 + assert result.confidence_scores.neutral < 0.02 + assert result.confidence_scores.negative < 0.02 + + async def test_storage(self): + """Test that we can upload and download from blob storage""" + account_name = uuid4().hex + container_client = await self.blob_service_client.create_container(account_name) + blob_name = uuid4().hex + blob_data = b"012345" + try: + assert await container_client.exists() + blob_client = container_client.get_blob_client(blob_name) + await blob_client.upload_blob(blob_data) + + # make a new client so we don't have cached data + blob_client = BlobClient( + account_url=self.blob_service_client.url, + container_name=container_client.container_name, + blob_name=blob_name, + credential=container_client.credential, + max_single_get_size=1, + max_chunk_get_size=1, + transport=PyodideTransport(), + ) + assert await blob_client.exists() + downloader = await blob_client.download_blob() + i = 0 + async for chunk in downloader.chunks(): + assert chunk == bytes(str(i), "utf-8") + i += 1 + assert i == len(blob_data) + + except Exception: + await container_client.delete_container() + raise + else: + await container_client.delete_container() diff --git a/sdk/core/azure-core/samples/pyodide_integration/data/hello-world.gz b/sdk/core/azure-core/samples/pyodide_integration/data/hello-world.gz new file mode 100644 index 0000000000000000000000000000000000000000..34336ab5dd953ee01a608522d648b83797414fc9 GIT binary patch literal 44 zcmb2|=HR&IdLxN}IU_YECttTbzbGe#;k3uulPAM9&g-A@e8|M0Yt6Bqhk=0s0CRN@ Ai~s-t literal 0 HcmV?d00001 diff --git a/sdk/core/azure-core/samples/pyodide_integration/example-env b/sdk/core/azure-core/samples/pyodide_integration/example-env new file mode 100644 index 000000000000..57f925f76d42 --- /dev/null +++ b/sdk/core/azure-core/samples/pyodide_integration/example-env @@ -0,0 +1,4 @@ +TEXT_ANALYTICS_KEY= +TEXT_ANALYTICS_ENDPOINT= +BLOB_SERVICE_KEY= +BLOB_SERVICE_ENDPOINT=https://.blob.core.windows.net diff --git a/sdk/core/azure-core/samples/pyodide_integration/index.html b/sdk/core/azure-core/samples/pyodide_integration/index.html new file mode 100644 index 000000000000..9391e05df16a --- /dev/null +++ b/sdk/core/azure-core/samples/pyodide_integration/index.html @@ -0,0 +1,89 @@ + + + + + + + + + + diff --git a/sdk/core/azure-core/samples/pyodide_integration/readme.md b/sdk/core/azure-core/samples/pyodide_integration/readme.md new file mode 100644 index 000000000000..ea4eeff70885 --- /dev/null +++ b/sdk/core/azure-core/samples/pyodide_integration/readme.md @@ -0,0 +1,43 @@ +# Integration Testing + +## Running + +Once you have set up your Azure Resources and your `.env` file, navigate to the `/sdk/core/azure-core`. Make a wheel of core by running + +```bash +python setup.py bdist-wheel +``` + +and run + +```python +python -h http.server +``` + +to set up the test server. Then, from a Chromium-based browser such as Edge, go to [`http://localhost:8000/samples/pyodide_integration`](http://localhost:8000/samples/test_pyodide_integration) and the tests will be run in the browser. Dev tip: keep your browser's devtools open. If you make a change to the source code, remember to rebuild your wheel. Note that you might have to update `requirements.txt` depending on the version of `azure-core`. Just make sure the url to `azure-core` in `requirements.txt` matches that of the wheel in `../../dist/`. + +## Adding tests + +Add tests in `browser_test.py`. I couldn't get `pytest` or `unittest` to cooperate with me, so I made my own little async testing framework (`async_test.py`). If you are creating new files to test or new packages, update the `TEST_FILES` and `PACKAGES` variables in `index.html`, import the test case, and run it. + +## Sensitive values + +To run the tests, you need a `.env` folder in this directory with your sensitive values. +see `example-env`. You can then access the values as environment variables using `os.getenv`. You will +need to have your `textanalytics` key and endpoint as well as your Blob Storage key and url. + +## Dependencies + +All all packages listed in `requirements.txt` will be available in the testing environment. + +## Azure Resources + +You need your own Text Analytics and Blob Storage accounts to run these tests. Blob storage requirest some additional configuration to work. To set up Blob Storage, navigate to your storage client homepage and go to the `Resource Sharing (CORS)` tab. Create a rule with the following values + +| Allowed origins | Allowed methods | Allowed headers | Exposed headers | Max age | +|-----------------|-----------------|-----------------|-----------------|---------| +| `*` | All | `*` | See below | `3600` | + +For exposed headers, put + +> Server,Content-Range,ETag,Last-Modified,Accept-Ranges,x-ms-*,enc diff --git a/sdk/core/azure-core/samples/pyodide_integration/requirements.txt b/sdk/core/azure-core/samples/pyodide_integration/requirements.txt new file mode 100644 index 000000000000..97135659a4df --- /dev/null +++ b/sdk/core/azure-core/samples/pyodide_integration/requirements.txt @@ -0,0 +1,5 @@ +python-dotenv +http://localhost:8000/dist/azure_core-1.25.0-py3-none-any.whl +azure-ai-textanalytics +azure-storage-blob +azure-ai-formrecognizer From fdd87c1706b51b0d04ccffdc308d9018388bb3a6 Mon Sep 17 00:00:00 2001 From: Steven Jin Date: Thu, 4 Aug 2022 09:53:14 -0700 Subject: [PATCH 32/32] update integration tests --- sdk/core/azure-core/samples/pyodide_integration/browser.py | 3 +-- sdk/core/azure-core/samples/pyodide_integration/readme.md | 2 +- sdk/core/azure-core/tests/test_pyodide_transport.py | 4 ++-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/sdk/core/azure-core/samples/pyodide_integration/browser.py b/sdk/core/azure-core/samples/pyodide_integration/browser.py index 5e10689bd6cd..80578d5c5571 100644 --- a/sdk/core/azure-core/samples/pyodide_integration/browser.py +++ b/sdk/core/azure-core/samples/pyodide_integration/browser.py @@ -3,8 +3,7 @@ from azure.ai.textanalytics.aio import TextAnalyticsClient from azure.core.credentials import AzureKeyCredential -from azure.core.pipeline.transport import HttpRequest -from azure.core.pipeline.transport.pyodide import PyodideTransport +from azure.core.pipeline.transport import HttpRequest, PyodideTransport from azure.storage.blob.aio import BlobClient, BlobServiceClient # pylint: disable=import-error diff --git a/sdk/core/azure-core/samples/pyodide_integration/readme.md b/sdk/core/azure-core/samples/pyodide_integration/readme.md index ea4eeff70885..939fecdfc05a 100644 --- a/sdk/core/azure-core/samples/pyodide_integration/readme.md +++ b/sdk/core/azure-core/samples/pyodide_integration/readme.md @@ -5,7 +5,7 @@ Once you have set up your Azure Resources and your `.env` file, navigate to the `/sdk/core/azure-core`. Make a wheel of core by running ```bash -python setup.py bdist-wheel +python setup.py bdist_wheel ``` and run diff --git a/sdk/core/azure-core/tests/test_pyodide_transport.py b/sdk/core/azure-core/tests/test_pyodide_transport.py index 3424be4cdbf3..e7abb2800979 100644 --- a/sdk/core/azure-core/tests/test_pyodide_transport.py +++ b/sdk/core/azure-core/tests/test_pyodide_transport.py @@ -64,9 +64,9 @@ def transport(self, mock_pyodide_module, mock_js_module): ("js", mock_js_module), ) with mock.patch.dict(sys.modules, patch_dict): - import azure.core.pipeline.transport._pyodide + import azure.core.pipeline.transport - yield azure.core.pipeline.transport._pyodide + yield azure.core.pipeline.transport @pytest.fixture() def pipeline(self, transport):