Tests/pyodide integration tests

sdk/core/azure-core/CHANGELOG.md

@@ -6,6 +6,7 @@ Azure-core is supported on Python 3.7 or later. For more details, please read ou
 
 ### Features Added
 
+- Added Pyodide-compatible transport.
 - Added `CaseInsensitiveDict` implementation in `azure.core.utils` removing dependency on `requests` and `aiohttp`
 
 ### Breaking Changes

sdk/core/azure-core/azure/core/pipeline/transport/pyodide.py

@@ -0,0 +1,160 @@
+# --------------------------------------------------------------------------
+#
+# Copyright (c) Microsoft Corporation. All rights reserved.
+#
+# The MIT License (MIT)
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the ""Software""), to
+# deal in the Software without restriction, including without limitation the
+# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+# sell copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+#
+# --------------------------------------------------------------------------
+
+from collections.abc import AsyncIterator
+from io import BytesIO
+
+import js # pylint: disable=import-error
+from pyodide import JsException  # pylint: disable=import-error
+from pyodide.http import pyfetch  # pylint: disable=import-error
+
+from azure.core.exceptions import HttpResponseError
+from azure.core.pipeline import Pipeline
+from azure.core.utils import CaseInsensitiveDict
+
+from ...rest._http_response_impl_async import AsyncHttpResponseImpl
+from . import HttpRequest
+from ._requests_asyncio import AsyncioRequestsTransport
+
+
+class PyodideTransportResponse(AsyncHttpResponseImpl):
+    """Async response object for the `PyodideTransport`."""
+
+    def __init__(self, **kwargs):
+        super(PyodideTransportResponse, self).__init__(**kwargs)
+        # clone to avoid reading from the same `FetchResponse` a second time in `load_body`.
+        self._js_stream = self.internal_response.clone().js_response.body
+        self._js_reader = None
+
+    async def close(self) -> None:
+        """We don't actually have control over closing connections in the browser, so we just pretend
+        to close.
+        """
+        self._is_closed = True
+
+    async def load_body(self) -> None:
+        """Load the body of the response."""
+        if self._content is None:
+            self._content = await self._internal_response.bytes()
+
+    def body(self) -> bytes:
+        """The body is just the content."""
+        return self.content
+
+class PyodideStreamDownloadGenerator(AsyncIterator):
+    """Simple stream download generator that returns the contents of
+    a request.
+    """
+
+    # pylint: disable=unused-argument
+    def __init__(self, pipeline: Pipeline, response: PyodideTransportResponse, *_, **kwargs):
+        self._block_size = response._block_size
+        self.response = response
+        # use this to efficiently store bytes.
+        if kwargs.pop("decompress", False):
+            self._js_reader = response._js_stream.pipeThrough(js.DecompressionStream.new("gzip")).getReader()
+        else:
+            self._js_reader = response._js_stream.getReader()
+        self._stream = BytesIO()
+
+        self._closed = False
+        # We cannot control how many bytes we get from `response.reader`. `self.buffer_left`
+        # indicates how many unread bytes there are in `self.stream`
+        self.buffer_left = 0
+        self.done = False
+
+    async def __anext__(self) -> bytes:
+        """Get the next block of bytes."""
+        if self._closed:
+            raise StopAsyncIteration()
+
+        # remember the initial stream position
+        start_pos = self._stream.tell()
+        # move stream position to the end
+        self._stream.read()
+        # read from reader until there is no more data or we have `self._block_size` unread bytes.
+        while self.buffer_left < self._block_size:
+            read = await self._js_reader.read()
+            if read.done:
+                self._closed = True
+                break
+            self.buffer_left += self._stream.write(bytes(read.value))
+
+        # move the stream position back to where we started
+        self._stream.seek(start_pos)
+        self.buffer_left -= self._block_size
+        return self._stream.read(self._block_size)
+
+class PyodideTransport(AsyncioRequestsTransport):
+    """Implements a basic HTTP sender using the Pyodide Javascript Fetch API.
+
+    WARNING: Pyodide is still an alpha technology. As such, this transport
+    is highly experimental and subject to breaking changes. This transport was
+    built around Pyodide version 0.20.0.
+    """
+
+    async def send(self, request: HttpRequest, **kwargs) -> PyodideTransportResponse:
+        """Send request object according to configuration.
+
+        :param request: The request object to be sent.
+        :type request: ~azure.core.pipeline.transport.HttpRequest
+        :return: An HTTPResponse object.
+        :rtype: PyodideResponseTransport
+        """
+        stream_response = kwargs.pop("stream_response", False)
+        endpoint = request.url
+        request_headers = dict(request.headers)
+        init = {
+            "method": request.method,
+            "headers": request_headers,
+            "body": request.data,
+            "files": request.files,
+            "verify": kwargs.pop("connection_verify", self.connection_config.verify),
+            "cert": kwargs.pop("connection_cert", self.connection_config.cert),
+            "allow_redirects": False,
+            **kwargs,
+        }
+
+        try:
+            response = await pyfetch(endpoint, **init)
+        except JsException as error:
+            raise HttpResponseError(error, error=error)
+
+        headers = CaseInsensitiveDict(response.js_response.headers)
+        transport_response = PyodideTransportResponse(
+            request=request,
+            internal_response=response,
+            block_size=self.connection_config.data_block_size,
+            status_code=response.status,
+            reason=response.status_text,
+            content_type=headers.get("content-type"),
+            headers=headers,
+            stream_download_generator=PyodideStreamDownloadGenerator,
+        )
+        if not stream_response:
+            await transport_response.load_body()
+
+        return transport_response

sdk/core/azure-core/samples/test_pyodide_integration/async_testing.py

@@ -0,0 +1,27 @@
+import traceback
+import sys
+
+
+class AsyncTestSuite:
+    """Async test cases
+    Test must be asynchronous and follow the pattern `test*`.
+    """
+
+    async def run(self):
+        """Run the tests an print the results."""
+        print("".join(("-" * 8, type(self).__name__, "-" * 8)))
+        for method_name in dir(self):
+            if not method_name.startswith("test"):
+                continue
+            print(method_name, end="... ")
+            try:
+                await getattr(self, method_name)()
+            except AssertionError:
+                print("FAIL")
+                traceback.print_exception(*sys.exc_info())
+            except Exception:  # pylint: disable=broad-except
+                print("ERROR")
+                traceback.print_exception(*sys.exc_info())
+            else:
+                print("PASS")
+            print()

sdk/core/azure-core/samples/test_pyodide_integration/browser_test.py

@@ -0,0 +1,95 @@
+from unittest.mock import _patch_dict, patch
+from uuid import uuid4
+
+from azure.ai.textanalytics.aio import TextAnalyticsClient
+from azure.core.credentials import AzureKeyCredential
+from azure.core.pipeline.transport import HttpRequest
+from azure.core.pipeline.transport.pyodide import PyodideTransport
+from azure.storage.blob.aio import BlobClient, BlobServiceClient
+
+# pylint: disable=import-error
+from async_testing import AsyncTestSuite
+
+
+class PyodideTransportIntegrationTestSuite(AsyncTestSuite):
+    """Integration tests for the Pyodide transport."""
+
+    text_analytics_client: TextAnalyticsClient
+    blob_service_client: BlobServiceClient
+
+    def __init__(
+        self,
+        text_analytics_key: str,
+        text_analytics_endpoint: str,
+        blob_service_key: str,
+        blob_service_endpoint: str,
+    ):
+        self.text_analytics_client = TextAnalyticsClient(
+            endpoint=text_analytics_endpoint,
+            credential=AzureKeyCredential(text_analytics_key),
+            transport=PyodideTransport(),
+        )
+        self.blob_service_client = BlobServiceClient(
+            blob_service_endpoint, blob_service_key, transport=PyodideTransport()
+        )
+
+    async def test_decompress_generator(self):
+        """Test that we can decompress streams properly."""
+        url = "data/hello-world.gz"
+        request = HttpRequest(method="GET", url=url)
+        transport = PyodideTransport()
+        response = await transport.send(request, stream_response=True)
+        data = b"".join([x async for x in response.iter_bytes()])
+        assert data == b"hello world\n"
+
+        response = await transport.send(request, stream_response=True)
+        data = b"".join([x async for x in response.iter_raw()])
+        assert data != b"hello world\n"
+
+
+    async def test_sentiment_analysis(self):
+        """Test that sentiment analysis works."""
+        results = await self.text_analytics_client.analyze_sentiment(
+            ["good great amazing"]
+        )
+        assert len(results) == 1
+        result = results[0]
+        assert result.sentiment == "positive"
+        assert result.confidence_scores.positive > 0.98
+        assert result.confidence_scores.neutral < 0.02
+        assert result.confidence_scores.negative < 0.02
+
+    async def test_storage(self):
+        """Test that we can upload and download from blob storage"""
+        account_name = uuid4().hex
+        container_client = await self.blob_service_client.create_container(account_name)
+        blob_name = uuid4().hex
+        blob_data = b"012345"
+        try:
+            assert await container_client.exists()
+            blob_client = container_client.get_blob_client(blob_name)
+            await blob_client.upload_blob(blob_data)
+
+            # make a new client so we don't have cached data
+            blob_client = BlobClient(
+                account_url=self.blob_service_client.url,
+                container_name=container_client.container_name,
+                blob_name=blob_name,
+                credential=container_client.credential,
+                max_single_get_size=1,
+                max_chunk_get_size=1,
+                transport=PyodideTransport(),
+            )
+            assert await blob_client.exists()
+            downloader = await blob_client.download_blob()
+            i = 0
+            async for chunk in downloader.chunks():
+                assert chunk == bytes(str(i), "utf-8")
+                i += 1
+            assert i == len(blob_data)
+
+        except Exception:
+            await container_client.delete_container()
+            raise
+        else:
+            await container_client.delete_container()

sdk/core/azure-core/samples/test_pyodide_integration/example-env

@@ -0,0 +1,4 @@
+TEXT_ANALYTICS_KEY=
+TEXT_ANALYTICS_ENDPOINT=
+BLOB_SERVICE_KEY=
+BLOB_SERVICE_ENDPOINT=https://<storage client name>.blob.core.windows.net

sdk/core/azure-core/samples/test_pyodide_integration/index.html

@@ -0,0 +1,89 @@
+<html>
+  <head>
+    <meta charset="UTF-8" />
+    <script src="https://cdn.jsdelivr.net/pyodide/v0.20.0/full/pyodide.js"></script>
+  </head>
+  <body>
+    <code id="output"></code>
+    <script>
+      async function main() {
+        output = document.getElementById("output");
+
+        pyodide = await loadPyodide({
+          stdout: (x) =>
+            (output.innerHTML += x.replace(/\s/, "&nbsp") + "<br />"),
+          stderr: (x) =>
+            (output.innerHTML += x.replace(/\s/, "&nbsp") + "<br />"),
+        });
+        await pyodide.loadPackage("micropip");
+        await pyodide.runPythonAsync(`
+          import micropip
+          from pyodide.http import pyfetch
+
+          TEST_FILES = [
+            "browser_test.py",
+            "async_testing.py",
+            ".env",
+            "requirements.txt",
+          ]
+
+          for filename in TEST_FILES:
+            res = await pyfetch(filename, cache="no-cache");
+            # get the name of the file
+            with open(filename, "wb") as f:
+              f.write(await res.bytes())
+
+          with open("requirements.txt") as f:
+            # filter comments out
+            requirements = [r for r in f.read().split("\\n") if not r.startswith("#") and r]
+            await micropip.install(requirements)
+
+          from dotenv import load_dotenv
+          from types import ModuleType
+          import sys
+          import os
+
+          load_dotenv()  # Load environment variables
+
+          TEXT_ANALYTICS_KEY = os.getenv("TEXT_ANALYTICS_KEY")
+          TEXT_ANALYTICS_ENDPOINT = os.getenv("TEXT_ANALYTICS_ENDPOINT")
+          BLOB_SERVICE_KEY = os.getenv("BLOB_SERVICE_KEY")
+          BLOB_SERVICE_ENDPOINT = os.getenv("BLOB_SERVICE_ENDPOINT")
+
+          if not TEXT_ANALYTICS_KEY:
+            print("Invalid textanalytics key")
+            raise ValueError("Invalid textanalytics key")
+
+          if not TEXT_ANALYTICS_ENDPOINT:
+            print("Invalid textanalytics endpoint")
+            raise ValueError("Invalid textanalytics endpoint")
+
+          if not BLOB_SERVICE_KEY:
+            print("Invalid blob service key")
+            raise ValueError("Invalid blob service key")
+
+          if not BLOB_SERVICE_ENDPOINT:
+            print("Invalid blob service endpoint")
+            raise ValueError("Invalid blob service endpoint")
+
+          # Would have liked to use unittest.mock.patch here, but it doesn't seem
+          # to work in Pyodide.
+
+          fake_aiohttp = ModuleType("AioHttp")
+          fake_aiohttp.ClientPayloadError = Exception
+          sys.modules["aiohttp"] =  fake_aiohttp
+          from browser_test import PyodideTransportIntegrationTestSuite
+
+          test_case = PyodideTransportIntegrationTestSuite(
+            text_analytics_endpoint=TEXT_ANALYTICS_ENDPOINT,
+            text_analytics_key=TEXT_ANALYTICS_KEY,
+            blob_service_key=BLOB_SERVICE_KEY,
+            blob_service_endpoint=BLOB_SERVICE_ENDPOINT,
+          )
+          await test_case.run()
+        `);
+      }
+      main();
+    </script>
+  </body>
+</html>