Add Experimental Pyodide Transport

sdk/core/azure-core/CHANGELOG.md

@@ -16,6 +16,7 @@ Azure-core is supported on Python 3.7 or later. For more details, please read ou
 
 ### Features Added
 
+- Added **experimental** Pyodide-compatible transport.
 - Added `CaseInsensitiveDict` implementation in `azure.core.utils` removing dependency on `requests` and `aiohttp`
 
 ## 1.24.2 (2022-06-30)

sdk/core/azure-core/azure/core/pipeline/transport/__init__.py

@@ -43,6 +43,8 @@
     'TrioRequestsTransportResponse',
     'AioHttpTransport',
     'AioHttpTransportResponse',
+    'PyodideTransport',
+    'PyodideTransportResponse',
 ]
 
 # pylint: disable=unused-import, redefined-outer-name, no-member, too-many-statements, too-many-branches
@@ -102,6 +104,18 @@ def __getattr__(name):
             transport = TrioRequestsTransportResponse
         except ImportError:
             raise ImportError("trio package is not installed")
+    if name == 'PyodideTransport':
+        try:
+            from ._pyodide import PyodideTransport
+            transport = PyodideTransport
+        except ImportError:
+            raise ImportError("pyodide package is not installed")
+    if name == 'PyodideTransportResponse':
+        try:
+            from ._pyodide import PyodideTransportResponse
+            transport = PyodideTransportResponse
+        except ImportError:
+            raise ImportError("pyodide package is not installed")
     if transport:
         return transport
     raise AttributeError(f"module 'azure.core.pipeline.transport' has no attribute {name}")

sdk/core/azure-core/samples/pyodide_integration/async_testing.py

@@ -0,0 +1,27 @@
+import traceback
+import sys
+
+
+class AsyncTestSuite:
+    """Async test cases
+    Test must be asynchronous and follow the pattern `test*`.
+    """
+
+    async def run(self):
+        """Run the tests an print the results."""
+        print("".join(("-" * 8, type(self).__name__, "-" * 8)))
+        for method_name in dir(self):
+            if not method_name.startswith("test"):
+                continue
+            print(method_name, end="... ")
+            try:
+                await getattr(self, method_name)()
+            except AssertionError:
+                print("FAIL")
+                traceback.print_exception(*sys.exc_info())
+            except Exception:  # pylint: disable=broad-except
+                print("ERROR")
+                traceback.print_exception(*sys.exc_info())
+            else:
+                print("PASS")
+            print()

sdk/core/azure-core/samples/pyodide_integration/browser.py

@@ -0,0 +1,104 @@
+from unittest.mock import _patch_dict, patch
+from uuid import uuid4
+
+from azure.ai.textanalytics.aio import TextAnalyticsClient
+from azure.core.credentials import AzureKeyCredential
+from azure.core.pipeline.transport import HttpRequest, PyodideTransport
+from azure.storage.blob.aio import BlobClient, BlobServiceClient
+
+# pylint: disable=import-error
+from async_testing import AsyncTestSuite
+
+
+class PyodideTransportIntegrationTestSuite(AsyncTestSuite):
+    """Integration tests for the Pyodide transport."""
+
+    text_analytics_client: TextAnalyticsClient
+    blob_service_client: BlobServiceClient
+
+    def __init__(
+        self,
+        text_analytics_key: str,
+        text_analytics_endpoint: str,
+        blob_service_key: str,
+        blob_service_endpoint: str,
+    ):
+        self.text_analytics_client = TextAnalyticsClient(
+            endpoint=text_analytics_endpoint,
+            credential=AzureKeyCredential(text_analytics_key),
+            transport=PyodideTransport(),
+        )
+        self.blob_service_client = BlobServiceClient(
+            blob_service_endpoint, blob_service_key, transport=PyodideTransport()
+        )
+
+    async def test_decompress_generator(self):
+        """Test that we can decompress streams properly."""
+        url = "data/hello-world.gz"
+        request = HttpRequest(method="GET", url=url)
+        transport = PyodideTransport()
+        response = await transport.send(request, stream_response=True)
+        response.headers["enc"] = "deflate"
+        data = b"".join([x async for x in response.iter_bytes()])
+        assert data == b"hello world\n"
+
+        response = await transport.send(request, stream_response=True)
+        data = b"".join([x async for x in response.iter_raw()])
+        assert data != b"hello world\n"
+
+        response = await transport.send(request, stream_response=True)
+        data = b"".join([x async for x in response.iter_bytes()])
+        assert data != b"hello world\n"
+
+        response = await transport.send(request, stream_response=True)
+        response.headers["enc"] = "deflate"
+        data = b"".join([x async for x in response.iter_bytes()])
+        assert data == b"hello world\n"
+
+
+    async def test_sentiment_analysis(self):
+        """Test that sentiment analysis works."""
+        results = await self.text_analytics_client.analyze_sentiment(
+            ["good great amazing"]
+        )
+        assert len(results) == 1
+        result = results[0]
+        assert result.sentiment == "positive"
+        assert result.confidence_scores.positive > 0.98
+        assert result.confidence_scores.neutral < 0.02
+        assert result.confidence_scores.negative < 0.02
+
+    async def test_storage(self):
+        """Test that we can upload and download from blob storage"""
+        account_name = uuid4().hex
+        container_client = await self.blob_service_client.create_container(account_name)
+        blob_name = uuid4().hex
+        blob_data = b"012345"
+        try:
+            assert await container_client.exists()
+            blob_client = container_client.get_blob_client(blob_name)
+            await blob_client.upload_blob(blob_data)
+
+            # make a new client so we don't have cached data
+            blob_client = BlobClient(
+                account_url=self.blob_service_client.url,
+                container_name=container_client.container_name,
+                blob_name=blob_name,
+                credential=container_client.credential,
+                max_single_get_size=1,
+                max_chunk_get_size=1,
+                transport=PyodideTransport(),
+            )
+            assert await blob_client.exists()
+            downloader = await blob_client.download_blob()
+            i = 0
+            async for chunk in downloader.chunks():
+                assert chunk == bytes(str(i), "utf-8")
+                i += 1
+            assert i == len(blob_data)
+
+        except Exception:
+            await container_client.delete_container()
+            raise
+        else:
+            await container_client.delete_container()

sdk/core/azure-core/samples/pyodide_integration/example-env

@@ -0,0 +1,4 @@
+TEXT_ANALYTICS_KEY=
+TEXT_ANALYTICS_ENDPOINT=
+BLOB_SERVICE_KEY=
+BLOB_SERVICE_ENDPOINT=https://<storage client name>.blob.core.windows.net

sdk/core/azure-core/samples/pyodide_integration/index.html

@@ -0,0 +1,89 @@
+<html>
+  <head>
+    <meta charset="UTF-8" />
+    <script src="https://cdn.jsdelivr.net/pyodide/v0.20.0/full/pyodide.js"></script>
+  </head>
+  <body>
+    <code id="output"></code>
+    <script>
+      async function main() {
+        output = document.getElementById("output");
+
+        pyodide = await loadPyodide({
+          stdout: (x) =>
+            (output.innerHTML += x.replace(/\s/, "&nbsp") + "<br />"),
+          stderr: (x) =>
+            (output.innerHTML += x.replace(/\s/, "&nbsp") + "<br />"),
+        });
+        await pyodide.loadPackage("micropip");
+        await pyodide.runPythonAsync(`
+          import micropip
+          from pyodide.http import pyfetch
+
+          TEST_FILES = [
+            "browser.py",
+            "async_testing.py",
+            ".env",
+            "requirements.txt",
+          ]
+
+          for filename in TEST_FILES:
+            res = await pyfetch(filename, cache="no-cache");
+            # get the name of the file
+            with open(filename, "wb") as f:
+              f.write(await res.bytes())
+
+          with open("requirements.txt") as f:
+            # filter comments out
+            requirements = [r for r in f.read().split("\\n") if not r.startswith("#") and r]
+            await micropip.install(requirements)
+
+          from dotenv import load_dotenv
+          from types import ModuleType
+          import sys
+          import os
+
+          load_dotenv()  # Load environment variables
+
+          TEXT_ANALYTICS_KEY = os.getenv("TEXT_ANALYTICS_KEY")
+          TEXT_ANALYTICS_ENDPOINT = os.getenv("TEXT_ANALYTICS_ENDPOINT")
+          BLOB_SERVICE_KEY = os.getenv("BLOB_SERVICE_KEY")
+          BLOB_SERVICE_ENDPOINT = os.getenv("BLOB_SERVICE_ENDPOINT")
+
+          if not TEXT_ANALYTICS_KEY:
+            print("Invalid textanalytics key")
+            raise ValueError("Invalid textanalytics key")
+
+          if not TEXT_ANALYTICS_ENDPOINT:
+            print("Invalid textanalytics endpoint")
+            raise ValueError("Invalid textanalytics endpoint")
+
+          if not BLOB_SERVICE_KEY:
+            print("Invalid blob service key")
+            raise ValueError("Invalid blob service key")
+
+          if not BLOB_SERVICE_ENDPOINT:
+            print("Invalid blob service endpoint")
+            raise ValueError("Invalid blob service endpoint")
+
+          # Would have liked to use unittest.mock.patch here, but it doesn't seem
+          # to work in Pyodide.
+
+          fake_aiohttp = ModuleType("AioHttp")
+          fake_aiohttp.ClientPayloadError = Exception
+          sys.modules["aiohttp"] =  fake_aiohttp
+          from browser import PyodideTransportIntegrationTestSuite
+
+          test_case = PyodideTransportIntegrationTestSuite(
+            text_analytics_endpoint=TEXT_ANALYTICS_ENDPOINT,
+            text_analytics_key=TEXT_ANALYTICS_KEY,
+            blob_service_key=BLOB_SERVICE_KEY,
+            blob_service_endpoint=BLOB_SERVICE_ENDPOINT,
+          )
+          await test_case.run()
+        `);
+      }
+      main();
+    </script>
+  </body>
+</html>

sdk/core/azure-core/samples/pyodide_integration/readme.md

@@ -0,0 +1,43 @@
+# Integration Testing
+
+## Running
+
+Once you have set up your Azure Resources and your `.env` file, navigate to the `/sdk/core/azure-core`. Make a wheel of core by running
+
+```bash
+python setup.py bdist_wheel
+```
+
+and run
+
+```python
+python -h http.server
+```
+
+to set up the test server. Then, from a Chromium-based browser such as Edge, go to [`http://localhost:8000/samples/pyodide_integration`](http://localhost:8000/samples/test_pyodide_integration) and the tests will be run in the browser. Dev tip: keep your browser's devtools open. If you make a change to the source code, remember to rebuild your wheel. Note that you might have to update `requirements.txt` depending on the version of `azure-core`. Just make sure the url to `azure-core` in `requirements.txt` matches that of the wheel in `../../dist/`.
+
+## Adding tests
+
+Add tests in `browser_test.py`. I couldn't get `pytest` or `unittest` to cooperate with me, so I made my own little async testing framework (`async_test.py`). If you are creating new files to test or new packages, update the `TEST_FILES` and `PACKAGES` variables in `index.html`, import the test case, and run it.
+
+## Sensitive values
+
+To run the tests, you need a `.env` folder in this directory with your sensitive values.
+see `example-env`. You can then access the values as environment variables using `os.getenv`. You will
+need to have your `textanalytics` key and endpoint as well as your Blob Storage key and url.
+
+## Dependencies
+
+All all packages listed in `requirements.txt` will be available in the testing environment.
+
+## Azure Resources
+
+You need your own Text Analytics and Blob Storage accounts to run these tests. Blob storage requirest some additional configuration to work. To set up Blob Storage, navigate to your storage client homepage and go to the `Resource Sharing (CORS)` tab. Create a rule with the following values
+
+| Allowed origins | Allowed methods | Allowed headers | Exposed headers | Max age |
+|-----------------|-----------------|-----------------|-----------------|---------|
+| `*`             | All             | `*`             | See below       | `3600`  |
+
+For exposed headers, put
+
+> Server,Content-Range,ETag,Last-Modified,Accept-Ranges,x-ms-*,enc

sdk/core/azure-core/samples/pyodide_integration/requirements.txt

@@ -0,0 +1,5 @@
+python-dotenv
+http://localhost:8000/dist/azure_core-1.25.0-py3-none-any.whl
+azure-ai-textanalytics
+azure-storage-blob
+azure-ai-formrecognizer