diff --git a/.vscode/cspell.json b/.vscode/cspell.json index e802842e0496..e583204b0953 100644 --- a/.vscode/cspell.json +++ b/.vscode/cspell.json @@ -233,6 +233,7 @@ "myfile", "nazsdk", "nbytes", + "nbsp", "noarch", "NOPERM", "northcentralus", @@ -262,8 +263,10 @@ "prevsnapshot", "pschema", "PSECRET", + "pyfetch", "pyfuncitem", "pygobject", + "pyodide", "parameterizing", "pytyped", "pytz", diff --git a/eng/.docsettings.yml b/eng/.docsettings.yml index f4e9d81bdd17..3e0092e7ad2b 100644 --- a/eng/.docsettings.yml +++ b/eng/.docsettings.yml @@ -149,6 +149,7 @@ known_content_issues: - ['sdk/textanalytics/azure-ai-textanalytics/swagger/README.md', '#4554'] - ['sdk/appconfiguration/azure-appconfiguration/swagger/README.md', '#4554'] - ['sdk/core/azure-core/tests/testserver_tests/coretestserver/README.md', '#4554'] + - ['sdk/core/azure-core-experimental/README.md', '#4554'] package_indexing_exclusion_list: - 'azure-sdk-tools' - 'azure-template' diff --git a/eng/ignore-links.txt b/eng/ignore-links.txt index 5e57f7127f69..12081cc9ec6e 100644 --- a/eng/ignore-links.txt +++ b/eng/ignore-links.txt @@ -6,3 +6,4 @@ https://docs.microsoft.com/samples/azure-samples/azure-samples-python-management https://pypi.org/project/azure-messaging-webpubsubservice/ https://github.com/Azure/azure-rest-api-specs-pr https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/communication/azure-communication-rooms/samples +http://localhost:8000/samples/pyodide_integration diff --git a/pylintrc b/pylintrc index 7799d771f6fb..1a1e7abcf55a 100644 --- a/pylintrc +++ b/pylintrc @@ -54,4 +54,7 @@ method-name-hint=lower_case_with_underscores function-name-hint=lower_case_with_underscores argument-name-hint=lower_case_with_underscores variable-name-hint=lower_case_with_underscores -inlinevar-name-hint=lower_case_with_underscores (short is OK) \ No newline at end of file +inlinevar-name-hint=lower_case_with_underscores (short is OK) + +[TYPECHECK] +generated-members=js.* \ No newline at end of file diff --git a/sdk/core/azure-core-experimental/CHANGELOG.md b/sdk/core/azure-core-experimental/CHANGELOG.md new file mode 100644 index 000000000000..31ec168df3b6 --- /dev/null +++ b/sdk/core/azure-core-experimental/CHANGELOG.md @@ -0,0 +1,9 @@ +# Release History + +## 1.0.0b1 (2022-10-06) + +Initial release + +### Features Added + +- pyodide implementation of azure-core transport protocol diff --git a/sdk/core/azure-core-experimental/LICENSE b/sdk/core/azure-core-experimental/LICENSE new file mode 100644 index 000000000000..63447fd8bbbf --- /dev/null +++ b/sdk/core/azure-core-experimental/LICENSE @@ -0,0 +1,21 @@ +Copyright (c) Microsoft Corporation. + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/sdk/core/azure-core-experimental/MANIFEST.in b/sdk/core/azure-core-experimental/MANIFEST.in new file mode 100644 index 000000000000..efb83171670c --- /dev/null +++ b/sdk/core/azure-core-experimental/MANIFEST.in @@ -0,0 +1,9 @@ +recursive-include tests *.py +include *.md +include LICENSE +include azure/__init__.py +include azure/core/__init__.py +include azure/core/experimental/__init__.py +include azure/core/experimental/transport/*.py +recursive-include samples *.py +include azure/core/experimental/py.typed diff --git a/sdk/core/azure-core-experimental/README.md b/sdk/core/azure-core-experimental/README.md new file mode 100644 index 000000000000..defd2b834f17 --- /dev/null +++ b/sdk/core/azure-core-experimental/README.md @@ -0,0 +1,24 @@ + +# Azure Core Experimental client library for Python + +Azure Core Experimental client library provides experimental features that are not ready for stable release. The main reason might be because the feature depends on some other packages that there is no stable version available yet. If its dependant package introduces breaking changes, the features maybe break. It is not recommended to use azure core experimental library in production environment. + +## Getting started + +Install the azure core experimental package with [pip](https://pypi.org/project/pip/): + +```bash +pip install azure-core-experimental +``` + +## Features list + +- pyodide transport + +## Contributing + +This project welcomes contributions and suggestions. Most contributions require you to agree to a Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us the rights to use your contribution. For details, visit https://cla.microsoft.com. + +When you submit a pull request, a CLA-bot will automatically determine whether you need to provide a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions provided by the bot. You will only need to do this once across all repos using our CLA. + +This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. diff --git a/sdk/core/azure-core-experimental/azure/__init__.py b/sdk/core/azure-core-experimental/azure/__init__.py new file mode 100644 index 000000000000..0d1f7edf5dc6 --- /dev/null +++ b/sdk/core/azure-core-experimental/azure/__init__.py @@ -0,0 +1 @@ +__path__ = __import__('pkgutil').extend_path(__path__, __name__) # type: ignore diff --git a/sdk/core/azure-core-experimental/azure/core/__init__.py b/sdk/core/azure-core-experimental/azure/core/__init__.py new file mode 100644 index 000000000000..0d1f7edf5dc6 --- /dev/null +++ b/sdk/core/azure-core-experimental/azure/core/__init__.py @@ -0,0 +1 @@ +__path__ = __import__('pkgutil').extend_path(__path__, __name__) # type: ignore diff --git a/sdk/core/azure-core-experimental/azure/core/experimental/__init__.py b/sdk/core/azure-core-experimental/azure/core/experimental/__init__.py new file mode 100644 index 000000000000..a5db9f961fc2 --- /dev/null +++ b/sdk/core/azure-core-experimental/azure/core/experimental/__init__.py @@ -0,0 +1,28 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- + +from ._version import VERSION +__version__ = VERSION diff --git a/sdk/core/azure-core-experimental/azure/core/experimental/_version.py b/sdk/core/azure-core-experimental/azure/core/experimental/_version.py new file mode 100644 index 000000000000..ac9f392f513e --- /dev/null +++ b/sdk/core/azure-core-experimental/azure/core/experimental/_version.py @@ -0,0 +1,6 @@ +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ + +VERSION = "1.0.0b1" diff --git a/sdk/core/azure-core-experimental/azure/core/experimental/py.typed b/sdk/core/azure-core-experimental/azure/core/experimental/py.typed new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/sdk/core/azure-core-experimental/azure/core/experimental/transport/__init__.py b/sdk/core/azure-core-experimental/azure/core/experimental/transport/__init__.py new file mode 100644 index 000000000000..b7d3501810c4 --- /dev/null +++ b/sdk/core/azure-core-experimental/azure/core/experimental/transport/__init__.py @@ -0,0 +1,45 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- +import sys + +# pylint: disable=undefined-all-variable + +if sys.version_info >= (3, 7): + __all__ = [ + 'PyodideTransport', + ] + + def __dir__(): + return __all__ + + def __getattr__(name): + if name == 'PyodideTransport': + try: + from ._pyodide import PyodideTransport + return PyodideTransport + except ImportError: + raise ImportError("pyodide package is not installed") + raise AttributeError(f"module 'azure.core.experimental.transport' has no attribute {name}") diff --git a/sdk/core/azure-core-experimental/azure/core/experimental/transport/_pyodide.py b/sdk/core/azure-core-experimental/azure/core/experimental/transport/_pyodide.py new file mode 100644 index 000000000000..cb1220a29d63 --- /dev/null +++ b/sdk/core/azure-core-experimental/azure/core/experimental/transport/_pyodide.py @@ -0,0 +1,158 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- + +from collections.abc import AsyncIterator +from io import BytesIO + +import js # pylint: disable=import-error +from pyodide import JsException # pylint: disable=import-error +from pyodide.http import pyfetch # pylint: disable=import-error + +from azure.core.exceptions import HttpResponseError +from azure.core.pipeline import Pipeline +from azure.core.utils import CaseInsensitiveDict + +from azure.core.rest._http_response_impl_async import AsyncHttpResponseImpl +from azure.core.pipeline.transport import HttpRequest, AsyncioRequestsTransport + + +class PyodideTransportResponse(AsyncHttpResponseImpl): + """Async response object for the `PyodideTransport`.""" + + + def _js_stream(self): + """So we get a fresh stream every time.""" + return self._internal_response.clone().js_response.body + + async def close(self) -> None: + """We don't actually have control over closing connections in the browser, so we just pretend + to close. + """ + self._is_closed = True + + def body(self) -> bytes: + """The body is just the content.""" + return self.content + + async def load_body(self) -> None: + """Backcompat""" + if self._content is None: + self._content = await self._internal_response.clone().bytes() + +class PyodideStreamDownloadGenerator(AsyncIterator): + """Simple stream download generator that returns the contents of + a request. + """ + + # pylint: disable=unused-argument + def __init__(self, pipeline: Pipeline, response: PyodideTransportResponse, *_, **kwargs): + self._block_size = response._block_size + self.response = response + # use this to efficiently store bytes. + if kwargs.pop("decompress", False) and self.response.headers.get("enc", None) in ("gzip", "deflate"): + self._js_reader = response._js_stream().pipeThrough(js.DecompressionStream.new("gzip")).getReader() + else: + self._js_reader = response._js_stream().getReader() + self._stream = BytesIO() + + self._closed = False + # We cannot control how many bytes we get from `response.reader`. `self._buffer_left` + # indicates how many unread bytes there are in `self.stream` + self._buffer_left = 0 + self.done = False + + async def __anext__(self) -> bytes: + """Get the next block of bytes.""" + if self._closed: + raise StopAsyncIteration() + + # remember the initial stream position + start_pos = self._stream.tell() + # move stream position to the end + self._stream.read() + # read from reader until there is no more data or we have `self._block_size` unread bytes. + while self._buffer_left < self._block_size: + read = await self._js_reader.read() + if read.done: + self._closed = True + break + self._buffer_left += self._stream.write(bytes(read.value)) + + # move the stream position back to where we started + self._stream.seek(start_pos) + self._buffer_left -= self._block_size + return self._stream.read(self._block_size) + +class PyodideTransport(AsyncioRequestsTransport): + """**This object is experimental**, meaning it may be changed in a future release + or might break with a future Pyodide release. This transport was built with Pyodide + version 0.20.0. + + Implements a basic HTTP sender using the Pyodide Javascript Fetch API. + """ + + async def send(self, request: HttpRequest, **kwargs) -> PyodideTransportResponse: + """Send request object according to configuration. + + :param request: The request object to be sent. + :type request: ~azure.core.pipeline.transport.HttpRequest + :return: An HTTPResponse object. + :rtype: PyodideResponseTransport + """ + stream_response = kwargs.pop("stream_response", False) + endpoint = request.url + request_headers = dict(request.headers) + init = { + "method": request.method, + "headers": request_headers, + "body": request.data, + "files": request.files, + "verify": kwargs.pop("connection_verify", self.connection_config.verify), + "cert": kwargs.pop("connection_cert", self.connection_config.cert), + "allow_redirects": False, + **kwargs, + } + + try: + response = await pyfetch(endpoint, **init) + except JsException as error: + raise HttpResponseError(error, error=error) + + headers = CaseInsensitiveDict(response.js_response.headers) + transport_response = PyodideTransportResponse( + request=request, + internal_response=response, + block_size=self.connection_config.data_block_size, + status_code=response.status, + reason=response.status_text, + content_type=headers.get("content-type"), + headers=headers, + stream_download_generator=PyodideStreamDownloadGenerator, + ) + if not stream_response: + await transport_response.load_body() + + return transport_response diff --git a/sdk/core/azure-core-experimental/dev_requirements.txt b/sdk/core/azure-core-experimental/dev_requirements.txt new file mode 100644 index 000000000000..019715d963e1 --- /dev/null +++ b/sdk/core/azure-core-experimental/dev_requirements.txt @@ -0,0 +1,5 @@ +-e ../../../tools/azure-sdk-tools +../azure-core +-e ../../../tools/azure-devtools +js +pyodide==0.19.0a1; python_version >= '3.8' diff --git a/sdk/core/azure-core-experimental/samples/pyodide_integration/async_testing.py b/sdk/core/azure-core-experimental/samples/pyodide_integration/async_testing.py new file mode 100644 index 000000000000..1e3eff2f9901 --- /dev/null +++ b/sdk/core/azure-core-experimental/samples/pyodide_integration/async_testing.py @@ -0,0 +1,27 @@ +import traceback +import sys + + +class AsyncTestSuite: + """Async test cases + Test must be asynchronous and follow the pattern `test*`. + """ + + async def run(self): + """Run the tests an print the results.""" + print("".join(("-" * 8, type(self).__name__, "-" * 8))) + for method_name in dir(self): + if not method_name.startswith("test"): + continue + print(method_name, end="... ") + try: + await getattr(self, method_name)() + except AssertionError: + print("FAIL") + traceback.print_exception(*sys.exc_info()) + except Exception: # pylint: disable=broad-except + print("ERROR") + traceback.print_exception(*sys.exc_info()) + else: + print("PASS") + print() diff --git a/sdk/core/azure-core-experimental/samples/pyodide_integration/browser.py b/sdk/core/azure-core-experimental/samples/pyodide_integration/browser.py new file mode 100644 index 000000000000..d91762fbf0d8 --- /dev/null +++ b/sdk/core/azure-core-experimental/samples/pyodide_integration/browser.py @@ -0,0 +1,130 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +# -------------------------------------------------------------------------- + +from uuid import uuid4 + +from azure.ai.textanalytics.aio import TextAnalyticsClient +from azure.core.credentials import AzureKeyCredential +from azure.core.pipeline.transport import HttpRequest +from azure.core.experimental.transport import PyodideTransport +from azure.storage.blob.aio import BlobClient, BlobServiceClient + +# pylint: disable=import-error +from async_testing import AsyncTestSuite + + +class PyodideTransportIntegrationTestSuite(AsyncTestSuite): + """Integration tests for the Pyodide transport.""" + + text_analytics_client: TextAnalyticsClient + blob_service_client: BlobServiceClient + + def __init__( + self, + text_analytics_key: str, + text_analytics_endpoint: str, + blob_service_key: str, + blob_service_endpoint: str, + ): + self.text_analytics_client = TextAnalyticsClient( + endpoint=text_analytics_endpoint, + credential=AzureKeyCredential(text_analytics_key), + transport=PyodideTransport(), + ) + self.blob_service_client = BlobServiceClient( + blob_service_endpoint, blob_service_key, transport=PyodideTransport() + ) + + async def test_decompress_generator(self): + """Test that we can decompress streams properly.""" + url = "data/hello-world.gz" + request = HttpRequest(method="GET", url=url) + transport = PyodideTransport() + response = await transport.send(request, stream_response=True) + response.headers["enc"] = "deflate" + data = b"".join([x async for x in response.iter_bytes()]) + assert data == b"hello world\n" + + response = await transport.send(request, stream_response=True) + data = b"".join([x async for x in response.iter_raw()]) + assert data != b"hello world\n" + + response = await transport.send(request, stream_response=True) + data = b"".join([x async for x in response.iter_bytes()]) + assert data != b"hello world\n" + + response = await transport.send(request, stream_response=True) + response.headers["enc"] = "deflate" + data = b"".join([x async for x in response.iter_bytes()]) + assert data == b"hello world\n" + + + async def test_sentiment_analysis(self): + """Test that sentiment analysis works.""" + results = await self.text_analytics_client.analyze_sentiment( + ["good great amazing"] + ) + assert len(results) == 1 + result = results[0] + assert result.sentiment == "positive" + assert result.confidence_scores.positive > 0.98 + assert result.confidence_scores.neutral < 0.02 + assert result.confidence_scores.negative < 0.02 + + async def test_storage(self): + """Test that we can upload and download from blob storage""" + account_name = uuid4().hex + container_client = await self.blob_service_client.create_container(account_name) + blob_name = uuid4().hex + blob_data = b"012345" + try: + assert await container_client.exists() + blob_client = container_client.get_blob_client(blob_name) + await blob_client.upload_blob(blob_data) + + # make a new client so we don't have cached data + blob_client = BlobClient( + account_url=self.blob_service_client.url, + container_name=container_client.container_name, + blob_name=blob_name, + credential=container_client.credential, + max_single_get_size=1, + max_chunk_get_size=1, + transport=PyodideTransport(), + ) + assert await blob_client.exists() + downloader = await blob_client.download_blob() + i = 0 + async for chunk in downloader.chunks(): + assert chunk == bytes(str(i), "utf-8") + i += 1 + assert i == len(blob_data) + + except Exception: + await container_client.delete_container() + raise + else: + await container_client.delete_container() diff --git a/sdk/core/azure-core-experimental/samples/pyodide_integration/data/hello-world.gz b/sdk/core/azure-core-experimental/samples/pyodide_integration/data/hello-world.gz new file mode 100644 index 000000000000..34336ab5dd95 Binary files /dev/null and b/sdk/core/azure-core-experimental/samples/pyodide_integration/data/hello-world.gz differ diff --git a/sdk/core/azure-core-experimental/samples/pyodide_integration/example.env b/sdk/core/azure-core-experimental/samples/pyodide_integration/example.env new file mode 100644 index 000000000000..57f925f76d42 --- /dev/null +++ b/sdk/core/azure-core-experimental/samples/pyodide_integration/example.env @@ -0,0 +1,4 @@ +TEXT_ANALYTICS_KEY= +TEXT_ANALYTICS_ENDPOINT= +BLOB_SERVICE_KEY= +BLOB_SERVICE_ENDPOINT=https://.blob.core.windows.net diff --git a/sdk/core/azure-core-experimental/samples/pyodide_integration/index.html b/sdk/core/azure-core-experimental/samples/pyodide_integration/index.html new file mode 100644 index 000000000000..9391e05df16a --- /dev/null +++ b/sdk/core/azure-core-experimental/samples/pyodide_integration/index.html @@ -0,0 +1,89 @@ + + + + + + + + + + diff --git a/sdk/core/azure-core-experimental/samples/pyodide_integration/readme.md b/sdk/core/azure-core-experimental/samples/pyodide_integration/readme.md new file mode 100644 index 000000000000..314f29666fb4 --- /dev/null +++ b/sdk/core/azure-core-experimental/samples/pyodide_integration/readme.md @@ -0,0 +1,43 @@ +# Integration Testing + +## Running + +Once you have set up your Azure Resources and your `.env` file, navigate to the `/sdk/core/azure-core`. Make a wheel of core by running + +```bash +python setup.py bdist_wheel +``` + +and run + +```python +python -h http.server +``` + +to set up the test server. Then, from a Chromium-based browser such as Edge, go to [`http://localhost:8000/samples/pyodide_integration`](http://localhost:8000/samples/pyodide_integration) and the tests will be run in the browser. Dev tip: keep your browser's devtools open. If you make a change to the source code, remember to rebuild your wheel. Note that you might have to update `requirements.txt` depending on the version of `azure-core`. Just make sure the url to `azure-core` in `requirements.txt` matches that of the wheel in `../../dist/`. + +## Adding tests + +Add tests in `browser_test.py`. I couldn't get `pytest` or `unittest` to cooperate with me, so I made my own little async testing framework (`async_test.py`). If you are creating new files to test or new packages, update the `TEST_FILES` and `PACKAGES` variables in `index.html`, import the test case, and run it. + +## Sensitive values + +To run the tests, you need a `.env` folder in this directory with your sensitive values. +see `example-env`. You can then access the values as environment variables using `os.getenv`. You will +need to have your `textanalytics` key and endpoint as well as your Blob Storage key and url. + +## Dependencies + +All all packages listed in `requirements.txt` will be available in the testing environment. + +## Azure Resources + +You need your own Text Analytics and Blob Storage accounts to run these tests. Blob storage requires some additional configuration to work. To set up Blob Storage, navigate to your storage client homepage and go to the `Resource Sharing (CORS)` tab. Create a rule with the following values + +| Allowed origins | Allowed methods | Allowed headers | Exposed headers | Max age | +|-----------------|-----------------|-----------------|-----------------|---------| +| `*` | All | `*` | See below | `3600` | + +For exposed headers, put + +> Server,Content-Range,ETag,Last-Modified,Accept-Ranges,x-ms-*,enc diff --git a/sdk/core/azure-core-experimental/samples/pyodide_integration/requirements.txt b/sdk/core/azure-core-experimental/samples/pyodide_integration/requirements.txt new file mode 100644 index 000000000000..97135659a4df --- /dev/null +++ b/sdk/core/azure-core-experimental/samples/pyodide_integration/requirements.txt @@ -0,0 +1,5 @@ +python-dotenv +http://localhost:8000/dist/azure_core-1.25.0-py3-none-any.whl +azure-ai-textanalytics +azure-storage-blob +azure-ai-formrecognizer diff --git a/sdk/core/azure-core-experimental/sdk_packaging.toml b/sdk/core/azure-core-experimental/sdk_packaging.toml new file mode 100644 index 000000000000..e7687fdae93b --- /dev/null +++ b/sdk/core/azure-core-experimental/sdk_packaging.toml @@ -0,0 +1,2 @@ +[packaging] +auto_update = false \ No newline at end of file diff --git a/sdk/core/azure-core-experimental/setup.py b/sdk/core/azure-core-experimental/setup.py new file mode 100644 index 000000000000..2dfc82062d89 --- /dev/null +++ b/sdk/core/azure-core-experimental/setup.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python + +#------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +#-------------------------------------------------------------------------- + +import re +import os.path +from io import open +from setuptools import find_packages, setup # type: ignore + +# Change the PACKAGE_NAME only to change folder and different name +PACKAGE_NAME = "azure-core-experimental" +PACKAGE_PPRINT_NAME = "Core Experimental" + +package_folder_path = "azure/core/experimental" + +# Version extraction inspired from 'requests' +with open(os.path.join(package_folder_path, '_version.py'), 'r') as fd: + version = re.search(r'^VERSION\s*=\s*[\'"]([^\'"]*)[\'"]', # type: ignore + fd.read(), re.MULTILINE).group(1) + +if not version: + raise RuntimeError('Cannot find version information') + +with open('README.md', encoding='utf-8') as f: + readme = f.read() +with open('CHANGELOG.md', encoding='utf-8') as f: + changelog = f.read() + +setup( + name=PACKAGE_NAME, + version=version, + description='Microsoft Azure {} Library for Python'.format(PACKAGE_PPRINT_NAME), + long_description=readme + '\n\n' + changelog, + long_description_content_type='text/markdown', + license='MIT License', + author='Microsoft Corporation', + author_email='azpysdkhelp@microsoft.com', + url='https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/core/azure-core-experimental', + classifiers=[ + "Development Status :: 4 - Beta", + 'Programming Language :: Python', + 'Programming Language :: Python :: 3 :: Only', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'License :: OSI Approved :: MIT License', + ], + zip_safe=False, + packages=[ + 'azure.core.experimental', + ], + include_package_data=True, + package_data={ + 'pytyped': ['py.typed'], + }, + python_requires=">=3.7", + install_requires=[ + 'azure-core<2.0.0,>=1.25.0', + ], +) diff --git a/sdk/core/azure-core-experimental/tests/test_pyodide_transport.py b/sdk/core/azure-core-experimental/tests/test_pyodide_transport.py new file mode 100644 index 000000000000..7b84222fa908 --- /dev/null +++ b/sdk/core/azure-core-experimental/tests/test_pyodide_transport.py @@ -0,0 +1,170 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +# -------------------------------------------------------------------------- +"""Tests that mock the browser layer.""" +import asyncio +import sys +from typing import NamedTuple +from unittest import mock + +import pytest +from azure.core.exceptions import HttpResponseError +from azure.core.pipeline._base_async import AsyncPipeline +from azure.core.pipeline.policies._retry_async import AsyncRetryPolicy +from azure.core.rest import HttpRequest + +PLACEHOLDER_ENDPOINT = "https://my-resource-group.cognitiveservices.azure.com/" + + +class TestPyodideTransportClass: + """Unittest for the Pyodide transport.""" + + @pytest.fixture() + def mock_pyodide_module(self): + """Create a mock for the Pyodide module.""" + mock_pyodide_module = mock.Mock() + mock_pyodide_module.http.pyfetch = mock.Mock() + mock_pyodide_module.JsException = type("JsException", (Exception,), {}) + return mock_pyodide_module + + @pytest.fixture() + def mock_js_module(self): + """Mock the `js` module""" + return mock.Mock() + + @pytest.fixture() + def transport(self, mock_pyodide_module, mock_js_module): + """Add the mock Pyodide module to `sys.modules` and import our transport.""" + # Use patch so we don't clutter up the `sys.modules` namespace. + patch_dict = ( + ("pyodide", mock_pyodide_module), + ("pyodide.http", mock_pyodide_module.http), + ("js", mock_js_module), + ) + with mock.patch.dict(sys.modules, patch_dict): + import azure.core.experimental.transport + + yield azure.core.experimental.transport + + @pytest.fixture() + def pipeline(self, transport): + """Create a pipeline to test.""" + return AsyncPipeline(transport.PyodideTransport(), [AsyncRetryPolicy()]) + + @pytest.fixture() + def mock_pyfetch(self, mock_pyodide_module): + """Utility fixture for less typing.""" + return mock_pyodide_module.http.pyfetch + + def create_mock_response( + self, body: bytes, headers: dict, status: int, status_text: str + ) -> mock.Mock: + """Create a mock response object that mimics `pyodide.http.FetchResponse`""" + mock_response = mock.Mock() + mock_response.body = body + mock_response.js_response.headers = headers + mock_response.status = status + mock_response.status_text = status_text + bytes_promise = asyncio.Future() + bytes_promise.set_result(body) + mock_response.bytes = mock.Mock() + mock_response.bytes.return_value = bytes_promise + mock_response.clone.return_value = mock_response + + response_promise = asyncio.Future() + response_promise.set_result(mock_response) + return response_promise + + @pytest.mark.skipif(sys.version_info < (3, 8), reason="pyodide needs py 3.8+") + @pytest.mark.asyncio + async def test_successful_send(self, mock_pyfetch, mock_pyodide_module, pipeline): + """Test that a successful send returns the correct values.""" + # setup data + mock_pyfetch.reset_mock() + method = "POST" + headers = {"key": "value"} + data = b"data" + request = HttpRequest( + method=method, url=PLACEHOLDER_ENDPOINT, headers=headers, data=data + ) + response_body = b"0123" + response_headers = {"header": "value"} + response_status = 200 + response_text = "OK" + mock_response = self.create_mock_response( + body=response_body, + headers=response_headers, + status=response_status, + status_text=response_text, + ) + mock_pyodide_module.http.pyfetch.return_value = mock_response + response = (await pipeline.run(request=request)).http_response + # Check that the pipeline processed the data correctly. + await response.load_body() + assert response.body() == response_body + assert response.status_code == response_status + assert response.headers == response_headers + assert response.reason == response_text + + assert not response._is_closed + await response.close() + assert response._is_closed + + # Check that the call had the correct arguments. + mock_pyfetch.assert_called_once() + args = mock_pyfetch.call_args[0] + kwargs = mock_pyfetch.call_args[1] + assert len(args) == 1 + assert args[0] == PLACEHOLDER_ENDPOINT + assert kwargs["method"] == method + assert kwargs["body"] == data + assert not kwargs["allow_redirects"] + assert kwargs["headers"]["key"] == "value" + assert kwargs["headers"]["Content-Length"] == str(len(data)) + assert kwargs["verify"] + assert kwargs["cert"] is None + assert not kwargs["files"] + + @pytest.mark.skipif(sys.version_info < (3, 8), reason="pyodide needs py 3.8+") + @pytest.mark.asyncio + async def test_unsuccessful_send(self, mock_pyfetch, mock_pyodide_module, pipeline): + """Test that the pipeline is failing correctly.""" + mock_pyfetch.reset_mock() + mock_pyfetch.side_effect = mock_pyodide_module.JsException + retry_total = 3 + request = HttpRequest(method="GET", url=PLACEHOLDER_ENDPOINT) + with pytest.raises(HttpResponseError): + await pipeline.run(request) + # 3 retries plus the original request. + assert mock_pyfetch.call_count == retry_total + 1 + + @pytest.mark.skipif(sys.version_info < (3, 8), reason="pyodide needs py 3.8+") + def test_valid_import(self, transport): + """Test that we can import Pyodide classes from `azure.core.pipeline.transport` + Adding the transport fixture will mock the Pyodide modules in `sys.modules`. + """ + # Use patch so we don't clutter up the `sys.modules` namespace. + import azure.core.experimental.transport as transport + assert transport.PyodideTransport diff --git a/shared_requirements.txt b/shared_requirements.txt index a6e160b38afa..8505205d6cbd 100644 --- a/shared_requirements.txt +++ b/shared_requirements.txt @@ -507,3 +507,4 @@ opentelemetry-sdk<2.0.0,>=1.5.0,!=1.10a0 #override azure-mgmt-redis azure-mgmt-core>=1.3.2,<2.0.0 #override azure-mgmt-resourcegraph azure-mgmt-core>=1.3.2,<2.0.0 #override azure-mgmt-resourcegraph msrest>=0.7.1 +#override azure-core-experimental azure-core<2.0.0,>=1.25.0