diff --git a/.agents/skills/unihttp/references/backends.md b/.agents/skills/unihttp/references/backends.md index a3d5eeb..660e849 100644 --- a/.agents/skills/unihttp/references/backends.md +++ b/.agents/skills/unihttp/references/backends.md @@ -14,6 +14,7 @@ base class and the install extra change. | requests | `RequestsSyncClient` | — | `unihttp[requests]` | | niquests | `NiquestsSyncClient` | `NiquestsAsyncClient` | `unihttp[niquests]` | | zapros | `ZaprosSyncClient` | `ZaprosAsyncClient` | `unihttp[zapros]` | +| urllib | `UrllibSyncClient` | — | — (standard library) | Import the class from its module, e.g. `from unihttp.clients.aiohttp import AiohttpAsyncClient`. @@ -27,6 +28,9 @@ Import the class from its module, e.g. - **`httpx`** when you want one library that does both sync and async, or HTTP/2. - **`niquests`** as a drop-in `requests` successor with sync + async. - **`zapros`** when the project already standardizes on it. +- **`urllib`** (`UrllibSyncClient`) when you cannot add any third-party + dependency — it is powered entirely by the standard library. Prefer a real + backend for demanding workloads (connection pooling, HTTP/2, async). ## Sync vs async consequences diff --git a/README.md b/README.md index 1dbdff8..b4a0371 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ - **Declarative**: Define API methods using standard Python type hints. - **Type-Safe**: Full support for static type checking. -- **Backend Agnostic**: Works with `httpx`, `httpx2`, `aiohttp`, `requests`, `niquests` and `zapros`. +- **Backend Agnostic**: Works with `httpx`, `httpx2`, `aiohttp`, `requests`, `niquests`, `zapros` and the standard library `urllib`. - **Extensible**: Powerful middleware and error handling systems. ## Installation @@ -60,6 +60,10 @@ pip install "unihttp[aiohttp]" # For Aiohttp (Async) support pip install "unihttp[zapros]" # For Zapros (Sync/Async) support ``` +The standard library `urllib` backend (`UrllibSyncClient`) requires no extra — +it works out of the box with a plain `pip install unihttp`, which is handy for +dependency-free environments. + ## Serialization Backends `unihttp` allows you to choose your preferred serialization framework: diff --git a/src/unihttp/clients/urllib.py b/src/unihttp/clients/urllib.py new file mode 100644 index 0000000..92f3844 --- /dev/null +++ b/src/unihttp/clients/urllib.py @@ -0,0 +1,185 @@ +import json +import urllib.error +import urllib.request +import uuid +from collections.abc import Callable +from http.cookies import SimpleCookie +from typing import Any +from urllib.parse import urlencode, urljoin + +from unihttp.clients.base import BaseSyncClient +from unihttp.exceptions import NetworkError, RequestTimeoutError +from unihttp.http import UploadFile +from unihttp.http.request import HTTPRequest +from unihttp.http.response import HTTPResponse +from unihttp.middlewares.base import Middleware +from unihttp.serialize import RequestDumper, ResponseLoader + + +class UrllibSyncClient(BaseSyncClient): + """Synchronous client implementation using the standard library `urllib`. + + Requires no third-party HTTP dependency — handy for constrained or + dependency-free environments. For anything demanding, prefer a real backend + such as `requests`, `httpx` or `niquests`. + """ + + def __init__( + self, + base_url: str, + request_dumper: RequestDumper, + response_loader: ResponseLoader, + middleware: list[Middleware] | None = None, + opener: urllib.request.OpenerDirector | None = None, + timeout: float | None = None, + json_dumps: Callable[[Any], str] = json.dumps, + json_loads: Callable[[str | bytes | bytearray], Any] = json.loads, + ): + super().__init__( + base_url=base_url, + request_dumper=request_dumper, + response_loader=response_loader, + middleware=middleware, + json_dumps=json_dumps, + json_loads=json_loads, + ) + + self._opener = opener if opener is not None else urllib.request.build_opener() + self._timeout = timeout + + def _normalize_file(self, key: str, item: Any) -> tuple[str, str | None, bytes, str]: + """Normalize a single file entry to (field, filename, content, type).""" + filename: str | None = None + content_type = "application/octet-stream" + + if isinstance(item, UploadFile): + filename, content, content_type = item.to_tuple() + elif isinstance(item, tuple): + if len(item) == 2: + filename, content = item + else: + filename, content, content_type = item + else: + content = item + + if hasattr(content, "read"): + content = content.read() + if isinstance(content, str): + content = content.encode() + + return key, filename, content, content_type + + def _encode_multipart(self, form: Any, files: dict[str, Any]) -> tuple[bytes, str]: + """Encode form fields and files into a multipart/form-data body.""" + boundary = uuid.uuid4().hex + boundary_bytes = boundary.encode() + lines: list[bytes] = [] + + if form: + for key, value in form.items(): + lines.extend(( + b"--" + boundary_bytes, + f'Content-Disposition: form-data; name="{key}"'.encode(), + b"", + str(value).encode(), + )) + + for key, value in files.items(): + items = value if isinstance(value, list) else [value] + for item in items: + field, filename, content, content_type = self._normalize_file(key, item) + disposition = f'Content-Disposition: form-data; name="{field}"' + if filename: + disposition += f'; filename="{filename}"' + lines.extend(( + b"--" + boundary_bytes, + disposition.encode(), + f"Content-Type: {content_type}".encode(), + b"", + content, + )) + + lines.extend((b"--" + boundary_bytes + b"--", b"")) + + body = b"\r\n".join(lines) + return body, f"multipart/form-data; boundary={boundary}" + + def _extract_cookies(self, headers: Any) -> dict[str, str]: + """Parse Set-Cookie response headers into a simple mapping.""" + cookies: dict[str, str] = {} + for raw in headers.get_all("Set-Cookie") or []: + parsed: SimpleCookie = SimpleCookie() + parsed.load(raw) + for name, morsel in parsed.items(): + cookies[name] = morsel.value + return cookies + + def _build_url(self, request: HTTPRequest) -> str: + url = urljoin(self.base_url, request.url) + if request.query: + query_string = urlencode(request.query, doseq=True) + separator = "&" if "?" in url else "?" + url = f"{url}{separator}{query_string}" + return url + + def _prepare_body( + self, request: HTTPRequest, headers: dict[str, str] + ) -> bytes | None: + """Encode the request payload and set the matching Content-Type header.""" + if request.body and (request.form or request.file): + raise ValueError( + "Cannot use Body with Form or File. " + "Use Form for fields in multipart requests." + ) + + if request.file: + body, content_type = self._encode_multipart(request.form, request.file) + headers.setdefault("Content-Type", content_type) + return body + if request.form: + headers.setdefault("Content-Type", "application/x-www-form-urlencoded") + return urlencode(request.form, doseq=True).encode() + if request.body: + headers.setdefault("Content-Type", "application/json") + return self.json_dumps(request.body).encode() + return None + + def make_request(self, request: HTTPRequest) -> HTTPResponse: + headers = dict(request.header) + body = self._prepare_body(request, headers) + + req = urllib.request.Request( # noqa: S310 # base_url is developer-controlled + url=self._build_url(request), + data=body, + headers=headers, + method=request.method, + ) + + try: + raw: Any = self._opener.open(req, timeout=self._timeout) + except urllib.error.HTTPError as e: + # HTTPError is itself a valid response object for non-2xx statuses. + raw = e + except urllib.error.URLError as e: + if isinstance(e.reason, TimeoutError): + raise RequestTimeoutError(str(e)) from e + raise NetworkError(str(e)) from e + except TimeoutError as e: + raise RequestTimeoutError(str(e)) from e + + content = raw.read() + + response_data: Any = None + if content: + try: + response_data = self.json_loads(content) + except (ValueError, TypeError): + response_data = content + + return HTTPResponse( + status_code=raw.getcode(), + headers=dict(raw.headers.items()), + cookies=self._extract_cookies(raw.headers), + data=response_data, + raw_response=raw, + ) diff --git a/tests/test_clients/test_urllib.py b/tests/test_clients/test_urllib.py new file mode 100644 index 0000000..30546b0 --- /dev/null +++ b/tests/test_clients/test_urllib.py @@ -0,0 +1,283 @@ +import io +import socket +import urllib.error +import urllib.request +from email.message import Message +from unittest.mock import MagicMock + +import pytest +from unihttp.clients.urllib import UrllibSyncClient +from unihttp.exceptions import NetworkError, RequestTimeoutError +from unihttp.http import UploadFile +from unihttp.http.request import HTTPRequest + + +class FakeHeaders(Message): + """A minimal email.message.Message-based headers container.""" + + def __init__(self, headers=None, cookies=None): + super().__init__() + for key, value in (headers or {}).items(): + self[key] = value + for cookie in cookies or []: + self["Set-Cookie"] = cookie + + +class FakeResponse: + def __init__(self, status=200, headers=None, body=b"", cookies=None): + self._status = status + self.headers = FakeHeaders(headers, cookies) + self._body = body + + def getcode(self): + return self._status + + def read(self): + return self._body + + +@pytest.fixture +def mock_opener(): + return MagicMock(spec=urllib.request.OpenerDirector) + + +def make_client(dumper, loader, opener): + return UrllibSyncClient( + base_url="http://base", + request_dumper=dumper, + response_loader=loader, + opener=opener, + ) + + +def test_urllib_make_request(mock_request_dumper, mock_response_loader, mock_opener): + client = make_client(mock_request_dumper, mock_response_loader, mock_opener) + + mock_opener.open.return_value = FakeResponse( + status=200, + headers={"Content-Type": "application/json"}, + body=b'{"key": "value"}', + cookies=["session=abc; Path=/"], + ) + + request = HTTPRequest( + url="/test", + method="POST", + header={"Auth": "123"}, + path={}, + query={"q": "1"}, + body={"data": "abc"}, + file={}, + form={}, + ) + + response = client.make_request(request) + + sent_req = mock_opener.open.call_args[0][0] + assert sent_req.get_full_url() == "http://base/test?q=1" + assert sent_req.get_method() == "POST" + assert sent_req.data == b'{"data": "abc"}' + assert sent_req.get_header("Content-type") == "application/json" + assert sent_req.get_header("Auth") == "123" + + assert response.status_code == 200 + assert response.data == {"key": "value"} + assert response.headers["Content-Type"] == "application/json" + assert response.cookies == {"session": "abc"} + + +def test_urllib_context_manager(mock_request_dumper, mock_response_loader, mock_opener): + with make_client(mock_request_dumper, mock_response_loader, mock_opener) as client: + assert isinstance(client, UrllibSyncClient) + + +def test_urllib_network_error(mock_request_dumper, mock_response_loader, mock_opener): + client = make_client(mock_request_dumper, mock_response_loader, mock_opener) + mock_opener.open.side_effect = urllib.error.URLError("Connection refused") + + request = HTTPRequest("/url", "GET", {}, {}, {}, {}, {}, {}) + + with pytest.raises(NetworkError): + client.make_request(request) + + +def test_urllib_timeout_error_from_urlerror( + mock_request_dumper, mock_response_loader, mock_opener +): + client = make_client(mock_request_dumper, mock_response_loader, mock_opener) + mock_opener.open.side_effect = urllib.error.URLError(socket.timeout("timed out")) + + request = HTTPRequest("/url", "GET", {}, {}, {}, {}, {}, {}) + + with pytest.raises(RequestTimeoutError): + client.make_request(request) + + +def test_urllib_timeout_error_direct( + mock_request_dumper, mock_response_loader, mock_opener +): + client = make_client(mock_request_dumper, mock_response_loader, mock_opener) + mock_opener.open.side_effect = TimeoutError("timed out") + + request = HTTPRequest("/url", "GET", {}, {}, {}, {}, {}, {}) + + with pytest.raises(RequestTimeoutError): + client.make_request(request) + + +def test_urllib_http_error_is_response( + mock_request_dumper, mock_response_loader, mock_opener +): + client = make_client(mock_request_dumper, mock_response_loader, mock_opener) + + http_error = urllib.error.HTTPError( + url="http://base/test", + code=404, + msg="Not Found", + hdrs=FakeHeaders({"Content-Type": "application/json"}), + fp=io.BytesIO(b'{"error": "not found"}'), + ) + mock_opener.open.side_effect = http_error + + request = HTTPRequest("/test", "GET", {}, {}, {}, {}, {}, {}) + + response = client.make_request(request) + + assert response.status_code == 404 + assert response.data == {"error": "not found"} + assert not response.ok + assert response.is_client_error + + +def test_urllib_body_and_form_error( + mock_request_dumper, mock_response_loader, mock_opener +): + client = make_client(mock_request_dumper, mock_response_loader, mock_opener) + + request = HTTPRequest( + url="/test", + method="POST", + header={}, + path={}, + query={}, + body={"some": "body"}, + file=None, + form={"some": "form"}, + ) + + with pytest.raises(ValueError, match="Cannot use Body with Form or File"): + client.make_request(request) + + +def test_urllib_form_only(mock_request_dumper, mock_response_loader, mock_opener): + client = make_client(mock_request_dumper, mock_response_loader, mock_opener) + mock_opener.open.return_value = FakeResponse(status=200, body=b"{}") + + request = HTTPRequest( + url="/form", + method="POST", + header={}, + path={}, + query={}, + body=None, + file=None, + form={"key": "val"}, + ) + + client.make_request(request) + + sent_req = mock_opener.open.call_args[0][0] + assert sent_req.data == b"key=val" + assert sent_req.get_header("Content-type") == "application/x-www-form-urlencoded" + + +def test_urllib_multipart_with_file( + mock_request_dumper, mock_response_loader, mock_opener +): + client = make_client(mock_request_dumper, mock_response_loader, mock_opener) + mock_opener.open.return_value = FakeResponse(status=200, body=b"{}") + + request = HTTPRequest( + url="/upload", + method="POST", + header={}, + path={}, + query={}, + body=None, + file={"doc": UploadFile(b"file-content", filename="a.txt")}, + form={"caption": "hello"}, + ) + + client.make_request(request) + + sent_req = mock_opener.open.call_args[0][0] + content_type = sent_req.get_header("Content-type") + assert content_type.startswith("multipart/form-data; boundary=") + assert b'name="caption"' in sent_req.data + assert b"hello" in sent_req.data + assert b'name="doc"; filename="a.txt"' in sent_req.data + assert b"file-content" in sent_req.data + + +def test_urllib_non_json_body_falls_back_to_bytes( + mock_request_dumper, mock_response_loader, mock_opener +): + client = make_client(mock_request_dumper, mock_response_loader, mock_opener) + mock_opener.open.return_value = FakeResponse(status=200, body=b"plain text") + + request = HTTPRequest("/text", "GET", {}, {}, {}, None, {}, None) + + response = client.make_request(request) + + assert response.data == b"plain text" + + +def test_urllib_file_formats(mock_request_dumper, mock_response_loader, mock_opener): + client = make_client(mock_request_dumper, mock_response_loader, mock_opener) + mock_opener.open.return_value = FakeResponse(status=200, body=b"{}") + + request = HTTPRequest( + url="/upload", + method="POST", + header={}, + path={}, + query={}, + body=None, + file={ + "raw": b"bytes-content", + "stream": io.BytesIO(b"stream-content"), + "pair": ("pair.txt", b"pair-content"), + "triple": ("triple.bin", b"triple-content", "application/x-thing"), + "many": [b"one", ("two.txt", b"two")], + "text": ("note.txt", "string-content"), + }, + form=None, + ) + + client.make_request(request) + + data = mock_opener.open.call_args[0][0].data + assert b'name="raw"' in data + assert b"bytes-content" in data + assert b"stream-content" in data + assert b'filename="pair.txt"' in data + assert b'filename="triple.bin"' in data + assert b"application/x-thing" in data + assert b"one" in data + assert b'filename="two.txt"' in data + assert b"string-content" in data + + +def test_urllib_no_body_get(mock_request_dumper, mock_response_loader, mock_opener): + client = make_client(mock_request_dumper, mock_response_loader, mock_opener) + mock_opener.open.return_value = FakeResponse(status=204, body=b"") + + request = HTTPRequest("/ping", "GET", {}, {}, {}, None, {}, None) + + response = client.make_request(request) + + sent_req = mock_opener.open.call_args[0][0] + assert sent_req.data is None + assert sent_req.get_method() == "GET" + assert response.status_code == 204 + assert response.data is None diff --git a/tests/test_integration/test_real_clients.py b/tests/test_integration/test_real_clients.py index 989ff90..cae8a43 100644 --- a/tests/test_integration/test_real_clients.py +++ b/tests/test_integration/test_real_clients.py @@ -8,6 +8,7 @@ from unihttp.clients.aiohttp import AiohttpAsyncClient from unihttp.clients.httpx import HTTPXAsyncClient from unihttp.clients.requests import RequestsSyncClient +from unihttp.clients.urllib import UrllibSyncClient from unihttp.clients.zapros import ZaprosAsyncClient from unihttp.method import BaseMethod from unihttp.serialize import RequestDumper, ResponseLoader @@ -104,3 +105,15 @@ def test_requests_real_echo(integration_server, real_dumper, real_loader): assert result["body"] == {"sync": "true"} assert result["headers"]["X-Test"] == "requests" + + +@pytest.mark.skip(reason="Sync client blocks the event loop of the async server fixture") +def test_urllib_real_echo(integration_server, real_dumper, real_loader): + base_url = str(integration_server.make_url("/")) + + with UrllibSyncClient(base_url, real_dumper, real_loader) as client: + method = EchoMethod(body={"sync": "true"}, headers={"X-Test": "urllib"}) + result = client.call_method(method) + + assert result["body"] == {"sync": "true"} + assert result["headers"]["X-Test"] == "urllib"