From c41270ea722743ddccdab2c636237f74e96b89a2 Mon Sep 17 00:00:00 2001 From: Julian Soreavis <263610811+soreavis@users.noreply.github.com> Date: Sun, 19 Jul 2026 22:12:38 +0200 Subject: [PATCH] gh-43521: Add an opt-in gzip Content-Encoding handler to urllib.request Add urllib.request.HTTPGzipHandler, an opt-in handler that requests gzip-compressed responses and transparently decodes them. The handler is not one of the default build_opener() handlers, so urlopen()'s behaviour is unchanged unless it is added explicitly. When the request has no Accept-Encoding header, the handler adds "Accept-Encoding: gzip"; if the response then carries "Content-Encoding: gzip", the body is decoded incrementally as it is read and the Content-Encoding and Content-Length headers, which describe the compressed body, are removed. A request that already carries an Accept-Encoding header is left untouched, so a caller that means to handle the compressed response itself still receives it. Only gzip is handled, and the zlib module is required. --- Doc/library/urllib.request.rst | 25 +++ Doc/whatsnew/3.16.rst | 10 ++ Lib/test/test_urllib2.py | 143 ++++++++++++++++++ Lib/urllib/request.py | 109 ++++++++++++- ...6-07-19-12-00-00.gh-issue-43521.aYO2xG.rst | 5 + 5 files changed, 291 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2026-07-19-12-00-00.gh-issue-43521.aYO2xG.rst diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst index 95e4d2627c8b23..83d50f9e368f65 100644 --- a/Doc/library/urllib.request.rst +++ b/Doc/library/urllib.request.rst @@ -512,6 +512,31 @@ The following classes are provided: Process HTTP error responses. +.. class:: HTTPGzipHandler() + + Request and transparently decode gzip-compressed responses. + + This handler is not one of the handlers installed by :func:`build_opener` + by default; pass it explicitly to opt in:: + + opener = urllib.request.build_opener(urllib.request.HTTPGzipHandler) + with opener.open("http://www.example.com/") as response: + body = response.read() + + Unless the request already has an :mailheader:`Accept-Encoding` header, + the handler adds ``Accept-Encoding: gzip``. If the response is then sent + with ``Content-Encoding: gzip``, its body is decoded as it is read and the + :mailheader:`Content-Encoding` and :mailheader:`Content-Length` headers, + which describe the compressed body, are removed. If the request already + carries an :mailheader:`Accept-Encoding` header, the response is returned + unchanged, so a caller that asked for compressed data can decode it itself. + + Only ``gzip`` is supported. The :mod:`zlib` module is required; + constructing the handler without it raises :exc:`ImportError`. + + .. versionadded:: next + + .. _request-objects: Request Objects diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index c06e6930ac78bf..e5e03fcbf1b887 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -539,6 +539,16 @@ tkinter with no unit suffix to :class:`int` or :class:`float`. (Contributed by Serhiy Storchaka in :gh:`153513`.) +urllib.request +-------------- + +* Add :class:`~urllib.request.HTTPGzipHandler`, an opt-in handler that + requests gzip-compressed responses (by adding an ``Accept-Encoding: gzip`` + header) and transparently decodes them. It is not installed by + :func:`~urllib.request.build_opener` by default; add it explicitly to opt + in. + (Contributed by Julian Soreavis in :gh:`43521`.) + xml --- diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index d2fd111f6d9de0..82527bbde07655 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -1893,6 +1893,149 @@ def test_invalid_closed(self): self.assertTrue(conn.fakesock.closed, "Connection not closed") +class FakeGzipResponse: + # Minimal stand-in for the http.client.HTTPResponse that a response + # processor sees: headers plus a readable, closable body. + def __init__(self, data, headers, code=200): + self._file = io.BytesIO(data) + self.headers = headers + self.code = code + self.msg = "OK" + self.url = "http://example.com/" + self.closed = False + + def read(self, amt=-1): + return self._file.read(amt) + + def close(self): + self.closed = True + self._file.close() + + +class HTTPGzipHandlerTests(unittest.TestCase): + + body = b"The quick brown fox jumps over the lazy dog.\n" * 200 + + def make_response(self, data, content_encoding="gzip", content_length=True): + import email.message + headers = email.message.Message() + if content_encoding is not None: + headers["Content-Encoding"] = content_encoding + if content_length: + headers["Content-Length"] = str(len(data)) + headers["Content-Type"] = "text/plain" + return FakeGzipResponse(data, headers) + + def gzip_bytes(self, data): + import gzip + return gzip.compress(data) + + def opened_request(self, **kwargs): + h = urllib.request.HTTPGzipHandler() + req = Request("http://example.com/", **kwargs) + h.http_request(req) + return h, req + + def test_not_installed_by_default(self): + # Opt-in: build_opener() does not install it, but accepts it explicitly. + o = urllib.request.build_opener() + self.assertFalse(any(isinstance(h, urllib.request.HTTPGzipHandler) + for h in o.handlers)) + o = urllib.request.build_opener(urllib.request.HTTPGzipHandler) + self.assertTrue(any(isinstance(h, urllib.request.HTTPGzipHandler) + for h in o.handlers)) + + def test_request_adds_accept_encoding(self): + h, req = self.opened_request() + self.assertEqual(req.get_header("Accept-encoding"), "gzip") + + def test_request_keeps_user_accept_encoding(self): + h, req = self.opened_request(headers={"Accept-Encoding": "identity"}) + self.assertEqual(req.get_header("Accept-encoding"), "identity") + + def test_response_is_decoded(self): + h, req = self.opened_request() + response = self.make_response(self.gzip_bytes(self.body)) + with h.http_response(req, response) as result: + self.assertEqual(result.read(), self.body) + + def test_response_headers_stripped(self): + h, req = self.opened_request() + response = self.make_response(self.gzip_bytes(self.body)) + result = h.http_response(req, response) + info = result.info() + self.assertIsNone(info.get("Content-Encoding")) + self.assertIsNone(info.get("Content-Length")) + self.assertEqual(info.get("Content-Type"), "text/plain") + self.assertEqual(result.status, 200) + result.close() + + def test_response_streamed_in_chunks(self): + h, req = self.opened_request() + response = self.make_response(self.gzip_bytes(self.body)) + result = h.http_response(req, response) + chunks = [] + while chunk := result.read(64): + self.assertLessEqual(len(chunk), 64) + chunks.append(chunk) + self.assertEqual(b"".join(chunks), self.body) + result.close() + + def test_multiple_gzip_members_decoded(self): + # A response may be several concatenated gzip members; all of them + # must be decoded, not just the first. + h, req = self.opened_request() + comp = self.gzip_bytes(b"AAA") + self.gzip_bytes(b"BBB") + response = self.make_response(comp) + with h.http_response(req, response) as result: + self.assertEqual(result.read(), b"AAABBB") + + def test_trailing_junk_after_member_ignored(self): + # Bytes after the final member that are not another gzip member are + # tolerated: the body decodes without error and is not corrupted. + h, req = self.opened_request() + comp = self.gzip_bytes(self.body) + b"trailing garbage" + response = self.make_response(comp) + with h.http_response(req, response) as result: + self.assertEqual(result.read(), self.body) + + def test_user_accept_encoding_passes_through(self): + # A caller-supplied Accept-Encoding is left to the caller to decode: + # the gzipped response is returned untouched. + h, req = self.opened_request(headers={"Accept-Encoding": "gzip"}) + response = self.make_response(self.gzip_bytes(self.body)) + result = h.http_response(req, response) + self.assertIs(result, response) + self.assertEqual(result.headers.get("Content-Encoding"), "gzip") + + def test_non_gzip_response_passes_through(self): + h, req = self.opened_request() + response = self.make_response(self.body, content_encoding=None) + result = h.http_response(req, response) + self.assertIs(result, response) + + def test_truncated_stream_raises(self): + h, req = self.opened_request() + comp = self.gzip_bytes(self.body) + response = self.make_response(comp[:len(comp) // 2]) + result = h.http_response(req, response) + with self.assertRaises(EOFError): + result.read() + result.close() + + def test_close_propagates(self): + h, req = self.opened_request() + response = self.make_response(self.gzip_bytes(self.body)) + result = h.http_response(req, response) + result.close() + self.assertTrue(response.closed) + + def test_requires_zlib(self): + with mock.patch.object(urllib.request, "zlib", None): + with self.assertRaises(ImportError): + urllib.request.HTTPGzipHandler() + + class MiscTests(unittest.TestCase): def opener_has_handler(self, opener, handler_class): diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index 660301fef61258..c10648558bc338 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -84,6 +84,7 @@ import base64 import bisect import contextlib +import copy import email import hashlib import http.client @@ -113,6 +114,11 @@ else: _have_ssl = True +try: + import zlib +except ImportError: + zlib = None + __all__ = [ # Classes 'Request', 'OpenerDirector', 'BaseHandler', 'HTTPDefaultErrorHandler', @@ -122,7 +128,7 @@ 'HTTPBasicAuthHandler', 'ProxyBasicAuthHandler', 'AbstractDigestAuthHandler', 'HTTPDigestAuthHandler', 'ProxyDigestAuthHandler', 'HTTPHandler', 'FileHandler', 'FTPHandler', 'CacheFTPHandler', 'DataHandler', - 'UnknownHandler', 'HTTPErrorProcessor', + 'UnknownHandler', 'HTTPErrorProcessor', 'HTTPGzipHandler', # Functions 'urlopen', 'install_opener', 'build_opener', 'pathname2url', 'url2pathname', 'getproxies', @@ -1392,6 +1398,107 @@ def http_response(self, request, response): https_request = http_request https_response = http_response + +class _GzipReader(io.BufferedIOBase): + """Incrementally decode a gzip Content-Encoding response body.""" + + def __init__(self, fp): + self._fp = fp + self._decompressor = zlib.decompressobj(16 + zlib.MAX_WBITS) + self._buffer = b'' + self._leftover = b'' + self._got_data = False + self._eof = False + + def readable(self): + return True + + def _refill(self): + while True: + raw = self._leftover or self._fp.read(io.DEFAULT_BUFFER_SIZE) + self._leftover = b'' + if not raw: + self._eof = True + data = self._decompressor.flush() + if self._got_data and not self._decompressor.eof: + raise EOFError('compressed response ended before the ' + 'end-of-stream marker was reached') + return data + self._got_data = True + if self._decompressor.eof: + # The previous member ended; the remaining bytes start the + # next concatenated member. Decode it with a fresh + # decompressor, and tolerate trailing bytes that are not + # another member, as the reader did before. + self._decompressor = zlib.decompressobj(16 + zlib.MAX_WBITS) + try: + data = self._decompressor.decompress(raw) + except zlib.error: + self._eof = True + return b'' + else: + data = self._decompressor.decompress(raw) + self._leftover = self._decompressor.unused_data + if data: + return data + + def read(self, size=-1): + if size is None: + size = -1 + while not self._eof and (size < 0 or len(self._buffer) < size): + self._buffer += self._refill() + if size < 0: + data, self._buffer = self._buffer, b'' + else: + data, self._buffer = self._buffer[:size], self._buffer[size:] + return data + + def close(self): + try: + fp = self._fp + if fp is not None: + self._fp = None + fp.close() + finally: + super().close() + + +class HTTPGzipHandler(BaseHandler): + """Request and transparently decode gzip Content-Encoding responses. + + This handler is not installed by build_opener() by default; add it + explicitly to opt in. It sends Accept-Encoding: gzip unless the request + already carries an Accept-Encoding header, and decodes the response only + when it added that header, so a caller supplying its own Accept-Encoding + still receives the raw response. + """ + + def __init__(self): + if zlib is None: + raise ImportError('the zlib module is required for gzip decoding') + + def http_request(self, request): + if not request.has_header('Accept-encoding'): + request.add_unredirected_header('Accept-encoding', 'gzip') + request._gzip_injected = True + return request + + def http_response(self, request, response): + if (getattr(request, '_gzip_injected', False) and + response.headers.get('Content-Encoding', '').lower() == 'gzip'): + headers = copy.copy(response.headers) + del headers['Content-Encoding'] + del headers['Content-Length'] + result = addinfourl(_GzipReader(response), headers, + response.url, response.code) + result.msg = response.msg + return result + return response + + https_request = http_request + https_response = http_response + + class UnknownHandler(BaseHandler): def unknown_open(self, req): type = req.type diff --git a/Misc/NEWS.d/next/Library/2026-07-19-12-00-00.gh-issue-43521.aYO2xG.rst b/Misc/NEWS.d/next/Library/2026-07-19-12-00-00.gh-issue-43521.aYO2xG.rst new file mode 100644 index 00000000000000..bacfaf32f1a815 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-19-12-00-00.gh-issue-43521.aYO2xG.rst @@ -0,0 +1,5 @@ +Add :class:`urllib.request.HTTPGzipHandler`, an opt-in handler that sends an +``Accept-Encoding: gzip`` request header and transparently decodes +gzip-compressed responses. It is not one of the default +:func:`~urllib.request.build_opener` handlers, so it must be added +explicitly.