Skip to content

bpo-43260: io: Prevent large data remains in textio buffer. #24592

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
methane merged 8 commits into from
Feb 21, 2021
Merged

bpo-43260: io: Prevent large data remains in textio buffer. #24592

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
3ba68b0
bpo-43260: io: Prevent large data remains in textio.
methane Feb 20, 2021
5ffbe8f
Add news entry
methane Feb 20, 2021
ccda9bc
Use chunk_size instead of heuristic 1MiB
methane Feb 20, 2021
a17a9f4
Use Eryk's idea
methane Feb 20, 2021
54ff77e
fix comment
methane Feb 20, 2021
960b928
<=
methane Feb 20, 2021
9572c1e
concatenate
methane Feb 20, 2021
05ab6e0
Add test
methane Feb 20, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions Misc/NEWS.d/next/Library/2021-02-20-12-15-29.bpo-43260.6znAas.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Fix TextIOWrapper can not flush internal buffer forever after very large
text is written.
20 changes: 17 additions & 3 deletions Modules/_io/textio.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1585,6 +1585,8 @@ _textiowrapper_writeflush(textio *self)
ret = PyObject_CallMethodOneArg(self->buffer, _PyIO_str_write, b);
} while (ret == NULL && _PyIO_trap_eintr());
Py_DECREF(b);
// NOTE: We cleared buffer but we don't know how many bytes are actually written
// when an error occurred.
if (ret == NULL)
return -1;
Py_DECREF(ret);
Expand Down Expand Up @@ -1642,7 +1644,10 @@ _io_TextIOWrapper_write_impl(textio *self, PyObject *text)

/* XXX What if we were just reading? */
if (self->encodefunc != NULL) {
if (PyUnicode_IS_ASCII(text) && is_asciicompat_encoding(self->encodefunc)) {
if (PyUnicode_IS_ASCII(text) &&
// See bpo-43260
PyUnicode_GET_LENGTH(text) < self->chunk_size &&
is_asciicompat_encoding(self->encodefunc)) {
b = text;
Py_INCREF(b);
}
Expand All @@ -1651,8 +1656,9 @@ _io_TextIOWrapper_write_impl(textio *self, PyObject *text)
}
self->encoding_start_of_stream = 0;
}
else
else {
b = PyObject_CallMethodOneArg(self->encoder, _PyIO_str_encode, text);
}

Py_DECREF(text);
if (b == NULL)
Expand All @@ -1677,6 +1683,14 @@ _io_TextIOWrapper_write_impl(textio *self, PyObject *text)
self->pending_bytes_count = 0;
self->pending_bytes = b;
}
else if (self->pending_bytes_count + bytes_len > self->chunk_size) {
// Prevent to concatinate more than chunk_size data.
if (_textiowrapper_writeflush(self) < 0) {
Py_DECREF(b);
return NULL;
}
self->pending_bytes = b;
}
else if (!PyList_CheckExact(self->pending_bytes)) {
PyObject *list = PyList_New(2);
if (list == NULL) {
Expand All @@ -1696,7 +1710,7 @@ _io_TextIOWrapper_write_impl(textio *self, PyObject *text)
}

self->pending_bytes_count += bytes_len;
if (self->pending_bytes_count > self->chunk_size || needflush ||
if (self->pending_bytes_count >= self->chunk_size || needflush ||
text_needflush) {
if (_textiowrapper_writeflush(self) < 0)
return NULL;
Expand Down