diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index e007dc7e39e030..c0d53eca4ff022 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -125,6 +125,8 @@ def test_ignore(self): self.module.filterwarnings("ignore", category=UserWarning) self.module.warn("FilterTests.test_ignore", UserWarning) self.assertEqual(len(w), 0) + self.assertEqual(list(__warningregistry__.keys()), + ['version']) def test_ignore_after_default(self): with original_warnings.catch_warnings(record=True, @@ -705,6 +707,7 @@ def test_default_action(self): self.module.defaultaction = "ignore" __warningregistry__ = {} registry = {} + self.module._filters_mutated() self.module.warn_explicit(message, UserWarning, "", 44, registry=registry) self.assertEqual(len(w), 0) @@ -847,13 +850,14 @@ def test_issue31416(self): # bad warnings.filters or warnings.defaultaction. wmod = self.module with original_warnings.catch_warnings(module=wmod): - wmod.filters = [(None, None, Warning, None, 0)] with self.assertRaises(TypeError): + wmod._add_filter(None, None, Warning, None, 0) wmod.warn_explicit('foo', Warning, 'bar', 1) - wmod.filters = [] + with original_warnings.catch_warnings(module=wmod): with support.swap_attr(wmod, 'defaultaction', None), \ self.assertRaises(TypeError): + wmod.resetwarnings() wmod.warn_explicit('foo', Warning, 'bar', 1) @support.cpython_only @@ -1011,14 +1015,14 @@ def test_catch_warnings_defaults(self): with wmod.catch_warnings(module=wmod) as w: self.assertIsNone(w) self.assertIs(wmod.showwarning, orig_showwarning) - self.assertIsNot(wmod.filters, orig_filters) + self.assertIs(wmod.filters, orig_filters) self.assertIs(wmod.filters, orig_filters) if wmod is sys.modules['warnings']: # Ensure the default module is this one with wmod.catch_warnings() as w: self.assertIsNone(w) self.assertIs(wmod.showwarning, orig_showwarning) - self.assertIsNot(wmod.filters, orig_filters) + self.assertIs(wmod.filters, orig_filters) self.assertIs(wmod.filters, orig_filters) def test_record_override_showwarning_before(self): diff --git a/Lib/warnings.py b/Lib/warnings.py index b2605f84aec0f5..f743d02ff35d6a 100644 --- a/Lib/warnings.py +++ b/Lib/warnings.py @@ -137,8 +137,16 @@ def filterwarnings(action, message="", category=Warning, module="", lineno=0, assert isinstance(module, str), "module must be a string" assert isinstance(lineno, int) and lineno >= 0, \ "lineno must be an int >= 0" - _add_filter(action, re.compile(message, re.I), category, - re.compile(module), lineno, append=append) + + if message: + regex = re.compile(message, re.I) + else: + regex = None + if module: + module = re.compile(module) + else: + module = None + _add_filter(action, regex, category, module, lineno, append=append) def simplefilter(action, category=Warning, lineno=0, append=False): """Insert a simple entry into the list of warnings filters (at the front). @@ -157,22 +165,26 @@ def simplefilter(action, category=Warning, lineno=0, append=False): _add_filter(action, None, category, None, lineno, append=append) def _add_filter(*item, append): + global filters # Remove possible duplicate filters, so new one will be placed # in correct place. If append=True and duplicate exists, do nothing. + filters_list = list(filters) if not append: try: - filters.remove(item) + filters_list.remove(item) except ValueError: pass - filters.insert(0, item) + filters_list.insert(0, item) else: - if item not in filters: - filters.append(item) + if item not in filters_list: + filters_list.append(item) + filters = tuple(filters_list) _filters_mutated() def resetwarnings(): """Clear the list of warning filters, so that no filters are active.""" - filters[:] = [] + global filters + filters = () _filters_mutated() class _OptionError(Exception): @@ -353,7 +365,6 @@ def warn_explicit(message, category, filename, lineno, action = defaultaction # Early exit actions if action == "ignore": - registry[key] = 1 return # Prime the linecache for formatting, in case the @@ -455,7 +466,7 @@ def __enter__(self): raise RuntimeError("Cannot enter %r twice" % self) self._entered = True self._filters = self._module.filters - self._module.filters = self._filters[:] + self._module.filters = tuple(self._filters) self._module._filters_mutated() self._showwarning = self._module.showwarning self._showwarnmsg_impl = self._module._showwarnmsg_impl @@ -492,8 +503,9 @@ def __exit__(self, *exc_info): defaultaction = _defaultaction onceregistry = _onceregistry _warnings_defaults = True + filters = tuple(filters) except ImportError: - filters = [] + filters = () defaultaction = "default" onceregistry = {} diff --git a/Misc/NEWS.d/next/Library/2017-11-21-16-05-35.bpo-27535.JLhcNz.rst b/Misc/NEWS.d/next/Library/2017-11-21-16-05-35.bpo-27535.JLhcNz.rst new file mode 100644 index 00000000000000..51bcfb7d76953b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-21-16-05-35.bpo-27535.JLhcNz.rst @@ -0,0 +1,4 @@ +The warnings module doesn't leak memory anymore in the hidden warnings +registry for the "ignore" action of warnings filters. warn_explicit() +function doesn't add the warning key to the registry anymore for the +"ignore" action. diff --git a/Python/_warnings.c b/Python/_warnings.c index f2110edc52d196..38922d716cb0e1 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -35,7 +35,7 @@ check_matched(PyObject *obj, PyObject *arg) A NULL return value can mean false or an error. */ static PyObject * -get_warnings_attr(const char *attr, int try_import) +get_warnings_attr(_Py_Identifier *attr_id, int try_import) { static PyObject *warnings_str = NULL; PyObject *warnings_module, *obj; @@ -64,7 +64,7 @@ get_warnings_attr(const char *attr, int try_import) return NULL; } - obj = PyObject_GetAttrString(warnings_module, attr); + obj = _PyObject_GetAttrId(warnings_module, attr_id); Py_DECREF(warnings_module); if (obj == NULL && PyErr_ExceptionMatches(PyExc_AttributeError)) { PyErr_Clear(); @@ -77,8 +77,9 @@ static PyObject * get_once_registry(void) { PyObject *registry; + _Py_IDENTIFIER(onceregistry); - registry = get_warnings_attr("onceregistry", 0); + registry = get_warnings_attr(&PyId_onceregistry, 0); if (registry == NULL) { if (PyErr_Occurred()) return NULL; @@ -98,18 +99,65 @@ get_once_registry(void) } -static PyObject * +typedef struct { + char *action; + PyObject *message; + PyObject *category; + PyObject *module; + Py_ssize_t lineno; +} WarningFilter; + +typedef struct { + Py_ssize_t nfilter; + WarningFilter *filters; + long version; + char *default_action; +} WarningFilters; + +#define WarningFilters_INIT (WarningFilters){-1, NULL, -1, NULL} + +static WarningFilters cache_filters = WarningFilters_INIT; + + +static char* +unicode_as_ascii(PyObject *unicode) +{ + PyObject *bytes = PyUnicode_AsASCIIString(unicode); + if (bytes == NULL) { + return NULL; + } + + char *str = _PyMem_Strdup(PyBytes_AS_STRING(bytes)); + Py_DECREF(bytes); + if (str == NULL) { + PyErr_NoMemory(); + return NULL; + } + return str; +} + + +static const char* get_default_action(void) { PyObject *default_action; + _Py_IDENTIFIER(defaultaction); - default_action = get_warnings_attr("defaultaction", 0); + default_action = get_warnings_attr(&PyId_defaultaction, 0); if (default_action == NULL) { if (PyErr_Occurred()) { return NULL; } assert(_PyRuntime.warnings.default_action); - return _PyRuntime.warnings.default_action; + + char *action_str = unicode_as_ascii(_PyRuntime.warnings.default_action); + if (action_str == NULL) { + return NULL; + } + PyMem_Free(cache_filters.default_action); + cache_filters.default_action = action_str; + + return action_str; } if (!PyUnicode_Check(default_action)) { PyErr_Format(PyExc_TypeError, @@ -120,104 +168,201 @@ get_default_action(void) return NULL; } Py_SETREF(_PyRuntime.warnings.default_action, default_action); - return default_action; + + char *action_str = unicode_as_ascii(default_action); + if (action_str == NULL) { + return NULL; + } + PyMem_Free(cache_filters.default_action); + cache_filters.default_action = action_str; + + return action_str; +} + + +static void +warning_filters_clear(WarningFilters *filters) +{ + for (Py_ssize_t i=0; i < filters->nfilter; i++) { + WarningFilter *filter = &filters->filters[i]; + + PyMem_Free(filter->action); + Py_DECREF(filter->message); + Py_DECREF(filter->category); + Py_DECREF(filter->module); + } + PyMem_Free(filters->filters); + PyMem_Free(filters->default_action); + *filters = WarningFilters_INIT; } /* The item is a new reference. */ -static PyObject* -get_filter(PyObject *category, PyObject *text, Py_ssize_t lineno, - PyObject *module, PyObject **item) +static int +get_filter(PyObject *item, WarningFilter *filter) +{ + PyObject *action, *msg, *cat, *mod, *lineno_obj; + Py_ssize_t lineno; + + /* Python code: action, msg, cat, mod, lineno = item */ + action = PyTuple_GET_ITEM(item, 0); + msg = PyTuple_GET_ITEM(item, 1); + cat = PyTuple_GET_ITEM(item, 2); + mod = PyTuple_GET_ITEM(item, 3); + lineno_obj = PyTuple_GET_ITEM(item, 4); + + if (!PyUnicode_Check(action)) { + PyErr_Format(PyExc_TypeError, + "action must be a string, not '%.200s'", + Py_TYPE(action)->tp_name); + return -1; + } + + lineno = PyLong_AsSsize_t(lineno_obj); + if (lineno == -1 && PyErr_Occurred()) { + return -1; + } + + char *action_str = unicode_as_ascii(action); + if (action_str == NULL) { + return -1; + } + + if (strcmp(action_str, "always") != 0 + && strcmp(action_str, "ignore") == 0 + && strcmp(action_str, "once") == 0 + && strcmp(action_str, "module") == 0 + && strcmp(action_str, "default") != 0) { + PyErr_Format(PyExc_RuntimeError, + "Unrecognized action (%R) in warnings.filters:\n %R", + action, item); + PyMem_Free(action_str); + return -1; + } + + filter->action = action_str; + Py_INCREF(msg); + filter->message = msg; + Py_INCREF(cat); + filter->category = cat; + Py_INCREF(mod); + filter->module = mod; + filter->lineno = lineno; + return 0; +} + +static WarningFilters* +get_filters(void) { - PyObject *action; - Py_ssize_t i; - PyObject *warnings_filters; + if (cache_filters.version == _PyRuntime.warnings.filters_version) { + return &cache_filters; + } + - warnings_filters = get_warnings_attr("filters", 0); + /* Update _PyRuntime.warnings.filters */ + _Py_IDENTIFIER(filters); + PyObject *warnings_filters = get_warnings_attr(&PyId_filters, 0); if (warnings_filters == NULL) { - if (PyErr_Occurred()) + if (PyErr_Occurred()) { return NULL; + } } else { Py_SETREF(_PyRuntime.warnings.filters, warnings_filters); } - PyObject *filters = _PyRuntime.warnings.filters; - if (filters == NULL || !PyList_Check(filters)) { + PyObject *filters_tuple = _PyRuntime.warnings.filters; + if (filters_tuple == NULL || !PyTuple_Check(filters_tuple)) { PyErr_SetString(PyExc_ValueError, - MODULE_NAME ".filters must be a list"); + MODULE_NAME ".filters must be a tuple"); return NULL; } - /* _PyRuntime.warnings.filters could change while we are iterating over it. */ - for (i = 0; i < PyList_GET_SIZE(filters); i++) { - PyObject *tmp_item, *action, *msg, *cat, *mod, *ln_obj; - Py_ssize_t ln; - int is_subclass, good_msg, good_mod; + WarningFilters new_filters = WarningFilters_INIT; + new_filters.nfilter = 0; + new_filters.version = _PyRuntime.warnings.filters_version; - tmp_item = PyList_GET_ITEM(filters, i); - if (!PyTuple_Check(tmp_item) || PyTuple_GET_SIZE(tmp_item) != 5) { + /* _PyRuntime.warnings.filters could change while we are iterating over it. */ + Py_ssize_t nfilter = PyTuple_GET_SIZE(filters_tuple); + for (Py_ssize_t i = 0; i < nfilter; i++) { + PyObject *item = PyTuple_GET_ITEM(filters_tuple, i); + if (!PyTuple_Check(item) || PyTuple_GET_SIZE(item) != 5) { PyErr_Format(PyExc_ValueError, MODULE_NAME ".filters item %zd isn't a 5-tuple", i); return NULL; } - /* Python code: action, msg, cat, mod, ln = item */ - Py_INCREF(tmp_item); - action = PyTuple_GET_ITEM(tmp_item, 0); - msg = PyTuple_GET_ITEM(tmp_item, 1); - cat = PyTuple_GET_ITEM(tmp_item, 2); - mod = PyTuple_GET_ITEM(tmp_item, 3); - ln_obj = PyTuple_GET_ITEM(tmp_item, 4); - - if (!PyUnicode_Check(action)) { - PyErr_Format(PyExc_TypeError, - "action must be a string, not '%.200s'", - Py_TYPE(action)->tp_name); - Py_DECREF(tmp_item); - return NULL; + /* Keep a strong reference */ + Py_INCREF(item); + WarningFilter filter; + if (get_filter(item, &filter) < 0) { + Py_DECREF(item); + goto error; + } + Py_DECREF(item); + + WarningFilter *new_array; + Py_ssize_t size; + + size = (new_filters.nfilter + 1) * sizeof(new_filters.filters[0]); + new_array = PyMem_Realloc(new_filters.filters, size); + if (new_array == NULL) { + PyErr_NoMemory(); + goto error; } + new_filters.filters = new_array; + new_filters.filters[new_filters.nfilter] = filter; + new_filters.nfilter++; + } - good_msg = check_matched(msg, text); + /* FIXME: retry or raise an error in that case? */ + assert(new_filters.version == _PyRuntime.warnings.filters_version); + + warning_filters_clear(&cache_filters); + cache_filters = new_filters; + + return &cache_filters; + +error: + warning_filters_clear(&new_filters); + return NULL; +} + + +static const char* +get_action(PyObject *category, PyObject *text, Py_ssize_t lineno, + PyObject *module) +{ + WarningFilters *filters = get_filters(); + if (filters == NULL) { + return NULL; + } + + for (Py_ssize_t i = 0; i < filters->nfilter; i++) { + WarningFilter *filter = &filters->filters[i]; + int is_subclass, good_msg, good_mod; + + good_msg = check_matched(filter->message, text); if (good_msg == -1) { - Py_DECREF(tmp_item); return NULL; } - good_mod = check_matched(mod, module); + good_mod = check_matched(filter->module, module); if (good_mod == -1) { - Py_DECREF(tmp_item); return NULL; } - is_subclass = PyObject_IsSubclass(category, cat); + is_subclass = PyObject_IsSubclass(category, filter->category); if (is_subclass == -1) { - Py_DECREF(tmp_item); - return NULL; - } - - ln = PyLong_AsSsize_t(ln_obj); - if (ln == -1 && PyErr_Occurred()) { - Py_DECREF(tmp_item); return NULL; } - if (good_msg && is_subclass && good_mod && (ln == 0 || lineno == ln)) { - *item = tmp_item; - return action; + if (good_msg && is_subclass && good_mod && (filter->lineno == 0 || lineno == filter->lineno)) { + return filter->action; } - - Py_DECREF(tmp_item); } - action = get_default_action(); - if (action != NULL) { - Py_INCREF(Py_None); - *item = Py_None; - return action; - } - - return NULL; + return get_default_action(); } @@ -389,11 +534,13 @@ call_show_warning(PyObject *category, PyObject *text, PyObject *message, PyObject *sourceline, PyObject *source) { PyObject *show_fn, *msg, *res, *warnmsg_cls = NULL; + _Py_IDENTIFIER(_showwarnmsg); + _Py_IDENTIFIER(WarningMessage); /* If the source parameter is set, try to get the Python implementation. The Python implementation is able to log the traceback where the source was allocated, whereas the C implementation doesn't. */ - show_fn = get_warnings_attr("_showwarnmsg", source != NULL); + show_fn = get_warnings_attr(&PyId__showwarnmsg, source != NULL); if (show_fn == NULL) { if (PyErr_Occurred()) return -1; @@ -407,7 +554,7 @@ call_show_warning(PyObject *category, PyObject *text, PyObject *message, goto error; } - warnmsg_cls = get_warnings_attr("WarningMessage", 0); + warnmsg_cls = get_warnings_attr(&PyId_WarningMessage, 0); if (warnmsg_cls == NULL) { if (!PyErr_Occurred()) { PyErr_SetString(PyExc_RuntimeError, @@ -445,8 +592,7 @@ warn_explicit(PyObject *category, PyObject *message, PyObject *source) { PyObject *key = NULL, *text = NULL, *result = NULL, *lineno_obj = NULL; - PyObject *item = NULL; - PyObject *action; + const char *action; int rc; /* module can be None if a warning is emitted late during Python shutdown. @@ -511,25 +657,31 @@ warn_explicit(PyObject *category, PyObject *message, /* Else this warning hasn't been generated before. */ } - action = get_filter(category, text, lineno, module, &item); + action = get_action(category, text, lineno, module); if (action == NULL) goto cleanup; - if (_PyUnicode_EqualToASCIIString(action, "error")) { + if (strcmp(action, "error") == 0) { PyErr_SetObject(category, message); goto cleanup; } + if (strcmp(action, "ignore") == 0) { + goto return_none; + } + /* Store in the registry that we've been here, *except* when the action is "always". */ rc = 0; - if (!_PyUnicode_EqualToASCIIString(action, "always")) { + if (strcmp(action, "always") != 0) { if (registry != NULL && registry != Py_None && - PyDict_SetItem(registry, key, Py_True) < 0) + PyDict_SetItem(registry, key, Py_True) < 0) { goto cleanup; - else if (_PyUnicode_EqualToASCIIString(action, "ignore")) + } + + if (strcmp(action, "ignore") == 0) goto return_none; - else if (_PyUnicode_EqualToASCIIString(action, "once")) { + else if (strcmp(action, "once") == 0) { if (registry == NULL || registry == Py_None) { registry = get_once_registry(); if (registry == NULL) @@ -538,16 +690,13 @@ warn_explicit(PyObject *category, PyObject *message, /* _PyRuntime.warnings.once_registry[(text, category)] = 1 */ rc = update_registry(registry, text, category, 0); } - else if (_PyUnicode_EqualToASCIIString(action, "module")) { + else if (strcmp(action, "module") == 0) { /* registry[(text, category, 0)] = 1 */ if (registry != NULL && registry != Py_None) rc = update_registry(registry, text, category, 0); } - else if (!_PyUnicode_EqualToASCIIString(action, "default")) { - PyErr_Format(PyExc_RuntimeError, - "Unrecognized action (%R) in warnings.filters:\n %R", - action, item); - goto cleanup; + else { + /* action == "default" */ } } @@ -566,7 +715,6 @@ warn_explicit(PyObject *category, PyObject *message, Py_INCREF(result); cleanup: - Py_XDECREF(item); Py_XDECREF(key); Py_XDECREF(text); Py_XDECREF(lineno_obj); @@ -940,6 +1088,10 @@ static PyObject * warnings_filters_mutated(PyObject *self, PyObject *args) { _PyRuntime.warnings.filters_version++; + warning_filters_clear(&cache_filters); + if (get_filters() == NULL) { + return NULL; + } Py_RETURN_NONE; } @@ -1257,7 +1409,10 @@ init_filters(void) } } - return filters; + PyObject *filters_tuple = PyList_AsTuple(filters); + Py_DECREF(filters); + + return filters_tuple; } static struct PyModuleDef warningsmodule = {