From d618f6ff3b3b52fe66f6c888da914a0d4f271ac3 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 30 Jan 2020 10:23:12 +0100 Subject: [PATCH 1/4] bpo-39500: Document PyUnicode_IsIdentifier() function PyUnicode_IsIdentifier() now raises an exception and return -1 on error, rather than calling Py_FatalError(). --- Doc/c-api/unicode.rst | 7 +++++++ Objects/typeobject.c | 6 +++++- Objects/unicodeobject.c | 23 +++++++++++++++-------- Parser/tokenizer.c | 4 ++++ 4 files changed, 31 insertions(+), 9 deletions(-) diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 77f123cf1f2c08c..6d361dcdd3800d1 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -239,6 +239,13 @@ access internal read-only data of Unicode objects: Part of the old-style Unicode API, please migrate to using the :c:func:`PyUnicode_nBYTE_DATA` family of macros. +.. c:function:: int PyUnicode_IsIdentifier(PyObject *o) + + Return ``1`` if the string is a valid identifier according to the language + definition, section :ref:`identifiers`. Return ``0`` otherwise. + + Raise an exception and return ``-1`` on error. + Unicode Character Properties """""""""""""""""""""""""""" diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 8422a3c5a38c46c..52c8c09dd5f6c35 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -2241,7 +2241,11 @@ valid_identifier(PyObject *s) Py_TYPE(s)->tp_name); return 0; } - if (!PyUnicode_IsIdentifier(s)) { + int identifier = PyUnicode_IsIdentifier(s); + if (identifier < 0) { + return 0 + } + if (!identifier) { PyErr_SetString(PyExc_TypeError, "__slots__ must be identifiers"); return 0; diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 1ec2accdb09f202..0f262ccd9548303 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -12209,13 +12209,13 @@ PyUnicode_IsIdentifier(PyObject *self) Py_UCS4 first; if (PyUnicode_READY(self) == -1) { - Py_FatalError("identifier not ready"); - return 0; + return -1; } - /* Special case for empty strings */ - if (PyUnicode_GET_LENGTH(self) == 0) + if (PyUnicode_GET_LENGTH(self) == 0) { + /* an empty string is not a valid identifier */ return 0; + } kind = PyUnicode_KIND(self); data = PyUnicode_DATA(self); @@ -12228,12 +12228,15 @@ PyUnicode_IsIdentifier(PyObject *self) to check just for these, except that _ must be allowed as starting an identifier. */ first = PyUnicode_READ(kind, data, 0); - if (!_PyUnicode_IsXidStart(first) && first != 0x5F /* LOW LINE */) + if (!_PyUnicode_IsXidStart(first) && first != 0x5F /* LOW LINE */) { return 0; + } - for (i = 1; i < PyUnicode_GET_LENGTH(self); i++) - if (!_PyUnicode_IsXidContinue(PyUnicode_READ(kind, data, i))) + for (i = 1; i < PyUnicode_GET_LENGTH(self); i++) { + if (!_PyUnicode_IsXidContinue(PyUnicode_READ(kind, data, i))) { return 0; + } + } return 1; } @@ -12250,7 +12253,11 @@ static PyObject * unicode_isidentifier_impl(PyObject *self) /*[clinic end generated code: output=fe585a9666572905 input=2d807a104f21c0c5]*/ { - return PyBool_FromLong(PyUnicode_IsIdentifier(self)); + int identifier = PyUnicode_IsIdentifier(self); + if (identifier < 0) { + return NULL; + } + return PyBool_FromLong(identifier); } /*[clinic input] diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index f73c32684c7b73f..b5e3f54ce8b249a 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -1078,6 +1078,10 @@ verify_identifier(struct tok_state *tok) return 0; } result = PyUnicode_IsIdentifier(s); + if (result < 0) { + /* silently ignore error */ + PyErr_Clear(); + } Py_DECREF(s); if (result == 0) tok->done = E_IDENTIFIER; From d253c672c5a1e3da814e5150d3fb3aa8110eb7a0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 30 Jan 2020 12:24:59 +0100 Subject: [PATCH 2/4] Fix typo --- Objects/typeobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 52c8c09dd5f6c35..cbc7c81e83ace45 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -2243,7 +2243,7 @@ valid_identifier(PyObject *s) } int identifier = PyUnicode_IsIdentifier(s); if (identifier < 0) { - return 0 + return 0; } if (!identifier) { PyErr_SetString(PyExc_TypeError, From 9e4042ddb95706512f9bacf4dc10b0c07cad33ab Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 30 Jan 2020 12:45:44 +0100 Subject: [PATCH 3/4] Document the behavior change --- Doc/c-api/unicode.rst | 4 ++++ Doc/whatsnew/3.9.rst | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 6d361dcdd3800d1..93b52170f1d894b 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -246,6 +246,10 @@ access internal read-only data of Unicode objects: Raise an exception and return ``-1`` on error. + .. versionchanged:: 3.9 + The function now returns ``-1`` on error, instead of calling + :c:func:`Py_FatalError`. + Unicode Character Properties """""""""""""""""""""""""""" diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index c8f407751ec5e25..a5733ce7b81904f 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -339,6 +339,10 @@ Build and C API Changes functions are now required to build Python. (Contributed by Victor Stinner in :issue:`39395`.) +* The :c:func:`PyUnicode_IsIdentifier` function now returns ``-1`` on error, + instead of calling :c:func:`Py_FatalError`. + (Contributed by Victor Stinner in :issue:`39500`.) + Deprecated ========== From 602d7ac834e9d9c3eb8ebfe55898a41cb7a8ab30 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 30 Jan 2020 22:54:19 +0100 Subject: [PATCH 4/4] Add assertion in verify_identifier() --- Parser/tokenizer.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index b5e3f54ce8b249a..a885a69e23c4cab 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -1078,6 +1078,7 @@ verify_identifier(struct tok_state *tok) return 0; } result = PyUnicode_IsIdentifier(s); + assert(result >= 0); if (result < 0) { /* silently ignore error */ PyErr_Clear();