From 646115f0483ea74d94da0f20b85dd8351108864a Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Wed, 3 Jun 2026 11:45:55 +0200 Subject: [PATCH] [GR-75636] Support text signatures for C extensions --- .../src/tests/cpyext/test_functions.py | 76 +++++++++++- .../modules/cext/CFunctionDocUtils.java | 112 ++++++++++++++++++ .../cext/PythonCextAbstractBuiltins.java | 4 +- .../modules/cext/PythonCextDescrBuiltins.java | 4 +- .../modules/cext/PythonCextFuncBuiltins.java | 6 +- .../cext/PythonCextMethodBuiltins.java | 3 +- .../modules/cext/PythonCextTypeBuiltins.java | 6 +- .../builtins/objects/cext/capi/CExtNodes.java | 5 +- .../function/AbstractFunctionBuiltins.java | 8 +- .../objects/function/PBuiltinFunction.java | 5 + 10 files changed, 212 insertions(+), 17 deletions(-) create mode 100644 graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/CFunctionDocUtils.java diff --git a/graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_functions.py b/graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_functions.py index 6d838a2124..8a4a80c3c2 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_functions.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_functions.py @@ -41,7 +41,14 @@ import sys import unittest -from . import CPyExtType, CPyExtTestCase, CPyExtFunction, unhandled_error_compare, CPyExtHeapType +from . import ( + CPyExtFunction, + CPyExtHeapType, + CPyExtTestCase, + CPyExtType, + compile_module_from_string, + unhandled_error_compare, +) DIR = os.path.dirname(__file__) @@ -563,6 +570,73 @@ def _ref_hash_not_implemented(args): # test calling m_meth class TestPyCFunction(unittest.TestCase): + def test_docstring_text_signature(self): + module = compile_module_from_string(r""" + #define PY_SSIZE_T_CLEAN + #include + + static PyObject *with_signature(PyObject *self, PyObject *arg) { + Py_RETURN_NONE; + } + + static PyObject *without_signature(PyObject *self, PyObject *arg) { + Py_RETURN_NONE; + } + + static PyObject *without_doc(PyObject *self, PyObject *unused) { + Py_RETURN_NONE; + } + + static PyMethodDef module_methods[] = { + {"with_signature", with_signature, METH_O, + "with_signature($module, value, /)\n" + "--\n\n" + "Return module function metadata."}, + {"without_signature", without_signature, METH_O, "Return a plain docstring."}, + {"without_doc", without_doc, METH_NOARGS, NULL}, + {NULL, NULL, 0, NULL} + }; + + static PyModuleDef module_def = { + PyModuleDef_HEAD_INIT, + "textsignature", + "", + -1, + module_methods, + NULL, NULL, NULL, NULL + }; + + PyMODINIT_FUNC + PyInit_textsignature(void) + { + return PyModule_Create(&module_def); + } + """, "textsignature") + + assert module.with_signature.__doc__ == "Return module function metadata." + assert module.with_signature.__text_signature__ == "($module, value, /)" + assert module.without_signature.__doc__ == "Return a plain docstring." + assert module.without_signature.__text_signature__ is None + assert module.without_doc.__doc__ is None + assert module.without_doc.__text_signature__ is None + + TypeWithTextSignature = CPyExtType( + "TypeWithTextSignature", + """ + static PyObject *method_with_signature(PyObject *self, PyObject *arg) { + Py_RETURN_NONE; + } + """, + tp_methods=""" + {"method_with_signature", method_with_signature, METH_O, + "method_with_signature($self, value, /)\\n" + "--\\n\\n" + "Return type method metadata."} + """, + ) + assert TypeWithTextSignature.method_with_signature.__doc__ == "Return type method metadata." + assert TypeWithTextSignature.method_with_signature.__text_signature__ == "($self, value, /)" + test_PyCFunction_NewEx_non_string_module = CPyExtFunction( lambda args: 1, lambda: ( diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/CFunctionDocUtils.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/CFunctionDocUtils.java new file mode 100644 index 0000000000..97533eafb9 --- /dev/null +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/CFunctionDocUtils.java @@ -0,0 +1,112 @@ +/* + * Copyright (c) 2026, 2026, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The Universal Permissive License (UPL), Version 1.0 + * + * Subject to the condition set forth below, permission is hereby granted to any + * person obtaining a copy of this software, associated documentation and/or + * data (collectively the "Software"), free of charge and under any and all + * copyright rights in the Software, and any and all patent rights owned or + * freely licensable by each licensor hereunder covering either (i) the + * unmodified Software as contributed to or provided by such licensor, or (ii) + * the Larger Works (as defined below), to deal in both + * + * (a) the Software, and + * + * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if + * one is included with the Software each a "Larger Work" to which the Software + * is contributed by such licensors), + * + * without restriction, including without limitation the rights to copy, create + * derivative works of, display, perform, and distribute the Software and make, + * use, sell, offer for sale, import, export, have made, and have sold the + * Software and the Larger Work(s), and to sublicense the foregoing rights on + * either these or other terms. + * + * This license is subject to the following condition: + * + * The above copyright notice and either this complete permission notice or at a + * minimum a reference to the UPL must be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package com.oracle.graal.python.builtins.modules.cext; + +import static com.oracle.graal.python.nodes.SpecialAttributeNames.T___DOC__; +import static com.oracle.graal.python.nodes.SpecialAttributeNames.T___TEXT_SIGNATURE__; +import static com.oracle.graal.python.util.PythonUtils.toTruffleStringUncached; + +import com.oracle.graal.python.builtins.objects.PNone; +import com.oracle.graal.python.builtins.objects.function.PBuiltinFunction; +import com.oracle.graal.python.nodes.attributes.WriteAttributeToPythonObjectNode; +import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary; +import com.oracle.truffle.api.strings.TruffleString; + +public final class CFunctionDocUtils { + + private static final String SIGNATURE_END_MARKER = ")\n--\n\n"; + private static final int SIGNATURE_END_MARKER_LENGTH = SIGNATURE_END_MARKER.length(); + + private CFunctionDocUtils() { + } + + @TruffleBoundary + public static void writeDocAndTextSignature(PBuiltinFunction function, TruffleString name, Object docObj) { + Object doc = PNone.NONE; + Object textSignature = PNone.NONE; + if (docObj instanceof TruffleString) { + TruffleString docTruffleString = (TruffleString) docObj; + String docString = docTruffleString.toJavaStringUncached(); + int start = findSignature(name.toJavaStringUncached(), docString); + int end = start >= 0 ? skipSignature(docString, start) : -1; + if (end < 0) { + doc = docString.isEmpty() ? PNone.NONE : docTruffleString; + } else { + int textSignatureEnd = end - SIGNATURE_END_MARKER_LENGTH + 1; + doc = end == docString.length() ? PNone.NONE : toTruffleStringUncached(docString.substring(end)); + textSignature = toTruffleStringUncached(docString.substring(start, textSignatureEnd)); + } + } + WriteAttributeToPythonObjectNode.executeUncached(function, T___DOC__, doc); + WriteAttributeToPythonObjectNode.executeUncached(function, T___TEXT_SIGNATURE__, textSignature); + } + + /* + * Matches CPython's find_signature: the internal doc must start with the callable name followed + * by the first '(' of the signature. + */ + private static int findSignature(String name, String doc) { + int dot = name.lastIndexOf('.'); + if (dot != -1) { + name = name.substring(dot + 1); + } + int length = name.length(); + if (!doc.startsWith(name) || doc.length() <= length || doc.charAt(length) != '(') { + return -1; + } + return length; + } + + /* + * Matches CPython's skip_signature: a blank line before the marker invalidates the signature. + */ + private static int skipSignature(String doc, int start) { + for (int i = start; i < doc.length(); i++) { + if (doc.startsWith(SIGNATURE_END_MARKER, i)) { + return i + SIGNATURE_END_MARKER_LENGTH; + } + if (doc.charAt(i) == '\n' && i + 1 < doc.length() && doc.charAt(i + 1) == '\n') { + return -1; + } + } + return -1; + } +} diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/PythonCextAbstractBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/PythonCextAbstractBuiltins.java index e2ddc17e46..a8338b0625 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/PythonCextAbstractBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/PythonCextAbstractBuiltins.java @@ -793,12 +793,12 @@ static int PyObject_SetDoc(long objPtr, long valuePtr) { Object obj = NativeToPythonNode.executeRawUncached(objPtr); Object value = CharPtrToPythonNode.getUncached().execute(valuePtr); if (obj instanceof PBuiltinFunction builtinFunction) { - WriteAttributeToPythonObjectNode.executeUncached(builtinFunction, T___DOC__, value); + CFunctionDocUtils.writeDocAndTextSignature(builtinFunction, builtinFunction.getName(), value); return 1; } if (obj instanceof PBuiltinMethod builtinMethod) { PBuiltinFunction builtinFunction = builtinMethod.getBuiltinFunction(); - WriteAttributeToPythonObjectNode.executeUncached(builtinFunction, T___DOC__, value); + CFunctionDocUtils.writeDocAndTextSignature(builtinFunction, builtinFunction.getName(), value); return 1; } if (obj instanceof GetSetDescriptor descriptor) { diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/PythonCextDescrBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/PythonCextDescrBuiltins.java index 12e097e065..afd94851c0 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/PythonCextDescrBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/PythonCextDescrBuiltins.java @@ -49,7 +49,6 @@ import static com.oracle.graal.python.builtins.objects.cext.capi.transitions.ArgDescriptor.PyObjectTransfer; import static com.oracle.graal.python.builtins.objects.cext.capi.transitions.ArgDescriptor.PyTypeObject; import static com.oracle.graal.python.nodes.HiddenAttr.METHOD_DEF_PTR; -import static com.oracle.graal.python.nodes.SpecialAttributeNames.T___DOC__; import static com.oracle.graal.python.nodes.SpecialAttributeNames.T___NAME__; import com.oracle.graal.python.PythonLanguage; @@ -104,9 +103,10 @@ public static long GraalPyPrivate_Descr_NewClassMethod(long methodDefPtr, long n Object type = NativeToPythonClassInternalNode.executeUncached(typeRaw); PBuiltinFunction func = MethodDescriptorWrapper.createWrapperFunction(language, name, methPtr, type, flags); assert func != null; + WriteAttributeToPythonObjectNode.executeUncached(func, T___NAME__, name); + CFunctionDocUtils.writeDocAndTextSignature(func, name, doc); PDecoratedMethod classMethod = PFactory.createBuiltinClassmethodFromCallableObj(language, func); WriteAttributeToPythonObjectNode.executeUncached(classMethod, T___NAME__, name); - WriteAttributeToPythonObjectNode.executeUncached(classMethod, T___DOC__, doc); HiddenAttr.WriteLongNode.executeUncached(classMethod, METHOD_DEF_PTR, methodDefPtr); return PythonToNativeInternalNode.executeUncached(classMethod, true); } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/PythonCextFuncBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/PythonCextFuncBuiltins.java index f913868ba0..548a0420d6 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/PythonCextFuncBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/PythonCextFuncBuiltins.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2022, 2026, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 @@ -45,7 +45,6 @@ import static com.oracle.graal.python.builtins.objects.cext.capi.transitions.ArgDescriptor.ConstCharPtrAsTruffleString; import static com.oracle.graal.python.builtins.objects.cext.capi.transitions.ArgDescriptor.PyObject; import static com.oracle.graal.python.builtins.objects.cext.capi.transitions.ArgDescriptor.PyObjectTransfer; -import static com.oracle.graal.python.nodes.SpecialAttributeNames.T___DOC__; import com.oracle.graal.python.PythonLanguage; import com.oracle.graal.python.builtins.modules.cext.PythonCextBuiltins.CApiBinaryBuiltinNode; @@ -108,7 +107,8 @@ private static PNone setDoc(Object functionObj, TruffleString doc) { } else { throw CompilerDirectives.shouldNotReachHere("Unexpected object passed to GraalPyCFunction_SetDoc"); } - function.setAttribute(T___DOC__, doc != null ? doc : PNone.NONE); + CFunctionDocUtils.writeDocAndTextSignature(function, function.getName(), + doc != null ? doc : PNone.NO_VALUE); return PNone.NO_VALUE; } } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/PythonCextMethodBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/PythonCextMethodBuiltins.java index a08e21271b..666682f120 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/PythonCextMethodBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/PythonCextMethodBuiltins.java @@ -50,7 +50,6 @@ import static com.oracle.graal.python.builtins.objects.cext.capi.transitions.ArgDescriptor.PyTypeObject; import static com.oracle.graal.python.builtins.objects.cext.common.CExtContext.METH_METHOD; import static com.oracle.graal.python.runtime.nativeaccess.NativeMemory.NULLPTR; -import static com.oracle.graal.python.nodes.SpecialAttributeNames.T___DOC__; import static com.oracle.graal.python.nodes.SpecialAttributeNames.T___MODULE__; import static com.oracle.graal.python.nodes.SpecialAttributeNames.T___NAME__; @@ -83,7 +82,7 @@ static PythonBuiltinObject cFunctionNewExMethodNode(PythonLanguage language, lon PBuiltinFunction func = MethodDescriptorWrapper.createWrapperFunction(language, name, methPtr, PNone.NO_VALUE, flags); HiddenAttr.WriteLongNode.executeUncached(func, METHOD_DEF_PTR, methodDefPtr); WriteAttributeToPythonObjectNode.executeUncached(func, T___NAME__, name); - WriteAttributeToPythonObjectNode.executeUncached(func, T___DOC__, doc); + CFunctionDocUtils.writeDocAndTextSignature(func, name, doc); PBuiltinMethod method; if (cls != PNone.NO_VALUE) { method = PFactory.createBuiltinMethod(language, self, func, cls); diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/PythonCextTypeBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/PythonCextTypeBuiltins.java index 1292f9ba6e..f410a21b0d 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/PythonCextTypeBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/PythonCextTypeBuiltins.java @@ -293,6 +293,10 @@ int trace(long ptr) { private static PythonBuiltinObject typeAddMethod(PythonLanguage language, long methodDefPtr, TruffleString name, long methPtr, int flags, Object type, Object doc) { assert doc == PNone.NO_VALUE || doc instanceof TruffleString; PBuiltinFunction func = MethodDescriptorWrapper.createWrapperFunction(language, name, methPtr, type, flags); + if (func != null) { + WriteAttributeToPythonObjectNode.executeUncached(func, T___NAME__, name); + CFunctionDocUtils.writeDocAndTextSignature(func, name, doc); + } if (CExtContext.isMethClass(flags)) { if (CExtContext.isMethStatic(flags)) { assert func == null; @@ -303,8 +307,6 @@ private static PythonBuiltinObject typeAddMethod(PythonLanguage language, long m } else if (CExtContext.isMethStatic(flags)) { return PFactory.createStaticmethodFromCallableObj(language, func); } - WriteAttributeToPythonObjectNode.executeUncached(func, T___NAME__, name); - WriteAttributeToPythonObjectNode.executeUncached(func, T___DOC__, doc); HiddenAttr.WriteLongNode.executeUncached(func, METHOD_DEF_PTR, methodDefPtr); return func; } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CExtNodes.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CExtNodes.java index c1b77950b4..3dc09002aa 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CExtNodes.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CExtNodes.java @@ -84,6 +84,7 @@ import com.oracle.graal.python.PythonLanguage; import com.oracle.graal.python.builtins.PythonBuiltinClassType; +import com.oracle.graal.python.builtins.modules.cext.CFunctionDocUtils; import com.oracle.graal.python.builtins.objects.PNone; import com.oracle.graal.python.builtins.objects.PythonAbstractObject; import com.oracle.graal.python.builtins.objects.bytes.PByteArray; @@ -1199,9 +1200,7 @@ static PBuiltinFunction createLegacyMethod(long methodDefPtr, int element, Pytho PBuiltinFunction function = PFactory.createBuiltinFunction(language, methodName, null, PythonUtils.EMPTY_OBJECT_ARRAY, kwDefaults, flags, rootNode); HiddenAttr.WriteLongNode.executeUncached(function, METHOD_DEF_PTR, methodDefPtr); - // write doc string; we need to directly write to the storage otherwise it is disallowed - // writing to builtin types. - WriteAttributeToPythonObjectNode.executeUncached(function, SpecialAttributeNames.T___DOC__, methodDoc); + CFunctionDocUtils.writeDocAndTextSignature(function, methodName, methodDoc); return function; } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/function/AbstractFunctionBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/function/AbstractFunctionBuiltins.java index 273a1b29b3..44e6836809 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/function/AbstractFunctionBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/function/AbstractFunctionBuiltins.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2025, Oracle and/or its affiliates. + * Copyright (c) 2017, 2026, Oracle and/or its affiliates. * Copyright (c) 2014, Regents of the University of California * * All rights reserved. @@ -348,8 +348,12 @@ static Object setFunction(PFunction self, Object value, @Specialization(guards = "isNoValue(none)") @TruffleBoundary - static TruffleString getBuiltin(PBuiltinFunction self, @SuppressWarnings("unused") PNone none, + static Object getBuiltin(PBuiltinFunction self, @SuppressWarnings("unused") PNone none, @Bind Node inliningTarget) { + Object storedSignature = ReadAttributeFromObjectNode.getUncached().execute(self, T___TEXT_SIGNATURE__); + if (storedSignature != PNone.NO_VALUE) { + return storedSignature; + } Signature signature = self.getSignature(); if (signature.isHidden()) { throw PRaiseNode.raiseStatic(inliningTarget, AttributeError, ErrorMessages.HAS_NO_ATTR, self, T___TEXT_SIGNATURE__); diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/function/PBuiltinFunction.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/function/PBuiltinFunction.java index 32d6f00b9f..a41da7f09d 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/function/PBuiltinFunction.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/function/PBuiltinFunction.java @@ -26,6 +26,7 @@ package com.oracle.graal.python.builtins.objects.function; import static com.oracle.graal.python.nodes.SpecialAttributeNames.T___DOC__; +import static com.oracle.graal.python.nodes.SpecialAttributeNames.T___TEXT_SIGNATURE__; import static com.oracle.graal.python.nodes.StringLiterals.T_DOT; import java.util.Arrays; @@ -256,6 +257,10 @@ public PBuiltinFunction boundToObject(PythonBuiltinClassType klass, PythonLangua } else { PBuiltinFunction func = PFactory.createBuiltinFunction(language, this, klass); func.setAttribute(T___DOC__, getAttribute(T___DOC__)); + Object textSignature = getAttribute(T___TEXT_SIGNATURE__); + if (textSignature != PNone.NO_VALUE) { + func.setAttribute(T___TEXT_SIGNATURE__, textSignature); + } return func; } }