Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions bindings/pyroot/cppyy/CPyCppyy/src/CPPMethod.cxx
Original file line number Diff line number Diff line change
Expand Up @@ -321,15 +321,15 @@ void CPyCppyy::CPPMethod::SetPyError_(PyObject* msg)
if (!isCppExc) {
// this is the case where no Python error has occurred yet, and we set a new
// error with context info
PyErr_Format(errtype, "%s =>\n %s: %s", cdoc, cname, cmsg ? cmsg : "");
PyErr_Format(errtype, "Failed to call \"%s\" =>\n %s: %s", cdoc, cname, cmsg ? cmsg : "");
} else {
// augment the top message with context information
PyObject *&topMessage = ((CPPExcInstance*)evalue)->fTopMessage;
Py_XDECREF(topMessage);
if (msg) {
topMessage = CPyCppyy_PyText_FromFormat("%s =>\n %s: %s | ", cdoc, cname, cmsg);
topMessage = CPyCppyy_PyText_FromFormat("Failed to call \"%s\" =>\n %s: %s | ", cdoc, cname, cmsg);
} else {
topMessage = CPyCppyy_PyText_FromFormat("%s =>\n %s: ", cdoc, cname);
topMessage = CPyCppyy_PyText_FromFormat("Failed to call \"%s\" =>\n %s: ", cdoc, cname);
}
// restore the updated error
#if PY_VERSION_HEX >= 0x030c0000
Expand Down
3 changes: 2 additions & 1 deletion bindings/pyroot/cppyy/CPyCppyy/src/Converters.cxx
Original file line number Diff line number Diff line change
Expand Up @@ -2688,7 +2688,8 @@ static void* PyFunction_AsCPointer(PyObject* pyobject,
if (pytmpl->fTemplateArgs)
fullname += CPyCppyy_PyText_AsString(pytmpl->fTemplateArgs);
Cppyy::TCppScope_t scope = ((CPPClass*)pytmpl->fTI->fPyClass)->fCppType;
Cppyy::TCppMethod_t cppmeth = Cppyy::GetMethodTemplate(scope, fullname, signature);
std::ostringstream diagnostics;
Cppyy::TCppMethod_t cppmeth = Cppyy::GetMethodTemplate(scope, fullname, signature, diagnostics);
if (cppmeth) {
void* fptr = (void*)Cppyy::GetFunctionAddress(cppmeth, false);
if (fptr) return fptr;
Expand Down
4 changes: 2 additions & 2 deletions bindings/pyroot/cppyy/CPyCppyy/src/Cppyy.h
Original file line number Diff line number Diff line change
Expand Up @@ -229,8 +229,8 @@ namespace Cppyy {
CPPYY_IMPORT
bool IsMethodTemplate(TCppScope_t scope, TCppIndex_t imeth);
CPPYY_IMPORT
TCppMethod_t GetMethodTemplate(
TCppScope_t scope, const std::string& name, const std::string& proto);
TCppMethod_t GetMethodTemplate(TCppScope_t scope, const std::string &name, const std::string &proto,
std::ostringstream &diagnostics);

CPPYY_IMPORT
TCppIndex_t GetGlobalOperator(
Expand Down
58 changes: 44 additions & 14 deletions bindings/pyroot/cppyy/CPyCppyy/src/TemplateProxy.cxx
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@

// Standard
#include <algorithm>

#include <sstream>

namespace CPyCppyy {

Expand Down Expand Up @@ -177,9 +177,11 @@ PyObject* TemplateProxy::Instantiate(const std::string& fname,
proto = name_v1.substr(1, name_v1.size()-2);
}

// the following causes instantiation as necessary
std::ostringstream diagnostics;

// the following causes instantiation as necessary
Cppyy::TCppScope_t scope = ((CPPClass*)fTI->fPyClass)->fCppType;
Cppyy::TCppMethod_t cppmeth = Cppyy::GetMethodTemplate(scope, fname, proto);
Cppyy::TCppMethod_t cppmeth = Cppyy::GetMethodTemplate(scope, fname, proto, diagnostics);
if (cppmeth) { // overload stops here
// A successful instantiation needs to be cached to pre-empt future instantiations. There
// are two names involved, the original asked (which may be partial) and the received.
Expand All @@ -203,12 +205,14 @@ PyObject* TemplateProxy::Instantiate(const std::string& fname,
pos = proto.find("initializer_list", pos + 6);
}

Cppyy::TCppMethod_t m2 = Cppyy::GetMethodTemplate(scope, fname, proto);
std::ostringstream diagnostics2;
Cppyy::TCppMethod_t m2 = Cppyy::GetMethodTemplate(scope, fname, proto, diagnostics2);
if (m2 && m2 != cppmeth) {
// replace if the new method with vector was found; otherwise just continue
// with the previously found method with initializer_list.
cppmeth = m2;
resname = Cppyy::GetMethodFullName(cppmeth);
// replace if the new method with vector was found; otherwise just continue
// with the previously found method with initializer_list.
cppmeth = m2;
resname = Cppyy::GetMethodFullName(cppmeth);
diagnostics = std::move(diagnostics2);
}
}

Expand All @@ -226,6 +230,10 @@ PyObject* TemplateProxy::Instantiate(const std::string& fname,
Py_DECREF(pyol);
Py_DECREF(pycachename);
Py_DECREF(dct);

PyErr_Format(PyExc_TypeError, "Failed to instantiate \"%s(%s)\"\n%s", fname.c_str(), proto.c_str(),
diagnostics.str().c_str());

return nullptr;
}

Expand Down Expand Up @@ -296,10 +304,19 @@ PyObject* TemplateProxy::Instantiate(const std::string& fname,
PyObject* pymeth =
CPPOverload_Type.tp_descr_get(pyol, bNeedsRebind ? fSelf : nullptr, (PyObject*)&CPPOverload_Type);
Py_DECREF(pyol);
// check if diagnostics contains only spaces since this can happen as a result of clang indenting
const bool emptydiag = diagnostics.str().find_first_not_of(' ') == diagnostics.str().npos;
if (!emptydiag) {
std::ostringstream warnmsg;
warnmsg << "Compiler warnings during instantiation of \"" << fname << "(" << proto << ")\"\n"
<< diagnostics.str();
PyErr_WarnEx(PyExc_Warning, warnmsg.str().c_str(), 1);
}
return pymeth;
}

PyErr_Format(PyExc_TypeError, "Failed to instantiate \"%s(%s)\"", fname.c_str(), proto.c_str());
PyErr_Format(PyExc_TypeError, "Failed to instantiate \"%s(%s)\"\n%s", fname.c_str(), proto.c_str(),
diagnostics.str().c_str());
return nullptr;
}

Expand Down Expand Up @@ -801,6 +818,8 @@ static PyObject* tpp_overload(TemplateProxy* pytmpl, PyObject* args)
Cppyy::TCppMethod_t cppmeth = (Cppyy::TCppMethod_t) 0;
std::string proto;

std::ostringstream diagnostics;

if (PyArg_ParseTuple(args, const_cast<char*>("s|i:__overload__"), &sigarg, &want_const)) {
want_const = PyTuple_GET_SIZE(args) == 1 ? -1 : want_const;

Expand All @@ -818,12 +837,12 @@ static PyObject* tpp_overload(TemplateProxy* pytmpl, PyObject* args)

scope = ((CPPClass*)pytmpl->fTI->fPyClass)->fCppType;
cppmeth = Cppyy::GetMethodTemplate(
scope, pytmpl->fTI->fCppName, proto.substr(1, proto.size()-2));
scope, pytmpl->fTI->fCppName, proto.substr(1, proto.size()-2), diagnostics);
} else if (PyArg_ParseTuple(args, const_cast<char*>("ss:__overload__"), &sigarg, &tmplarg)) {
scope = ((CPPClass*)pytmpl->fTI->fPyClass)->fCppType;
std::string full_name = std::string(pytmpl->fTI->fCppName) + "<" + tmplarg + ">";

cppmeth = Cppyy::GetMethodTemplate(scope, full_name, sigarg);
cppmeth = Cppyy::GetMethodTemplate(scope, full_name, sigarg, diagnostics);
} else if (PyArg_ParseTuple(args, const_cast<char*>("O|i:__overload__"), &sigarg_tuple, &want_const)) {
PyErr_Clear();
want_const = PyTuple_GET_SIZE(args) == 1 ? -1 : want_const;
Expand Down Expand Up @@ -854,20 +873,31 @@ static PyObject* tpp_overload(TemplateProxy* pytmpl, PyObject* args)
proto.push_back('>');

scope = ((CPPClass*)pytmpl->fTI->fPyClass)->fCppType;
cppmeth = Cppyy::GetMethodTemplate(
scope, pytmpl->fTI->fCppName, proto.substr(1, proto.size()-2));
cppmeth =
Cppyy::GetMethodTemplate(scope, pytmpl->fTI->fCppName, proto.substr(1, proto.size() - 2), diagnostics);
} else {
PyErr_Format(PyExc_TypeError, "Unexpected arguments to __overload__");
return nullptr;
}

// else attempt instantiation
if (!cppmeth) {
return nullptr;
PyErr_Format(PyExc_TypeError, "Failed in template overload resolution \"%s(%s)\"\n%s",
pytmpl->fTI->fCppName.c_str(), proto.c_str(), diagnostics.str().c_str());
return nullptr;
}

PyErr_Clear();

const bool emptydiag = diagnostics.str().find_first_not_of(' ') == diagnostics.str().npos;
if (!emptydiag) {
std::ostringstream warnmsg;
warnmsg << "Compiler warnings during template overload resolution of \"" << pytmpl->fTI->fCppName << "(" << proto
<< ")\"\n"
<< diagnostics.str();
PyErr_WarnEx(PyExc_Warning, warnmsg.str().c_str(), 1);
}

// TODO: the next step should be consolidated with Instantiate()
PyCallable* meth = nullptr;
if (Cppyy::IsNamespace(scope)) {
Expand Down
3 changes: 2 additions & 1 deletion bindings/pyroot/cppyy/CPyCppyy/src/Utility.cxx
Original file line number Diff line number Diff line change
Expand Up @@ -378,7 +378,8 @@ CPyCppyy::PyCallable* CPyCppyy::Utility::FindBinaryOperator(
else { fname << "not_implemented<"; }
fname << lcname << ", " << rcname << ">";
proto << "const " << lcname << "&, const " << rcname;
Cppyy::TCppMethod_t method = Cppyy::GetMethodTemplate(s_intern, fname.str(), proto.str());
std::ostringstream diagnostics;
Cppyy::TCppMethod_t method = Cppyy::GetMethodTemplate(s_intern, fname.str(), proto.str(), diagnostics);
if (method) pyfunc = new CPPFunction(s_intern, method);
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1088,9 +1088,11 @@ char* Cppyy::CallS(
cstr = cppstring_to_cstring(*cppresult);
*length = cppresult->size();
cppresult->std::string::~basic_string();
} else
*length = 0;
free((void*)cppresult);
free((void *)cppresult);
} else {
*length = 0;
free((void *)cppresult);
Comment thread
vepadulano marked this conversation as resolved.
}
return cstr;
}

Expand Down Expand Up @@ -2101,55 +2103,80 @@ bool template_compare(std::string n1, std::string n2) {
return n2.compare(0, n1.size(), n1) == 0;
}

Cppyy::TCppMethod_t Cppyy::GetMethodTemplate(
TCppScope_t scope, const std::string& name, const std::string& proto)
Cppyy::TCppMethod_t Cppyy::GetMethodTemplate(TCppScope_t scope, const std::string &name, const std::string &proto,
std::ostringstream &diagnostics)
{
// There is currently no clean way of extracting a templated method out of ROOT/meta
// for a variety of reasons, none of them fundamental. The game played below is to
// first get any pre-existing functions already managed by ROOT/meta, but if that fails,
// to do an explicit lookup that ignores the prototype (i.e. the full name should be
// enough), and finally to ignore the template arguments part of the name as this fails
// in cling if there are default parameters.
TFunction* func = nullptr; ClassInfo_t* cl = nullptr;

// It would be possible to get the prototype from the created functions and use that to
// do a new lookup, after which ROOT/meta will manage the function. However, neither
// TFunction::GetPrototype() nor TFunction::GetSignature() is of the proper form, so
// we'll/ manage the new TFunctions instead and will assume that they are cached on the
// calling side to prevent multiple creations.

// redirect diagnostics, taking the lock to make sure no other calls pollute the results
R__WRITE_LOCKGUARD(ROOT::gCoreMutex);

const std::string diagnosticsold = diagnostics.str();
TInterpreter::RedirectDiagnostics redirectRAII(gInterpreter, diagnostics, /*enableColors*/ true, /*indent*/ 4);
Comment on lines +2123 to +2126

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wonder how this interacts with the future rebase of clingwrapper on top of CppInterop @aaronj0 @Vipul-Cariappa . Do we have similar infrastructure for retrieving interpreter diagnostics without leaking implementation details here?

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We currently do not have a way of doing this. But it should be relatively easy to implement. The question of "without leaking implementation details", not sure about that. If the question is: Can we propagate why a template instantiation fails? Yes, we propagate the reason.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe a clearer rephrasing of my question would be "can we obtain the same interpreter diagnostics in clingwrapper.cxx, without using any ROOT facility (i.e. R__WRITE_LOCKGUARD, ROOT::gCoreMutex, TInterpreter in this case), so that this diagnostic can become something generic and still give us the chance one day of outsourcing clingwrapper.cxx?"

@aaronj0 aaronj0 Dec 1, 2025

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

"can we obtain the same interpreter diagnostics in clingwrapper.cxx, without using any ROOT facility (i.e. R__WRITE_LOCKGUARD, ROOT::gCoreMutex, TInterpreter in this case)

The way I see it, we will still need the gCore mutex for the clingwrapper API to continue supporting ROOT threading model. Which is not a problem since we still link against Core in the design that migrates to InterOp.

so that this diagnostic can become something generic and still give us the chance one day of outsourcing clingwrapper.cxx

As Vipul mentioned, obtaining the diagnostic itself is not yet supported, but it can indeed be done generically, i.e., based on Clang's diagnostics engine (like this PR does). Based on that, I would comment that the right place for the very nice TClingDiagnosticsRAII implemented by this PR is probably CppInterOp, which solves the concern on whether this can be generalized and outsourced. In my view, clingwrapper is the one part that will/should contain ROOT-specific extensions on top of CppInterOp. @vgvassilev can comment on the changes in Cling's LookupHelper, but on first glance, that looks good to me.

One way to proceed is to keep the current approach and, as part of the migration, upstream the diagnostics from TCling to CppInterOp, and revert the changes in TInterpreter, since we can then interact with the Cpp::Interpreter directly. If the need for this PR is not immediate, the easier option is to revisit it on top of the migration PR. Ideally, we shouldn't grow further the interactions between Core and the interpreter via metacling for cppyy use cases. Apart from that, this PR heads in the right direction and proposes some quite nice additions.


TFunction *func = nullptr;
ClassInfo_t *cl = nullptr;
if (scope == (TCppScope_t)GLOBAL_HANDLE) {
func = gROOT->GetGlobalFunctionWithPrototype(name.c_str(), proto.c_str());
if (func && name.back() == '>') {
// make sure that all template parameters match (more are okay, e.g. defaults or
// ones derived from the arguments or variadic templates)
if (!template_compare(name, func->GetName()))
func = nullptr; // happens if implicit conversion matches the overload
}
func = gROOT->GetGlobalFunctionWithPrototype(name.c_str(), proto.c_str());
if (func && name.back() == '>') {
// make sure that all template parameters match (more are okay, e.g. defaults or
// ones derived from the arguments or variadic templates)
if (!template_compare(name, func->GetName()))
func = nullptr; // happens if implicit conversion matches the overload
}
} else {
TClassRef& cr = type_from_handle(scope);
if (cr.GetClass()) {
func = cr->GetMethodWithPrototype(name.c_str(), proto.c_str());
if (!func) {
cl = cr->GetClassInfo();
TClassRef &cr = type_from_handle(scope);
if (cr.GetClass()) {
func = cr->GetMethodWithPrototype(name.c_str(), proto.c_str());
if (!func) {
cl = cr->GetClassInfo();
// try base classes to cover a common 'using' case (TODO: this is stupid and misses
// out on base classes; fix that with improved access to Cling)
TCppIndex_t nbases = GetNumBases(scope);
for (TCppIndex_t i = 0; i < nbases; ++i) {
TClassRef& base = type_from_handle(GetScope(GetBaseName(scope, i)));
if (base.GetClass()) {
func = base->GetMethodWithPrototype(name.c_str(), proto.c_str());
if (func) break;
}
TCppIndex_t nbases = GetNumBases(scope);
for (TCppIndex_t i = 0; i < nbases; ++i) {
TClassRef &base = type_from_handle(GetScope(GetBaseName(scope, i)));
if (base.GetClass()) {
func = base->GetMethodWithPrototype(name.c_str(), proto.c_str());
if (func)
break;
}
}
}
}
}

if (!func && name.back() == '>' && (cl || scope == (TCppScope_t)GLOBAL_HANDLE)) {
// try again, ignoring proto in case full name is complete template
auto declid = gInterpreter->GetFunction(cl, name.c_str());
if (declid) {
auto existing = gMethodTemplates.find(declid);
if (existing == gMethodTemplates.end()) {
auto cw = new_CallWrapper(declid, name);
existing = gMethodTemplates.insert(std::make_pair(declid, cw)).first;
}
return (TCppMethod_t)existing->second;
std::ostringstream diagnostics2;
TInterpreter::RedirectDiagnostics redirectRAII2(gInterpreter, diagnostics2, /*enableColors*/ true, /*indent*/ 4);

auto declid = gInterpreter->GetFunction(cl, name.c_str());
if (declid) {
auto existing = gMethodTemplates.find(declid);
if (existing == gMethodTemplates.end()) {
auto cw = new_CallWrapper(declid, name);
existing = gMethodTemplates.insert(std::make_pair(declid, cw)).first;
}
// replace diagnostics to suppress spurious errors or warnings from the previous
// failed lookup
diagnostics = std::ostringstream();
diagnostics << diagnosticsold;
diagnostics << diagnostics2.str();

return (TCppMethod_t)existing->second;
}
diagnostics << diagnostics2.str();
}

if (func) {
Expand All @@ -2165,15 +2192,24 @@ Cppyy::TCppMethod_t Cppyy::GetMethodTemplate(
if (name.back() == '>') {
auto pos = name.find('<');
if (pos != std::string::npos) {
TCppMethod_t cppmeth = GetMethodTemplate(scope, name.substr(0, pos), proto);
std::ostringstream diagnostics2;
TCppMethod_t cppmeth = GetMethodTemplate(scope, name.substr(0, pos), proto, diagnostics2);
if (cppmeth) {
// allow if requested template names match up to the result
const std::string& alt = GetMethodFullName(cppmeth);
if (name.size() < alt.size() && alt.find('<') == pos) {
if (template_compare(name, alt))
if (template_compare(name, alt)){
// replace diagnostics to suppress spurious errors or warnings
// from the previous failed lookup
diagnostics = std::ostringstream();
diagnostics << diagnosticsold;
diagnostics << diagnostics2.str();
return cppmeth;
}

}
}
diagnostics << diagnostics2.str();
}
}

Expand Down Expand Up @@ -2636,5 +2672,3 @@ long long Cppyy::GetEnumDataValue(TCppEnum_t etype, TCppIndex_t idata)
TEnumConstant* ecst = (TEnumConstant*)((TEnum*)etype)->GetConstants()->At((int)idata);
return (long long)ecst->GetValue();
}


Original file line number Diff line number Diff line change
Expand Up @@ -226,8 +226,8 @@ namespace Cppyy {
RPY_EXPORTED
bool IsMethodTemplate(TCppScope_t scope, TCppIndex_t imeth);
RPY_EXPORTED
TCppMethod_t GetMethodTemplate(
TCppScope_t scope, const std::string& name, const std::string& proto);
TCppMethod_t GetMethodTemplate(TCppScope_t scope, const std::string &name, const std::string &proto,
std::ostringstream &diagnostics);

RPY_EXPORTED
TCppIndex_t GetGlobalOperator(
Expand Down
2 changes: 2 additions & 0 deletions bindings/pyroot/pythonizations/test/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -191,3 +191,5 @@ if(NOT MSVC OR CMAKE_SIZEOF_VOID_P EQUAL 8 OR win_broken_tests)
# https://github.com/root-project/root/issues/9809
ROOT_ADD_PYUNITTEST(pyroot_array_numpy_views array_conversions.py PYTHON_DEPS numpy)
endif()

ROOT_ADD_PYUNITTEST(gh_11854 gh_11854.py)
Loading
Loading