diff --git a/bindings/pyroot/cppyy/CPyCppyy/setup.py b/bindings/pyroot/cppyy/CPyCppyy/setup.py index 320d3ca9122df..0bab79eb338d5 100755 --- a/bindings/pyroot/cppyy/CPyCppyy/setup.py +++ b/bindings/pyroot/cppyy/CPyCppyy/setup.py @@ -9,7 +9,7 @@ except ImportError: has_wheel = False -requirements = ['cppyy-cling==6.30.0', 'cppyy-backend==1.15.2'] +requirements = ['cppyy-cling==6.32.8', 'cppyy-backend==1.15.3'] setup_requirements = ['wheel'] if 'build' in sys.argv or 'install' in sys.argv: setup_requirements += requirements @@ -80,7 +80,7 @@ def build_extension(self, ext): setup( name='CPyCppyy', - version='1.12.16', + version='1.13.0', description='Cling-based Python-C++ bindings for CPython', long_description=long_description, diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/CPPInstance.h b/bindings/pyroot/cppyy/CPyCppyy/src/CPPInstance.h index f61c041e2777d..85012f8ba5391 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/CPPInstance.h +++ b/bindings/pyroot/cppyy/CPyCppyy/src/CPPInstance.h @@ -27,20 +27,21 @@ class CPPInstance { public: enum EFlags { kDefault = 0x0000, - kNoWrapConv = 0x0001, - kIsOwner = 0x0002, - kIsExtended = 0x0004, - kIsReference = 0x0008, - kIsRValue = 0x0010, - kIsLValue = 0x0020, - kIsValue = 0x0040, - kIsPtrPtr = 0x0080, - kIsArray = 0x0100, - kIsSmartPtr = 0x0200, - kNoMemReg = 0x0400, - kHasLifeLine = 0x0800, - kIsRegulated = 0x1000, - kIsActual = 0x2000 }; + kNoWrapConv = 0x0001, // use type as-is (eg. no smart ptr wrap) + kIsOwner = 0x0002, // Python instance owns C++ object/memory + kIsExtended = 0x0004, // has extended data + kIsValue = 0x0008, // was created from a by-value return + kIsReference = 0x0010, // represents one indirection + kIsArray = 0x0020, // represents an array of objects + kIsSmartPtr = 0x0040, // is or embeds a smart pointer + kIsPtrPtr = 0x0080, // represents two indirections + kIsRValue = 0x0100, // can be used as an r-value + kIsLValue = 0x0200, // can be used as an l-value + kNoMemReg = 0x0400, // do not register with memory regulator + kIsRegulated = 0x0800, // is registered with memory regulator + kIsActual = 0x1000, // has been downcasted to actual type + kHasLifeLine = 0x2000, // has a life line set + }; public: // public, as the python C-API works with C structs PyObject_HEAD diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/CPPMethod.cxx b/bindings/pyroot/cppyy/CPyCppyy/src/CPPMethod.cxx index 097fb527cab19..823e8ac60a694 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/CPPMethod.cxx +++ b/bindings/pyroot/cppyy/CPyCppyy/src/CPPMethod.cxx @@ -611,15 +611,30 @@ PyObject* CPyCppyy::CPPMethod::GetArgDefault(int iarg, bool silent) PyObject* gdct = *dctptr; PyObject* scope = nullptr; - if (defvalue.find("::") != std::string::npos) { - // try to tickle scope creation, just in case - scope = CreateScopeProxy(defvalue.substr(0, defvalue.rfind('('))); - if (!scope) PyErr_Clear(); - - // rename '::' -> '.' - TypeManip::cppscope_to_pyscope(defvalue); + if (defvalue.rfind('(') != std::string::npos) { // constructor-style call + // try to tickle scope creation, just in case, first look in the scope where + // the function lives, then in the global scope + std::string possible_scope = defvalue.substr(0, defvalue.rfind('(')); + if (!Cppyy::IsBuiltin(possible_scope)) { + std::string cand_scope = Cppyy::GetScopedFinalName(fScope)+"::"+possible_scope; + scope = CreateScopeProxy(cand_scope); + if (!scope) { + PyErr_Clear(); + // search within the global scope instead + scope = CreateScopeProxy(possible_scope); + if (!scope) PyErr_Clear(); + } else { + // re-scope the scope; alternatively, the expression could be + // compiled in the dictionary of the function's namespace, but + // that would affect arguments passed to the constructor, too + defvalue = cand_scope + defvalue.substr(defvalue.rfind('('), std::string::npos); + } + } } + // replace '::' -> '.' + TypeManip::cppscope_to_pyscope(defvalue); + if (!scope) { // a couple of common cases that python doesn't like (technically, 'L' is okay with older // pythons, but C long will always fit in Python int, so no need to bother) @@ -956,13 +971,6 @@ PyObject* CPyCppyy::CPPMethod::Execute(void* self, ptrdiff_t offset, CallContext result = ExecuteProtected(self, offset, ctxt); } -// TODO: the following is dreadfully slow and dead-locks on Apache: revisit -// raising exceptions through callbacks by using magic returns -// if (result && Utility::PyErr_Occurred_WithGIL()) { -// // can happen in the case of a CINT error: trigger exception processing -// Py_DECREF(result); -// result = 0; -// } else if (!result && PyErr_Occurred()) if (!result && PyErr_Occurred()) SetPyError_(0); diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/CPPOverload.cxx b/bindings/pyroot/cppyy/CPyCppyy/src/CPPOverload.cxx index bbae42606bd0c..96af011bdde7e 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/CPPOverload.cxx +++ b/bindings/pyroot/cppyy/CPyCppyy/src/CPPOverload.cxx @@ -697,7 +697,7 @@ static PyObject* mp_call(CPPOverload* pymeth, PyObject* args, PyObject* kwds) continue; // did not set implicit conversion, so don't try again PyObject* result = methods[i]->Call(im_self, args, nargsf, kwds, &ctxt); - if (result != 0) { + if (result) { // success: update the dispatch map for subsequent calls if (!memoized_pc) dispatchMap.push_back(std::make_pair(sighash, methods[i])); diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/Converters.cxx b/bindings/pyroot/cppyy/CPyCppyy/src/Converters.cxx index f7ccf76488edd..c127604a6e3d6 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/Converters.cxx +++ b/bindings/pyroot/cppyy/CPyCppyy/src/Converters.cxx @@ -880,6 +880,13 @@ CPPYY_IMPL_BASIC_CONVERTER_NI( CPPYY_IMPL_BASIC_CHAR_CONVERTER(Char, char, CHAR_MIN, CHAR_MAX) CPPYY_IMPL_BASIC_CHAR_CONVERTER(UChar, unsigned char, 0, UCHAR_MAX) +PyObject* CPyCppyy::SCharAsIntConverter::FromMemory(void* address) +{ +// special case to be used with arrays: return a Python int instead of str +// (following the same convention as module array.array) + return PyInt_FromLong((long)*((signed char*)address)); +} + PyObject* CPyCppyy::UCharAsIntConverter::FromMemory(void* address) { // special case to be used with arrays: return a Python int instead of str @@ -3428,6 +3435,7 @@ static struct InitConvFactories_t { gf["unsigned char"] = (cf_t)+[](cdims_t) { static UCharConverter c{}; return &c; }; gf["const unsigned char&"] = (cf_t)+[](cdims_t) { static ConstUCharRefConverter c{}; return &c; }; gf["unsigned char&"] = (cf_t)+[](cdims_t) { static UCharRefConverter c{}; return &c; }; + gf["SCharAsInt"] = (cf_t)+[](cdims_t) { static SCharAsIntConverter c{}; return &c; }; gf["UCharAsInt"] = (cf_t)+[](cdims_t) { static UCharAsIntConverter c{}; return &c; }; gf["wchar_t"] = (cf_t)+[](cdims_t) { static WCharConverter c{}; return &c; }; gf["char16_t"] = (cf_t)+[](cdims_t) { static Char16Converter c{}; return &c; }; @@ -3481,11 +3489,12 @@ static struct InitConvFactories_t { // pointer/array factories gf["bool ptr"] = (cf_t)+[](cdims_t d) { return new BoolArrayConverter{d}; }; - gf["const signed char[]"] = (cf_t)+[](cdims_t d) { return new SCharArrayConverter{d}; }; - gf["signed char[]"] = gf["const signed char[]"]; + gf["signed char ptr"] = (cf_t)+[](cdims_t d) { return new SCharArrayConverter{d}; }; gf["signed char**"] = (cf_t)+[](cdims_t) { return new SCharArrayConverter{{UNKNOWN_SIZE, UNKNOWN_SIZE}}; }; gf["const unsigned char*"] = (cf_t)+[](cdims_t d) { return new UCharArrayConverter{d}; }; gf["unsigned char ptr"] = (cf_t)+[](cdims_t d) { return new UCharArrayConverter{d}; }; + gf["SCharAsInt*"] = gf["signed char ptr"]; + gf["SCharAsInt[]"] = gf["signed char ptr"]; gf["UCharAsInt*"] = gf["unsigned char ptr"]; gf["UCharAsInt[]"] = gf["unsigned char ptr"]; #if __cplusplus > 201402L diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/DeclareConverters.h b/bindings/pyroot/cppyy/CPyCppyy/src/DeclareConverters.h index 609e130e2fc0b..5d26c9e569a59 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/DeclareConverters.h +++ b/bindings/pyroot/cppyy/CPyCppyy/src/DeclareConverters.h @@ -72,6 +72,11 @@ protected: \ CPPYY_DECLARE_BASIC_CONVERTER(Long); CPPYY_DECLARE_BASIC_CONVERTER(Bool); CPPYY_DECLARE_BASIC_CONVERTER(Char); +class SCharAsIntConverter : public CharConverter { +public: + using CharConverter::CharConverter; + virtual PyObject* FromMemory(void*); +}; CPPYY_DECLARE_BASIC_CONVERTER(UChar); class UCharAsIntConverter : public UCharConverter { public: diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/DeclareExecutors.h b/bindings/pyroot/cppyy/CPyCppyy/src/DeclareExecutors.h index f8167762f533b..0b4d81000c637 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/DeclareExecutors.h +++ b/bindings/pyroot/cppyy/CPyCppyy/src/DeclareExecutors.h @@ -63,6 +63,7 @@ public: \ } CPPYY_ARRAY_DECL_EXEC(Void); CPPYY_ARRAY_DECL_EXEC(Bool); +CPPYY_ARRAY_DECL_EXEC(SChar); CPPYY_ARRAY_DECL_EXEC(UChar); #if __cplusplus > 201402L CPPYY_ARRAY_DECL_EXEC(Byte); @@ -115,8 +116,6 @@ class InstanceExecutor : public Executor { class IteratorExecutor : public InstanceExecutor { public: IteratorExecutor(Cppyy::TCppType_t klass); - virtual PyObject* Execute( - Cppyy::TCppMethod_t, Cppyy::TCppObject_t, CallContext*); }; CPPYY_DECL_EXEC(Constructor); diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/Executors.cxx b/bindings/pyroot/cppyy/CPyCppyy/src/Executors.cxx index dd6b71d1d504e..5e948467718b9 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/Executors.cxx +++ b/bindings/pyroot/cppyy/CPyCppyy/src/Executors.cxx @@ -508,6 +508,7 @@ PyObject* CPyCppyy::name##ArrayExecutor::Execute( \ } CPPYY_IMPL_ARRAY_EXEC(Bool, bool, ) +CPPYY_IMPL_ARRAY_EXEC(SChar, signed char, ) CPPYY_IMPL_ARRAY_EXEC(UChar, unsigned char, ) #if __cplusplus > 201402L CPPYY_IMPL_ARRAY_EXEC(Byte, std::byte, ) @@ -632,24 +633,9 @@ PyObject* CPyCppyy::InstanceExecutor::Execute( CPyCppyy::IteratorExecutor::IteratorExecutor(Cppyy::TCppType_t klass) : InstanceExecutor(klass) { - fFlags |= CPPInstance::kNoWrapConv; // adds to flags from base class + fFlags |= CPPInstance::kNoMemReg | CPPInstance::kNoWrapConv; // adds to flags from base class } -//---------------------------------------------------------------------------- -PyObject* CPyCppyy::IteratorExecutor::Execute( - Cppyy::TCppMethod_t method, Cppyy::TCppObject_t self, CallContext* ctxt) -{ - PyObject* iter = this->InstanceExecutor::Execute(method, self, ctxt); - if (iter && ctxt->fPyContext) { - // set life line to tie iterator life time to the container (which may - // be a temporary) - std::ostringstream attr_name; - attr_name << "__" << (intptr_t)iter; - if (PyObject_SetAttrString(ctxt->fPyContext, (char*)attr_name.str().c_str(), iter)) - PyErr_Clear(); - } - return iter; -} //---------------------------------------------------------------------------- PyObject* CPyCppyy::InstanceRefExecutor::Execute( @@ -1090,7 +1076,8 @@ struct InitExecFactories_t { gf["const char*&"] = (ef_t)+[](cdims_t) { static CStringRefExecutor e{}; return &e; }; gf["char*&"] = gf["const char*&"]; gf["const signed char*"] = gf["const char*"]; - gf["signed char*"] = gf["char*"]; + //gf["signed char*"] = gf["char*"]; + gf["signed char ptr"] = (ef_t)+[](cdims_t d) { return new SCharArrayExecutor{d}; }; gf["wchar_t*"] = (ef_t)+[](cdims_t) { static WCStringExecutor e{}; return &e;}; gf["char16_t*"] = (ef_t)+[](cdims_t) { static CString16Executor e{}; return &e;}; gf["char32_t*"] = (ef_t)+[](cdims_t) { static CString32Executor e{}; return &e;}; diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/LowLevelViews.cxx b/bindings/pyroot/cppyy/CPyCppyy/src/LowLevelViews.cxx index 0f31cb8bfa58a..29917d684431d 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/LowLevelViews.cxx +++ b/bindings/pyroot/cppyy/CPyCppyy/src/LowLevelViews.cxx @@ -998,7 +998,7 @@ template<> struct typecode_traits { template<> struct typecode_traits { static constexpr const char* format = "b"; static constexpr const char* name = "char"; }; template<> struct typecode_traits { - static constexpr const char* format = "b"; static constexpr const char* name = "signed char"; }; + static constexpr const char* format = "b"; static constexpr const char* name = "SCharAsInt"; }; template<> struct typecode_traits { static constexpr const char* format = "B"; static constexpr const char* name = "UCharAsInt"; }; #if __cplusplus > 201402L diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/MemoryRegulator.cxx b/bindings/pyroot/cppyy/CPyCppyy/src/MemoryRegulator.cxx index 37cc394a5c23d..8103c1cb1357a 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/MemoryRegulator.cxx +++ b/bindings/pyroot/cppyy/CPyCppyy/src/MemoryRegulator.cxx @@ -206,7 +206,7 @@ bool CPyCppyy::MemoryRegulator::UnregisterPyObject(CPPInstance* pyobj, PyObject* if (!(pyobj && pyclass)) return false; - Cppyy::TCppObject_t cppobj = pyobj->GetObject(); + Cppyy::TCppObject_t cppobj = pyobj->IsSmart() ? pyobj->GetObjectRaw() : pyobj->GetObject(); if (!cppobj) return false; diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/ProxyWrappers.cxx b/bindings/pyroot/cppyy/CPyCppyy/src/ProxyWrappers.cxx index 0c1c0e002ecfd..037660498ed7e 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/ProxyWrappers.cxx +++ b/bindings/pyroot/cppyy/CPyCppyy/src/ProxyWrappers.cxx @@ -834,23 +834,13 @@ PyObject* CPyCppyy::BindCppObjectNoCast(Cppyy::TCppObject_t address, if (!pyclass) return nullptr; // error has been set in CreateScopeProxy - bool isRef = flags & CPPInstance::kIsReference; - bool isValue = flags & CPPInstance::kIsValue; + bool noReg = flags & (CPPInstance::kNoMemReg|CPPInstance::kNoWrapConv); + bool isRef = flags & CPPInstance::kIsReference; + void* r_address = isRef ? (address ? *(void**)address : nullptr) : address; -// TODO: make sure that a consistent address is used (may have to be done in BindCppObject) - if (address && !isValue /* always fresh */ && !(flags & (CPPInstance::kNoWrapConv|CPPInstance::kNoMemReg))) { - PyObject* oldPyObject = MemoryRegulator::RetrievePyObject( - isRef ? *(void**)address : address, pyclass); - - // ptr-ptr requires old object to be a reference to enable re-use - if (oldPyObject && (!(flags & CPPInstance::kIsPtrPtr) || - ((CPPInstance*)oldPyObject)->fFlags & CPPInstance::kIsReference)) { - return oldPyObject; - } - } - -// if smart, instantiate a Python-side object of the underlying type, carrying the smartptr - PyObject* smart_type = (flags != CPPInstance::kNoWrapConv && (((CPPClass*)pyclass)->fFlags & CPPScope::kIsSmart)) ? pyclass : nullptr; +// check whether the object to be bound is a smart pointer that needs embedding + PyObject* smart_type = (!(flags & CPPInstance::kNoWrapConv) && \ + (((CPPClass*)pyclass)->fFlags & CPPScope::kIsSmart)) ? pyclass : nullptr; if (smart_type) { pyclass = CreateScopeProxy(((CPPSmartClass*)smart_type)->fUnderlyingType); if (!pyclass) { @@ -860,6 +850,29 @@ PyObject* CPyCppyy::BindCppObjectNoCast(Cppyy::TCppObject_t address, } } +// TODO: make sure that a consistent address is used (may have to be done in BindCppObject) + if (address && !(flags & CPPInstance::kIsValue) /* always fresh */ && !noReg) { + PyObject* oldPyObject = MemoryRegulator::RetrievePyObject(r_address, pyclass); + + // embedded smart pointers are registered with the class of the underlying type, b/c + // there is no Python proxy for the smart pointer itself in that case; check whether + // the result found matches the smart-ness and if so, whether the smart pointer found + // is the same as the one requested (note: ptr-ptr requires old object to be a + // reference to enable re-use) + if (oldPyObject) { + CPPInstance* o_pyobj = ((CPPInstance*)oldPyObject); + + if ((bool)smart_type == o_pyobj->IsSmart() && \ + r_address == (smart_type ? o_pyobj->GetObjectRaw() : o_pyobj->GetObject()) && \ + (!(flags & CPPInstance::kIsPtrPtr) || (o_pyobj->fFlags & CPPInstance::kIsReference))) { + Py_DECREF(pyclass); + return oldPyObject; + } else { // reference or naked v.s. smart pointer mismatch + Py_DECREF(oldPyObject); + } + } + } + // instantiate an object of this class PyObject* args = PyTuple_New(0); CPPInstance* pyobj = @@ -875,9 +888,10 @@ PyObject* CPyCppyy::BindCppObjectNoCast(Cppyy::TCppObject_t address, if (smart_type) pyobj->SetSmart(smart_type); - // do not register null pointers, references (?), or direct usage of smart pointers or iterators - if (address && !isRef && !(flags & (CPPInstance::kNoWrapConv|CPPInstance::kNoMemReg))) - MemoryRegulator::RegisterPyObject(pyobj, pyobj->GetObject()); + // do not register null pointers, references (regulated objects can be referenced, but + // referenced objects can not be regulated), or direct usage of smart pointers or iterators + if (address && !isRef && !noReg) + MemoryRegulator::RegisterPyObject(pyobj, smart_type ? pyobj->GetObjectRaw() : pyobj->GetObject()); } // successful completion; wrap exception options to make them raiseable, normal return otherwise diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/Pythonize.cxx b/bindings/pyroot/cppyy/CPyCppyy/src/Pythonize.cxx index 8559b2ebfe7ff..cb07f80dd6388 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/Pythonize.cxx +++ b/bindings/pyroot/cppyy/CPyCppyy/src/Pythonize.cxx @@ -1237,7 +1237,7 @@ PyObject* STLStringDecode(CPPInstance* self, PyObject* args, PyObject* kwds) return nullptr; char* keywords[] = {(char*)"encoding", (char*)"errors", (char*)nullptr}; - const char* encoding; const char* errors; + const char* encoding = nullptr; const char* errors = nullptr; if (!PyArg_ParseTupleAndKeywords(args, kwds, const_cast("s|s"), keywords, &encoding, &errors)) return nullptr; diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/Utility.cxx b/bindings/pyroot/cppyy/CPyCppyy/src/Utility.cxx index 03eda4eaaedc5..7cfb76032a332 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/Utility.cxx +++ b/bindings/pyroot/cppyy/CPyCppyy/src/Utility.cxx @@ -122,6 +122,10 @@ namespace { } } initOperatorMapping_; + inline std::string full_scope(const std::string& tpname) { + return tpname[0] == ':' ? tpname : "::"+tpname; + } + } // unnamed namespace @@ -484,7 +488,7 @@ static bool AddTypeName(std::string& tmpl_name, PyObject* tn, PyObject* arg, } if (CPPScope_Check(tn)) { - tmpl_name.append(Cppyy::GetScopedFinalName(((CPPClass*)tn)->fCppType)); + tmpl_name.append(full_scope(Cppyy::GetScopedFinalName(((CPPClass*)tn)->fCppType))); if (arg) { // try to specialize the type match for the given object CPPInstance* pyobj = (CPPInstance*)arg; @@ -505,7 +509,7 @@ static bool AddTypeName(std::string& tmpl_name, PyObject* tn, PyObject* arg, } if (tn == (PyObject*)&CPPOverload_Type) { - PyObject* tpName = arg ? \ + PyObject* tpName = arg ? \ PyObject_GetAttr(arg, PyStrings::gCppName) : \ CPyCppyy_PyText_FromString("void* (*)(...)"); tmpl_name.append(CPyCppyy_PyText_AsString(tpName)); @@ -529,7 +533,7 @@ static bool AddTypeName(std::string& tmpl_name, PyObject* tn, PyObject* arg, for (Py_ssize_t i = 0; i < (PyList_GET_SIZE(values)-1); ++i) { if (i) tpn << ", "; PyObject* item = PyList_GET_ITEM(values, i); - tpn << (CPPScope_Check(item) ? ClassName(item) : AnnotationAsText(item)); + tpn << (CPPScope_Check(item) ? full_scope(ClassName(item)) : AnnotationAsText(item)); } Py_DECREF(values); @@ -547,7 +551,8 @@ static bool AddTypeName(std::string& tmpl_name, PyObject* tn, PyObject* arg, PyObject* tpName = PyObject_GetAttr(arg, PyStrings::gCppName); if (tpName) { - tmpl_name.append(CPyCppyy_PyText_AsString(tpName)); + const char* cname = CPyCppyy_PyText_AsString(tpName); + tmpl_name.append(CPPScope_Check(arg) ? full_scope(cname) : cname); Py_DECREF(tpName); return true; } @@ -1017,7 +1022,11 @@ std::string CPyCppyy::Utility::ClassName(PyObject* pyobj) static std::set sIteratorTypes; bool CPyCppyy::Utility::IsSTLIterator(const std::string& classname) { -// attempt to recognize STL iterators (TODO: probably belongs in the backend) +// attempt to recognize STL iterators (TODO: probably belongs in the backend), using +// a couple of common container classes with different iterator protocols (note that +// mapping iterators are handled separately in the pythonizations) as exemplars (the +// actual, resolved, names will be compiler-specific) that are picked b/c they are +// baked into the CoreLegacy dictionary if (sIteratorTypes.empty()) { std::string tt = "::"; for (auto c : {"std::vector", "std::list", "std::deque"}) { diff --git a/bindings/pyroot/cppyy/cppyy/LICENSE.txt b/bindings/pyroot/cppyy/cppyy/LICENSE.txt index 60d53483fb73f..512f66c87d496 100644 --- a/bindings/pyroot/cppyy/cppyy/LICENSE.txt +++ b/bindings/pyroot/cppyy/cppyy/LICENSE.txt @@ -52,6 +52,7 @@ source code): Aditi Dutta Shaheed Haque Aaron Jomy + Jonas Rembser Toby StClere-Smithe Stefan Wunsch diff --git a/bindings/pyroot/cppyy/cppyy/doc/source/changelog.rst b/bindings/pyroot/cppyy/cppyy/doc/source/changelog.rst index 033f819e17f4f..127358fb609b1 100644 --- a/bindings/pyroot/cppyy/cppyy/doc/source/changelog.rst +++ b/bindings/pyroot/cppyy/cppyy/doc/source/changelog.rst @@ -10,8 +10,8 @@ See :doc:`packages `, for details on the package structure. PyPy support lags CPython support. -master ------- +2024-12-16: 3.5.0 +----------------- * Fix buffering problems with std::string_view's on Python str objects * Fix potential buffering problems in creation of initializer lists @@ -25,6 +25,10 @@ master * Extend API to define executor and converter aliases * Use importlib.metadata instead of pkg_resources for py3.11 and later * Added out-of-bounds handling for small char-based enums +* Fixed a leak check in the generic STL iterator protocol +* Represent arrays of ``signed char`` as low level views returning bytes +* Improve memory regulator's handling of smart pointers +* Version PCHs with the C++ standard version * Fixes for py3.12 and py3.13 * Upgrade backend to Clang16 diff --git a/bindings/pyroot/cppyy/cppyy/installer/cppyy_monkey_patch.py b/bindings/pyroot/cppyy/cppyy/installer/cppyy_monkey_patch.py index 206822082c8d3..697eff678b7de 100644 --- a/bindings/pyroot/cppyy/cppyy/installer/cppyy_monkey_patch.py +++ b/bindings/pyroot/cppyy/cppyy/installer/cppyy_monkey_patch.py @@ -18,7 +18,7 @@ def get_requires_for_build_wheel(*args, **kwds): try: import __pypy__, sys version = sys.pypy_version_info - requirements = ['cppyy-cling==6.30.0'] + requirements = ['cppyy-cling==6.32.8'] if version[0] == 5: if version[1] <= 9: requirements = ['cppyy-cling<6.12'] @@ -32,7 +32,7 @@ def get_requires_for_build_wheel(*args, **kwds): requirements = ['cppyy-cling<=6.18.2.3'] except ImportError: # CPython - requirements = ['cppyy-backend==1.15.2', 'cppyy-cling==6.30.0'] + requirements = ['cppyy-backend==1.15.3', 'cppyy-cling==6.32.8'] return requirements + _get_requires_for_build_wheel(*args, **kwds) diff --git a/bindings/pyroot/cppyy/cppyy/python/cppyy/__init__.py b/bindings/pyroot/cppyy/cppyy/python/cppyy/__init__.py index a7456a0c8b124..976afdb7da770 100644 --- a/bindings/pyroot/cppyy/cppyy/python/cppyy/__init__.py +++ b/bindings/pyroot/cppyy/cppyy/python/cppyy/__init__.py @@ -50,21 +50,25 @@ 'set_debug', # enable/disable debug output ] -from ._version import __version__ - -import ctypes, os, sys, sysconfig, warnings +import ctypes +import os +import sys +import sysconfig +import warnings if not 'CLING_STANDARD_PCH' in os.environ: def _set_pch(): try: import cppyy_backend as cpb - local_pch = os.path.join(os.path.dirname(__file__), 'allDict.cxx.pch.'+str(cpb.__version__)) + local_pch = os.path.join( + os.path.dirname(__file__), 'allDict.cxx.pch.'+str(cpb.__version__)) if os.path.exists(local_pch): os.putenv('CLING_STANDARD_PCH', local_pch) os.environ['CLING_STANDARD_PCH'] = local_pch except (ImportError, AttributeError): pass - _set_pch(); del _set_pch + _set_pch() + del _set_pch try: import __pypy__ @@ -73,6 +77,9 @@ def _set_pch(): except ImportError: ispypy = False +from . import _typemap +from ._version import __version__ + # import separately instead of in the above try/except block for easier to # understand tracebacks if ispypy: @@ -87,7 +94,6 @@ def _set_pch(): #- external typemap ---------------------------------------------------------- -from . import _typemap _typemap.initialize(_backend) # also creates (u)int8_t mapper try: @@ -166,7 +172,7 @@ def __getitem__(self, cls): return py_make_smartptr(cls, self.ptrcls) except AttributeError: pass - if type(cls) == str and not cls in ('int', 'float'): + if isinstance(cls, str) and not cls in ('int', 'float'): return py_make_smartptr(getattr(gbl, cls), self.ptrcls) return self.maker[cls] @@ -178,27 +184,38 @@ def __getitem__(self, cls): #--- interface to Cling ------------------------------------------------------ class _stderr_capture(object): def __init__(self): - self._capture = not gbl.gDebug and True or False - self.err = "" + self._capture = not gbl.gDebug and True or False + self.err = "" def __enter__(self): if self._capture: - _begin_capture_stderr() + _begin_capture_stderr() return self def __exit__(self, tp, val, trace): if self._capture: self.err = _end_capture_stderr() +def _cling_report(msg, errcode, msg_is_error=False): + # errcode should be authorative, but at least on MacOS, Cling does not report an + # error when it should, so also check for the typical compilation signature that + # Cling puts out as an indicator than an error occurred + if 'input_line' in msg: + if 'warning' in msg and not 'error' in msg: + warnings.warn(msg, SyntaxWarning) + msg_is_error=False + + if 'error' in msg: + errcode = 1 + + if errcode or (msg and msg_is_error): + raise SyntaxError('Failed to parse the given C++ code%s' % msg) + def cppdef(src): """Declare C++ source to Cling.""" with _stderr_capture() as err: errcode = gbl.gInterpreter.Declare(src) - if not errcode or err.err: - if 'warning' in err.err.lower() and not 'error' in err.err.lower(): - warnings.warn(err.err, SyntaxWarning) - return True - raise SyntaxError('Failed to parse the given C++ code%s' % err.err) + _cling_report(err.err, int(not errcode), msg_is_error=True) return True def cppexec(stmt): @@ -214,11 +231,11 @@ def cppexec(stmt): gbl.gInterpreter.ProcessLine(stmt, ctypes.pointer(errcode)) except Exception as e: sys.stderr.write("%s\n\n" % str(e)) - if not errcode.value: errcode.value = 1 + if not errcode.value: + errcode.value = 1 - if errcode.value: - raise SyntaxError('Failed to parse the given C++ code%s' % err.err) - elif err.err and err.err[1:] != '\n': + _cling_report(err.err, errcode.value) + if err.err and err.err[1:] != '\n': sys.stderr.write(err.err[1:]) return True @@ -331,7 +348,8 @@ def add_library_path(path): if apipath_extra is None: ldversion = sysconfig.get_config_var('LDVERSION') - if not ldversion: ldversion = sys.version[:3] + if not ldversion: + ldversion = sys.version[:3] apipath_extra = os.path.join(os.path.dirname(apipath), 'site', 'python'+ldversion) if not os.path.exists(os.path.join(apipath_extra, 'CPyCppyy')): @@ -357,7 +375,9 @@ def add_library_path(path): if apipath_extra.lower() != 'none': if not os.path.exists(os.path.join(apipath_extra, 'CPyCppyy')): - warnings.warn("CPyCppyy API not found (tried: %s); set CPPYY_API_PATH envar to the 'CPyCppyy' API directory to fix" % apipath_extra) + warnings.warn("CPyCppyy API not found (tried: %s); " + "set CPPYY_API_PATH envar to the 'CPyCppyy' API directory to fix" + % apipath_extra) else: add_include_path(apipath_extra) @@ -366,15 +386,20 @@ def add_library_path(path): if os.getenv('CONDA_PREFIX'): # MacOS, Linux include_path = os.path.join(os.getenv('CONDA_PREFIX'), 'include') - if os.path.exists(include_path): add_include_path(include_path) + if os.path.exists(include_path): + add_include_path(include_path) # Windows include_path = os.path.join(os.getenv('CONDA_PREFIX'), 'Library', 'include') - if os.path.exists(include_path): add_include_path(include_path) + if os.path.exists(include_path): + add_include_path(include_path) -# assuming that we are in PREFIX/lib/python/site-packages/cppyy, add PREFIX/include to the search path -include_path = os.path.abspath(os.path.join(os.path.dirname(__file__), *(4*[os.path.pardir]+['include']))) -if os.path.exists(include_path): add_include_path(include_path) +# assuming that we are in PREFIX/lib/python/site-packages/cppyy, +# add PREFIX/include to the search path +include_path = os.path.abspath( + os.path.join(os.path.dirname(__file__), *(4*[os.path.pardir]+['include']))) +if os.path.exists(include_path): + add_include_path(include_path) del include_path, apipath, ispypy @@ -392,7 +417,7 @@ def set_debug(enable=True): gbl.gDebug = 0 def _get_name(tt): - if type(tt) == str: + if isinstance(tt, str): return tt try: ttname = tt.__cpp_name__ @@ -403,7 +428,7 @@ def _get_name(tt): _sizes = {} def sizeof(tt): """Returns the storage size (in chars) of C++ type .""" - if not isinstance(tt, type) and not type(tt) == str: + if not isinstance(tt, type) and not isinstance(tt, str): tt = type(tt) try: return _sizes[tt] @@ -434,8 +459,9 @@ def typeid(tt): def multi(*bases): # after six, see also _typemap.py """Resolve metaclasses for multiple inheritance.""" # contruct a "no conflict" meta class; the '_meta' is needed by convention - nc_meta = type.__new__(type, 'cppyy_nc_meta', tuple(type(b) for b in bases if type(b) is not type), {}) + nc_meta = type.__new__( + type, 'cppyy_nc_meta', tuple(type(b) for b in bases if type(b) is not type), {}) class faux_meta(type): - def __new__(cls, name, this_bases, d): + def __new__(mcs, name, this_bases, d): return nc_meta(name, bases, d) return type.__new__(faux_meta, 'faux_meta', (), {}) diff --git a/bindings/pyroot/cppyy/cppyy/python/cppyy/__pyinstaller/hook-cppyy.py b/bindings/pyroot/cppyy/cppyy/python/cppyy/__pyinstaller/hook-cppyy.py index 2e0a98b1f53b0..4819b6ab00b01 100644 --- a/bindings/pyroot/cppyy/cppyy/python/cppyy/__pyinstaller/hook-cppyy.py +++ b/bindings/pyroot/cppyy/cppyy/python/cppyy/__pyinstaller/hook-cppyy.py @@ -4,7 +4,7 @@ # # See also setup.cfg. -__all__ = ['data'] +__all__ = ['datas'] def _backend_files(): diff --git a/bindings/pyroot/cppyy/cppyy/python/cppyy/_cpython_cppyy.py b/bindings/pyroot/cppyy/cppyy/python/cppyy/_cpython_cppyy.py index 8d6f1e0f2c0fd..4245828387fa2 100644 --- a/bindings/pyroot/cppyy/cppyy/python/cppyy/_cpython_cppyy.py +++ b/bindings/pyroot/cppyy/cppyy/python/cppyy/_cpython_cppyy.py @@ -1,6 +1,9 @@ """ CPython-specific touch-ups """ +import ctypes +import sys + from . import _stdcpp_fix from cppyy_backend import loader @@ -23,12 +26,10 @@ _backend._cpp_backend = c # explicitly expose APIs from libcppyy -import ctypes _w = ctypes.CDLL(_backend.__file__, ctypes.RTLD_GLOBAL) # some beautification for inspect (only on p2) -import sys if sys.hexversion < 0x3000000: # TODO: this reliese on CPPOverload cooking up a func_code object, which atm # is simply not implemented for p3 :/ @@ -39,14 +40,14 @@ inspect._old_isfunction = inspect.isfunction def isfunction(object): - if type(object) == _backend.CPPOverload and not object.im_class: + if isinstance(object, _backend.CPPOverload) and not object.im_class: return True return inspect._old_isfunction(object) inspect.isfunction = isfunction inspect._old_ismethod = inspect.ismethod def ismethod(object): - if type(object) == _backend.CPPOverload: + if isinstance(object, _backend.CPPOverload): return True return inspect._old_ismethod(object) inspect.ismethod = ismethod @@ -70,7 +71,7 @@ def __repr__(self): def __getitem__(self, *args): # multi-argument to [] becomes a single tuple argument - if args and type(args[0]) is tuple: + if args and isinstance(args[0], tuple): args = args[0] # if already instantiated, return the existing class @@ -82,7 +83,7 @@ def __getitem__(self, *args): # construct the type name from the types or their string representation newargs = [self.__name__] for arg in args: - if type(arg) == str: + if isinstance(arg, str): arg = ','.join(map(lambda x: x.strip(), arg.split(','))) newargs.append(arg) pyclass = _backend.MakeCppTemplateClass(*newargs) @@ -95,11 +96,13 @@ def __getitem__(self, *args): if 'reserve' in pyclass.__dict__: def iadd(self, ll): self.reserve(len(ll)) - for x in ll: self.push_back(x) + for x in ll: + self.push_back(x) return self else: def iadd(self, ll): - for x in ll: self.push_back(x) + for x in ll: + self.push_back(x) return self pyclass.__iadd__ = iadd @@ -114,7 +117,7 @@ def __call__(self, *args): # most common cases are covered if args: args0 = args[0] - if args0 and (type(args0) is tuple or type(args0) is list): + if args0 and isinstance(args0, (tuple, list)): t = type(args0[0]) if t is float: t = 'double' @@ -125,7 +128,7 @@ def __call__(self, *args): if self.__name__ in self.stl_unrolled_types: return self[tuple(type(a) for a in args0)](*args0) - if args0 and type(args0) is dict: + if args0 and isinstance(args0, dict): if self.__name__ in self.stl_mapping_types: try: pair = args0.items().__iter__().__next__() @@ -167,7 +170,8 @@ def add_default_paths(): if os.path.exists(lib_path): gSystem.AddDynamicPath(lib_path) # assuming that we are in PREFIX/lib/python/site-packages/cppyy, add PREFIX/lib to the search path - lib_path = os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir, os.path.pardir)) + lib_path = os.path.abspath( + os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir, os.path.pardir)) if os.path.exists(lib_path): gSystem.AddDynamicPath(lib_path) try: diff --git a/bindings/pyroot/cppyy/cppyy/python/cppyy/_pypy_cppyy.py b/bindings/pyroot/cppyy/cppyy/python/cppyy/_pypy_cppyy.py index 78ad08a188ab6..1e57791a67680 100644 --- a/bindings/pyroot/cppyy/cppyy/python/cppyy/_pypy_cppyy.py +++ b/bindings/pyroot/cppyy/cppyy/python/cppyy/_pypy_cppyy.py @@ -3,7 +3,8 @@ from . import _stdcpp_fix -import os, sys +import os +import sys from cppyy_backend import loader __all__ = [ @@ -46,14 +47,13 @@ def fixup_legacy(): #- exports ------------------------------------------------------------------- -import sys _thismodule = sys.modules[__name__] for name in __all__: try: setattr(_thismodule, name, getattr(_backend, name)) except AttributeError: pass -del name, sys +del name nullptr = _backend.nullptr def load_reflection_info(name): diff --git a/bindings/pyroot/cppyy/cppyy/python/cppyy/_pythonization.py b/bindings/pyroot/cppyy/cppyy/python/cppyy/_pythonization.py index d49b90f8a06f8..f8b2a4d0f870f 100644 --- a/bindings/pyroot/cppyy/cppyy/python/cppyy/_pythonization.py +++ b/bindings/pyroot/cppyy/cppyy/python/cppyy/_pythonization.py @@ -1,6 +1,8 @@ """ Pythonization API. """ +import re + __all__ = [ 'add_pythonization', 'remove_pythonization', @@ -79,7 +81,6 @@ def __call__(self, obj): return delattr(obj, self.attr) def __init__(self, match_class, orig_attribute, new_attribute, keep_orig): - import re self.match_class = re.compile(match_class) self.match_attr = re.compile(orig_attribute) self.new_attr = new_attribute @@ -98,7 +99,6 @@ def __call__(self, obj, name): # def rename_attribute(match_class, orig_attribute, new_attribute, keep_orig=False): # class method_pythonizor: # def __init__(self, match_class, orig_attribute, new_attribute, keep_orig): -# import re # self.match_class = re.compile(match_class) # self.match_attr = re.compile(orig_attribute) # self.new_attr = new_attribute @@ -125,7 +125,6 @@ def __call__(self, obj, name): def add_overload(match_class, match_method, overload): class method_pythonizor(object): def __init__(self, match_class, match_method, overload): - import re self.match_class = re.compile(match_class) self.match_method = re.compile(match_method) self.overload = overload @@ -134,21 +133,20 @@ def __call__(self, obj, name): if not self.match_class.match(name): return for k in dir(obj): #.__dict__: - try: - tmp = getattr(obj, k) - except: - continue - if self.match_method.match(k): - try: - tmp.__add_overload__(overload) - except AttributeError: pass + try: + tmp = getattr(obj, k) + except AttributeError: + continue + if self.match_method.match(k): + try: + tmp.__add_overload__(overload) + except AttributeError: pass return method_pythonizor(match_class, match_method, overload) def compose_method(match_class, match_method, g): class composition_pythonizor(object): def __init__(self, match_class, match_method, g): - import re self.match_class = re.compile(match_class) self.match_method = re.compile(match_method) self.g = g @@ -162,7 +160,7 @@ def __call__(self, obj, name): continue try: f = getattr(obj, k) - except: + except AttributeError: continue def make_fun(f, g): def h(self, *args, **kwargs): @@ -176,7 +174,6 @@ def h(self, *args, **kwargs): def set_method_property(match_class, match_method, prop, value): class method_pythonizor(object): def __init__(self, match_class, match_method, prop, value): - import re self.match_class = re.compile(match_class) self.match_method = re.compile(match_method) self.prop = prop @@ -188,7 +185,7 @@ def __call__(self, obj, name): for k in dir(obj): #.__dict__: try: tmp = getattr(obj, k) - except: + except AttributeError: continue if self.match_method.match(k): setattr(tmp, self.prop, self.value) @@ -198,7 +195,6 @@ def __call__(self, obj, name): def make_property(match_class, match_get, match_set=None, match_del=None, prop_name=None): class property_pythonizor(object): def __init__(self, match_class, match_get, match_set, match_del, prop_name): - import re self.match_class = re.compile(match_class) self.match_get = re.compile(match_get) @@ -222,10 +218,14 @@ def __init__(self, match_class, match_get, match_set, match_del, prop_name): self.match_many = match_many_getters if not (self.match_many or prop_name): - raise ValueError("If not matching properties by regex, need a property name with exactly one substitution field") + raise ValueError( + "If not matching properties by regex, " + "need a property name with exactly one substitution field") if self.match_many and prop_name: if prop_name.format(').!:(') == prop_name: - raise ValueError("If matching properties by regex and providing a property name, the name needs exactly one substitution field") + raise ValueError( + "If matching properties by regex and providing a property name, " + "the name needs exactly one substitution field") self.prop_name = prop_name @@ -263,7 +263,7 @@ def __call__(self, obj, name): match = self.match_get.match(k) try: tmp = getattr(obj, k) - except: + except AttributeError: continue if match and hasattr(tmp, '__call__'): if self.match_many: @@ -278,7 +278,7 @@ def __call__(self, obj, name): match = self.match_set.match(k) try: tmp = getattr(obj, k) - except: + except AttributeError: continue if match and hasattr(tmp, '__call__'): if self.match_many: @@ -293,7 +293,7 @@ def __call__(self, obj, name): match = self.match_del.match(k) try: tmp = getattr(obj, k) - except: + except AttributeError: continue if match and hasattr(tmp, '__call__'): if self.match_many: @@ -313,7 +313,6 @@ def __call__(self, obj, name): names += list(named_deleters.keys()) names = set(names) - properties = [] for name in names: if name in named_getters: fget = self.make_get_del_proxy(named_getters[name]) diff --git a/bindings/pyroot/cppyy/cppyy/python/cppyy/_typemap.py b/bindings/pyroot/cppyy/cppyy/python/cppyy/_typemap.py index 3aefbf4eee797..fd09510c4af8f 100644 --- a/bindings/pyroot/cppyy/cppyy/python/cppyy/_typemap.py +++ b/bindings/pyroot/cppyy/cppyy/python/cppyy/_typemap.py @@ -2,7 +2,9 @@ for typedef-ed C++ builtin types. """ +import ctypes import sys +import types def _create_mapper(cls, extra_dct=None): def mapper(name, scope): @@ -13,7 +15,8 @@ def mapper(name, scope): cppname = name modname = 'cppyy.gbl' dct = {'__cpp_name__' : cppname, '__module__' : modname} - if extra_dct: dct.update(extra_dct) + if extra_dct: + dct.update(extra_dct) return type(name, (cls,), dct) return mapper @@ -46,7 +49,15 @@ def with_metaclass(meta, *bases): class metaclass(type): def __new__(cls, name, this_bases, d): - return meta(name, bases, d) + if sys.version_info[:2] >= (3, 7): + # This version introduced PEP 560 that requires a bit + # of extra care (we mimic what is done by __build_class__). + resolved_bases = types.resolve_bases(bases) + if resolved_bases is not bases: + d['__orig_bases__'] = bases + else: + resolved_bases = bases + return meta(name, resolved_bases, d) @classmethod def __prepare__(cls, name, this_bases): @@ -55,9 +66,10 @@ def __prepare__(cls, name, this_bases): # --- end from six.py class _BoolMeta(type): - def __call__(self, val = bool()): - if val: return True - else: return False + def __call__(cls, val = bool()): + if val: + return True + return False class _Bool(with_metaclass(_BoolMeta, object)): pass @@ -99,7 +111,6 @@ def initialize(backend): tm[tp] = float_tm # void* - import ctypes def voidp_init(self, arg=0): import cppyy, ctypes if arg == cppyy.nullptr: arg = 0 diff --git a/bindings/pyroot/cppyy/cppyy/python/cppyy/_version.py b/bindings/pyroot/cppyy/cppyy/python/cppyy/_version.py index f71b21a5dd538..01bd03cec6483 100644 --- a/bindings/pyroot/cppyy/cppyy/python/cppyy/_version.py +++ b/bindings/pyroot/cppyy/cppyy/python/cppyy/_version.py @@ -1 +1 @@ -__version__ = '3.1.2' +__version__ = '3.5.0' diff --git a/bindings/pyroot/cppyy/cppyy/python/cppyy/interactive.py b/bindings/pyroot/cppyy/cppyy/python/cppyy/interactive.py index a9be6c3ec282b..0b4bdd7ce471f 100644 --- a/bindings/pyroot/cppyy/cppyy/python/cppyy/interactive.py +++ b/bindings/pyroot/cppyy/cppyy/python/cppyy/interactive.py @@ -13,7 +13,8 @@ def __init__(self, hook_okay): self._hook_okay = hook_okay def __getattr__(self, attr): - import cppyy, sys + import cppyy + if attr == '__all__': # copy all exported items from cppyy itself for v in cppyy.__all__: @@ -25,10 +26,9 @@ def __getattr__(self, attr): caller = sys.modules[sys._getframe(1).f_globals['__name__']] cppyy._backend._set_cpp_lazy_lookup(caller.__dict__) return cppyy.__all__ - else: - self.__dict__['g'] = cppyy.gbl - self.__dict__['std'] = cppyy.gbl.std - return ['g', 'std']+cppyy.__all__ + self.__dict__['g'] = cppyy.gbl + self.__dict__['std'] = cppyy.gbl.std + return ['g', 'std']+cppyy.__all__ return getattr(cppyy, attr) sys.modules['cppyy.interactive'] = InteractiveLazy(\ diff --git a/bindings/pyroot/cppyy/cppyy/python/cppyy/ll.py b/bindings/pyroot/cppyy/cppyy/python/cppyy/ll.py index 49f5f04f04edb..9fb6034ce0843 100644 --- a/bindings/pyroot/cppyy/cppyy/python/cppyy/ll.py +++ b/bindings/pyroot/cppyy/cppyy/python/cppyy/ll.py @@ -36,11 +36,12 @@ # convenience functions to create C-style argv/argc def argv(): - argc = len(sys.argv) + """Return C's argv for use with cppyy/ctypes.""" cargsv = (ctypes.c_char_p * len(sys.argv))(*(x.encode() for x in sys.argv)) return ctypes.POINTER(ctypes.c_char_p)(cargsv) def argc(): + """Return C's argc for use with cppyy/ctypes.""" return len(sys.argv) # import low-level python converters @@ -91,7 +92,8 @@ def __call__(self, size, managed=False): res = self.func[self.array_type](size) try: res.reshape((size,)+res.shape[1:]) - if managed: res.__python_owns__ = True + if managed: + res.__python_owns__ = True except AttributeError: res.__reshape__((size,)) if managed: diff --git a/bindings/pyroot/cppyy/cppyy/python/cppyy/numba_ext.py b/bindings/pyroot/cppyy/cppyy/python/cppyy/numba_ext.py index 980dbdae9125d..a83985b909094 100644 --- a/bindings/pyroot/cppyy/cppyy/python/cppyy/numba_ext.py +++ b/bindings/pyroot/cppyy/cppyy/python/cppyy/numba_ext.py @@ -17,9 +17,7 @@ from llvmlite import ir from numba.extending import make_attribute_wrapper -import itertools import re -import inspect # setuptools entry point for Numba def _init_extension(): @@ -72,7 +70,7 @@ def resolve_const_types(val): return re.match(r'const\s+(.+)\s*\*', val).group(1) def cpp2numba(val): - if type(val) != str: + if not isinstance(val, str): # TODO: distinguish ptr/ref/byval # TODO: Only metaclasses/proxies end up here since # ref cases makes the RETURN_TYPE from reflex a string @@ -114,7 +112,7 @@ def numba2cpp(val): def numba_arg_convertor(args): args_cpp = [] - for i, arg in enumerate(list(args)): + for arg in list(args): # If the user explicitly passes an argument using numba CPointer, the regex match is used # to detect the pass by reference since the dispatcher always returns typeref[val*] match = re.search(r"typeref\[(.*?)\*\]", str(arg)) @@ -128,7 +126,7 @@ def numba_arg_convertor(args): def to_ref(type_list): ref_list = [] - for i, l in enumerate(type_list): + for l in type_list: ref_list.append(l + '&') return ref_list @@ -165,9 +163,8 @@ def cpp2ir(val): elif val != "char*" and val[-1] == "*": if val.startswith('const'): return ir.PointerType(cpp2ir(resolve_const_types(val))) - else: - type_2 = _cpp2ir[val[:-1]] - return ir.PointerType(type_2) + type_2 = _cpp2ir[val[:-1]] + return ir.PointerType(type_2) # # C++ function pointer -> Numba @@ -197,7 +194,8 @@ def get_call_type(self, context, args, kwds): except KeyError: pass - ol = CppFunctionNumbaType(self._func.__overload__(numba_arg_convertor(args)), self._is_method) + ol = CppFunctionNumbaType( + self._func.__overload__(numba_arg_convertor(args)), self._is_method) thistype = None if self._is_method: @@ -222,7 +220,8 @@ def get_call_type(self, context, args, kwds): @nb_iutils.lower_builtin(ol, *args) def lower_external_call(context, builder, sig, args, - ty=nb_types.ExternalFunctionPointer(extsig, ol.get_pointer), pyval=self._func, is_method=self._is_method): + ty=nb_types.ExternalFunctionPointer(extsig, ol.get_pointer), + pyval=self._func, is_method=self._is_method): ptrty = context.get_function_pointer_type(ty) ptrval = context.add_dynamic_addr( builder, ty.get_pointer(pyval), info=str(pyval)) @@ -237,9 +236,11 @@ def get_call_signatures(self): def get_impl_key(self, sig): return self._impl_keys[sig.args] - #TODO : Remove the redundancy of __overload__ matching and use this function to only obtain the address given the matched overload + # TODO: Remove the redundancy of __overload__ matching and use this function + # to only obtain the address given the matched overload def get_pointer(self, func): - if func is None: func = self._func + if func is None: + func = self._func ol = func.__overload__(numba_arg_convertor(self.sig.args)) @@ -340,14 +341,14 @@ def generic_resolve(self, typ, attr): try: f = getattr(typ._scope, attr) - if type(f) == cpp_types.Function: + if isinstance(f, cpp_types.Function): ft = CppFunctionNumbaType(f, is_method=True) except AttributeError: pass try: f = typ._scope.__dict__[attr] - if type(f) == cpp_types.DataMember: + if isinstance(f, cpp_types.DataMember): ct = f.__cpp_reflex__(cpp_refl.TYPE) ft = cpp2numba(ct) except AttributeError: @@ -363,7 +364,7 @@ def cppclass_getattr_impl(context, builder, typ, val, attr): # TODO: the following relies on the fact that numba will first lower the # field access, then immediately lower the call; and that the `val` loads # the struct representing the C++ object. Neither need be stable. - if attr in typ._scope.__dict__ and type(typ._scope.__dict__[attr]) == cpp_types.DataMember: + if attr in typ._scope.__dict__ and isinstance(typ._scope.__dict__[attr], cpp_types.DataMember): dm = typ._scope.__dict__[attr] ct = dm.__cpp_reflex__(cpp_refl.TYPE) offset = dm.__cpp_reflex__(cpp_refl.OFFSET) @@ -435,7 +436,8 @@ def get_data_type(self): # struct is split in a series of byte members to get the total size right # and to allow addressing at the correct offsets. if self._data_type is None: - self._data_type = ir.LiteralStructType([ir_byte for i in range(self._sizeof)], packed=True) + self._data_type = \ + ir.LiteralStructType([ir_byte for i in range(self._sizeof)], packed=True) return self._data_type # return: representation used for return argument. @@ -482,11 +484,11 @@ class ImplClassType(CppClassNumbaType): member_methods = dict() for name, field in val.__dict__.items(): - if type(field) == cpp_types.DataMember: + if isinstance(field, cpp_types.DataMember): data_members.append(CppDataMemberInfo( name, field.__cpp_reflex__(cpp_refl.OFFSET), field.__cpp_reflex__(cpp_refl.TYPE)) ) - elif type(field) == cpp_types.Function: + elif isinstance(field, cpp_types.Function): member_methods[name] = field.__cpp_reflex__(cpp_refl.RETURN_TYPE) # TODO: this refresh is needed b/c the scope type is registered as a @@ -521,13 +523,13 @@ def traverse_types(self): # value: representation inside function body. Maybe stored in stack. # The representation here are flexible. def get_value_type(self): - # the C++ object, b/c through a proxy, is always accessed by pointer; it is represented - # as a pointer to POD to allow indexing by Numba for data member type checking, but the - # address offsetting for loading data member values is independent (see get(), below), - # so the exact layout need not match a POD + # the C++ object, b/c through a proxy, is always accessed by pointer; it is + # represented as a pointer to POD to allow indexing by Numba for data member + # type checking, but the address offsetting for loading data member values is + # independent (see get(), below), so the exact layout need not match a POD - # TODO: this doesn't work for real PODs, b/c those are unpacked into their elements and - # passed through registers + # TODO: this doesn't work for real PODs, b/c those are unpacked into their elements + # and passed through registers return ir.PointerType(super(ImplClassModel, self).get_value_type()) # argument: representation used for function argument. Needs to be builtin type, @@ -609,14 +611,13 @@ def box_instance(typ, val, c): global cppyy_from_voidptr - if type(val) == ir.Constant: + if isinstance(val, ir.Constant): if val.constant == ir.Undefined: assert not "Value passed to instance boxing is undefined" return NULL implclass = make_implclass(c.context, c.builder, typ) classobj = c.pyapi.unserialize(c.pyapi.serialize_object(cpp_types.Instance)) - pyobj = c.context.get_argument_type(nb_types.pyobject) box_list = [] @@ -630,7 +631,8 @@ def box_instance(typ, val, c): box_res = c.pyapi.call_function_objargs( classobj, tuple(box_list) ) - # Required for nopython mode, numba nrt requres each member box call to decref since it steals the reference + # Required for nopython mode, numba nrt requres each member box call to decref + # since it steals the reference for i in box_list: c.pyapi.decref(i) @@ -650,4 +652,4 @@ def typeof_instance(val, c): except KeyError: pass # Pass the val itself to obtain Cling address of the CPPInstance for reference to C++ objects - return typeof_scope(val, c, Qualified.instance) \ No newline at end of file + return typeof_scope(val, c, Qualified.instance) diff --git a/bindings/pyroot/cppyy/cppyy/setup.py b/bindings/pyroot/cppyy/cppyy/setup.py index ef9398a94ed21..0bb4ed7449b46 100755 --- a/bindings/pyroot/cppyy/cppyy/setup.py +++ b/bindings/pyroot/cppyy/cppyy/setup.py @@ -8,7 +8,7 @@ try: import __pypy__, sys version = sys.pypy_version_info - requirements = ['cppyy-backend==1.15.2', 'cppyy-cling==6.30.0'] + requirements = ['cppyy-backend==1.15.3', 'cppyy-cling==6.32.8'] if version[0] == 5: if version[1] <= 9: requirements = ['cppyy-backend<0.3', 'cppyy-cling<6.12'] @@ -23,7 +23,7 @@ requirements = ['cppyy-backend<=1.10', 'cppyy-cling<=6.18.2.3'] except ImportError: # CPython - requirements = ['CPyCppyy==1.12.16', 'cppyy-backend==1.15.2', 'cppyy-cling==6.30.0'] + requirements = ['CPyCppyy==1.13.0', 'cppyy-backend==1.15.3', 'cppyy-cling==6.32.8'] setup_requirements = ['wheel'] if 'build' in sys.argv or 'install' in sys.argv: diff --git a/bindings/pyroot/cppyy/cppyy/test/test_cpp11features.py b/bindings/pyroot/cppyy/cppyy/test/test_cpp11features.py index 986360c5a861e..0e7a208e3d658 100644 --- a/bindings/pyroot/cppyy/cppyy/test/test_cpp11features.py +++ b/bindings/pyroot/cppyy/cppyy/test/test_cpp11features.py @@ -490,3 +490,44 @@ class Inherit(cppyy.gbl.unique_ptr_data.Example): a = Inherit() # Test whether this attribute was inherited assert a.y == 66. + + def test18_unique_ptr_identity(self): + """std::unique_ptr identity preservation""" + + import cppyy + + cppyy.cppdef("""\ + namespace UniqueIdentity { + struct A { + A(int _a) : a(_a) {} + int a; + }; + + std::unique_ptr create() { return std::make_unique(37); } + + struct Consumer { + public: + Consumer(std::unique_ptr & ptr) : fPtr{std::move(ptr)} { + ptr.reset(); + } + + const A& get() const { return *fPtr; } + const std::unique_ptr& pget() const { return fPtr; } + + private: + std::unique_ptr fPtr; + }; }""") + + ns = cppyy.gbl.UniqueIdentity + + x = ns.create() + assert x.a == 37 + + c = ns.Consumer(x) + x = c.get() + assert x.a == 37 + + p1 = c.pget() + p2 = c.pget() + assert p1 is p2 + diff --git a/bindings/pyroot/cppyy/cppyy/test/test_leakcheck.py b/bindings/pyroot/cppyy/cppyy/test/test_leakcheck.py index a205d3e51a619..80cf3c2aba7ac 100644 --- a/bindings/pyroot/cppyy/cppyy/test/test_leakcheck.py +++ b/bindings/pyroot/cppyy/cppyy/test/test_leakcheck.py @@ -2,12 +2,6 @@ from pytest import mark, skip from .support import setup_make, pylong, pyunicode -currpath = py.path.local(__file__).dirpath() -test_dct = str(currpath.join("datatypesDict")) - -def setup_module(mod): - setup_make("datatypes") - nopsutil = False try: import psutil @@ -20,9 +14,6 @@ class TestLEAKCHECK: def setup_class(cls): import cppyy, psutil - cls.test_dct = test_dct - cls.memory = cppyy.load_reflection_info(cls.test_dct) - cls.process = psutil.Process(os.getpid()) def runit(self, N, scope, func, *args, **kwds): @@ -46,27 +37,41 @@ def check_func(self, scope, func, *args, **kwds): # function that is to be found on each call python-side tmpl_args = kwds.pop('tmpl_args', None) - # warmup function (TOOD: why doesn't once suffice?) - for i in range(8): # actually, 2 seems to be enough - if tmpl_args is None: - getattr(scope, func)(*args, **kwds) - else: - getattr(scope, func)[tmpl_args](*args, **kwds) + # warmup function + gc.collect() + if tmpl_args is None: + getattr(scope, func)(*args, **kwds) + else: + getattr(scope, func)[tmpl_args](*args, **kwds) # number of iterations N = 100000 + # The use of arena's, free-lists, etc. means that checking rss remains + # unreliable, unless looking for consistent jumps, so the leak check will + # be run M times and only considered failed if it "leaks" every time. In + # actual practice, the number of fails is 0, 1, or M. Note that the total + # number of gc objects tracked is always required to remain the same. + M = 3 + # leak check - gc.collect() - last = self.process.memory_info().rss + fail = 0 + for i in range(M): + gc.collect() + pre = len(gc.get_objects()) + last = self.process.memory_info().rss - if tmpl_args is None: - self.runit(N, scope, func, *args, **kwds) - else: - self.runit_template(N, scope, func, tmpl_args, *args, **kwds) + if tmpl_args is None: + self.runit(N, scope, func, *args, **kwds) + else: + self.runit_template(N, scope, func, tmpl_args, *args, **kwds) - gc.collect() - assert last == self.process.memory_info().rss + gc.collect() + assert len(gc.get_objects()) == pre + if last < self.process.memory_info().rss: + fail += 1 + + assert fail < M def test01_free_functions(self): """Leak test of free functions""" @@ -228,6 +233,7 @@ def test06_dir(self): self.check_func(cppyy.gbl, '__dir__', cppyy.gbl) def test07_string_handling(self): + """Leak check of returning an std::string by value""" import cppyy @@ -235,14 +241,34 @@ def test07_string_handling(self): namespace LeakCheck { class Leaker { public: - const std::string leak_string(std::size_t size) const { - std::string result; - result.reserve(size); - return result; - } + const std::string leak_string(std::size_t size) const { + std::string result; + result.reserve(size); + return result; + } }; }""") ns = cppyy.gbl.LeakCheck obj = ns.Leaker() self.check_func(obj, 'leak_string', 2048) + + def test08_list_creation(self): + """Leak check of creating a python list from an std::list""" + + import cppyy + + cppyy.cppdef("""\ + namespace LeakCheck { + std::list list_by_value() { return std::list(3); } + } """) + + ns = cppyy.gbl.LeakCheck + + def wrapped_list_by_value(): + return list(ns.list_by_value()) + + ns.leak_list = wrapped_list_by_value + + self.check_func(ns, 'leak_list') + diff --git a/bindings/pyroot/cppyy/cppyy/test/test_numba.py b/bindings/pyroot/cppyy/cppyy/test/test_numba.py index 7fdfe7355a2fc..d36866c369733 100644 --- a/bindings/pyroot/cppyy/cppyy/test/test_numba.py +++ b/bindings/pyroot/cppyy/cppyy/test/test_numba.py @@ -366,8 +366,9 @@ def test10_returning_a_reference(self): cppyy.cppdef(""" int64_t& ref_add(int64_t x, int64_t y) { int64_t c = x + y; - static int64_t result = c; - return c; + static int64_t result = 0; + result = c; + return result; } """) diff --git a/bindings/pyroot/cppyy/cppyy/test/test_stltypes.py b/bindings/pyroot/cppyy/cppyy/test/test_stltypes.py index b42fa0b08b16a..65e5d3ae0383a 100644 --- a/bindings/pyroot/cppyy/cppyy/test/test_stltypes.py +++ b/bindings/pyroot/cppyy/cppyy/test/test_stltypes.py @@ -789,6 +789,34 @@ def test23_copy_conversion(self): for f, d in zip(x, v): assert f == d + def test24_byte_vectors(self): + """Vectors of "byte" types should return low level views""" + + import cppyy + import cppyy.types + + vector = cppyy.gbl.std.vector + + for ctype in ('unsigned char', 'signed char', 'int8_t', 'uint8_t'): + vc = vector[ctype](range(10)) + data = vc.data() + + assert type(data) == cppyy.types.LowLevelView + assert len(data) == 10 + + for i, d in enumerate(data): + assert d == i + + for ctype in ('signed char', 'int8_t'): + vc = vector[ctype](range(-5, 5, 1)) + data = vc.data() + + assert type(data) == cppyy.types.LowLevelView + assert len(data) == 10 + + for i, d in zip(range(-5, 5, 1), data): + assert d == i + class TestSTLSTRING: def setup_class(cls): diff --git a/bindings/pyroot/cppyy/cppyy/test/test_templates.py b/bindings/pyroot/cppyy/cppyy/test/test_templates.py index 6498fc60d2c79..ecb58ed1c1982 100644 --- a/bindings/pyroot/cppyy/cppyy/test/test_templates.py +++ b/bindings/pyroot/cppyy/cppyy/test/test_templates.py @@ -1276,6 +1276,36 @@ class A { assert cppyy.gbl.FailedTypeDeducer.B['double']().result() == 5. assert cppyy.gbl.FailedTypeDeducer.B[int]().result() == 5 + def test06_type_deduction_and_scoping(self): + """Possible shadowing of types used in template construction""" + + import cppyy + + cppyy.cppdef(r""" + namespace ShadowX { + class ShadowC {}; + } + + namespace ShadowY { + namespace ShadowZ { + template void f() {} + } + + namespace ShadowX { + class ShadowD {}; + } + }""") + + ns = cppyy.gbl.ShadowY.ShadowZ + C = cppyy.gbl.ShadowX.ShadowC + + # lookup of shadowed class will fail + raises(TypeError, ns.f.__getitem__(C.__cpp_name__)) + + # direct instantiation now succeeds + ns.f[C]() + ns.f['::'+C.__cpp_name__]() + class TestTEMPLATE_TYPE_REDUCTION: def setup_class(cls): diff --git a/bindings/pyroot/cppyy/patches/CPyCppyy-Avoid-nullptr-dereferencing.patch b/bindings/pyroot/cppyy/patches/CPyCppyy-Avoid-nullptr-dereferencing.patch new file mode 100644 index 0000000000000..b1f7d651c4d27 --- /dev/null +++ b/bindings/pyroot/cppyy/patches/CPyCppyy-Avoid-nullptr-dereferencing.patch @@ -0,0 +1,43 @@ +From 38983c14210be41b134abb6713912412b9c31cbd Mon Sep 17 00:00:00 2001 +From: Jonas Rembser +Date: Wed, 18 Dec 2024 03:02:32 +0100 +Subject: [PATCH] Avoid `nullptr` dereferencing in + CPyCppyy::BindCppObjectNoCast + +Closes the following issue: + + * wlav/cppyy#281 +--- + bindings/pyroot/cppyy/CPyCppyy/src/CPPMethod.cxx | 2 -- + bindings/pyroot/cppyy/CPyCppyy/src/ProxyWrappers.cxx | 2 +- + 2 files changed, 1 insertion(+), 3 deletions(-) + +diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/CPPMethod.cxx b/bindings/pyroot/cppyy/CPyCppyy/src/CPPMethod.cxx +index f70d8f1d0e..823e8ac60a 100644 +--- a/bindings/pyroot/cppyy/CPyCppyy/src/CPPMethod.cxx ++++ b/bindings/pyroot/cppyy/CPyCppyy/src/CPPMethod.cxx +@@ -611,8 +611,6 @@ PyObject* CPyCppyy::CPPMethod::GetArgDefault(int iarg, bool silent) + PyObject* gdct = *dctptr; + PyObject* scope = nullptr; + +- std::string possible_scope = defvalue.substr(0, defvalue.rfind('(')); +- + if (defvalue.rfind('(') != std::string::npos) { // constructor-style call + // try to tickle scope creation, just in case, first look in the scope where + // the function lives, then in the global scope +diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/ProxyWrappers.cxx b/bindings/pyroot/cppyy/CPyCppyy/src/ProxyWrappers.cxx +index 693ad7b367..037660498e 100644 +--- a/bindings/pyroot/cppyy/CPyCppyy/src/ProxyWrappers.cxx ++++ b/bindings/pyroot/cppyy/CPyCppyy/src/ProxyWrappers.cxx +@@ -836,7 +836,7 @@ PyObject* CPyCppyy::BindCppObjectNoCast(Cppyy::TCppObject_t address, + + bool noReg = flags & (CPPInstance::kNoMemReg|CPPInstance::kNoWrapConv); + bool isRef = flags & CPPInstance::kIsReference; +- void* r_address = isRef ? *(void**)address : address; ++ void* r_address = isRef ? (address ? *(void**)address : nullptr) : address; + + // check whether the object to be bound is a smart pointer that needs embedding + PyObject* smart_type = (!(flags & CPPInstance::kNoWrapConv) && \ +-- +2.47.0 + diff --git a/bindings/pyroot/cppyy/patches/CPyCppyy-Perform-function-style-casts.patch b/bindings/pyroot/cppyy/patches/CPyCppyy-Perform-function-style-casts.patch new file mode 100644 index 0000000000000..2d2213b2ea758 --- /dev/null +++ b/bindings/pyroot/cppyy/patches/CPyCppyy-Perform-function-style-casts.patch @@ -0,0 +1,26 @@ +From 6e6725637d4a0d0402fac38a08f7176991d29ac9 Mon Sep 17 00:00:00 2001 +From: Jonas Rembser +Date: Tue, 17 Dec 2024 13:17:36 +0100 +Subject: [PATCH] Perform function-style casts when returning multi-keyword + types + +--- + bindings/pyroot/cppyy/CPyCppyy/src/Dispatcher.cxx | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/Dispatcher.cxx b/bindings/pyroot/cppyy/CPyCppyy/src/Dispatcher.cxx +index db388575db..cdef2b8c7b 100644 +--- a/bindings/pyroot/cppyy/CPyCppyy/src/Dispatcher.cxx ++++ b/bindings/pyroot/cppyy/CPyCppyy/src/Dispatcher.cxx +@@ -45,7 +45,7 @@ static inline void InjectMethod(Cppyy::TCppMethod_t method, const std::string& m + " return"; + if (retType != "void") { + if (retType.back() != '*') +- code << " " << CPyCppyy::TypeManip::remove_const(retType) << "{}"; ++ code << " (" << CPyCppyy::TypeManip::remove_const(retType) << "){}"; + else + code << " nullptr"; + } +-- +2.47.0 + diff --git a/bindings/pyroot/cppyy/patches/CPyCppyy-Support-conversion-from-str-to-char.patch b/bindings/pyroot/cppyy/patches/CPyCppyy-Support-conversion-from-str-to-char.patch new file mode 100644 index 0000000000000..de17b0bac0f14 --- /dev/null +++ b/bindings/pyroot/cppyy/patches/CPyCppyy-Support-conversion-from-str-to-char.patch @@ -0,0 +1,126 @@ +From 7d741bcd61fe523b67e29e9086f4b297c41d2588 Mon Sep 17 00:00:00 2001 +From: Jonas Rembser +Date: Wed, 18 Dec 2024 02:53:11 +0100 +Subject: [PATCH] Support conversion from `str` to `char[]` + +This is for the ROOT usecase, where `char[]` is sometimes used as column +types in TTrees to contain fixed-sized string labels. +--- + .../pyroot/cppyy/CPyCppyy/src/Converters.cxx | 74 ++++++++++++------- + .../cppyy/CPyCppyy/src/DeclareConverters.h | 1 + + 2 files changed, 50 insertions(+), 25 deletions(-) + +diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/Converters.cxx b/bindings/pyroot/cppyy/CPyCppyy/src/Converters.cxx +index fa62c04f47..c127604a6e 100644 +--- a/bindings/pyroot/cppyy/CPyCppyy/src/Converters.cxx ++++ b/bindings/pyroot/cppyy/CPyCppyy/src/Converters.cxx +@@ -1617,6 +1617,42 @@ bool CPyCppyy::VoidArrayConverter::ToMemory(PyObject* value, void* address, PyOb + return true; + } + ++namespace { ++ ++// Copy a buffer to memory address with an array converter. ++template ++bool ToArrayFromBuffer(PyObject* owner, void* address, PyObject* ctxt, ++ const void * buf, Py_ssize_t buflen, ++ CPyCppyy::dims_t& shape, bool isFixed) ++{ ++ if (buflen == 0) ++ return false; ++ ++ Py_ssize_t oldsz = 1; ++ for (Py_ssize_t idim = 0; idim < shape.ndim(); ++idim) { ++ if (shape[idim] == CPyCppyy::UNKNOWN_SIZE) { ++ oldsz = -1; ++ break; ++ } ++ oldsz *= shape[idim]; ++ } ++ if (shape.ndim() != CPyCppyy::UNKNOWN_SIZE && 0 < oldsz && oldsz < buflen) { ++ PyErr_SetString(PyExc_ValueError, "buffer too large for value"); ++ return false; ++ } ++ ++ if (isFixed) ++ memcpy(*(type**)address, buf, (0 < buflen ? buflen : 1)*sizeof(type)); ++ else { ++ *(type**)address = (type*)buf; ++ shape.ndim(1); ++ shape[0] = buflen; ++ SetLifeLine(ctxt, owner, (intptr_t)address); ++ } ++ return true; ++} ++ ++} + + //---------------------------------------------------------------------------- + #define CPPYY_IMPL_ARRAY_CONVERTER(name, ctype, type, code, suffix) \ +@@ -1697,31 +1733,7 @@ bool CPyCppyy::name##ArrayConverter::ToMemory( \ + if (fShape.ndim() <= 1 || fIsFixed) { \ + void* buf = nullptr; \ + Py_ssize_t buflen = Utility::GetBuffer(value, code, sizeof(type), buf);\ +- if (buflen == 0) \ +- return false; \ +- \ +- Py_ssize_t oldsz = 1; \ +- for (Py_ssize_t idim = 0; idim < fShape.ndim(); ++idim) { \ +- if (fShape[idim] == UNKNOWN_SIZE) { \ +- oldsz = -1; \ +- break; \ +- } \ +- oldsz *= fShape[idim]; \ +- } \ +- if (fShape.ndim() != UNKNOWN_SIZE && 0 < oldsz && oldsz < buflen) { \ +- PyErr_SetString(PyExc_ValueError, "buffer too large for value"); \ +- return false; \ +- } \ +- \ +- if (fIsFixed) \ +- memcpy(*(type**)address, buf, (0 < buflen ? buflen : 1)*sizeof(type));\ +- else { \ +- *(type**)address = (type*)buf; \ +- fShape.ndim(1); \ +- fShape[0] = buflen; \ +- SetLifeLine(ctxt, value, (intptr_t)address); \ +- } \ +- \ ++ return ToArrayFromBuffer(value, address, ctxt, buf, buflen, fShape, fIsFixed);\ + } else { /* multi-dim, non-flat array; assume structure matches */ \ + void* buf = nullptr; /* TODO: GetBuffer() assumes flat? */ \ + Py_ssize_t buflen = Utility::GetBuffer(value, code, sizeof(void*), buf);\ +@@ -1820,6 +1832,18 @@ PyObject* CPyCppyy::CStringArrayConverter::FromMemory(void* address) + return CreateLowLevelViewString(*(const char***)address, fShape); + } + ++//---------------------------------------------------------------------------- ++bool CPyCppyy::CStringArrayConverter::ToMemory(PyObject* value, void* address, PyObject* ctxt) ++{ ++// As a special array converter, the CStringArrayConverter one can also copy strings in the array, ++// and not only buffers. ++ Py_ssize_t len; ++ if (const char* cstr = CPyCppyy_PyText_AsStringAndSize(value, &len)) { ++ return ToArrayFromBuffer(value, address, ctxt, cstr, len, fShape, fIsFixed); ++ } ++ return SCharArrayConverter::ToMemory(value, address, ctxt); ++} ++ + //---------------------------------------------------------------------------- + PyObject* CPyCppyy::NonConstCStringArrayConverter::FromMemory(void* address) + { +diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/DeclareConverters.h b/bindings/pyroot/cppyy/CPyCppyy/src/DeclareConverters.h +index 82a0e5d23b..5d26c9e569 100644 +--- a/bindings/pyroot/cppyy/CPyCppyy/src/DeclareConverters.h ++++ b/bindings/pyroot/cppyy/CPyCppyy/src/DeclareConverters.h +@@ -237,6 +237,7 @@ public: + using SCharArrayConverter::SCharArrayConverter; + virtual bool SetArg(PyObject*, Parameter&, CallContext* = nullptr); + virtual PyObject* FromMemory(void* address); ++ virtual bool ToMemory(PyObject*, void*, PyObject* = nullptr); + + private: + std::vector fBuffer; +-- +2.47.0 + diff --git a/bindings/pyroot/cppyy/patches/CPyCppyy-Use-PyMapping_GetOptionalItemString-where-necessary.patch b/bindings/pyroot/cppyy/patches/CPyCppyy-Use-PyMapping_GetOptionalItemString-where-necessary.patch new file mode 100644 index 0000000000000..13e913e7beda5 --- /dev/null +++ b/bindings/pyroot/cppyy/patches/CPyCppyy-Use-PyMapping_GetOptionalItemString-where-necessary.patch @@ -0,0 +1,44 @@ +From 386290cd2470da3ba20f0b12a6cfb81ed423471e Mon Sep 17 00:00:00 2001 +From: Jonas Rembser +Date: Wed, 18 Dec 2024 02:51:40 +0100 +Subject: [PATCH] Use PyMapping_GetOptionalItemString where necessary with + Python 3.13 + +With Python 3.13, some lookup methods like `PyMapping_GetItemString` and +`PyObject_GetAttrString` became more strict. They are now always +throwing an exception in case the attribute is not found. + +To make these optional lookups work again, the `GetOptional` family of +functions needs to be used. + +See: + * https://docs.python.org/3/c-api/object.html#c.PyObject_GetOptionalAttrString + * https://docs.python.org/3/c-api/mapping.html#c.PyMapping_GetOptionalItemString + +This is the upstream version of the following ROOT commit: + + * root-project/root@e78450dc45ed868b7a52a0 +--- + bindings/pyroot/cppyy/CPyCppyy/src/Dispatcher.cxx | 5 +++++ + 1 file changed, 5 insertions(+) + +diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/Dispatcher.cxx b/bindings/pyroot/cppyy/CPyCppyy/src/Dispatcher.cxx +index cdef2b8c7b..06731d6d85 100644 +--- a/bindings/pyroot/cppyy/CPyCppyy/src/Dispatcher.cxx ++++ b/bindings/pyroot/cppyy/CPyCppyy/src/Dispatcher.cxx +@@ -484,7 +484,12 @@ bool CPyCppyy::InsertDispatcher(CPPScope* klass, PyObject* bases, PyObject* dct, + // Python class to keep the inheritance tree intact) + for (const auto& name : protected_names) { + PyObject* disp_dct = PyObject_GetAttr(disp_proxy, PyStrings::gDict); ++#if PY_VERSION_HEX < 0x30d00f0 + PyObject* pyf = PyMapping_GetItemString(disp_dct, (char*)name.c_str()); ++#else ++ PyObject* pyf = nullptr; ++ PyMapping_GetOptionalItemString(disp_dct, (char*)name.c_str(), &pyf); ++#endif + if (pyf) { + PyObject_SetAttrString((PyObject*)klass, (char*)name.c_str(), pyf); + Py_DECREF(pyf); +-- +2.47.0 + diff --git a/bindings/pyroot/cppyy/patches/cppyy-Don-t-enable-cling-autoloading.patch b/bindings/pyroot/cppyy/patches/cppyy-Don-t-enable-cling-autoloading.patch index ddd6775bd9b52..bf807b5be5d29 100644 --- a/bindings/pyroot/cppyy/patches/cppyy-Don-t-enable-cling-autoloading.patch +++ b/bindings/pyroot/cppyy/patches/cppyy-Don-t-enable-cling-autoloading.patch @@ -1,6 +1,6 @@ -From 800deb0450dd692e6ef49ec76d6bc8ee83f29630 Mon Sep 17 00:00:00 2001 +From 435b1e67acbc9c7348874030c8ad8721c917cd87 Mon Sep 17 00:00:00 2001 From: Jonas Rembser -Date: Tue, 12 Mar 2024 03:26:23 +0100 +Date: Tue, 17 Dec 2024 13:13:32 +0100 Subject: [PATCH] [cppyy] Don't enable cling autoloading --- @@ -8,10 +8,10 @@ Subject: [PATCH] [cppyy] Don't enable cling autoloading 1 file changed, 5 deletions(-) diff --git a/bindings/pyroot/cppyy/cppyy/python/cppyy/__init__.py b/bindings/pyroot/cppyy/cppyy/python/cppyy/__init__.py -index fdc5be8dc8..957443289d 100644 +index 47b0ff1aab..976afdb7da 100644 --- a/bindings/pyroot/cppyy/cppyy/python/cppyy/__init__.py +++ b/bindings/pyroot/cppyy/cppyy/python/cppyy/__init__.py -@@ -86,11 +86,6 @@ sys.modules['cppyy.gbl'] = gbl +@@ -93,11 +93,6 @@ sys.modules['cppyy.gbl'] = gbl sys.modules['cppyy.gbl.std'] = gbl.std @@ -21,8 +21,8 @@ index fdc5be8dc8..957443289d 100644 - - #- external typemap ---------------------------------------------------------- - from . import _typemap _typemap.initialize(_backend) # also creates (u)int8_t mapper + -- -2.44.0 +2.47.0 diff --git a/bindings/pyroot/cppyy/patches/cppyy-No-CppyyLegacy-namespace.patch b/bindings/pyroot/cppyy/patches/cppyy-No-CppyyLegacy-namespace.patch index eb20ce0344b08..900eb06f6cc01 100644 --- a/bindings/pyroot/cppyy/patches/cppyy-No-CppyyLegacy-namespace.patch +++ b/bindings/pyroot/cppyy/patches/cppyy-No-CppyyLegacy-namespace.patch @@ -1,6 +1,6 @@ -From e990e7bcb5a119ad619c847d61d29cae4e447ec1 Mon Sep 17 00:00:00 2001 +From a92b052837426ec0b6bb4c91128c5455380c50bb Mon Sep 17 00:00:00 2001 From: Jonas Rembser -Date: Tue, 12 Mar 2024 01:35:52 +0100 +Date: Tue, 17 Dec 2024 13:11:06 +0100 Subject: [PATCH] [cppyy] Don't use `CppyyLegacy` namespace --- @@ -8,19 +8,19 @@ Subject: [PATCH] [cppyy] Don't use `CppyyLegacy` namespace 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bindings/pyroot/cppyy/cppyy/python/cppyy/__init__.py b/bindings/pyroot/cppyy/cppyy/python/cppyy/__init__.py -index 1a2afbd211..2a169cadee 100644 +index b5ea3d087f..47b0ff1aab 100644 --- a/bindings/pyroot/cppyy/cppyy/python/cppyy/__init__.py +++ b/bindings/pyroot/cppyy/cppyy/python/cppyy/__init__.py -@@ -178,7 +178,7 @@ del make_smartptr +@@ -189,7 +189,7 @@ del make_smartptr #--- interface to Cling ------------------------------------------------------ class _stderr_capture(object): def __init__(self): -- self._capture = not gbl.CppyyLegacy.gDebug and True or False -+ self._capture = not gbl.gDebug and True or False - self.err = "" +- self._capture = not gbl.CppyyLegacy.gDebug and True or False ++ self._capture = not gbl.gDebug and True or False + self.err = "" def __enter__(self): -@@ -242,8 +242,8 @@ def load_library(name): +@@ -264,8 +264,8 @@ def load_library(name): with _stderr_capture() as err: gSystem = gbl.gSystem if name[:3] != 'lib': @@ -31,7 +31,7 @@ index 1a2afbd211..2a169cadee 100644 name = 'lib'+name sc = gSystem.Load(name) if sc == -1: -@@ -378,9 +378,9 @@ def add_autoload_map(fname): +@@ -417,9 +417,9 @@ def add_autoload_map(fname): def set_debug(enable=True): """Enable/disable debug output.""" if enable: @@ -42,7 +42,7 @@ index 1a2afbd211..2a169cadee 100644 + gbl.gDebug = 0 def _get_name(tt): - if type(tt) == str: + if isinstance(tt, str): -- -2.44.0 +2.47.0 diff --git a/bindings/pyroot/cppyy/patches/cppyy-Remove-Windows-workaround.patch b/bindings/pyroot/cppyy/patches/cppyy-Remove-Windows-workaround.patch index 9ca7d0b76f904..0de68d5d9decc 100644 --- a/bindings/pyroot/cppyy/patches/cppyy-Remove-Windows-workaround.patch +++ b/bindings/pyroot/cppyy/patches/cppyy-Remove-Windows-workaround.patch @@ -1,6 +1,6 @@ -From c5c0a50babb33db7f0b193fc824fca2b02ed6b41 Mon Sep 17 00:00:00 2001 +From 096daee5b75ace6438e924c6d72bdb0065f626e0 Mon Sep 17 00:00:00 2001 From: Jonas Rembser -Date: Tue, 12 Mar 2024 09:39:45 +0100 +Date: Tue, 17 Dec 2024 13:06:45 +0100 Subject: [PATCH] [cppyy] Remove unneeded `std::endl` workaround for Windows --- @@ -8,11 +8,11 @@ Subject: [PATCH] [cppyy] Remove unneeded `std::endl` workaround for Windows 1 file changed, 7 deletions(-) diff --git a/bindings/pyroot/cppyy/cppyy/python/cppyy/__init__.py b/bindings/pyroot/cppyy/cppyy/python/cppyy/__init__.py -index 2a169cadee..957443289d 100644 +index 50c148621b..b5ea3d087f 100644 --- a/bindings/pyroot/cppyy/cppyy/python/cppyy/__init__.py +++ b/bindings/pyroot/cppyy/cppyy/python/cppyy/__init__.py -@@ -430,10 +430,3 @@ def multi(*bases): # after six, see also _typemap.py - def __new__(cls, name, this_bases, d): +@@ -470,10 +470,3 @@ def multi(*bases): # after six, see also _typemap.py + def __new__(mcs, name, this_bases, d): return nc_meta(name, bases, d) return type.__new__(faux_meta, 'faux_meta', (), {}) - @@ -23,5 +23,5 @@ index 2a169cadee..957443289d 100644 - std::basic_ostream>& __cdecl std::endl>( - std::basic_ostream>&);""") -- -2.44.0 +2.47.0 diff --git a/bindings/pyroot/cppyy/sync-upstream b/bindings/pyroot/cppyy/sync-upstream index c5d5c15e36851..0cf57856d28aa 100755 --- a/bindings/pyroot/cppyy/sync-upstream +++ b/bindings/pyroot/cppyy/sync-upstream @@ -1,4 +1,4 @@ -#!/usr/bin/bash +#!/usr/bin/env bash # Please run in this directory. @@ -10,26 +10,8 @@ mv cppyy/CMakeLists.txt cppyy_CMakeLists.txt rm -rf CPyCppyy rm -rf cppyy -git clone https://github.com/wlav/CPyCppyy.git -git clone https://github.com/wlav/cppyy.git - -# In case you want to merge other branches first: - -cd CPyCppyy - -git remote rename origin wlav -git remote add guitargeek https://github.com/guitargeek/CPyCppyy.git -git fetch guitargeek - -git checkout -b sync - -rebase_topic () { - git rebase sync $1 && git branch -D sync && git checkout -b sync -} - -rebase_topic guitargeek/string_converters - -cd .. +git clone --depth 1 --branch CPyCppyy-1.13.0 https://github.com/wlav/CPyCppyy.git +git clone --depth 1 --branch cppyy-3.5.0 https://github.com/wlav/cppyy.git rm -rf CPyCppyy/.git rm -rf cppyy/.git @@ -46,6 +28,10 @@ git apply patches/CPyCppyy-Always-convert-returned-std-string.patch git apply patches/CPyCppyy-Disable-implicit-conversion-to-smart-ptr.patch git apply patches/CPyCppyy-TString_converter.patch git apply patches/CPyCppyy-Prevent-construction-of-agg-init-for-tuple.patch +git apply patches/CPyCppyy-Support-conversion-from-str-to-char.patch # https://github.com/wlav/CPyCppyy/pull/21 +git apply patches/CPyCppyy-Perform-function-style-casts.patch # https://github.com/wlav/CPyCppyy/pull/34 +git apply patches/CPyCppyy-Use-PyMapping_GetOptionalItemString-where-necessary.patch # https://github.com/wlav/CPyCppyy/pull/44 +git apply patches/CPyCppyy-Avoid-nullptr-dereferencing.patch # https://github.com/wlav/CPyCppyy/pull/45 git apply patches/cppyy-No-CppyyLegacy-namespace.patch git apply patches/cppyy-Remove-Windows-workaround.patch git apply patches/cppyy-Don-t-enable-cling-autoloading.patch