[Python] Sync with cppyy 3.5.0

bindings/pyroot/cppyy/CPyCppyy/setup.py

@@ -9,7 +9,7 @@
 except ImportError:
     has_wheel = False
 
-requirements = ['cppyy-cling==6.30.0', 'cppyy-backend==1.15.2']
+requirements = ['cppyy-cling==6.32.8', 'cppyy-backend==1.15.3']
 setup_requirements = ['wheel']
 if 'build' in sys.argv or 'install' in sys.argv:
     setup_requirements += requirements
@@ -80,7 +80,7 @@ def build_extension(self, ext):
 
 setup(
     name='CPyCppyy',
-    version='1.12.16',
+    version='1.13.0',
     description='Cling-based Python-C++ bindings for CPython',
     long_description=long_description,
 

bindings/pyroot/cppyy/CPyCppyy/src/CPPInstance.h

@@ -27,20 +27,21 @@ class CPPInstance {
 public:
     enum EFlags {
         kDefault     = 0x0000,
-        kNoWrapConv  = 0x0001,
-        kIsOwner     = 0x0002,
-        kIsExtended  = 0x0004,
-        kIsReference = 0x0008,
-        kIsRValue    = 0x0010,
-        kIsLValue    = 0x0020,
-        kIsValue     = 0x0040,
-        kIsPtrPtr    = 0x0080,
-        kIsArray     = 0x0100,
-        kIsSmartPtr  = 0x0200,
-        kNoMemReg    = 0x0400,
-        kHasLifeLine = 0x0800,
-        kIsRegulated = 0x1000,
-        kIsActual    = 0x2000 };
+        kNoWrapConv  = 0x0001,    // use type as-is (eg. no smart ptr wrap)
+        kIsOwner     = 0x0002,    // Python instance owns C++ object/memory
+        kIsExtended  = 0x0004,    // has extended data
+        kIsValue     = 0x0008,    // was created from a by-value return
+        kIsReference = 0x0010,    // represents one indirection
+        kIsArray     = 0x0020,    // represents an array of objects
+        kIsSmartPtr  = 0x0040,    // is or embeds a smart pointer
+        kIsPtrPtr    = 0x0080,    // represents two indirections
+        kIsRValue    = 0x0100,    // can be used as an r-value
+        kIsLValue    = 0x0200,    // can be used as an l-value
+        kNoMemReg    = 0x0400,    // do not register with memory regulator
+        kIsRegulated = 0x0800,    // is registered with memory regulator
+        kIsActual    = 0x1000,    // has been downcasted to actual type
+        kHasLifeLine = 0x2000,    // has a life line set
+    };
 
 public:                 // public, as the python C-API works with C structs
     PyObject_HEAD

bindings/pyroot/cppyy/CPyCppyy/src/CPPMethod.cxx

@@ -611,15 +611,30 @@ PyObject* CPyCppyy::CPPMethod::GetArgDefault(int iarg, bool silent)
         PyObject* gdct = *dctptr;
         PyObject* scope = nullptr;
 
-        if (defvalue.find("::") != std::string::npos) {
-        // try to tickle scope creation, just in case
-            scope = CreateScopeProxy(defvalue.substr(0, defvalue.rfind('(')));
-            if (!scope) PyErr_Clear();
-
-        // rename '::' -> '.'
-            TypeManip::cppscope_to_pyscope(defvalue);
+        if (defvalue.rfind('(') != std::string::npos) {    // constructor-style call
+        // try to tickle scope creation, just in case, first look in the scope where
+        // the function lives, then in the global scope
+            std::string possible_scope = defvalue.substr(0, defvalue.rfind('('));
+            if (!Cppyy::IsBuiltin(possible_scope)) {
+                std::string cand_scope = Cppyy::GetScopedFinalName(fScope)+"::"+possible_scope;
+                scope = CreateScopeProxy(cand_scope);
+                if (!scope) {
+                    PyErr_Clear();
+                // search within the global scope instead
+                    scope = CreateScopeProxy(possible_scope);
+                    if (!scope) PyErr_Clear();
+                } else {
+                // re-scope the scope; alternatively, the expression could be
+                // compiled in the dictionary of the function's namespace, but
+                // that would affect arguments passed to the constructor, too
+                    defvalue = cand_scope + defvalue.substr(defvalue.rfind('('), std::string::npos);
+                }
+            }
         }
 
+    // replace '::' -> '.'
+        TypeManip::cppscope_to_pyscope(defvalue);
+
         if (!scope) {
         // a couple of common cases that python doesn't like (technically, 'L' is okay with older
         // pythons, but C long will always fit in Python int, so no need to bother)
@@ -956,13 +971,6 @@ PyObject* CPyCppyy::CPPMethod::Execute(void* self, ptrdiff_t offset, CallContext
         result = ExecuteProtected(self, offset, ctxt);
     }
 
-// TODO: the following is dreadfully slow and dead-locks on Apache: revisit
-// raising exceptions through callbacks by using magic returns
-//    if (result && Utility::PyErr_Occurred_WithGIL()) {
-//    // can happen in the case of a CINT error: trigger exception processing
-//        Py_DECREF(result);
-//        result = 0;
-//    } else if (!result && PyErr_Occurred())
     if (!result && PyErr_Occurred())
         SetPyError_(0);
 

bindings/pyroot/cppyy/CPyCppyy/src/CPPOverload.cxx

@@ -697,7 +697,7 @@ static PyObject* mp_call(CPPOverload* pymeth, PyObject* args, PyObject* kwds)
                 continue;    // did not set implicit conversion, so don't try again
 
             PyObject* result = methods[i]->Call(im_self, args, nargsf, kwds, &ctxt);
-            if (result != 0) {
+            if (result) {
             // success: update the dispatch map for subsequent calls
                 if (!memoized_pc)
                     dispatchMap.push_back(std::make_pair(sighash, methods[i]));

bindings/pyroot/cppyy/CPyCppyy/src/Converters.cxx

@@ -880,6 +880,13 @@ CPPYY_IMPL_BASIC_CONVERTER_NI(
 CPPYY_IMPL_BASIC_CHAR_CONVERTER(Char,  char,          CHAR_MIN,  CHAR_MAX)
 CPPYY_IMPL_BASIC_CHAR_CONVERTER(UChar, unsigned char,        0, UCHAR_MAX)
 
+PyObject* CPyCppyy::SCharAsIntConverter::FromMemory(void* address)
+{
+// special case to be used with arrays: return a Python int instead of str
+// (following the same convention as module array.array)
+    return PyInt_FromLong((long)*((signed char*)address));
+}
+
 PyObject* CPyCppyy::UCharAsIntConverter::FromMemory(void* address)
 {
 // special case to be used with arrays: return a Python int instead of str
@@ -3428,6 +3435,7 @@ static struct InitConvFactories_t {
         gf["unsigned char"] =               (cf_t)+[](cdims_t) { static UCharConverter c{};          return &c; };
         gf["const unsigned char&"] =        (cf_t)+[](cdims_t) { static ConstUCharRefConverter c{};  return &c; };
         gf["unsigned char&"] =              (cf_t)+[](cdims_t) { static UCharRefConverter c{};       return &c; };
+        gf["SCharAsInt"] =                  (cf_t)+[](cdims_t) { static SCharAsIntConverter c{};     return &c; };
         gf["UCharAsInt"] =                  (cf_t)+[](cdims_t) { static UCharAsIntConverter c{};     return &c; };
         gf["wchar_t"] =                     (cf_t)+[](cdims_t) { static WCharConverter c{};          return &c; };
         gf["char16_t"] =                    (cf_t)+[](cdims_t) { static Char16Converter c{};         return &c; };
@@ -3481,11 +3489,12 @@ static struct InitConvFactories_t {
 
     // pointer/array factories
         gf["bool ptr"] =                    (cf_t)+[](cdims_t d) { return new BoolArrayConverter{d}; };
-        gf["const signed char[]"] =         (cf_t)+[](cdims_t d) { return new SCharArrayConverter{d}; };
-        gf["signed char[]"] =               gf["const signed char[]"];
+        gf["signed char ptr"] =             (cf_t)+[](cdims_t d) { return new SCharArrayConverter{d}; };
         gf["signed char**"] =               (cf_t)+[](cdims_t)   { return new SCharArrayConverter{{UNKNOWN_SIZE, UNKNOWN_SIZE}}; };
         gf["const unsigned char*"] =        (cf_t)+[](cdims_t d) { return new UCharArrayConverter{d}; };
         gf["unsigned char ptr"] =           (cf_t)+[](cdims_t d) { return new UCharArrayConverter{d}; };
+        gf["SCharAsInt*"] =                 gf["signed char ptr"];
+        gf["SCharAsInt[]"] =                gf["signed char ptr"];
         gf["UCharAsInt*"] =                 gf["unsigned char ptr"];
         gf["UCharAsInt[]"] =                gf["unsigned char ptr"];
 #if __cplusplus > 201402L

bindings/pyroot/cppyy/CPyCppyy/src/DeclareConverters.h

@@ -72,6 +72,11 @@ protected:                                                                   \
 CPPYY_DECLARE_BASIC_CONVERTER(Long);
 CPPYY_DECLARE_BASIC_CONVERTER(Bool);
 CPPYY_DECLARE_BASIC_CONVERTER(Char);
+class SCharAsIntConverter : public CharConverter {
+public:
+    using CharConverter::CharConverter;
+    virtual PyObject* FromMemory(void*);
+};
 CPPYY_DECLARE_BASIC_CONVERTER(UChar);
 class UCharAsIntConverter : public UCharConverter {
 public:

bindings/pyroot/cppyy/CPyCppyy/src/DeclareExecutors.h

@@ -63,6 +63,7 @@ public:                                                                      \
 }
 CPPYY_ARRAY_DECL_EXEC(Void);
 CPPYY_ARRAY_DECL_EXEC(Bool);
+CPPYY_ARRAY_DECL_EXEC(SChar);
 CPPYY_ARRAY_DECL_EXEC(UChar);
 #if __cplusplus > 201402L
 CPPYY_ARRAY_DECL_EXEC(Byte);
@@ -115,8 +116,6 @@ class InstanceExecutor : public Executor {
 class IteratorExecutor : public InstanceExecutor {
 public:
     IteratorExecutor(Cppyy::TCppType_t klass);
-    virtual PyObject* Execute(
-        Cppyy::TCppMethod_t, Cppyy::TCppObject_t, CallContext*);
 };
 
 CPPYY_DECL_EXEC(Constructor);

bindings/pyroot/cppyy/CPyCppyy/src/Executors.cxx

@@ -508,6 +508,7 @@ PyObject* CPyCppyy::name##ArrayExecutor::Execute(                            \
 }
 
 CPPYY_IMPL_ARRAY_EXEC(Bool,     bool,                    )
+CPPYY_IMPL_ARRAY_EXEC(SChar,    signed char,             )
 CPPYY_IMPL_ARRAY_EXEC(UChar,    unsigned char,           )
 #if __cplusplus > 201402L
 CPPYY_IMPL_ARRAY_EXEC(Byte,     std::byte,               )
@@ -632,24 +633,9 @@ PyObject* CPyCppyy::InstanceExecutor::Execute(
 CPyCppyy::IteratorExecutor::IteratorExecutor(Cppyy::TCppType_t klass) :
     InstanceExecutor(klass)
 {
-    fFlags |= CPPInstance::kNoWrapConv;     // adds to flags from base class
+    fFlags |= CPPInstance::kNoMemReg | CPPInstance::kNoWrapConv;     // adds to flags from base class
 }
 
-//----------------------------------------------------------------------------
-PyObject* CPyCppyy::IteratorExecutor::Execute(
-    Cppyy::TCppMethod_t method, Cppyy::TCppObject_t self, CallContext* ctxt)
-{
-    PyObject* iter = this->InstanceExecutor::Execute(method, self, ctxt);
-    if (iter && ctxt->fPyContext) {
-    // set life line to tie iterator life time to the container (which may
-    // be a temporary)
-        std::ostringstream attr_name;
-        attr_name << "__" << (intptr_t)iter;
-        if (PyObject_SetAttrString(ctxt->fPyContext, (char*)attr_name.str().c_str(), iter))
-            PyErr_Clear();
-    }
-    return iter;
-}
 
 //----------------------------------------------------------------------------
 PyObject* CPyCppyy::InstanceRefExecutor::Execute(
@@ -1090,7 +1076,8 @@ struct InitExecFactories_t {
         gf["const char*&"] =                (ef_t)+[](cdims_t) { static CStringRefExecutor e{};     return &e; };
         gf["char*&"] =                      gf["const char*&"];
         gf["const signed char*"] =          gf["const char*"];
-        gf["signed char*"] =                gf["char*"];
+        //gf["signed char*"] =                gf["char*"];
+        gf["signed char ptr"] =             (ef_t)+[](cdims_t d) { return new SCharArrayExecutor{d};    };
         gf["wchar_t*"] =                    (ef_t)+[](cdims_t) { static WCStringExecutor e{};    return &e;};
         gf["char16_t*"] =                   (ef_t)+[](cdims_t) { static CString16Executor e{};   return &e;};
         gf["char32_t*"] =                   (ef_t)+[](cdims_t) { static CString32Executor e{};   return &e;};

bindings/pyroot/cppyy/CPyCppyy/src/LowLevelViews.cxx

@@ -998,7 +998,7 @@ template<> struct typecode_traits<bool> {
 template<> struct typecode_traits<char> {
     static constexpr const char* format = "b"; static constexpr const char* name = "char"; };
 template<> struct typecode_traits<signed char> {
-    static constexpr const char* format = "b"; static constexpr const char* name = "signed char"; };
+    static constexpr const char* format = "b"; static constexpr const char* name = "SCharAsInt"; };
 template<> struct typecode_traits<unsigned char> {
     static constexpr const char* format = "B"; static constexpr const char* name = "UCharAsInt"; };
 #if __cplusplus > 201402L

bindings/pyroot/cppyy/CPyCppyy/src/MemoryRegulator.cxx

@@ -206,7 +206,7 @@ bool CPyCppyy::MemoryRegulator::UnregisterPyObject(CPPInstance* pyobj, PyObject*
     if (!(pyobj && pyclass))
         return false;
 
-    Cppyy::TCppObject_t cppobj = pyobj->GetObject();
+    Cppyy::TCppObject_t cppobj = pyobj->IsSmart() ? pyobj->GetObjectRaw() : pyobj->GetObject();
     if (!cppobj)
         return false;
 

bindings/pyroot/cppyy/CPyCppyy/src/ProxyWrappers.cxx

@@ -834,23 +834,13 @@ PyObject* CPyCppyy::BindCppObjectNoCast(Cppyy::TCppObject_t address,
     if (!pyclass)
         return nullptr;                 // error has been set in CreateScopeProxy
 
-    bool isRef   = flags & CPPInstance::kIsReference;
-    bool isValue = flags & CPPInstance::kIsValue;
+    bool noReg      = flags & (CPPInstance::kNoMemReg|CPPInstance::kNoWrapConv);
+    bool isRef      = flags & CPPInstance::kIsReference;
+    void* r_address = isRef ? (address ? *(void**)address : nullptr) : address;
 
-// TODO: make sure that a consistent address is used (may have to be done in BindCppObject)
-    if (address && !isValue /* always fresh */ && !(flags & (CPPInstance::kNoWrapConv|CPPInstance::kNoMemReg))) {
-        PyObject* oldPyObject = MemoryRegulator::RetrievePyObject(
-            isRef ? *(void**)address : address, pyclass);
-
-    // ptr-ptr requires old object to be a reference to enable re-use
-        if (oldPyObject && (!(flags & CPPInstance::kIsPtrPtr) ||
-                ((CPPInstance*)oldPyObject)->fFlags & CPPInstance::kIsReference)) {
-            return oldPyObject;
-        }
-    }
-
-// if smart, instantiate a Python-side object of the underlying type, carrying the smartptr
-    PyObject* smart_type = (flags != CPPInstance::kNoWrapConv && (((CPPClass*)pyclass)->fFlags & CPPScope::kIsSmart)) ? pyclass : nullptr;
+// check whether the object to be bound is a smart pointer that needs embedding
+    PyObject* smart_type = (!(flags & CPPInstance::kNoWrapConv) && \
+                            (((CPPClass*)pyclass)->fFlags & CPPScope::kIsSmart)) ? pyclass : nullptr;
     if (smart_type) {
         pyclass = CreateScopeProxy(((CPPSmartClass*)smart_type)->fUnderlyingType);
         if (!pyclass) {
@@ -860,6 +850,29 @@ PyObject* CPyCppyy::BindCppObjectNoCast(Cppyy::TCppObject_t address,
         }
     }
 
+// TODO: make sure that a consistent address is used (may have to be done in BindCppObject)
+    if (address && !(flags & CPPInstance::kIsValue) /* always fresh */ && !noReg) {
+        PyObject* oldPyObject = MemoryRegulator::RetrievePyObject(r_address, pyclass);
+
+    // embedded smart pointers are registered with the class of the underlying type, b/c
+    // there is no Python proxy for the smart pointer itself in that case; check whether
+    // the result found matches the smart-ness and if so, whether the smart pointer found
+    // is the same as the one requested (note: ptr-ptr requires old object to be a
+    // reference to enable re-use)
+        if (oldPyObject) {
+            CPPInstance* o_pyobj = ((CPPInstance*)oldPyObject);
+
+            if ((bool)smart_type == o_pyobj->IsSmart() && \
+                    r_address == (smart_type ? o_pyobj->GetObjectRaw() : o_pyobj->GetObject()) && \
+                    (!(flags & CPPInstance::kIsPtrPtr) || (o_pyobj->fFlags & CPPInstance::kIsReference))) {
+                Py_DECREF(pyclass);
+                return oldPyObject;
+            } else {      // reference or naked v.s. smart pointer mismatch
+                Py_DECREF(oldPyObject);
+            }
+        }
+    }
+
 // instantiate an object of this class
     PyObject* args = PyTuple_New(0);
     CPPInstance* pyobj =
@@ -875,9 +888,10 @@ PyObject* CPyCppyy::BindCppObjectNoCast(Cppyy::TCppObject_t address,
         if (smart_type)
             pyobj->SetSmart(smart_type);
 
-    // do not register null pointers, references (?), or direct usage of smart pointers or iterators
-        if (address && !isRef && !(flags & (CPPInstance::kNoWrapConv|CPPInstance::kNoMemReg)))
-            MemoryRegulator::RegisterPyObject(pyobj, pyobj->GetObject());
+    // do not register null pointers, references (regulated objects can be referenced, but
+    // referenced objects can not be regulated), or direct usage of smart pointers or iterators
+        if (address && !isRef && !noReg)
+            MemoryRegulator::RegisterPyObject(pyobj, smart_type ? pyobj->GetObjectRaw() : pyobj->GetObject());
     }
 
 // successful completion; wrap exception options to make them raiseable, normal return otherwise

bindings/pyroot/cppyy/CPyCppyy/src/Pythonize.cxx

@@ -1237,7 +1237,7 @@ PyObject* STLStringDecode(CPPInstance* self, PyObject* args, PyObject* kwds)
         return nullptr;
 
     char* keywords[] = {(char*)"encoding", (char*)"errors", (char*)nullptr};
-    const char* encoding; const char* errors;
+    const char* encoding = nullptr; const char* errors = nullptr;
     if (!PyArg_ParseTupleAndKeywords(args, kwds,
             const_cast<char*>("s|s"), keywords, &encoding, &errors))
         return nullptr;

bindings/pyroot/cppyy/CPyCppyy/src/Utility.cxx

@@ -122,6 +122,10 @@ namespace {
         }
     } initOperatorMapping_;
 
+    inline std::string full_scope(const std::string& tpname) {
+        return tpname[0] == ':' ? tpname : "::"+tpname;
+    }
+
 } // unnamed namespace
 
 
@@ -484,7 +488,7 @@ static bool AddTypeName(std::string& tmpl_name, PyObject* tn, PyObject* arg,
     }
 
     if (CPPScope_Check(tn)) {
-        tmpl_name.append(Cppyy::GetScopedFinalName(((CPPClass*)tn)->fCppType));
+        tmpl_name.append(full_scope(Cppyy::GetScopedFinalName(((CPPClass*)tn)->fCppType)));
         if (arg) {
         // try to specialize the type match for the given object
             CPPInstance* pyobj = (CPPInstance*)arg;
@@ -505,7 +509,7 @@ static bool AddTypeName(std::string& tmpl_name, PyObject* tn, PyObject* arg,
     }
 
     if (tn == (PyObject*)&CPPOverload_Type) {
-        PyObject* tpName =  arg ? \
+        PyObject* tpName = arg ? \
             PyObject_GetAttr(arg, PyStrings::gCppName) : \
             CPyCppyy_PyText_FromString("void* (*)(...)");
         tmpl_name.append(CPyCppyy_PyText_AsString(tpName));
@@ -529,7 +533,7 @@ static bool AddTypeName(std::string& tmpl_name, PyObject* tn, PyObject* arg,
                     for (Py_ssize_t i = 0; i < (PyList_GET_SIZE(values)-1); ++i) {
                         if (i) tpn << ", ";
                         PyObject* item = PyList_GET_ITEM(values, i);
-                        tpn << (CPPScope_Check(item) ?  ClassName(item) : AnnotationAsText(item));
+                        tpn << (CPPScope_Check(item) ? full_scope(ClassName(item)) : AnnotationAsText(item));
                     }
                     Py_DECREF(values);
 
@@ -547,7 +551,8 @@ static bool AddTypeName(std::string& tmpl_name, PyObject* tn, PyObject* arg,
 
         PyObject* tpName = PyObject_GetAttr(arg, PyStrings::gCppName);
         if (tpName) {
-            tmpl_name.append(CPyCppyy_PyText_AsString(tpName));
+            const char* cname = CPyCppyy_PyText_AsString(tpName);
+            tmpl_name.append(CPPScope_Check(arg) ? full_scope(cname) : cname);
             Py_DECREF(tpName);
             return true;
         }
@@ -1017,7 +1022,11 @@ std::string CPyCppyy::Utility::ClassName(PyObject* pyobj)
 static std::set<std::string> sIteratorTypes;
 bool CPyCppyy::Utility::IsSTLIterator(const std::string& classname)
 {
-// attempt to recognize STL iterators (TODO: probably belongs in the backend)
+// attempt to recognize STL iterators (TODO: probably belongs in the backend), using
+// a couple of common container classes with different iterator protocols (note that
+// mapping iterators are handled separately in the pythonizations) as exemplars (the
+// actual, resolved, names will be compiler-specific) that are picked b/c they are
+// baked into the CoreLegacy dictionary
     if (sIteratorTypes.empty()) {
         std::string tt = "<int>::";
         for (auto c : {"std::vector", "std::list", "std::deque"}) {

bindings/pyroot/cppyy/cppyy/LICENSE.txt

@@ -52,6 +52,7 @@ source code):
   Aditi Dutta
   Shaheed Haque
   Aaron Jomy
+  Jonas Rembser
   Toby StClere-Smithe
   Stefan Wunsch