[PyROOT] Make namespace pythonizations lazy

bindings/pyroot_experimental/pythonizations/python/ROOT/_facade.py

@@ -179,3 +179,43 @@ def _setattr(self, name, val):
     @property
     def __version__(self):
         return self.gROOT.GetVersion()
+
+    # Overload VecOps namespace
+    # The property gets the C++ namespace, adds the pythonizations and
+    # eventually deletes itself so that following calls go directly
+    # to the C++ namespace. This mechanic ensures that we pythonize the
+    # namespace lazily.
+    @property
+    def VecOps(self):
+        ns = self._fallback_getattr('VecOps')
+        try:
+            from libROOTPythonizations import AsRVec
+            ns.AsRVec = AsRVec
+        except:
+            raise Exception('Failed to pythonize the namespace VecOps')
+        del type(self).VecOps
+        return ns
+
+    # Overload RDF namespace
+    @property
+    def RDF(self):
+        ns = self._fallback_getattr('RDF')
+        try:
+            from libROOTPythonizations import MakeNumpyDataFrame
+            ns.MakeNumpyDataFrame = MakeNumpyDataFrame
+        except:
+            raise Exception('Failed to pythonize the namespace RDF')
+        del type(self).RDF
+        return ns
+
+    # Overload TMVA namespace
+    @property
+    def TMVA(self):
+        ns = self._fallback_getattr('TMVA')
+        try:
+            from libROOTPythonizations import AsRTensor
+            ns.Experimental.AsRTensor = AsRTensor
+        except:
+            raise Exception('Failed to pythonize the namespace TMVA')
+        del type(self).TMVA
+        return ns

bindings/pyroot_experimental/pythonizations/python/ROOT/pythonization/_rbdt.py

@@ -9,14 +9,7 @@
 ################################################################################
 
 from ROOT import pythonization
-from libROOTPythonizations import AsRVec
-
-
-try:
-    from libROOTPythonizations import AsRTensor
-    has_rtensor = True
-except:
-    has_rtensor = False
+from cppyy import gbl as gbl_namespace
 
 
 def Compute(self, x):
@@ -29,11 +22,11 @@ def Compute(self, x):
     # numpy.array is a factory and the actual type of a numpy array is numpy.ndarray
     if isinstance(x, np.ndarray):
         if len(x.shape) == 1:
-            x_ = AsRVec(x)
+            x_ = gbl_namespace.VecOps.AsRVec(x)
             y = self._OriginalCompute(x_)
             return np.asarray(y)
         elif len(x.shape) == 2:
-            x_ = AsRTensor(x)
+            x_ = gbl_namespace.TMVA.Experimental.AsRTensor(x)
             y = self._OriginalCompute(x_)
             return np.asarray(y)
         else:
@@ -49,7 +42,7 @@ def pythonize_rbdt(klass, name):
     # klass: class to be pythonized
     # name: name of the class
 
-    if name.startswith("TMVA::Experimental::RBDT") and has_rtensor:
+    if name.startswith("TMVA::Experimental::RBDT"):
         klass._OriginalCompute = klass.Compute
         klass.Compute = Compute
 

bindings/pyroot_experimental/pythonizations/python/ROOT/pythonization/_rdataframe.py

@@ -160,11 +160,3 @@ def pythonize_rdataframe(klass, name):
             setattr(klass, method_name, partialmethod(_histo_profile, fixed_args))
 
     return True
-
-# Add MakeNumpyDataFrame feature as free function to the ROOT module
-try:
-    from libROOTPythonizations import MakeNumpyDataFrame
-    import cppyy
-    cppyy.gbl.ROOT.RDF.MakeNumpyDataFrame = MakeNumpyDataFrame
-except:
-    pass

bindings/pyroot_experimental/pythonizations/python/ROOT/pythonization/_rtensor.py

@@ -130,11 +130,3 @@ def pythonize_rtensor(klass, name):
         klass.__getitem__ = RTensorGetitem
 
     return True
-
-# Add AsRTensor feature as free function to the ROOT module
-try:
-    from libROOTPythonizations import AsRTensor
-    import cppyy
-    cppyy.gbl.TMVA.Experimental.AsRTensor = AsRTensor
-except:
-    pass

bindings/pyroot_experimental/pythonizations/python/ROOT/pythonization/_rvec.py

@@ -9,7 +9,7 @@
 ################################################################################
 
 from ROOT import pythonization
-from libROOTPythonizations import GetEndianess, GetDataPointer, GetSizeOfType, AsRVec
+from libROOTPythonizations import GetEndianess, GetDataPointer, GetSizeOfType
 
 
 _array_interface_dtype_map = {
@@ -64,8 +64,3 @@ def pythonize_rvec(klass, name):
         add_array_interface_property(klass, name)
 
     return True
-
-
-# Add AsRVec feature as free function to the ROOT module
-import cppyy
-cppyy.gbl.ROOT.VecOps.AsRVec = AsRVec

bindings/pyroot_experimental/pythonizations/test/CMakeLists.txt

@@ -4,6 +4,13 @@
 # For the licensing terms see $ROOTSYS/LICENSE.
 # For the list of contributors see $ROOTSYS/README/CREDITS.
 
+# Test library loads during importing ROOT
+# Testing only the Linux systems is sufficient to detect unwanted links to libraries at import time.
+# Mac (and potentially Windows) pull in many system libraries which makes this test very complex.
+if (NOT APPLE AND NOT WIN32)
+    ROOT_ADD_PYUNITTEST(pyroot_import_load_libs import_load_libs.py)
+endif()
+
 # Test ROOT module
 ROOT_ADD_PYUNITTEST(pyroot_root_module root_module.py)
 

bindings/pyroot_experimental/pythonizations/test/import_load_libs.py

@@ -0,0 +1,118 @@
+import unittest
+import re
+import os
+
+
+class ImportLoadLibs(unittest.TestCase):
+    """
+    Test which libraries are loaded during importing ROOT
+    """
+
+    # The whitelist is a list of regex expressions that mark wanted libraries
+    # Note that the regex has to result in an exact match with the library name.
+    known_libs = [
+            # libCore and dependencies
+            'libCore',
+            'libm',
+            'liblz4',
+            'liblzma',
+            'libzstd',
+            'libz',
+            'libpthread',
+            'libc',
+            'libdl',
+            'libpcre',
+            # libCling and dependencies
+            'libCling.*',
+            'librt',
+            'libncurses.*',
+            'libtinfo', # by libncurses (on some older platforms)
+            # libTree and dependencies
+            'libTree',
+            'libThread',
+            'libRIO',
+            'libNet',
+            'libImt',
+            'libMathCore',
+            'libssl',
+            'libcrypt.*', # by libssl
+            'libtbb',
+            # On centos7 libssl links against kerberos pulling in all dependencies below, removed with libssl1.1.0
+            'libgssapi_krb5',
+            'libkrb5',
+            'libk5crypto',
+            'libkrb5support',
+            'libselinux',
+            'libkeyutils',
+            'libcom_err',
+            'libresolv',
+            # cppyy and Python libraries
+            'libcppyy.*',
+            'libROOTPythonizations.*',
+            'libpython.*',
+            'libutil.*',
+            '.*cpython.*',
+            '_.*',
+            '.*module',
+            'operator',
+            'cStringIO',
+            'binascii',
+            'libbz2',
+            # System libraries and others
+            'libnss_.*',
+            'ld.*',
+            'libffi',
+            ]
+
+    # Verbose mode of the test
+    verbose = False
+
+    def test_import(self):
+        """
+        Test libraries loaded after importing ROOT
+        """
+        import ROOT
+        libs = str(ROOT.gSystem.GetLibraries())
+
+        if self.verbose:
+            print("Initial output from ROOT.gSystem.GetLibraries():\n" + libs)
+
+        # Split paths
+        libs = libs.split(' ')
+
+        # Get library name without full path and .so* suffix
+        libs = [os.path.basename(l).split('.so')[0] for l in libs \
+                if not l.startswith('-l') and not l.startswith('-L')]
+
+        # Check that the loaded libraries are white listed
+        bad_libs = []
+        good_libs = []
+        matched_re = []
+        for l in libs:
+            matched = False
+            for r in self.known_libs:
+                m = re.match(r, l)
+                if m:
+                    if m.group(0) == l:
+                        matched = True
+                        good_libs.append(l)
+                        matched_re.append(r)
+                        break
+            if not matched:
+                bad_libs.append(l)
+
+        if self.verbose:
+            print('Found whitelisted libraries after importing ROOT with the shown regex match:')
+            for l, r in zip(good_libs, matched_re):
+                print(' - {} ({})'.format(l, r))
+            import sys
+            sys.stdout.flush()
+
+        if bad_libs:
+            raise Exception('Found not whitelisted libraries after importing ROOT:' \
+                    + '\n - ' + '\n - '.join(bad_libs) \
+                    + '\nIf the test fails with a library that is loaded on purpose, please add it to the whitelist.')
+
+
+if __name__ == '__main__':
+    unittest.main()

bindings/pyroot_experimental/pythonizations/test/rdataframe_makenumpy.py

@@ -20,7 +20,7 @@ def test_dtypes(self):
         """
         for dtype in self.dtypes:
             data = {"x": np.array([1, 2, 3], dtype=dtype)}
-            df = ROOT.ROOT.RDF.MakeNumpyDataFrame(data)
+            df = ROOT.RDF.MakeNumpyDataFrame(data)
             self.assertEqual(df.Mean("x").GetValue(), 2)
 
     def test_multiple_columns(self):
@@ -30,7 +30,7 @@ def test_multiple_columns(self):
         data = {}
         for dtype in self.dtypes:
             data[dtype] = np.array([1, 2, 3], dtype=dtype)
-        df = ROOT.ROOT.RDF.MakeNumpyDataFrame(data)
+        df = ROOT.RDF.MakeNumpyDataFrame(data)
         colnames = df.GetColumnNames()
         # Test column names
         for dtype in colnames:
@@ -47,7 +47,7 @@ def test_refcount(self):
         self.assertEqual(sys.getrefcount(data), 2)
         self.assertEqual(sys.getrefcount(data["x"]), 2)
 
-        df = ROOT.ROOT.RDF.MakeNumpyDataFrame(data)
+        df = ROOT.RDF.MakeNumpyDataFrame(data)
         self.assertTrue(hasattr(df, "__data__"))
         self.assertEqual(sys.getrefcount(df), 2)
 
@@ -58,7 +58,7 @@ def test_transformations(self):
         Test the use of transformations
         """
         data = {"x": np.array([1, 2, 3], dtype="float32")}
-        df = ROOT.ROOT.RDF.MakeNumpyDataFrame(data)
+        df = ROOT.RDF.MakeNumpyDataFrame(data)
         df2 = df.Filter("x>1").Define("y", "2*x")
         self.assertEqual(df2.Mean("x").GetValue(), 2.5)
         self.assertEqual(df2.Mean("y").GetValue(), 5)
@@ -68,7 +68,7 @@ def test_delete_dict(self):
         Test behaviour with data dictionary going out of scope
         """
         data = {"x": np.array([1, 2, 3], dtype="float32")}
-        df = ROOT.ROOT.RDF.MakeNumpyDataFrame(data)
+        df = ROOT.RDF.MakeNumpyDataFrame(data)
         del data
         self.assertEqual(df.Mean("x").GetValue(), 2)
 
@@ -78,15 +78,15 @@ def test_delete_numpy_array(self):
         """
         x = np.array([1, 2, 3], dtype="float32")
         data = {"x": x}
-        df = ROOT.ROOT.RDF.MakeNumpyDataFrame(data)
+        df = ROOT.RDF.MakeNumpyDataFrame(data)
         del x
         self.assertEqual(df.Mean("x").GetValue(), 2)
 
     def test_inplace_dict(self):
         """
         Test behaviour with inplace dictionary
         """
-        df = ROOT.ROOT.RDF.MakeNumpyDataFrame({"x": np.array([1, 2, 3], dtype="float32")})
+        df = ROOT.RDF.MakeNumpyDataFrame({"x": np.array([1, 2, 3], dtype="float32")})
         self.assertEqual(df.Mean("x").GetValue(), 2)
 
     def test_lifetime_numpy_array(self):
@@ -96,7 +96,7 @@ def test_lifetime_numpy_array(self):
         x = np.array([1, 2, 3], dtype="float32")
         ref1 = sys.getrefcount(x)
 
-        df = ROOT.ROOT.RDF.MakeNumpyDataFrame({"x": x})
+        df = ROOT.RDF.MakeNumpyDataFrame({"x": x})
         ref2 = sys.getrefcount(x)
         self.assertEqual(ref2, ref1 + 1)
 
@@ -115,7 +115,7 @@ def test_lifetime_datasource(self):
 
         # Data source has dictionary with RVecs attached, which take a reference
         # to the numpy array
-        df = ROOT.ROOT.RDF.MakeNumpyDataFrame({"x": x})
+        df = ROOT.RDF.MakeNumpyDataFrame({"x": x})
         m = df.Mean("x")
         ref2 = sys.getrefcount(x)
         self.assertEqual(ref1 + 1, ref2)