[WIP][PyROOT] Move interoperability functions to ROOT.Numpy namespace

bindings/pyroot_experimental/pythonizations/python/ROOT/_facade.py

@@ -179,3 +179,33 @@ def _setattr(self, name, val):
     @property
     def __version__(self):
         return self.gROOT.GetVersion()
+
+    # Inject numpy pythonizations in the Numpy "namespace"
+    @property
+    def Numpy(self):
+        # Dummy objects to inject the pythonizations
+        class Numpy: pass
+        class Experimental: pass
+        Numpy.Experimental = Experimental
+
+        # Add pythonizations
+        try:
+            from libROOTPythonizations import AsRVec
+            Numpy.AsRVec = AsRVec
+        except:
+            pass
+        try:
+            from libROOTPythonizations import AsRTensor
+            Numpy.Experimental.AsRTensor = AsRTensor
+        except:
+            pass
+        try:
+            from libROOTPythonizations import MakeNumpyDataFrame
+            Numpy.MakeDataFrame = MakeNumpyDataFrame
+        except:
+            pass
+
+        # Add the pythonized dummy object to the ROOT facade and override this property
+        # so that we run the setup only once
+        self.__dict__['Numpy'] = Numpy
+        return Numpy

bindings/pyroot_experimental/pythonizations/python/ROOT/pythonization/_rbdt.py

@@ -9,14 +9,6 @@
 ################################################################################
 
 from ROOT import pythonization
-from libROOTPythonizations import AsRVec
-
-
-try:
-    from libROOTPythonizations import AsRTensor
-    has_rtensor = True
-except:
-    has_rtensor = False
 
 
 def Compute(self, x):
@@ -29,10 +21,12 @@ def Compute(self, x):
     # numpy.array is a factory and the actual type of a numpy array is numpy.ndarray
     if isinstance(x, np.ndarray):
         if len(x.shape) == 1:
+            from libROOTPythonizations import AsRVec
             x_ = AsRVec(x)
             y = self._OriginalCompute(x_)
             return np.asarray(y)
         elif len(x.shape) == 2:
+            from libROOTPythonizations import AsRTensor
             x_ = AsRTensor(x)
             y = self._OriginalCompute(x_)
             return np.asarray(y)
@@ -49,7 +43,7 @@ def pythonize_rbdt(klass, name):
     # klass: class to be pythonized
     # name: name of the class
 
-    if name.startswith("TMVA::Experimental::RBDT") and has_rtensor:
+    if name.startswith("TMVA::Experimental::RBDT"):
         klass._OriginalCompute = klass.Compute
         klass.Compute = Compute
 

bindings/pyroot_experimental/pythonizations/python/ROOT/pythonization/_rdataframe.py

@@ -160,11 +160,3 @@ def pythonize_rdataframe(klass, name):
             setattr(klass, method_name, partialmethod(_histo_profile, fixed_args))
 
     return True
-
-# Add MakeNumpyDataFrame feature as free function to the ROOT module
-try:
-    from libROOTPythonizations import MakeNumpyDataFrame
-    import cppyy
-    cppyy.gbl.ROOT.RDF.MakeNumpyDataFrame = MakeNumpyDataFrame
-except:
-    pass

bindings/pyroot_experimental/pythonizations/python/ROOT/pythonization/_rtensor.py

@@ -130,11 +130,3 @@ def pythonize_rtensor(klass, name):
         klass.__getitem__ = RTensorGetitem
 
     return True
-
-# Add AsRTensor feature as free function to the ROOT module
-try:
-    from libROOTPythonizations import AsRTensor
-    import cppyy
-    cppyy.gbl.TMVA.Experimental.AsRTensor = AsRTensor
-except:
-    pass

bindings/pyroot_experimental/pythonizations/python/ROOT/pythonization/_rvec.py

@@ -9,7 +9,7 @@
 ################################################################################
 
 from ROOT import pythonization
-from libROOTPythonizations import GetEndianess, GetDataPointer, GetSizeOfType, AsRVec
+from libROOTPythonizations import GetEndianess, GetDataPointer, GetSizeOfType
 
 
 _array_interface_dtype_map = {
@@ -64,8 +64,3 @@ def pythonize_rvec(klass, name):
         add_array_interface_property(klass, name)
 
     return True
-
-
-# Add AsRVec feature as free function to the ROOT module
-import cppyy
-cppyy.gbl.ROOT.VecOps.AsRVec = AsRVec

bindings/pyroot_experimental/pythonizations/test/CMakeLists.txt

@@ -4,6 +4,9 @@
 # For the licensing terms see $ROOTSYS/LICENSE.
 # For the list of contributors see $ROOTSYS/README/CREDITS.
 
+# Test library loads during importing ROOT
+ROOT_ADD_PYUNITTEST(pyroot_import_load_libs import_load_libs.py)
+
 # Test ROOT module
 ROOT_ADD_PYUNITTEST(pyroot_root_module root_module.py)
 

bindings/pyroot_experimental/pythonizations/test/import_load_libs.py

@@ -0,0 +1,123 @@
+import unittest
+import re
+import os
+
+
+class ImportLoadLibs(unittest.TestCase):
+    """
+    Test which libraries are loaded during importing ROOT
+    """
+
+    # The whitelist is a list of regex expressions that mark wanted libraries
+    # Note that the regex has to result in an exact match with the library name.
+    known_libs = [
+            # libCore and dependencies
+            'libCore',
+            'libm',
+            'liblz4',
+            'liblzma.*',
+            'libzstd',
+            'libz.*',
+            'libpthread',
+            'libc',
+            'libdl',
+            'libpcre',
+            # libCling and dependencies
+            'libCling',
+            'librt',
+            'libncurses.*',
+            'libtinfo', # by libncurses (on some platforms)
+            # libTree and dependencies
+            'libTree.*',
+            'libThread.*',
+            'libRIO.*',
+            'libNet.*',
+            'libImt.*',
+            'libMathCore.*',
+            'libssl.*',
+            'libcrypt.*', # by libssl
+            'libtbb.*',
+            # On centos7 libssl links against kerberos pulling in all dependencies below, removed with libssl1.1.0
+            'libgssapi_krb5',
+            'libkrb5',
+            'libk5crypto',
+            'libkrb5support',
+            'libselinux',
+            'libkeyutils',
+            'libcom_err',
+            'libresolv',
+            # cppyy and Python libraries
+            'libcppyy.*',
+            'libROOTPythonizations.*',
+            'libpython.*',
+            'libutil.*',
+            '.*cpython.*',
+            '_.*',
+            '.*module',
+            'operator',
+            'cStringIO',
+            'binascii',
+            'libbz2.*',
+            # System libraries and others
+            'libnss_.*',
+            'ld.*',
+            'libffi',
+            # Mac specific libraries
+            'Python',
+            'libMobileGestalt.*',
+            ]
+
+    # Verbose mode of the test
+    verbose = True
+
+    def test_import(self):
+        """
+        Test libraries loaded after importing ROOT
+        """
+        import ROOT
+        libs = str(ROOT.gSystem.GetLibraries())
+
+        if self.verbose:
+            print("Initial output from ROOT.gSystem.GetLibraries():\n" + libs)
+
+        # Split paths
+        libs = libs.split(' ')
+        for l in libs:
+            print(l)
+
+        # Get library name without full path and .so* suffix
+        libs = [os.path.basename(l).split('.so')[0] for l in libs \
+                if not l.startswith('-l') and not l.startswith('-L')]
+
+        # Check that the loaded libraries are white listed
+        bad_libs = []
+        good_libs = []
+        matched_re = []
+        for l in libs:
+            matched = False
+            for r in self.known_libs:
+                m = re.match(r, l)
+                if m:
+                    if m.group(0) == l:
+                        matched = True
+                        good_libs.append(l)
+                        matched_re.append(r)
+                        break
+            if not matched:
+                bad_libs.append(l)
+
+        if self.verbose:
+            print('Found whitelisted libraries after importing ROOT with the shown regex match:')
+            for l, r in zip(good_libs, matched_re):
+                print(' - {} ({})'.format(l, r))
+            import sys
+            sys.stdout.flush()
+
+        if bad_libs:
+            raise Exception('Found not whitelisted libraries after importing ROOT:' \
+                    + '\n - ' + '\n - '.join(bad_libs) \
+                    + '\nIf the test fails with a library that is loaded on purpose, please add it to the whitelist.')
+
+
+if __name__ == '__main__':
+    unittest.main()

bindings/pyroot_experimental/pythonizations/test/rdataframe_makenumpy.py

@@ -20,7 +20,7 @@ def test_dtypes(self):
         """
         for dtype in self.dtypes:
             data = {"x": np.array([1, 2, 3], dtype=dtype)}
-            df = ROOT.ROOT.RDF.MakeNumpyDataFrame(data)
+            df = ROOT.Numpy.MakeDataFrame(data)
             self.assertEqual(df.Mean("x").GetValue(), 2)
 
     def test_multiple_columns(self):
@@ -30,7 +30,7 @@ def test_multiple_columns(self):
         data = {}
         for dtype in self.dtypes:
             data[dtype] = np.array([1, 2, 3], dtype=dtype)
-        df = ROOT.ROOT.RDF.MakeNumpyDataFrame(data)
+        df = ROOT.Numpy.MakeDataFrame(data)
         colnames = df.GetColumnNames()
         # Test column names
         for dtype in colnames:
@@ -47,7 +47,7 @@ def test_refcount(self):
         self.assertEqual(sys.getrefcount(data), 2)
         self.assertEqual(sys.getrefcount(data["x"]), 2)
 
-        df = ROOT.ROOT.RDF.MakeNumpyDataFrame(data)
+        df = ROOT.Numpy.MakeDataFrame(data)
         self.assertTrue(hasattr(df, "__data__"))
         self.assertEqual(sys.getrefcount(df), 2)
 
@@ -58,7 +58,7 @@ def test_transformations(self):
         Test the use of transformations
         """
         data = {"x": np.array([1, 2, 3], dtype="float32")}
-        df = ROOT.ROOT.RDF.MakeNumpyDataFrame(data)
+        df = ROOT.Numpy.MakeDataFrame(data)
         df2 = df.Filter("x>1").Define("y", "2*x")
         self.assertEqual(df2.Mean("x").GetValue(), 2.5)
         self.assertEqual(df2.Mean("y").GetValue(), 5)
@@ -68,7 +68,7 @@ def test_delete_dict(self):
         Test behaviour with data dictionary going out of scope
         """
         data = {"x": np.array([1, 2, 3], dtype="float32")}
-        df = ROOT.ROOT.RDF.MakeNumpyDataFrame(data)
+        df = ROOT.Numpy.MakeDataFrame(data)
         del data
         self.assertEqual(df.Mean("x").GetValue(), 2)
 
@@ -78,15 +78,15 @@ def test_delete_numpy_array(self):
         """
         x = np.array([1, 2, 3], dtype="float32")
         data = {"x": x}
-        df = ROOT.ROOT.RDF.MakeNumpyDataFrame(data)
+        df = ROOT.Numpy.MakeDataFrame(data)
         del x
         self.assertEqual(df.Mean("x").GetValue(), 2)
 
     def test_inplace_dict(self):
         """
         Test behaviour with inplace dictionary
         """
-        df = ROOT.ROOT.RDF.MakeNumpyDataFrame({"x": np.array([1, 2, 3], dtype="float32")})
+        df = ROOT.Numpy.MakeDataFrame({"x": np.array([1, 2, 3], dtype="float32")})
         self.assertEqual(df.Mean("x").GetValue(), 2)
 
     def test_lifetime_numpy_array(self):
@@ -96,7 +96,7 @@ def test_lifetime_numpy_array(self):
         x = np.array([1, 2, 3], dtype="float32")
         ref1 = sys.getrefcount(x)
 
-        df = ROOT.ROOT.RDF.MakeNumpyDataFrame({"x": x})
+        df = ROOT.Numpy.MakeDataFrame({"x": x})
         ref2 = sys.getrefcount(x)
         self.assertEqual(ref2, ref1 + 1)
 
@@ -115,7 +115,7 @@ def test_lifetime_datasource(self):
 
         # Data source has dictionary with RVecs attached, which take a reference
         # to the numpy array
-        df = ROOT.ROOT.RDF.MakeNumpyDataFrame({"x": x})
+        df = ROOT.Numpy.MakeDataFrame({"x": x})
         m = df.Mean("x")
         ref2 = sys.getrefcount(x)
         self.assertEqual(ref1 + 1, ref2)

bindings/pyroot_experimental/pythonizations/test/rtensor.py

@@ -27,7 +27,7 @@ def test_dtypes(self):
         """
         for dtype in self.dtypes:
             np_obj = np.array([[1, 2], [3, 4], [5, 6]], dtype=dtype)
-            root_obj = ROOT.TMVA.Experimental.AsRTensor(np_obj)
+            root_obj = ROOT.Numpy.Experimental.AsRTensor(np_obj)
             self.assertTrue(check_shape(root_obj, np_obj))
             np_obj[0,0] = 42
             self.assertTrue(root_obj[0,0] == 42)
@@ -37,12 +37,12 @@ def test_memoryLayout(self):
         Test adoption of the memory layout
         """
         np_obj = np.array([[1, 2], [3, 4], [5, 6]])
-        root_obj = ROOT.TMVA.Experimental.AsRTensor(np_obj)
+        root_obj = ROOT.Numpy.Experimental.AsRTensor(np_obj)
         self.assertTrue(np_obj.flags.c_contiguous)
         self.assertEqual(root_obj.GetMemoryLayout(), 1)
 
         np_obj2 = np_obj.T
-        root_obj2 = ROOT.TMVA.Experimental.AsRTensor(np_obj2)
+        root_obj2 = ROOT.Numpy.Experimental.AsRTensor(np_obj2)
         self.assertTrue(np_obj2.flags.f_contiguous)
         self.assertEqual(root_obj2.GetMemoryLayout(), 2)
 
@@ -54,7 +54,7 @@ def test_strides(self):
         in bytes.
         """
         np_obj = np.array([[1, 2], [3, 4], [5, 6]], dtype="float32")
-        root_obj = ROOT.TMVA.Experimental.AsRTensor(np_obj)
+        root_obj = ROOT.Numpy.Experimental.AsRTensor(np_obj)
 
         np_strides = np_obj.strides
         root_strides = root_obj.GetStrides()
@@ -63,7 +63,7 @@ def test_strides(self):
             self.assertEqual(x, y * 4)
 
         np_obj = np_obj.T
-        root_obj = ROOT.TMVA.Experimental.AsRTensor(np_obj)
+        root_obj = ROOT.Numpy.Experimental.AsRTensor(np_obj)
         np_strides = np_obj.strides
         root_strides = root_obj.GetStrides()
         self.assertEqual(len(np_strides), len(root_strides))