diff --git a/bindings/pyroot_experimental/pythonizations/python/ROOT/_facade.py b/bindings/pyroot_experimental/pythonizations/python/ROOT/_facade.py index 33c77e5007c73..39cba55e00de3 100644 --- a/bindings/pyroot_experimental/pythonizations/python/ROOT/_facade.py +++ b/bindings/pyroot_experimental/pythonizations/python/ROOT/_facade.py @@ -179,3 +179,33 @@ def _setattr(self, name, val): @property def __version__(self): return self.gROOT.GetVersion() + + # Inject numpy pythonizations in the Numpy "namespace" + @property + def Numpy(self): + # Dummy objects to inject the pythonizations + class Numpy: pass + class Experimental: pass + Numpy.Experimental = Experimental + + # Add pythonizations + try: + from libROOTPythonizations import AsRVec + Numpy.AsRVec = AsRVec + except: + pass + try: + from libROOTPythonizations import AsRTensor + Numpy.Experimental.AsRTensor = AsRTensor + except: + pass + try: + from libROOTPythonizations import MakeNumpyDataFrame + Numpy.MakeDataFrame = MakeNumpyDataFrame + except: + pass + + # Add the pythonized dummy object to the ROOT facade and override this property + # so that we run the setup only once + self.__dict__['Numpy'] = Numpy + return Numpy diff --git a/bindings/pyroot_experimental/pythonizations/python/ROOT/pythonization/_rbdt.py b/bindings/pyroot_experimental/pythonizations/python/ROOT/pythonization/_rbdt.py index 4b4b54b206b7c..5e4de0a9673f4 100644 --- a/bindings/pyroot_experimental/pythonizations/python/ROOT/pythonization/_rbdt.py +++ b/bindings/pyroot_experimental/pythonizations/python/ROOT/pythonization/_rbdt.py @@ -9,14 +9,6 @@ ################################################################################ from ROOT import pythonization -from libROOTPythonizations import AsRVec - - -try: - from libROOTPythonizations import AsRTensor - has_rtensor = True -except: - has_rtensor = False def Compute(self, x): @@ -29,10 +21,12 @@ def Compute(self, x): # numpy.array is a factory and the actual type of a numpy array is numpy.ndarray if isinstance(x, np.ndarray): if len(x.shape) == 1: + from libROOTPythonizations import AsRVec x_ = AsRVec(x) y = self._OriginalCompute(x_) return np.asarray(y) elif len(x.shape) == 2: + from libROOTPythonizations import AsRTensor x_ = AsRTensor(x) y = self._OriginalCompute(x_) return np.asarray(y) @@ -49,7 +43,7 @@ def pythonize_rbdt(klass, name): # klass: class to be pythonized # name: name of the class - if name.startswith("TMVA::Experimental::RBDT") and has_rtensor: + if name.startswith("TMVA::Experimental::RBDT"): klass._OriginalCompute = klass.Compute klass.Compute = Compute diff --git a/bindings/pyroot_experimental/pythonizations/python/ROOT/pythonization/_rdataframe.py b/bindings/pyroot_experimental/pythonizations/python/ROOT/pythonization/_rdataframe.py index c964a9d78974f..2248fd595eb0f 100644 --- a/bindings/pyroot_experimental/pythonizations/python/ROOT/pythonization/_rdataframe.py +++ b/bindings/pyroot_experimental/pythonizations/python/ROOT/pythonization/_rdataframe.py @@ -160,11 +160,3 @@ def pythonize_rdataframe(klass, name): setattr(klass, method_name, partialmethod(_histo_profile, fixed_args)) return True - -# Add MakeNumpyDataFrame feature as free function to the ROOT module -try: - from libROOTPythonizations import MakeNumpyDataFrame - import cppyy - cppyy.gbl.ROOT.RDF.MakeNumpyDataFrame = MakeNumpyDataFrame -except: - pass diff --git a/bindings/pyroot_experimental/pythonizations/python/ROOT/pythonization/_rtensor.py b/bindings/pyroot_experimental/pythonizations/python/ROOT/pythonization/_rtensor.py index 6ac10a8cc0a93..051543228d521 100644 --- a/bindings/pyroot_experimental/pythonizations/python/ROOT/pythonization/_rtensor.py +++ b/bindings/pyroot_experimental/pythonizations/python/ROOT/pythonization/_rtensor.py @@ -130,11 +130,3 @@ def pythonize_rtensor(klass, name): klass.__getitem__ = RTensorGetitem return True - -# Add AsRTensor feature as free function to the ROOT module -try: - from libROOTPythonizations import AsRTensor - import cppyy - cppyy.gbl.TMVA.Experimental.AsRTensor = AsRTensor -except: - pass diff --git a/bindings/pyroot_experimental/pythonizations/python/ROOT/pythonization/_rvec.py b/bindings/pyroot_experimental/pythonizations/python/ROOT/pythonization/_rvec.py index af294acd1d9c3..0f05285d6bbbc 100644 --- a/bindings/pyroot_experimental/pythonizations/python/ROOT/pythonization/_rvec.py +++ b/bindings/pyroot_experimental/pythonizations/python/ROOT/pythonization/_rvec.py @@ -9,7 +9,7 @@ ################################################################################ from ROOT import pythonization -from libROOTPythonizations import GetEndianess, GetDataPointer, GetSizeOfType, AsRVec +from libROOTPythonizations import GetEndianess, GetDataPointer, GetSizeOfType _array_interface_dtype_map = { @@ -64,8 +64,3 @@ def pythonize_rvec(klass, name): add_array_interface_property(klass, name) return True - - -# Add AsRVec feature as free function to the ROOT module -import cppyy -cppyy.gbl.ROOT.VecOps.AsRVec = AsRVec diff --git a/bindings/pyroot_experimental/pythonizations/test/CMakeLists.txt b/bindings/pyroot_experimental/pythonizations/test/CMakeLists.txt index eb2ebbf0c3223..6a802207e987f 100644 --- a/bindings/pyroot_experimental/pythonizations/test/CMakeLists.txt +++ b/bindings/pyroot_experimental/pythonizations/test/CMakeLists.txt @@ -4,6 +4,9 @@ # For the licensing terms see $ROOTSYS/LICENSE. # For the list of contributors see $ROOTSYS/README/CREDITS. +# Test library loads during importing ROOT +ROOT_ADD_PYUNITTEST(pyroot_import_load_libs import_load_libs.py) + # Test ROOT module ROOT_ADD_PYUNITTEST(pyroot_root_module root_module.py) diff --git a/bindings/pyroot_experimental/pythonizations/test/import_load_libs.py b/bindings/pyroot_experimental/pythonizations/test/import_load_libs.py new file mode 100644 index 0000000000000..6a277f285383c --- /dev/null +++ b/bindings/pyroot_experimental/pythonizations/test/import_load_libs.py @@ -0,0 +1,123 @@ +import unittest +import re +import os + + +class ImportLoadLibs(unittest.TestCase): + """ + Test which libraries are loaded during importing ROOT + """ + + # The whitelist is a list of regex expressions that mark wanted libraries + # Note that the regex has to result in an exact match with the library name. + known_libs = [ + # libCore and dependencies + 'libCore', + 'libm', + 'liblz4', + 'liblzma.*', + 'libzstd', + 'libz.*', + 'libpthread', + 'libc', + 'libdl', + 'libpcre', + # libCling and dependencies + 'libCling', + 'librt', + 'libncurses.*', + 'libtinfo', # by libncurses (on some platforms) + # libTree and dependencies + 'libTree.*', + 'libThread.*', + 'libRIO.*', + 'libNet.*', + 'libImt.*', + 'libMathCore.*', + 'libssl.*', + 'libcrypt.*', # by libssl + 'libtbb.*', + # On centos7 libssl links against kerberos pulling in all dependencies below, removed with libssl1.1.0 + 'libgssapi_krb5', + 'libkrb5', + 'libk5crypto', + 'libkrb5support', + 'libselinux', + 'libkeyutils', + 'libcom_err', + 'libresolv', + # cppyy and Python libraries + 'libcppyy.*', + 'libROOTPythonizations.*', + 'libpython.*', + 'libutil.*', + '.*cpython.*', + '_.*', + '.*module', + 'operator', + 'cStringIO', + 'binascii', + 'libbz2.*', + # System libraries and others + 'libnss_.*', + 'ld.*', + 'libffi', + # Mac specific libraries + 'Python', + 'libMobileGestalt.*', + ] + + # Verbose mode of the test + verbose = True + + def test_import(self): + """ + Test libraries loaded after importing ROOT + """ + import ROOT + libs = str(ROOT.gSystem.GetLibraries()) + + if self.verbose: + print("Initial output from ROOT.gSystem.GetLibraries():\n" + libs) + + # Split paths + libs = libs.split(' ') + for l in libs: + print(l) + + # Get library name without full path and .so* suffix + libs = [os.path.basename(l).split('.so')[0] for l in libs \ + if not l.startswith('-l') and not l.startswith('-L')] + + # Check that the loaded libraries are white listed + bad_libs = [] + good_libs = [] + matched_re = [] + for l in libs: + matched = False + for r in self.known_libs: + m = re.match(r, l) + if m: + if m.group(0) == l: + matched = True + good_libs.append(l) + matched_re.append(r) + break + if not matched: + bad_libs.append(l) + + if self.verbose: + print('Found whitelisted libraries after importing ROOT with the shown regex match:') + for l, r in zip(good_libs, matched_re): + print(' - {} ({})'.format(l, r)) + import sys + sys.stdout.flush() + + if bad_libs: + raise Exception('Found not whitelisted libraries after importing ROOT:' \ + + '\n - ' + '\n - '.join(bad_libs) \ + + '\nIf the test fails with a library that is loaded on purpose, please add it to the whitelist.') + + +if __name__ == '__main__': + unittest.main() diff --git a/bindings/pyroot_experimental/pythonizations/test/rdataframe_makenumpy.py b/bindings/pyroot_experimental/pythonizations/test/rdataframe_makenumpy.py index 43173b8fa97b1..b21f53fc2436c 100644 --- a/bindings/pyroot_experimental/pythonizations/test/rdataframe_makenumpy.py +++ b/bindings/pyroot_experimental/pythonizations/test/rdataframe_makenumpy.py @@ -20,7 +20,7 @@ def test_dtypes(self): """ for dtype in self.dtypes: data = {"x": np.array([1, 2, 3], dtype=dtype)} - df = ROOT.ROOT.RDF.MakeNumpyDataFrame(data) + df = ROOT.Numpy.MakeDataFrame(data) self.assertEqual(df.Mean("x").GetValue(), 2) def test_multiple_columns(self): @@ -30,7 +30,7 @@ def test_multiple_columns(self): data = {} for dtype in self.dtypes: data[dtype] = np.array([1, 2, 3], dtype=dtype) - df = ROOT.ROOT.RDF.MakeNumpyDataFrame(data) + df = ROOT.Numpy.MakeDataFrame(data) colnames = df.GetColumnNames() # Test column names for dtype in colnames: @@ -47,7 +47,7 @@ def test_refcount(self): self.assertEqual(sys.getrefcount(data), 2) self.assertEqual(sys.getrefcount(data["x"]), 2) - df = ROOT.ROOT.RDF.MakeNumpyDataFrame(data) + df = ROOT.Numpy.MakeDataFrame(data) self.assertTrue(hasattr(df, "__data__")) self.assertEqual(sys.getrefcount(df), 2) @@ -58,7 +58,7 @@ def test_transformations(self): Test the use of transformations """ data = {"x": np.array([1, 2, 3], dtype="float32")} - df = ROOT.ROOT.RDF.MakeNumpyDataFrame(data) + df = ROOT.Numpy.MakeDataFrame(data) df2 = df.Filter("x>1").Define("y", "2*x") self.assertEqual(df2.Mean("x").GetValue(), 2.5) self.assertEqual(df2.Mean("y").GetValue(), 5) @@ -68,7 +68,7 @@ def test_delete_dict(self): Test behaviour with data dictionary going out of scope """ data = {"x": np.array([1, 2, 3], dtype="float32")} - df = ROOT.ROOT.RDF.MakeNumpyDataFrame(data) + df = ROOT.Numpy.MakeDataFrame(data) del data self.assertEqual(df.Mean("x").GetValue(), 2) @@ -78,7 +78,7 @@ def test_delete_numpy_array(self): """ x = np.array([1, 2, 3], dtype="float32") data = {"x": x} - df = ROOT.ROOT.RDF.MakeNumpyDataFrame(data) + df = ROOT.Numpy.MakeDataFrame(data) del x self.assertEqual(df.Mean("x").GetValue(), 2) @@ -86,7 +86,7 @@ def test_inplace_dict(self): """ Test behaviour with inplace dictionary """ - df = ROOT.ROOT.RDF.MakeNumpyDataFrame({"x": np.array([1, 2, 3], dtype="float32")}) + df = ROOT.Numpy.MakeDataFrame({"x": np.array([1, 2, 3], dtype="float32")}) self.assertEqual(df.Mean("x").GetValue(), 2) def test_lifetime_numpy_array(self): @@ -96,7 +96,7 @@ def test_lifetime_numpy_array(self): x = np.array([1, 2, 3], dtype="float32") ref1 = sys.getrefcount(x) - df = ROOT.ROOT.RDF.MakeNumpyDataFrame({"x": x}) + df = ROOT.Numpy.MakeDataFrame({"x": x}) ref2 = sys.getrefcount(x) self.assertEqual(ref2, ref1 + 1) @@ -115,7 +115,7 @@ def test_lifetime_datasource(self): # Data source has dictionary with RVecs attached, which take a reference # to the numpy array - df = ROOT.ROOT.RDF.MakeNumpyDataFrame({"x": x}) + df = ROOT.Numpy.MakeDataFrame({"x": x}) m = df.Mean("x") ref2 = sys.getrefcount(x) self.assertEqual(ref1 + 1, ref2) diff --git a/bindings/pyroot_experimental/pythonizations/test/rtensor.py b/bindings/pyroot_experimental/pythonizations/test/rtensor.py index 231dcb031e426..c661527fe96e5 100644 --- a/bindings/pyroot_experimental/pythonizations/test/rtensor.py +++ b/bindings/pyroot_experimental/pythonizations/test/rtensor.py @@ -27,7 +27,7 @@ def test_dtypes(self): """ for dtype in self.dtypes: np_obj = np.array([[1, 2], [3, 4], [5, 6]], dtype=dtype) - root_obj = ROOT.TMVA.Experimental.AsRTensor(np_obj) + root_obj = ROOT.Numpy.Experimental.AsRTensor(np_obj) self.assertTrue(check_shape(root_obj, np_obj)) np_obj[0,0] = 42 self.assertTrue(root_obj[0,0] == 42) @@ -37,12 +37,12 @@ def test_memoryLayout(self): Test adoption of the memory layout """ np_obj = np.array([[1, 2], [3, 4], [5, 6]]) - root_obj = ROOT.TMVA.Experimental.AsRTensor(np_obj) + root_obj = ROOT.Numpy.Experimental.AsRTensor(np_obj) self.assertTrue(np_obj.flags.c_contiguous) self.assertEqual(root_obj.GetMemoryLayout(), 1) np_obj2 = np_obj.T - root_obj2 = ROOT.TMVA.Experimental.AsRTensor(np_obj2) + root_obj2 = ROOT.Numpy.Experimental.AsRTensor(np_obj2) self.assertTrue(np_obj2.flags.f_contiguous) self.assertEqual(root_obj2.GetMemoryLayout(), 2) @@ -54,7 +54,7 @@ def test_strides(self): in bytes. """ np_obj = np.array([[1, 2], [3, 4], [5, 6]], dtype="float32") - root_obj = ROOT.TMVA.Experimental.AsRTensor(np_obj) + root_obj = ROOT.Numpy.Experimental.AsRTensor(np_obj) np_strides = np_obj.strides root_strides = root_obj.GetStrides() @@ -63,7 +63,7 @@ def test_strides(self): self.assertEqual(x, y * 4) np_obj = np_obj.T - root_obj = ROOT.TMVA.Experimental.AsRTensor(np_obj) + root_obj = ROOT.Numpy.Experimental.AsRTensor(np_obj) np_strides = np_obj.strides root_strides = root_obj.GetStrides() self.assertEqual(len(np_strides), len(root_strides)) diff --git a/bindings/pyroot_experimental/pythonizations/test/rvec_asrvec.py b/bindings/pyroot_experimental/pythonizations/test/rvec_asrvec.py index bccfe4e7ebc53..0fe856e6300d9 100644 --- a/bindings/pyroot_experimental/pythonizations/test/rvec_asrvec.py +++ b/bindings/pyroot_experimental/pythonizations/test/rvec_asrvec.py @@ -44,7 +44,7 @@ def test_dtypes(self): """ for dtype in self.dtypes: np_obj = np.empty(2, dtype=dtype) - root_obj = ROOT.ROOT.VecOps.AsRVec(np_obj) + root_obj = ROOT.Numpy.AsRVec(np_obj) self.check_memory_adoption(root_obj, np_obj) self.check_size(2, root_obj) @@ -53,7 +53,7 @@ def test_multidim(self): Test adoption of multi-dimensional numpy arrays """ np_obj = np.array([[1, 2], [3, 4]], dtype="float32") - rvec = ROOT.ROOT.VecOps.AsRVec(np_obj) + rvec = ROOT.Numpy.AsRVec(np_obj) self.assertEqual(rvec.size(), 4) def test_size_zero(self): @@ -61,7 +61,7 @@ def test_size_zero(self): Test adoption of numpy array with size 0 """ np_obj = np.array([], dtype="float32") - rvec = ROOT.ROOT.VecOps.AsRVec(np_obj) + rvec = ROOT.Numpy.AsRVec(np_obj) self.assertEqual(rvec.size(), 0) def test_adopt_rvec(self): @@ -70,7 +70,7 @@ def test_adopt_rvec(self): """ rvec = ROOT.ROOT.VecOps.RVec("float")(1) rvec[0] = 42 - rvec2 = ROOT.ROOT.VecOps.AsRVec(rvec) + rvec2 = ROOT.Numpy.AsRVec(rvec) self.assertEqual(rvec.size(), rvec2.size()) self.assertEqual(rvec[0], rvec2[0]) rvec2[0] = 43 @@ -81,7 +81,7 @@ def test_ownership(self): Test ownership of returned RVec (to be owned by Python) """ np_obj = np.array([1, 2]) - rvec = ROOT.ROOT.VecOps.AsRVec(np_obj) + rvec = ROOT.Numpy.AsRVec(np_obj) self.assertEqual(rvec.__python_owns__, True) def test_attribute_adopted(self): @@ -89,7 +89,7 @@ def test_attribute_adopted(self): Test __adopted__ attribute of returned RVecs """ np_obj = np.array([1, 2]) - rvec = ROOT.ROOT.VecOps.AsRVec(np_obj) + rvec = ROOT.Numpy.AsRVec(np_obj) self.assertTrue(hasattr(rvec, "__adopted__")) self.assertEqual(id(rvec.__adopted__), id(np_obj)) @@ -104,7 +104,7 @@ def test_refcount(self): is decreased. """ np_obj = np.array([1, 2]) - rvec = ROOT.ROOT.VecOps.AsRVec(np_obj) + rvec = ROOT.Numpy.AsRVec(np_obj) self.assertEqual(sys.getrefcount(rvec), 2) self.assertEqual(sys.getrefcount(np_obj), 3) del rvec