From fcd4bcaab7185db7c2403b077d55fd1d5600122b Mon Sep 17 00:00:00 2001 From: Stefan Wunsch Date: Tue, 31 Mar 2020 10:51:57 +0200 Subject: [PATCH 1/2] [PyROOT] Add test checking loaded libraries after import --- .../pythonizations/test/CMakeLists.txt | 7 ++ .../pythonizations/test/import_load_libs.py | 118 ++++++++++++++++++ 2 files changed, 125 insertions(+) create mode 100644 bindings/pyroot_experimental/pythonizations/test/import_load_libs.py diff --git a/bindings/pyroot_experimental/pythonizations/test/CMakeLists.txt b/bindings/pyroot_experimental/pythonizations/test/CMakeLists.txt index eb2ebbf0c3223..7c047c0c589aa 100644 --- a/bindings/pyroot_experimental/pythonizations/test/CMakeLists.txt +++ b/bindings/pyroot_experimental/pythonizations/test/CMakeLists.txt @@ -4,6 +4,13 @@ # For the licensing terms see $ROOTSYS/LICENSE. # For the list of contributors see $ROOTSYS/README/CREDITS. +# Test library loads during importing ROOT +# Testing only the Linux systems is sufficient to detect unwanted links to libraries at import time. +# Mac (and potentially Windows) pull in many system libraries which makes this test very complex. +if (NOT APPLE AND NOT WIN32) + ROOT_ADD_PYUNITTEST(pyroot_import_load_libs import_load_libs.py) +endif() + # Test ROOT module ROOT_ADD_PYUNITTEST(pyroot_root_module root_module.py) diff --git a/bindings/pyroot_experimental/pythonizations/test/import_load_libs.py b/bindings/pyroot_experimental/pythonizations/test/import_load_libs.py new file mode 100644 index 0000000000000..efcdf42fae308 --- /dev/null +++ b/bindings/pyroot_experimental/pythonizations/test/import_load_libs.py @@ -0,0 +1,118 @@ +import unittest +import re +import os + + +class ImportLoadLibs(unittest.TestCase): + """ + Test which libraries are loaded during importing ROOT + """ + + # The whitelist is a list of regex expressions that mark wanted libraries + # Note that the regex has to result in an exact match with the library name. + known_libs = [ + # libCore and dependencies + 'libCore', + 'libm', + 'liblz4', + 'liblzma', + 'libzstd', + 'libz', + 'libpthread', + 'libc', + 'libdl', + 'libpcre', + # libCling and dependencies + 'libCling.*', + 'librt', + 'libncurses.*', + 'libtinfo', # by libncurses (on some older platforms) + # libTree and dependencies + 'libTree', + 'libThread', + 'libRIO', + 'libNet', + 'libImt', + 'libMathCore', + 'libssl', + 'libcrypt.*', # by libssl + 'libtbb', + # On centos7 libssl links against kerberos pulling in all dependencies below, removed with libssl1.1.0 + 'libgssapi_krb5', + 'libkrb5', + 'libk5crypto', + 'libkrb5support', + 'libselinux', + 'libkeyutils', + 'libcom_err', + 'libresolv', + # cppyy and Python libraries + 'libcppyy.*', + 'libROOTPythonizations.*', + 'libpython.*', + 'libutil.*', + '.*cpython.*', + '_.*', + '.*module', + 'operator', + 'cStringIO', + 'binascii', + 'libbz2', + # System libraries and others + 'libnss_.*', + 'ld.*', + 'libffi', + ] + + # Verbose mode of the test + verbose = False + + def test_import(self): + """ + Test libraries loaded after importing ROOT + """ + import ROOT + libs = str(ROOT.gSystem.GetLibraries()) + + if self.verbose: + print("Initial output from ROOT.gSystem.GetLibraries():\n" + libs) + + # Split paths + libs = libs.split(' ') + + # Get library name without full path and .so* suffix + libs = [os.path.basename(l).split('.so')[0] for l in libs \ + if not l.startswith('-l') and not l.startswith('-L')] + + # Check that the loaded libraries are white listed + bad_libs = [] + good_libs = [] + matched_re = [] + for l in libs: + matched = False + for r in self.known_libs: + m = re.match(r, l) + if m: + if m.group(0) == l: + matched = True + good_libs.append(l) + matched_re.append(r) + break + if not matched: + bad_libs.append(l) + + if self.verbose: + print('Found whitelisted libraries after importing ROOT with the shown regex match:') + for l, r in zip(good_libs, matched_re): + print(' - {} ({})'.format(l, r)) + import sys + sys.stdout.flush() + + if bad_libs: + raise Exception('Found not whitelisted libraries after importing ROOT:' \ + + '\n - ' + '\n - '.join(bad_libs) \ + + '\nIf the test fails with a library that is loaded on purpose, please add it to the whitelist.') + + +if __name__ == '__main__': + unittest.main() From 40d33a8414cfb88c8cfcb154a7a6fde5ab2cdf09 Mon Sep 17 00:00:00 2001 From: Stefan Wunsch Date: Fri, 3 Apr 2020 09:30:36 +0200 Subject: [PATCH 2/2] [PyROOT] Make namespace pythonizations lazy * Remove dependency on VecOps, RDataFrame and TMVA for import ROOT * Inject namespace overload via property in the ROOT facade * Take care that we have proper error handling if, e.g., TMVA is not there --- .../pythonizations/python/ROOT/_facade.py | 40 +++++++++++++++++++ .../python/ROOT/pythonization/_rbdt.py | 15 ++----- .../python/ROOT/pythonization/_rdataframe.py | 8 ---- .../python/ROOT/pythonization/_rtensor.py | 8 ---- .../python/ROOT/pythonization/_rvec.py | 7 +--- .../test/rdataframe_makenumpy.py | 18 ++++----- .../pythonizations/test/rvec_asrvec.py | 16 ++++---- 7 files changed, 62 insertions(+), 50 deletions(-) diff --git a/bindings/pyroot_experimental/pythonizations/python/ROOT/_facade.py b/bindings/pyroot_experimental/pythonizations/python/ROOT/_facade.py index 33c77e5007c73..8ec47b300c9af 100644 --- a/bindings/pyroot_experimental/pythonizations/python/ROOT/_facade.py +++ b/bindings/pyroot_experimental/pythonizations/python/ROOT/_facade.py @@ -179,3 +179,43 @@ def _setattr(self, name, val): @property def __version__(self): return self.gROOT.GetVersion() + + # Overload VecOps namespace + # The property gets the C++ namespace, adds the pythonizations and + # eventually deletes itself so that following calls go directly + # to the C++ namespace. This mechanic ensures that we pythonize the + # namespace lazily. + @property + def VecOps(self): + ns = self._fallback_getattr('VecOps') + try: + from libROOTPythonizations import AsRVec + ns.AsRVec = AsRVec + except: + raise Exception('Failed to pythonize the namespace VecOps') + del type(self).VecOps + return ns + + # Overload RDF namespace + @property + def RDF(self): + ns = self._fallback_getattr('RDF') + try: + from libROOTPythonizations import MakeNumpyDataFrame + ns.MakeNumpyDataFrame = MakeNumpyDataFrame + except: + raise Exception('Failed to pythonize the namespace RDF') + del type(self).RDF + return ns + + # Overload TMVA namespace + @property + def TMVA(self): + ns = self._fallback_getattr('TMVA') + try: + from libROOTPythonizations import AsRTensor + ns.Experimental.AsRTensor = AsRTensor + except: + raise Exception('Failed to pythonize the namespace TMVA') + del type(self).TMVA + return ns diff --git a/bindings/pyroot_experimental/pythonizations/python/ROOT/pythonization/_rbdt.py b/bindings/pyroot_experimental/pythonizations/python/ROOT/pythonization/_rbdt.py index 4b4b54b206b7c..2250127ed1bb6 100644 --- a/bindings/pyroot_experimental/pythonizations/python/ROOT/pythonization/_rbdt.py +++ b/bindings/pyroot_experimental/pythonizations/python/ROOT/pythonization/_rbdt.py @@ -9,14 +9,7 @@ ################################################################################ from ROOT import pythonization -from libROOTPythonizations import AsRVec - - -try: - from libROOTPythonizations import AsRTensor - has_rtensor = True -except: - has_rtensor = False +from cppyy import gbl as gbl_namespace def Compute(self, x): @@ -29,11 +22,11 @@ def Compute(self, x): # numpy.array is a factory and the actual type of a numpy array is numpy.ndarray if isinstance(x, np.ndarray): if len(x.shape) == 1: - x_ = AsRVec(x) + x_ = gbl_namespace.VecOps.AsRVec(x) y = self._OriginalCompute(x_) return np.asarray(y) elif len(x.shape) == 2: - x_ = AsRTensor(x) + x_ = gbl_namespace.TMVA.Experimental.AsRTensor(x) y = self._OriginalCompute(x_) return np.asarray(y) else: @@ -49,7 +42,7 @@ def pythonize_rbdt(klass, name): # klass: class to be pythonized # name: name of the class - if name.startswith("TMVA::Experimental::RBDT") and has_rtensor: + if name.startswith("TMVA::Experimental::RBDT"): klass._OriginalCompute = klass.Compute klass.Compute = Compute diff --git a/bindings/pyroot_experimental/pythonizations/python/ROOT/pythonization/_rdataframe.py b/bindings/pyroot_experimental/pythonizations/python/ROOT/pythonization/_rdataframe.py index c964a9d78974f..2248fd595eb0f 100644 --- a/bindings/pyroot_experimental/pythonizations/python/ROOT/pythonization/_rdataframe.py +++ b/bindings/pyroot_experimental/pythonizations/python/ROOT/pythonization/_rdataframe.py @@ -160,11 +160,3 @@ def pythonize_rdataframe(klass, name): setattr(klass, method_name, partialmethod(_histo_profile, fixed_args)) return True - -# Add MakeNumpyDataFrame feature as free function to the ROOT module -try: - from libROOTPythonizations import MakeNumpyDataFrame - import cppyy - cppyy.gbl.ROOT.RDF.MakeNumpyDataFrame = MakeNumpyDataFrame -except: - pass diff --git a/bindings/pyroot_experimental/pythonizations/python/ROOT/pythonization/_rtensor.py b/bindings/pyroot_experimental/pythonizations/python/ROOT/pythonization/_rtensor.py index 6ac10a8cc0a93..051543228d521 100644 --- a/bindings/pyroot_experimental/pythonizations/python/ROOT/pythonization/_rtensor.py +++ b/bindings/pyroot_experimental/pythonizations/python/ROOT/pythonization/_rtensor.py @@ -130,11 +130,3 @@ def pythonize_rtensor(klass, name): klass.__getitem__ = RTensorGetitem return True - -# Add AsRTensor feature as free function to the ROOT module -try: - from libROOTPythonizations import AsRTensor - import cppyy - cppyy.gbl.TMVA.Experimental.AsRTensor = AsRTensor -except: - pass diff --git a/bindings/pyroot_experimental/pythonizations/python/ROOT/pythonization/_rvec.py b/bindings/pyroot_experimental/pythonizations/python/ROOT/pythonization/_rvec.py index af294acd1d9c3..0f05285d6bbbc 100644 --- a/bindings/pyroot_experimental/pythonizations/python/ROOT/pythonization/_rvec.py +++ b/bindings/pyroot_experimental/pythonizations/python/ROOT/pythonization/_rvec.py @@ -9,7 +9,7 @@ ################################################################################ from ROOT import pythonization -from libROOTPythonizations import GetEndianess, GetDataPointer, GetSizeOfType, AsRVec +from libROOTPythonizations import GetEndianess, GetDataPointer, GetSizeOfType _array_interface_dtype_map = { @@ -64,8 +64,3 @@ def pythonize_rvec(klass, name): add_array_interface_property(klass, name) return True - - -# Add AsRVec feature as free function to the ROOT module -import cppyy -cppyy.gbl.ROOT.VecOps.AsRVec = AsRVec diff --git a/bindings/pyroot_experimental/pythonizations/test/rdataframe_makenumpy.py b/bindings/pyroot_experimental/pythonizations/test/rdataframe_makenumpy.py index 43173b8fa97b1..56402e48df6a6 100644 --- a/bindings/pyroot_experimental/pythonizations/test/rdataframe_makenumpy.py +++ b/bindings/pyroot_experimental/pythonizations/test/rdataframe_makenumpy.py @@ -20,7 +20,7 @@ def test_dtypes(self): """ for dtype in self.dtypes: data = {"x": np.array([1, 2, 3], dtype=dtype)} - df = ROOT.ROOT.RDF.MakeNumpyDataFrame(data) + df = ROOT.RDF.MakeNumpyDataFrame(data) self.assertEqual(df.Mean("x").GetValue(), 2) def test_multiple_columns(self): @@ -30,7 +30,7 @@ def test_multiple_columns(self): data = {} for dtype in self.dtypes: data[dtype] = np.array([1, 2, 3], dtype=dtype) - df = ROOT.ROOT.RDF.MakeNumpyDataFrame(data) + df = ROOT.RDF.MakeNumpyDataFrame(data) colnames = df.GetColumnNames() # Test column names for dtype in colnames: @@ -47,7 +47,7 @@ def test_refcount(self): self.assertEqual(sys.getrefcount(data), 2) self.assertEqual(sys.getrefcount(data["x"]), 2) - df = ROOT.ROOT.RDF.MakeNumpyDataFrame(data) + df = ROOT.RDF.MakeNumpyDataFrame(data) self.assertTrue(hasattr(df, "__data__")) self.assertEqual(sys.getrefcount(df), 2) @@ -58,7 +58,7 @@ def test_transformations(self): Test the use of transformations """ data = {"x": np.array([1, 2, 3], dtype="float32")} - df = ROOT.ROOT.RDF.MakeNumpyDataFrame(data) + df = ROOT.RDF.MakeNumpyDataFrame(data) df2 = df.Filter("x>1").Define("y", "2*x") self.assertEqual(df2.Mean("x").GetValue(), 2.5) self.assertEqual(df2.Mean("y").GetValue(), 5) @@ -68,7 +68,7 @@ def test_delete_dict(self): Test behaviour with data dictionary going out of scope """ data = {"x": np.array([1, 2, 3], dtype="float32")} - df = ROOT.ROOT.RDF.MakeNumpyDataFrame(data) + df = ROOT.RDF.MakeNumpyDataFrame(data) del data self.assertEqual(df.Mean("x").GetValue(), 2) @@ -78,7 +78,7 @@ def test_delete_numpy_array(self): """ x = np.array([1, 2, 3], dtype="float32") data = {"x": x} - df = ROOT.ROOT.RDF.MakeNumpyDataFrame(data) + df = ROOT.RDF.MakeNumpyDataFrame(data) del x self.assertEqual(df.Mean("x").GetValue(), 2) @@ -86,7 +86,7 @@ def test_inplace_dict(self): """ Test behaviour with inplace dictionary """ - df = ROOT.ROOT.RDF.MakeNumpyDataFrame({"x": np.array([1, 2, 3], dtype="float32")}) + df = ROOT.RDF.MakeNumpyDataFrame({"x": np.array([1, 2, 3], dtype="float32")}) self.assertEqual(df.Mean("x").GetValue(), 2) def test_lifetime_numpy_array(self): @@ -96,7 +96,7 @@ def test_lifetime_numpy_array(self): x = np.array([1, 2, 3], dtype="float32") ref1 = sys.getrefcount(x) - df = ROOT.ROOT.RDF.MakeNumpyDataFrame({"x": x}) + df = ROOT.RDF.MakeNumpyDataFrame({"x": x}) ref2 = sys.getrefcount(x) self.assertEqual(ref2, ref1 + 1) @@ -115,7 +115,7 @@ def test_lifetime_datasource(self): # Data source has dictionary with RVecs attached, which take a reference # to the numpy array - df = ROOT.ROOT.RDF.MakeNumpyDataFrame({"x": x}) + df = ROOT.RDF.MakeNumpyDataFrame({"x": x}) m = df.Mean("x") ref2 = sys.getrefcount(x) self.assertEqual(ref1 + 1, ref2) diff --git a/bindings/pyroot_experimental/pythonizations/test/rvec_asrvec.py b/bindings/pyroot_experimental/pythonizations/test/rvec_asrvec.py index bccfe4e7ebc53..ac48151baf245 100644 --- a/bindings/pyroot_experimental/pythonizations/test/rvec_asrvec.py +++ b/bindings/pyroot_experimental/pythonizations/test/rvec_asrvec.py @@ -44,7 +44,7 @@ def test_dtypes(self): """ for dtype in self.dtypes: np_obj = np.empty(2, dtype=dtype) - root_obj = ROOT.ROOT.VecOps.AsRVec(np_obj) + root_obj = ROOT.VecOps.AsRVec(np_obj) self.check_memory_adoption(root_obj, np_obj) self.check_size(2, root_obj) @@ -53,7 +53,7 @@ def test_multidim(self): Test adoption of multi-dimensional numpy arrays """ np_obj = np.array([[1, 2], [3, 4]], dtype="float32") - rvec = ROOT.ROOT.VecOps.AsRVec(np_obj) + rvec = ROOT.VecOps.AsRVec(np_obj) self.assertEqual(rvec.size(), 4) def test_size_zero(self): @@ -61,16 +61,16 @@ def test_size_zero(self): Test adoption of numpy array with size 0 """ np_obj = np.array([], dtype="float32") - rvec = ROOT.ROOT.VecOps.AsRVec(np_obj) + rvec = ROOT.VecOps.AsRVec(np_obj) self.assertEqual(rvec.size(), 0) def test_adopt_rvec(self): """ Test adoption of RVecs """ - rvec = ROOT.ROOT.VecOps.RVec("float")(1) + rvec = ROOT.VecOps.RVec("float")(1) rvec[0] = 42 - rvec2 = ROOT.ROOT.VecOps.AsRVec(rvec) + rvec2 = ROOT.VecOps.AsRVec(rvec) self.assertEqual(rvec.size(), rvec2.size()) self.assertEqual(rvec[0], rvec2[0]) rvec2[0] = 43 @@ -81,7 +81,7 @@ def test_ownership(self): Test ownership of returned RVec (to be owned by Python) """ np_obj = np.array([1, 2]) - rvec = ROOT.ROOT.VecOps.AsRVec(np_obj) + rvec = ROOT.VecOps.AsRVec(np_obj) self.assertEqual(rvec.__python_owns__, True) def test_attribute_adopted(self): @@ -89,7 +89,7 @@ def test_attribute_adopted(self): Test __adopted__ attribute of returned RVecs """ np_obj = np.array([1, 2]) - rvec = ROOT.ROOT.VecOps.AsRVec(np_obj) + rvec = ROOT.VecOps.AsRVec(np_obj) self.assertTrue(hasattr(rvec, "__adopted__")) self.assertEqual(id(rvec.__adopted__), id(np_obj)) @@ -104,7 +104,7 @@ def test_refcount(self): is decreased. """ np_obj = np.array([1, 2]) - rvec = ROOT.ROOT.VecOps.AsRVec(np_obj) + rvec = ROOT.VecOps.AsRVec(np_obj) self.assertEqual(sys.getrefcount(rvec), 2) self.assertEqual(sys.getrefcount(np_obj), 3) del rvec