From 210ddc92da965a2184d10230f0c70d3ef9a9be59 Mon Sep 17 00:00:00 2001 From: Ralf Gommers Date: Wed, 1 Nov 2023 18:34:46 +0100 Subject: [PATCH] Add Netlib BLAS (libblas/libcblas) and LAPACK (liblapack/liblapacke) dependencies Also a bit of refactoring and improvements to OpenBLAS and MKL logic to reduce the amount of unnecessary checks. --- mesonbuild/dependencies/__init__.py | 2 + mesonbuild/dependencies/blas_lapack.py | 314 +++++++++++++++++++++---- 2 files changed, 271 insertions(+), 45 deletions(-) diff --git a/mesonbuild/dependencies/__init__.py b/mesonbuild/dependencies/__init__.py index 97bd797c1e88..545c7ce8f1c3 100644 --- a/mesonbuild/dependencies/__init__.py +++ b/mesonbuild/dependencies/__init__.py @@ -238,6 +238,8 @@ def __init__(self, name: str, environment: 'Environment', kwargs: T.Dict[str, T. # From blas_lapack: 'accelerate': 'blas_lapack', + 'blas': 'blas_lapack', + 'lapack': 'blas_lapack', 'mkl': 'blas_lapack', 'openblas': 'blas_lapack', diff --git a/mesonbuild/dependencies/blas_lapack.py b/mesonbuild/dependencies/blas_lapack.py index 42fe4bea93d5..92c24725a054 100644 --- a/mesonbuild/dependencies/blas_lapack.py +++ b/mesonbuild/dependencies/blas_lapack.py @@ -302,6 +302,28 @@ """ +def check_blas_machine_file(self, name: str, props: dict) -> T.Tuple[bool, T.List[str]]: + # TBD: do we need to support multiple extra dirs? + incdir = props.get(f'{name}_includedir') + assert incdir is None or isinstance(incdir, str) + libdir = props.get(f'{name}_librarydir') + assert libdir is None or isinstance(libdir, str) + + if incdir and libdir: + has_dirs = True + if not Path(incdir).is_absolute() or not Path(libdir).is_absolute(): + raise mesonlib.MesonException('Paths given for openblas_includedir and ' + 'openblas_librarydir in machine file must be absolute') + return has_dirs, [libdir, incdir] + elif incdir or libdir: + raise mesonlib.MesonException('Both openblas_includedir *and* openblas_librarydir ' + 'have to be set in your machine file (one is not enough)') + else: + raise mesonlib.MesonBugException('issue with openblas dependency detection, should not ' + 'be possible to reach this else clause') + return (False, []) + + class BLASLAPACKMixin(): def parse_modules(self, kwargs: T.Dict[str, T.Any]) -> None: modules: T.List[str] = mesonlib.extract_as_list(kwargs, 'modules') @@ -322,14 +344,17 @@ def parse_modules(self, kwargs: T.Dict[str, T.Any]) -> None: self.needs_lapack = 'lapack' in modules self.needs_lapacke = 'lapacke' in modules - def check_symbols(self, compile_args, suffix=None) -> None: + def check_symbols(self, compile_args, suffix=None, check_cblas=True, + check_lapacke=True, lapack_only=False) -> None: # verify that we've found the right LP64/ILP64 interface - symbols = ['dgemm_'] - if self.needs_cblas: + symbols = [] + if not lapack_only: + symbols += ['dgemm_'] + if check_cblas and self.needs_cblas: symbols += ['cblas_dgemm'] if self.needs_lapack: symbols += ['zungqr_'] - if self.needs_lapacke: + if check_lapacke and self.needs_lapacke: symbols += ['LAPACKE_zungqr'] if suffix is None: @@ -416,13 +441,14 @@ def detect(self, lib_dirs: T.Optional[T.List[str]] = None, inc_dirs: T.Optional[ for libname in libnames: link_arg = self.clib_compiler.find_library(libname, self.env, lib_dirs) - incdir_args = [f'-I{inc_dir}' for inc_dir in inc_dirs] - for hdr in ['openblas_config.h', 'openblas/openblas_config.h']: - found_header, _ = self.clib_compiler.has_header(hdr, '', self.env, dependencies=[self], - extra_args=incdir_args) - if found_header: - self._openblas_config_header = hdr - break + if link_arg: + incdir_args = [f'-I{inc_dir}' for inc_dir in inc_dirs] + for hdr in ['openblas_config.h', 'openblas/openblas_config.h']: + found_header, _ = self.clib_compiler.has_header(hdr, '', self.env, dependencies=[self], + extra_args=incdir_args) + if found_header: + self._openblas_config_header = hdr + break if link_arg and found_header: if not self.probe_symbols(link_arg): @@ -433,25 +459,10 @@ def detect(self, lib_dirs: T.Optional[T.List[str]] = None, inc_dirs: T.Optional[ break def detect_openblas_machine_file(self, props: dict) -> None: - # TBD: do we need to support multiple extra dirs? - incdir = props.get('openblas_includedir') - assert incdir is None or isinstance(incdir, str) - libdir = props.get('openblas_librarydir') - assert libdir is None or isinstance(libdir, str) - - if incdir and libdir: - self.is_found = True - if not Path(incdir).is_absolute() or not Path(libdir).is_absolute(): - raise mesonlib.MesonException('Paths given for openblas_includedir and ' - 'openblas_librarydir in machine file must be absolute') - elif incdir or libdir: - raise mesonlib.MesonException('Both openblas_includedir *and* openblas_librarydir ' - 'have to be set in your machine file (one is not enough)') - else: - raise mesonlib.MesonBugException('issue with openblas dependency detection, should not ' - 'be possible to reach this else clause') - - self.detect([libdir], [incdir]) + has_dirs, _dirs = check_blas_machine_file('openblas', props) + if has_dirs: + libdir, incdir = _dirs + self.detect([libdir], [incdir]) def detect_openblas_version(self) -> str: v, _ = self.clib_compiler.get_define('OPENBLAS_VERSION', @@ -476,7 +487,7 @@ def __init__(self, name: str, env: 'Environment', kwargs: T.Dict[str, T.Any]) -> super().__init__(name, env, kwargs) - if not self.probe_symbols(self.link_args): + if self.is_found and not self.probe_symbols(self.link_args): self.is_found = False @@ -489,19 +500,224 @@ def __init__(self, name: str, env: 'Environment', kwargs: T.Dict[str, T.Any], if self.interface == 'ilp64': self.is_found = False - elif not self.probe_symbols(self.link_args): + elif self.is_found and not self.probe_symbols(self.link_args): self.is_found = False -class NetlibPkgConfigDependency(BLASLAPACKMixin, PkgConfigDependency): +class NetlibMixin(): + def get_symbol_suffix(self) -> str: + self._ilp64_suffix = '' # Handle `64_` suffix, or custom suffixes? + return '' if self.interface == 'lp64' else self._ilp64_suffix + + def probe_symbols(self, compile_args, check_cblas=True, check_lapacke=True, + lapack_only=False) -> bool: + """Most ILP64 BLAS builds will not use a suffix, but the new standard will be _64 + (see Reference-LAPACK/lapack#666). Check which one we're dealing with""" + if self.interface == 'lp64': + return self.check_symbols(compile_args, check_cblas=check_cblas, + check_lapacke=check_lapacke, lapack_only=lapack_only) + + if self.check_symbols(compile_args, '_64', check_cblas=check_cblas, + check_lapacke=check_lapacke, lapack_only=lapack_only): + self._ilp64_suffix = '_64' + elif self.check_symbols(compile_args, '', check_cblas=check_cblas, + check_lapacke=check_lapacke, lapack_only=lapack_only): + self._ilp64_suffix = '' + else: + return False + return True + + +class NetlibBLASPkgConfigDependency(BLASLAPACKMixin, NetlibMixin, PkgConfigDependency): def __init__(self, name: str, env: 'Environment', kwargs: T.Dict[str, T.Any]) -> None: - # TODO: add 'cblas' - super().__init__('blas', env, kwargs) + # TODO: add ILP64 - needs factory function like for OpenBLAS + super().__init__(name, env, kwargs) self.feature_since = ('1.3.0', '') self.parse_modules(kwargs) - def get_symbol_suffix(self) -> str: - return '' + if self.is_found: + if self.needs_cblas: + # `name` may be 'blas' or 'blas64'; CBLAS library naming should be consistent + # with BLAS library naming, so just prepend 'c' and try to detect it. + try: + cblas_pc = PkgConfigDependency('c'+name, env, kwargs) + if cblas_pc.found(): + self.link_args += cblas_pc.link_args + self.compile_args += cblas_pc.compile_args + except DependencyException: + pass + + if not self.probe_symbols(self.link_args): + self.is_found = False + + +class NetlibBLASSystemDependency(BLASLAPACKMixin, NetlibMixin, SystemDependency): + def __init__(self, name: str, environment: 'Environment', kwargs: T.Dict[str, T.Any]) -> None: + super().__init__(name, environment, kwargs) + self.feature_since = ('1.3.0', '') + self.parse_modules(kwargs) + + # First, look for paths specified in a machine file + props = self.env.properties[self.for_machine].properties + if any(x in props for x in ['blas_includedir', 'blas_librarydir']): + self.detect_blas_machine_file(props) + + # Then look in standard directories by attempting to link + if not self.is_found: + extra_libdirs: T.List[str] = [] + self.detect(extra_libdirs) + + if self.is_found: + self.version = 'unknown' # no way to derive this from standard headers + + def detect(self, lib_dirs: T.Optional[T.List[str]] = None, inc_dirs: T.Optional[T.List[str]] = None) -> None: + if lib_dirs is None: + lib_dirs = [] + if inc_dirs is None: + inc_dirs = [] + + if self.interface == 'lp64': + libnames = ['blas'] + cblas_headers = ['cblas.h'] + elif self.interface == 'ilp64': + libnames = ['blas64', 'blas'] + cblas_headers = ['cblas_64.h', 'cblas.h'] + + for libname in libnames: + link_arg = self.clib_compiler.find_library(libname, self.env, lib_dirs) + if not link_arg: + continue + + # libblas may include CBLAS symbols (Debian builds it like this), + # but more often than not there's a separate libcblas library. Handle both cases. + if not self.probe_symbols(link_arg, check_cblas=False): + continue + if self.needs_cblas: + cblas_in_blas = self.probe_symbols(link_arg, check_cblas=True) + + if self.needs_cblas and not cblas_in_blas: + # We found libblas and it does not contain CBLAS symbols, so we need libcblas + cblas_libname = 'c' + libname + link_arg_cblas = self.clib_compiler.find_library(cblas_libname, self.env, lib_dirs) + if link_arg_cblas: + link_arg.extend(link_arg_cblas) + else: + # We didn't find CBLAS + continue + + self.is_found = True + self.link_args += link_arg + if self.needs_cblas: + incdir_args = [f'-I{inc_dir}' for inc_dir in inc_dirs] + for hdr in cblas_headers: + found_header, _ = self.clib_compiler.has_header(hdr, '', self.env, dependencies=[self], + extra_args=incdir_args) + if found_header: + # If we don't get here, we found the library but not the header - this may + # be okay, since projects may ship their own CBLAS header for portability) + self.compile_args += incdir_args + break + + def detect_blas_machine_file(self, props: dict) -> None: + has_dirs, _dirs = check_blas_machine_file('blas', props) + if has_dirs: + libdir, incdir = _dirs + self.detect([libdir], [incdir]) + + +class NetlibLAPACKPkgConfigDependency(BLASLAPACKMixin, NetlibMixin, PkgConfigDependency): + def __init__(self, name: str, env: 'Environment', kwargs: T.Dict[str, T.Any]) -> None: + # TODO: add ILP64 (needs factory function like for OpenBLAS) + super().__init__(name, env, kwargs) + self.feature_since = ('1.3.0', '') + self.parse_modules(kwargs) + + if self.is_found: + if self.needs_lapacke: + # Similar to CBLAS: there may be a separate liblapacke.so + try: + lapacke_pc = PkgConfigDependency(name+'e', env, kwargs) + if lapacke_pc.found(): + self.link_args += lapacke_pc.link_args + self.compile_args += lapacke_pc.compile_args + except DependencyException: + pass + + if not self.probe_symbols(self.link_args, lapack_only=True): + self.is_found = False + + +class NetlibLAPACKSystemDependency(BLASLAPACKMixin, NetlibMixin, SystemDependency): + def __init__(self, name: str, environment: 'Environment', kwargs: T.Dict[str, T.Any]) -> None: + super().__init__(name, environment, kwargs) + self.feature_since = ('1.3.0', '') + self.parse_modules(kwargs) + + # First, look for paths specified in a machine file + props = self.env.properties[self.for_machine].properties + if any(x in props for x in ['lapack_includedir', 'lapack_librarydir']): + self.detect_lapack_machine_file(props) + + # Then look in standard directories by attempting to link + if not self.is_found: + extra_libdirs: T.List[str] = [] + self.detect(extra_libdirs) + + if self.is_found: + self.version = 'unknown' # no way to derive this from standard headers + + def detect(self, lib_dirs: T.Optional[T.List[str]] = None, inc_dirs: T.Optional[T.List[str]] = None) -> None: + if lib_dirs is None: + lib_dirs = [] + if inc_dirs is None: + inc_dirs = [] + + if self.interface == 'lp64': + libnames = ['lapack'] + lapacke_headers = ['lapacke.h'] + elif self.interface == 'ilp64': + libnames = ['lapack64', 'lapack'] + lapacke_headers = ['lapacke_64.h', 'lapacke.h'] + + for libname in libnames: + link_arg = self.clib_compiler.find_library(libname, self.env, lib_dirs) + if not link_arg: + continue + + if not self.probe_symbols(link_arg, check_lapacke=False, lapack_only=True): + continue + if self.needs_lapacke: + lapacke_in_lapack = self.probe_symbols(link_arg, check_lapacke=True, lapack_only=True) + + if self.needs_lapacke and not lapacke_in_lapack: + # We found liblapack and it does not contain LAPACKE symbols, so we need liblapacke + lapacke_libname = libname + 'e' + link_arg_lapacke = self.clib_compiler.find_library(lapacke_libname, self.env, lib_dirs) + if link_arg_lapacke: + link_arg.extend(link_arg_lapacke) + else: + # We didn't find LAPACKE + continue + + self.is_found = True + self.link_args += link_arg + if self.needs_lapacke: + incdir_args = [f'-I{inc_dir}' for inc_dir in inc_dirs] + for hdr in lapacke_headers: + found_header, _ = self.clib_compiler.has_header(hdr, '', self.env, dependencies=[self], + extra_args=incdir_args) + if found_header: + # If we don't get here, we found the library but not the header - this may + # be okay, since projects may ship their own LAPACKE header for portability) + self.compile_args += incdir_args + break + + def detect_lapack_machine_file(self, props: dict) -> None: + has_dirs, _dirs = check_blas_machine_file('lapack', props) + if has_dirs: + libdir, incdir = _dirs + self.detect([libdir], [incdir]) + class AccelerateSystemDependency(BLASLAPACKMixin, SystemDependency): @@ -683,9 +899,10 @@ def detect_sdl(self) -> None: mlog.warning(f'MKLROOT env var set to {mklroot}, but not pointing to an MKL install') link_arg = self.clib_compiler.find_library('mkl_rt', self.env, lib_dirs) - incdir_args = [f'-I{inc_dir}' for inc_dir in inc_dirs] - found_header, _ = self.clib_compiler.has_header('mkl_version.h', '', self.env, - dependencies=[self], extra_args=incdir_args) + if link_arg: + incdir_args = [f'-I{inc_dir}' for inc_dir in inc_dirs] + found_header, _ = self.clib_compiler.has_header('mkl_version.h', '', self.env, + dependencies=[self], extra_args=incdir_args) if link_arg and found_header: self.is_found = True self.compile_args += incdir_args @@ -734,11 +951,18 @@ def openblas_factory(env: 'Environment', for_machine: 'MachineChoice', packages['openblas'] = openblas_factory -packages['netlib-blas'] = netlib_factory = DependencyFactory( - 'netlib-blas', - [DependencyMethods.PKGCONFIG], #, DependencyMethods.SYSTEM], - #system_class=NetlibSystemDependency, - pkgconfig_class=NetlibPkgConfigDependency, +packages['blas'] = netlib_factory = DependencyFactory( + 'blas', + [DependencyMethods.PKGCONFIG, DependencyMethods.SYSTEM], + pkgconfig_class=NetlibBLASPkgConfigDependency, + system_class=NetlibBLASSystemDependency, +) + +packages['lapack'] = netlib_factory = DependencyFactory( + 'lapack', + [DependencyMethods.PKGCONFIG, DependencyMethods.SYSTEM], + pkgconfig_class=NetlibLAPACKPkgConfigDependency, + system_class=NetlibLAPACKSystemDependency, )