DistArray conversion to/fro Eigen::Tensor

INSTALL.md

@@ -34,13 +34,13 @@ Both methods are supported. However, for most users we _strongly_ recommend to b
 
 - [CMake](https://cmake.org/), version 3.15 or higher; if CUDA support is needed, CMake 3.18 or higher is required.
 - [Git](https://git-scm.com/) 1.8 or later (required to obtain TiledArray and MADNESS source code from GitHub)
-- [Eigen](http://eigen.tuxfamily.org/), version 3.3 or higher; if CUDA is enabled then 3.3.7 is required (will be downloaded automatically, if missing)
+- [Eigen](http://eigen.tuxfamily.org/), version 3.3.5 or higher; if CUDA is enabled then 3.3.7 is required (will be downloaded automatically, if missing)
 - [Boost libraries](www.boost.org/), version 1.59 or higher (will be downloaded automatically, if missing). The following principal Boost components are used:
   - Boost.Iterator: header-only
   - Boost.Container: header-only
   - Boost.Test: header-only or (optionally) as a compiled library, *only used for unit testing*
   - Boost.Range: header-only, *only used for unit testing*
-- [BTAS](https://github.com/ValeevGroup/BTAS), tag e540f265e1d16cc35b1b6726470160ed0046c530 . If usable BTAS installation is not found, TiledArray will download and compile
+- [BTAS](https://github.com/ValeevGroup/BTAS), tag 5702259d5d207fe5a8e0c975c3cf1f610dcf381a . If usable BTAS installation is not found, TiledArray will download and compile
   BTAS from source. *This is the recommended way to compile BTAS for all users*.
 - [MADNESS](https://github.com/m-a-d-n-e-s-s/madness), tag b8069a25460b696d21cc8b739a91c9249da26575 .
   Only the MADworld runtime and BLAS/LAPACK C API component of MADNESS is used by TiledArray.

external/eigen.cmake

@@ -49,11 +49,8 @@ if (TARGET TiledArray_Eigen)
 
   # INTERFACE libraries cannot be used as CMAKE_REQUIRED_LIBRARIES, so must manually transfer deps info
   get_property(EIGEN3_INCLUDE_DIRS TARGET TiledArray_Eigen PROPERTY INTERFACE_INCLUDE_DIRECTORIES)
-  if (NOT MADNESS_INTERNAL_INCLUDE_DIRS)
-    message(FATAL_ERROR "eigen.cmake must be loaded after calling detect_MADNESS_config()")
-  endif()
   list(APPEND CMAKE_REQUIRED_INCLUDES ${EIGEN3_INCLUDE_DIRS} ${PROJECT_BINARY_DIR}/src ${PROJECT_SOURCE_DIR}/src
-       ${MADNESS_INTERNAL_INCLUDE_DIRS} ${LAPACK_INCLUDE_DIRS})
+       ${LAPACK_INCLUDE_DIRS})
   list(APPEND CMAKE_REQUIRED_LIBRARIES ${LAPACK_LIBRARIES})
   foreach(_def ${LAPACK_COMPILE_DEFINITIONS})
     list(APPEND CMAKE_REQUIRED_DEFINITIONS "-D${_def}")
@@ -86,8 +83,11 @@ elseif(TA_EXPERT)
 
 else()
 
-  set(Eigen3_VERSION 3.3.7)
-  set(EIGEN3_URL_HASH MD5=b9e98a200d2455f06db9c661c5610496)
+  # last resort: unpack and copy to install dir
+  # N.B. NOT building via FetchContent since Eigen3 is not subprojectable due to polluting global namespace of targets (e.g. lapack, check, etc.)
+
+  set(Eigen3_VERSION ${TA_INSTALL_EIGEN_VERSION})
+  set(EIGEN3_URL_HASH ${TA_INSTALL_EIGEN_URL_HASH})
   set(EIGEN3_URL https://gitlab.com/libeigen/eigen/-/archive/${Eigen3_VERSION}/eigen-${Eigen3_VERSION}.tar.bz2)
 
   include(ExternalProject)
@@ -144,17 +144,5 @@ endif()
 # finish configuring TiledArray_Eigen and install
 if (TARGET TiledArray_Eigen)
   set(TiledArray_Eigen_VERSION "${Eigen3_VERSION}" CACHE STRING "Eigen3_VERSION of the library interfaced by TiledArray_Eigen target")
-  # TiledArray_Eigen uses LAPACK/MKL
-  # N.B. used to ... seems to be disabled
-#  target_link_libraries(TiledArray_Eigen INTERFACE ${LAPACK_LIBRARIES})
-#  target_include_directories(TiledArray_Eigen INTERFACE ${LAPACK_INCLUDE_DIRS})
-#  target_compile_definitions(TiledArray_Eigen INTERFACE ${LAPACK_COMPILE_DEFINITIONS})
-#  target_compile_options(TiledArray_Eigen INTERFACE ${LAPACK_COMPILE_OPTIONS})
-#  # Eigen's prototypes for BLAS interface libraries do not match MADNESS cblas
-#  if (MADNESS_HAS_MKL)
-#    # target_compile_definitions(TiledArray_Eigen INTERFACE EIGEN_USE_MKL EIGEN_USE_BLAS)
-#  else(MADNESS_HAS_MKL)
-#    # target_compile_definitions(TiledArray_Eigen INTERFACE EIGEN_USE_BLAS)
-#  endif(MADNESS_HAS_MKL)
   install(TARGETS TiledArray_Eigen EXPORT tiledarray COMPONENT tiledarray)
 endif(TARGET TiledArray_Eigen)

external/versions.cmake

@@ -12,20 +12,20 @@ set(TA_INSTALL_BOOST_URL_HASH 882b48708d211a5f48e60b0124cf5863c1534cd544ecd0664b
 set(TA_INSTALL_BOOST_PREVIOUS_URL_HASH 882b48708d211a5f48e60b0124cf5863c1534cd544ecd0664bb534a4b5d506e9)
 
 # N.B. may need to update INSTALL.md manually with the CUDA-specific version
-set(TA_TRACKED_EIGEN_VERSION 3.3)
+set(TA_TRACKED_EIGEN_VERSION 3.3.5)
 set(TA_TRACKED_EIGEN_PREVIOUS_VERSION 3.3)
-set(TA_INSTALL_EIGEN_VERSION 3.3.7)
+set(TA_INSTALL_EIGEN_VERSION 3.4.0)
 set(TA_INSTALL_EIGEN_PREVIOUS_VERSION 3.3.7)
-set(TA_INSTALL_EIGEN_URL_HASH b9e98a200d2455f06db9c661c5610496)
-set(TA_INSTALL_EIGEN_PREVIOUS_URL_HASH b9e98a200d2455f06db9c661c5610496)
+set(TA_INSTALL_EIGEN_URL_HASH SHA256=b4c198460eba6f28d34894e3a5710998818515104d6e74e5cc331ce31e46e626)
+set(TA_INSTALL_EIGEN_PREVIOUS_URL_HASH MD5=b9e98a200d2455f06db9c661c5610496)
 
 set(TA_TRACKED_MADNESS_TAG b8069a25460b696d21cc8b739a91c9249da26575)
 set(TA_TRACKED_MADNESS_PREVIOUS_TAG f9aa38e4f46c5ea6ca6bbceb945beae5230f9ad0)
 set(TA_TRACKED_MADNESS_VERSION 0.10.1)
 set(TA_TRACKED_MADNESS_PREVIOUS_VERSION 0.10.1)
 
-set(TA_TRACKED_BTAS_TAG e540f265e1d16cc35b1b6726470160ed0046c530)
-set(TA_TRACKED_BTAS_PREVIOUS_TAG 6db4794185166fae4fc235f05b05e863a82a9dec)
+set(TA_TRACKED_BTAS_TAG 5702259d5d207fe5a8e0c975c3cf1f610dcf381a)
+set(TA_TRACKED_BTAS_PREVIOUS_TAG e540f265e1d16cc35b1b6726470160ed0046c530)
 
 set(TA_TRACKED_CUTT_TAG 0e8685bf82910bc7435835f846e88f1b39f47f09)
 set(TA_TRACKED_CUTT_PREVIOUS_TAG 592198b93c93b7ca79e7900b9a9f2e79f9dafec3)

python/CMakeLists.txt

@@ -12,22 +12,10 @@ project(python-tiledarray)
 set(CMAKE_CXX_STANDARD 17)
 add_compile_options(-Wall)
 
-find_package(Eigen3 3.3 REQUIRED)
-
-# find_package(MPI REQUIRED)
-# include_directories(${MPI_INCLUDE_PATH})
-
-find_package(Boost REQUIRED) # spirit absent from cmake
-# include_directories(${BOOST_INCLUDE_DIRS})
-
-#find_package (Python COMPONENTS Interpreter Development)
-#include_directories(${PROJECT_SOURCE_DIR}/pybind11/include)
-#python_add_library(python-tiledarray MODULE src/tiledarray.cc WITH_SOABI)
-
-#add_subdirectory(pybind11)
 pybind11_add_module(python-tiledarray MODULE src/tiledarray.cc)
 
 # mpi c libs need come from tiledarray
+# Eigen and Boost also transitively come from tiledarray
 target_link_libraries(python-tiledarray PRIVATE tiledarray)
 
 # pybind uses sized deallocations when using C++17, clang requires -fsized-deallocation compiler flag to enable them
@@ -43,14 +31,6 @@ set_target_properties(
   OUTPUT_NAME tiledarray
   )
 
-if (Eigen3::Eigen)
-  target_link_libraries(python-tiledarray INTERFACE Eigen3::Eigen)
-else()
-  include_directories(${EIGEN3_INCLUDE_DIR})
-endif()
-
-target_link_libraries(python-tiledarray INTERFACE Boost::boost)
-
 # tests
 if (BUILD_TESTING)
   # build step

src/TiledArray/conversions/btas.h

@@ -115,7 +115,7 @@ namespace detail {
 /// TiledArray::DistArray
 
 /// \tparam DistArray_ a TiledArray::DistArray type
-/// \tparam TArgs the type pack in btas::Tensor<TArgs...> type
+/// \tparam BTAS_Tensor_ a btas::Tensor type
 /// \param src The btas::Tensor object whose block will be copied
 /// \param dst The array that will hold the result
 /// \param i The index of the tile to be copied
@@ -124,7 +124,7 @@ namespace detail {
 /// counter.
 template <typename DistArray_, typename BTAS_Tensor_>
 void counted_btas_subtensor_to_tensor(const BTAS_Tensor_* src, DistArray_* dst,
-                                      const typename DistArray_::ordinal_type i,
+                                      const typename Range::index_type i,
                                       madness::AtomicInt* counter) {
   typename DistArray_::value_type tensor(dst->trange().make_tile_range(i));
   btas_subtensor_to_tensor(*src, tensor);
@@ -134,8 +134,8 @@ void counted_btas_subtensor_to_tensor(const BTAS_Tensor_* src, DistArray_* dst,
 
 /// Task function for assigning a tensor to an Eigen submatrix
 
-/// \tparam Tensor_ a TiledArray::Tensor type
-/// \tparam TArgs the type pack in btas::Tensor<TArgs...> type
+/// \tparam TA_Tensor_ a TiledArray::Tensor type
+/// \tparam BTAS_Tensor_ a btas::Tensor type
 /// \param src The source tensor
 /// \param dst The destination tensor
 /// \param counter The task counter
@@ -171,10 +171,14 @@ inline auto make_shape<false>(World&, const TiledArray::TiledRange&) {
 /// has sparse policy, a sparse map with large norm is created to ensure all the
 /// values from \c src copy to the \c DistArray_ object. The copy operation is
 /// done in parallel, and this function will block until all elements of
-/// \c src have been copied into the result array tiles. The size of
-/// \c world.size() must be equal to 1 or \c replicate must be equal to
-/// \c true . If \c replicate is \c true, it is your responsibility to ensure
-/// that the data in \c src is identical on all nodes. Upon completion,
+/// \c src have been copied into the result array tiles.
+/// Each tile is created
+/// using the local contents of \c src, hence
+/// it is your responsibility to ensure that the data in \c src
+/// is distributed correctly among the ranks. If in doubt, you should replicate
+/// \c src among the ranks prior to calling this.
+///
+/// Upon completion,
 /// if the \c DistArray_ object has sparse policy truncate() is called.\n
 /// Usage:
 /// \code
@@ -201,18 +205,20 @@ inline auto make_shape<false>(World&, const TiledArray::TiledRange&) {
 /// \param[in] trange The tiled range of the new array
 /// \param[in] src The btas::Tensor<TArgs..> object whose contents will be
 /// copied to the result.
-/// \param[in] replicated \c true indicates that the
-/// result array should be a
-///            replicated array [default = false].
+/// \param replicated if true, the result will be replicated
+///        [default = false].
+/// \param pmap the process map object [default=null]; initialized to the
+/// default if \p replicated is false, or a replicated pmap if \p replicated
+/// is true; ignored if \p replicated is true and \c world.size()>1
 /// \return A \c DistArray_ object that is a copy of \c src
 /// \throw TiledArray::Exception When world size is greater than 1
 /// \note If using 2 or more World ranks, set \c replicated=true and make sure
 /// \c matrix is the same on each rank!
 template <typename DistArray_, typename T, typename Range, typename Storage>
-DistArray_ btas_tensor_to_array(World& world,
-                                const TiledArray::TiledRange& trange,
-                                const btas::Tensor<T, Range, Storage>& src,
-                                bool replicated = false) {
+DistArray_ btas_tensor_to_array(
+    World& world, const TiledArray::TiledRange& trange,
+    const btas::Tensor<T, Range, Storage>& src, bool replicated = false,
+    std::shared_ptr<typename DistArray_::pmap_interface> pmap = {}) {
   // Test preconditions
   const auto rank = trange.tiles_range().rank();
   TA_ASSERT(rank == src.range().rank() &&
@@ -229,35 +235,26 @@ DistArray_ btas_tensor_to_array(World& world,
   using Policy_ = typename DistArray_::policy_type;
   const auto is_sparse = !is_dense_v<Policy_>;
 
-  // Check that this is not a distributed computing environment
-  if (!replicated)
-    TA_ASSERT(
-        world.size() == 1 &&
-        "An array can be created from a btas::Tensor if the number of World "
-        "ranks is greater than 1 only when replicated=true.");
-
   // Make a shape, only used if making a sparse array
   using Shape_ = typename DistArray_::shape_type;
   Shape_ shape = detail::make_shape<is_sparse>(world, trange);
 
   // Create a new tensor
-  DistArray_ array =
-      (replicated && (world.size() > 1)
-           ? DistArray_(
-                 world, trange, shape,
-                 std::static_pointer_cast<typename DistArray_::pmap_interface>(
-                     std::make_shared<detail::ReplicatedPmap>(
-                         world, trange.tiles_range().volume())))
-           : DistArray_(world, trange, shape));
+  if (replicated && (world.size() > 1))
+    pmap = std::static_pointer_cast<typename DistArray_::pmap_interface>(
+        std::make_shared<detail::ReplicatedPmap>(
+            world, trange.tiles_range().volume()));
+  DistArray_ array = (pmap ? DistArray_(world, trange, shape, pmap)
+                           : DistArray_(world, trange, shape));
 
   // Spawn copy tasks
   madness::AtomicInt counter;
   counter = 0;
   std::int64_t n = 0;
-  for (typename DistArray_::ordinal_type i = 0; i < array.size(); ++i) {
+  for (auto&& acc : array) {
     world.taskq.add(
         &detail::counted_btas_subtensor_to_tensor<DistArray_, Tensor_>, &src,
-        &array, i, &counter);
+        &array, acc.index(), &counter);
     ++n;
   }
 
@@ -276,32 +273,38 @@ DistArray_ btas_tensor_to_array(World& world,
 /// object. The copy operation is done in parallel, and this function will block
 /// until all elements of \c src have been copied into the result array tiles.
 /// The size of \c src.world().size() must be equal to 1 or \c src must be a
-/// replicated TiledArray::DistArray. Usage: \code TiledArray::TArrayD
+/// replicated TiledArray::DistArray. Usage:
+/// \code
+/// TiledArray::TArrayD
 /// array(world, trange);
 /// // Set tiles of array ...
 ///
 /// auto t = array_to_btas_tensor(array);
 /// \endcode
 /// \tparam Tile the tile type of \c src
 /// \tparam Policy the policy type of \c src
+/// \tparam Range_ the range type of the result (either, btas::RangeNd or
+///         TiledArray::Range)
+/// \tparam Storage_ the storage type of the result
 /// \param[in] src The TiledArray::DistArray<Tile,Policy> object whose contents
-/// will be copied to the result. \return A \c btas::Tensor object that is a
-/// copy of \c src \throw TiledArray::Exception When world size is greater than
-/// 1 and \c src is not replicated
+/// will be copied to the result.
+/// \return A \c btas::Tensor object that is a copy of \c src
+/// \throw TiledArray::Exception When world size is greater than
+///        1 and \c src is not replicated
 /// \param[in] target_rank the rank on which to create the BTAS tensor
 ///            containing the data of \c src ; if \c target_rank=-1 then
 ///            create the BTAS tensor on every rank (this requires
 ///            that \c src.is_replicated()==true )
 /// \return BTAS tensor object containing the data of \c src , if my rank equals
 ///         \c target_rank or \c target_rank==-1 ,
 ///         default-initialized BTAS tensor otherwise.
-template <typename Tile, typename Policy,
-          typename Storage = std::vector<typename Tile::value_type>>
-btas::Tensor<typename Tile::value_type, btas::DEFAULT::range, Storage>
-array_to_btas_tensor(const TiledArray::DistArray<Tile, Policy>& src,
-                     int target_rank = -1) {
+template <typename Tile, typename Policy, typename Range_ = TiledArray::Range,
+          typename Storage_ = btas::DEFAULT::storage<typename Tile::value_type>>
+btas::Tensor<typename Tile::value_type, Range_, Storage_> array_to_btas_tensor(
+    const TiledArray::DistArray<Tile, Policy>& src, int target_rank = -1) {
   // Test preconditions
-  if (target_rank == -1 && !src.pmap()->is_replicated())
+  if (target_rank == -1 && src.world().size() > 1 &&
+      !src.pmap()->is_replicated())
     TA_ASSERT(
         src.world().size() == 1 &&
         "TiledArray::array_to_btas_tensor(): a non-replicated array can only "
@@ -310,7 +313,7 @@ array_to_btas_tensor(const TiledArray::DistArray<Tile, Policy>& src,
 
   using result_type =
       btas::Tensor<typename TiledArray::DistArray<Tile, Policy>::element_type,
-                   btas::DEFAULT::range, Storage>;
+                   Range_, Storage_>;
   using result_range_type = typename result_type::range_type;
 
   // Construct the result