From 1f6f10d654be0dac1be056ae5025ac0a9d6dfe96 Mon Sep 17 00:00:00 2001 From: Alex Lindsay Date: Tue, 18 Feb 2020 16:47:15 -0800 Subject: [PATCH] Dispatch to allgather_packed_range from allgather for dynamic size types This works currently by: - Adding a `is_fixed_type` bool to StandardType classes - doing SFINAE on Communicator::allgather based on `is_fixed_type` This also adds default `typename Enable = void` template arguments to `StandardType` and `Packing`. This is useful for enabling/disabling specializations based on whether data members are fixed sizes or not. I'm thinking of an example like `DualNumber` where that class may be fixed-size depending on whether D is fixed-size --- src/Makefile.am | 1 + src/parallel/include/timpi/communicator.h | 30 +++-- src/parallel/include/timpi/packing.h | 3 +- src/parallel/include/timpi/packing_forward.h | 40 ++++++ .../include/timpi/parallel_implementation.h | 23 +++- src/parallel/include/timpi/standard_type.h | 33 ++++- test/Makefile.am | 9 ++ test/dispatch_to_packed_unit.C | 124 ++++++++++++++++++ test/packed_range_unit.C | 21 ++- 9 files changed, 264 insertions(+), 20 deletions(-) create mode 100644 src/parallel/include/timpi/packing_forward.h create mode 100644 test/dispatch_to_packed_unit.C diff --git a/src/Makefile.am b/src/Makefile.am index d2fe48a..87edce1 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -77,6 +77,7 @@ include_HEADERS += parallel/include/timpi/communicator.h include_HEADERS += parallel/include/timpi/data_type.h include_HEADERS += parallel/include/timpi/message_tag.h include_HEADERS += parallel/include/timpi/op_function.h +include_HEADERS += parallel/include/timpi/packing_forward.h include_HEADERS += parallel/include/timpi/packing.h include_HEADERS += parallel/include/timpi/parallel_communicator_specializations include_HEADERS += parallel/include/timpi/parallel_implementation.h diff --git a/src/parallel/include/timpi/communicator.h b/src/parallel/include/timpi/communicator.h index 11cfe4d..fa701b1 100644 --- a/src/parallel/include/timpi/communicator.h +++ b/src/parallel/include/timpi/communicator.h @@ -20,17 +20,19 @@ #define TIMPI_COMMUNICATOR_H // TIMPI includes -#include "timpi/data_type.h" +#include "timpi/standard_type.h" #include "timpi/message_tag.h" #include "timpi/timpi_config.h" #include "timpi/request.h" #include "timpi/status.h" +#include "timpi/packing_forward.h" // C++ includes #include #include // shared_ptr #include #include +#include // C++ includes needed for parallel_communicator_specializations // @@ -38,15 +40,6 @@ #include // for specializations #include -// FIXME: This *should* be in TIMPI namespace but we have libMesh -// users which already partially specialized it -namespace libMesh { -namespace Parallel { -template -class Packing; -} -} - namespace TIMPI { @@ -827,9 +820,22 @@ class Communicator /** * Take a vector of length \p this->size(), and fill in - * \p recv[processor_id] = the value of \p send on that processor + * \p recv[processor_id] = the value of \p send on that processor. This + * overload works on fixed size types */ - template + template ::is_fixed_type, + int>::type = 0> + inline void allgather(const T & send, + std::vector & recv) const; + + /** + * Take a vector of length \p this->size(), and fill in + * \p recv[processor_id] = the value of \p send on that processor. This + * overload works on potentially dynamically sized types, and dispatches + * to \p allgather_packed_range + */ + template ::is_fixed_type, + int>::type = 0> inline void allgather(const T & send, std::vector & recv) const; diff --git a/src/parallel/include/timpi/packing.h b/src/parallel/include/timpi/packing.h index 09e2b66..e71d0e4 100644 --- a/src/parallel/include/timpi/packing.h +++ b/src/parallel/include/timpi/packing.h @@ -21,6 +21,7 @@ // TIMPI Includes #include "timpi/timpi_assert.h" +#include "timpi/packing_forward.h" // C++ includes #include @@ -43,7 +44,7 @@ namespace Parallel * Users will need to specialize this class for their particular data * types. */ -template +template class Packing { public: // Should be an MPI sendable type in specializations, e.g. diff --git a/src/parallel/include/timpi/packing_forward.h b/src/parallel/include/timpi/packing_forward.h new file mode 100644 index 0000000..432412a --- /dev/null +++ b/src/parallel/include/timpi/packing_forward.h @@ -0,0 +1,40 @@ +// The TIMPI Message-Passing Parallelism Library. +// Copyright (C) 2002-2019 Benjamin S. Kirk, John W. Peterson, Roy H. Stogner + +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. + +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. + +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + + +#ifndef TIMPI_PACKING_FORWARD_H +#define TIMPI_PACKING_FORWARD_H + +// FIXME: This *should* be in TIMPI namespace but we have libMesh +// users which already partially specialized it +namespace libMesh +{ +namespace Parallel +{ +/** + * Define data types and (un)serialization functions for use when + * encoding a potentially-variable-size object of type T. + * + * Users will need to specialize this class for their particular data + * types. + */ +template +class Packing; +} +} + +#endif // TIMPI_PACKING_FORWARD_H diff --git a/src/parallel/include/timpi/parallel_implementation.h b/src/parallel/include/timpi/parallel_implementation.h index 60dd0af..e35da53 100644 --- a/src/parallel/include/timpi/parallel_implementation.h +++ b/src/parallel/include/timpi/parallel_implementation.h @@ -2712,7 +2712,8 @@ inline void Communicator::gather(const unsigned int root_id, -template +template ::is_fixed_type, int>::type> inline void Communicator::allgather(const T & sendval, std::vector & recv) const { @@ -2734,6 +2735,26 @@ inline void Communicator::allgather(const T & sendval, recv[0] = sendval; } +template ::is_fixed_type, int>::type> +inline void Communicator::allgather(const T & sendval, + std::vector & recv) const +{ + TIMPI_LOG_SCOPE ("allgather()","Parallel"); + + timpi_assert(this->size()); + recv.resize(this->size()); + + unsigned int comm_size = this->size(); + if (comm_size > 1) + { + std::vector range = {sendval}; + + allgather_packed_range((void *)(NULL), range.begin(), range.end(), recv.begin()); + } + else if (comm_size > 0) + recv[0] = sendval; +} template diff --git a/src/parallel/include/timpi/standard_type.h b/src/parallel/include/timpi/standard_type.h index 9f4d615..c50b660 100644 --- a/src/parallel/include/timpi/standard_type.h +++ b/src/parallel/include/timpi/standard_type.h @@ -56,12 +56,11 @@ struct standardtype_dependent_false : std::false_type * More complicated data types may need to provide a pointer-to-T so * that we can use MPI_Address without constructing a new T. */ -template +template class StandardType : public DataType { - // Get a slightly better compiler diagnostic - static_assert(standardtype_dependent_false::value, - "Only specializations of StandardType may be used, did you forget to include a header file (e.g. parallel_algebra.h)?"); +public: + static const bool is_fixed_type = false; /* * The unspecialized class is useless, so we make its constructor @@ -87,6 +86,8 @@ class StandardType : public DataType public: \ explicit \ StandardType(const cxxtype * = nullptr) : DataType(mpitype) {} \ + \ + static const bool is_fixed_type = true; \ } #else @@ -98,6 +99,8 @@ class StandardType : public DataType public: \ explicit \ StandardType(const cxxtype * = nullptr) : DataType() {} \ + \ + static const bool is_fixed_type = true; \ } #endif @@ -140,6 +143,8 @@ TIMPI_STANDARD_TYPE(long double,MPI_LONG_DOUBLE); ~StandardType() { this->free(); } + + static const bool is_fixed_type = true; }; # endif #else @@ -211,6 +216,9 @@ class StandardType> : public DataType } ~StandardType() { this->free(); } + + static const bool is_fixed_type = StandardType::is_fixed_type + && StandardType::is_fixed_type; }; @@ -283,6 +291,19 @@ void FillDisplacementArray::fill } +template +struct CheckAllFixedTypes +{ + static const bool is_fixed_type = StandardType::is_fixed_type && + CheckAllFixedTypes::is_fixed_type; +}; + +template +struct CheckAllFixedTypes +{ + static const bool is_fixed_type = StandardType::is_fixed_type; +}; + template class StandardType> : public DataType { @@ -347,6 +368,8 @@ class StandardType> : public DataType } ~StandardType() { this->free(); } + + static const bool is_fixed_type = CheckAllFixedTypes::is_fixed_type; }; @@ -359,6 +382,8 @@ class StandardType> : public DataType DataType(StandardType(nullptr), 2) {} ~StandardType() { this->free(); } + + static const bool is_fixed_type = StandardType::is_fixed_type; }; } // namespace TIMPI diff --git a/test/Makefile.am b/test/Makefile.am index 5b87c32..d256753 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -15,15 +15,18 @@ if BUILD_DBG_MODE packed_range_unit_dbg_SOURCES = packed_range_unit.C parallel_sync_unit_dbg_SOURCES = parallel_sync_unit.C parallel_unit_dbg_SOURCES = parallel_unit.C + dispatch_to_packed_unit_dbg_SOURCES = dispatch_to_packed_unit.C message_tag_unit_dbg_LDFLAGS = $(top_builddir)/src/libtimpi_dbg.la packed_range_unit_dbg_LDFLAGS = $(top_builddir)/src/libtimpi_dbg.la parallel_sync_unit_dbg_LDFLAGS = $(top_builddir)/src/libtimpi_dbg.la parallel_unit_dbg_LDFLAGS = $(top_builddir)/src/libtimpi_dbg.la + dispatch_to_packed_unit_dbg_LDFLAGS = $(top_builddir)/src/libtimpi_dbg.la TESTS += message_tag_unit-dbg TESTS += packed_range_unit-dbg TESTS += parallel_sync_unit-dbg TESTS += parallel_unit-dbg + TESTS += dispatch_to_packed_unit-dbg endif if BUILD_DEVEL_MODE @@ -31,15 +34,18 @@ if BUILD_DEVEL_MODE packed_range_unit_devel_SOURCES = packed_range_unit.C parallel_sync_unit_devel_SOURCES = parallel_sync_unit.C parallel_unit_devel_SOURCES = parallel_unit.C + dispatch_to_packed_unit_devel_SOURCES = dispatch_to_packed_unit.C message_tag_unit_devel_LDFLAGS = $(top_builddir)/src/libtimpi_devel.la packed_range_unit_devel_LDFLAGS = $(top_builddir)/src/libtimpi_devel.la parallel_sync_unit_devel_LDFLAGS = $(top_builddir)/src/libtimpi_devel.la parallel_unit_devel_LDFLAGS = $(top_builddir)/src/libtimpi_devel.la + dispatch_to_packed_unit_devel_LDFLAGS = $(top_builddir)/src/libtimpi_devel.la TESTS += message_tag_unit-devel TESTS += packed_range_unit-devel TESTS += parallel_sync_unit-devel TESTS += parallel_unit-devel + TESTS += dispatch_to_packed_unit-devel endif if BUILD_OPT_MODE @@ -47,15 +53,18 @@ if BUILD_OPT_MODE packed_range_unit_opt_SOURCES = packed_range_unit.C parallel_sync_unit_opt_SOURCES = parallel_sync_unit.C parallel_unit_opt_SOURCES = parallel_unit.C + dispatch_to_packed_unit_opt_SOURCES = dispatch_to_packed_unit.C message_tag_unit_opt_LDFLAGS = $(top_builddir)/src/libtimpi_opt.la packed_range_unit_opt_LDFLAGS = $(top_builddir)/src/libtimpi_opt.la parallel_sync_unit_opt_LDFLAGS = $(top_builddir)/src/libtimpi_opt.la parallel_unit_opt_LDFLAGS = $(top_builddir)/src/libtimpi_opt.la + dispatch_to_packed_unit_opt_LDFLAGS = $(top_builddir)/src/libtimpi_opt.la TESTS += message_tag_unit-opt TESTS += packed_range_unit-opt TESTS += parallel_sync_unit-opt TESTS += parallel_unit-opt + TESTS += dispatch_to_packed_unit-opt endif check_PROGRAMS = $(TESTS) diff --git a/test/dispatch_to_packed_unit.C b/test/dispatch_to_packed_unit.C new file mode 100644 index 0000000..e2d31bd --- /dev/null +++ b/test/dispatch_to_packed_unit.C @@ -0,0 +1,124 @@ +#include + +#include +#include +#include + +#define TIMPI_UNIT_ASSERT(expr) \ + if (!(expr)) \ + timpi_error() + +namespace TIMPI +{ +template +class StandardType> : public DataType +{ +public: + static const bool is_fixed_type = false; + +private: + StandardType(const std::set * example = nullptr); +}; +} + +namespace libMesh { +namespace Parallel { +template +class Packing> { +public: + + static const unsigned int size_bytes = 4; + + typedef T buffer_type; + + static unsigned int + get_set_len (typename std::vector::const_iterator in) + { + unsigned int set_len = reinterpret_cast(in[size_bytes-1]); + for (signed int i=size_bytes-2; i >= 0; --i) + { + set_len *= 256; + set_len += reinterpret_cast(in[i]); + } + return set_len; + } + + + static unsigned int + packed_size (typename std::vector::const_iterator in) + { + return get_set_len(in) + size_bytes; + } + + static unsigned int packable_size + (const std::set & s, + const void *) + { + return s.size() + size_bytes; + } + + + template + static void pack (const std::set & b, Iter data_out, + const void *) + { + unsigned int set_len = b.size(); + for (unsigned int i=0; i != size_bytes; ++i) + { + *data_out++ = (set_len % 256); + set_len /= 256; + } + std::copy(b.begin(), b.end(), data_out); + } + + static std::set + unpack (typename std::vector::const_iterator in, void *) + { + unsigned int set_len = get_set_len(in); + + return std::set(in + size_bytes, in + size_bytes + set_len); + } + +}; + + +} // namespace Parallel + +} // namespace libMesh + +using namespace TIMPI; + +Communicator *TestCommWorld; + + void testContainerAllGather() + { + std::vector> vals; + const unsigned int my_rank = TestCommWorld->rank(); + + std::vector data_vec(my_rank + 1); + std::iota(data_vec.begin(), data_vec.end(), 0); + TestCommWorld->allgather(std::set(data_vec.begin(), data_vec.end()), vals); + + const std::size_t comm_size = TestCommWorld->size(); + const std::size_t vec_size = vals.size(); + TIMPI_UNIT_ASSERT(comm_size == vec_size); + + for (std::size_t i = 0; i < vec_size; ++i) + { + const auto & current_set = vals[i]; + unsigned int value = 0; + for (auto number : current_set) + TIMPI_UNIT_ASSERT(number == value++); + } + } + + +int main(int argc, const char * const * argv) +{ + TIMPI::TIMPIInit init(argc, argv); + TestCommWorld = &init.comm(); + + testContainerAllGather(); + + return 0; +} diff --git a/test/packed_range_unit.C b/test/packed_range_unit.C index cc2f4fb..6f8c7c4 100644 --- a/test/packed_range_unit.C +++ b/test/packed_range_unit.C @@ -5,7 +5,7 @@ #define TIMPI_UNIT_ASSERT(expr) \ if (!(expr)) \ - timpi_error(); + timpi_error() template struct null_output_iterator @@ -27,7 +27,6 @@ struct null_output_iterator null_output_iterator & operator*() { return *this; } }; - namespace libMesh { namespace Parallel { template @@ -144,6 +143,23 @@ Communicator *TestCommWorld; (std::string*)NULL); } + void testContainerAllGather() + { + // This method uses a specialized allgather method that is only defined + // when we have MPI +#ifdef TIMPI_HAVE_MPI + std::vector vals; + const unsigned int my_rank = TestCommWorld->rank(); + TestCommWorld->allgather(std::string(my_rank+1, '0' + my_rank), vals); + + const std::size_t comm_size = TestCommWorld->size(); + const std::size_t vec_size = vals.size(); + TIMPI_UNIT_ASSERT(comm_size == vec_size); + + for (std::size_t i = 0; i < vec_size; ++i) + TIMPI_UNIT_ASSERT(vals[i] == std::string(i + 1, '0' + i)); +#endif + } void testContainerSendReceive() { @@ -189,6 +205,7 @@ int main(int argc, const char * const * argv) testNullAllGather(); testNullSendReceive(); + testContainerAllGather(); testContainerSendReceive(); return 0;