From 25ecebf00d8b6f53f9f7b210a6532347f1505c84 Mon Sep 17 00:00:00 2001 From: "E. G. Patrick Bos" Date: Wed, 9 Jun 2021 15:44:52 +0200 Subject: [PATCH 1/9] Add builtin ZeroMQ for RooFit::MultiProcess (later) --- roofit/CMakeLists.txt | 1 + roofit/builtin_zeromq/CMakeLists.txt | 35 ++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 roofit/builtin_zeromq/CMakeLists.txt diff --git a/roofit/CMakeLists.txt b/roofit/CMakeLists.txt index 890df0340dde2..01478c67ca3d5 100644 --- a/roofit/CMakeLists.txt +++ b/roofit/CMakeLists.txt @@ -5,6 +5,7 @@ # For the list of contributors see $ROOTSYS/README/CREDITS. add_subdirectory(batchcompute) +add_subdirectory(builtin_zeromq) add_subdirectory(roofitcore) add_subdirectory(roofit) if(mathmore) diff --git a/roofit/builtin_zeromq/CMakeLists.txt b/roofit/builtin_zeromq/CMakeLists.txt new file mode 100644 index 0000000000000..8407e20d1deae --- /dev/null +++ b/roofit/builtin_zeromq/CMakeLists.txt @@ -0,0 +1,35 @@ +include(ExternalProject) + +set(ZeroMQ_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/BUILTIN_ZeroMQ-prefix) +set(ZeroMQ_LIBNAME ${CMAKE_STATIC_LIBRARY_PREFIX}zmq${CMAKE_STATIC_LIBRARY_SUFFIX}) + +set(ZeroMQ_LIBRARY ${ZeroMQ_PREFIX}/lib/${ZeroMQ_LIBNAME} CACHE INTERNAL "" FORCE) +set(ZeroMQ_LIBRARIES ${ZeroMQ_LIBRARY} CACHE INTERNAL "" FORCE) + +ExternalProject_Add(BUILTIN_ZeroMQ + URL https://github.com/zeromq/libzmq/archive/refs/tags/v4.3.2.tar.gz + CMAKE_ARGS + -DCMAKE_INSTALL_PREFIX= + -DWITH_PERF_TOOL=OFF + -DZMQ_BUILD_TESTS=OFF + -DPOLLER=select + BUILD_BYPRODUCTS ${ZeroMQ_LIBRARIES} + ) + +set(ZeroMQ_FOUND TRUE CACHE BOOL "" FORCE) + +set(ZeroMQ_INCLUDE_DIR ${ZeroMQ_PREFIX}/include CACHE INTERNAL "" FORCE) +set(ZeroMQ_INCLUDE_DIRS ${ZeroMQ_PREFIX}/include CACHE INTERNAL "" FORCE) + +add_library(ZeroMQ INTERFACE) +target_include_directories(ZeroMQ INTERFACE $) +target_link_libraries(ZeroMQ INTERFACE $) +add_dependencies(ZeroMQ BUILTIN_ZeroMQ) + +add_library(ZeroMQ::ZeroMQ ALIAS ZeroMQ) + +set(ZeroMQ_DIR ${ZeroMQ_PREFIX}/share/cmake/ZeroMQ CACHE INTERNAL "" FORCE) + +# also need source and build directories for ppoll in RooFitZMQ +set(ZeroMQ_SOURCE_DIR ${ZeroMQ_PREFIX}/src/BUILTIN_ZeroMQ/src CACHE INTERNAL "" FORCE) +set(ZeroMQ_BUILD_DIR ${ZeroMQ_PREFIX}/src/BUILTIN_ZeroMQ-build CACHE INTERNAL "" FORCE) \ No newline at end of file From 856308b47307fbe48742be1af191079baf5719a4 Mon Sep 17 00:00:00 2001 From: "E. G. Patrick Bos" Date: Thu, 10 Jun 2021 14:07:22 +0200 Subject: [PATCH 2/9] suppress warnings in ZeroMQ builtin build --- roofit/builtin_zeromq/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roofit/builtin_zeromq/CMakeLists.txt b/roofit/builtin_zeromq/CMakeLists.txt index 8407e20d1deae..3f64bcb19ac8e 100644 --- a/roofit/builtin_zeromq/CMakeLists.txt +++ b/roofit/builtin_zeromq/CMakeLists.txt @@ -13,6 +13,8 @@ ExternalProject_Add(BUILTIN_ZeroMQ -DWITH_PERF_TOOL=OFF -DZMQ_BUILD_TESTS=OFF -DPOLLER=select + # TODO: remove -w flag when updating to v4.3.5 or higher, current warnings should be gone by then + -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}\ -w BUILD_BYPRODUCTS ${ZeroMQ_LIBRARIES} ) From 8b59b0bb8b5fb57f65a267b303663a95f0c496ea Mon Sep 17 00:00:00 2001 From: "E. G. Patrick Bos" Date: Thu, 10 Jun 2021 15:52:41 +0200 Subject: [PATCH 3/9] add cppzmq to ZeroMQ builtin --- roofit/builtin_zeromq/CMakeLists.txt | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/roofit/builtin_zeromq/CMakeLists.txt b/roofit/builtin_zeromq/CMakeLists.txt index 3f64bcb19ac8e..9b769f3d00e95 100644 --- a/roofit/builtin_zeromq/CMakeLists.txt +++ b/roofit/builtin_zeromq/CMakeLists.txt @@ -18,6 +18,8 @@ ExternalProject_Add(BUILTIN_ZeroMQ BUILD_BYPRODUCTS ${ZeroMQ_LIBRARIES} ) +ExternalProject_Get_Property(BUILTIN_ZeroMQ install_dir) + set(ZeroMQ_FOUND TRUE CACHE BOOL "" FORCE) set(ZeroMQ_INCLUDE_DIR ${ZeroMQ_PREFIX}/include CACHE INTERNAL "" FORCE) @@ -34,4 +36,11 @@ set(ZeroMQ_DIR ${ZeroMQ_PREFIX}/share/cmake/ZeroMQ CACHE INTERNAL "" FORCE) # also need source and build directories for ppoll in RooFitZMQ set(ZeroMQ_SOURCE_DIR ${ZeroMQ_PREFIX}/src/BUILTIN_ZeroMQ/src CACHE INTERNAL "" FORCE) -set(ZeroMQ_BUILD_DIR ${ZeroMQ_PREFIX}/src/BUILTIN_ZeroMQ-build CACHE INTERNAL "" FORCE) \ No newline at end of file +set(ZeroMQ_BUILD_DIR ${ZeroMQ_PREFIX}/src/BUILTIN_ZeroMQ-build CACHE INTERNAL "" FORCE) + +ExternalProject_Add(BUILTIN_cppzmq + URL https://github.com/zeromq/cppzmq/archive/refs/tags/v4.7.1.tar.gz + CMAKE_ARGS + -DCMAKE_INSTALL_PREFIX=${install_dir} + -DCPPZMQ_BUILD_TESTS=OFF + ) From 1042726634b5e0a10a333642d4fb81d3e5de6a95 Mon Sep 17 00:00:00 2001 From: "E. G. Patrick Bos" Date: Thu, 10 Jun 2021 15:55:06 +0200 Subject: [PATCH 4/9] add RooFitZMQ package The files in here are modified versions of the ZMQ convenience functions from https://gitlab.cern.ch/raaij/generate_and_sort, contributed by @roelaaij. --- roofit/CMakeLists.txt | 1 + roofit/roofitZMQ/CMakeLists.txt | 40 ++++ roofit/roofitZMQ/inc/LinkDef.h | 1 + roofit/roofitZMQ/inc/RooFit_ZMQ/Utility.h | 39 ++++ .../roofitZMQ/inc/RooFit_ZMQ/ZeroMQPoller.h | 48 ++++ roofit/roofitZMQ/inc/RooFit_ZMQ/ZeroMQSvc.h | 214 ++++++++++++++++++ roofit/roofitZMQ/inc/RooFit_ZMQ/functions.h | 111 +++++++++ roofit/roofitZMQ/src/ZeroMQPoller.cpp | 133 +++++++++++ roofit/roofitZMQ/src/ZeroMQSvc.cpp | 103 +++++++++ roofit/roofitZMQ/src/functions.cpp | 9 + 10 files changed, 699 insertions(+) create mode 100644 roofit/roofitZMQ/CMakeLists.txt create mode 100644 roofit/roofitZMQ/inc/LinkDef.h create mode 100644 roofit/roofitZMQ/inc/RooFit_ZMQ/Utility.h create mode 100644 roofit/roofitZMQ/inc/RooFit_ZMQ/ZeroMQPoller.h create mode 100644 roofit/roofitZMQ/inc/RooFit_ZMQ/ZeroMQSvc.h create mode 100644 roofit/roofitZMQ/inc/RooFit_ZMQ/functions.h create mode 100644 roofit/roofitZMQ/src/ZeroMQPoller.cpp create mode 100644 roofit/roofitZMQ/src/ZeroMQSvc.cpp create mode 100644 roofit/roofitZMQ/src/functions.cpp diff --git a/roofit/CMakeLists.txt b/roofit/CMakeLists.txt index 01478c67ca3d5..886761c71ee09 100644 --- a/roofit/CMakeLists.txt +++ b/roofit/CMakeLists.txt @@ -6,6 +6,7 @@ add_subdirectory(batchcompute) add_subdirectory(builtin_zeromq) +add_subdirectory(roofitZMQ) add_subdirectory(roofitcore) add_subdirectory(roofit) if(mathmore) diff --git a/roofit/roofitZMQ/CMakeLists.txt b/roofit/roofitZMQ/CMakeLists.txt new file mode 100644 index 0000000000000..c861a7471120a --- /dev/null +++ b/roofit/roofitZMQ/CMakeLists.txt @@ -0,0 +1,40 @@ +############################################################################ +# CMakeLists.txt file for building ROOT roofitcore/ZMQ package +# @author Patrick Bos, NL eScience Center +############################################################################ + +#if(NOT ZeroMQ_FOUND) +# find_package(ZeroMQ REQUIRED) +#endif() + +ROOT_STANDARD_LIBRARY_PACKAGE(RooFitZMQ + HEADERS + RooFit_ZMQ/Utility.h + RooFit_ZMQ/ZeroMQSvc.h + RooFit_ZMQ/functions.h + RooFit_ZMQ/ZeroMQPoller.h +# RooFit_ZMQ/ppoll.h + SOURCES + src/ZeroMQSvc.cpp + src/ZeroMQPoller.cpp + src/functions.cpp +# src/ppoll.cpp + DICTIONARY_OPTIONS +# -I${ZeroMQ_INCLUDE_DIR} + -writeEmptyRootPCM + DEPENDENCIES + Core # TVersionCheck, RegisterModule, these are always called from G__RooFitZMQ.cxx.o + LIBRARIES +# ${ZeroMQ_LIBRARY} + ZeroMQ + ) + + +target_link_libraries(RooFitZMQ PRIVATE ZeroMQ) +target_include_directories(RooFitZMQ PUBLIC $) +target_include_directories(RooFitZMQ PUBLIC $) +target_include_directories(RooFitZMQ PUBLIC $) + +#if(testing) +# ROOT_ADD_TEST_SUBDIRECTORY(test) +#endif() diff --git a/roofit/roofitZMQ/inc/LinkDef.h b/roofit/roofitZMQ/inc/LinkDef.h new file mode 100644 index 0000000000000..5495493c90b99 --- /dev/null +++ b/roofit/roofitZMQ/inc/LinkDef.h @@ -0,0 +1 @@ +// something \ No newline at end of file diff --git a/roofit/roofitZMQ/inc/RooFit_ZMQ/Utility.h b/roofit/roofitZMQ/inc/RooFit_ZMQ/Utility.h new file mode 100644 index 0000000000000..a329805967df6 --- /dev/null +++ b/roofit/roofitZMQ/inc/RooFit_ZMQ/Utility.h @@ -0,0 +1,39 @@ +#ifndef SERIALIZE_UTILITY_H +#define SERIALIZE_UTILITY_H 1 + +#if defined(__clang__) && (__clang_major__ > 3 || (__clang_major__ == 3 && __clang_minor__ >= 10)) +#define HAVE_TRIVIALLY_COPYABLE 1 +#elif defined(__GNUC__) && __GNUC__ >= 5 +#define HAVE_TRIVIALLY_COPYABLE 1 +#else +#undef HAVE_TRIVIALLY_COPYABLE +#endif + +#include + +namespace ZMQ { + namespace Detail { + +#if defined(HAVE_TRIVIALLY_COPYABLE) + template + using simple_object = std::is_trivially_copyable; +#else + template + using simple_object = std::is_pod; +#endif + +// is trivial + template + struct is_trivial : std::conditional< + simple_object< + typename std::decay::type + >::value, + std::true_type, + std::false_type + >::type { + }; + + } +} + +#endif // SERIALIZE_UTILITY_H diff --git a/roofit/roofitZMQ/inc/RooFit_ZMQ/ZeroMQPoller.h b/roofit/roofitZMQ/inc/RooFit_ZMQ/ZeroMQPoller.h new file mode 100644 index 0000000000000..4d14444c01553 --- /dev/null +++ b/roofit/roofitZMQ/inc/RooFit_ZMQ/ZeroMQPoller.h @@ -0,0 +1,48 @@ +#ifndef ZEROMQPOLLER_H +#define ZEROMQPOLLER_H 1 +#include +#include +#include +#include + +#include "RooFit_ZMQ/ZeroMQSvc.h" +#include "RooFit_ZMQ/functions.h" + +class ZeroMQPoller { +public: + + using entry_t = std::tuple; + // The key is what zmq::socket_t stores inside, and what goes into + // pollitem_t through zmq::socket_t's conversion to void* operator + using sockets_t = std::unordered_map; + + using fd_entry_t = std::tuple; + using fds_t = std::unordered_map; + + using free_t = std::deque; + + ZeroMQPoller() = default; + + std::vector> poll(int timeo = -1); + std::vector> ppoll(int timeo, const sigset_t * sigmask_); + + size_t size() const; + + size_t register_socket(zmq::socket_t& socket, zmq::PollType type); + size_t register_socket(int fd, zmq::PollType type); + + size_t unregister_socket(zmq::socket_t& socket); + size_t unregister_socket(int fd); + +private: + + // Vector of (socket, flags) + std::vector m_items; + sockets_t m_sockets; + fds_t m_fds; + + // free slots in items + free_t m_free; +}; + +#endif // ZEROMQPOLLER_H diff --git a/roofit/roofitZMQ/inc/RooFit_ZMQ/ZeroMQSvc.h b/roofit/roofitZMQ/inc/RooFit_ZMQ/ZeroMQSvc.h new file mode 100644 index 0000000000000..0eace37b7f6c4 --- /dev/null +++ b/roofit/roofitZMQ/inc/RooFit_ZMQ/ZeroMQSvc.h @@ -0,0 +1,214 @@ +#ifndef ZEROMQ_IZEROMQSVC_H +#define ZEROMQ_IZEROMQSVC_H 1 + +// Include files +// from STL +#include +#include +#include +#include +#include +#include // std::cerr + +// ZeroMQ +#include +#include "RooFit_ZMQ/Utility.h" +#include "RooFit_ZMQ/functions.h" + +// debugging +#include // getpid + +namespace ZMQ { + +struct TimeOutException : std::exception { + TimeOutException() = default; + TimeOutException(const TimeOutException&) = default; + ~TimeOutException() = default; + TimeOutException& operator=(const TimeOutException&) = default; +}; + +struct MoreException : std::exception { + MoreException() = default; + MoreException(const MoreException&) = default; + ~MoreException() = default; + MoreException& operator=(const MoreException&) = default; +}; + +} + +template +struct ZmqLingeringSocketPtrDeleter { + void operator()(zmq::socket_t* socket) { + auto period = PERIOD; + + int tries = 0; + int max_tries = 3; + while(true) { + try { + // the actual work this function should do, plus the delete socket below: + if (socket) socket->setsockopt(ZMQ_LINGER, &period, sizeof(period)); + break; + } catch (zmq::error_t& e) { + if (++tries == max_tries + || e.num() == EINVAL || e.num() == ETERM || e.num() == ENOTSOCK // not recoverable from here + ) { + std::cerr << "ERROR in ZeroMQSvc::socket: " << e.what() << " (errno: " << e.num() << ")\n"; + throw; + } + std::cerr << "RETRY " << tries << "/" << (max_tries - 1) << " in ZmqLingeringSocketPtrDeleter: call interrupted (errno: " << e.num() << ")\n"; + } + } + + delete socket; + } +}; + +template +using ZmqLingeringSocketPtr = std::unique_ptr>; + + +// We retry send and receive only on EINTR, all other errors are either fatal, or can only +// be handled at the caller. +template +auto retry_send(zmq::socket_t& socket, int max_tries, args_t ...args) -> decltype(socket.send(args...)) { + int tries = 0; + while(true) { + try { + // the actual work this function should do, all the rest is error handling: + return socket.send(args...); + } catch (zmq::error_t& e) { + if (++tries == max_tries + || e.num() != EINTR // only recoverable error + ) { +// std::cerr << "ERROR in ZeroMQSvc::send (retry_send) on pid " << getpid() << ": " << e.what() << " (errno: " << e.num() << ")\n"; + throw; + } + std::cerr << "RETRY " << tries << "/" << (max_tries - 1) << " in ZeroMQSvc::send (retry_send) on pid " << getpid() << ": " << e.what() << ")\n"; + } + } +} + +template +auto retry_recv(zmq::socket_t& socket, int max_tries, args_t ...args) -> decltype(socket.recv(args...)) { + int tries = 0; + while(true) { + try { + // the actual work this function should do, all the rest is error handling: + return socket.recv(args...); + } catch (zmq::error_t& e) { + if (++tries == max_tries + || e.num() != EINTR // only recoverable error + ) { +// std::cerr << "ERROR in ZeroMQSvc::recv (retry_recv) on pid " << getpid() << ": " << e.what() << " (errno: " << e.num() << ")\n"; + throw; + } + std::cerr << "RETRY " << tries << "/" << (max_tries - 1) << " in ZeroMQSvc::recv (retry_recv) on pid " << getpid() << ": " << e.what() << ")\n"; + } + } +} + + +/** @class IZeroMQSvc IZeroMQSvc.h ZeroMQ/IZeroMQSvc.h + * + * + * @author + * @date 2015-06-22 + */ +class ZeroMQSvc { + // Note on error handling: + // Creating message_t can throw, but only when memory ran out (errno ENOMEM), + // and that is something only the caller can fix, so we don't catch it here. + +public: + + enum Encoding { + Text = 0, + Binary + }; + + Encoding encoding() const; + void setEncoding(const Encoding& e); + zmq::context_t& context() const; + zmq::socket_t socket(int type) const; + zmq::socket_t* socket_ptr(int type) const; + void close_context() const; + + // decode message with ZMQ, POD version + template ::value + && ZMQ::Detail::is_trivial::value, T>::type* = nullptr> + T decode(const zmq::message_t& msg) const { + T object; + memcpy(&object, msg.data(), msg.size()); + return object; + } + + // decode ZMQ message, string version + template ::value, T>::type* = nullptr> + std::string decode(const zmq::message_t& msg) const { + std::string r(msg.size() + 1, char{}); + r.assign(static_cast(msg.data()), msg.size()); + return r; + } + + // receive message with ZMQ, general version + // FIXME: what to do with flags=0.... more is a pointer, that might prevent conversion + template ::value), T>::type* = nullptr> + T receive(zmq::socket_t& socket, int flags = 0, bool* more = nullptr) const { + // receive message + zmq::message_t msg; + auto nbytes = retry_recv(socket, 2, &msg, flags); + if (0 == nbytes) { + throw ZMQ::TimeOutException{}; + } + if (more) *more = msg.more(); + + // decode message + return decode(msg); + } + + // receive message with ZMQ + template ::value, T>::type* = nullptr> + T receive(zmq::socket_t& socket, int flags = 0, bool* more = nullptr) const { + // receive message + zmq::message_t msg; + auto nbytes = retry_recv(socket, 2, &msg, flags); + if (0 == nbytes) { + throw ZMQ::TimeOutException{}; + } + if (more) *more = msg.more(); + return msg; + } + + // encode message to ZMQ + template ::value + && ZMQ::Detail::is_trivial::value, T>::type* = nullptr> + zmq::message_t encode(const T& item, std::function sizeFun = ZMQ::defaultSizeOf) const { + size_t s = sizeFun(item); + zmq::message_t msg{s}; + memcpy((void *)msg.data(), &item, s); + return msg; + } + + zmq::message_t encode(const char* item) const; + zmq::message_t encode(const std::string& item) const; + + // Send message with ZMQ + template ::value, T>::type* = nullptr> + bool send(zmq::socket_t& socket, const T& item, int flags = 0) const { + return retry_send(socket, 1, encode(item), flags); + } + + bool send(zmq::socket_t& socket, const char* item, int flags = 0) const; + bool send(zmq::socket_t& socket, zmq::message_t& msg, int flags = 0) const; + bool send(zmq::socket_t& socket, zmq::message_t&& msg, int flags = 0) const; + +private: + + Encoding m_enc = Text; + mutable zmq::context_t* m_context = nullptr; + +}; + +ZeroMQSvc& zmqSvc(); + +#endif // ZEROMQ_IZEROMQSVC_H diff --git a/roofit/roofitZMQ/inc/RooFit_ZMQ/functions.h b/roofit/roofitZMQ/inc/RooFit_ZMQ/functions.h new file mode 100644 index 0000000000000..2199275b1d272 --- /dev/null +++ b/roofit/roofitZMQ/inc/RooFit_ZMQ/functions.h @@ -0,0 +1,111 @@ +#ifndef ZEROMQ_FUNCTIONS_H +#define ZEROMQ_FUNCTIONS_H 1 + +#include + +namespace zmq { + + enum SocketTypes { + PAIR = ZMQ_PAIR, + PUB = ZMQ_PUB, + SUB = ZMQ_SUB, + REQ = ZMQ_REQ, + REP = ZMQ_REP, + DEALER = ZMQ_DEALER, + ROUTER = ZMQ_ROUTER, + PULL = ZMQ_PULL, + PUSH = ZMQ_PUSH, + XPUB = ZMQ_XPUB, + XSUB = ZMQ_XSUB, + STREAM = ZMQ_STREAM + }; + + enum PollType : short { + POLLIN = ZMQ_POLLIN, + POLLOUT = ZMQ_POLLOUT + }; + + enum SocketOptions { + AFFINITY = ZMQ_AFFINITY, + IDENTITY = ZMQ_IDENTITY, + SUBSCRIBE = ZMQ_SUBSCRIBE, + UNSUBSCRIBE = ZMQ_UNSUBSCRIBE, + RATE = ZMQ_RATE, + RECOVERY_IVL = ZMQ_RECOVERY_IVL, + SNDBUF = ZMQ_SNDBUF, + RCVBUF = ZMQ_RCVBUF, + RCVMORE = ZMQ_RCVMORE, + FD = ZMQ_FD, + EVENTS = ZMQ_EVENTS, + TYPE = ZMQ_TYPE, + LINGER = ZMQ_LINGER, + RECONNECT_IVL = ZMQ_RECONNECT_IVL, + BACKLOG = ZMQ_BACKLOG, + RECONNECT_IVL_MAX = ZMQ_RECONNECT_IVL_MAX, + MAXMSGSIZE = ZMQ_MAXMSGSIZE, + SNDHWM = ZMQ_SNDHWM, + RCVHWM = ZMQ_RCVHWM, + MULTICAST_HOPS = ZMQ_MULTICAST_HOPS, + RCVTIMEO = ZMQ_RCVTIMEO, + SNDTIMEO = ZMQ_SNDTIMEO, + LAST_ENDPOINT = ZMQ_LAST_ENDPOINT, + ROUTER_MANDATORY = ZMQ_ROUTER_MANDATORY, + TCP_KEEPALIVE = ZMQ_TCP_KEEPALIVE, + TCP_KEEPALIVE_CNT = ZMQ_TCP_KEEPALIVE_CNT, + TCP_KEEPALIVE_IDLE = ZMQ_TCP_KEEPALIVE_IDLE, + TCP_KEEPALIVE_INTVL = ZMQ_TCP_KEEPALIVE_INTVL, + IMMEDIATE = ZMQ_IMMEDIATE, + XPUB_VERBOSE = ZMQ_XPUB_VERBOSE, + ROUTER_RAW = ZMQ_ROUTER_RAW, + IPV6 = ZMQ_IPV6, + MECHANISM = ZMQ_MECHANISM, + PLAIN_SERVER = ZMQ_PLAIN_SERVER, + PLAIN_USERNAME = ZMQ_PLAIN_USERNAME, + PLAIN_PASSWORD = ZMQ_PLAIN_PASSWORD, + CURVE_SERVER = ZMQ_CURVE_SERVER, + CURVE_PUBLICKEY = ZMQ_CURVE_PUBLICKEY, + CURVE_SECRETKEY = ZMQ_CURVE_SECRETKEY, + CURVE_SERVERKEY = ZMQ_CURVE_SERVERKEY, + PROBE_ROUTER = ZMQ_PROBE_ROUTER, + REQ_CORRELATE = ZMQ_REQ_CORRELATE, + REQ_RELAXED = ZMQ_REQ_RELAXED, + CONFLATE = ZMQ_CONFLATE, + ZAP_DOMAIN = ZMQ_ZAP_DOMAIN, + ROUTER_HANDOVER = ZMQ_ROUTER_HANDOVER, + TOS = ZMQ_TOS, + CONNECT_RID = ZMQ_CONNECT_RID, + GSSAPI_SERVER = ZMQ_GSSAPI_SERVER, + GSSAPI_PRINCIPAL = ZMQ_GSSAPI_PRINCIPAL, + GSSAPI_SERVICE_PRINCIPAL = ZMQ_GSSAPI_SERVICE_PRINCIPAL, + GSSAPI_PLAINTEXT = ZMQ_GSSAPI_PLAINTEXT, + HANDSHAKE_IVL = ZMQ_HANDSHAKE_IVL, + SOCKS_PROXY = ZMQ_SOCKS_PROXY, + XPUB_NODROP = ZMQ_XPUB_NODROP + }; + + // Message options + enum MessageOptions { + MORE = ZMQ_MORE, + SRCFD = ZMQ_SRCFD, + SHARED = ZMQ_SHARED + }; + + // Send/recv options. + enum SendRecvOptions { + DONTWAIT = ZMQ_DONTWAIT, + SNDMORE = ZMQ_SNDMORE + }; +} + +namespace ZMQ { + +template +size_t defaultSizeOf(const T&) { + return sizeof(T); +} + +size_t stringLength(const char& cs); + +} + +#endif // ZEROMQ_FUNCTIONS_H diff --git a/roofit/roofitZMQ/src/ZeroMQPoller.cpp b/roofit/roofitZMQ/src/ZeroMQPoller.cpp new file mode 100644 index 0000000000000..12eafaf09392d --- /dev/null +++ b/roofit/roofitZMQ/src/ZeroMQPoller.cpp @@ -0,0 +1,133 @@ +/* + * ROOT + * Copyright (c) 2018, Patrick Bos + * Distributed under the terms of the BSD 3-Clause License. + * + * The full license is in the file LICENSE, distributed with this software. + */ + +#include + +#include "RooFit_ZMQ/ZeroMQPoller.h" + +std::vector> ZeroMQPoller::poll(int timeo) { + std::vector> r; + if (m_items.empty()) { + throw std::runtime_error("No sockets registered"); + } + int n = 0; + while (true) { + try { + n = zmq::poll(&m_items[0], m_items.size(), timeo); + if (n == 0) return r; + break; + } catch (const zmq::error_t& e) { + std::cerr << "in ZeroMQPoller::poll on PID " << getpid() << ": " << e.what() << std::endl; + if (e.num() != EINTR) { + throw; + } + } + } + // TODO: replace this with ranges::v3::zip + for (size_t i = 0; i < m_items.size(); ++i) { + void* socket = m_items[i].socket; + size_t index = 0; + int flags = 0; + if (socket == nullptr) { + // an fd was registered + std::tie(index, flags) = m_fds[m_items[i].fd]; + } else { + // a socket was registered + const zmq::socket_t* s; + std::tie(index, flags, s) = m_sockets[socket]; + } + if (m_items[i].revents & short(flags)) { + r.emplace_back(index, flags); + } + } + return r; +} + +size_t ZeroMQPoller::size() const { + return m_items.size(); +} + +size_t ZeroMQPoller::register_socket(zmq::socket_t& socket, zmq::PollType type) { + zmq::socket_t* s = &socket; + auto it = m_sockets.find(s); + if (it != m_sockets.end()) { + return std::get<0>(it->second); + } + size_t index = m_free.empty() ? m_items.size() : m_free.front(); + if (!m_free.empty()) m_free.pop_front(); + // NOTE: tis uses the conversion-to-void* operator of + // zmq::socket_t, which returns the wrapped object + m_items.push_back({socket, 0, type, 0}); + + // We need to lookup by the pointer to the object wrapped by zmq::socket_t + m_sockets.emplace(m_items.back().socket, std::make_tuple(index, type, s)); + return index; +} + +size_t ZeroMQPoller::register_socket(int fd, zmq::PollType type) { + auto it = m_fds.find(fd); + if (it != m_fds.end()) { + return std::get<0>(it->second); + } + size_t index = m_free.empty() ? m_items.size() : m_free.front(); + if (!m_free.empty()) m_free.pop_front(); + // NOTE: tis uses the conversion-to-void* operator of + // zmq::socket_t, which returns the wrapped object + m_items.push_back({nullptr, fd, type, 0}); + + // We need to lookup by the pointer to the object wrapped by zmq::socket_t + m_fds.emplace(fd, std::make_tuple(index, type)); + return index; +} + + +size_t ZeroMQPoller::unregister_socket(zmq::socket_t& socket) { + if (!m_sockets.count(socket.operator void*())) { + throw std::out_of_range("Socket is not registered"); + } + // Remove from m_sockets + // Can't search by the key of m_sockets, as that is the wrapped + // object, but have to use the pointer to the wrapper + // (zmq::socket_t) + auto it = std::find_if(begin(m_sockets), end(m_sockets), + [&socket](const decltype(m_sockets)::value_type& entry) { + return &socket == std::get<2>(entry.second); + }); + auto index = std::get<0>(it->second); + m_free.push_back(index); + m_sockets.erase(it); + + // Remove from m_items + auto iit = std::find_if(begin(m_items), end(m_items), [&it](const zmq::pollitem_t& item) { + return it->first == item.socket; + }); + assert(iit != end(m_items)); + m_items.erase(iit); + + return index; +} + +size_t ZeroMQPoller::unregister_socket(int fd) { + if (!m_fds.count(fd)) { + throw std::out_of_range("fileno is not registered"); + } + // Remove from m_fds + auto it = m_fds.find(fd); + auto index = std::get<0>(it->second); + m_free.push_back(index); + m_fds.erase(it); + + // Remove from m_items + auto iit = std::find_if(begin(m_items), end(m_items), [&it](const zmq::pollitem_t& item) { + return it->first == item.fd; + }); + assert(iit != end(m_items)); + m_items.erase(iit); + + return index; +} diff --git a/roofit/roofitZMQ/src/ZeroMQSvc.cpp b/roofit/roofitZMQ/src/ZeroMQSvc.cpp new file mode 100644 index 0000000000000..b69695349e0c3 --- /dev/null +++ b/roofit/roofitZMQ/src/ZeroMQSvc.cpp @@ -0,0 +1,103 @@ +//#include +#include // std::ref +#include + +#include "RooFit_ZMQ/ZeroMQSvc.h" + +ZeroMQSvc& zmqSvc() { + static std::unique_ptr svc; + if (!svc) { + svc = std::make_unique(); + } + return *svc; +} + +ZeroMQSvc::Encoding ZeroMQSvc::encoding() const { + return m_enc; +} + +void ZeroMQSvc::setEncoding(const ZeroMQSvc::Encoding &e) { + m_enc = e; +} + +zmq::context_t& ZeroMQSvc::context() const { + if (!m_context) { + try { + m_context = new zmq::context_t; + } catch (zmq::error_t& e) { + std::cerr << "ERROR: Creating ZeroMQ context failed. This only happens when PGM initialization failed or when a nullptr was returned from zmq_ctx_new because the created context was invalid. Contact ZMQ experts when this happens, because it shouldn't.\n"; + throw e; + } + } + return *m_context; +} + +zmq::socket_t ZeroMQSvc::socket(int type) const { + try { + // the actual work this function should do, all the rest is error handling: + return zmq::socket_t{context(), type}; + } catch (zmq::error_t& e) { + // all zmq errors not recoverable from here, only at call site + std::cerr << "ERROR in ZeroMQSvc::socket: " << e.what() << " (errno: " << e.num() << ")\n"; + throw e; + } +} + +zmq::socket_t* ZeroMQSvc::socket_ptr(int type) const { + try { + // the actual work this function should do, all the rest is error handling: + return new zmq::socket_t(context(), type); + } catch (zmq::error_t& e) { + // all zmq errors not recoverable from here, only at call site + std::cerr << "ERROR in ZeroMQSvc::socket_ptr: " << e.what() << " (errno: " << e.num() << ")\n"; + throw e; + } +} + +void ZeroMQSvc::close_context() const { + if (m_context) { + delete m_context; + m_context = nullptr; + } +} + + +zmq::message_t ZeroMQSvc::encode(const char* item) const { + std::function fun = ZMQ::stringLength; + return encode(*item, fun); +} + +zmq::message_t ZeroMQSvc::encode(const std::string& item) const { + return encode(item.c_str()); +} + +//zmq::message_t ZeroMQSvc::encode(const TObject& item) const { +// auto deleteBuffer = []( void* data, void* /* hint */ ) -> void { +// delete [] (char*)data; +// }; +// +// TBufferFile buffer(TBuffer::kWrite); +// buffer.WriteObject(&item); +// +// // Create message and detach buffer +// // This is the only ZMQ thing that can throw, and only when memory ran out +// // (errno ENOMEM), and that is something only the caller can fix, so we don't +// // catch it here: +// zmq::message_t message(buffer.Buffer(), buffer.Length(), deleteBuffer); +// buffer.DetachBuffer(); +// +// return message; +//} + + +bool ZeroMQSvc::send(zmq::socket_t& socket, const char* item, int flags) const { + return retry_send(socket, 2, encode(item), flags); +} + +bool ZeroMQSvc::send(zmq::socket_t& socket, zmq::message_t& msg, int flags) const { + return retry_send(socket, 2, std::ref(msg), flags); +} + +bool ZeroMQSvc::send(zmq::socket_t& socket, zmq::message_t&& msg, int flags) const { + return retry_send(socket, 2, std::move(msg), flags); +} diff --git a/roofit/roofitZMQ/src/functions.cpp b/roofit/roofitZMQ/src/functions.cpp new file mode 100644 index 0000000000000..ebb6199bfdb5d --- /dev/null +++ b/roofit/roofitZMQ/src/functions.cpp @@ -0,0 +1,9 @@ +#include +#include "RooFit_ZMQ/ZeroMQSvc.h" +#include "RooFit_ZMQ/functions.h" + +namespace ZMQ { + size_t stringLength(const char& cs) { + return strlen(&cs); + } +} From 4476eb3316969f110f6f9f35c72abf75fcb89aa9 Mon Sep 17 00:00:00 2001 From: "E. G. Patrick Bos" Date: Thu, 10 Jun 2021 16:27:29 +0200 Subject: [PATCH 5/9] add RooFitZMQ tests --- roofit/roofitZMQ/CMakeLists.txt | 6 +- roofit/roofitZMQ/test/CMakeLists.txt | 6 + roofit/roofitZMQ/test/test_HWM.cxx | 130 ++++++++ roofit/roofitZMQ/test/test_ZMQ.cpp | 306 ++++++++++++++++++ .../test/test_ZMQ_load_balancing.cxx | 133 ++++++++ roofit/roofitZMQ/test/test_polling.cxx | 149 +++++++++ 6 files changed, 727 insertions(+), 3 deletions(-) create mode 100644 roofit/roofitZMQ/test/CMakeLists.txt create mode 100644 roofit/roofitZMQ/test/test_HWM.cxx create mode 100644 roofit/roofitZMQ/test/test_ZMQ.cpp create mode 100644 roofit/roofitZMQ/test/test_ZMQ_load_balancing.cxx create mode 100644 roofit/roofitZMQ/test/test_polling.cxx diff --git a/roofit/roofitZMQ/CMakeLists.txt b/roofit/roofitZMQ/CMakeLists.txt index c861a7471120a..8174ecb952418 100644 --- a/roofit/roofitZMQ/CMakeLists.txt +++ b/roofit/roofitZMQ/CMakeLists.txt @@ -35,6 +35,6 @@ target_include_directories(RooFitZMQ PUBLIC $) target_include_directories(RooFitZMQ PUBLIC $) -#if(testing) -# ROOT_ADD_TEST_SUBDIRECTORY(test) -#endif() +if(testing) + ROOT_ADD_TEST_SUBDIRECTORY(test) +endif() diff --git a/roofit/roofitZMQ/test/CMakeLists.txt b/roofit/roofitZMQ/test/CMakeLists.txt new file mode 100644 index 0000000000000..b1da06f39da1f --- /dev/null +++ b/roofit/roofitZMQ/test/CMakeLists.txt @@ -0,0 +1,6 @@ +#find_package(ZeroMQ REQUIRED) + +ROOT_ADD_GTEST(test_RooFitZMQ test_ZMQ.cpp LIBRARIES RooFitZMQ ${ZeroMQ_LIBRARY}) +ROOT_ADD_GTEST(test_RooFitZMQ_polling test_polling.cxx LIBRARIES RooFitZMQ ${ZeroMQ_LIBRARY}) +ROOT_ADD_GTEST(test_RooFitZMQ_HWM test_HWM.cxx LIBRARIES RooFitZMQ ${ZeroMQ_LIBRARY}) +ROOT_ADD_GTEST(test_RooFitZMQ_load_balancing test_ZMQ_load_balancing.cxx LIBRARIES RooFitZMQ ${ZeroMQ_LIBRARY}) diff --git a/roofit/roofitZMQ/test/test_HWM.cxx b/roofit/roofitZMQ/test/test_HWM.cxx new file mode 100644 index 0000000000000..7ef11b5c3f5ed --- /dev/null +++ b/roofit/roofitZMQ/test/test_HWM.cxx @@ -0,0 +1,130 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2016-2021, Netherlands eScience Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "gtest/gtest.h" + +#include // fork, usleep + +#include "RooFit_ZMQ/ZeroMQSvc.h" + +class HighWaterMarkTest : public ::testing::Test { +protected: + void SetUp() override { + do { + child_pid = fork(); + } while (child_pid == -1); // retry if fork fails + + if (child_pid > 0) { // parent + pusher.reset(zmqSvc().socket_ptr(zmq::PUSH)); + if (set_hwm) { + auto rc = zmq_setsockopt(*pusher, ZMQ_SNDHWM, &hwm, sizeof hwm); + assert(rc == 0); + } + pusher->bind("ipc:///tmp/ZMQ_test_fork_polling_P2C.ipc"); + } else { // child + puller.reset(zmqSvc().socket_ptr(zmq::PULL)); + if (set_hwm) { + auto rc = zmq_setsockopt(*puller, ZMQ_RCVHWM, &hwm, sizeof hwm); + assert(rc == 0); + } + puller->connect("ipc:///tmp/ZMQ_test_fork_polling_P2C.ipc"); + } + } + + void TearDown() override { + if (child_pid > 0) { // parent + // wait for child + int status = 0; + pid_t pid; + do { + pid = waitpid(child_pid, &status, 0); + } while (-1 == pid && EINTR == errno); // retry on interrupted system call + if (0 != status) { + if (WIFEXITED(status)) { printf("exited, status=%d\n", WEXITSTATUS(status)); } else if (WIFSIGNALED(status)) { printf("killed by signal %d\n", WTERMSIG(status)); } else if (WIFSTOPPED(status)) { printf("stopped by signal %d\n", WSTOPSIG(status)); } else if (WIFCONTINUED(status)) { printf("continued\n"); } + } + if (-1 == pid) { + throw std::runtime_error(std::string("waitpid, errno ") + std::to_string(errno)); + } + pusher.reset(nullptr); + zmqSvc().close_context(); + } else { // child + puller.reset(nullptr); + zmqSvc().close_context(); + } + } + + void run_test() { + std::size_t max_sends = 2000; + + if (child_pid > 0) { // parent + // start test + for (std::size_t ix = 0; ix < max_sends; ++ix) { + zmqSvc().send(*pusher, 0.1f); + zmqSvc().send(*pusher, 1); + zmqSvc().send(*pusher, true); + if (ix % 100 == 0) { + printf("parent at ix = %lu\n", ix); + } + } + } else { // child + // wait a few seconds before reading to allow the parent to overflow the HWM + printf("child waiting for 2 seconds...\n"); + sleep(2); + printf("child starts receiving\n"); + + for (std::size_t ix = 0; ix < max_sends; ++ix) { + zmqSvc().receive(*puller); + zmqSvc().receive(*puller); + zmqSvc().receive(*puller); + if (ix % 100 == 0) { + printf("child at ix = %lu\n", ix); + } + } + } + } + + pid_t child_pid {0}; + ZmqLingeringSocketPtr<> pusher, puller; + bool set_hwm = false; + int hwm = 0; +}; + + +TEST_F(HighWaterMarkTest, demonstrateHittingDefaultHWM) { + run_test(); +} + +class HighWaterMarkZeroTest : public HighWaterMarkTest { + void SetUp() override { + set_hwm = true; + hwm = 0; + HighWaterMarkTest::SetUp(); + } +}; + +TEST_F(HighWaterMarkZeroTest, zeroHWM) { + run_test(); +} + +class HighWaterMark2kTest : public HighWaterMarkTest { + void SetUp() override { + set_hwm = true; + hwm = 2000; + HighWaterMarkTest::SetUp(); + } +}; + +TEST_F(HighWaterMark2kTest, HWM2k) { + run_test(); +} diff --git a/roofit/roofitZMQ/test/test_ZMQ.cpp b/roofit/roofitZMQ/test/test_ZMQ.cpp new file mode 100644 index 0000000000000..985d1b91c5902 --- /dev/null +++ b/roofit/roofitZMQ/test/test_ZMQ.cpp @@ -0,0 +1,306 @@ +/***************************************************************************** + * Project: RooFit * + * Package: RooFitCore * + * @(#)root/roofitcore:$Id$ + * Authors: * + * PB, Patrick Bos, NL eScience Center, p.bos@esciencecenter.nl * + * * + * Redistribution and use in source and binary forms, * + * with or without modification, are permitted according to the terms * + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) * + *****************************************************************************/ + +#include "gtest/gtest.h" + +#include // fork, usleep + +#include + +#include "RooFit_ZMQ/ZeroMQSvc.h" + + +// N.B.: wait_for_child is identically defined in RooFit::MultiProcess, but we copy it here to reduce module +// interdependencies. It also requires an extra include: +#include // kill, SIGKILL + +int wait_for_child(pid_t child_pid, bool may_throw, int retries_before_killing) +{ + int status = 0; + int patience = retries_before_killing; + pid_t tmp; + do { + if (patience-- < 1) { + ::kill(child_pid, SIGKILL); + } + tmp = waitpid(child_pid, &status, WNOHANG); + } while (tmp == 0 // child has not yet changed state, try again + || (-1 == tmp && EINTR == errno) // retry on interrupted system call + ); + + if (patience < 1) { + std::cout << "Had to send PID " << child_pid << " " << (-patience + 1) << " SIGKILLs"; + } + + if (0 != status) { + if (WIFEXITED(status)) { + printf("exited, status=%d\n", WEXITSTATUS(status)); + } else if (WIFSIGNALED(status)) { + printf("killed by signal %d\n", WTERMSIG(status)); + } else if (WIFSTOPPED(status)) { + printf("stopped by signal %d\n", WSTOPSIG(status)); + } else if (WIFCONTINUED(status)) { + printf("continued\n"); + } + } + + if (-1 == tmp && may_throw) + throw std::runtime_error(std::string("waitpid, errno ") + std::to_string(errno)); + + return status; +} + +void elaborate_bind(const ZmqLingeringSocketPtr<>& socket, std::string name) { + try { + socket->bind(name); + } catch (const zmq::error_t& e) { + if (e.num() == EADDRINUSE) { + std::cerr << "address already in use, retrying bind in 500ms\n"; + usleep(500000); + try { + socket->bind(name); + } catch (const zmq::error_t& e2) { + if (e2.num() == EADDRINUSE) { + std::cerr + << "again: address already in use, aborting; please check whether there are any remaining improperly exited processes (zombies) around or whether some other program is using port 6660\n"; + } + throw e2; + } + // Sometimes, the socket from the previous test needs some time to close, so + // we introduce a latency here. A more robust and fast approach might be to + // do the following on the bind side: + // 1. first try another port, e.g. increase by one + // 2. if that doesn't work, do the latency and retry the original port + // The connect side then also needs to change, because it doesn't know which + // port the bind side will bind to. The connect side could try connecting to + // both options asynchronously, and then in a loop check both for signs of + // life. If one comes alive, transfer ownership of that pointer to the pointer + // you want to eventually use (`socket`) and that's it. + } else { + throw e; + } + } +} + +class AllSocketTypes : public ::testing::TestWithParam< std::tuple, std::pair /* socket_names */> > {}; + +TEST_P(AllSocketTypes, forkHandshake) { + auto socket_names = std::get<2>(GetParam()); + pid_t child_pid {0}; + do { + child_pid = fork(); + } while (child_pid == -1); // retry if fork fails + + if (child_pid > 0) { // master + ZmqLingeringSocketPtr<> socket; + socket.reset(zmqSvc().socket_ptr(std::get<1>(GetParam()).first)); + elaborate_bind(socket, socket_names.second); + // bind is on the master process to avoid zombie children to hold on to binds + + // start test + zmqSvc().send(*socket, std::string("breaker breaker")); + + auto receipt = zmqSvc().receive(*socket); + + EXPECT_EQ(receipt, 1212); + +// socket->close(); // this gives exception of type zmq::error_t: Socket operation on non-socket + socket.reset(nullptr); + zmqSvc().close_context(); // if you don't close context in parent process as well, the next repeat will hang + + wait_for_child(child_pid, true, 5); + } else { // child + ZmqLingeringSocketPtr<> socket; + socket.reset(zmqSvc().socket_ptr(std::get<1>(GetParam()).second)); + socket->connect(socket_names.first); + + // start test + auto receipt = zmqSvc().receive(*socket); + if (receipt == "breaker breaker") { + zmqSvc().send(*socket, 1212); + } + // take care, don't just use _exit, it will not cleanly destroy context etc! + // if you really need to, at least close and destroy everything properly +// socket->close(); // this gives exception of type zmq::error_t: Socket operation on non-socket + socket.reset(nullptr); + zmqSvc().close_context(); + _Exit(0); + } +} + +std::string ipc {"ipc:///tmp/ZMQ_test_fork.ipc"}; +std::string tcp_server {"tcp://127.0.0.1:6660"}; +std::string tcp_client {"tcp://*:6660"}; +auto socket_name_options = ::testing::Values( + std::make_pair(tcp_server, tcp_client), + std::make_pair(ipc, ipc) + ); + + +INSTANTIATE_TEST_SUITE_P(REQREP, AllSocketTypes, + ::testing::Combine(::testing::Range(0, 10), // repeat to probe connection stability + ::testing::Values(std::make_pair(zmq::REQ, zmq::REP)), + socket_name_options + )); +INSTANTIATE_TEST_SUITE_P(PAIRPAIR, AllSocketTypes, + ::testing::Combine(::testing::Range(0, 10), // repeat to probe connection stability + ::testing::Values(std::make_pair(zmq::PAIR, zmq::PAIR)), + socket_name_options + )); + + +class AsyncSocketTypes : public ::testing::TestWithParam< std::tuple, std::pair /* socket_names */, bool /* expect_throw */> > {}; + +TEST_P(AsyncSocketTypes, forkMultiSendReceive) { + bool expect_throw = std::get<3>(GetParam()); + ZmqLingeringSocketPtr<> socket; + auto socket_names = std::get<2>(GetParam()); + pid_t child_pid {0}; + do { + child_pid = fork(); + } while (child_pid == -1); // retry if fork fails + + if (child_pid > 0) { // master + socket.reset(zmqSvc().socket_ptr(std::get<1>(GetParam()).first)); + elaborate_bind(socket, socket_names.second); + // bind is on the master process to avoid zombie children to hold on to binds + + // start test: send 2 things, receive 1, send 1 more, finish + zmqSvc().send(*socket, std::string("breaker breaker")); + + if (expect_throw) { + EXPECT_ANY_THROW(zmqSvc().send(*socket, std::string("anybody out there?"))); + // NOTE: also in case of a throw, be sure to properly close down the connection! + // Otherwise, you may get zombies waiting for a reply. + socket.reset(nullptr); + zmqSvc().close_context(); // if you don't close context in parent process as well, the next repeat will hang + wait_for_child(child_pid, true, 5); + return; + } else { + EXPECT_NO_THROW(zmqSvc().send(*socket, std::string("anybody out there?"))); + } + + auto receipt = zmqSvc().receive(*socket); + + EXPECT_EQ(receipt, 1212); + + zmqSvc().send(*socket, std::string("kthxbye")); + +// socket->close(); // this gives exception of type zmq::error_t: Socket operation on non-socket + socket.reset(nullptr); + zmqSvc().close_context(); // if you don't close context in parent process as well, the next repeat will hang + + wait_for_child(child_pid, true, 5); + } else { // child + socket.reset(zmqSvc().socket_ptr(std::get<1>(GetParam()).second)); + socket->connect(socket_names.first); + + // start test, receive something + auto receipt1 = zmqSvc().receive(*socket); + auto receipt2 = zmqSvc().receive(*socket); + if (receipt1 == "breaker breaker" && receipt2 == "anybody out there?") { + zmqSvc().send(*socket, 1212); + } + auto receipt3 = zmqSvc().receive(*socket); + if (receipt3 != "kthxbye") { + std::cerr << "did not receive final reply correctly\n"; + } + + // take care, don't just use _exit, it will not cleanly destroy context etc! + // if you really need to, at least close and destroy everything properly +// socket->close(); // this gives exception of type zmq::error_t: Socket operation on non-socket + socket.reset(nullptr); + zmqSvc().close_context(); + _Exit(0); + } +} + +TEST_P(AsyncSocketTypes, forkIgnoreSomeMessages) { + bool expect_throw = std::get<3>(GetParam()); + ZmqLingeringSocketPtr<> socket; + auto socket_names = std::get<2>(GetParam()); + pid_t child_pid {0}; + do { + child_pid = fork(); + } while (child_pid == -1); // retry if fork fails + + if (child_pid > 0) { // master + socket.reset(zmqSvc().socket_ptr(std::get<1>(GetParam()).first)); + elaborate_bind(socket, socket_names.second); + // bind is on the master process to avoid zombie children to hold on to binds + + // start test: send 2 things, receive 1, send 1 more, finish + zmqSvc().send(*socket, std::string("breaker breaker")); + + if (expect_throw) { + EXPECT_ANY_THROW(zmqSvc().send(*socket, std::string("anybody out there?"))); + // NOTE: also in case of a throw, be sure to properly close down the connection! + // Otherwise, you may get zombies waiting for a reply. + socket.reset(nullptr); + zmqSvc().close_context(); // if you don't close context in parent process as well, the next repeat will hang + wait_for_child(child_pid, true, 5); + return; + } else { + EXPECT_NO_THROW(zmqSvc().send(*socket, std::string("anybody out there?"))); + } + + auto receipt = zmqSvc().receive(*socket); + + EXPECT_EQ(receipt, 1212); + + zmqSvc().send(*socket, std::string("kthxbye")); + +// socket->close(); // this gives exception of type zmq::error_t: Socket operation on non-socket + socket.reset(nullptr); + zmqSvc().close_context(); // if you don't close context in parent process as well, the next repeat will hang + + wait_for_child(child_pid, true, 5); + } else { // child + socket.reset(zmqSvc().socket_ptr(std::get<1>(GetParam()).second)); + socket->connect(socket_names.first); + + // start test, receive first thing + auto receipt = zmqSvc().receive(*socket); + if (receipt == "breaker breaker") { + zmqSvc().send(*socket, 1212); + } + + // ignore the rest of the sent messages, but give the other end a second to + // actually send its stuff, instead of hanging in retry_send because the + // connection has died; a better solution would be if retry_send (in + // ZeroMQSvc::send) had a callback mechanism that could be used to break + // out when a child has died, but ok + sleep(1); + + // take care, don't just use _exit, it will not cleanly destroy context etc! + // if you really need to, at least close and destroy everything properly +// socket->close(); // this gives exception of type zmq::error_t: Socket operation on non-socket + socket.reset(nullptr); + zmqSvc().close_context(); + _Exit(0); + } +} + + +INSTANTIATE_TEST_SUITE_P(PAIRPAIR, AsyncSocketTypes, + ::testing::Combine(::testing::Range(0, 10), // repeat to probe connection stability + ::testing::Values(std::make_pair(zmq::PAIR, zmq::PAIR)), + socket_name_options, + ::testing::Values(false) // don't expect throw + )); + +INSTANTIATE_TEST_SUITE_P(REQREP, AsyncSocketTypes, + ::testing::Combine(::testing::Values(0), // no repeats, we only care about the throw + ::testing::Values(std::make_pair(zmq::REQ, zmq::REP)), + socket_name_options, + ::testing::Values(true) // expect throw + )); diff --git a/roofit/roofitZMQ/test/test_ZMQ_load_balancing.cxx b/roofit/roofitZMQ/test/test_ZMQ_load_balancing.cxx new file mode 100644 index 0000000000000..ebb4680357104 --- /dev/null +++ b/roofit/roofitZMQ/test/test_ZMQ_load_balancing.cxx @@ -0,0 +1,133 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2016-2021, Netherlands eScience Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "gtest/gtest.h" + +#include // fork, usleep + +#include "RooFit_ZMQ/ZeroMQSvc.h" +#include "RooFit_ZMQ/ZeroMQPoller.h" + +class ZMQPushPullTest : public ::testing::Test { +protected: + std::size_t N_children = 4; + std::size_t max_sends = 20; + + void SetUp() override { + for (std::size_t i = 0; i < N_children; ++i) { + do { + child_pid = fork(); + } while (child_pid == -1); // retry if fork fails + if (child_pid == 0) { // child + child_id = i; + break; + } else { + child_pids.push_back(child_pid); + } + } + + if (child_pid > 0) { // parent + pusher.reset(zmqSvc().socket_ptr(zmq::PUSH)); + pusher->bind("ipc:///tmp/ZMQ_test_push_pull_P2C.ipc"); + } else { // child + puller.reset(zmqSvc().socket_ptr(zmq::PULL)); + puller->connect("ipc:///tmp/ZMQ_test_push_pull_P2C.ipc"); + + poller.register_socket(*puller, zmq::POLLIN); + } + } + + void TearDown() override { + if (child_pid > 0) { // parent + // wait for children + int status = 0; + pid_t pid; + for (pid_t child_pid_i : child_pids) { + do { + pid = waitpid(child_pid_i, &status, 0); + } while (-1 == pid && EINTR == errno); // retry on interrupted system call + if (0 != status) { + if (WIFEXITED(status)) { printf("exited, status=%d\n", WEXITSTATUS(status)); } else if (WIFSIGNALED(status)) { printf("killed by signal %d\n", WTERMSIG(status)); } else if (WIFSTOPPED(status)) { printf("stopped by signal %d\n", WSTOPSIG(status)); } else if (WIFCONTINUED(status)) { printf("continued\n"); } + } + if (-1 == pid) { + throw std::runtime_error(std::string("waitpid, errno ") + std::to_string(errno)); + } + } + pusher.reset(nullptr); + zmqSvc().close_context(); + } else { // child + puller.reset(nullptr); + zmqSvc().close_context(); + } + } + + void run_parent() { + // start test + usleep(1000); // wait a second so that all pull sockets are connected for round-robin distribution + // if you don't wait a second above, the push socket will "round-robin" all the messages to just one or two connected sockets + for (std::size_t ix = 0; ix < max_sends; ++ix) { + zmqSvc().send(*pusher, 0); + } + for (std::size_t ix = 0; ix < N_children; ++ix) { + // end by sending some 1's to all children, to let them know the sending is over + zmqSvc().send(*pusher, 1); + } + } + + void run_child() { + std::size_t count = 0; + for (std::size_t ix = 0; ix < max_sends; ++ix) { + auto r = poller.poll(2000); + if (r.empty()) { + printf("poller of child %d timed out after 2 seconds\n", child_id); + break; + } + auto value = zmqSvc().receive(*puller, ZMQ_DONTWAIT); + usleep(200); // "do some work" + printf("value on child %d: %d\n", child_id, value); + if (value == 1) { + printf("child %d got value %d, done here\n", child_id, value); + break; + } + ++count; + } + printf("child %d got %lu values\n", child_id, count); + } + + pid_t child_pid {0}; + int child_id = -1; + std::vector child_pids; + ZmqLingeringSocketPtr<> pusher, puller; + ZeroMQPoller poller; +}; + + +/// This test shows how push-pull is unsuited for load balancing; messages are just sent to the first available pull socket without any dynamic load balancing +TEST_F(ZMQPushPullTest, demoRoundRobin) { + if (child_pid > 0) { + run_parent(); + } else { + run_child(); + } +} + +/// This test tries to see whether push-pull can be made to work as a bit of a load balancer, using a low HWM at the receiver +TEST_F(ZMQPushPullTest, demoHWM1LoadBalancing) { + if (child_pid > 0) { + run_parent(); + } else { + puller->setsockopt(ZMQ_RCVHWM, 1); + run_child(); + } +} diff --git a/roofit/roofitZMQ/test/test_polling.cxx b/roofit/roofitZMQ/test/test_polling.cxx new file mode 100644 index 0000000000000..9250b2ee6018b --- /dev/null +++ b/roofit/roofitZMQ/test/test_polling.cxx @@ -0,0 +1,149 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2016-2019, Netherlands eScience Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ + + +#include "gtest/gtest.h" + +#include // fork, usleep +#include // signal blocking +#include // strsignal() + +#include + +#include "RooFit_ZMQ/ZeroMQSvc.h" +#include "RooFit_ZMQ/ZeroMQPoller.h" + + +static bool terminated = false; + +void handle_sigterm(int signum) { + terminated = true; + std::cout << "handled signal " << strsignal(signum) << " on PID " << getpid() << std::endl; +} + + +TEST(Polling, doublePoll) { + pid_t child_pid {0}; + do { + child_pid = fork(); + } while (child_pid == -1); // retry if fork fails + + if (child_pid > 0) { // master + sigset_t sigmask, sigmask_old; + sigemptyset(&sigmask); + sigaddset(&sigmask, SIGCHLD); + sigprocmask(SIG_BLOCK, &sigmask, &sigmask_old); + +// std::cout << "master PID: " << getpid() << std::endl; + ZmqLingeringSocketPtr<> pusher, puller; + pusher.reset(zmqSvc().socket_ptr(zmq::PUSH)); + pusher->bind("ipc:///tmp/ZMQ_test_fork_polling_M2C.ipc"); + puller.reset(zmqSvc().socket_ptr(zmq::PULL)); + puller->bind("ipc:///tmp/ZMQ_test_fork_polling_C2M.ipc"); + + ZeroMQPoller poller1, poller2; + poller1.register_socket(*puller, zmq::POLLIN); + poller2.register_socket(*puller, zmq::POLLIN); + + // start test + zmqSvc().send(*pusher, std::string("breaker breaker")); + + auto result1a = poller1.poll(-1); + auto result1b = poller1.poll(-1); + auto result2 = poller2.poll(-1); + EXPECT_EQ(result1a.size(), result1b.size()); + EXPECT_EQ(result1a.size(), result2.size()); + + auto receipt = zmqSvc().receive(*puller, ZMQ_DONTWAIT); + + EXPECT_EQ(receipt, 1212); + + kill(child_pid, SIGTERM); + + // socket->close(); // this gives exception of type zmq::error_t: Socket operation on non-socket + pusher.reset(nullptr); + puller.reset(nullptr); + zmqSvc().close_context(); // if you don't close context in parent process as well, the next repeat will hang + + // wait for child + int status = 0; + pid_t pid; + do { + pid = waitpid(child_pid, &status, 0); + } while (-1 == pid && EINTR == errno); // retry on interrupted system call + + if (0 != status) { + if (WIFEXITED(status)) { printf("exited, status=%d\n", WEXITSTATUS(status)); } else if (WIFSIGNALED(status)) { printf("killed by signal %d\n", WTERMSIG(status)); } else if (WIFSTOPPED(status)) { printf("stopped by signal %d\n", WSTOPSIG(status)); } else if (WIFCONTINUED(status)) { printf("continued\n"); } + } + + if (-1 == pid) { + throw std::runtime_error(std::string("waitpid, errno ") + std::to_string(errno)); + } + + sigprocmask(SIG_SETMASK, &sigmask_old, nullptr); + } else { // child +// std::cout << "child PID: " << getpid() << std::endl; + + sigset_t sigmask, sigmask_old; + sigemptyset(&sigmask); + sigaddset(&sigmask, SIGTERM); + sigprocmask(SIG_BLOCK, &sigmask, &sigmask_old); + + struct sigaction sa; + memset (&sa, '\0', sizeof(sa)); + sa.sa_handler = handle_sigterm; + + if (sigaction(SIGTERM, &sa, NULL) < 0) { + std::perror("sigaction failed"); + std::exit(1); + } + + ZmqLingeringSocketPtr<> puller, pusher; + puller.reset(zmqSvc().socket_ptr(zmq::PULL)); + puller->connect("ipc:///tmp/ZMQ_test_fork_polling_M2C.ipc"); + pusher.reset(zmqSvc().socket_ptr(zmq::PUSH)); + pusher->connect("ipc:///tmp/ZMQ_test_fork_polling_C2M.ipc"); + + ZeroMQPoller poller1, poller2; + poller1.register_socket(*puller, zmq::POLLIN); + poller2.register_socket(*puller, zmq::POLLIN); + + // start test + auto result1a = poller1.poll(-1); + auto result1b = poller1.poll(-1); + auto result2 = poller2.poll(-1); + EXPECT_EQ(result1a.size(), result1b.size()); + EXPECT_EQ(result1a.size(), result2.size()); + + auto receipt = zmqSvc().receive(*puller, ZMQ_DONTWAIT); + if (receipt == "breaker breaker") { + zmqSvc().send(*pusher, 1212); + } + // take care, don't just use _exit, it will not cleanly destroy context etc! + // if you really need to, at least close and destroy everything properly + // socket->close(); // this gives exception of type zmq::error_t: Socket operation on non-socket + + sigprocmask(SIG_SETMASK, &sigmask_old, nullptr); + + while (!terminated) {} + +// std::cout << "child terminated" << std::endl; + + puller.reset(nullptr); + pusher.reset(nullptr); + zmqSvc().close_context(); + _Exit(0); + } + +} From fcb329f06dd5b7858ebee9050684a40b847124b4 Mon Sep 17 00:00:00 2001 From: "E. G. Patrick Bos" Date: Thu, 10 Jun 2021 16:54:54 +0200 Subject: [PATCH 6/9] add ppoll to RooFitZMQ This is a reimplementation of ZMQ's poll function that mimics the POSIX ppoll function. The difference with regular poll is that ppoll blocks incoming POSIX signals, allowing graceful handling of these signals. We use this in RooFit::MultiProcess to shut down forked child processes. The motivation behind this addition is discussed more in this blog post: https://blog.esciencecenter.nl/combining-zeromq-posix-signals-b754f6f29cd6, or more succinctly in Martin Sustrik's blog post, in the edit at the bottom: https://250bpm.com/blog:12/ --- roofit/roofitZMQ/CMakeLists.txt | 6 +- roofit/roofitZMQ/inc/RooFit_ZMQ/ppoll.h | 29 ++ roofit/roofitZMQ/src/ppoll.cpp | 345 ++++++++++++++++++++++++ 3 files changed, 376 insertions(+), 4 deletions(-) create mode 100644 roofit/roofitZMQ/inc/RooFit_ZMQ/ppoll.h create mode 100644 roofit/roofitZMQ/src/ppoll.cpp diff --git a/roofit/roofitZMQ/CMakeLists.txt b/roofit/roofitZMQ/CMakeLists.txt index 8174ecb952418..81fee8d871a92 100644 --- a/roofit/roofitZMQ/CMakeLists.txt +++ b/roofit/roofitZMQ/CMakeLists.txt @@ -13,19 +13,17 @@ ROOT_STANDARD_LIBRARY_PACKAGE(RooFitZMQ RooFit_ZMQ/ZeroMQSvc.h RooFit_ZMQ/functions.h RooFit_ZMQ/ZeroMQPoller.h -# RooFit_ZMQ/ppoll.h + RooFit_ZMQ/ppoll.h SOURCES src/ZeroMQSvc.cpp src/ZeroMQPoller.cpp src/functions.cpp -# src/ppoll.cpp + src/ppoll.cpp DICTIONARY_OPTIONS -# -I${ZeroMQ_INCLUDE_DIR} -writeEmptyRootPCM DEPENDENCIES Core # TVersionCheck, RegisterModule, these are always called from G__RooFitZMQ.cxx.o LIBRARIES -# ${ZeroMQ_LIBRARY} ZeroMQ ) diff --git a/roofit/roofitZMQ/inc/RooFit_ZMQ/ppoll.h b/roofit/roofitZMQ/inc/RooFit_ZMQ/ppoll.h new file mode 100644 index 0000000000000..c00a1ccfebd7f --- /dev/null +++ b/roofit/roofitZMQ/inc/RooFit_ZMQ/ppoll.h @@ -0,0 +1,29 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2016-2019, Netherlands eScience Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ +#ifndef ROOT_ROOFIT_ZMQ_ppoll +#define ROOT_ROOFIT_ZMQ_ppoll + +#include +#include + +namespace ZMQ { + +int zmq_ppoll (zmq_pollitem_t *items_, int nitems_, long timeout_, const sigset_t * sigmask_); +int ppoll(zmq_pollitem_t *items_, size_t nitems_, long timeout_, const sigset_t * sigmask_); +int ppoll(std::vector &items, long timeout_, const sigset_t * sigmask_); +class ppoll_error_t : public zmq::error_t {}; + +} + +#endif // ROOT_ROOFIT_ZMQ_ppoll diff --git a/roofit/roofitZMQ/src/ppoll.cpp b/roofit/roofitZMQ/src/ppoll.cpp new file mode 100644 index 0000000000000..5cfcda16222e1 --- /dev/null +++ b/roofit/roofitZMQ/src/ppoll.cpp @@ -0,0 +1,345 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2016-2019, Netherlands eScience Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include // pselect +#include // timespec +#include + +#include "likely.hpp" // unlikely +#include "clock.hpp" // zmq::clock_t +#include "err.hpp" // zmq_assert +#include "polling_util.hpp" // zmq::optimized_fd_set_t, zmq::compute_timeout, zmq::timeout_t +#include "fd.hpp" // zmq::fd_t + +#include + + +namespace ZMQ { + +int zmq_ppoll (zmq_pollitem_t *items_, int nitems_, long timeout_, + const sigset_t * sigmask_) +{ +#if defined ZMQ_HAVE_POLLER + // if poller is present, use that if there is at least 1 thread-safe socket, + // otherwise fall back to the previous implementation as it's faster. + for (int i = 0; i != nitems_; i++) { + if (items_[i].socket + && as_socket_base_t (items_[i].socket)->is_thread_safe ()) { + return zmq_poller_poll (items_, nitems_, timeout_); + } + } +#endif // ZMQ_HAVE_POLLER +#if defined ZMQ_POLL_BASED_ON_POLL || defined ZMQ_POLL_BASED_ON_SELECT + if (unlikely (nitems_ < 0)) { + errno = EINVAL; + return -1; + } + if (unlikely (nitems_ == 0)) { + if (timeout_ == 0) + return 0; +#if defined ZMQ_HAVE_WINDOWS + Sleep (timeout_ > 0 ? timeout_ : INFINITE); + return 0; +#elif defined ZMQ_HAVE_VXWORKS + struct timespec ns_; + ns_.tv_sec = timeout_ / 1000; + ns_.tv_nsec = timeout_ % 1000 * 1000000; + return nanosleep (&ns_, 0); +#else + return usleep (timeout_ * 1000); +#endif + } + if (!items_) { + errno = EFAULT; + return -1; + } + + zmq::clock_t clock; + uint64_t now = 0; + uint64_t end = 0; +#if defined ZMQ_POLL_BASED_ON_POLL + zmq::fast_vector_t pollfds (nitems_); + + // Build pollset for poll () system call. + for (int i = 0; i != nitems_; i++) { + // If the poll item is a 0MQ socket, we poll on the file descriptor + // retrieved by the ZMQ_FD socket option. + if (items_[i].socket) { + size_t zmq_fd_size = sizeof (zmq::fd_t); + if (zmq_getsockopt (items_[i].socket, ZMQ_FD, &pollfds[i].fd, + &zmq_fd_size) + == -1) { + return -1; + } + pollfds[i].events = items_[i].events ? POLLIN : 0; + } + // Else, the poll item is a raw file descriptor. Just convert the + // events to normal POLLIN/POLLOUT for poll (). + else { + pollfds[i].fd = items_[i].fd; + pollfds[i].events = + (items_[i].events & ZMQ_POLLIN ? POLLIN : 0) + | (items_[i].events & ZMQ_POLLOUT ? POLLOUT : 0) + | (items_[i].events & ZMQ_POLLPRI ? POLLPRI : 0); + } + } +#else + // Ensure we do not attempt to select () on more than FD_SETSIZE + // file descriptors. + // TODO since this function is called by a client, we could return errno EINVAL/ENOMEM/... here + zmq_assert (nitems_ <= FD_SETSIZE); + + zmq::optimized_fd_set_t pollset_in (nitems_); + FD_ZERO (pollset_in.get ()); + zmq::optimized_fd_set_t pollset_out (nitems_); + FD_ZERO (pollset_out.get ()); + zmq::optimized_fd_set_t pollset_err (nitems_); + FD_ZERO (pollset_err.get ()); + + zmq::fd_t maxfd = 0; + + // Build the fd_sets for passing to select (). + for (int i = 0; i != nitems_; i++) { + // If the poll item is a 0MQ socket we are interested in input on the + // notification file descriptor retrieved by the ZMQ_FD socket option. + if (items_[i].socket) { + size_t zmq_fd_size = sizeof (zmq::fd_t); + zmq::fd_t notify_fd; + if (zmq_getsockopt (items_[i].socket, ZMQ_FD, ¬ify_fd, + &zmq_fd_size) + == -1) + return -1; + if (items_[i].events) { + FD_SET (notify_fd, pollset_in.get ()); + if (maxfd < notify_fd) + maxfd = notify_fd; + } + } + // Else, the poll item is a raw file descriptor. Convert the poll item + // events to the appropriate fd_sets. + else { + if (items_[i].events & ZMQ_POLLIN) + FD_SET (items_[i].fd, pollset_in.get ()); + if (items_[i].events & ZMQ_POLLOUT) + FD_SET (items_[i].fd, pollset_out.get ()); + if (items_[i].events & ZMQ_POLLERR) + FD_SET (items_[i].fd, pollset_err.get ()); + if (maxfd < items_[i].fd) + maxfd = items_[i].fd; + } + } + + zmq::optimized_fd_set_t inset (nitems_); + zmq::optimized_fd_set_t outset (nitems_); + zmq::optimized_fd_set_t errset (nitems_); +#endif + + bool first_pass = true; + int nevents = 0; + + while (true) { +#if defined ZMQ_POLL_BASED_ON_POLL + + // Compute the timeout for the subsequent poll. + zmq::timeout_t timeout = + zmq::compute_timeout (first_pass, timeout_, now, end); + + // Wait for events. + { + int rc = ppoll (&pollfds[0], nitems_, timeout); + if (rc == -1 && errno == EINTR) { + return -1; + } + errno_assert (rc >= 0); + } + // Check for the events. + for (int i = 0; i != nitems_; i++) { + items_[i].revents = 0; + + // The poll item is a 0MQ socket. Retrieve pending events + // using the ZMQ_EVENTS socket option. + if (items_[i].socket) { + size_t zmq_events_size = sizeof (uint32_t); + uint32_t zmq_events; + if (zmq_getsockopt (items_[i].socket, ZMQ_EVENTS, &zmq_events, + &zmq_events_size) + == -1) { + return -1; + } + if ((items_[i].events & ZMQ_POLLOUT) + && (zmq_events & ZMQ_POLLOUT)) + items_[i].revents |= ZMQ_POLLOUT; + if ((items_[i].events & ZMQ_POLLIN) + && (zmq_events & ZMQ_POLLIN)) + items_[i].revents |= ZMQ_POLLIN; + } + // Else, the poll item is a raw file descriptor, simply convert + // the events to zmq_pollitem_t-style format. + else { + if (pollfds[i].revents & POLLIN) + items_[i].revents |= ZMQ_POLLIN; + if (pollfds[i].revents & POLLOUT) + items_[i].revents |= ZMQ_POLLOUT; + if (pollfds[i].revents & POLLPRI) + items_[i].revents |= ZMQ_POLLPRI; + if (pollfds[i].revents & ~(POLLIN | POLLOUT | POLLPRI)) + items_[i].revents |= ZMQ_POLLERR; + } + + if (items_[i].revents) + nevents++; + } + +#else + + // Compute the timeout for the subsequent poll. + struct timespec timeout; + struct timespec *ptimeout; + if (first_pass) { + timeout.tv_sec = 0; + timeout.tv_nsec = 0; + ptimeout = &timeout; + } else if (timeout_ < 0) + ptimeout = NULL; + else { + timeout.tv_sec = static_cast ((end - now) / 1000); + timeout.tv_nsec = static_cast ((end - now) % 1000 * 1000000); + ptimeout = &timeout; + } + + // Wait for events. Ignore interrupts if there's infinite timeout. + while (true) { + memcpy (inset.get (), pollset_in.get (), + zmq::valid_pollset_bytes (*pollset_in.get ())); + memcpy (outset.get (), pollset_out.get (), + zmq::valid_pollset_bytes (*pollset_out.get ())); + memcpy (errset.get (), pollset_err.get (), + zmq::valid_pollset_bytes (*pollset_err.get ())); +#if defined ZMQ_HAVE_WINDOWS + int rc = + select (0, inset.get (), outset.get (), errset.get (), ptimeout); + if (unlikely (rc == SOCKET_ERROR)) { + errno = zmq::wsa_error_to_errno (WSAGetLastError ()); + wsa_assert (errno == ENOTSOCK); + return -1; + } +#else + int rc = pselect (maxfd + 1, inset.get (), outset.get (), + errset.get (), ptimeout, sigmask_); + if (unlikely (rc == -1)) { + errno_assert (errno == EINTR || errno == EBADF); + return -1; + } +#endif + break; + } + + // Check for the events. + for (int i = 0; i != nitems_; i++) { + items_[i].revents = 0; + + // The poll item is a 0MQ socket. Retrieve pending events + // using the ZMQ_EVENTS socket option. + if (items_[i].socket) { + size_t zmq_events_size = sizeof (uint32_t); + uint32_t zmq_events; + if (zmq_getsockopt (items_[i].socket, ZMQ_EVENTS, &zmq_events, + &zmq_events_size) + == -1) + return -1; + if ((items_[i].events & ZMQ_POLLOUT) + && (zmq_events & ZMQ_POLLOUT)) + items_[i].revents |= ZMQ_POLLOUT; + if ((items_[i].events & ZMQ_POLLIN) + && (zmq_events & ZMQ_POLLIN)) + items_[i].revents |= ZMQ_POLLIN; + } + // Else, the poll item is a raw file descriptor, simply convert + // the events to zmq_pollitem_t-style format. + else { + if (FD_ISSET (items_[i].fd, inset.get ())) + items_[i].revents |= ZMQ_POLLIN; + if (FD_ISSET (items_[i].fd, outset.get ())) + items_[i].revents |= ZMQ_POLLOUT; + if (FD_ISSET (items_[i].fd, errset.get ())) + items_[i].revents |= ZMQ_POLLERR; + } + + if (items_[i].revents) + nevents++; + } +#endif + + // If timeout is zero, exit immediately whether there are events or not. + if (timeout_ == 0) + break; + + // If there are events to return, we can exit immediately. + if (nevents) + break; + + // At this point we are meant to wait for events but there are none. + // If timeout is infinite we can just loop until we get some events. + if (timeout_ < 0) { + if (first_pass) + first_pass = false; + continue; + } + + // The timeout is finite and there are no events. In the first pass + // we get a timestamp of when the polling have begun. (We assume that + // first pass have taken negligible time). We also compute the time + // when the polling should time out. + if (first_pass) { + now = clock.now_ms (); + end = now + timeout_; + if (now == end) + break; + first_pass = false; + continue; + } + + // Find out whether timeout have expired. + now = clock.now_ms (); + if (now >= end) + break; + } + + return nevents; +#else + // Exotic platforms that support neither poll() nor select(). + errno = ENOTSUP; + return -1; +#endif +} + + +// This function can throw, so wrap in try-catch! +int ppoll(zmq_pollitem_t *items_, size_t nitems_, long timeout_, const sigset_t * sigmask_) +{ + int rc = zmq_ppoll(items_, static_cast(nitems_), timeout_, sigmask_); + if (rc < 0) + throw ppoll_error_t(); + return rc; +} + + +// This function can throw, so wrap in try-catch! +int ppoll(std::vector &items, long timeout_, const sigset_t * sigmask_) +{ + return ppoll(items.data(), items.size(), timeout_, sigmask_); +} + +} + From 81b3ad64556edfc2c1b5880070edee21af47af27 Mon Sep 17 00:00:00 2001 From: "E. G. Patrick Bos" Date: Thu, 10 Jun 2021 20:34:43 +0200 Subject: [PATCH 7/9] move ZeroMQ builtin to builtins directory and add build option The option builtin_zeromq is set to ON by default and controls whether RooFitZMQ is built. At some point, we should also add the option to use a system installed ZeroMQ, so that people can avoid the built-in, if they wish, e.g. for the conda-forge build. --- {roofit/builtin_zeromq => builtins/zeromq}/CMakeLists.txt | 2 ++ cmake/modules/RootBuildOptions.cmake | 2 ++ cmake/modules/SearchInstalledSoftware.cmake | 7 +++++++ roofit/CMakeLists.txt | 5 +++-- 4 files changed, 14 insertions(+), 2 deletions(-) rename {roofit/builtin_zeromq => builtins/zeromq}/CMakeLists.txt (96%) diff --git a/roofit/builtin_zeromq/CMakeLists.txt b/builtins/zeromq/CMakeLists.txt similarity index 96% rename from roofit/builtin_zeromq/CMakeLists.txt rename to builtins/zeromq/CMakeLists.txt index 9b769f3d00e95..5ec3b60f59a22 100644 --- a/roofit/builtin_zeromq/CMakeLists.txt +++ b/builtins/zeromq/CMakeLists.txt @@ -31,6 +31,7 @@ target_link_libraries(ZeroMQ INTERFACE $) add_dependencies(ZeroMQ BUILTIN_ZeroMQ) add_library(ZeroMQ::ZeroMQ ALIAS ZeroMQ) +add_library(libzmq ALIAS ZeroMQ) set(ZeroMQ_DIR ${ZeroMQ_PREFIX}/share/cmake/ZeroMQ CACHE INTERNAL "" FORCE) @@ -43,4 +44,5 @@ ExternalProject_Add(BUILTIN_cppzmq CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${install_dir} -DCPPZMQ_BUILD_TESTS=OFF + DEPENDS BUILTIN_ZeroMQ ) diff --git a/cmake/modules/RootBuildOptions.cmake b/cmake/modules/RootBuildOptions.cmake index 1cc9f48926367..cd157236e552e 100644 --- a/cmake/modules/RootBuildOptions.cmake +++ b/cmake/modules/RootBuildOptions.cmake @@ -107,6 +107,7 @@ ROOT_BUILD_OPTION(builtin_vdt OFF "Build VDT internally (requires network)") ROOT_BUILD_OPTION(builtin_veccore OFF "Build VecCore internally (requires network)") ROOT_BUILD_OPTION(builtin_xrootd OFF "Build XRootD internally (requires network)") ROOT_BUILD_OPTION(builtin_xxhash OFF "Build bundled copy of xxHash") +ROOT_BUILD_OPTION(builtin_zeromq ON "Build ZeroMQ internally (requires network)") ROOT_BUILD_OPTION(builtin_zlib OFF "Build bundled copy of zlib") ROOT_BUILD_OPTION(builtin_zstd OFF "Build included libzstd, or use system libzstd") ROOT_BUILD_OPTION(ccache OFF "Enable ccache usage for speeding up builds") @@ -299,6 +300,7 @@ if(builtin_all) set(builtin_veccore_defvalue ON) set(builtin_xrootd_defvalue ON) set(builtin_xxhash_defvalue ON) + set(builtin_zeromq_defvalue ON) set(builtin_zlib_defvalue ON) set(builtin_zstd_defvalue ON) endif() diff --git a/cmake/modules/SearchInstalledSoftware.cmake b/cmake/modules/SearchInstalledSoftware.cmake index c64ab05414e21..0624a00fbb3a3 100644 --- a/cmake/modules/SearchInstalledSoftware.cmake +++ b/cmake/modules/SearchInstalledSoftware.cmake @@ -1767,6 +1767,13 @@ if (mpi) endif() endif() +#---Check for ZeroMQ----------------------------------------------------------- + +if(builtin_zeromq) + list(APPEND ROOT_BUILTINS ZeroMQ) + add_subdirectory(builtins/zeromq) +endif() + #---Download googletest-------------------------------------------------------------- if (testing) # FIXME: Remove our version of gtest in roottest. We can reuse this one. diff --git a/roofit/CMakeLists.txt b/roofit/CMakeLists.txt index 886761c71ee09..cec391f22a510 100644 --- a/roofit/CMakeLists.txt +++ b/roofit/CMakeLists.txt @@ -5,8 +5,9 @@ # For the list of contributors see $ROOTSYS/README/CREDITS. add_subdirectory(batchcompute) -add_subdirectory(builtin_zeromq) -add_subdirectory(roofitZMQ) +if (builtin_zeromq) + add_subdirectory(roofitZMQ) +endif(builtin_zeromq) add_subdirectory(roofitcore) add_subdirectory(roofit) if(mathmore) From 2bddfa40876fbe257d2db621c870e98e3c86bb0f Mon Sep 17 00:00:00 2001 From: "E. G. Patrick Bos" Date: Fri, 11 Jun 2021 11:43:14 +0200 Subject: [PATCH 8/9] add ZeroMQPoller::ppoll implementation Should have been added in ppoll commit before. --- roofit/roofitZMQ/src/ZeroMQPoller.cpp | 31 +++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/roofit/roofitZMQ/src/ZeroMQPoller.cpp b/roofit/roofitZMQ/src/ZeroMQPoller.cpp index 12eafaf09392d..29a0dd7d610c7 100644 --- a/roofit/roofitZMQ/src/ZeroMQPoller.cpp +++ b/roofit/roofitZMQ/src/ZeroMQPoller.cpp @@ -8,6 +8,7 @@ #include +#include "RooFit_ZMQ/ppoll.h" #include "RooFit_ZMQ/ZeroMQPoller.h" std::vector> ZeroMQPoller::poll(int timeo) { @@ -48,6 +49,36 @@ std::vector> ZeroMQPoller::poll(int timeo) { return r; } +// This function can throw (from inside ZMQ::ppoll), so wrap in try-catch! +std::vector> ZeroMQPoller::ppoll(int timeo, const sigset_t * sigmask_) { + if (m_items.empty()) { + throw std::runtime_error("No sockets registered"); + } + + std::vector> r; + + auto n = ZMQ::ppoll(m_items, timeo, sigmask_); + if (n == 0) return r; + + for (auto& m_item : m_items) { + size_t index = 0; + int flags = 0; + if (m_item.socket == nullptr) { + // an fd was registered + std::tie(index, flags) = m_fds[m_item.fd]; + } else { + // a socket was registered + const zmq::socket_t* s; + std::tie(index, flags, s) = m_sockets[m_item.socket]; + } + if (m_item.revents & short(flags)) { + r.emplace_back(index, flags); + } + } + return r; +} + + size_t ZeroMQPoller::size() const { return m_items.size(); } From fc0ac88c7e2bb74304a90fa9e02cecd62a023f47 Mon Sep 17 00:00:00 2001 From: "E. G. Patrick Bos" Date: Fri, 11 Jun 2021 12:16:52 +0200 Subject: [PATCH 9/9] add RooFit::MultiProcess RooFit::MultiProcess is a task-based parallelization framework that will be used to parallelize gradients in RooFit/Minuit2 fits. It uses forked processes for parallelization, as opposed to threads. We chose this approach because A) the existing RooRealMPFE parallelization framework already made use of forks as well, so we had something to build on and B) it was at the time deemed infeasible to check the entire RooFit code for thread-safety. Moreover, we use MultiProcess to parallelize gradients -- i.e. the tasks to be executed in parallel are partial derivatives -- and these are sufficiently large tasks that communication in between tasks is not a big concern in the big fits that we aimed to parallelize. The communication between processes is done using ZeroMQ, accessed through convenience functions in RooFitZMQ. Note: test_Job still fails, it has to be updated to the latest communication framework. --- roofit/CMakeLists.txt | 1 + roofit/multiprocess/CMakeLists.txt | 40 ++ roofit/multiprocess/inc/LinkDef.h | 1 + .../inc/RooFit/MultiProcess/Job.h | 130 +++++ .../inc/RooFit/MultiProcess/JobManager.h | 91 ++++ .../inc/RooFit/MultiProcess/Messenger.h | 208 ++++++++ .../inc/RooFit/MultiProcess/Messenger_decl.h | 185 +++++++ .../inc/RooFit/MultiProcess/ProcessManager.h | 78 +++ .../inc/RooFit/MultiProcess/Queue.h | 46 ++ .../inc/RooFit/MultiProcess/types.h | 29 ++ .../inc/RooFit/MultiProcess/util.h | 34 ++ .../inc/RooFit/MultiProcess/worker.h | 27 + roofit/multiprocess/src/Job.cxx | 65 +++ roofit/multiprocess/src/JobManager.cxx | 264 ++++++++++ roofit/multiprocess/src/Messenger.cxx | 481 ++++++++++++++++++ roofit/multiprocess/src/ProcessManager.cxx | 286 +++++++++++ roofit/multiprocess/src/Queue.cxx | 224 ++++++++ roofit/multiprocess/src/util.cxx | 130 +++++ roofit/multiprocess/src/worker.cxx | 228 +++++++++ roofit/multiprocess/test/CMakeLists.txt | 13 + roofit/multiprocess/test/test_Job.cxx | 252 +++++++++ roofit/multiprocess/test/test_Messenger.cxx | 116 +++++ .../multiprocess/test/test_ProcessManager.cxx | 57 +++ roofit/multiprocess/test/test_worker.cxx | 29 ++ roofit/multiprocess/test/utils.h | 171 +++++++ 25 files changed, 3186 insertions(+) create mode 100644 roofit/multiprocess/CMakeLists.txt create mode 100644 roofit/multiprocess/inc/LinkDef.h create mode 100644 roofit/multiprocess/inc/RooFit/MultiProcess/Job.h create mode 100644 roofit/multiprocess/inc/RooFit/MultiProcess/JobManager.h create mode 100644 roofit/multiprocess/inc/RooFit/MultiProcess/Messenger.h create mode 100644 roofit/multiprocess/inc/RooFit/MultiProcess/Messenger_decl.h create mode 100644 roofit/multiprocess/inc/RooFit/MultiProcess/ProcessManager.h create mode 100644 roofit/multiprocess/inc/RooFit/MultiProcess/Queue.h create mode 100644 roofit/multiprocess/inc/RooFit/MultiProcess/types.h create mode 100644 roofit/multiprocess/inc/RooFit/MultiProcess/util.h create mode 100644 roofit/multiprocess/inc/RooFit/MultiProcess/worker.h create mode 100644 roofit/multiprocess/src/Job.cxx create mode 100644 roofit/multiprocess/src/JobManager.cxx create mode 100644 roofit/multiprocess/src/Messenger.cxx create mode 100644 roofit/multiprocess/src/ProcessManager.cxx create mode 100644 roofit/multiprocess/src/Queue.cxx create mode 100644 roofit/multiprocess/src/util.cxx create mode 100644 roofit/multiprocess/src/worker.cxx create mode 100644 roofit/multiprocess/test/CMakeLists.txt create mode 100644 roofit/multiprocess/test/test_Job.cxx create mode 100644 roofit/multiprocess/test/test_Messenger.cxx create mode 100644 roofit/multiprocess/test/test_ProcessManager.cxx create mode 100644 roofit/multiprocess/test/test_worker.cxx create mode 100644 roofit/multiprocess/test/utils.h diff --git a/roofit/CMakeLists.txt b/roofit/CMakeLists.txt index cec391f22a510..6715e07edd8a9 100644 --- a/roofit/CMakeLists.txt +++ b/roofit/CMakeLists.txt @@ -7,6 +7,7 @@ add_subdirectory(batchcompute) if (builtin_zeromq) add_subdirectory(roofitZMQ) + add_subdirectory(multiprocess) endif(builtin_zeromq) add_subdirectory(roofitcore) add_subdirectory(roofit) diff --git a/roofit/multiprocess/CMakeLists.txt b/roofit/multiprocess/CMakeLists.txt new file mode 100644 index 0000000000000..702e4e611f45e --- /dev/null +++ b/roofit/multiprocess/CMakeLists.txt @@ -0,0 +1,40 @@ +############################################################################ +# CMakeLists.txt file for building ROOT RooFitMultiProcess package +# @author Patrick Bos, Netherlands eScience Center +############################################################################ + +#FIND_PACKAGE(ZeroMQ REQUIRED) + +ROOT_STANDARD_LIBRARY_PACKAGE(RooFitMultiProcess + HEADERS + RooFit/MultiProcess/worker.h + RooFit/MultiProcess/Messenger_decl.h + RooFit/MultiProcess/Messenger.h + RooFit/MultiProcess/ProcessManager.h + RooFit/MultiProcess/JobManager.h + RooFit/MultiProcess/util.h + RooFit/MultiProcess/Queue.h + RooFit/MultiProcess/types.h + RooFit/MultiProcess/Job.h + SOURCES + src/worker.cxx + src/Messenger.cxx + src/ProcessManager.cxx + src/util.cxx + src/Queue.cxx + src/JobManager.cxx + src/Job.cxx + DICTIONARY_OPTIONS + "-writeEmptyRootPCM" + DEPENDENCIES + RooFitZMQ + LIBRARIES + ZeroMQ +) + +target_link_libraries(RooFitMultiProcess PRIVATE ZeroMQ) +target_include_directories(RooFitMultiProcess PUBLIC $) + +if(testing) + ROOT_ADD_TEST_SUBDIRECTORY(test) +endif() diff --git a/roofit/multiprocess/inc/LinkDef.h b/roofit/multiprocess/inc/LinkDef.h new file mode 100644 index 0000000000000..6a9ddbee0696a --- /dev/null +++ b/roofit/multiprocess/inc/LinkDef.h @@ -0,0 +1 @@ +// ceci n'est pas un fichier \ No newline at end of file diff --git a/roofit/multiprocess/inc/RooFit/MultiProcess/Job.h b/roofit/multiprocess/inc/RooFit/MultiProcess/Job.h new file mode 100644 index 0000000000000..423071fabfeb1 --- /dev/null +++ b/roofit/multiprocess/inc/RooFit/MultiProcess/Job.h @@ -0,0 +1,130 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2016-2019, Netherlands eScience Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ +#ifndef ROOT_ROOFIT_MultiProcess_Job_decl +#define ROOT_ROOFIT_MultiProcess_Job_decl + +#include +#include + +namespace RooFit { +namespace MultiProcess { + +// forward declaration +class JobManager; + +/* + * @brief interface class for defining the actual work that must be done + * + * Think of "job" as in "employment", e.g. the job of a baker, which + * involves *tasks* like baking and selling bread. The Job must define the + * tasks through its execution (evaluate_task), based on a task index argument. + * + * Classes inheriting from Job must implement the pure virtual methods: + * - void evaluate_task(std::size_t task) + * - void update_real(std::size_t ix, double val, bool is_constant) + * - void update_bool(std::size_t ix, bool value) + * - void receive_task_result_on_queue(std::size_t task, std::size_t worker_id) + * - void send_back_results_from_queue_to_master() + * - void clear_results() + * - void receive_results_on_master() + * - void send_back_task_result_from_worker(std::size_t task) + * + * The latter five deal with sending results back from workers to master. + * update_real is used to sync parameters that changed on master to the workers + * before the next set of tasks is queued; it must have a part that updates the + * actual parameters on the worker process, but can optionally also be used to + * send them from the master to the queue (Q2W is handled by the Queue::loop). + * An example implementation: + + + + * + * The type of result from each task is strongly dependent on the Job at hand + * and so Job does not provide a default results vector or anything like this. + * It is up to the inheriting class to implement this. We would have liked a + * template parameter task_result_t, so that we could also provide a default + * "boilerplate" calculate function to show a typical Job use-case of all the + * above infrastructure. This is not trivial, because the JobManager has to + * keep a list of Job pointers, so if there would be different template + * instantiations of Jobs, this would complicate this list. As an example, + * a boilerplate calculation could look something like this: + * +return_type CertainJob::evaluate() { + if (get_manager()->process_manager().is_master()) { + // update parameters that changed since last calculation (or creation if first time) + for (size_t ix = 0; ix < changed_parameters.size(); ++ix) { // changed_parameters would be a member of CertainJob + auto msg = RooFit::MultiProcess::M2Q::update_real; + get_manager()->messenger().send_from_master_to_queue(msg, id, ix, + changed_parameters[ix].getVal(), + changed_parameters[ix].isConstant()); + } + + // master fills queue with tasks + for (std::size_t ix = 0; ix < N_tasks; ++ix) { // N_tasks would also be a member of CertainJob + JobTask job_task(id, ix); + get_manager()->queue().add(job_task); + } + + // wait for task results back from workers to master + gather_worker_results(); + + // possibly do some final manipulation of the results that were gathered + } + return result; // or not, it could be stored and accessed later; also here, result would be a member of CertainJob +} + * + * Child classes should refrain from direct access to the JobManager instance + * (through JobManager::instance), but rather use the here provided + * Job::get_manager(). This function starts the worker_loop on the worker when + * first called, meaning that the workers will not + */ +class Job { +public: + explicit Job(); + Job(const Job &other); + + ~Job(); + + virtual void evaluate_task(std::size_t task) = 0; + virtual void update_real(std::size_t ix, double val, bool is_constant) = 0; + virtual void update_bool(std::size_t ix, bool value) = 0; + virtual void update_state(); + + virtual void send_back_task_result_from_worker(std::size_t task) = 0; + virtual void receive_task_result_on_queue(std::size_t task, std::size_t worker_id) = 0; + virtual void send_back_results_from_queue_to_master() = 0; + // after results have been retrieved, they may need to be cleared to ensure + // they won't be retrieved the next time again, e.g. when using a map to + // collect results; if not needed it can just be left empty + virtual void clear_results() = 0; + virtual void receive_results_on_master() = 0; + virtual bool receive_task_result_on_master(const zmq::message_t & message) = 0; + + void gather_worker_results(); + +protected: + JobManager *get_manager(); + + std::size_t id; + +private: + // do not use _manager directly, it must first be initialized! use get_manager() + JobManager *_manager = nullptr; +}; + +} // namespace MultiProcess +} // namespace RooFit + +#endif // ROOT_ROOFIT_MultiProcess_Job_decl diff --git a/roofit/multiprocess/inc/RooFit/MultiProcess/JobManager.h b/roofit/multiprocess/inc/RooFit/MultiProcess/JobManager.h new file mode 100644 index 0000000000000..c3524138ae50a --- /dev/null +++ b/roofit/multiprocess/inc/RooFit/MultiProcess/JobManager.h @@ -0,0 +1,91 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2016-2019, Netherlands eScience Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ +#ifndef ROOT_ROOFIT_MultiProcess_JobManager +#define ROOT_ROOFIT_MultiProcess_JobManager + +#include // unique_ptr +#include + +#include "RooFit/MultiProcess/types.h" + +namespace RooFit { +namespace MultiProcess { + +// forward definitions +class ProcessManager; +class Messenger; +class Queue; +class Job; + +/* + * @brief Main point of access for all MultiProcess infrastructure + * + * This class mainly serves as the access point to the multi-process infrastructure + * for Jobs. It is meant to be used as a singleton that holds and connects the other + * infrastructural classes: the messenger, process manager, worker and queue loops. + * + * It is important that the user of this class, particularly the one that calls + * instance() first, calls activate_loops() soon after, because everything that is + * done in between instance() and activate_loops() will be executed on all processes. + * This may be useful in some cases, but in general, one will probably want to always + * use the JobManager in its full capacity, including the queue and worker loops. + * This is the way the Job class uses this class, see + * + * The default number of processes is set using std::thread::hardware_concurrency(). + * To change it, simply set JobManager::default_N_workers to a different value + * before creation of a new instance. + */ + +class JobManager { +public: + static JobManager *instance(); + static bool is_instantiated(); + + ~JobManager(); + + static std::size_t add_job_object(Job *job_object); + static Job *get_job_object(std::size_t job_object_id); + static bool remove_job_object(std::size_t job_object_id); + + ProcessManager & process_manager() const; + Messenger & messenger() const; + Queue & queue() const; + + void retrieve(std::size_t requesting_job_id); + void results_from_queue_to_master(); + + void activate(); + bool is_activated() const; + +private: + explicit JobManager(std::size_t N_workers); + + std::unique_ptr process_manager_ptr; + std::unique_ptr messenger_ptr; + std::unique_ptr queue_ptr; + bool activated = false; + + static std::map job_objects; + static std::size_t job_counter; + static std::unique_ptr _instance; + +public: + static unsigned int default_N_workers; // no need for getters/setters, just public +}; + +} // namespace MultiProcess +} // namespace RooFit + +#endif // ROOT_ROOFIT_MultiProcess_JobManager diff --git a/roofit/multiprocess/inc/RooFit/MultiProcess/Messenger.h b/roofit/multiprocess/inc/RooFit/MultiProcess/Messenger.h new file mode 100644 index 0000000000000..69163933b0db0 --- /dev/null +++ b/roofit/multiprocess/inc/RooFit/MultiProcess/Messenger.h @@ -0,0 +1,208 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2016-2019, Netherlands eScience Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ +#ifndef ROOT_ROOFIT_MultiProcess_Messenger +#define ROOT_ROOFIT_MultiProcess_Messenger + +#include "RooFit/MultiProcess/Messenger_decl.h" + +namespace RooFit { +namespace MultiProcess { + +// -- WORKER - QUEUE COMMUNICATION -- + +template +void Messenger::send_from_worker_to_queue(T item, Ts... items) +{ + #ifndef NDEBUG + std::stringstream ss; + ss << "PID " << getpid() << " sends W2Q " << item; + debug_print(ss.str()); + #endif + + zmqSvc().send(*this_worker_qw_push, item, send_flag); + // if (sizeof...(items) > 0) { // this will only work with if constexpr, c++17 + send_from_worker_to_queue(items...); +} + +template +value_t Messenger::receive_from_worker_on_queue(std::size_t this_worker_id) +{ + qw_pull_poller[this_worker_id].ppoll(-1, &ppoll_sigmask); + auto value = zmqSvc().receive(*qw_pull[this_worker_id], ZMQ_DONTWAIT); + + #ifndef NDEBUG + std::stringstream ss; + ss << "PID " << getpid() << " receives W(" << this_worker_id << ")2Q " << value; + debug_print(ss.str()); + #endif + + return value; +} + +template +void Messenger::send_from_queue_to_worker(std::size_t this_worker_id, T item, Ts... items) +{ +#ifndef NDEBUG + std::stringstream ss; + ss << "PID " << getpid() << " sends Q2W(" << this_worker_id << ") " << item; + debug_print(ss.str()); +#endif + + zmqSvc().send(*qw_push[this_worker_id], item, send_flag); + // if (sizeof...(items) > 0) { // this will only work with if constexpr, c++17 + send_from_queue_to_worker(this_worker_id, items...); +} + +template +value_t Messenger::receive_from_queue_on_worker() +{ + qw_pull_poller[0].ppoll(-1, &ppoll_sigmask); + auto value = zmqSvc().receive(*this_worker_qw_pull, ZMQ_DONTWAIT); + + #ifndef NDEBUG + std::stringstream ss; + ss << "PID " << getpid() << " receives Q2W " << value; + debug_print(ss.str()); + #endif + + return value; +} + + +// -- QUEUE - MASTER COMMUNICATION -- + +template +void Messenger::send_from_queue_to_master(T item, Ts... items) +{ + #ifndef NDEBUG + std::stringstream ss; + ss << "PID " << getpid() << " sends Q2M " << item; + debug_print(ss.str()); + #endif + + zmqSvc().send(*mq_push, item, send_flag); + // if (sizeof...(items) > 0) { // this will only work with if constexpr, c++17 + send_from_queue_to_master(items...); +} + +template +value_t Messenger::receive_from_queue_on_master() +{ + mq_pull_poller.ppoll(-1, &ppoll_sigmask); + auto value = zmqSvc().receive(*mq_pull, ZMQ_DONTWAIT); + + #ifndef NDEBUG + std::stringstream ss; + ss << "PID " << getpid() << " receives Q2M " << value; + debug_print(ss.str()); + #endif + + return value; +} + +template +void Messenger::send_from_master_to_queue(T item, Ts... items) +{ + #ifndef NDEBUG + std::stringstream ss; + ss << "PID " << getpid() << " sends M2Q " << item; + debug_print(ss.str()); + #endif + + zmqSvc().send(*mq_push, item, send_flag); + // if (sizeof...(items) > 0) { // this will only work with if constexpr, c++17 + send_from_master_to_queue(items...); +} + +template +value_t Messenger::receive_from_master_on_queue() +{ + mq_pull_poller.ppoll(-1, &ppoll_sigmask); + auto value = zmqSvc().receive(*mq_pull, ZMQ_DONTWAIT); + + #ifndef NDEBUG + std::stringstream ss; + ss << "PID " << getpid() << " receives M2Q " << value; + debug_print(ss.str()); + #endif + + return value; +} + + +// -- MASTER - WORKER COMMUNICATION -- + +template +void Messenger::publish_from_master_to_workers(T item, Ts... items) +{ +#ifndef NDEBUG + std::stringstream ss; + ss << "PID " << getpid() << " sends M2W " << item; + debug_print(ss.str()); +#endif + + zmqSvc().send(*mw_pub, item, send_flag); + // if (sizeof...(items) > 0) { // this will only work with if constexpr, c++17 + publish_from_master_to_workers(items...); +} + +template +value_t Messenger::receive_from_master_on_worker() +{ + mw_sub_poller.ppoll(-1, &ppoll_sigmask); + auto value = zmqSvc().receive(*mw_sub, ZMQ_DONTWAIT); + +#ifndef NDEBUG + std::stringstream ss; + ss << "PID " << getpid() << " receives M2W " << value; + debug_print(ss.str()); +#endif + + return value; +} + +template +void Messenger::send_from_worker_to_master(T item, Ts... items) +{ +#ifndef NDEBUG + std::stringstream ss; + ss << "PID " << getpid() << " sends M2W " << item; + debug_print(ss.str()); +#endif + + zmqSvc().send(*wm_push, item, send_flag); + // if (sizeof...(items) > 0) { // this will only work with if constexpr, c++17 + send_from_worker_to_master(items...); +} + +template +value_t Messenger::receive_from_worker_on_master() +{ + wm_pull_poller.ppoll(-1, &ppoll_sigmask); + auto value = zmqSvc().receive(*wm_pull, ZMQ_DONTWAIT); + +#ifndef NDEBUG + std::stringstream ss; + ss << "PID " << getpid() << " receives M2W " << value; + debug_print(ss.str()); +#endif + + return value; +} + +} // namespace MultiProcess +} // namespace RooFit + +#endif // ROOT_ROOFIT_MultiProcess_Messenger diff --git a/roofit/multiprocess/inc/RooFit/MultiProcess/Messenger_decl.h b/roofit/multiprocess/inc/RooFit/MultiProcess/Messenger_decl.h new file mode 100644 index 0000000000000..80d690658b241 --- /dev/null +++ b/roofit/multiprocess/inc/RooFit/MultiProcess/Messenger_decl.h @@ -0,0 +1,185 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2016-2019, Netherlands eScience Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ +#ifndef ROOT_ROOFIT_MultiProcess_Messenger_decl +#define ROOT_ROOFIT_MultiProcess_Messenger_decl + +#include +#include +#include // sigprocmask, sigset_t, etc + +#include "RooFit_ZMQ/ZeroMQSvc.h" +#include "RooFit_ZMQ/ZeroMQPoller.h" +#include "RooFit/MultiProcess/ProcessManager.h" + +namespace RooFit { +namespace MultiProcess { + +void set_socket_immediate(ZmqLingeringSocketPtr<> &socket); + +// test messages +enum class X2X : int { + ping = -1, + pong = -2, + initial_value = 0 +}; + +class Messenger { +public: + explicit Messenger(const ProcessManager &process_manager); + ~Messenger(); + + void test_connections(const ProcessManager &process_manager); + + enum class test_snd_pipes { + M2Q, + Q2M, + Q2W, + W2Q + }; + + enum class test_rcv_pipes { + fromQonM, + fromMonQ, + fromWonQ, + fromQonW, + }; + + std::pair create_queue_poller(); + std::pair create_worker_poller(); + + // -- WORKER - QUEUE COMMUNICATION -- + + void send_from_worker_to_queue(); + template + void send_from_worker_to_queue(T item, Ts... items); + template + value_t receive_from_worker_on_queue(std::size_t this_worker_id); + void send_from_queue_to_worker(std::size_t this_worker_id); + template + void send_from_queue_to_worker(std::size_t this_worker_id, T item, Ts... items); + template + value_t receive_from_queue_on_worker(); + + // -- QUEUE - MASTER COMMUNICATION -- + + void send_from_queue_to_master(); + + template + void send_from_queue_to_master(T item, Ts... items); + template + value_t receive_from_queue_on_master(); + void send_from_master_to_queue(); + + template + void send_from_master_to_queue(T item, Ts... items); + template + value_t receive_from_master_on_queue(); + + // -- MASTER - WORKER COMMUNICATION -- + + void publish_from_master_to_workers(); + template + void publish_from_master_to_workers(T item, Ts... items); + template + value_t receive_from_master_on_worker(); + + void send_from_worker_to_master(); + template + void send_from_worker_to_master(T item, Ts... items); + template + value_t receive_from_worker_on_master(); + + void test_receive(X2X expected_ping_value, test_rcv_pipes rcv_pipe, std::size_t worker_id); + void test_send(X2X ping_value, test_snd_pipes snd_pipe, std::size_t worker_id); + + sigset_t ppoll_sigmask; +// std::size_t N_available_polled_results = 0; + + void set_send_flag(int flag); + +private: + void debug_print(std::string s); + + // push + std::vector> qw_push; + ZmqLingeringSocketPtr<> this_worker_qw_push; + ZmqLingeringSocketPtr<> mq_push; + // pollers for all push sockets + std::vector qw_push_poller; + ZeroMQPoller mq_push_poller; + // pull + std::vector> qw_pull; + ZmqLingeringSocketPtr<> this_worker_qw_pull; + ZmqLingeringSocketPtr<> mq_pull; + // pollers for all pull sockets + std::vector qw_pull_poller; + ZeroMQPoller mq_pull_poller; + + // test to circumvent queue for parameter updating + ZmqLingeringSocketPtr<> mw_pub; + ZmqLingeringSocketPtr<> mw_sub; + ZeroMQPoller mw_sub_poller; + // test to circumvent queue for result retrieving + ZmqLingeringSocketPtr<> wm_push; + ZmqLingeringSocketPtr<> wm_pull; + ZeroMQPoller wm_pull_poller; + + // destruction flags to distinguish between different process-type setups: + bool close_MQ_on_destruct_ = false; + bool close_this_QW_on_destruct_ = false; + bool close_QW_container_on_destruct_ = false; + + int send_flag = 0; +}; + + +// Messages from master to queue +enum class M2Q : int { + terminate = 100, + enqueue = 10, + retrieve = 11, + update_real = 12, + // update_cat = 13, + update_bool = 14, +}; + +// Messages from queue to master +enum class Q2M : int { retrieve_rejected = 20, retrieve_accepted = 21, retrieve_later = 22 }; + +// Messages from worker to queue +enum class W2Q : int { dequeue = 30, send_result = 31 }; + +// Messages from queue to worker +enum class Q2W : int { + terminate = 400, + dequeue_rejected = 40, + dequeue_accepted = 41, + result_received = 43, + update_real = 44, + // update_cat = 45 + update_bool = 46, +}; + +// stream output operators for debugging +std::ostream &operator<<(std::ostream &out, const M2Q value); +std::ostream &operator<<(std::ostream &out, const Q2M value); +std::ostream &operator<<(std::ostream &out, const Q2W value); +std::ostream &operator<<(std::ostream &out, const W2Q value); +std::ostream &operator<<(std::ostream &out, const X2X value); + +} // namespace MultiProcess +} // namespace RooFit + +#endif // ROOT_ROOFIT_MultiProcess_Messenger_decl diff --git a/roofit/multiprocess/inc/RooFit/MultiProcess/ProcessManager.h b/roofit/multiprocess/inc/RooFit/MultiProcess/ProcessManager.h new file mode 100644 index 0000000000000..7c83546edecf0 --- /dev/null +++ b/roofit/multiprocess/inc/RooFit/MultiProcess/ProcessManager.h @@ -0,0 +1,78 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2016-2019, Netherlands eScience Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ +#ifndef ROOT_ROOFIT_MultiProcess_ProcessManager +#define ROOT_ROOFIT_MultiProcess_ProcessManager + +#include // pid_t +#include // sig_atomic_t and for sigterm handling on child processes (in ProcessManager.cxx) +#include + +// forward declaration +class Queue; + +namespace RooFit { +namespace MultiProcess { + +class ProcessManager { + friend Queue; + +public: + explicit ProcessManager(std::size_t N_workers); + ~ProcessManager(); + + bool is_initialized() const; + + void terminate() noexcept; +// void terminate_workers(); + void wait_for_sigterm_then_exit(); + + bool is_master() const; + bool is_queue() const; + bool is_worker() const; + std::size_t worker_id() const; + std::size_t N_workers() const; + + void identify_processes() const; + + static void handle_sigterm(int signum); + static bool sigterm_received(); + + // for debugging/testing: + pid_t get_queue_pid() {return queue_pid;} + std::vector get_worker_pids() {return worker_pids;} + +private: + void initialize_processes(bool cpu_pinning = true); + void shutdown_processes(); + + bool _is_master = false; + bool _is_queue = false; + bool _is_worker = false; + std::size_t _worker_id; + std::size_t _N_workers; + + // master must wait for workers after completion, for which it needs their PIDs + std::vector worker_pids; + pid_t queue_pid; + + bool initialized = false; + + static volatile sig_atomic_t _sigterm_received; +}; + +} // namespace MultiProcess +} // namespace RooFit + +#endif // ROOT_ROOFIT_MultiProcess_ProcessManager diff --git a/roofit/multiprocess/inc/RooFit/MultiProcess/Queue.h b/roofit/multiprocess/inc/RooFit/MultiProcess/Queue.h new file mode 100644 index 0000000000000..bd44ac1c36a96 --- /dev/null +++ b/roofit/multiprocess/inc/RooFit/MultiProcess/Queue.h @@ -0,0 +1,46 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2016-2019, Netherlands eScience Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ +#ifndef ROOT_ROOFIT_MultiProcess_Queue +#define ROOT_ROOFIT_MultiProcess_Queue + +#include + +#include "RooFit/MultiProcess/types.h" +#include "RooFit/MultiProcess/Messenger.h" + +namespace RooFit { +namespace MultiProcess { + +class Queue { +public: + bool pop(JobTask &job_task); + void add(JobTask job_task); + + void loop(); + + bool process_master_message(M2Q message); + void process_worker_message(std::size_t this_worker_id, W2Q message); + +private: + std::queue _queue; + std::size_t N_tasks = 0; // total number of received tasks + std::size_t N_tasks_completed = 0; + std::size_t N_tasks_at_workers = 0; +}; + +} // namespace MultiProcess +} // namespace RooFit + +#endif // ROOT_ROOFIT_MultiProcess_Queue diff --git a/roofit/multiprocess/inc/RooFit/MultiProcess/types.h b/roofit/multiprocess/inc/RooFit/MultiProcess/types.h new file mode 100644 index 0000000000000..6b0f0cab87ece --- /dev/null +++ b/roofit/multiprocess/inc/RooFit/MultiProcess/types.h @@ -0,0 +1,29 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2016-2019, Netherlands eScience Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ +#ifndef ROOT_ROOFIT_MultiProcess_types +#define ROOT_ROOFIT_MultiProcess_types + +#include // pair + +namespace RooFit { +namespace MultiProcess { + +// some helper types +using Task = std::size_t; +using JobTask = std::pair; // combined job_object and task identifier type + +} // namespace MultiProcess +} // namespace RooFit + +#endif // ROOT_ROOFIT_MultiProcess_types diff --git a/roofit/multiprocess/inc/RooFit/MultiProcess/util.h b/roofit/multiprocess/inc/RooFit/MultiProcess/util.h new file mode 100644 index 0000000000000..4f5f5f287857a --- /dev/null +++ b/roofit/multiprocess/inc/RooFit/MultiProcess/util.h @@ -0,0 +1,34 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2016-2019, Netherlands eScience Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef ROOT_ROOFIT_MultiProcess_util +#define ROOT_ROOFIT_MultiProcess_util + +#include // getpid, pid_t + +#include // for ZMQ::ppoll_error_t +#include + +namespace RooFit { +namespace MultiProcess { + +int wait_for_child(pid_t child_pid, bool may_throw, int retries_before_killing); + +enum class zmq_ppoll_error_response { abort, unknown_eintr, retry }; +zmq_ppoll_error_response handle_zmq_ppoll_error(ZMQ::ppoll_error_t &e); +std::tuple>, bool> careful_ppoll(ZeroMQPoller &poller, const sigset_t &ppoll_sigmask, std::size_t max_tries = 2); + +} // namespace MultiProcess +} // namespace RooFit +#endif // ROOT_ROOFIT_MultiProcess_util diff --git a/roofit/multiprocess/inc/RooFit/MultiProcess/worker.h b/roofit/multiprocess/inc/RooFit/MultiProcess/worker.h new file mode 100644 index 0000000000000..2fc2eaa7ef5ae --- /dev/null +++ b/roofit/multiprocess/inc/RooFit/MultiProcess/worker.h @@ -0,0 +1,27 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2016-2019, Netherlands eScience Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ +#ifndef ROOT_ROOFIT_MultiProcess_worker +#define ROOT_ROOFIT_MultiProcess_worker + +namespace RooFit { +namespace MultiProcess { + +void worker_loop(); +bool is_worker_loop_running(); + +} // namespace MultiProcess +} // namespace RooFit + +#endif // ROOT_ROOFIT_MultiProcess_worker diff --git a/roofit/multiprocess/src/Job.cxx b/roofit/multiprocess/src/Job.cxx new file mode 100644 index 0000000000000..9f3cd7aacf1a1 --- /dev/null +++ b/roofit/multiprocess/src/Job.cxx @@ -0,0 +1,65 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2016-2019, Netherlands eScience Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "RooFit/MultiProcess/JobManager.h" +#include "RooFit/MultiProcess/Messenger.h" +#include "RooFit/MultiProcess/worker.h" +#include "RooFit/MultiProcess/Job.h" +#include "RooFit/MultiProcess/Queue.h" + +namespace RooFit { +namespace MultiProcess { + +Job::Job() +{ + id = JobManager::add_job_object(this); +} + +Job::Job(const Job &other) + : _manager(other._manager) +{ + id = JobManager::add_job_object(this); +} + +Job::~Job() +{ + JobManager::remove_job_object(id); +} + +// This function is necessary here, because the Job knows about the number +// of workers, so only from the Job can the JobManager be instantiated. +JobManager *Job::get_manager() +{ + if (!_manager) { + _manager = JobManager::instance(); + } + + if (!_manager->is_activated()) { + _manager->activate(); + } + + return _manager; +} + +void Job::gather_worker_results() +{ + get_manager()->retrieve(id); +} + +void Job::update_state() {} + + +} // namespace MultiProcess +} // namespace RooFit diff --git a/roofit/multiprocess/src/JobManager.cxx b/roofit/multiprocess/src/JobManager.cxx new file mode 100644 index 0000000000000..d65a821d39c66 --- /dev/null +++ b/roofit/multiprocess/src/JobManager.cxx @@ -0,0 +1,264 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2016-2019, Netherlands eScience Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ +#include // getpid + +#include // std::thread::hardware_concurrency() + +#include "ROOT/RMakeUnique.hxx" // make_unique in C++11 + +#include "RooFit/MultiProcess/ProcessManager.h" +#include "RooFit/MultiProcess/Messenger.h" +#include "RooFit/MultiProcess/JobManager.h" +#include "RooFit/MultiProcess/Job.h" +#include "RooFit/MultiProcess/Queue.h" // complete type for JobManager::queue() +#include "RooFit/MultiProcess/worker.h" +#include "RooFit/MultiProcess/util.h" + + +namespace RooFit { +namespace MultiProcess { + +// static function +JobManager* JobManager::instance() { + if (!JobManager::is_instantiated()) { + assert(default_N_workers != 0); + _instance.reset(new JobManager(default_N_workers)); // can't use make_unique, because ctor is private + _instance->messenger().test_connections(_instance->process_manager()); + // set send to non blocking on all processes after checking the connections are working: + _instance->messenger().set_send_flag(ZMQ_DONTWAIT); + } + return _instance.get(); +} + + +// static function +bool JobManager::is_instantiated() { + return static_cast(_instance); +} + +// (private) constructor +// Don't construct JobManager objects manually, use the static instance if +// you need to run multiple jobs. +JobManager::JobManager(std::size_t N_workers) { + queue_ptr = std::make_unique(); + process_manager_ptr = std::make_unique(N_workers); + messenger_ptr = std::make_unique(*process_manager_ptr); +} + +JobManager::~JobManager() { + // The instance gets created by some Job. Once all Jobs are gone, the + // JM will get destroyed. In this case, the job_objects map should have + // been emptied. This check makes sure: + assert(JobManager::job_objects.empty()); + messenger_ptr.reset(nullptr); + process_manager_ptr.reset(nullptr); + queue_ptr.reset(nullptr); +} + + +// static function +// returns job_id for added job_object +std::size_t JobManager::add_job_object(Job *job_object) { + if (JobManager::is_instantiated()) { + if (_instance->process_manager().is_initialized()) { + std::stringstream ss; + ss << "Cannot add Job to JobManager instantiation, forking has already taken place! Instance object at raw ptr " << _instance.get(); + throw std::logic_error("Cannot add Job to JobManager instantiation, forking has already taken place! Call terminate() on the instance before adding new Jobs."); + } + } + std::size_t job_id = job_counter++; + job_objects[job_id] = job_object; + return job_id; +} + +// static function +Job* JobManager::get_job_object(std::size_t job_object_id) { + return job_objects[job_object_id]; +} + +// static function +bool JobManager::remove_job_object(std::size_t job_object_id) { + bool removed_succesfully = job_objects.erase(job_object_id) == 1; + if (job_objects.empty()) { + _instance.reset(nullptr); + } + return removed_succesfully; +} + + +ProcessManager & JobManager::process_manager() const { + return *process_manager_ptr; +} + + +Messenger & JobManager::messenger() const { + return *messenger_ptr; +} + + +Queue & JobManager::queue() const { + return *queue_ptr; +} + + +//void JobManager::retrieve() { +// if (process_manager().is_master()) { +// bool carry_on = true; +// while (carry_on) { +// messenger().send_from_master_to_queue(M2Q::retrieve); +// try { +// auto handshake = messenger().receive_from_queue_on_master(); +// switch (handshake) { +// case Q2M::retrieve_accepted: { +// carry_on = false; +// auto N_jobs = messenger().receive_from_queue_on_master(); +// for (std::size_t job_ix = 0; job_ix < N_jobs; ++job_ix) { +// auto job_object_id = messenger().receive_from_queue_on_master(); +// JobManager::get_job_object(job_object_id)->receive_results_on_master(); +// } +// } break; +// case Q2M::retrieve_later: { +// carry_on = true; +// } break; +// case Q2M::retrieve_rejected: { +// carry_on = false; +// throw std::logic_error( +// "Master sent M2Q::retrieve, but queue had no tasks yet: Q2M::retrieve_rejected. Aborting!"); +// } break; +// } +// } catch (ZMQ::ppoll_error_t &e) { +// zmq_ppoll_error_response response; +// try { +// response = handle_zmq_ppoll_error(e); +// } catch (std::logic_error& e) { +// printf("JobManager::retrieve got unhandleable ZMQ::ppoll_error_t\n"); +// throw; +// } +// if (response == zmq_ppoll_error_response::abort) { +// throw std::logic_error("in JobManager::retrieve: master received a SIGTERM, aborting"); +// } else if (response == zmq_ppoll_error_response::unknown_eintr) { +// printf("EINTR in JobManager::retrieve, continuing\n"); +// continue; +// } else if (response == zmq_ppoll_error_response::retry) { +// printf("EAGAIN from ppoll in JobManager::retrieve, continuing\n"); +// continue; +// } +// } catch (zmq::error_t& e) { +// printf("unhandled zmq::error_t (not a ppoll_error_t) in JobManager::retrieve with errno %d: %s\n", e.num(), e.what()); +// throw; +// } +// +// } +// } +//} + + +void JobManager::retrieve(std::size_t requesting_job_id) { + if (process_manager().is_master()) { +// auto get_time = []() { +// return std::chrono::duration_cast( +// std::chrono::high_resolution_clock::now().time_since_epoch()) +// .count(); +// }; +// decltype(get_time()) t1, t2; + + bool job_fully_retrieved = false; + while (not job_fully_retrieved) { + try { +// t1 = get_time(); + auto task_result_message = messenger().receive_from_worker_on_master(); + auto job_object_id = *reinterpret_cast(task_result_message.data()); // job_id must always be the first element of the result message! + bool this_job_fully_retrieved = JobManager::get_job_object(job_object_id)->receive_task_result_on_master(task_result_message); + if (requesting_job_id == job_object_id) { + job_fully_retrieved = this_job_fully_retrieved; + } +// t2 = get_time(); +// printf("wallclock [master] retrieve got task %lu: %f\n", *(reinterpret_cast(task_result_message.data()) + 1), (t2 - t1) / 1.e9); + } catch (ZMQ::ppoll_error_t &e) { + zmq_ppoll_error_response response; + try { + response = handle_zmq_ppoll_error(e); + } catch (std::logic_error& e) { + printf("JobManager::retrieve got unhandleable ZMQ::ppoll_error_t\n"); + throw; + } + if (response == zmq_ppoll_error_response::abort) { + throw std::logic_error("in JobManager::retrieve: master received a SIGTERM, aborting"); + } else if (response == zmq_ppoll_error_response::unknown_eintr) { + printf("EINTR in JobManager::retrieve, continuing\n"); + continue; + } else if (response == zmq_ppoll_error_response::retry) { + printf("EAGAIN from ppoll in JobManager::retrieve, continuing\n"); + continue; + } + } catch (zmq::error_t& e) { + printf("unhandled zmq::error_t (not a ppoll_error_t) in JobManager::retrieve with errno %d: %s\n", e.num(), e.what()); + throw; + } + } + } +} + + +void JobManager::results_from_queue_to_master() +{ + assert(process_manager().is_queue()); + messenger().send_from_queue_to_master(Q2M::retrieve_accepted, JobManager::job_objects.size()); + for (auto job_tuple : JobManager::job_objects) { + messenger().send_from_queue_to_master(job_tuple.first); // job id + job_tuple.second->send_back_results_from_queue_to_master(); // N_job_tasks, task_ids and results + job_tuple.second->clear_results(); + } +} + + +void JobManager::activate() +{ + // This function exists purely because activation from the constructor is + // impossible; the constructor must return a constructed instance, which it + // can't do if it's stuck in an infinite loop. This means the Job that first + // creates the JobManager instance must also activate it (or any other user + // of this class). + // This should be called soon after creation of instance, because everything + // between construction and activation gets executed both on the master + // process and on the slaves. + + activated = true; + + if (process_manager().is_queue()) { + queue().loop(); + std::_Exit(0); + } + + if (!is_worker_loop_running() && process_manager().is_worker()) { + RooFit::MultiProcess::worker_loop(); + std::_Exit(0); + } +} + + +bool JobManager::is_activated() const +{ + return activated; +} + +// initialize static members +std::map JobManager::job_objects; +std::size_t JobManager::job_counter = 0; +std::unique_ptr JobManager::_instance {nullptr}; +unsigned int JobManager::default_N_workers = std::thread::hardware_concurrency(); + +} // namespace MultiProcess +} // namespace RooFit diff --git a/roofit/multiprocess/src/Messenger.cxx b/roofit/multiprocess/src/Messenger.cxx new file mode 100644 index 0000000000000..911535506034d --- /dev/null +++ b/roofit/multiprocess/src/Messenger.cxx @@ -0,0 +1,481 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2016-2019, Netherlands eScience Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include // sigprocmask etc +#include // sprintf + +#include +#include "RooFit/MultiProcess/Messenger.h" + +namespace RooFit { +namespace MultiProcess { + +void set_socket_immediate(ZmqLingeringSocketPtr<> &socket) +{ + int optval = 1; + socket->setsockopt(ZMQ_IMMEDIATE, &optval, sizeof(optval)); +} + +Messenger::Messenger(const ProcessManager &process_manager) +{ + sigemptyset(&ppoll_sigmask); + + // high water mark for master-queue sending, which can be quite a busy channel, especially at the start of a run + int hwm = 0; + // create zmq connections (zmq context is automatically created in the ZeroMQSvc class and maintained as singleton) + // and pollers where necessary + try { + if (process_manager.is_master()) { + mq_push.reset(zmqSvc().socket_ptr(zmq::PUSH)); + auto rc = zmq_setsockopt (*mq_push, ZMQ_SNDHWM, &hwm, sizeof hwm); + assert (rc == 0); + mq_push->bind("ipc:///tmp/roofitMP_from_master_to_queue"); + + mq_push_poller.register_socket(*mq_push, zmq::POLLOUT); + + mq_pull.reset(zmqSvc().socket_ptr(zmq::PULL)); + rc = zmq_setsockopt (*mq_pull, ZMQ_RCVHWM, &hwm, sizeof hwm); + assert (rc == 0); + mq_pull->bind("ipc:///tmp/roofitMP_from_queue_to_master"); + + mq_pull_poller.register_socket(*mq_pull, zmq::POLLIN); + + mw_pub.reset(zmqSvc().socket_ptr(zmq::PUB)); + rc = zmq_setsockopt (*mw_pub, ZMQ_SNDHWM, &hwm, sizeof hwm); + assert (rc == 0); + mw_pub->bind("ipc:///tmp/roofitMP_from_master_to_workers"); + + wm_pull.reset(zmqSvc().socket_ptr(zmq::PULL)); + rc = zmq_setsockopt (*wm_pull, ZMQ_RCVHWM, &hwm, sizeof hwm); + assert (rc == 0); + wm_pull->bind("ipc:///tmp/roofitMP_from_workers_to_master"); + wm_pull_poller.register_socket(*wm_pull, zmq::POLLIN); + + close_MQ_on_destruct_ = true; + + // make sure all subscribers are connected + ZmqLingeringSocketPtr<> subscriber_ping_socket {zmqSvc().socket_ptr(zmq::REP)}; + subscriber_ping_socket->bind("ipc:///tmp/roofitMP_subscriber_ping_socket"); + ZeroMQPoller subscriber_ping_poller; + subscriber_ping_poller.register_socket(*subscriber_ping_socket, zmq::POLLIN); + std::size_t N_subscribers_confirmed = 0; + while (N_subscribers_confirmed < process_manager.N_workers()) { + zmqSvc().send(*mw_pub, false); + auto poll_results = subscriber_ping_poller.poll(0); +// for (auto& poll_result : poll_results) { + for (std::size_t ix = 0; ix < poll_results.size(); ++ix) { + auto request = zmqSvc().receive(*subscriber_ping_socket, zmq::DONTWAIT); + assert(request == "present"); + zmqSvc().send(*subscriber_ping_socket, "roger"); + ++N_subscribers_confirmed; + } + } + zmqSvc().send(*mw_pub, true); + + } else if (process_manager.is_queue()) { + // first the queue-worker sockets + // do resize instead of reserve so that the unique_ptrs are initialized + // (to nullptr) so that we can do reset below, alternatively you can do + // push/emplace_back with move or something + qw_push.resize(process_manager.N_workers()); + qw_pull.resize(process_manager.N_workers()); + qw_push_poller.resize(process_manager.N_workers()); + qw_pull_poller.resize(process_manager.N_workers()); + for (std::size_t ix = 0; ix < process_manager.N_workers(); ++ix) { + std::stringstream push_name, pull_name; + // push + qw_push[ix].reset(zmqSvc().socket_ptr(zmq::PUSH)); + push_name << "ipc:///tmp/roofitMP_from_queue_to_worker_" << ix; + qw_push[ix]->bind(push_name.str()); + + qw_push_poller[ix].register_socket(*qw_push[ix], zmq::POLLOUT); + + // pull + qw_pull[ix].reset(zmqSvc().socket_ptr(zmq::PULL)); + pull_name << "ipc:///tmp/roofitMP_from_worker_" << ix << "_to_queue"; + qw_pull[ix]->bind(pull_name.str()); + + qw_pull_poller[ix].register_socket(*qw_pull[ix], zmq::POLLIN); + } + + // then the master-queue sockets + mq_push.reset(zmqSvc().socket_ptr(zmq::PUSH)); + auto rc = zmq_setsockopt (*mq_push, ZMQ_SNDHWM, &hwm, sizeof hwm); + assert (rc == 0); + mq_push->connect("ipc:///tmp/roofitMP_from_queue_to_master"); + + mq_push_poller.register_socket(*mq_push, zmq::POLLOUT); + + mq_pull.reset(zmqSvc().socket_ptr(zmq::PULL)); + rc = zmq_setsockopt (*mq_pull, ZMQ_RCVHWM, &hwm, sizeof hwm); + assert (rc == 0); + mq_pull->connect("ipc:///tmp/roofitMP_from_master_to_queue"); + + mq_pull_poller.register_socket(*mq_pull, zmq::POLLIN); + + close_MQ_on_destruct_ = true; + close_QW_container_on_destruct_ = true; + } else if (process_manager.is_worker()) { + // we only need one queue-worker pipe on the worker + qw_push_poller.resize(1); + qw_pull_poller.resize(1); + + std::stringstream push_name, pull_name; + // push + this_worker_qw_push.reset(zmqSvc().socket_ptr(zmq::PUSH)); + push_name << "ipc:///tmp/roofitMP_from_worker_" << process_manager.worker_id() << "_to_queue"; + this_worker_qw_push->connect(push_name.str()); + + qw_push_poller[0].register_socket(*this_worker_qw_push, zmq::POLLOUT); + + // pull + this_worker_qw_pull.reset(zmqSvc().socket_ptr(zmq::PULL)); + pull_name << "ipc:///tmp/roofitMP_from_queue_to_worker_" << process_manager.worker_id(); + this_worker_qw_pull->connect(pull_name.str()); + + qw_pull_poller[0].register_socket(*this_worker_qw_pull, zmq::POLLIN); + + mw_sub.reset(zmqSvc().socket_ptr(zmq::SUB)); + auto rc = zmq_setsockopt (*mw_sub, ZMQ_RCVHWM, &hwm, sizeof hwm); + assert (rc == 0); + rc = zmq_setsockopt(*mw_sub, ZMQ_SUBSCRIBE, "", 0); + assert (rc == 0); + mw_sub->connect("ipc:///tmp/roofitMP_from_master_to_workers"); + mw_sub_poller.register_socket(*mw_sub, zmq::POLLIN); + + wm_push.reset(zmqSvc().socket_ptr(zmq::PUSH)); + rc = zmq_setsockopt (*wm_push, ZMQ_SNDHWM, &hwm, sizeof hwm); + assert (rc == 0); + wm_push->connect("ipc:///tmp/roofitMP_from_workers_to_master"); + + // check publisher connection and then wait until all subscribers are connected + ZmqLingeringSocketPtr<> subscriber_ping_socket {zmqSvc().socket_ptr(zmq::REQ)}; + subscriber_ping_socket->connect("ipc:///tmp/roofitMP_subscriber_ping_socket"); + auto all_connected = zmqSvc().receive(*mw_sub); + zmqSvc().send(*subscriber_ping_socket, "present"); + auto reply = zmqSvc().receive(*subscriber_ping_socket); + assert(reply == "roger"); + + while (!all_connected) { + all_connected = zmqSvc().receive(*mw_sub); + } + + close_this_QW_on_destruct_ = true; + } else { + // should never get here + throw std::runtime_error("Messenger ctor: I'm neither master, nor queue, nor a worker"); + } + } catch (zmq::error_t &e) { + std::cerr << e.what() << " -- errnum: " << e.num() << std::endl; + throw; + }; +} + +Messenger::~Messenger() { + printf("Messenger dtor on PID %d\n", getpid()); + if (close_MQ_on_destruct_) { + try { + mq_push.reset(nullptr); + mq_pull.reset(nullptr); + mw_pub.reset(nullptr); + wm_pull.reset(nullptr); + } catch (const std::exception& e) { + std::cerr << "WARNING: something in Messenger dtor threw an exception! Original exception message:\n" << e.what() << std::endl; + } + } + if (close_this_QW_on_destruct_) { + this_worker_qw_push.reset(nullptr); + this_worker_qw_pull.reset(nullptr); + mw_sub.reset(nullptr); + wm_push.reset(nullptr); + } + if (close_QW_container_on_destruct_) { + for (auto& socket : qw_push) { + socket.reset(nullptr); + } + for (auto& socket : qw_pull) { + socket.reset(nullptr); + } + } + zmqSvc().close_context(); +} + + +void Messenger::test_send(X2X ping_value, test_snd_pipes snd_pipe, std::size_t worker_id) { + try { + switch (snd_pipe) { + case test_snd_pipes::M2Q: { + send_from_master_to_queue(ping_value); + break; + } + case test_snd_pipes::Q2M : { + send_from_queue_to_master(ping_value); + break; + } + case test_snd_pipes::Q2W: { + send_from_queue_to_worker(worker_id, ping_value); + break; + } + case test_snd_pipes::W2Q: { + send_from_worker_to_queue(ping_value); + break; + } + } + } catch (zmq::error_t &e) { + if (e.num() == EAGAIN) { + throw std::runtime_error("Messenger::test_connections: SEND over master-queue connection timed out!"); + } else { + throw; + } + } +} + + +void Messenger::test_receive(X2X expected_ping_value, test_rcv_pipes rcv_pipe, std::size_t worker_id) { + X2X handshake = X2X::initial_value; + + std::size_t max_tries = 3, tries = 0; + bool carry_on = true; + while (carry_on && (tries++ < max_tries)) { + try { + switch (rcv_pipe) { + case test_rcv_pipes::fromMonQ: { + handshake = receive_from_master_on_queue(); + break; + } + case test_rcv_pipes::fromQonM: { + handshake = receive_from_queue_on_master(); + break; + } + case test_rcv_pipes::fromQonW: { + handshake = receive_from_queue_on_worker(); + break; + } + case test_rcv_pipes::fromWonQ: { + handshake = receive_from_worker_on_queue(worker_id); + break; + } + } + carry_on = false; + } catch (ZMQ::ppoll_error_t &e) { + auto response = handle_zmq_ppoll_error(e); + if (response == zmq_ppoll_error_response::abort) { + throw std::runtime_error("EINTR in test_receive and SIGTERM received, aborting\n"); + } else if (response == zmq_ppoll_error_response::unknown_eintr) { + printf("EINTR in test_receive but no SIGTERM received, try %lu\n", tries); + continue; + } else if (response == zmq_ppoll_error_response::retry) { + printf("EAGAIN in test_receive, try %lu\n", tries); + continue; + } + } catch (zmq::error_t &e) { + if (e.num() == EAGAIN) { + throw std::runtime_error("Messenger::test_connections: RECEIVE over master-queue connection timed out!"); + } else { + printf("unhandled zmq::error_t (not a ppoll_error_t) in Messenger::test_receive with errno %d: %s\n", e.num(), e.what()); + throw; + } + } + } + + if (handshake != expected_ping_value) { + throw std::runtime_error("Messenger::test_connections: RECEIVE over master-queue connection failed, did not receive pong!"); + } +} + + +void Messenger::test_connections(const ProcessManager &process_manager) { + // Before blocking SIGTERM, set the signal handler, so we can also check after blocking whether a signal occurred + // In our case, we already set it in the ProcessManager after forking to the queue and worker processes. + sigset_t sigmask; + sigemptyset(&sigmask); + sigaddset(&sigmask, SIGTERM); + sigprocmask(SIG_BLOCK, &sigmask, &ppoll_sigmask); + + if (process_manager.is_master()) { + test_send(X2X::ping, test_snd_pipes::M2Q, -1); + test_receive(X2X::pong, test_rcv_pipes::fromQonM, -1); + test_receive(X2X::ping, test_rcv_pipes::fromQonM, -1); + test_send(X2X::pong, test_snd_pipes::M2Q, -1); + } else if (process_manager.is_queue()) { + ZeroMQPoller poller; + std::size_t mq_index; + std::tie(poller, mq_index) = create_queue_poller(); + + for (std::size_t ix = 0; ix < process_manager.N_workers(); ++ix) { + test_send(X2X::ping, test_snd_pipes::Q2W, ix); + } + + while (!process_manager.sigterm_received() && (poller.size() > 0)) { + // poll: wait until status change (-1: infinite timeout) + std::vector> poll_result; + bool abort; + std::tie(poll_result, abort) = careful_ppoll(poller, ppoll_sigmask); + if (abort) break; + + // then process incoming messages from sockets + for (auto readable_socket : poll_result) { + // message comes from the master/queue socket (first element): + if (readable_socket.first == mq_index) { + test_receive(X2X::ping, test_rcv_pipes::fromMonQ, -1); + test_send(X2X::pong, test_snd_pipes::Q2M, -1); + test_send(X2X::ping, test_snd_pipes::Q2M, -1); + test_receive(X2X::pong, test_rcv_pipes::fromMonQ, -1); + poller.unregister_socket(*mq_pull); + } else { // from a worker socket + // TODO: dangerous assumption for this_worker_id, may become invalid if we allow multiple queue_loops on the same process! + auto this_worker_id = readable_socket.first - 1; // TODO: replace with a more reliable lookup + + test_receive(X2X::pong, test_rcv_pipes::fromWonQ, this_worker_id); + test_receive(X2X::ping, test_rcv_pipes::fromWonQ, this_worker_id); + test_send(X2X::pong, test_snd_pipes::Q2W, this_worker_id); + + poller.unregister_socket(*qw_pull[this_worker_id]); + } + } + } + + } else if (process_manager.is_worker()) { + test_receive(X2X::ping, test_rcv_pipes::fromQonW, -1); + test_send(X2X::pong, test_snd_pipes::W2Q, -1); + test_send(X2X::ping, test_snd_pipes::W2Q, -1); + test_receive(X2X::pong, test_rcv_pipes::fromQonW, -1); + } else { + // should never get here + throw std::runtime_error("Messenger::test_connections: I'm neither master, nor queue, nor a worker"); + } + + // clean up signal management modifications + sigprocmask(SIG_SETMASK, &ppoll_sigmask, nullptr); + printf("done with test_connections on PID %d\n", getpid()); +} + + +std::pair Messenger::create_queue_poller() { + ZeroMQPoller poller; + std::size_t mq_index = poller.register_socket(*mq_pull, zmq::POLLIN); + for (auto &s : qw_pull) { + poller.register_socket(*s, zmq::POLLIN); + } + return {std::move(poller), mq_index}; +} + + +std::pair Messenger::create_worker_poller() { + ZeroMQPoller poller; + poller.register_socket(*this_worker_qw_pull, zmq::POLLIN); + std::size_t mw_sub_index = poller.register_socket(*mw_sub, zmq::POLLIN); + return {std::move(poller), mw_sub_index}; +} + + +// -- WORKER - QUEUE COMMUNICATION -- + +void Messenger::send_from_worker_to_queue() {} + +void Messenger::send_from_queue_to_worker(std::size_t /*this_worker_id*/) {} + +// -- QUEUE - MASTER COMMUNICATION -- + +void Messenger::send_from_queue_to_master() {} + +void Messenger::send_from_master_to_queue() {} + +void Messenger::set_send_flag(int flag) { + if (flag == 0 || flag == ZMQ_DONTWAIT || flag == ZMQ_SNDMORE || flag == (ZMQ_DONTWAIT | ZMQ_SNDMORE)) { + send_flag = flag; + } else { + throw std::runtime_error("in Messenger::set_send_flag: trying to set illegal flag, see zmq_send API for allowed flags"); + } +} + +// -- MASTER - WORKER COMMUNICATION -- + +void Messenger::publish_from_master_to_workers() {} + +void Messenger::send_from_worker_to_master() {} + + +// for debugging +#define PROCESS_VAL(p) case(p): s = #p; break; + +std::ostream& operator<<(std::ostream& out, const M2Q value){ + std::string s; + switch(value){ + PROCESS_VAL(M2Q::terminate); + PROCESS_VAL(M2Q::enqueue); + PROCESS_VAL(M2Q::retrieve); + PROCESS_VAL(M2Q::update_real); + PROCESS_VAL(M2Q::update_bool); + default: s = std::to_string(static_cast(value)); + } + return out << s; +} + +std::ostream& operator<<(std::ostream& out, const Q2M value){ + std::string s; + switch(value){ + PROCESS_VAL(Q2M::retrieve_rejected); + PROCESS_VAL(Q2M::retrieve_accepted); + PROCESS_VAL(Q2M::retrieve_later); + default: s = std::to_string(static_cast(value)); + } + return out << s; +} + +std::ostream& operator<<(std::ostream& out, const W2Q value){ + std::string s; + switch(value){ + PROCESS_VAL(W2Q::dequeue); + PROCESS_VAL(W2Q::send_result); + default: s = std::to_string(static_cast(value)); + } + return out << s; +} + +std::ostream& operator<<(std::ostream& out, const Q2W value){ + std::string s; + switch(value){ + PROCESS_VAL(Q2W::terminate); + PROCESS_VAL(Q2W::dequeue_rejected); + PROCESS_VAL(Q2W::dequeue_accepted); + PROCESS_VAL(Q2W::update_real); + PROCESS_VAL(Q2W::result_received); + PROCESS_VAL(Q2W::update_bool); + default: s = std::to_string(static_cast(value)); + } + return out << s; +} + +std::ostream& operator<<(std::ostream& out, const X2X value){ + std::string s; + switch(value){ + PROCESS_VAL(X2X::ping); + PROCESS_VAL(X2X::pong); + default: s = std::to_string(static_cast(value)); + } + return out << s; +} + +#undef PROCESS_VAL + +void Messenger::debug_print(std::string /*s*/) +{ +// printf("%s\n", s.c_str()); +// std::cerr << s << std::endl; +} + +} // namespace MultiProcess +} // namespace RooFit diff --git a/roofit/multiprocess/src/ProcessManager.cxx b/roofit/multiprocess/src/ProcessManager.cxx new file mode 100644 index 0000000000000..6720443b9dddd --- /dev/null +++ b/roofit/multiprocess/src/ProcessManager.cxx @@ -0,0 +1,286 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2016-2019, Netherlands eScience Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include // for strsignal +#include // for wait +#include +#include + +#include "RooFit/MultiProcess/util.h" +#include "RooFit/MultiProcess/JobManager.h" +#include "RooFit/MultiProcess/Messenger.h" +#include "RooFit/MultiProcess/ProcessManager.h" + +namespace RooFit { +namespace MultiProcess { + + +ProcessManager::ProcessManager(std::size_t N_workers) : _N_workers(N_workers) { + // This class defines three types of processes: + // 1. master: the initial main process. It defines and enqueues tasks + // and processes results. + // 2. workers: a pool of processes that will try to take tasks from the + // queue. These are forked from master. + // 3. queue: communication between the other types (necessarily) goes + // through this process. This process runs the queue_loop and + // maintains the queue of tasks. + + initialize_processes(); +} + +ProcessManager::~ProcessManager() +{ + if (is_master()) { + terminate(); + } else { + wait_for_sigterm_then_exit(); + } +} + + +// static member initialization +volatile sig_atomic_t ProcessManager::_sigterm_received = 0; + +// static function +void ProcessManager::handle_sigterm(int signum) { + // We need this to tell the children to die, because we can't talk + // to them anymore during JobManager destruction, because that kills + // the Messenger first. We do that with SIGTERMs. The sigterm_received() + // should be checked in message loops to stop them when it's true. + _sigterm_received = 1; + printf("handled %s on PID %d\n", strsignal(signum), getpid()); +} + +// static function +bool ProcessManager::sigterm_received() { + if (_sigterm_received > 0) { + return true; + } else { + return false; + } +} + +void ProcessManager::initialize_processes(bool cpu_pinning) { + // Initialize processes; + // ... first workers: + worker_pids.resize(_N_workers); + pid_t child_pid {}; + for (std::size_t ix = 0; ix < _N_workers; ++ix) { + child_pid = fork(); + if (!child_pid) { // we're on the worker + _is_worker = true; + _worker_id = ix; + break; + } else { // we're on master + worker_pids[ix] = child_pid; + } + } + + // ... then queue: + if (child_pid) { // we're on master + queue_pid = fork(); + if (!queue_pid) { // we're now on queue + _is_queue = true; + } else { + _is_master = true; + } + } + +// printf("ATTACH NOOOOOOW PID %d\n", getpid()); +// sleep(30); + + // set the sigterm handler on the child processes + if (!_is_master) { + struct sigaction sa; + memset (&sa, '\0', sizeof(sa)); + sa.sa_handler = ProcessManager::handle_sigterm; + + if (sigaction(SIGTERM, &sa, NULL) < 0) { + std::perror("sigaction failed"); + std::exit(1); + } + } + + if (cpu_pinning) { +#if defined(__APPLE__) + #ifndef NDEBUG + static bool affinity_warned = false; + if (is_master() & !affinity_warned) { + std::cout << "CPU affinity cannot be set on macOS" << std::endl; + affinity_warned = true; + } + #endif // NDEBUG +#elif defined(_WIN32) + #ifndef NDEBUG + if (is_master()) std::cerr << "WARNING: CPU affinity setting not implemented on Windows, continuing..." << std::endl; + #endif // NDEBUG +#else + cpu_set_t mask; + // zero all bits in mask + CPU_ZERO(&mask); + // set correct bit + std::size_t set_cpu; + if (is_master()) { + set_cpu = _N_workers + 1; + } else if (is_queue()) { + set_cpu = _N_workers; + } else { + set_cpu = _worker_id; + } + CPU_SET(set_cpu, &mask); + #ifndef NDEBUG + // sched_setaffinity returns 0 on success + if (sched_setaffinity(0, sizeof(mask), &mask) == -1) { + std::cerr << "WARNING: Could not set CPU affinity, continuing..." << std::endl; + } else { + std::cerr << "CPU affinity set to cpu " << set_cpu << " in process " << getpid() << std::endl; + } + #endif // NDEBUG +#endif + } + +#ifndef NDEBUG + identify_processes(); +#endif // NDEBUG + + initialized = true; +} + +bool ProcessManager::is_initialized() const { + return initialized; +} + + +void ProcessManager::terminate() noexcept { + try { + if (is_master() && is_initialized()) { +// JobManager::instance()->messenger().send_from_master_to_queue(M2Q::terminate); + shutdown_processes(); + } + } catch (const std::exception& e) { + std::cerr << "WARNING: something in ProcessManager::terminate threw an exception! Original exception message:\n" << e.what() << std::endl; + } +} + + +void ProcessManager::wait_for_sigterm_then_exit() { + if (!is_master()) { + while(!sigterm_received()) {} + std::_Exit(0); + } +} + + +//void ProcessManager::terminate_workers() { +// if (is_queue()) { +// for(std::size_t worker_ix = 0; worker_ix < _N_workers; ++worker_ix) { +// JobManager::instance()->messenger().send_from_queue_to_worker(worker_ix, Q2W::terminate); +// } +// JobManager::instance()->messenger().close_queue_worker_connections(); +// } +//} + + +int chill_wait() { + int status = 0; + pid_t pid; + do { + pid = wait(&status); + } while (-1 == pid && EINTR == errno); // retry on interrupted system call + + if (0 != status) { + if (WIFEXITED(status)) { + printf("exited, status=%d\n", WEXITSTATUS(status)); + } else if (WIFSIGNALED(status)) { + printf("killed by signal %d\n", WTERMSIG(status)); + } else if (WIFSTOPPED(status)) { + printf("stopped by signal %d\n", WSTOPSIG(status)); + } else if (WIFCONTINUED(status)) { + printf("continued\n"); + } + } + + if (-1 == pid) { + if (errno == ECHILD) { + printf("chill_wait: no children (got ECHILD error code from wait call), done\n"); + } else { + throw std::runtime_error(std::string("chill_wait: error in wait call: ") + strerror(errno) + std::string(", errno ") + std::to_string(errno)); + } + } + + return pid; +} + +void ProcessManager::shutdown_processes() { + if (is_master()) { + // terminate all children + std::unordered_set children; + children.insert(queue_pid); + kill(queue_pid, SIGTERM); + for (auto pid : worker_pids) { + kill(pid, SIGTERM); + children.insert(pid); + } + // then wait for them to actually die and clean out the zombies + while (!children.empty()) { + pid_t pid = chill_wait(); + children.erase(pid); + } + } + + initialized = false; +} + + +// Getters + +bool ProcessManager::is_master() const { + return _is_master; +} + +bool ProcessManager::is_queue() const { + return _is_queue; +} + +bool ProcessManager::is_worker() const { + return _is_worker; +} + +std::size_t ProcessManager::worker_id() const { + return _worker_id; +} + +std::size_t ProcessManager::N_workers() const { + return _N_workers; +} + + +// Debugging + +void ProcessManager::identify_processes() const { + // identify yourselves (for debugging) + if (_is_worker) { + printf("I'm a worker, PID %d\n", getpid()); + } else if (_is_master) { + printf("I'm master, PID %d\n", getpid()); + } else if (_is_queue) { + printf("I'm queue, PID %d\n", getpid()); + } else { + printf("I'm not master, queue or worker, weird! PID %d\n", getpid()); + } +} + +} // namespace MultiProcess +} // namespace RooFit diff --git a/roofit/multiprocess/src/Queue.cxx b/roofit/multiprocess/src/Queue.cxx new file mode 100644 index 0000000000000..5ea167812de59 --- /dev/null +++ b/roofit/multiprocess/src/Queue.cxx @@ -0,0 +1,224 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2016-2019, Netherlands eScience Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "RooFit/MultiProcess/JobManager.h" +#include "RooFit/MultiProcess/ProcessManager.h" +#include "RooFit/MultiProcess/Job.h" // complete Job object for JobManager::get_job_object() +#include "RooFit/MultiProcess/Queue.h" +#include "RooFit/MultiProcess/util.h" + +namespace RooFit { +namespace MultiProcess { + +// Have a worker ask for a task-message from the queue +bool Queue::pop(JobTask &job_task) +{ + if (_queue.empty()) { + return false; + } else { + job_task = _queue.front(); + _queue.pop(); + return true; + } +} + +// Enqueue a task +void Queue::add(JobTask job_task) +{ + if (JobManager::instance()->process_manager().is_master()) { +// not necessary, the job adds tasks, so it is already activated there; TODO: remove commented out lines +// if (!JobManager::instance()->is_activated()) { +// JobManager::instance()->activate(); +// } + JobManager::instance()->messenger().send_from_master_to_queue(M2Q::enqueue, job_task.first, job_task.second); + } else if (JobManager::instance()->process_manager().is_queue()) { + _queue.push(job_task); + } else { + throw std::logic_error("calling Communicator::to_master_queue from slave process"); + } +} + + +bool Queue::process_master_message(M2Q message) +{ + bool carry_on = true; + + switch (message) { + case M2Q::terminate: { + carry_on = false; + break; + } + + case M2Q::enqueue: { + // enqueue task + auto job_object_id = JobManager::instance()->messenger().receive_from_master_on_queue(); + auto task = JobManager::instance()->messenger().receive_from_master_on_queue(); + JobTask job_task(job_object_id, task); + add(job_task); + N_tasks++; + break; + } + + case M2Q::retrieve: { + // retrieve task results after queue is empty and all + // tasks have been completed + if (_queue.empty() && N_tasks_at_workers == 0 && N_tasks_completed == 0) { + JobManager::instance()->messenger().send_from_queue_to_master(Q2M::retrieve_rejected); // handshake message: no tasks enqueued, premature retrieve! + } else if (_queue.empty() && N_tasks_completed == N_tasks) { + JobManager::instance()->results_from_queue_to_master(); + // reset number of received and completed tasks + N_tasks = 0; + N_tasks_completed = 0; + } else { + JobManager::instance()->messenger().send_from_queue_to_master(Q2M::retrieve_later); // handshake message: tasks not done yet, try again + } + break; + } + + case M2Q::update_real: { +// auto get_time = []() { +// return std::chrono::duration_cast( +// std::chrono::high_resolution_clock::now().time_since_epoch()) +// .count(); +// }; +// auto t1 = get_time(); + auto job_id = JobManager::instance()->messenger().receive_from_master_on_queue(); + auto ix = JobManager::instance()->messenger().receive_from_master_on_queue(); + auto val = JobManager::instance()->messenger().receive_from_master_on_queue(); + auto is_constant = JobManager::instance()->messenger().receive_from_master_on_queue(); +// std::cout << "\ngot ix = " << ix << ", job_id = " << job_id << ", val = " << val << ", is_constant = " << is_constant << " in queue loop from M2Q::update_real" << std::endl; + for (std::size_t worker_ix = 0; worker_ix < JobManager::instance()->process_manager().N_workers(); ++worker_ix) { + JobManager::instance()->messenger().send_from_queue_to_worker(worker_ix, Q2W::update_real, job_id, ix, val, is_constant); + } +// auto t2 = get_time(); +// std::cout << "update_real on queue: " << (t2 - t1) / 1.e9 << "s (from " << t1 << " to " << t2 << "ns)" << std::endl; + break; + } + + case M2Q::update_bool: { + auto job_id = JobManager::instance()->messenger().receive_from_master_on_queue(); + auto ix = JobManager::instance()->messenger().receive_from_master_on_queue(); + auto value = JobManager::instance()->messenger().receive_from_master_on_queue(); + for (std::size_t worker_ix = 0; worker_ix < JobManager::instance()->process_manager().N_workers(); ++worker_ix) { + JobManager::instance()->messenger().send_from_queue_to_worker(worker_ix, Q2W::update_bool, job_id, ix, value); + } + } + } + + return carry_on; +} + + +void Queue::process_worker_message(std::size_t this_worker_id, W2Q message) +{ + switch (message) { + case W2Q::dequeue: { + // dequeue task + JobTask job_task; + bool popped = pop(job_task); + if (popped) { + // Note: below two commands should be run atomically for thread safety (if that ever becomes an issue) + JobManager::instance()->messenger().send_from_queue_to_worker(this_worker_id, Q2W::dequeue_accepted, job_task.first, job_task.second); + ++N_tasks_at_workers; + } else { + JobManager::instance()->messenger().send_from_queue_to_worker(this_worker_id, Q2W::dequeue_rejected); + } + break; + } + + case W2Q::send_result: { + // receive back task result + auto job_object_id = JobManager::instance()->messenger().receive_from_worker_on_queue(this_worker_id); + auto task = JobManager::instance()->messenger().receive_from_worker_on_queue(this_worker_id); + JobManager::get_job_object(job_object_id)->receive_task_result_on_queue(task, this_worker_id); + JobManager::instance()->messenger().send_from_queue_to_worker(this_worker_id, Q2W::result_received); + N_tasks_completed++; + --N_tasks_at_workers; + break; + } + } +} + + +void Queue::loop() +{ + assert(JobManager::instance()->process_manager().is_queue()); + bool carry_on = true; + ZeroMQPoller poller; + std::size_t mq_index; + std::tie(poller, mq_index) = JobManager::instance()->messenger().create_queue_poller(); + + // Before blocking SIGTERM, set the signal handler, so we can also check after blocking whether a signal occurred + // In our case, we already set it in the ProcessManager after forking to the queue and worker processes. + + sigset_t sigmask; + sigemptyset(&sigmask); + sigaddset(&sigmask, SIGTERM); + sigprocmask(SIG_BLOCK, &sigmask, &JobManager::instance()->messenger().ppoll_sigmask); + + // Before doing anything, check whether we have received a terminate signal while blocking signals! + // In this case, we also do that in the while condition. + while (!ProcessManager::sigterm_received() && carry_on) { + try { // watch for zmq_error from ppoll caused by SIGTERM from master + // poll: wait until status change (-1: infinite timeout) + auto poll_result = poller.ppoll(-1, &JobManager::instance()->messenger().ppoll_sigmask); +// JobManager::instance()->messenger().N_available_polled_results = poll_result.size(); + // then process incoming messages from sockets + for (auto readable_socket : poll_result) { + // message comes from the master/queue socket (first element): + if (readable_socket.first == mq_index) { + auto message = JobManager::instance()->messenger().receive_from_master_on_queue(); + carry_on = process_master_message(message); + // on terminate, also stop for-loop, no need to check other + // sockets anymore: + if (!carry_on) { + break; + } + } else { // from a worker socket + // TODO: dangerous assumption for this_worker_id, may become invalid if we allow multiple queue_loops on the same process! + auto this_worker_id = readable_socket.first - 1; // TODO: replace with a more reliable lookup + auto message = JobManager::instance()->messenger().receive_from_worker_on_queue(this_worker_id); + process_worker_message(this_worker_id, message); + } + } + } catch (ZMQ::ppoll_error_t& e) { + zmq_ppoll_error_response response; + try { + response = handle_zmq_ppoll_error(e); + } catch (std::logic_error& e) { + printf("queue loop got unhandleable ZMQ::ppoll_error_t\n"); + throw; + } + if (response == zmq_ppoll_error_response::abort) { + break; + } else if (response == zmq_ppoll_error_response::unknown_eintr) { + printf("EINTR in queue loop but no SIGTERM received, continuing\n"); + continue; + } else if (response == zmq_ppoll_error_response::retry) { + printf("EAGAIN from ppoll in queue loop, continuing\n"); + continue; + } + } catch (zmq::error_t& e) { + printf("unhandled zmq::error_t (not a ppoll_error_t) in queue loop with errno %d: %s\n", e.num(), e.what()); + throw; + } + } + + // clean up signal management modifications + sigprocmask(SIG_SETMASK, &JobManager::instance()->messenger().ppoll_sigmask, nullptr); +} + +} // namespace MultiProcess +} // namespace RooFit diff --git a/roofit/multiprocess/src/util.cxx b/roofit/multiprocess/src/util.cxx new file mode 100644 index 0000000000000..ff9aa14120eac --- /dev/null +++ b/roofit/multiprocess/src/util.cxx @@ -0,0 +1,130 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2016-2019, Netherlands eScience Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include // kill, SIGKILL +#include // cerr, and indirectly WNOHANG, EINTR, W* macros +#include // runtime_error +#include // waitpid +#include + +#include +#include + +namespace RooFit { +namespace MultiProcess { + +int wait_for_child(pid_t child_pid, bool may_throw, int retries_before_killing) +{ + int status = 0; + int patience = retries_before_killing; + pid_t tmp; + do { + if (patience-- < 1) { + ::kill(child_pid, SIGKILL); + } + tmp = waitpid(child_pid, &status, WNOHANG); + } while (tmp == 0 // child has not yet changed state, try again + || (-1 == tmp && EINTR == errno) // retry on interrupted system call + ); + + if (patience < 1) { + std::cout << "Had to send PID " << child_pid << " " << (-patience + 1) << " SIGKILLs"; + } + + if (0 != status) { + if (WIFEXITED(status)) { + printf("exited, status=%d\n", WEXITSTATUS(status)); + } else if (WIFSIGNALED(status)) { + printf("killed by signal %d\n", WTERMSIG(status)); + } else if (WIFSTOPPED(status)) { + printf("stopped by signal %d\n", WSTOPSIG(status)); + } else if (WIFCONTINUED(status)) { + printf("continued\n"); + } + } + + if (-1 == tmp && may_throw) + throw std::runtime_error(std::string("waitpid, errno ") + std::to_string(errno)); + + return status; +} + +zmq_ppoll_error_response handle_zmq_ppoll_error(ZMQ::ppoll_error_t &e) +{ + if ((e.num() == EINTR) && (ProcessManager::sigterm_received())) { + // valid EINTR, because we want to exit and kill the processes on SIGTERM + return zmq_ppoll_error_response::abort; + } else if (e.num() == EINTR) { + // on other EINTRs, we retry (mostly this happens in debuggers) + // printf("got EINTR but no SIGTERM received"); + return zmq_ppoll_error_response::unknown_eintr; + } else if (e.num() == EAGAIN) { + // This can happen from recv if ppoll initially gets a read-ready signal for a socket, + // but the received data does not pass the checksum test, so the socket becomes unreadable + // again or from non-blocking send if the socket becomes unwritable either due to the HWM + // being reached or the socket not being connected (anymore). The latter case usually means + // the connection has been severed from the other side, meaning it has probably been killed + // and in that case the next ppoll call will probably also receive a SIGTERM, ending the + // loop. In case something else is wrong, this message will print multiple times, which + // should be taken as a cue for writing a bug report :) + // TODO: handle this more rigorously + // printf("EAGAIN (from either send or receive), try %d\n", tries); + return zmq_ppoll_error_response::retry; + } else { + char buffer[512]; + snprintf(buffer, 512, "handle_zmq_ppoll_error is out of options to handle exception, caught ZMQ::ppoll_error_t had errno %d and text: %s\n", + e.num(), e.what()); + throw std::logic_error(buffer); + } +} + +// returns a tuple containing first the poll result and second a boolean flag that tells the caller whether it should +// abort the enclosing loop +std::tuple>, bool> +careful_ppoll(ZeroMQPoller &poller, const sigset_t &ppoll_sigmask, std::size_t max_tries) +{ + std::size_t tries = 0; + std::vector> poll_result; + bool abort = true; + bool carry_on = true; + while (carry_on && (tries++ < max_tries)) { + if (tries > 1) { + printf("careful_ppoll try %lu\n", tries); + } + try { // watch for zmq_error from ppoll caused by SIGTERM from master + poll_result = poller.ppoll(-1, &ppoll_sigmask); + abort = false; + carry_on = false; + } catch (ZMQ::ppoll_error_t &e) { + auto response = handle_zmq_ppoll_error(e); + if (response == zmq_ppoll_error_response::abort) { + break; + } else if (response == zmq_ppoll_error_response::unknown_eintr) { + printf("EINTR in careful_ppoll but no SIGTERM received, try %lu\n", tries); + continue; + } else if (response == zmq_ppoll_error_response::retry) { + printf("EAGAIN in careful_ppoll (from either send or receive), try %lu\n", tries); + continue; + } + } + } + + if (tries == max_tries) { + printf("careful_ppoll reached maximum number of tries, %lu, please report as a bug\n", tries); + } + return {poll_result, abort}; +} + +} // namespace MultiProcess +} // namespace RooFit \ No newline at end of file diff --git a/roofit/multiprocess/src/worker.cxx b/roofit/multiprocess/src/worker.cxx new file mode 100644 index 0000000000000..86191dfd694a4 --- /dev/null +++ b/roofit/multiprocess/src/worker.cxx @@ -0,0 +1,228 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2016-2019, Netherlands eScience Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include // getpid, pid_t +#include // EINTR +#include +#include // sigprocmask etc + +#include "RooFit/MultiProcess/JobManager.h" +#include "RooFit/MultiProcess/types.h" +#include "RooFit/MultiProcess/Messenger.h" +#include "RooFit/MultiProcess/Job.h" +#include "RooFit/MultiProcess/util.h" + +#include "RooFit/MultiProcess/worker.h" + +namespace RooFit { +namespace MultiProcess { + +static bool worker_loop_running = false; + +bool is_worker_loop_running() +{ + return worker_loop_running; +} + +bool process_queue_message(Q2W message_q2w, bool &dequeue_acknowledged) +{ + bool carry_on = true; + + auto get_time = []() { + return std::chrono::duration_cast( + std::chrono::high_resolution_clock::now().time_since_epoch()) + .count(); + }; + decltype(get_time()) t1 = 0, t2 = 0; + + + switch (message_q2w) { + case Q2W::terminate: { + carry_on = false; + break; + } + + case Q2W::dequeue_rejected: { + t2 = get_time(); +// std::cout +// << "no work: worker " << JobManager::instance()->process_manager().worker_id() << " asked at " << t1 +// << " and got rejected at " << t2 << std::endl; + + dequeue_acknowledged = true; + break; + } + case Q2W::dequeue_accepted: { + dequeue_acknowledged = true; + auto job_id = JobManager::instance()->messenger().receive_from_queue_on_worker(); + auto task = JobManager::instance()->messenger().receive_from_queue_on_worker(); + + JobManager::get_job_object(job_id)->evaluate_task(task); + + // std::cout +// << "task done: worker " << JobManager::instance()->process_manager().worker_id() << " asked at " << t1 << ", started at " +// << t2 << " and finished at " << t3 << std::endl; + +// { +// // old +// JobManager::instance()->messenger().send_from_worker_to_queue(W2Q::send_result, job_id, task); +// JobManager::get_job_object(job_id)->send_back_task_result_from_worker(task); +// +// message_q2w = JobManager::instance()->messenger().receive_from_queue_on_worker(); +// if (message_q2w != Q2W::result_received) { +// std::cerr << "worker " << getpid() +// << " sent result, but did not receive Q2W::result_received handshake! Got " << message_q2w +// << " instead." << std::endl; +// throw std::runtime_error(""); +// } + +// } + { + // new +// JobManager::instance()->messenger().send_from_worker_to_master(job_id, task); + JobManager::get_job_object(job_id)->send_back_task_result_from_worker(task); + } + + // std::cout << "task done: worker " << JobManager::instance()->process_manager().worker_id() << " sent back results" << std::endl; + +// std::cout << "task done: worker " << JobManager::instance()->process_manager().worker_id() << " finished dequeue_accepted" << std::endl; + break; + } + + // previously this was non-work mode + + case Q2W::update_real: { + t1 = get_time(); + + auto job_id = JobManager::instance()->messenger().receive_from_queue_on_worker(); + auto ix = JobManager::instance()->messenger().receive_from_queue_on_worker(); + auto val = JobManager::instance()->messenger().receive_from_queue_on_worker(); + bool is_constant = JobManager::instance()->messenger().receive_from_queue_on_worker(); + JobManager::get_job_object(job_id)->update_real(ix, val, is_constant); + + t2 = get_time(); +// std::cout +// << "update_real on worker " << JobManager::instance()->process_manager().worker_id() << ": " << (t2 - t1) / 1.e9 +// << "s (from " << t1 << " to " << t2 << "ns)" << std::endl; + + break; + } + + case Q2W::update_bool: { + auto job_id = JobManager::instance()->messenger().receive_from_queue_on_worker(); + auto ix = JobManager::instance()->messenger().receive_from_queue_on_worker(); + auto value = JobManager::instance()->messenger().receive_from_queue_on_worker(); + JobManager::get_job_object(job_id)->update_bool(ix, value); + + break; + } + + case Q2W::result_received: { + std::cerr << "In worker_loop: " << message_q2w + << " message received, but should only be received as handshake!" << std::endl; + break; + } + } + + return carry_on; +} + +void worker_loop() +{ + assert(JobManager::instance()->process_manager().is_worker()); + worker_loop_running = true; + bool carry_on = true; +// Task task; + std::size_t job_id; + Q2W message_q2w; + + // use a flag to not ask twice + bool dequeue_acknowledged = true; + + ZeroMQPoller poller; + std::size_t mw_sub_index; + + std::tie(poller, mw_sub_index) = JobManager::instance()->messenger().create_worker_poller(); + + // Before blocking SIGTERM, set the signal handler, so we can also check after blocking whether a signal occurred + // In our case, we already set it in the ProcessManager after forking to the queue and worker processes. + + sigset_t sigmask; + sigemptyset(&sigmask); + sigaddset(&sigmask, SIGTERM); + sigprocmask(SIG_BLOCK, &sigmask, &JobManager::instance()->messenger().ppoll_sigmask); + + // Before doing anything, check whether we have received a terminate signal while blocking signals! + // In this case, we also do that in the while condition. + while (!ProcessManager::sigterm_received() && carry_on) { + try { // watch for error from ppoll (which is called inside receive functions) caused by SIGTERM from master + + // try to dequeue a task + if (dequeue_acknowledged) { // don't ask twice + JobManager::instance()->messenger().send_from_worker_to_queue(W2Q::dequeue); + dequeue_acknowledged = false; + } + +// // receive handshake +// message_q2w = JobManager::instance()->messenger().receive_from_queue_on_worker(); + // receive handshake, other message or update from sub socket + auto poll_result = poller.ppoll(-1, &JobManager::instance()->messenger().ppoll_sigmask); +// JobManager::instance()->messenger().N_available_polled_results = poll_result.size(); + // then process incoming messages from sockets + for (auto readable_socket : poll_result) { + // message comes from the master/queue socket (first element): + if (readable_socket.first == mw_sub_index) { + job_id = JobManager::instance()->messenger().receive_from_master_on_worker(); + JobManager::get_job_object(job_id)->update_state(); + } else { // from queue socket + message_q2w = JobManager::instance()->messenger().receive_from_queue_on_worker(); + carry_on = process_queue_message(message_q2w, dequeue_acknowledged); + // on terminate, also stop for-loop, no need to check other + // sockets anymore: + if (!carry_on) { + break; + } + } + } + + } catch (ZMQ::ppoll_error_t& e) { + zmq_ppoll_error_response response; + try { + response = handle_zmq_ppoll_error(e); + } catch (std::logic_error& e) { + printf("worker loop at PID %d got unhandleable ZMQ::ppoll_error_t\n", getpid()); + throw; + } + if (response == zmq_ppoll_error_response::abort) { + break; + } else if (response == zmq_ppoll_error_response::unknown_eintr) { + printf("EINTR in worker loop at PID %d but no SIGTERM received, continuing\n", getpid()); + continue; + } else if (response == zmq_ppoll_error_response::retry) { + printf("EAGAIN from ppoll in worker loop at PID %d, continuing\n", getpid()); + continue; + } + } catch (zmq::error_t& e) { + printf("unhandled zmq::error_t (not a ppoll_error_t) in worker loop at PID %d with errno %d: %s\n", getpid(), e.num(), e.what()); + throw; + } + } + // clean up signal management modifications + sigprocmask(SIG_SETMASK, &JobManager::instance()->messenger().ppoll_sigmask, nullptr); + + worker_loop_running = false; +} + +} // namespace MultiProcess +} // namespace RooFit diff --git a/roofit/multiprocess/test/CMakeLists.txt b/roofit/multiprocess/test/CMakeLists.txt new file mode 100644 index 0000000000000..7cc22c78e4c7d --- /dev/null +++ b/roofit/multiprocess/test/CMakeLists.txt @@ -0,0 +1,13 @@ +# @author Patrick Bos, NL eScience Center, 2019 + +#find_package(ZeroMQ REQUIRED) + +add_library(RooFit_multiprocess_testing_utils INTERFACE) +target_link_libraries(RooFit_multiprocess_testing_utils INTERFACE RooFitCore RooBatchCompute) + +ROOT_ADD_GTEST(test_RooFitMultiProcess_Job test_Job.cxx LIBRARIES RooFitMultiProcess ${ZeroMQ_LIBRARY} RooFit_multiprocess_testing_utils) +ROOT_ADD_GTEST(test_RooFitMultiProcess_ProcessManager test_ProcessManager.cxx LIBRARIES RooFitMultiProcess ${ZeroMQ_LIBRARY}) +ROOT_ADD_GTEST(test_RooFitMultiProcess_Messenger test_Messenger.cxx LIBRARIES RooFitMultiProcess ${ZeroMQ_LIBRARY}) + +#target_include_directories(test_RooFitMultiProcess_Job PUBLIC ${ZeroMQ_INCLUDE_DIR}) +#target_include_directories(test_RooFitMultiProcess_Messenger PUBLIC ${ZeroMQ_INCLUDE_DIR}) diff --git a/roofit/multiprocess/test/test_Job.cxx b/roofit/multiprocess/test/test_Job.cxx new file mode 100644 index 0000000000000..6b71f7e3369fa --- /dev/null +++ b/roofit/multiprocess/test/test_Job.cxx @@ -0,0 +1,252 @@ +/***************************************************************************** + * Project: RooFit * + * Package: RooFitCore * + * @(#)root/roofitcore:$Id$ + * Authors: * + * PB, Patrick Bos, NL eScience Center, p.bos@esciencecenter.nl * + * IP, Inti Pelupessy, NL eScience Center, i.pelupessy@esciencecenter.nl * + * VC, Vince Croft, DIANA / NYU, vincent.croft@cern.ch * + * * + * Redistribution and use in source and binary forms, * + * with or without modification, are permitted according to the terms * + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) * + *****************************************************************************/ + +#include +#include + +#include +#include // JobTask +// needed to complete type returned from... +#include // ... Job::get_manager() +#include // ... JobManager::process_manager() +#include // ... JobManager::queue() + +#include "gtest/gtest.h" +#include "utils.h" + +class xSquaredPlusBVectorSerial { +public: + xSquaredPlusBVectorSerial(double b, std::vector x_init) : _b(b), x(std::move(x_init)), result(x.size()) {} + + void evaluate() + { + // call evaluate_task for each task + for (std::size_t ix = 0; ix < x.size(); ++ix) { + result[ix] = std::pow(x[ix], 2) + _b; + } + } + + std::vector get_result() + { + evaluate(); + return result; + } + + // for simplicity of the example (avoiding getters/setters) we make data members public as well + double _b; + std::vector x; + std::vector result; +}; + +class xSquaredPlusBVectorParallel : public RooFit::MultiProcess::Job { +public: + xSquaredPlusBVectorParallel(xSquaredPlusBVectorSerial* serial) + : serial(serial) + { + } + + void evaluate() + { + if (get_manager()->process_manager().is_master()) { + // master fills queue with tasks + for (std::size_t task_id = 0; task_id < serial->x.size(); ++task_id) { + RooFit::MultiProcess::JobTask job_task(id, task_id); + get_manager()->queue().add(job_task); + } + + // wait for task results back from workers to master + gather_worker_results(); + } + } + + std::vector get_result() + { + evaluate(); + return serial->result; + } + + void update_real(std::size_t /*ix*/, double val, bool /*is_constant*/) override { + if (get_manager()->process_manager().is_worker()) { + serial->_b = val; + } + } + + void update_bool(std::size_t /*ix*/, bool /*value*/) override { + // pass + } + + // -- BEGIN plumbing -- + + void receive_task_result_on_queue(std::size_t task, std::size_t worker_id) override { + double result = get_manager()->messenger().template receive_from_worker_on_queue(worker_id); + serial->result[task] = result; + } + + void send_back_task_result_from_worker(std::size_t task) override { + get_manager()->messenger().template send_from_worker_to_queue(serial->result[task]); + } + + void send_back_results_from_queue_to_master() override { +// get_manager()->messenger().send_from_queue_to_master(serial->result.size()); + for (std::size_t task_ix = 0ul; task_ix < serial->result.size(); ++task_ix) { + get_manager()->messenger().send_from_queue_to_master(task_ix, serial->result[task_ix]); + } + } + + void clear_results() override { + // no need to clear any results cache since we just reuse the result vector on the queue + } + + void receive_results_on_master() override { +// std::size_t N_job_tasks = get_manager()->messenger().template receive_from_queue_on_master(); +// for (std::size_t task_ix = 0ul; task_ix < N_job_tasks; ++task_ix) { + for (std::size_t task_ix = 0ul; task_ix < serial->result.size(); ++task_ix) { + std::size_t task_id = get_manager()->messenger().template receive_from_queue_on_master(); + serial->result[task_id] = get_manager()->messenger().template receive_from_queue_on_master(); + } + } + + bool receive_task_result_on_master(const zmq::message_t & /*message*/) override { + // TODO: implement; this no-op placeholder is just to make everything compile first so I can check whether merge was successful + return true; + } + + // -- END plumbing -- + + +private: + void evaluate_task(std::size_t task) override + { + assert(get_manager()->process_manager().is_worker()); + serial->result[task] = std::pow(serial->x[task], 2) + serial->_b; + } + + xSquaredPlusBVectorSerial* serial; +}; + +class TestMPJob : public ::testing::TestWithParam { + // You can implement all the usual fixture class members here. + // To access the test parameter, call GetParam() from class + // TestWithParam. +}; + +TEST_P(TestMPJob, singleJobGetResult) +{ + // Simple test case: calculate x^2 + b, where x is a vector. This case does + // both a simple calculation (squaring the input vector x) and represents + // handling of state updates in b. + std::vector x{0, 1, 2, 3}; + double b_initial = 3.; + + // start serial test + + xSquaredPlusBVectorSerial x_sq_plus_b(b_initial, x); + + auto y = x_sq_plus_b.get_result(); + std::vector y_expected{3, 4, 7, 12}; + + EXPECT_EQ(Hex(y[0]), Hex(y_expected[0])); + EXPECT_EQ(Hex(y[1]), Hex(y_expected[1])); + EXPECT_EQ(Hex(y[2]), Hex(y_expected[2])); + EXPECT_EQ(Hex(y[3]), Hex(y_expected[3])); + + std::size_t NumCPU = GetParam(); + + // start parallel test + + xSquaredPlusBVectorParallel x_sq_plus_b_parallel(&x_sq_plus_b); + RooFit::MultiProcess::JobManager::default_N_workers = NumCPU; + + auto y_parallel = x_sq_plus_b_parallel.get_result(); + + EXPECT_EQ(Hex(y_parallel[0]), Hex(y_expected[0])); + EXPECT_EQ(Hex(y_parallel[1]), Hex(y_expected[1])); + EXPECT_EQ(Hex(y_parallel[2]), Hex(y_expected[2])); + EXPECT_EQ(Hex(y_parallel[3]), Hex(y_expected[3])); +} + + +TEST_P(TestMPJob, multiJobGetResult) +{ + // Simple test case: calculate x^2 + b, where x is a vector. This case does + // both a simple calculation (squaring the input vector x) and represents + // handling of state updates in b. + std::vector x{0, 1, 2, 3}; + double b_initial = 3.; + + std::vector y_expected{3, 4, 7, 12}; + + std::size_t NumCPU = GetParam(); + + // define jobs + xSquaredPlusBVectorSerial x_sq_plus_b(b_initial, x); + xSquaredPlusBVectorSerial x_sq_plus_b2(b_initial + 1, x); + xSquaredPlusBVectorParallel x_sq_plus_b_parallel(&x_sq_plus_b); + xSquaredPlusBVectorParallel x_sq_plus_b_parallel2(&x_sq_plus_b2); + RooFit::MultiProcess::JobManager::default_N_workers = NumCPU; + + // do stuff + auto y_parallel = x_sq_plus_b_parallel.get_result(); + auto y_parallel2 = x_sq_plus_b_parallel2.get_result(); + + EXPECT_EQ(Hex(y_parallel[0]), Hex(y_expected[0])); + EXPECT_EQ(Hex(y_parallel[1]), Hex(y_expected[1])); + EXPECT_EQ(Hex(y_parallel[2]), Hex(y_expected[2])); + EXPECT_EQ(Hex(y_parallel[3]), Hex(y_expected[3])); + + EXPECT_EQ(Hex(y_parallel2[0]), Hex(y_expected[0] + 1)); + EXPECT_EQ(Hex(y_parallel2[1]), Hex(y_expected[1] + 1)); + EXPECT_EQ(Hex(y_parallel2[2]), Hex(y_expected[2] + 1)); + EXPECT_EQ(Hex(y_parallel2[3]), Hex(y_expected[3] + 1)); +} + + +// TODO: implement update_real test! + +//TEST_P(TestMPJob, singleJobUpdateReal) +//{ +//// Simple test case: calculate x^2 + b, where x is a vector. This case does +//// both a simple calculation (squaring the input vector x) and represents +//// handling of state updates in b. +//std::vector x{0, 1, 2, 3}; +//double b_initial = 3.; +// +//// start serial test +// +//xSquaredPlusBVectorSerial x_sq_plus_b(b_initial, x); +// +//auto y = x_sq_plus_b.get_result(); +//std::vector y_expected{3, 4, 7, 12}; +// +//EXPECT_EQ(Hex(y[0]), Hex(y_expected[0])); +//EXPECT_EQ(Hex(y[1]), Hex(y_expected[1])); +//EXPECT_EQ(Hex(y[2]), Hex(y_expected[2])); +//EXPECT_EQ(Hex(y[3]), Hex(y_expected[3])); +// +//std::size_t NumCPU = GetParam(); +// +//// start parallel test +// +//xSquaredPlusBVectorParallel x_sq_plus_b_parallel(NumCPU, &x_sq_plus_b); +// +//auto y_parallel = x_sq_plus_b_parallel.get_result(); +// +//EXPECT_EQ(Hex(y_parallel[0]), Hex(y_expected[0])); +//EXPECT_EQ(Hex(y_parallel[1]), Hex(y_expected[1])); +//EXPECT_EQ(Hex(y_parallel[2]), Hex(y_expected[2])); +//EXPECT_EQ(Hex(y_parallel[3]), Hex(y_expected[3])); +//} + + +INSTANTIATE_TEST_SUITE_P(NumberOfWorkerProcesses, TestMPJob, ::testing::Values(1, 2, 3)); diff --git a/roofit/multiprocess/test/test_Messenger.cxx b/roofit/multiprocess/test/test_Messenger.cxx new file mode 100644 index 0000000000000..49b9936f4e30f --- /dev/null +++ b/roofit/multiprocess/test/test_Messenger.cxx @@ -0,0 +1,116 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2016-2019, Netherlands eScience Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "RooFit/MultiProcess/Messenger.h" +#include // ... JobManager::process_manager() + +#include "gtest/gtest.h" + +TEST(TestMPMessenger, Connections) +{ + RooFit::MultiProcess::ProcessManager pm(2); + RooFit::MultiProcess::Messenger messenger(pm); + messenger.test_connections(pm); +} + +TEST(TestMPMessenger, ConnectionsManualExit) +{ + // the point of this test is to see whether clean-up of ZeroMQ resources is done properly without calling any + // destructors (which is what happens when you call _Exit() instead of regularly ending the program by reaching the + // end of main()). + RooFit::MultiProcess::ProcessManager pm( 2); + RooFit::MultiProcess::Messenger messenger(pm); + messenger.test_connections(pm); + if (!pm.is_master()) { + std::_Exit(0); + } +} + + +#include + +TEST(TestMPMessenger, SigStop) +{ + // For some reason, I cannot get my debugger (lldb) to play nicely with ZeroMQ; every time I pause a process, it + // says it crashed because of SIGSTOP, but SIGSTOP should just pause it... let's see whether we can reproduce this + // behavior here and, if possible, fix it. + RooFit::MultiProcess::ProcessManager pm(2); + RooFit::MultiProcess::Messenger messenger(pm); +// messenger.test_connections(pm); + + // all this works fine without a sigmask, maybe the sigmask is the issue? let's try: + + + if (pm.is_master()) { + printf("first send message to queue...\n"); + messenger.send_from_master_to_queue(1); + printf("sleep for 2 seconds...\n"); + sleep(2); + printf("SIGSTOPping all children...\n"); + kill(pm.get_queue_pid(), SIGSTOP); + kill(pm.get_worker_pids()[0], SIGSTOP); + kill(pm.get_worker_pids()[1], SIGSTOP); + printf("send another message to queue...\n"); + messenger.send_from_master_to_queue(2); + printf("sleep for 2 seconds...\n"); + sleep(2); + printf("SIGCONT queue and worker 1...\n"); + kill(pm.get_queue_pid(), SIGCONT); + kill(pm.get_worker_pids()[0], SIGCONT); + printf("sleep for 2 seconds...\n"); + sleep(2); + printf("SIGCONT worker 2...\n"); + kill(pm.get_worker_pids()[1], SIGCONT); + EXPECT_EQ(messenger.receive_from_queue_on_master(), 3); + } + + if (pm.is_queue()) { + auto number = messenger.receive_from_master_on_queue(); + printf("received %d on queue\n", number); + number = messenger.receive_from_master_on_queue(); + printf("received %d on queue\n", number); + messenger.send_from_queue_to_master(3); + } + + if (!pm.is_master()) { + while (!pm.sigterm_received()) {} + } +} + +TEST(TestMPMessenger, StressPubSub) +{ + std::size_t N_workers = 64; + RooFit::MultiProcess::ProcessManager pm(N_workers); + RooFit::MultiProcess::Messenger messenger(pm); + + if (pm.is_master()) { + messenger.publish_from_master_to_workers("bert"); + std::size_t ernies = 0; + for (; ernies < N_workers; ++ernies) { + auto receipt = messenger.receive_from_worker_on_master(); + if (receipt != "ernie") { + printf("whoops, got %s instead of ernie!\n", receipt.c_str()); + FAIL(); + } + } + EXPECT_EQ(ernies, N_workers); + } else if (pm.is_worker()) { + auto receipt = messenger.receive_from_master_on_worker(); + if (receipt == "bert") { + messenger.send_from_worker_to_master("ernie"); + } else { + printf("no bert on worker %lu\n", pm.worker_id()); + } + } +} diff --git a/roofit/multiprocess/test/test_ProcessManager.cxx b/roofit/multiprocess/test/test_ProcessManager.cxx new file mode 100644 index 0000000000000..4660657e4c720 --- /dev/null +++ b/roofit/multiprocess/test/test_ProcessManager.cxx @@ -0,0 +1,57 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2016-2019, Netherlands eScience Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include // ... JobManager::process_manager() + +#include "gtest/gtest.h" + +TEST(TestMPProcessManager, birthAndDeath) +{ + auto pm = RooFit::MultiProcess::ProcessManager(2); +} + +// disabled: it is not working anymore (16 March 2021), due to changes introduced to fix Messenger::test_connections +TEST(TestMPProcessManager, DISABLED_multiBirth) +{ + // This test doesn't actually represent a current usecase in RooFit, but let's + // showcase the possibility anyway for future reference. + + // first create a regular pm like in the birthAndDeath test + auto pm = RooFit::MultiProcess::ProcessManager(2); + // then from each forked node, spin up another set of workers+queue! + auto pm2 = RooFit::MultiProcess::ProcessManager(2); +} + + +TEST(TestMPProcessManager, checkState) +{ + std::size_t N_workers = 2; + auto master_pid = getpid(); + auto pm = RooFit::MultiProcess::ProcessManager(N_workers); + ASSERT_TRUE(pm.is_initialized()); + ASSERT_EQ(pm.N_workers(), N_workers); + + if (pm.is_master()) { + EXPECT_EQ(getpid(), master_pid); + EXPECT_FALSE(pm.is_queue()); + EXPECT_FALSE(pm.is_worker()); + } else if (pm.is_queue()) { + EXPECT_FALSE(pm.is_master()); + EXPECT_FALSE(pm.is_worker()); + } else if (pm.is_worker()) { + EXPECT_FALSE(pm.is_master()); + EXPECT_FALSE(pm.is_queue()); + EXPECT_TRUE(pm.worker_id() == 0 || pm.worker_id() == 1); + } +} \ No newline at end of file diff --git a/roofit/multiprocess/test/test_worker.cxx b/roofit/multiprocess/test/test_worker.cxx new file mode 100644 index 0000000000000..949527a89bb87 --- /dev/null +++ b/roofit/multiprocess/test/test_worker.cxx @@ -0,0 +1,29 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2016-2019, Netherlands eScience Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "RooFit/MultiProcess/worker.h" + +#include "gtest/gtest.h" + +// TODO: implement something along these lines (the following is pseudocode) +//TEST(worker, loop) +//{ +// do_fork(); +// if (on_master()) { +// wait(); +// terminate_worker(); +// } else if (on_worker()) { +// worker_loop(); +// } +//} diff --git a/roofit/multiprocess/test/utils.h b/roofit/multiprocess/test/utils.h new file mode 100644 index 0000000000000..f4f72976e7226 --- /dev/null +++ b/roofit/multiprocess/test/utils.h @@ -0,0 +1,171 @@ +/***************************************************************************** + * Project: RooFit * + * Package: RooFitCore * + * @(#)root/roofitcore:$Id$ + * Authors: * + * PB, Patrick Bos, NL eScience Center, p.bos@esciencecenter.nl * + * * + * Redistribution and use in source and binary forms, * + * with or without modification, are permitted according to the terms * + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) * + *****************************************************************************/ + +#ifndef ROOT_ROOFIT_MultiProcess_tests_utils_h +#define ROOT_ROOFIT_MultiProcess_tests_utils_h + +#include + +#include // make_unique + +#include "RooWorkspace.h" +#include "RooRandom.h" +#include "RooAddPdf.h" +#include "RooDataSet.h" +#include "RooRealVar.h" // for the dynamic cast to have a complete type + + +std::tuple, std::unique_ptr> +generate_1D_gaussian_pdf_nll(RooWorkspace &w, unsigned long N_events) { + w.factory("Gaussian::g(x[-5,5],mu[0,-3,3],sigma[1])"); + + RooAbsPdf *pdf = w.pdf("g"); + RooRealVar *mu = w.var("mu"); + + RooDataSet *data = pdf->generate(RooArgSet(*w.var("x")), N_events); + mu->setVal(-2.9); + + std::unique_ptr nll {pdf->createNLL(*data)}; + + // save initial values for the start of all minimizations + std::unique_ptr values = std::make_unique(*mu, *pdf, *nll, "values"); + + return std::make_tuple(std::move(nll), std::move(values)); +} + +// return two unique_ptrs, the first because nll is a pointer, +// the second because RooArgSet doesn't have a move ctor +std::tuple, std::unique_ptr> +generate_ND_gaussian_pdf_nll(RooWorkspace &w, unsigned int n, unsigned long N_events) { + RooArgSet obs_set; + + // create gaussian parameters + double mean[n], sigma[n]; + for (unsigned ix = 0; ix < n; ++ix) { + mean[ix] = RooRandom::randomGenerator()->Gaus(0, 2); + sigma[ix] = 0.1 + abs(RooRandom::randomGenerator()->Gaus(0, 2)); + } + + // create gaussians and also the observables and parameters they depend on + for (unsigned ix = 0; ix < n; ++ix) { + std::ostringstream os; + os << "Gaussian::g" << ix + << "(x" << ix << "[-10,10]," + << "m" << ix << "[" << mean[ix] << ",-10,10]," + << "s" << ix << "[" << sigma[ix] << ",0.1,10])"; + w.factory(os.str().c_str()); + } + + // create uniform background signals on each observable + for (unsigned ix = 0; ix < n; ++ix) { + { + std::ostringstream os; + os << "Uniform::u" << ix << "(x" << ix << ")"; + w.factory(os.str().c_str()); + } + + // gather the observables in a list for data generation below + { + std::ostringstream os; + os << "x" << ix; + obs_set.add(*w.arg(os.str().c_str())); + } + } + + RooArgSet pdf_set = w.allPdfs(); + + // create event counts for all pdfs + RooArgSet count_set; + + // ... for the gaussians + for (unsigned ix = 0; ix < n; ++ix) { + std::stringstream os, os2; + os << "Nsig" << ix; + os2 << "#signal events comp " << ix; + RooRealVar a(os.str().c_str(), os2.str().c_str(), N_events/10, 0., 10*N_events); + w.import(a); + // gather in count_set + count_set.add(*w.arg(os.str().c_str())); + } + // ... and for the uniform background components + for (unsigned ix = 0; ix < n; ++ix) { + std::stringstream os, os2; + os << "Nbkg" << ix; + os2 << "#background events comp " << ix; + RooRealVar a(os.str().c_str(), os2.str().c_str(), N_events/10, 0., 10*N_events); + w.import(a); + // gather in count_set + count_set.add(*w.arg(os.str().c_str())); + } + + RooAddPdf* sum = new RooAddPdf("sum", "gaussians+uniforms", pdf_set, count_set); + w.import(*sum); // keep sum around after returning + + // --- Generate a toyMC sample from composite PDF --- + RooDataSet *data = sum->generate(obs_set, N_events); + + std::unique_ptr nll {sum->createNLL(*data)}; + + // set values randomly so that they actually need to do some fitting + for (unsigned ix = 0; ix < n; ++ix) { + { + std::ostringstream os; + os << "m" << ix; + dynamic_cast(w.arg(os.str().c_str()))->setVal(RooRandom::randomGenerator()->Gaus(0, 2)); + } + { + std::ostringstream os; + os << "s" << ix; + dynamic_cast(w.arg(os.str().c_str()))->setVal(0.1 + abs(RooRandom::randomGenerator()->Gaus(0, 2))); + } + } + + // gather all values of parameters, pdfs and nll here for easy + // saving and restoring + std::unique_ptr all_values = std::make_unique(pdf_set, count_set, "all_values"); + all_values->add(*nll); + all_values->add(*sum); + for (unsigned ix = 0; ix < n; ++ix) { + { + std::ostringstream os; + os << "m" << ix; + all_values->add(*w.arg(os.str().c_str())); + } + { + std::ostringstream os; + os << "s" << ix; + all_values->add(*w.arg(os.str().c_str())); + } + } + + return std::make_tuple(std::move(nll), std::move(all_values)); +} + + +class Hex { +public: + explicit Hex(double n) : number_(n) {} + operator double() const { return number_; } + bool operator==(const Hex& other) { + return double(*this) == double(other); + } + +private: + double number_; +}; + +::std::ostream& operator<<(::std::ostream& os, const Hex& hex) { + return os << std::hexfloat << double(hex) << std::defaultfloat; // whatever needed to print bar to os +} + + +#endif //ROOT_ROOFIT_MultiProcess_tests_utils_h