From 25ecebf00d8b6f53f9f7b210a6532347f1505c84 Mon Sep 17 00:00:00 2001 From: "E. G. Patrick Bos" Date: Wed, 9 Jun 2021 15:44:52 +0200 Subject: [PATCH 01/15] Add builtin ZeroMQ for RooFit::MultiProcess (later) --- roofit/CMakeLists.txt | 1 + roofit/builtin_zeromq/CMakeLists.txt | 35 ++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 roofit/builtin_zeromq/CMakeLists.txt diff --git a/roofit/CMakeLists.txt b/roofit/CMakeLists.txt index 890df0340dde2..01478c67ca3d5 100644 --- a/roofit/CMakeLists.txt +++ b/roofit/CMakeLists.txt @@ -5,6 +5,7 @@ # For the list of contributors see $ROOTSYS/README/CREDITS. add_subdirectory(batchcompute) +add_subdirectory(builtin_zeromq) add_subdirectory(roofitcore) add_subdirectory(roofit) if(mathmore) diff --git a/roofit/builtin_zeromq/CMakeLists.txt b/roofit/builtin_zeromq/CMakeLists.txt new file mode 100644 index 0000000000000..8407e20d1deae --- /dev/null +++ b/roofit/builtin_zeromq/CMakeLists.txt @@ -0,0 +1,35 @@ +include(ExternalProject) + +set(ZeroMQ_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/BUILTIN_ZeroMQ-prefix) +set(ZeroMQ_LIBNAME ${CMAKE_STATIC_LIBRARY_PREFIX}zmq${CMAKE_STATIC_LIBRARY_SUFFIX}) + +set(ZeroMQ_LIBRARY ${ZeroMQ_PREFIX}/lib/${ZeroMQ_LIBNAME} CACHE INTERNAL "" FORCE) +set(ZeroMQ_LIBRARIES ${ZeroMQ_LIBRARY} CACHE INTERNAL "" FORCE) + +ExternalProject_Add(BUILTIN_ZeroMQ + URL https://github.com/zeromq/libzmq/archive/refs/tags/v4.3.2.tar.gz + CMAKE_ARGS + -DCMAKE_INSTALL_PREFIX= + -DWITH_PERF_TOOL=OFF + -DZMQ_BUILD_TESTS=OFF + -DPOLLER=select + BUILD_BYPRODUCTS ${ZeroMQ_LIBRARIES} + ) + +set(ZeroMQ_FOUND TRUE CACHE BOOL "" FORCE) + +set(ZeroMQ_INCLUDE_DIR ${ZeroMQ_PREFIX}/include CACHE INTERNAL "" FORCE) +set(ZeroMQ_INCLUDE_DIRS ${ZeroMQ_PREFIX}/include CACHE INTERNAL "" FORCE) + +add_library(ZeroMQ INTERFACE) +target_include_directories(ZeroMQ INTERFACE $) +target_link_libraries(ZeroMQ INTERFACE $) +add_dependencies(ZeroMQ BUILTIN_ZeroMQ) + +add_library(ZeroMQ::ZeroMQ ALIAS ZeroMQ) + +set(ZeroMQ_DIR ${ZeroMQ_PREFIX}/share/cmake/ZeroMQ CACHE INTERNAL "" FORCE) + +# also need source and build directories for ppoll in RooFitZMQ +set(ZeroMQ_SOURCE_DIR ${ZeroMQ_PREFIX}/src/BUILTIN_ZeroMQ/src CACHE INTERNAL "" FORCE) +set(ZeroMQ_BUILD_DIR ${ZeroMQ_PREFIX}/src/BUILTIN_ZeroMQ-build CACHE INTERNAL "" FORCE) \ No newline at end of file From 856308b47307fbe48742be1af191079baf5719a4 Mon Sep 17 00:00:00 2001 From: "E. G. Patrick Bos" Date: Thu, 10 Jun 2021 14:07:22 +0200 Subject: [PATCH 02/15] suppress warnings in ZeroMQ builtin build --- roofit/builtin_zeromq/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roofit/builtin_zeromq/CMakeLists.txt b/roofit/builtin_zeromq/CMakeLists.txt index 8407e20d1deae..3f64bcb19ac8e 100644 --- a/roofit/builtin_zeromq/CMakeLists.txt +++ b/roofit/builtin_zeromq/CMakeLists.txt @@ -13,6 +13,8 @@ ExternalProject_Add(BUILTIN_ZeroMQ -DWITH_PERF_TOOL=OFF -DZMQ_BUILD_TESTS=OFF -DPOLLER=select + # TODO: remove -w flag when updating to v4.3.5 or higher, current warnings should be gone by then + -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}\ -w BUILD_BYPRODUCTS ${ZeroMQ_LIBRARIES} ) From 8b59b0bb8b5fb57f65a267b303663a95f0c496ea Mon Sep 17 00:00:00 2001 From: "E. G. Patrick Bos" Date: Thu, 10 Jun 2021 15:52:41 +0200 Subject: [PATCH 03/15] add cppzmq to ZeroMQ builtin --- roofit/builtin_zeromq/CMakeLists.txt | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/roofit/builtin_zeromq/CMakeLists.txt b/roofit/builtin_zeromq/CMakeLists.txt index 3f64bcb19ac8e..9b769f3d00e95 100644 --- a/roofit/builtin_zeromq/CMakeLists.txt +++ b/roofit/builtin_zeromq/CMakeLists.txt @@ -18,6 +18,8 @@ ExternalProject_Add(BUILTIN_ZeroMQ BUILD_BYPRODUCTS ${ZeroMQ_LIBRARIES} ) +ExternalProject_Get_Property(BUILTIN_ZeroMQ install_dir) + set(ZeroMQ_FOUND TRUE CACHE BOOL "" FORCE) set(ZeroMQ_INCLUDE_DIR ${ZeroMQ_PREFIX}/include CACHE INTERNAL "" FORCE) @@ -34,4 +36,11 @@ set(ZeroMQ_DIR ${ZeroMQ_PREFIX}/share/cmake/ZeroMQ CACHE INTERNAL "" FORCE) # also need source and build directories for ppoll in RooFitZMQ set(ZeroMQ_SOURCE_DIR ${ZeroMQ_PREFIX}/src/BUILTIN_ZeroMQ/src CACHE INTERNAL "" FORCE) -set(ZeroMQ_BUILD_DIR ${ZeroMQ_PREFIX}/src/BUILTIN_ZeroMQ-build CACHE INTERNAL "" FORCE) \ No newline at end of file +set(ZeroMQ_BUILD_DIR ${ZeroMQ_PREFIX}/src/BUILTIN_ZeroMQ-build CACHE INTERNAL "" FORCE) + +ExternalProject_Add(BUILTIN_cppzmq + URL https://github.com/zeromq/cppzmq/archive/refs/tags/v4.7.1.tar.gz + CMAKE_ARGS + -DCMAKE_INSTALL_PREFIX=${install_dir} + -DCPPZMQ_BUILD_TESTS=OFF + ) From 1042726634b5e0a10a333642d4fb81d3e5de6a95 Mon Sep 17 00:00:00 2001 From: "E. G. Patrick Bos" Date: Thu, 10 Jun 2021 15:55:06 +0200 Subject: [PATCH 04/15] add RooFitZMQ package The files in here are modified versions of the ZMQ convenience functions from https://gitlab.cern.ch/raaij/generate_and_sort, contributed by @roelaaij. --- roofit/CMakeLists.txt | 1 + roofit/roofitZMQ/CMakeLists.txt | 40 ++++ roofit/roofitZMQ/inc/LinkDef.h | 1 + roofit/roofitZMQ/inc/RooFit_ZMQ/Utility.h | 39 ++++ .../roofitZMQ/inc/RooFit_ZMQ/ZeroMQPoller.h | 48 ++++ roofit/roofitZMQ/inc/RooFit_ZMQ/ZeroMQSvc.h | 214 ++++++++++++++++++ roofit/roofitZMQ/inc/RooFit_ZMQ/functions.h | 111 +++++++++ roofit/roofitZMQ/src/ZeroMQPoller.cpp | 133 +++++++++++ roofit/roofitZMQ/src/ZeroMQSvc.cpp | 103 +++++++++ roofit/roofitZMQ/src/functions.cpp | 9 + 10 files changed, 699 insertions(+) create mode 100644 roofit/roofitZMQ/CMakeLists.txt create mode 100644 roofit/roofitZMQ/inc/LinkDef.h create mode 100644 roofit/roofitZMQ/inc/RooFit_ZMQ/Utility.h create mode 100644 roofit/roofitZMQ/inc/RooFit_ZMQ/ZeroMQPoller.h create mode 100644 roofit/roofitZMQ/inc/RooFit_ZMQ/ZeroMQSvc.h create mode 100644 roofit/roofitZMQ/inc/RooFit_ZMQ/functions.h create mode 100644 roofit/roofitZMQ/src/ZeroMQPoller.cpp create mode 100644 roofit/roofitZMQ/src/ZeroMQSvc.cpp create mode 100644 roofit/roofitZMQ/src/functions.cpp diff --git a/roofit/CMakeLists.txt b/roofit/CMakeLists.txt index 01478c67ca3d5..886761c71ee09 100644 --- a/roofit/CMakeLists.txt +++ b/roofit/CMakeLists.txt @@ -6,6 +6,7 @@ add_subdirectory(batchcompute) add_subdirectory(builtin_zeromq) +add_subdirectory(roofitZMQ) add_subdirectory(roofitcore) add_subdirectory(roofit) if(mathmore) diff --git a/roofit/roofitZMQ/CMakeLists.txt b/roofit/roofitZMQ/CMakeLists.txt new file mode 100644 index 0000000000000..c861a7471120a --- /dev/null +++ b/roofit/roofitZMQ/CMakeLists.txt @@ -0,0 +1,40 @@ +############################################################################ +# CMakeLists.txt file for building ROOT roofitcore/ZMQ package +# @author Patrick Bos, NL eScience Center +############################################################################ + +#if(NOT ZeroMQ_FOUND) +# find_package(ZeroMQ REQUIRED) +#endif() + +ROOT_STANDARD_LIBRARY_PACKAGE(RooFitZMQ + HEADERS + RooFit_ZMQ/Utility.h + RooFit_ZMQ/ZeroMQSvc.h + RooFit_ZMQ/functions.h + RooFit_ZMQ/ZeroMQPoller.h +# RooFit_ZMQ/ppoll.h + SOURCES + src/ZeroMQSvc.cpp + src/ZeroMQPoller.cpp + src/functions.cpp +# src/ppoll.cpp + DICTIONARY_OPTIONS +# -I${ZeroMQ_INCLUDE_DIR} + -writeEmptyRootPCM + DEPENDENCIES + Core # TVersionCheck, RegisterModule, these are always called from G__RooFitZMQ.cxx.o + LIBRARIES +# ${ZeroMQ_LIBRARY} + ZeroMQ + ) + + +target_link_libraries(RooFitZMQ PRIVATE ZeroMQ) +target_include_directories(RooFitZMQ PUBLIC $) +target_include_directories(RooFitZMQ PUBLIC $) +target_include_directories(RooFitZMQ PUBLIC $) + +#if(testing) +# ROOT_ADD_TEST_SUBDIRECTORY(test) +#endif() diff --git a/roofit/roofitZMQ/inc/LinkDef.h b/roofit/roofitZMQ/inc/LinkDef.h new file mode 100644 index 0000000000000..5495493c90b99 --- /dev/null +++ b/roofit/roofitZMQ/inc/LinkDef.h @@ -0,0 +1 @@ +// something \ No newline at end of file diff --git a/roofit/roofitZMQ/inc/RooFit_ZMQ/Utility.h b/roofit/roofitZMQ/inc/RooFit_ZMQ/Utility.h new file mode 100644 index 0000000000000..a329805967df6 --- /dev/null +++ b/roofit/roofitZMQ/inc/RooFit_ZMQ/Utility.h @@ -0,0 +1,39 @@ +#ifndef SERIALIZE_UTILITY_H +#define SERIALIZE_UTILITY_H 1 + +#if defined(__clang__) && (__clang_major__ > 3 || (__clang_major__ == 3 && __clang_minor__ >= 10)) +#define HAVE_TRIVIALLY_COPYABLE 1 +#elif defined(__GNUC__) && __GNUC__ >= 5 +#define HAVE_TRIVIALLY_COPYABLE 1 +#else +#undef HAVE_TRIVIALLY_COPYABLE +#endif + +#include + +namespace ZMQ { + namespace Detail { + +#if defined(HAVE_TRIVIALLY_COPYABLE) + template + using simple_object = std::is_trivially_copyable; +#else + template + using simple_object = std::is_pod; +#endif + +// is trivial + template + struct is_trivial : std::conditional< + simple_object< + typename std::decay::type + >::value, + std::true_type, + std::false_type + >::type { + }; + + } +} + +#endif // SERIALIZE_UTILITY_H diff --git a/roofit/roofitZMQ/inc/RooFit_ZMQ/ZeroMQPoller.h b/roofit/roofitZMQ/inc/RooFit_ZMQ/ZeroMQPoller.h new file mode 100644 index 0000000000000..4d14444c01553 --- /dev/null +++ b/roofit/roofitZMQ/inc/RooFit_ZMQ/ZeroMQPoller.h @@ -0,0 +1,48 @@ +#ifndef ZEROMQPOLLER_H +#define ZEROMQPOLLER_H 1 +#include +#include +#include +#include + +#include "RooFit_ZMQ/ZeroMQSvc.h" +#include "RooFit_ZMQ/functions.h" + +class ZeroMQPoller { +public: + + using entry_t = std::tuple; + // The key is what zmq::socket_t stores inside, and what goes into + // pollitem_t through zmq::socket_t's conversion to void* operator + using sockets_t = std::unordered_map; + + using fd_entry_t = std::tuple; + using fds_t = std::unordered_map; + + using free_t = std::deque; + + ZeroMQPoller() = default; + + std::vector> poll(int timeo = -1); + std::vector> ppoll(int timeo, const sigset_t * sigmask_); + + size_t size() const; + + size_t register_socket(zmq::socket_t& socket, zmq::PollType type); + size_t register_socket(int fd, zmq::PollType type); + + size_t unregister_socket(zmq::socket_t& socket); + size_t unregister_socket(int fd); + +private: + + // Vector of (socket, flags) + std::vector m_items; + sockets_t m_sockets; + fds_t m_fds; + + // free slots in items + free_t m_free; +}; + +#endif // ZEROMQPOLLER_H diff --git a/roofit/roofitZMQ/inc/RooFit_ZMQ/ZeroMQSvc.h b/roofit/roofitZMQ/inc/RooFit_ZMQ/ZeroMQSvc.h new file mode 100644 index 0000000000000..0eace37b7f6c4 --- /dev/null +++ b/roofit/roofitZMQ/inc/RooFit_ZMQ/ZeroMQSvc.h @@ -0,0 +1,214 @@ +#ifndef ZEROMQ_IZEROMQSVC_H +#define ZEROMQ_IZEROMQSVC_H 1 + +// Include files +// from STL +#include +#include +#include +#include +#include +#include // std::cerr + +// ZeroMQ +#include +#include "RooFit_ZMQ/Utility.h" +#include "RooFit_ZMQ/functions.h" + +// debugging +#include // getpid + +namespace ZMQ { + +struct TimeOutException : std::exception { + TimeOutException() = default; + TimeOutException(const TimeOutException&) = default; + ~TimeOutException() = default; + TimeOutException& operator=(const TimeOutException&) = default; +}; + +struct MoreException : std::exception { + MoreException() = default; + MoreException(const MoreException&) = default; + ~MoreException() = default; + MoreException& operator=(const MoreException&) = default; +}; + +} + +template +struct ZmqLingeringSocketPtrDeleter { + void operator()(zmq::socket_t* socket) { + auto period = PERIOD; + + int tries = 0; + int max_tries = 3; + while(true) { + try { + // the actual work this function should do, plus the delete socket below: + if (socket) socket->setsockopt(ZMQ_LINGER, &period, sizeof(period)); + break; + } catch (zmq::error_t& e) { + if (++tries == max_tries + || e.num() == EINVAL || e.num() == ETERM || e.num() == ENOTSOCK // not recoverable from here + ) { + std::cerr << "ERROR in ZeroMQSvc::socket: " << e.what() << " (errno: " << e.num() << ")\n"; + throw; + } + std::cerr << "RETRY " << tries << "/" << (max_tries - 1) << " in ZmqLingeringSocketPtrDeleter: call interrupted (errno: " << e.num() << ")\n"; + } + } + + delete socket; + } +}; + +template +using ZmqLingeringSocketPtr = std::unique_ptr>; + + +// We retry send and receive only on EINTR, all other errors are either fatal, or can only +// be handled at the caller. +template +auto retry_send(zmq::socket_t& socket, int max_tries, args_t ...args) -> decltype(socket.send(args...)) { + int tries = 0; + while(true) { + try { + // the actual work this function should do, all the rest is error handling: + return socket.send(args...); + } catch (zmq::error_t& e) { + if (++tries == max_tries + || e.num() != EINTR // only recoverable error + ) { +// std::cerr << "ERROR in ZeroMQSvc::send (retry_send) on pid " << getpid() << ": " << e.what() << " (errno: " << e.num() << ")\n"; + throw; + } + std::cerr << "RETRY " << tries << "/" << (max_tries - 1) << " in ZeroMQSvc::send (retry_send) on pid " << getpid() << ": " << e.what() << ")\n"; + } + } +} + +template +auto retry_recv(zmq::socket_t& socket, int max_tries, args_t ...args) -> decltype(socket.recv(args...)) { + int tries = 0; + while(true) { + try { + // the actual work this function should do, all the rest is error handling: + return socket.recv(args...); + } catch (zmq::error_t& e) { + if (++tries == max_tries + || e.num() != EINTR // only recoverable error + ) { +// std::cerr << "ERROR in ZeroMQSvc::recv (retry_recv) on pid " << getpid() << ": " << e.what() << " (errno: " << e.num() << ")\n"; + throw; + } + std::cerr << "RETRY " << tries << "/" << (max_tries - 1) << " in ZeroMQSvc::recv (retry_recv) on pid " << getpid() << ": " << e.what() << ")\n"; + } + } +} + + +/** @class IZeroMQSvc IZeroMQSvc.h ZeroMQ/IZeroMQSvc.h + * + * + * @author + * @date 2015-06-22 + */ +class ZeroMQSvc { + // Note on error handling: + // Creating message_t can throw, but only when memory ran out (errno ENOMEM), + // and that is something only the caller can fix, so we don't catch it here. + +public: + + enum Encoding { + Text = 0, + Binary + }; + + Encoding encoding() const; + void setEncoding(const Encoding& e); + zmq::context_t& context() const; + zmq::socket_t socket(int type) const; + zmq::socket_t* socket_ptr(int type) const; + void close_context() const; + + // decode message with ZMQ, POD version + template ::value + && ZMQ::Detail::is_trivial::value, T>::type* = nullptr> + T decode(const zmq::message_t& msg) const { + T object; + memcpy(&object, msg.data(), msg.size()); + return object; + } + + // decode ZMQ message, string version + template ::value, T>::type* = nullptr> + std::string decode(const zmq::message_t& msg) const { + std::string r(msg.size() + 1, char{}); + r.assign(static_cast(msg.data()), msg.size()); + return r; + } + + // receive message with ZMQ, general version + // FIXME: what to do with flags=0.... more is a pointer, that might prevent conversion + template ::value), T>::type* = nullptr> + T receive(zmq::socket_t& socket, int flags = 0, bool* more = nullptr) const { + // receive message + zmq::message_t msg; + auto nbytes = retry_recv(socket, 2, &msg, flags); + if (0 == nbytes) { + throw ZMQ::TimeOutException{}; + } + if (more) *more = msg.more(); + + // decode message + return decode(msg); + } + + // receive message with ZMQ + template ::value, T>::type* = nullptr> + T receive(zmq::socket_t& socket, int flags = 0, bool* more = nullptr) const { + // receive message + zmq::message_t msg; + auto nbytes = retry_recv(socket, 2, &msg, flags); + if (0 == nbytes) { + throw ZMQ::TimeOutException{}; + } + if (more) *more = msg.more(); + return msg; + } + + // encode message to ZMQ + template ::value + && ZMQ::Detail::is_trivial::value, T>::type* = nullptr> + zmq::message_t encode(const T& item, std::function sizeFun = ZMQ::defaultSizeOf) const { + size_t s = sizeFun(item); + zmq::message_t msg{s}; + memcpy((void *)msg.data(), &item, s); + return msg; + } + + zmq::message_t encode(const char* item) const; + zmq::message_t encode(const std::string& item) const; + + // Send message with ZMQ + template ::value, T>::type* = nullptr> + bool send(zmq::socket_t& socket, const T& item, int flags = 0) const { + return retry_send(socket, 1, encode(item), flags); + } + + bool send(zmq::socket_t& socket, const char* item, int flags = 0) const; + bool send(zmq::socket_t& socket, zmq::message_t& msg, int flags = 0) const; + bool send(zmq::socket_t& socket, zmq::message_t&& msg, int flags = 0) const; + +private: + + Encoding m_enc = Text; + mutable zmq::context_t* m_context = nullptr; + +}; + +ZeroMQSvc& zmqSvc(); + +#endif // ZEROMQ_IZEROMQSVC_H diff --git a/roofit/roofitZMQ/inc/RooFit_ZMQ/functions.h b/roofit/roofitZMQ/inc/RooFit_ZMQ/functions.h new file mode 100644 index 0000000000000..2199275b1d272 --- /dev/null +++ b/roofit/roofitZMQ/inc/RooFit_ZMQ/functions.h @@ -0,0 +1,111 @@ +#ifndef ZEROMQ_FUNCTIONS_H +#define ZEROMQ_FUNCTIONS_H 1 + +#include + +namespace zmq { + + enum SocketTypes { + PAIR = ZMQ_PAIR, + PUB = ZMQ_PUB, + SUB = ZMQ_SUB, + REQ = ZMQ_REQ, + REP = ZMQ_REP, + DEALER = ZMQ_DEALER, + ROUTER = ZMQ_ROUTER, + PULL = ZMQ_PULL, + PUSH = ZMQ_PUSH, + XPUB = ZMQ_XPUB, + XSUB = ZMQ_XSUB, + STREAM = ZMQ_STREAM + }; + + enum PollType : short { + POLLIN = ZMQ_POLLIN, + POLLOUT = ZMQ_POLLOUT + }; + + enum SocketOptions { + AFFINITY = ZMQ_AFFINITY, + IDENTITY = ZMQ_IDENTITY, + SUBSCRIBE = ZMQ_SUBSCRIBE, + UNSUBSCRIBE = ZMQ_UNSUBSCRIBE, + RATE = ZMQ_RATE, + RECOVERY_IVL = ZMQ_RECOVERY_IVL, + SNDBUF = ZMQ_SNDBUF, + RCVBUF = ZMQ_RCVBUF, + RCVMORE = ZMQ_RCVMORE, + FD = ZMQ_FD, + EVENTS = ZMQ_EVENTS, + TYPE = ZMQ_TYPE, + LINGER = ZMQ_LINGER, + RECONNECT_IVL = ZMQ_RECONNECT_IVL, + BACKLOG = ZMQ_BACKLOG, + RECONNECT_IVL_MAX = ZMQ_RECONNECT_IVL_MAX, + MAXMSGSIZE = ZMQ_MAXMSGSIZE, + SNDHWM = ZMQ_SNDHWM, + RCVHWM = ZMQ_RCVHWM, + MULTICAST_HOPS = ZMQ_MULTICAST_HOPS, + RCVTIMEO = ZMQ_RCVTIMEO, + SNDTIMEO = ZMQ_SNDTIMEO, + LAST_ENDPOINT = ZMQ_LAST_ENDPOINT, + ROUTER_MANDATORY = ZMQ_ROUTER_MANDATORY, + TCP_KEEPALIVE = ZMQ_TCP_KEEPALIVE, + TCP_KEEPALIVE_CNT = ZMQ_TCP_KEEPALIVE_CNT, + TCP_KEEPALIVE_IDLE = ZMQ_TCP_KEEPALIVE_IDLE, + TCP_KEEPALIVE_INTVL = ZMQ_TCP_KEEPALIVE_INTVL, + IMMEDIATE = ZMQ_IMMEDIATE, + XPUB_VERBOSE = ZMQ_XPUB_VERBOSE, + ROUTER_RAW = ZMQ_ROUTER_RAW, + IPV6 = ZMQ_IPV6, + MECHANISM = ZMQ_MECHANISM, + PLAIN_SERVER = ZMQ_PLAIN_SERVER, + PLAIN_USERNAME = ZMQ_PLAIN_USERNAME, + PLAIN_PASSWORD = ZMQ_PLAIN_PASSWORD, + CURVE_SERVER = ZMQ_CURVE_SERVER, + CURVE_PUBLICKEY = ZMQ_CURVE_PUBLICKEY, + CURVE_SECRETKEY = ZMQ_CURVE_SECRETKEY, + CURVE_SERVERKEY = ZMQ_CURVE_SERVERKEY, + PROBE_ROUTER = ZMQ_PROBE_ROUTER, + REQ_CORRELATE = ZMQ_REQ_CORRELATE, + REQ_RELAXED = ZMQ_REQ_RELAXED, + CONFLATE = ZMQ_CONFLATE, + ZAP_DOMAIN = ZMQ_ZAP_DOMAIN, + ROUTER_HANDOVER = ZMQ_ROUTER_HANDOVER, + TOS = ZMQ_TOS, + CONNECT_RID = ZMQ_CONNECT_RID, + GSSAPI_SERVER = ZMQ_GSSAPI_SERVER, + GSSAPI_PRINCIPAL = ZMQ_GSSAPI_PRINCIPAL, + GSSAPI_SERVICE_PRINCIPAL = ZMQ_GSSAPI_SERVICE_PRINCIPAL, + GSSAPI_PLAINTEXT = ZMQ_GSSAPI_PLAINTEXT, + HANDSHAKE_IVL = ZMQ_HANDSHAKE_IVL, + SOCKS_PROXY = ZMQ_SOCKS_PROXY, + XPUB_NODROP = ZMQ_XPUB_NODROP + }; + + // Message options + enum MessageOptions { + MORE = ZMQ_MORE, + SRCFD = ZMQ_SRCFD, + SHARED = ZMQ_SHARED + }; + + // Send/recv options. + enum SendRecvOptions { + DONTWAIT = ZMQ_DONTWAIT, + SNDMORE = ZMQ_SNDMORE + }; +} + +namespace ZMQ { + +template +size_t defaultSizeOf(const T&) { + return sizeof(T); +} + +size_t stringLength(const char& cs); + +} + +#endif // ZEROMQ_FUNCTIONS_H diff --git a/roofit/roofitZMQ/src/ZeroMQPoller.cpp b/roofit/roofitZMQ/src/ZeroMQPoller.cpp new file mode 100644 index 0000000000000..12eafaf09392d --- /dev/null +++ b/roofit/roofitZMQ/src/ZeroMQPoller.cpp @@ -0,0 +1,133 @@ +/* + * ROOT + * Copyright (c) 2018, Patrick Bos + * Distributed under the terms of the BSD 3-Clause License. + * + * The full license is in the file LICENSE, distributed with this software. + */ + +#include + +#include "RooFit_ZMQ/ZeroMQPoller.h" + +std::vector> ZeroMQPoller::poll(int timeo) { + std::vector> r; + if (m_items.empty()) { + throw std::runtime_error("No sockets registered"); + } + int n = 0; + while (true) { + try { + n = zmq::poll(&m_items[0], m_items.size(), timeo); + if (n == 0) return r; + break; + } catch (const zmq::error_t& e) { + std::cerr << "in ZeroMQPoller::poll on PID " << getpid() << ": " << e.what() << std::endl; + if (e.num() != EINTR) { + throw; + } + } + } + // TODO: replace this with ranges::v3::zip + for (size_t i = 0; i < m_items.size(); ++i) { + void* socket = m_items[i].socket; + size_t index = 0; + int flags = 0; + if (socket == nullptr) { + // an fd was registered + std::tie(index, flags) = m_fds[m_items[i].fd]; + } else { + // a socket was registered + const zmq::socket_t* s; + std::tie(index, flags, s) = m_sockets[socket]; + } + if (m_items[i].revents & short(flags)) { + r.emplace_back(index, flags); + } + } + return r; +} + +size_t ZeroMQPoller::size() const { + return m_items.size(); +} + +size_t ZeroMQPoller::register_socket(zmq::socket_t& socket, zmq::PollType type) { + zmq::socket_t* s = &socket; + auto it = m_sockets.find(s); + if (it != m_sockets.end()) { + return std::get<0>(it->second); + } + size_t index = m_free.empty() ? m_items.size() : m_free.front(); + if (!m_free.empty()) m_free.pop_front(); + // NOTE: tis uses the conversion-to-void* operator of + // zmq::socket_t, which returns the wrapped object + m_items.push_back({socket, 0, type, 0}); + + // We need to lookup by the pointer to the object wrapped by zmq::socket_t + m_sockets.emplace(m_items.back().socket, std::make_tuple(index, type, s)); + return index; +} + +size_t ZeroMQPoller::register_socket(int fd, zmq::PollType type) { + auto it = m_fds.find(fd); + if (it != m_fds.end()) { + return std::get<0>(it->second); + } + size_t index = m_free.empty() ? m_items.size() : m_free.front(); + if (!m_free.empty()) m_free.pop_front(); + // NOTE: tis uses the conversion-to-void* operator of + // zmq::socket_t, which returns the wrapped object + m_items.push_back({nullptr, fd, type, 0}); + + // We need to lookup by the pointer to the object wrapped by zmq::socket_t + m_fds.emplace(fd, std::make_tuple(index, type)); + return index; +} + + +size_t ZeroMQPoller::unregister_socket(zmq::socket_t& socket) { + if (!m_sockets.count(socket.operator void*())) { + throw std::out_of_range("Socket is not registered"); + } + // Remove from m_sockets + // Can't search by the key of m_sockets, as that is the wrapped + // object, but have to use the pointer to the wrapper + // (zmq::socket_t) + auto it = std::find_if(begin(m_sockets), end(m_sockets), + [&socket](const decltype(m_sockets)::value_type& entry) { + return &socket == std::get<2>(entry.second); + }); + auto index = std::get<0>(it->second); + m_free.push_back(index); + m_sockets.erase(it); + + // Remove from m_items + auto iit = std::find_if(begin(m_items), end(m_items), [&it](const zmq::pollitem_t& item) { + return it->first == item.socket; + }); + assert(iit != end(m_items)); + m_items.erase(iit); + + return index; +} + +size_t ZeroMQPoller::unregister_socket(int fd) { + if (!m_fds.count(fd)) { + throw std::out_of_range("fileno is not registered"); + } + // Remove from m_fds + auto it = m_fds.find(fd); + auto index = std::get<0>(it->second); + m_free.push_back(index); + m_fds.erase(it); + + // Remove from m_items + auto iit = std::find_if(begin(m_items), end(m_items), [&it](const zmq::pollitem_t& item) { + return it->first == item.fd; + }); + assert(iit != end(m_items)); + m_items.erase(iit); + + return index; +} diff --git a/roofit/roofitZMQ/src/ZeroMQSvc.cpp b/roofit/roofitZMQ/src/ZeroMQSvc.cpp new file mode 100644 index 0000000000000..b69695349e0c3 --- /dev/null +++ b/roofit/roofitZMQ/src/ZeroMQSvc.cpp @@ -0,0 +1,103 @@ +//#include +#include // std::ref +#include + +#include "RooFit_ZMQ/ZeroMQSvc.h" + +ZeroMQSvc& zmqSvc() { + static std::unique_ptr svc; + if (!svc) { + svc = std::make_unique(); + } + return *svc; +} + +ZeroMQSvc::Encoding ZeroMQSvc::encoding() const { + return m_enc; +} + +void ZeroMQSvc::setEncoding(const ZeroMQSvc::Encoding &e) { + m_enc = e; +} + +zmq::context_t& ZeroMQSvc::context() const { + if (!m_context) { + try { + m_context = new zmq::context_t; + } catch (zmq::error_t& e) { + std::cerr << "ERROR: Creating ZeroMQ context failed. This only happens when PGM initialization failed or when a nullptr was returned from zmq_ctx_new because the created context was invalid. Contact ZMQ experts when this happens, because it shouldn't.\n"; + throw e; + } + } + return *m_context; +} + +zmq::socket_t ZeroMQSvc::socket(int type) const { + try { + // the actual work this function should do, all the rest is error handling: + return zmq::socket_t{context(), type}; + } catch (zmq::error_t& e) { + // all zmq errors not recoverable from here, only at call site + std::cerr << "ERROR in ZeroMQSvc::socket: " << e.what() << " (errno: " << e.num() << ")\n"; + throw e; + } +} + +zmq::socket_t* ZeroMQSvc::socket_ptr(int type) const { + try { + // the actual work this function should do, all the rest is error handling: + return new zmq::socket_t(context(), type); + } catch (zmq::error_t& e) { + // all zmq errors not recoverable from here, only at call site + std::cerr << "ERROR in ZeroMQSvc::socket_ptr: " << e.what() << " (errno: " << e.num() << ")\n"; + throw e; + } +} + +void ZeroMQSvc::close_context() const { + if (m_context) { + delete m_context; + m_context = nullptr; + } +} + + +zmq::message_t ZeroMQSvc::encode(const char* item) const { + std::function fun = ZMQ::stringLength; + return encode(*item, fun); +} + +zmq::message_t ZeroMQSvc::encode(const std::string& item) const { + return encode(item.c_str()); +} + +//zmq::message_t ZeroMQSvc::encode(const TObject& item) const { +// auto deleteBuffer = []( void* data, void* /* hint */ ) -> void { +// delete [] (char*)data; +// }; +// +// TBufferFile buffer(TBuffer::kWrite); +// buffer.WriteObject(&item); +// +// // Create message and detach buffer +// // This is the only ZMQ thing that can throw, and only when memory ran out +// // (errno ENOMEM), and that is something only the caller can fix, so we don't +// // catch it here: +// zmq::message_t message(buffer.Buffer(), buffer.Length(), deleteBuffer); +// buffer.DetachBuffer(); +// +// return message; +//} + + +bool ZeroMQSvc::send(zmq::socket_t& socket, const char* item, int flags) const { + return retry_send(socket, 2, encode(item), flags); +} + +bool ZeroMQSvc::send(zmq::socket_t& socket, zmq::message_t& msg, int flags) const { + return retry_send(socket, 2, std::ref(msg), flags); +} + +bool ZeroMQSvc::send(zmq::socket_t& socket, zmq::message_t&& msg, int flags) const { + return retry_send(socket, 2, std::move(msg), flags); +} diff --git a/roofit/roofitZMQ/src/functions.cpp b/roofit/roofitZMQ/src/functions.cpp new file mode 100644 index 0000000000000..ebb6199bfdb5d --- /dev/null +++ b/roofit/roofitZMQ/src/functions.cpp @@ -0,0 +1,9 @@ +#include +#include "RooFit_ZMQ/ZeroMQSvc.h" +#include "RooFit_ZMQ/functions.h" + +namespace ZMQ { + size_t stringLength(const char& cs) { + return strlen(&cs); + } +} From 4476eb3316969f110f6f9f35c72abf75fcb89aa9 Mon Sep 17 00:00:00 2001 From: "E. G. Patrick Bos" Date: Thu, 10 Jun 2021 16:27:29 +0200 Subject: [PATCH 05/15] add RooFitZMQ tests --- roofit/roofitZMQ/CMakeLists.txt | 6 +- roofit/roofitZMQ/test/CMakeLists.txt | 6 + roofit/roofitZMQ/test/test_HWM.cxx | 130 ++++++++ roofit/roofitZMQ/test/test_ZMQ.cpp | 306 ++++++++++++++++++ .../test/test_ZMQ_load_balancing.cxx | 133 ++++++++ roofit/roofitZMQ/test/test_polling.cxx | 149 +++++++++ 6 files changed, 727 insertions(+), 3 deletions(-) create mode 100644 roofit/roofitZMQ/test/CMakeLists.txt create mode 100644 roofit/roofitZMQ/test/test_HWM.cxx create mode 100644 roofit/roofitZMQ/test/test_ZMQ.cpp create mode 100644 roofit/roofitZMQ/test/test_ZMQ_load_balancing.cxx create mode 100644 roofit/roofitZMQ/test/test_polling.cxx diff --git a/roofit/roofitZMQ/CMakeLists.txt b/roofit/roofitZMQ/CMakeLists.txt index c861a7471120a..8174ecb952418 100644 --- a/roofit/roofitZMQ/CMakeLists.txt +++ b/roofit/roofitZMQ/CMakeLists.txt @@ -35,6 +35,6 @@ target_include_directories(RooFitZMQ PUBLIC $) target_include_directories(RooFitZMQ PUBLIC $) -#if(testing) -# ROOT_ADD_TEST_SUBDIRECTORY(test) -#endif() +if(testing) + ROOT_ADD_TEST_SUBDIRECTORY(test) +endif() diff --git a/roofit/roofitZMQ/test/CMakeLists.txt b/roofit/roofitZMQ/test/CMakeLists.txt new file mode 100644 index 0000000000000..b1da06f39da1f --- /dev/null +++ b/roofit/roofitZMQ/test/CMakeLists.txt @@ -0,0 +1,6 @@ +#find_package(ZeroMQ REQUIRED) + +ROOT_ADD_GTEST(test_RooFitZMQ test_ZMQ.cpp LIBRARIES RooFitZMQ ${ZeroMQ_LIBRARY}) +ROOT_ADD_GTEST(test_RooFitZMQ_polling test_polling.cxx LIBRARIES RooFitZMQ ${ZeroMQ_LIBRARY}) +ROOT_ADD_GTEST(test_RooFitZMQ_HWM test_HWM.cxx LIBRARIES RooFitZMQ ${ZeroMQ_LIBRARY}) +ROOT_ADD_GTEST(test_RooFitZMQ_load_balancing test_ZMQ_load_balancing.cxx LIBRARIES RooFitZMQ ${ZeroMQ_LIBRARY}) diff --git a/roofit/roofitZMQ/test/test_HWM.cxx b/roofit/roofitZMQ/test/test_HWM.cxx new file mode 100644 index 0000000000000..7ef11b5c3f5ed --- /dev/null +++ b/roofit/roofitZMQ/test/test_HWM.cxx @@ -0,0 +1,130 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2016-2021, Netherlands eScience Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "gtest/gtest.h" + +#include // fork, usleep + +#include "RooFit_ZMQ/ZeroMQSvc.h" + +class HighWaterMarkTest : public ::testing::Test { +protected: + void SetUp() override { + do { + child_pid = fork(); + } while (child_pid == -1); // retry if fork fails + + if (child_pid > 0) { // parent + pusher.reset(zmqSvc().socket_ptr(zmq::PUSH)); + if (set_hwm) { + auto rc = zmq_setsockopt(*pusher, ZMQ_SNDHWM, &hwm, sizeof hwm); + assert(rc == 0); + } + pusher->bind("ipc:///tmp/ZMQ_test_fork_polling_P2C.ipc"); + } else { // child + puller.reset(zmqSvc().socket_ptr(zmq::PULL)); + if (set_hwm) { + auto rc = zmq_setsockopt(*puller, ZMQ_RCVHWM, &hwm, sizeof hwm); + assert(rc == 0); + } + puller->connect("ipc:///tmp/ZMQ_test_fork_polling_P2C.ipc"); + } + } + + void TearDown() override { + if (child_pid > 0) { // parent + // wait for child + int status = 0; + pid_t pid; + do { + pid = waitpid(child_pid, &status, 0); + } while (-1 == pid && EINTR == errno); // retry on interrupted system call + if (0 != status) { + if (WIFEXITED(status)) { printf("exited, status=%d\n", WEXITSTATUS(status)); } else if (WIFSIGNALED(status)) { printf("killed by signal %d\n", WTERMSIG(status)); } else if (WIFSTOPPED(status)) { printf("stopped by signal %d\n", WSTOPSIG(status)); } else if (WIFCONTINUED(status)) { printf("continued\n"); } + } + if (-1 == pid) { + throw std::runtime_error(std::string("waitpid, errno ") + std::to_string(errno)); + } + pusher.reset(nullptr); + zmqSvc().close_context(); + } else { // child + puller.reset(nullptr); + zmqSvc().close_context(); + } + } + + void run_test() { + std::size_t max_sends = 2000; + + if (child_pid > 0) { // parent + // start test + for (std::size_t ix = 0; ix < max_sends; ++ix) { + zmqSvc().send(*pusher, 0.1f); + zmqSvc().send(*pusher, 1); + zmqSvc().send(*pusher, true); + if (ix % 100 == 0) { + printf("parent at ix = %lu\n", ix); + } + } + } else { // child + // wait a few seconds before reading to allow the parent to overflow the HWM + printf("child waiting for 2 seconds...\n"); + sleep(2); + printf("child starts receiving\n"); + + for (std::size_t ix = 0; ix < max_sends; ++ix) { + zmqSvc().receive(*puller); + zmqSvc().receive(*puller); + zmqSvc().receive(*puller); + if (ix % 100 == 0) { + printf("child at ix = %lu\n", ix); + } + } + } + } + + pid_t child_pid {0}; + ZmqLingeringSocketPtr<> pusher, puller; + bool set_hwm = false; + int hwm = 0; +}; + + +TEST_F(HighWaterMarkTest, demonstrateHittingDefaultHWM) { + run_test(); +} + +class HighWaterMarkZeroTest : public HighWaterMarkTest { + void SetUp() override { + set_hwm = true; + hwm = 0; + HighWaterMarkTest::SetUp(); + } +}; + +TEST_F(HighWaterMarkZeroTest, zeroHWM) { + run_test(); +} + +class HighWaterMark2kTest : public HighWaterMarkTest { + void SetUp() override { + set_hwm = true; + hwm = 2000; + HighWaterMarkTest::SetUp(); + } +}; + +TEST_F(HighWaterMark2kTest, HWM2k) { + run_test(); +} diff --git a/roofit/roofitZMQ/test/test_ZMQ.cpp b/roofit/roofitZMQ/test/test_ZMQ.cpp new file mode 100644 index 0000000000000..985d1b91c5902 --- /dev/null +++ b/roofit/roofitZMQ/test/test_ZMQ.cpp @@ -0,0 +1,306 @@ +/***************************************************************************** + * Project: RooFit * + * Package: RooFitCore * + * @(#)root/roofitcore:$Id$ + * Authors: * + * PB, Patrick Bos, NL eScience Center, p.bos@esciencecenter.nl * + * * + * Redistribution and use in source and binary forms, * + * with or without modification, are permitted according to the terms * + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) * + *****************************************************************************/ + +#include "gtest/gtest.h" + +#include // fork, usleep + +#include + +#include "RooFit_ZMQ/ZeroMQSvc.h" + + +// N.B.: wait_for_child is identically defined in RooFit::MultiProcess, but we copy it here to reduce module +// interdependencies. It also requires an extra include: +#include // kill, SIGKILL + +int wait_for_child(pid_t child_pid, bool may_throw, int retries_before_killing) +{ + int status = 0; + int patience = retries_before_killing; + pid_t tmp; + do { + if (patience-- < 1) { + ::kill(child_pid, SIGKILL); + } + tmp = waitpid(child_pid, &status, WNOHANG); + } while (tmp == 0 // child has not yet changed state, try again + || (-1 == tmp && EINTR == errno) // retry on interrupted system call + ); + + if (patience < 1) { + std::cout << "Had to send PID " << child_pid << " " << (-patience + 1) << " SIGKILLs"; + } + + if (0 != status) { + if (WIFEXITED(status)) { + printf("exited, status=%d\n", WEXITSTATUS(status)); + } else if (WIFSIGNALED(status)) { + printf("killed by signal %d\n", WTERMSIG(status)); + } else if (WIFSTOPPED(status)) { + printf("stopped by signal %d\n", WSTOPSIG(status)); + } else if (WIFCONTINUED(status)) { + printf("continued\n"); + } + } + + if (-1 == tmp && may_throw) + throw std::runtime_error(std::string("waitpid, errno ") + std::to_string(errno)); + + return status; +} + +void elaborate_bind(const ZmqLingeringSocketPtr<>& socket, std::string name) { + try { + socket->bind(name); + } catch (const zmq::error_t& e) { + if (e.num() == EADDRINUSE) { + std::cerr << "address already in use, retrying bind in 500ms\n"; + usleep(500000); + try { + socket->bind(name); + } catch (const zmq::error_t& e2) { + if (e2.num() == EADDRINUSE) { + std::cerr + << "again: address already in use, aborting; please check whether there are any remaining improperly exited processes (zombies) around or whether some other program is using port 6660\n"; + } + throw e2; + } + // Sometimes, the socket from the previous test needs some time to close, so + // we introduce a latency here. A more robust and fast approach might be to + // do the following on the bind side: + // 1. first try another port, e.g. increase by one + // 2. if that doesn't work, do the latency and retry the original port + // The connect side then also needs to change, because it doesn't know which + // port the bind side will bind to. The connect side could try connecting to + // both options asynchronously, and then in a loop check both for signs of + // life. If one comes alive, transfer ownership of that pointer to the pointer + // you want to eventually use (`socket`) and that's it. + } else { + throw e; + } + } +} + +class AllSocketTypes : public ::testing::TestWithParam< std::tuple, std::pair /* socket_names */> > {}; + +TEST_P(AllSocketTypes, forkHandshake) { + auto socket_names = std::get<2>(GetParam()); + pid_t child_pid {0}; + do { + child_pid = fork(); + } while (child_pid == -1); // retry if fork fails + + if (child_pid > 0) { // master + ZmqLingeringSocketPtr<> socket; + socket.reset(zmqSvc().socket_ptr(std::get<1>(GetParam()).first)); + elaborate_bind(socket, socket_names.second); + // bind is on the master process to avoid zombie children to hold on to binds + + // start test + zmqSvc().send(*socket, std::string("breaker breaker")); + + auto receipt = zmqSvc().receive(*socket); + + EXPECT_EQ(receipt, 1212); + +// socket->close(); // this gives exception of type zmq::error_t: Socket operation on non-socket + socket.reset(nullptr); + zmqSvc().close_context(); // if you don't close context in parent process as well, the next repeat will hang + + wait_for_child(child_pid, true, 5); + } else { // child + ZmqLingeringSocketPtr<> socket; + socket.reset(zmqSvc().socket_ptr(std::get<1>(GetParam()).second)); + socket->connect(socket_names.first); + + // start test + auto receipt = zmqSvc().receive(*socket); + if (receipt == "breaker breaker") { + zmqSvc().send(*socket, 1212); + } + // take care, don't just use _exit, it will not cleanly destroy context etc! + // if you really need to, at least close and destroy everything properly +// socket->close(); // this gives exception of type zmq::error_t: Socket operation on non-socket + socket.reset(nullptr); + zmqSvc().close_context(); + _Exit(0); + } +} + +std::string ipc {"ipc:///tmp/ZMQ_test_fork.ipc"}; +std::string tcp_server {"tcp://127.0.0.1:6660"}; +std::string tcp_client {"tcp://*:6660"}; +auto socket_name_options = ::testing::Values( + std::make_pair(tcp_server, tcp_client), + std::make_pair(ipc, ipc) + ); + + +INSTANTIATE_TEST_SUITE_P(REQREP, AllSocketTypes, + ::testing::Combine(::testing::Range(0, 10), // repeat to probe connection stability + ::testing::Values(std::make_pair(zmq::REQ, zmq::REP)), + socket_name_options + )); +INSTANTIATE_TEST_SUITE_P(PAIRPAIR, AllSocketTypes, + ::testing::Combine(::testing::Range(0, 10), // repeat to probe connection stability + ::testing::Values(std::make_pair(zmq::PAIR, zmq::PAIR)), + socket_name_options + )); + + +class AsyncSocketTypes : public ::testing::TestWithParam< std::tuple, std::pair /* socket_names */, bool /* expect_throw */> > {}; + +TEST_P(AsyncSocketTypes, forkMultiSendReceive) { + bool expect_throw = std::get<3>(GetParam()); + ZmqLingeringSocketPtr<> socket; + auto socket_names = std::get<2>(GetParam()); + pid_t child_pid {0}; + do { + child_pid = fork(); + } while (child_pid == -1); // retry if fork fails + + if (child_pid > 0) { // master + socket.reset(zmqSvc().socket_ptr(std::get<1>(GetParam()).first)); + elaborate_bind(socket, socket_names.second); + // bind is on the master process to avoid zombie children to hold on to binds + + // start test: send 2 things, receive 1, send 1 more, finish + zmqSvc().send(*socket, std::string("breaker breaker")); + + if (expect_throw) { + EXPECT_ANY_THROW(zmqSvc().send(*socket, std::string("anybody out there?"))); + // NOTE: also in case of a throw, be sure to properly close down the connection! + // Otherwise, you may get zombies waiting for a reply. + socket.reset(nullptr); + zmqSvc().close_context(); // if you don't close context in parent process as well, the next repeat will hang + wait_for_child(child_pid, true, 5); + return; + } else { + EXPECT_NO_THROW(zmqSvc().send(*socket, std::string("anybody out there?"))); + } + + auto receipt = zmqSvc().receive(*socket); + + EXPECT_EQ(receipt, 1212); + + zmqSvc().send(*socket, std::string("kthxbye")); + +// socket->close(); // this gives exception of type zmq::error_t: Socket operation on non-socket + socket.reset(nullptr); + zmqSvc().close_context(); // if you don't close context in parent process as well, the next repeat will hang + + wait_for_child(child_pid, true, 5); + } else { // child + socket.reset(zmqSvc().socket_ptr(std::get<1>(GetParam()).second)); + socket->connect(socket_names.first); + + // start test, receive something + auto receipt1 = zmqSvc().receive(*socket); + auto receipt2 = zmqSvc().receive(*socket); + if (receipt1 == "breaker breaker" && receipt2 == "anybody out there?") { + zmqSvc().send(*socket, 1212); + } + auto receipt3 = zmqSvc().receive(*socket); + if (receipt3 != "kthxbye") { + std::cerr << "did not receive final reply correctly\n"; + } + + // take care, don't just use _exit, it will not cleanly destroy context etc! + // if you really need to, at least close and destroy everything properly +// socket->close(); // this gives exception of type zmq::error_t: Socket operation on non-socket + socket.reset(nullptr); + zmqSvc().close_context(); + _Exit(0); + } +} + +TEST_P(AsyncSocketTypes, forkIgnoreSomeMessages) { + bool expect_throw = std::get<3>(GetParam()); + ZmqLingeringSocketPtr<> socket; + auto socket_names = std::get<2>(GetParam()); + pid_t child_pid {0}; + do { + child_pid = fork(); + } while (child_pid == -1); // retry if fork fails + + if (child_pid > 0) { // master + socket.reset(zmqSvc().socket_ptr(std::get<1>(GetParam()).first)); + elaborate_bind(socket, socket_names.second); + // bind is on the master process to avoid zombie children to hold on to binds + + // start test: send 2 things, receive 1, send 1 more, finish + zmqSvc().send(*socket, std::string("breaker breaker")); + + if (expect_throw) { + EXPECT_ANY_THROW(zmqSvc().send(*socket, std::string("anybody out there?"))); + // NOTE: also in case of a throw, be sure to properly close down the connection! + // Otherwise, you may get zombies waiting for a reply. + socket.reset(nullptr); + zmqSvc().close_context(); // if you don't close context in parent process as well, the next repeat will hang + wait_for_child(child_pid, true, 5); + return; + } else { + EXPECT_NO_THROW(zmqSvc().send(*socket, std::string("anybody out there?"))); + } + + auto receipt = zmqSvc().receive(*socket); + + EXPECT_EQ(receipt, 1212); + + zmqSvc().send(*socket, std::string("kthxbye")); + +// socket->close(); // this gives exception of type zmq::error_t: Socket operation on non-socket + socket.reset(nullptr); + zmqSvc().close_context(); // if you don't close context in parent process as well, the next repeat will hang + + wait_for_child(child_pid, true, 5); + } else { // child + socket.reset(zmqSvc().socket_ptr(std::get<1>(GetParam()).second)); + socket->connect(socket_names.first); + + // start test, receive first thing + auto receipt = zmqSvc().receive(*socket); + if (receipt == "breaker breaker") { + zmqSvc().send(*socket, 1212); + } + + // ignore the rest of the sent messages, but give the other end a second to + // actually send its stuff, instead of hanging in retry_send because the + // connection has died; a better solution would be if retry_send (in + // ZeroMQSvc::send) had a callback mechanism that could be used to break + // out when a child has died, but ok + sleep(1); + + // take care, don't just use _exit, it will not cleanly destroy context etc! + // if you really need to, at least close and destroy everything properly +// socket->close(); // this gives exception of type zmq::error_t: Socket operation on non-socket + socket.reset(nullptr); + zmqSvc().close_context(); + _Exit(0); + } +} + + +INSTANTIATE_TEST_SUITE_P(PAIRPAIR, AsyncSocketTypes, + ::testing::Combine(::testing::Range(0, 10), // repeat to probe connection stability + ::testing::Values(std::make_pair(zmq::PAIR, zmq::PAIR)), + socket_name_options, + ::testing::Values(false) // don't expect throw + )); + +INSTANTIATE_TEST_SUITE_P(REQREP, AsyncSocketTypes, + ::testing::Combine(::testing::Values(0), // no repeats, we only care about the throw + ::testing::Values(std::make_pair(zmq::REQ, zmq::REP)), + socket_name_options, + ::testing::Values(true) // expect throw + )); diff --git a/roofit/roofitZMQ/test/test_ZMQ_load_balancing.cxx b/roofit/roofitZMQ/test/test_ZMQ_load_balancing.cxx new file mode 100644 index 0000000000000..ebb4680357104 --- /dev/null +++ b/roofit/roofitZMQ/test/test_ZMQ_load_balancing.cxx @@ -0,0 +1,133 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2016-2021, Netherlands eScience Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "gtest/gtest.h" + +#include // fork, usleep + +#include "RooFit_ZMQ/ZeroMQSvc.h" +#include "RooFit_ZMQ/ZeroMQPoller.h" + +class ZMQPushPullTest : public ::testing::Test { +protected: + std::size_t N_children = 4; + std::size_t max_sends = 20; + + void SetUp() override { + for (std::size_t i = 0; i < N_children; ++i) { + do { + child_pid = fork(); + } while (child_pid == -1); // retry if fork fails + if (child_pid == 0) { // child + child_id = i; + break; + } else { + child_pids.push_back(child_pid); + } + } + + if (child_pid > 0) { // parent + pusher.reset(zmqSvc().socket_ptr(zmq::PUSH)); + pusher->bind("ipc:///tmp/ZMQ_test_push_pull_P2C.ipc"); + } else { // child + puller.reset(zmqSvc().socket_ptr(zmq::PULL)); + puller->connect("ipc:///tmp/ZMQ_test_push_pull_P2C.ipc"); + + poller.register_socket(*puller, zmq::POLLIN); + } + } + + void TearDown() override { + if (child_pid > 0) { // parent + // wait for children + int status = 0; + pid_t pid; + for (pid_t child_pid_i : child_pids) { + do { + pid = waitpid(child_pid_i, &status, 0); + } while (-1 == pid && EINTR == errno); // retry on interrupted system call + if (0 != status) { + if (WIFEXITED(status)) { printf("exited, status=%d\n", WEXITSTATUS(status)); } else if (WIFSIGNALED(status)) { printf("killed by signal %d\n", WTERMSIG(status)); } else if (WIFSTOPPED(status)) { printf("stopped by signal %d\n", WSTOPSIG(status)); } else if (WIFCONTINUED(status)) { printf("continued\n"); } + } + if (-1 == pid) { + throw std::runtime_error(std::string("waitpid, errno ") + std::to_string(errno)); + } + } + pusher.reset(nullptr); + zmqSvc().close_context(); + } else { // child + puller.reset(nullptr); + zmqSvc().close_context(); + } + } + + void run_parent() { + // start test + usleep(1000); // wait a second so that all pull sockets are connected for round-robin distribution + // if you don't wait a second above, the push socket will "round-robin" all the messages to just one or two connected sockets + for (std::size_t ix = 0; ix < max_sends; ++ix) { + zmqSvc().send(*pusher, 0); + } + for (std::size_t ix = 0; ix < N_children; ++ix) { + // end by sending some 1's to all children, to let them know the sending is over + zmqSvc().send(*pusher, 1); + } + } + + void run_child() { + std::size_t count = 0; + for (std::size_t ix = 0; ix < max_sends; ++ix) { + auto r = poller.poll(2000); + if (r.empty()) { + printf("poller of child %d timed out after 2 seconds\n", child_id); + break; + } + auto value = zmqSvc().receive(*puller, ZMQ_DONTWAIT); + usleep(200); // "do some work" + printf("value on child %d: %d\n", child_id, value); + if (value == 1) { + printf("child %d got value %d, done here\n", child_id, value); + break; + } + ++count; + } + printf("child %d got %lu values\n", child_id, count); + } + + pid_t child_pid {0}; + int child_id = -1; + std::vector child_pids; + ZmqLingeringSocketPtr<> pusher, puller; + ZeroMQPoller poller; +}; + + +/// This test shows how push-pull is unsuited for load balancing; messages are just sent to the first available pull socket without any dynamic load balancing +TEST_F(ZMQPushPullTest, demoRoundRobin) { + if (child_pid > 0) { + run_parent(); + } else { + run_child(); + } +} + +/// This test tries to see whether push-pull can be made to work as a bit of a load balancer, using a low HWM at the receiver +TEST_F(ZMQPushPullTest, demoHWM1LoadBalancing) { + if (child_pid > 0) { + run_parent(); + } else { + puller->setsockopt(ZMQ_RCVHWM, 1); + run_child(); + } +} diff --git a/roofit/roofitZMQ/test/test_polling.cxx b/roofit/roofitZMQ/test/test_polling.cxx new file mode 100644 index 0000000000000..9250b2ee6018b --- /dev/null +++ b/roofit/roofitZMQ/test/test_polling.cxx @@ -0,0 +1,149 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2016-2019, Netherlands eScience Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ + + +#include "gtest/gtest.h" + +#include // fork, usleep +#include // signal blocking +#include // strsignal() + +#include + +#include "RooFit_ZMQ/ZeroMQSvc.h" +#include "RooFit_ZMQ/ZeroMQPoller.h" + + +static bool terminated = false; + +void handle_sigterm(int signum) { + terminated = true; + std::cout << "handled signal " << strsignal(signum) << " on PID " << getpid() << std::endl; +} + + +TEST(Polling, doublePoll) { + pid_t child_pid {0}; + do { + child_pid = fork(); + } while (child_pid == -1); // retry if fork fails + + if (child_pid > 0) { // master + sigset_t sigmask, sigmask_old; + sigemptyset(&sigmask); + sigaddset(&sigmask, SIGCHLD); + sigprocmask(SIG_BLOCK, &sigmask, &sigmask_old); + +// std::cout << "master PID: " << getpid() << std::endl; + ZmqLingeringSocketPtr<> pusher, puller; + pusher.reset(zmqSvc().socket_ptr(zmq::PUSH)); + pusher->bind("ipc:///tmp/ZMQ_test_fork_polling_M2C.ipc"); + puller.reset(zmqSvc().socket_ptr(zmq::PULL)); + puller->bind("ipc:///tmp/ZMQ_test_fork_polling_C2M.ipc"); + + ZeroMQPoller poller1, poller2; + poller1.register_socket(*puller, zmq::POLLIN); + poller2.register_socket(*puller, zmq::POLLIN); + + // start test + zmqSvc().send(*pusher, std::string("breaker breaker")); + + auto result1a = poller1.poll(-1); + auto result1b = poller1.poll(-1); + auto result2 = poller2.poll(-1); + EXPECT_EQ(result1a.size(), result1b.size()); + EXPECT_EQ(result1a.size(), result2.size()); + + auto receipt = zmqSvc().receive(*puller, ZMQ_DONTWAIT); + + EXPECT_EQ(receipt, 1212); + + kill(child_pid, SIGTERM); + + // socket->close(); // this gives exception of type zmq::error_t: Socket operation on non-socket + pusher.reset(nullptr); + puller.reset(nullptr); + zmqSvc().close_context(); // if you don't close context in parent process as well, the next repeat will hang + + // wait for child + int status = 0; + pid_t pid; + do { + pid = waitpid(child_pid, &status, 0); + } while (-1 == pid && EINTR == errno); // retry on interrupted system call + + if (0 != status) { + if (WIFEXITED(status)) { printf("exited, status=%d\n", WEXITSTATUS(status)); } else if (WIFSIGNALED(status)) { printf("killed by signal %d\n", WTERMSIG(status)); } else if (WIFSTOPPED(status)) { printf("stopped by signal %d\n", WSTOPSIG(status)); } else if (WIFCONTINUED(status)) { printf("continued\n"); } + } + + if (-1 == pid) { + throw std::runtime_error(std::string("waitpid, errno ") + std::to_string(errno)); + } + + sigprocmask(SIG_SETMASK, &sigmask_old, nullptr); + } else { // child +// std::cout << "child PID: " << getpid() << std::endl; + + sigset_t sigmask, sigmask_old; + sigemptyset(&sigmask); + sigaddset(&sigmask, SIGTERM); + sigprocmask(SIG_BLOCK, &sigmask, &sigmask_old); + + struct sigaction sa; + memset (&sa, '\0', sizeof(sa)); + sa.sa_handler = handle_sigterm; + + if (sigaction(SIGTERM, &sa, NULL) < 0) { + std::perror("sigaction failed"); + std::exit(1); + } + + ZmqLingeringSocketPtr<> puller, pusher; + puller.reset(zmqSvc().socket_ptr(zmq::PULL)); + puller->connect("ipc:///tmp/ZMQ_test_fork_polling_M2C.ipc"); + pusher.reset(zmqSvc().socket_ptr(zmq::PUSH)); + pusher->connect("ipc:///tmp/ZMQ_test_fork_polling_C2M.ipc"); + + ZeroMQPoller poller1, poller2; + poller1.register_socket(*puller, zmq::POLLIN); + poller2.register_socket(*puller, zmq::POLLIN); + + // start test + auto result1a = poller1.poll(-1); + auto result1b = poller1.poll(-1); + auto result2 = poller2.poll(-1); + EXPECT_EQ(result1a.size(), result1b.size()); + EXPECT_EQ(result1a.size(), result2.size()); + + auto receipt = zmqSvc().receive(*puller, ZMQ_DONTWAIT); + if (receipt == "breaker breaker") { + zmqSvc().send(*pusher, 1212); + } + // take care, don't just use _exit, it will not cleanly destroy context etc! + // if you really need to, at least close and destroy everything properly + // socket->close(); // this gives exception of type zmq::error_t: Socket operation on non-socket + + sigprocmask(SIG_SETMASK, &sigmask_old, nullptr); + + while (!terminated) {} + +// std::cout << "child terminated" << std::endl; + + puller.reset(nullptr); + pusher.reset(nullptr); + zmqSvc().close_context(); + _Exit(0); + } + +} From fcb329f06dd5b7858ebee9050684a40b847124b4 Mon Sep 17 00:00:00 2001 From: "E. G. Patrick Bos" Date: Thu, 10 Jun 2021 16:54:54 +0200 Subject: [PATCH 06/15] add ppoll to RooFitZMQ This is a reimplementation of ZMQ's poll function that mimics the POSIX ppoll function. The difference with regular poll is that ppoll blocks incoming POSIX signals, allowing graceful handling of these signals. We use this in RooFit::MultiProcess to shut down forked child processes. The motivation behind this addition is discussed more in this blog post: https://blog.esciencecenter.nl/combining-zeromq-posix-signals-b754f6f29cd6, or more succinctly in Martin Sustrik's blog post, in the edit at the bottom: https://250bpm.com/blog:12/ --- roofit/roofitZMQ/CMakeLists.txt | 6 +- roofit/roofitZMQ/inc/RooFit_ZMQ/ppoll.h | 29 ++ roofit/roofitZMQ/src/ppoll.cpp | 345 ++++++++++++++++++++++++ 3 files changed, 376 insertions(+), 4 deletions(-) create mode 100644 roofit/roofitZMQ/inc/RooFit_ZMQ/ppoll.h create mode 100644 roofit/roofitZMQ/src/ppoll.cpp diff --git a/roofit/roofitZMQ/CMakeLists.txt b/roofit/roofitZMQ/CMakeLists.txt index 8174ecb952418..81fee8d871a92 100644 --- a/roofit/roofitZMQ/CMakeLists.txt +++ b/roofit/roofitZMQ/CMakeLists.txt @@ -13,19 +13,17 @@ ROOT_STANDARD_LIBRARY_PACKAGE(RooFitZMQ RooFit_ZMQ/ZeroMQSvc.h RooFit_ZMQ/functions.h RooFit_ZMQ/ZeroMQPoller.h -# RooFit_ZMQ/ppoll.h + RooFit_ZMQ/ppoll.h SOURCES src/ZeroMQSvc.cpp src/ZeroMQPoller.cpp src/functions.cpp -# src/ppoll.cpp + src/ppoll.cpp DICTIONARY_OPTIONS -# -I${ZeroMQ_INCLUDE_DIR} -writeEmptyRootPCM DEPENDENCIES Core # TVersionCheck, RegisterModule, these are always called from G__RooFitZMQ.cxx.o LIBRARIES -# ${ZeroMQ_LIBRARY} ZeroMQ ) diff --git a/roofit/roofitZMQ/inc/RooFit_ZMQ/ppoll.h b/roofit/roofitZMQ/inc/RooFit_ZMQ/ppoll.h new file mode 100644 index 0000000000000..c00a1ccfebd7f --- /dev/null +++ b/roofit/roofitZMQ/inc/RooFit_ZMQ/ppoll.h @@ -0,0 +1,29 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2016-2019, Netherlands eScience Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ +#ifndef ROOT_ROOFIT_ZMQ_ppoll +#define ROOT_ROOFIT_ZMQ_ppoll + +#include +#include + +namespace ZMQ { + +int zmq_ppoll (zmq_pollitem_t *items_, int nitems_, long timeout_, const sigset_t * sigmask_); +int ppoll(zmq_pollitem_t *items_, size_t nitems_, long timeout_, const sigset_t * sigmask_); +int ppoll(std::vector &items, long timeout_, const sigset_t * sigmask_); +class ppoll_error_t : public zmq::error_t {}; + +} + +#endif // ROOT_ROOFIT_ZMQ_ppoll diff --git a/roofit/roofitZMQ/src/ppoll.cpp b/roofit/roofitZMQ/src/ppoll.cpp new file mode 100644 index 0000000000000..5cfcda16222e1 --- /dev/null +++ b/roofit/roofitZMQ/src/ppoll.cpp @@ -0,0 +1,345 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2016-2019, Netherlands eScience Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include // pselect +#include // timespec +#include + +#include "likely.hpp" // unlikely +#include "clock.hpp" // zmq::clock_t +#include "err.hpp" // zmq_assert +#include "polling_util.hpp" // zmq::optimized_fd_set_t, zmq::compute_timeout, zmq::timeout_t +#include "fd.hpp" // zmq::fd_t + +#include + + +namespace ZMQ { + +int zmq_ppoll (zmq_pollitem_t *items_, int nitems_, long timeout_, + const sigset_t * sigmask_) +{ +#if defined ZMQ_HAVE_POLLER + // if poller is present, use that if there is at least 1 thread-safe socket, + // otherwise fall back to the previous implementation as it's faster. + for (int i = 0; i != nitems_; i++) { + if (items_[i].socket + && as_socket_base_t (items_[i].socket)->is_thread_safe ()) { + return zmq_poller_poll (items_, nitems_, timeout_); + } + } +#endif // ZMQ_HAVE_POLLER +#if defined ZMQ_POLL_BASED_ON_POLL || defined ZMQ_POLL_BASED_ON_SELECT + if (unlikely (nitems_ < 0)) { + errno = EINVAL; + return -1; + } + if (unlikely (nitems_ == 0)) { + if (timeout_ == 0) + return 0; +#if defined ZMQ_HAVE_WINDOWS + Sleep (timeout_ > 0 ? timeout_ : INFINITE); + return 0; +#elif defined ZMQ_HAVE_VXWORKS + struct timespec ns_; + ns_.tv_sec = timeout_ / 1000; + ns_.tv_nsec = timeout_ % 1000 * 1000000; + return nanosleep (&ns_, 0); +#else + return usleep (timeout_ * 1000); +#endif + } + if (!items_) { + errno = EFAULT; + return -1; + } + + zmq::clock_t clock; + uint64_t now = 0; + uint64_t end = 0; +#if defined ZMQ_POLL_BASED_ON_POLL + zmq::fast_vector_t pollfds (nitems_); + + // Build pollset for poll () system call. + for (int i = 0; i != nitems_; i++) { + // If the poll item is a 0MQ socket, we poll on the file descriptor + // retrieved by the ZMQ_FD socket option. + if (items_[i].socket) { + size_t zmq_fd_size = sizeof (zmq::fd_t); + if (zmq_getsockopt (items_[i].socket, ZMQ_FD, &pollfds[i].fd, + &zmq_fd_size) + == -1) { + return -1; + } + pollfds[i].events = items_[i].events ? POLLIN : 0; + } + // Else, the poll item is a raw file descriptor. Just convert the + // events to normal POLLIN/POLLOUT for poll (). + else { + pollfds[i].fd = items_[i].fd; + pollfds[i].events = + (items_[i].events & ZMQ_POLLIN ? POLLIN : 0) + | (items_[i].events & ZMQ_POLLOUT ? POLLOUT : 0) + | (items_[i].events & ZMQ_POLLPRI ? POLLPRI : 0); + } + } +#else + // Ensure we do not attempt to select () on more than FD_SETSIZE + // file descriptors. + // TODO since this function is called by a client, we could return errno EINVAL/ENOMEM/... here + zmq_assert (nitems_ <= FD_SETSIZE); + + zmq::optimized_fd_set_t pollset_in (nitems_); + FD_ZERO (pollset_in.get ()); + zmq::optimized_fd_set_t pollset_out (nitems_); + FD_ZERO (pollset_out.get ()); + zmq::optimized_fd_set_t pollset_err (nitems_); + FD_ZERO (pollset_err.get ()); + + zmq::fd_t maxfd = 0; + + // Build the fd_sets for passing to select (). + for (int i = 0; i != nitems_; i++) { + // If the poll item is a 0MQ socket we are interested in input on the + // notification file descriptor retrieved by the ZMQ_FD socket option. + if (items_[i].socket) { + size_t zmq_fd_size = sizeof (zmq::fd_t); + zmq::fd_t notify_fd; + if (zmq_getsockopt (items_[i].socket, ZMQ_FD, ¬ify_fd, + &zmq_fd_size) + == -1) + return -1; + if (items_[i].events) { + FD_SET (notify_fd, pollset_in.get ()); + if (maxfd < notify_fd) + maxfd = notify_fd; + } + } + // Else, the poll item is a raw file descriptor. Convert the poll item + // events to the appropriate fd_sets. + else { + if (items_[i].events & ZMQ_POLLIN) + FD_SET (items_[i].fd, pollset_in.get ()); + if (items_[i].events & ZMQ_POLLOUT) + FD_SET (items_[i].fd, pollset_out.get ()); + if (items_[i].events & ZMQ_POLLERR) + FD_SET (items_[i].fd, pollset_err.get ()); + if (maxfd < items_[i].fd) + maxfd = items_[i].fd; + } + } + + zmq::optimized_fd_set_t inset (nitems_); + zmq::optimized_fd_set_t outset (nitems_); + zmq::optimized_fd_set_t errset (nitems_); +#endif + + bool first_pass = true; + int nevents = 0; + + while (true) { +#if defined ZMQ_POLL_BASED_ON_POLL + + // Compute the timeout for the subsequent poll. + zmq::timeout_t timeout = + zmq::compute_timeout (first_pass, timeout_, now, end); + + // Wait for events. + { + int rc = ppoll (&pollfds[0], nitems_, timeout); + if (rc == -1 && errno == EINTR) { + return -1; + } + errno_assert (rc >= 0); + } + // Check for the events. + for (int i = 0; i != nitems_; i++) { + items_[i].revents = 0; + + // The poll item is a 0MQ socket. Retrieve pending events + // using the ZMQ_EVENTS socket option. + if (items_[i].socket) { + size_t zmq_events_size = sizeof (uint32_t); + uint32_t zmq_events; + if (zmq_getsockopt (items_[i].socket, ZMQ_EVENTS, &zmq_events, + &zmq_events_size) + == -1) { + return -1; + } + if ((items_[i].events & ZMQ_POLLOUT) + && (zmq_events & ZMQ_POLLOUT)) + items_[i].revents |= ZMQ_POLLOUT; + if ((items_[i].events & ZMQ_POLLIN) + && (zmq_events & ZMQ_POLLIN)) + items_[i].revents |= ZMQ_POLLIN; + } + // Else, the poll item is a raw file descriptor, simply convert + // the events to zmq_pollitem_t-style format. + else { + if (pollfds[i].revents & POLLIN) + items_[i].revents |= ZMQ_POLLIN; + if (pollfds[i].revents & POLLOUT) + items_[i].revents |= ZMQ_POLLOUT; + if (pollfds[i].revents & POLLPRI) + items_[i].revents |= ZMQ_POLLPRI; + if (pollfds[i].revents & ~(POLLIN | POLLOUT | POLLPRI)) + items_[i].revents |= ZMQ_POLLERR; + } + + if (items_[i].revents) + nevents++; + } + +#else + + // Compute the timeout for the subsequent poll. + struct timespec timeout; + struct timespec *ptimeout; + if (first_pass) { + timeout.tv_sec = 0; + timeout.tv_nsec = 0; + ptimeout = &timeout; + } else if (timeout_ < 0) + ptimeout = NULL; + else { + timeout.tv_sec = static_cast ((end - now) / 1000); + timeout.tv_nsec = static_cast ((end - now) % 1000 * 1000000); + ptimeout = &timeout; + } + + // Wait for events. Ignore interrupts if there's infinite timeout. + while (true) { + memcpy (inset.get (), pollset_in.get (), + zmq::valid_pollset_bytes (*pollset_in.get ())); + memcpy (outset.get (), pollset_out.get (), + zmq::valid_pollset_bytes (*pollset_out.get ())); + memcpy (errset.get (), pollset_err.get (), + zmq::valid_pollset_bytes (*pollset_err.get ())); +#if defined ZMQ_HAVE_WINDOWS + int rc = + select (0, inset.get (), outset.get (), errset.get (), ptimeout); + if (unlikely (rc == SOCKET_ERROR)) { + errno = zmq::wsa_error_to_errno (WSAGetLastError ()); + wsa_assert (errno == ENOTSOCK); + return -1; + } +#else + int rc = pselect (maxfd + 1, inset.get (), outset.get (), + errset.get (), ptimeout, sigmask_); + if (unlikely (rc == -1)) { + errno_assert (errno == EINTR || errno == EBADF); + return -1; + } +#endif + break; + } + + // Check for the events. + for (int i = 0; i != nitems_; i++) { + items_[i].revents = 0; + + // The poll item is a 0MQ socket. Retrieve pending events + // using the ZMQ_EVENTS socket option. + if (items_[i].socket) { + size_t zmq_events_size = sizeof (uint32_t); + uint32_t zmq_events; + if (zmq_getsockopt (items_[i].socket, ZMQ_EVENTS, &zmq_events, + &zmq_events_size) + == -1) + return -1; + if ((items_[i].events & ZMQ_POLLOUT) + && (zmq_events & ZMQ_POLLOUT)) + items_[i].revents |= ZMQ_POLLOUT; + if ((items_[i].events & ZMQ_POLLIN) + && (zmq_events & ZMQ_POLLIN)) + items_[i].revents |= ZMQ_POLLIN; + } + // Else, the poll item is a raw file descriptor, simply convert + // the events to zmq_pollitem_t-style format. + else { + if (FD_ISSET (items_[i].fd, inset.get ())) + items_[i].revents |= ZMQ_POLLIN; + if (FD_ISSET (items_[i].fd, outset.get ())) + items_[i].revents |= ZMQ_POLLOUT; + if (FD_ISSET (items_[i].fd, errset.get ())) + items_[i].revents |= ZMQ_POLLERR; + } + + if (items_[i].revents) + nevents++; + } +#endif + + // If timeout is zero, exit immediately whether there are events or not. + if (timeout_ == 0) + break; + + // If there are events to return, we can exit immediately. + if (nevents) + break; + + // At this point we are meant to wait for events but there are none. + // If timeout is infinite we can just loop until we get some events. + if (timeout_ < 0) { + if (first_pass) + first_pass = false; + continue; + } + + // The timeout is finite and there are no events. In the first pass + // we get a timestamp of when the polling have begun. (We assume that + // first pass have taken negligible time). We also compute the time + // when the polling should time out. + if (first_pass) { + now = clock.now_ms (); + end = now + timeout_; + if (now == end) + break; + first_pass = false; + continue; + } + + // Find out whether timeout have expired. + now = clock.now_ms (); + if (now >= end) + break; + } + + return nevents; +#else + // Exotic platforms that support neither poll() nor select(). + errno = ENOTSUP; + return -1; +#endif +} + + +// This function can throw, so wrap in try-catch! +int ppoll(zmq_pollitem_t *items_, size_t nitems_, long timeout_, const sigset_t * sigmask_) +{ + int rc = zmq_ppoll(items_, static_cast(nitems_), timeout_, sigmask_); + if (rc < 0) + throw ppoll_error_t(); + return rc; +} + + +// This function can throw, so wrap in try-catch! +int ppoll(std::vector &items, long timeout_, const sigset_t * sigmask_) +{ + return ppoll(items.data(), items.size(), timeout_, sigmask_); +} + +} + From 81b3ad64556edfc2c1b5880070edee21af47af27 Mon Sep 17 00:00:00 2001 From: "E. G. Patrick Bos" Date: Thu, 10 Jun 2021 20:34:43 +0200 Subject: [PATCH 07/15] move ZeroMQ builtin to builtins directory and add build option The option builtin_zeromq is set to ON by default and controls whether RooFitZMQ is built. At some point, we should also add the option to use a system installed ZeroMQ, so that people can avoid the built-in, if they wish, e.g. for the conda-forge build. --- {roofit/builtin_zeromq => builtins/zeromq}/CMakeLists.txt | 2 ++ cmake/modules/RootBuildOptions.cmake | 2 ++ cmake/modules/SearchInstalledSoftware.cmake | 7 +++++++ roofit/CMakeLists.txt | 5 +++-- 4 files changed, 14 insertions(+), 2 deletions(-) rename {roofit/builtin_zeromq => builtins/zeromq}/CMakeLists.txt (96%) diff --git a/roofit/builtin_zeromq/CMakeLists.txt b/builtins/zeromq/CMakeLists.txt similarity index 96% rename from roofit/builtin_zeromq/CMakeLists.txt rename to builtins/zeromq/CMakeLists.txt index 9b769f3d00e95..5ec3b60f59a22 100644 --- a/roofit/builtin_zeromq/CMakeLists.txt +++ b/builtins/zeromq/CMakeLists.txt @@ -31,6 +31,7 @@ target_link_libraries(ZeroMQ INTERFACE $) add_dependencies(ZeroMQ BUILTIN_ZeroMQ) add_library(ZeroMQ::ZeroMQ ALIAS ZeroMQ) +add_library(libzmq ALIAS ZeroMQ) set(ZeroMQ_DIR ${ZeroMQ_PREFIX}/share/cmake/ZeroMQ CACHE INTERNAL "" FORCE) @@ -43,4 +44,5 @@ ExternalProject_Add(BUILTIN_cppzmq CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${install_dir} -DCPPZMQ_BUILD_TESTS=OFF + DEPENDS BUILTIN_ZeroMQ ) diff --git a/cmake/modules/RootBuildOptions.cmake b/cmake/modules/RootBuildOptions.cmake index 1cc9f48926367..cd157236e552e 100644 --- a/cmake/modules/RootBuildOptions.cmake +++ b/cmake/modules/RootBuildOptions.cmake @@ -107,6 +107,7 @@ ROOT_BUILD_OPTION(builtin_vdt OFF "Build VDT internally (requires network)") ROOT_BUILD_OPTION(builtin_veccore OFF "Build VecCore internally (requires network)") ROOT_BUILD_OPTION(builtin_xrootd OFF "Build XRootD internally (requires network)") ROOT_BUILD_OPTION(builtin_xxhash OFF "Build bundled copy of xxHash") +ROOT_BUILD_OPTION(builtin_zeromq ON "Build ZeroMQ internally (requires network)") ROOT_BUILD_OPTION(builtin_zlib OFF "Build bundled copy of zlib") ROOT_BUILD_OPTION(builtin_zstd OFF "Build included libzstd, or use system libzstd") ROOT_BUILD_OPTION(ccache OFF "Enable ccache usage for speeding up builds") @@ -299,6 +300,7 @@ if(builtin_all) set(builtin_veccore_defvalue ON) set(builtin_xrootd_defvalue ON) set(builtin_xxhash_defvalue ON) + set(builtin_zeromq_defvalue ON) set(builtin_zlib_defvalue ON) set(builtin_zstd_defvalue ON) endif() diff --git a/cmake/modules/SearchInstalledSoftware.cmake b/cmake/modules/SearchInstalledSoftware.cmake index c64ab05414e21..0624a00fbb3a3 100644 --- a/cmake/modules/SearchInstalledSoftware.cmake +++ b/cmake/modules/SearchInstalledSoftware.cmake @@ -1767,6 +1767,13 @@ if (mpi) endif() endif() +#---Check for ZeroMQ----------------------------------------------------------- + +if(builtin_zeromq) + list(APPEND ROOT_BUILTINS ZeroMQ) + add_subdirectory(builtins/zeromq) +endif() + #---Download googletest-------------------------------------------------------------- if (testing) # FIXME: Remove our version of gtest in roottest. We can reuse this one. diff --git a/roofit/CMakeLists.txt b/roofit/CMakeLists.txt index 886761c71ee09..cec391f22a510 100644 --- a/roofit/CMakeLists.txt +++ b/roofit/CMakeLists.txt @@ -5,8 +5,9 @@ # For the list of contributors see $ROOTSYS/README/CREDITS. add_subdirectory(batchcompute) -add_subdirectory(builtin_zeromq) -add_subdirectory(roofitZMQ) +if (builtin_zeromq) + add_subdirectory(roofitZMQ) +endif(builtin_zeromq) add_subdirectory(roofitcore) add_subdirectory(roofit) if(mathmore) From 2bddfa40876fbe257d2db621c870e98e3c86bb0f Mon Sep 17 00:00:00 2001 From: "E. G. Patrick Bos" Date: Fri, 11 Jun 2021 11:43:14 +0200 Subject: [PATCH 08/15] add ZeroMQPoller::ppoll implementation Should have been added in ppoll commit before. --- roofit/roofitZMQ/src/ZeroMQPoller.cpp | 31 +++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/roofit/roofitZMQ/src/ZeroMQPoller.cpp b/roofit/roofitZMQ/src/ZeroMQPoller.cpp index 12eafaf09392d..29a0dd7d610c7 100644 --- a/roofit/roofitZMQ/src/ZeroMQPoller.cpp +++ b/roofit/roofitZMQ/src/ZeroMQPoller.cpp @@ -8,6 +8,7 @@ #include +#include "RooFit_ZMQ/ppoll.h" #include "RooFit_ZMQ/ZeroMQPoller.h" std::vector> ZeroMQPoller::poll(int timeo) { @@ -48,6 +49,36 @@ std::vector> ZeroMQPoller::poll(int timeo) { return r; } +// This function can throw (from inside ZMQ::ppoll), so wrap in try-catch! +std::vector> ZeroMQPoller::ppoll(int timeo, const sigset_t * sigmask_) { + if (m_items.empty()) { + throw std::runtime_error("No sockets registered"); + } + + std::vector> r; + + auto n = ZMQ::ppoll(m_items, timeo, sigmask_); + if (n == 0) return r; + + for (auto& m_item : m_items) { + size_t index = 0; + int flags = 0; + if (m_item.socket == nullptr) { + // an fd was registered + std::tie(index, flags) = m_fds[m_item.fd]; + } else { + // a socket was registered + const zmq::socket_t* s; + std::tie(index, flags, s) = m_sockets[m_item.socket]; + } + if (m_item.revents & short(flags)) { + r.emplace_back(index, flags); + } + } + return r; +} + + size_t ZeroMQPoller::size() const { return m_items.size(); } From aa83033cc6c12a36dda5f77be9084b3f7594ce1b Mon Sep 17 00:00:00 2001 From: "E. G. Patrick Bos" Date: Fri, 11 Jun 2021 12:30:23 +0200 Subject: [PATCH 09/15] address DeepCode issues: properly rethrow exceptions --- roofit/roofitZMQ/src/ZeroMQSvc.cpp | 6 +++--- roofit/roofitZMQ/test/test_ZMQ.cpp | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/roofit/roofitZMQ/src/ZeroMQSvc.cpp b/roofit/roofitZMQ/src/ZeroMQSvc.cpp index b69695349e0c3..37753fb2f386b 100644 --- a/roofit/roofitZMQ/src/ZeroMQSvc.cpp +++ b/roofit/roofitZMQ/src/ZeroMQSvc.cpp @@ -26,7 +26,7 @@ zmq::context_t& ZeroMQSvc::context() const { m_context = new zmq::context_t; } catch (zmq::error_t& e) { std::cerr << "ERROR: Creating ZeroMQ context failed. This only happens when PGM initialization failed or when a nullptr was returned from zmq_ctx_new because the created context was invalid. Contact ZMQ experts when this happens, because it shouldn't.\n"; - throw e; + throw; } } return *m_context; @@ -39,7 +39,7 @@ zmq::socket_t ZeroMQSvc::socket(int type) const { } catch (zmq::error_t& e) { // all zmq errors not recoverable from here, only at call site std::cerr << "ERROR in ZeroMQSvc::socket: " << e.what() << " (errno: " << e.num() << ")\n"; - throw e; + throw; } } @@ -50,7 +50,7 @@ zmq::socket_t* ZeroMQSvc::socket_ptr(int type) const { } catch (zmq::error_t& e) { // all zmq errors not recoverable from here, only at call site std::cerr << "ERROR in ZeroMQSvc::socket_ptr: " << e.what() << " (errno: " << e.num() << ")\n"; - throw e; + throw; } } diff --git a/roofit/roofitZMQ/test/test_ZMQ.cpp b/roofit/roofitZMQ/test/test_ZMQ.cpp index 985d1b91c5902..2cf6751968df9 100644 --- a/roofit/roofitZMQ/test/test_ZMQ.cpp +++ b/roofit/roofitZMQ/test/test_ZMQ.cpp @@ -73,7 +73,7 @@ void elaborate_bind(const ZmqLingeringSocketPtr<>& socket, std::string name) { std::cerr << "again: address already in use, aborting; please check whether there are any remaining improperly exited processes (zombies) around or whether some other program is using port 6660\n"; } - throw e2; + throw; } // Sometimes, the socket from the previous test needs some time to close, so // we introduce a latency here. A more robust and fast approach might be to @@ -86,7 +86,7 @@ void elaborate_bind(const ZmqLingeringSocketPtr<>& socket, std::string name) { // life. If one comes alive, transfer ownership of that pointer to the pointer // you want to eventually use (`socket`) and that's it. } else { - throw e; + throw; } } } From c95ee938f37eb4548e03c1c84622ef297affac0a Mon Sep 17 00:00:00 2001 From: "E. G. Patrick Bos" Date: Fri, 11 Jun 2021 12:33:30 +0200 Subject: [PATCH 10/15] pass on ROOT cxx_compiler to ZeroMQ built-in As suggested by @guitargeek. --- builtins/zeromq/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/builtins/zeromq/CMakeLists.txt b/builtins/zeromq/CMakeLists.txt index 5ec3b60f59a22..8b12714b9d9cf 100644 --- a/builtins/zeromq/CMakeLists.txt +++ b/builtins/zeromq/CMakeLists.txt @@ -13,6 +13,7 @@ ExternalProject_Add(BUILTIN_ZeroMQ -DWITH_PERF_TOOL=OFF -DZMQ_BUILD_TESTS=OFF -DPOLLER=select + -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} # TODO: remove -w flag when updating to v4.3.5 or higher, current warnings should be gone by then -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}\ -w BUILD_BYPRODUCTS ${ZeroMQ_LIBRARIES} From 0c0dc78fbe78e198ace3814fd623802e79d9ea4c Mon Sep 17 00:00:00 2001 From: "E. G. Patrick Bos" Date: Fri, 11 Jun 2021 12:44:56 +0200 Subject: [PATCH 11/15] include header corresponding to implementation file before others As suggested by @guitargeek. --- roofit/roofitZMQ/src/ZeroMQPoller.cpp | 3 ++- roofit/roofitZMQ/src/ZeroMQSvc.cpp | 5 ++--- roofit/roofitZMQ/src/functions.cpp | 3 ++- roofit/roofitZMQ/src/ppoll.cpp | 4 ++-- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/roofit/roofitZMQ/src/ZeroMQPoller.cpp b/roofit/roofitZMQ/src/ZeroMQPoller.cpp index 29a0dd7d610c7..964961bf21613 100644 --- a/roofit/roofitZMQ/src/ZeroMQPoller.cpp +++ b/roofit/roofitZMQ/src/ZeroMQPoller.cpp @@ -6,10 +6,11 @@ * The full license is in the file LICENSE, distributed with this software. */ +#include "RooFit_ZMQ/ZeroMQPoller.h" + #include #include "RooFit_ZMQ/ppoll.h" -#include "RooFit_ZMQ/ZeroMQPoller.h" std::vector> ZeroMQPoller::poll(int timeo) { std::vector> r; diff --git a/roofit/roofitZMQ/src/ZeroMQSvc.cpp b/roofit/roofitZMQ/src/ZeroMQSvc.cpp index 37753fb2f386b..db6e0b7103bc5 100644 --- a/roofit/roofitZMQ/src/ZeroMQSvc.cpp +++ b/roofit/roofitZMQ/src/ZeroMQSvc.cpp @@ -1,9 +1,8 @@ -//#include +#include "RooFit_ZMQ/ZeroMQSvc.h" + #include // std::ref #include -#include "RooFit_ZMQ/ZeroMQSvc.h" - ZeroMQSvc& zmqSvc() { static std::unique_ptr svc; if (!svc) { diff --git a/roofit/roofitZMQ/src/functions.cpp b/roofit/roofitZMQ/src/functions.cpp index ebb6199bfdb5d..eded4cebd33d5 100644 --- a/roofit/roofitZMQ/src/functions.cpp +++ b/roofit/roofitZMQ/src/functions.cpp @@ -1,6 +1,7 @@ +#include "RooFit_ZMQ/functions.h" + #include #include "RooFit_ZMQ/ZeroMQSvc.h" -#include "RooFit_ZMQ/functions.h" namespace ZMQ { size_t stringLength(const char& cs) { diff --git a/roofit/roofitZMQ/src/ppoll.cpp b/roofit/roofitZMQ/src/ppoll.cpp index 5cfcda16222e1..a753ed9628143 100644 --- a/roofit/roofitZMQ/src/ppoll.cpp +++ b/roofit/roofitZMQ/src/ppoll.cpp @@ -12,6 +12,8 @@ * http://www.apache.org/licenses/LICENSE-2.0 */ +#include + #include // pselect #include // timespec #include @@ -22,8 +24,6 @@ #include "polling_util.hpp" // zmq::optimized_fd_set_t, zmq::compute_timeout, zmq::timeout_t #include "fd.hpp" // zmq::fd_t -#include - namespace ZMQ { From fe6ed68f5966c3ad4ef044c0206cd5b8997ff9ab Mon Sep 17 00:00:00 2001 From: "E. G. Patrick Bos" Date: Wed, 23 Jun 2021 17:15:21 +0200 Subject: [PATCH 12/15] add ZeroMQ to CI header copying Without this, the clang-tidy checks will fail because they cannot include zmq.hpp. --- .ci/copy_headers.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/copy_headers.sh b/.ci/copy_headers.sh index fd4ec82b96e9d..a6587b6ee1dbb 100755 --- a/.ci/copy_headers.sh +++ b/.ci/copy_headers.sh @@ -10,6 +10,6 @@ cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -Dall=On -Dtesting=On -Dx11=Off -Dalien # We need to prebuild a minimal set of targets which are responsible for header copy # or generation. make -j4 move_headers intrinsics_gen clang-tablegen-targets ClangDriverOptions \ - googletest Dictgen BaseTROOT + googletest Dictgen BaseTROOT BUILTIN_cppzmq ln -s $PWD/compile_commands.json $PWD/../root/ From bcdf91e076d583918e93242103f51117316e46d4 Mon Sep 17 00:00:00 2001 From: "E. G. Patrick Bos" Date: Wed, 23 Jun 2021 20:14:01 +0200 Subject: [PATCH 13/15] clang-format on RooFitZMQ --- roofit/roofitZMQ/inc/RooFit_ZMQ/Utility.h | 27 +- .../roofitZMQ/inc/RooFit_ZMQ/ZeroMQPoller.h | 12 +- roofit/roofitZMQ/inc/RooFit_ZMQ/ZeroMQSvc.h | 187 ++++--- roofit/roofitZMQ/inc/RooFit_ZMQ/functions.h | 171 +++--- roofit/roofitZMQ/inc/RooFit_ZMQ/ppoll.h | 11 +- roofit/roofitZMQ/src/ZeroMQPoller.cpp | 236 ++++---- roofit/roofitZMQ/src/ZeroMQSvc.cpp | 118 ++-- roofit/roofitZMQ/src/functions.cpp | 7 +- roofit/roofitZMQ/src/ppoll.cpp | 523 +++++++++--------- roofit/roofitZMQ/test/test_HWM.cxx | 51 +- roofit/roofitZMQ/test/test_ZMQ.cpp | 456 +++++++-------- .../test/test_ZMQ_load_balancing.cxx | 52 +- roofit/roofitZMQ/test/test_polling.cxx | 39 +- 13 files changed, 958 insertions(+), 932 deletions(-) diff --git a/roofit/roofitZMQ/inc/RooFit_ZMQ/Utility.h b/roofit/roofitZMQ/inc/RooFit_ZMQ/Utility.h index a329805967df6..58a9df24f6e7e 100644 --- a/roofit/roofitZMQ/inc/RooFit_ZMQ/Utility.h +++ b/roofit/roofitZMQ/inc/RooFit_ZMQ/Utility.h @@ -12,28 +12,23 @@ #include namespace ZMQ { - namespace Detail { +namespace Detail { #if defined(HAVE_TRIVIALLY_COPYABLE) - template - using simple_object = std::is_trivially_copyable; +template +using simple_object = std::is_trivially_copyable; #else - template - using simple_object = std::is_pod; +template +using simple_object = std::is_pod; #endif // is trivial - template - struct is_trivial : std::conditional< - simple_object< - typename std::decay::type - >::value, - std::true_type, - std::false_type - >::type { - }; +template +struct is_trivial + : std::conditional::type>::value, std::true_type, std::false_type>::type { +}; - } -} +} // namespace Detail +} // namespace ZMQ #endif // SERIALIZE_UTILITY_H diff --git a/roofit/roofitZMQ/inc/RooFit_ZMQ/ZeroMQPoller.h b/roofit/roofitZMQ/inc/RooFit_ZMQ/ZeroMQPoller.h index 4d14444c01553..a4709aca2c5f4 100644 --- a/roofit/roofitZMQ/inc/RooFit_ZMQ/ZeroMQPoller.h +++ b/roofit/roofitZMQ/inc/RooFit_ZMQ/ZeroMQPoller.h @@ -10,11 +10,10 @@ class ZeroMQPoller { public: - - using entry_t = std::tuple; + using entry_t = std::tuple; // The key is what zmq::socket_t stores inside, and what goes into // pollitem_t through zmq::socket_t's conversion to void* operator - using sockets_t = std::unordered_map; + using sockets_t = std::unordered_map; using fd_entry_t = std::tuple; using fds_t = std::unordered_map; @@ -24,18 +23,17 @@ class ZeroMQPoller { ZeroMQPoller() = default; std::vector> poll(int timeo = -1); - std::vector> ppoll(int timeo, const sigset_t * sigmask_); + std::vector> ppoll(int timeo, const sigset_t *sigmask_); size_t size() const; - size_t register_socket(zmq::socket_t& socket, zmq::PollType type); + size_t register_socket(zmq::socket_t &socket, zmq::PollType type); size_t register_socket(int fd, zmq::PollType type); - size_t unregister_socket(zmq::socket_t& socket); + size_t unregister_socket(zmq::socket_t &socket); size_t unregister_socket(int fd); private: - // Vector of (socket, flags) std::vector m_items; sockets_t m_sockets; diff --git a/roofit/roofitZMQ/inc/RooFit_ZMQ/ZeroMQSvc.h b/roofit/roofitZMQ/inc/RooFit_ZMQ/ZeroMQSvc.h index 0eace37b7f6c4..ef31e68f85b42 100644 --- a/roofit/roofitZMQ/inc/RooFit_ZMQ/ZeroMQSvc.h +++ b/roofit/roofitZMQ/inc/RooFit_ZMQ/ZeroMQSvc.h @@ -8,7 +8,7 @@ #include #include #include -#include // std::cerr +#include // std::cerr // ZeroMQ #include @@ -22,41 +22,44 @@ namespace ZMQ { struct TimeOutException : std::exception { TimeOutException() = default; - TimeOutException(const TimeOutException&) = default; + TimeOutException(const TimeOutException &) = default; ~TimeOutException() = default; - TimeOutException& operator=(const TimeOutException&) = default; + TimeOutException &operator=(const TimeOutException &) = default; }; struct MoreException : std::exception { MoreException() = default; - MoreException(const MoreException&) = default; + MoreException(const MoreException &) = default; ~MoreException() = default; - MoreException& operator=(const MoreException&) = default; + MoreException &operator=(const MoreException &) = default; }; -} +} // namespace ZMQ template struct ZmqLingeringSocketPtrDeleter { - void operator()(zmq::socket_t* socket) { + void operator()(zmq::socket_t *socket) + { auto period = PERIOD; int tries = 0; int max_tries = 3; - while(true) { - try { - // the actual work this function should do, plus the delete socket below: - if (socket) socket->setsockopt(ZMQ_LINGER, &period, sizeof(period)); - break; - } catch (zmq::error_t& e) { - if (++tries == max_tries - || e.num() == EINVAL || e.num() == ETERM || e.num() == ENOTSOCK // not recoverable from here - ) { - std::cerr << "ERROR in ZeroMQSvc::socket: " << e.what() << " (errno: " << e.num() << ")\n"; - throw; + while (true) { + try { + // the actual work this function should do, plus the delete socket below: + if (socket) + socket->setsockopt(ZMQ_LINGER, &period, sizeof(period)); + break; + } catch (zmq::error_t &e) { + if (++tries == max_tries || e.num() == EINVAL || e.num() == ETERM || + e.num() == ENOTSOCK // not recoverable from here + ) { + std::cerr << "ERROR in ZeroMQSvc::socket: " << e.what() << " (errno: " << e.num() << ")\n"; + throw; + } + std::cerr << "RETRY " << tries << "/" << (max_tries - 1) + << " in ZmqLingeringSocketPtrDeleter: call interrupted (errno: " << e.num() << ")\n"; } - std::cerr << "RETRY " << tries << "/" << (max_tries - 1) << " in ZmqLingeringSocketPtrDeleter: call interrupted (errno: " << e.num() << ")\n"; - } } delete socket; @@ -66,48 +69,50 @@ struct ZmqLingeringSocketPtrDeleter { template using ZmqLingeringSocketPtr = std::unique_ptr>; - // We retry send and receive only on EINTR, all other errors are either fatal, or can only // be handled at the caller. template -auto retry_send(zmq::socket_t& socket, int max_tries, args_t ...args) -> decltype(socket.send(args...)) { - int tries = 0; - while(true) { - try { - // the actual work this function should do, all the rest is error handling: - return socket.send(args...); - } catch (zmq::error_t& e) { - if (++tries == max_tries - || e.num() != EINTR // only recoverable error - ) { -// std::cerr << "ERROR in ZeroMQSvc::send (retry_send) on pid " << getpid() << ": " << e.what() << " (errno: " << e.num() << ")\n"; - throw; +auto retry_send(zmq::socket_t &socket, int max_tries, args_t... args) -> decltype(socket.send(args...)) +{ + int tries = 0; + while (true) { + try { + // the actual work this function should do, all the rest is error handling: + return socket.send(args...); + } catch (zmq::error_t &e) { + if (++tries == max_tries || e.num() != EINTR // only recoverable error + ) { + // std::cerr << "ERROR in ZeroMQSvc::send (retry_send) on pid " << getpid() << ": " << e.what() << " + // (errno: " << e.num() << ")\n"; + throw; + } + std::cerr << "RETRY " << tries << "/" << (max_tries - 1) << " in ZeroMQSvc::send (retry_send) on pid " + << getpid() << ": " << e.what() << ")\n"; } - std::cerr << "RETRY " << tries << "/" << (max_tries - 1) << " in ZeroMQSvc::send (retry_send) on pid " << getpid() << ": " << e.what() << ")\n"; - } - } + } } template -auto retry_recv(zmq::socket_t& socket, int max_tries, args_t ...args) -> decltype(socket.recv(args...)) { - int tries = 0; - while(true) { - try { - // the actual work this function should do, all the rest is error handling: - return socket.recv(args...); - } catch (zmq::error_t& e) { - if (++tries == max_tries - || e.num() != EINTR // only recoverable error - ) { -// std::cerr << "ERROR in ZeroMQSvc::recv (retry_recv) on pid " << getpid() << ": " << e.what() << " (errno: " << e.num() << ")\n"; - throw; +auto retry_recv(zmq::socket_t &socket, int max_tries, args_t... args) -> decltype(socket.recv(args...)) +{ + int tries = 0; + while (true) { + try { + // the actual work this function should do, all the rest is error handling: + return socket.recv(args...); + } catch (zmq::error_t &e) { + if (++tries == max_tries || e.num() != EINTR // only recoverable error + ) { + // std::cerr << "ERROR in ZeroMQSvc::recv (retry_recv) on pid " << getpid() << ": " << e.what() << " + // (errno: " << e.num() << ")\n"; + throw; + } + std::cerr << "RETRY " << tries << "/" << (max_tries - 1) << " in ZeroMQSvc::recv (retry_recv) on pid " + << getpid() << ": " << e.what() << ")\n"; } - std::cerr << "RETRY " << tries << "/" << (max_tries - 1) << " in ZeroMQSvc::recv (retry_recv) on pid " << getpid() << ": " << e.what() << ")\n"; - } - } + } } - /** @class IZeroMQSvc IZeroMQSvc.h ZeroMQ/IZeroMQSvc.h * * @@ -115,100 +120,102 @@ auto retry_recv(zmq::socket_t& socket, int max_tries, args_t ...args) -> decltyp * @date 2015-06-22 */ class ZeroMQSvc { - // Note on error handling: - // Creating message_t can throw, but only when memory ran out (errno ENOMEM), - // and that is something only the caller can fix, so we don't catch it here. + // Note on error handling: + // Creating message_t can throw, but only when memory ran out (errno ENOMEM), + // and that is something only the caller can fix, so we don't catch it here. public: - - enum Encoding { - Text = 0, - Binary - }; + enum Encoding { Text = 0, Binary }; Encoding encoding() const; - void setEncoding(const Encoding& e); - zmq::context_t& context() const; + void setEncoding(const Encoding &e); + zmq::context_t &context() const; zmq::socket_t socket(int type) const; - zmq::socket_t* socket_ptr(int type) const; + zmq::socket_t *socket_ptr(int type) const; void close_context() const; - // decode message with ZMQ, POD version - template ::value - && ZMQ::Detail::is_trivial::value, T>::type* = nullptr> - T decode(const zmq::message_t& msg) const { + // decode message with ZMQ, POD version + template ::value && ZMQ::Detail::is_trivial::value, T>::type + * = nullptr> + T decode(const zmq::message_t &msg) const + { T object; memcpy(&object, msg.data(), msg.size()); return object; } // decode ZMQ message, string version - template ::value, T>::type* = nullptr> - std::string decode(const zmq::message_t& msg) const { + template ::value, T>::type * = nullptr> + std::string decode(const zmq::message_t &msg) const + { std::string r(msg.size() + 1, char{}); - r.assign(static_cast(msg.data()), msg.size()); + r.assign(static_cast(msg.data()), msg.size()); return r; } // receive message with ZMQ, general version // FIXME: what to do with flags=0.... more is a pointer, that might prevent conversion - template ::value), T>::type* = nullptr> - T receive(zmq::socket_t& socket, int flags = 0, bool* more = nullptr) const { + template ::value), T>::type * = nullptr> + T receive(zmq::socket_t &socket, int flags = 0, bool *more = nullptr) const + { // receive message zmq::message_t msg; auto nbytes = retry_recv(socket, 2, &msg, flags); if (0 == nbytes) { throw ZMQ::TimeOutException{}; } - if (more) *more = msg.more(); + if (more) + *more = msg.more(); // decode message return decode(msg); } // receive message with ZMQ - template ::value, T>::type* = nullptr> - T receive(zmq::socket_t& socket, int flags = 0, bool* more = nullptr) const { + template ::value, T>::type * = nullptr> + T receive(zmq::socket_t &socket, int flags = 0, bool *more = nullptr) const + { // receive message zmq::message_t msg; auto nbytes = retry_recv(socket, 2, &msg, flags); if (0 == nbytes) { throw ZMQ::TimeOutException{}; } - if (more) *more = msg.more(); + if (more) + *more = msg.more(); return msg; } // encode message to ZMQ - template ::value - && ZMQ::Detail::is_trivial::value, T>::type* = nullptr> - zmq::message_t encode(const T& item, std::function sizeFun = ZMQ::defaultSizeOf) const { + template ::value && ZMQ::Detail::is_trivial::value, T>::type + * = nullptr> + zmq::message_t encode(const T &item, std::function sizeFun = ZMQ::defaultSizeOf) const + { size_t s = sizeFun(item); zmq::message_t msg{s}; memcpy((void *)msg.data(), &item, s); return msg; } - zmq::message_t encode(const char* item) const; - zmq::message_t encode(const std::string& item) const; + zmq::message_t encode(const char *item) const; + zmq::message_t encode(const std::string &item) const; // Send message with ZMQ - template ::value, T>::type* = nullptr> - bool send(zmq::socket_t& socket, const T& item, int flags = 0) const { + template ::value, T>::type * = nullptr> + bool send(zmq::socket_t &socket, const T &item, int flags = 0) const + { return retry_send(socket, 1, encode(item), flags); } - bool send(zmq::socket_t& socket, const char* item, int flags = 0) const; - bool send(zmq::socket_t& socket, zmq::message_t& msg, int flags = 0) const; - bool send(zmq::socket_t& socket, zmq::message_t&& msg, int flags = 0) const; + bool send(zmq::socket_t &socket, const char *item, int flags = 0) const; + bool send(zmq::socket_t &socket, zmq::message_t &msg, int flags = 0) const; + bool send(zmq::socket_t &socket, zmq::message_t &&msg, int flags = 0) const; private: - Encoding m_enc = Text; - mutable zmq::context_t* m_context = nullptr; - + mutable zmq::context_t *m_context = nullptr; }; -ZeroMQSvc& zmqSvc(); +ZeroMQSvc &zmqSvc(); #endif // ZEROMQ_IZEROMQSVC_H diff --git a/roofit/roofitZMQ/inc/RooFit_ZMQ/functions.h b/roofit/roofitZMQ/inc/RooFit_ZMQ/functions.h index 2199275b1d272..f57a1da69e52d 100644 --- a/roofit/roofitZMQ/inc/RooFit_ZMQ/functions.h +++ b/roofit/roofitZMQ/inc/RooFit_ZMQ/functions.h @@ -5,107 +5,98 @@ namespace zmq { - enum SocketTypes { - PAIR = ZMQ_PAIR, - PUB = ZMQ_PUB, - SUB = ZMQ_SUB, - REQ = ZMQ_REQ, - REP = ZMQ_REP, - DEALER = ZMQ_DEALER, - ROUTER = ZMQ_ROUTER, - PULL = ZMQ_PULL, - PUSH = ZMQ_PUSH, - XPUB = ZMQ_XPUB, - XSUB = ZMQ_XSUB, - STREAM = ZMQ_STREAM - }; +enum SocketTypes { + PAIR = ZMQ_PAIR, + PUB = ZMQ_PUB, + SUB = ZMQ_SUB, + REQ = ZMQ_REQ, + REP = ZMQ_REP, + DEALER = ZMQ_DEALER, + ROUTER = ZMQ_ROUTER, + PULL = ZMQ_PULL, + PUSH = ZMQ_PUSH, + XPUB = ZMQ_XPUB, + XSUB = ZMQ_XSUB, + STREAM = ZMQ_STREAM +}; - enum PollType : short { - POLLIN = ZMQ_POLLIN, - POLLOUT = ZMQ_POLLOUT - }; +enum PollType : short { POLLIN = ZMQ_POLLIN, POLLOUT = ZMQ_POLLOUT }; - enum SocketOptions { - AFFINITY = ZMQ_AFFINITY, - IDENTITY = ZMQ_IDENTITY, - SUBSCRIBE = ZMQ_SUBSCRIBE, - UNSUBSCRIBE = ZMQ_UNSUBSCRIBE, - RATE = ZMQ_RATE, - RECOVERY_IVL = ZMQ_RECOVERY_IVL, - SNDBUF = ZMQ_SNDBUF, - RCVBUF = ZMQ_RCVBUF, - RCVMORE = ZMQ_RCVMORE, - FD = ZMQ_FD, - EVENTS = ZMQ_EVENTS, - TYPE = ZMQ_TYPE, - LINGER = ZMQ_LINGER, - RECONNECT_IVL = ZMQ_RECONNECT_IVL, - BACKLOG = ZMQ_BACKLOG, - RECONNECT_IVL_MAX = ZMQ_RECONNECT_IVL_MAX, - MAXMSGSIZE = ZMQ_MAXMSGSIZE, - SNDHWM = ZMQ_SNDHWM, - RCVHWM = ZMQ_RCVHWM, - MULTICAST_HOPS = ZMQ_MULTICAST_HOPS, - RCVTIMEO = ZMQ_RCVTIMEO, - SNDTIMEO = ZMQ_SNDTIMEO, - LAST_ENDPOINT = ZMQ_LAST_ENDPOINT, - ROUTER_MANDATORY = ZMQ_ROUTER_MANDATORY, - TCP_KEEPALIVE = ZMQ_TCP_KEEPALIVE, - TCP_KEEPALIVE_CNT = ZMQ_TCP_KEEPALIVE_CNT, - TCP_KEEPALIVE_IDLE = ZMQ_TCP_KEEPALIVE_IDLE, - TCP_KEEPALIVE_INTVL = ZMQ_TCP_KEEPALIVE_INTVL, - IMMEDIATE = ZMQ_IMMEDIATE, - XPUB_VERBOSE = ZMQ_XPUB_VERBOSE, - ROUTER_RAW = ZMQ_ROUTER_RAW, - IPV6 = ZMQ_IPV6, - MECHANISM = ZMQ_MECHANISM, - PLAIN_SERVER = ZMQ_PLAIN_SERVER, - PLAIN_USERNAME = ZMQ_PLAIN_USERNAME, - PLAIN_PASSWORD = ZMQ_PLAIN_PASSWORD, - CURVE_SERVER = ZMQ_CURVE_SERVER, - CURVE_PUBLICKEY = ZMQ_CURVE_PUBLICKEY, - CURVE_SECRETKEY = ZMQ_CURVE_SECRETKEY, - CURVE_SERVERKEY = ZMQ_CURVE_SERVERKEY, - PROBE_ROUTER = ZMQ_PROBE_ROUTER, - REQ_CORRELATE = ZMQ_REQ_CORRELATE, - REQ_RELAXED = ZMQ_REQ_RELAXED, - CONFLATE = ZMQ_CONFLATE, - ZAP_DOMAIN = ZMQ_ZAP_DOMAIN, - ROUTER_HANDOVER = ZMQ_ROUTER_HANDOVER, - TOS = ZMQ_TOS, - CONNECT_RID = ZMQ_CONNECT_RID, - GSSAPI_SERVER = ZMQ_GSSAPI_SERVER, - GSSAPI_PRINCIPAL = ZMQ_GSSAPI_PRINCIPAL, - GSSAPI_SERVICE_PRINCIPAL = ZMQ_GSSAPI_SERVICE_PRINCIPAL, - GSSAPI_PLAINTEXT = ZMQ_GSSAPI_PLAINTEXT, - HANDSHAKE_IVL = ZMQ_HANDSHAKE_IVL, - SOCKS_PROXY = ZMQ_SOCKS_PROXY, - XPUB_NODROP = ZMQ_XPUB_NODROP - }; +enum SocketOptions { + AFFINITY = ZMQ_AFFINITY, + IDENTITY = ZMQ_IDENTITY, + SUBSCRIBE = ZMQ_SUBSCRIBE, + UNSUBSCRIBE = ZMQ_UNSUBSCRIBE, + RATE = ZMQ_RATE, + RECOVERY_IVL = ZMQ_RECOVERY_IVL, + SNDBUF = ZMQ_SNDBUF, + RCVBUF = ZMQ_RCVBUF, + RCVMORE = ZMQ_RCVMORE, + FD = ZMQ_FD, + EVENTS = ZMQ_EVENTS, + TYPE = ZMQ_TYPE, + LINGER = ZMQ_LINGER, + RECONNECT_IVL = ZMQ_RECONNECT_IVL, + BACKLOG = ZMQ_BACKLOG, + RECONNECT_IVL_MAX = ZMQ_RECONNECT_IVL_MAX, + MAXMSGSIZE = ZMQ_MAXMSGSIZE, + SNDHWM = ZMQ_SNDHWM, + RCVHWM = ZMQ_RCVHWM, + MULTICAST_HOPS = ZMQ_MULTICAST_HOPS, + RCVTIMEO = ZMQ_RCVTIMEO, + SNDTIMEO = ZMQ_SNDTIMEO, + LAST_ENDPOINT = ZMQ_LAST_ENDPOINT, + ROUTER_MANDATORY = ZMQ_ROUTER_MANDATORY, + TCP_KEEPALIVE = ZMQ_TCP_KEEPALIVE, + TCP_KEEPALIVE_CNT = ZMQ_TCP_KEEPALIVE_CNT, + TCP_KEEPALIVE_IDLE = ZMQ_TCP_KEEPALIVE_IDLE, + TCP_KEEPALIVE_INTVL = ZMQ_TCP_KEEPALIVE_INTVL, + IMMEDIATE = ZMQ_IMMEDIATE, + XPUB_VERBOSE = ZMQ_XPUB_VERBOSE, + ROUTER_RAW = ZMQ_ROUTER_RAW, + IPV6 = ZMQ_IPV6, + MECHANISM = ZMQ_MECHANISM, + PLAIN_SERVER = ZMQ_PLAIN_SERVER, + PLAIN_USERNAME = ZMQ_PLAIN_USERNAME, + PLAIN_PASSWORD = ZMQ_PLAIN_PASSWORD, + CURVE_SERVER = ZMQ_CURVE_SERVER, + CURVE_PUBLICKEY = ZMQ_CURVE_PUBLICKEY, + CURVE_SECRETKEY = ZMQ_CURVE_SECRETKEY, + CURVE_SERVERKEY = ZMQ_CURVE_SERVERKEY, + PROBE_ROUTER = ZMQ_PROBE_ROUTER, + REQ_CORRELATE = ZMQ_REQ_CORRELATE, + REQ_RELAXED = ZMQ_REQ_RELAXED, + CONFLATE = ZMQ_CONFLATE, + ZAP_DOMAIN = ZMQ_ZAP_DOMAIN, + ROUTER_HANDOVER = ZMQ_ROUTER_HANDOVER, + TOS = ZMQ_TOS, + CONNECT_RID = ZMQ_CONNECT_RID, + GSSAPI_SERVER = ZMQ_GSSAPI_SERVER, + GSSAPI_PRINCIPAL = ZMQ_GSSAPI_PRINCIPAL, + GSSAPI_SERVICE_PRINCIPAL = ZMQ_GSSAPI_SERVICE_PRINCIPAL, + GSSAPI_PLAINTEXT = ZMQ_GSSAPI_PLAINTEXT, + HANDSHAKE_IVL = ZMQ_HANDSHAKE_IVL, + SOCKS_PROXY = ZMQ_SOCKS_PROXY, + XPUB_NODROP = ZMQ_XPUB_NODROP +}; - // Message options - enum MessageOptions { - MORE = ZMQ_MORE, - SRCFD = ZMQ_SRCFD, - SHARED = ZMQ_SHARED - }; +// Message options +enum MessageOptions { MORE = ZMQ_MORE, SRCFD = ZMQ_SRCFD, SHARED = ZMQ_SHARED }; - // Send/recv options. - enum SendRecvOptions { - DONTWAIT = ZMQ_DONTWAIT, - SNDMORE = ZMQ_SNDMORE - }; -} +// Send/recv options. +enum SendRecvOptions { DONTWAIT = ZMQ_DONTWAIT, SNDMORE = ZMQ_SNDMORE }; +} // namespace zmq namespace ZMQ { template -size_t defaultSizeOf(const T&) { +size_t defaultSizeOf(const T &) +{ return sizeof(T); } -size_t stringLength(const char& cs); +size_t stringLength(const char &cs); -} +} // namespace ZMQ #endif // ZEROMQ_FUNCTIONS_H diff --git a/roofit/roofitZMQ/inc/RooFit_ZMQ/ppoll.h b/roofit/roofitZMQ/inc/RooFit_ZMQ/ppoll.h index c00a1ccfebd7f..2f1edaf9ae78a 100644 --- a/roofit/roofitZMQ/inc/RooFit_ZMQ/ppoll.h +++ b/roofit/roofitZMQ/inc/RooFit_ZMQ/ppoll.h @@ -19,11 +19,12 @@ namespace ZMQ { -int zmq_ppoll (zmq_pollitem_t *items_, int nitems_, long timeout_, const sigset_t * sigmask_); -int ppoll(zmq_pollitem_t *items_, size_t nitems_, long timeout_, const sigset_t * sigmask_); -int ppoll(std::vector &items, long timeout_, const sigset_t * sigmask_); -class ppoll_error_t : public zmq::error_t {}; +int zmq_ppoll(zmq_pollitem_t *items_, int nitems_, long timeout_, const sigset_t *sigmask_); +int ppoll(zmq_pollitem_t *items_, size_t nitems_, long timeout_, const sigset_t *sigmask_); +int ppoll(std::vector &items, long timeout_, const sigset_t *sigmask_); +class ppoll_error_t : public zmq::error_t { +}; -} +} // namespace ZMQ #endif // ROOT_ROOFIT_ZMQ_ppoll diff --git a/roofit/roofitZMQ/src/ZeroMQPoller.cpp b/roofit/roofitZMQ/src/ZeroMQPoller.cpp index 964961bf21613..d08ba2e70f20f 100644 --- a/roofit/roofitZMQ/src/ZeroMQPoller.cpp +++ b/roofit/roofitZMQ/src/ZeroMQPoller.cpp @@ -12,46 +12,49 @@ #include "RooFit_ZMQ/ppoll.h" -std::vector> ZeroMQPoller::poll(int timeo) { - std::vector> r; - if (m_items.empty()) { - throw std::runtime_error("No sockets registered"); - } - int n = 0; - while (true) { - try { - n = zmq::poll(&m_items[0], m_items.size(), timeo); - if (n == 0) return r; - break; - } catch (const zmq::error_t& e) { - std::cerr << "in ZeroMQPoller::poll on PID " << getpid() << ": " << e.what() << std::endl; - if (e.num() != EINTR) { - throw; - } - } - } - // TODO: replace this with ranges::v3::zip - for (size_t i = 0; i < m_items.size(); ++i) { - void* socket = m_items[i].socket; - size_t index = 0; - int flags = 0; - if (socket == nullptr) { - // an fd was registered - std::tie(index, flags) = m_fds[m_items[i].fd]; - } else { - // a socket was registered - const zmq::socket_t* s; - std::tie(index, flags, s) = m_sockets[socket]; - } - if (m_items[i].revents & short(flags)) { - r.emplace_back(index, flags); - } - } - return r; +std::vector> ZeroMQPoller::poll(int timeo) +{ + std::vector> r; + if (m_items.empty()) { + throw std::runtime_error("No sockets registered"); + } + int n = 0; + while (true) { + try { + n = zmq::poll(&m_items[0], m_items.size(), timeo); + if (n == 0) + return r; + break; + } catch (const zmq::error_t &e) { + std::cerr << "in ZeroMQPoller::poll on PID " << getpid() << ": " << e.what() << std::endl; + if (e.num() != EINTR) { + throw; + } + } + } + // TODO: replace this with ranges::v3::zip + for (size_t i = 0; i < m_items.size(); ++i) { + void *socket = m_items[i].socket; + size_t index = 0; + int flags = 0; + if (socket == nullptr) { + // an fd was registered + std::tie(index, flags) = m_fds[m_items[i].fd]; + } else { + // a socket was registered + const zmq::socket_t *s; + std::tie(index, flags, s) = m_sockets[socket]; + } + if (m_items[i].revents & short(flags)) { + r.emplace_back(index, flags); + } + } + return r; } // This function can throw (from inside ZMQ::ppoll), so wrap in try-catch! -std::vector> ZeroMQPoller::ppoll(int timeo, const sigset_t * sigmask_) { +std::vector> ZeroMQPoller::ppoll(int timeo, const sigset_t *sigmask_) +{ if (m_items.empty()) { throw std::runtime_error("No sockets registered"); } @@ -59,9 +62,10 @@ std::vector> ZeroMQPoller::ppoll(int timeo, const sigset_ std::vector> r; auto n = ZMQ::ppoll(m_items, timeo, sigmask_); - if (n == 0) return r; + if (n == 0) + return r; - for (auto& m_item : m_items) { + for (auto &m_item : m_items) { size_t index = 0; int flags = 0; if (m_item.socket == nullptr) { @@ -69,7 +73,7 @@ std::vector> ZeroMQPoller::ppoll(int timeo, const sigset_ std::tie(index, flags) = m_fds[m_item.fd]; } else { // a socket was registered - const zmq::socket_t* s; + const zmq::socket_t *s; std::tie(index, flags, s) = m_sockets[m_item.socket]; } if (m_item.revents & short(flags)) { @@ -79,87 +83,89 @@ std::vector> ZeroMQPoller::ppoll(int timeo, const sigset_ return r; } - -size_t ZeroMQPoller::size() const { - return m_items.size(); +size_t ZeroMQPoller::size() const +{ + return m_items.size(); } -size_t ZeroMQPoller::register_socket(zmq::socket_t& socket, zmq::PollType type) { - zmq::socket_t* s = &socket; - auto it = m_sockets.find(s); - if (it != m_sockets.end()) { - return std::get<0>(it->second); - } - size_t index = m_free.empty() ? m_items.size() : m_free.front(); - if (!m_free.empty()) m_free.pop_front(); - // NOTE: tis uses the conversion-to-void* operator of - // zmq::socket_t, which returns the wrapped object - m_items.push_back({socket, 0, type, 0}); - - // We need to lookup by the pointer to the object wrapped by zmq::socket_t - m_sockets.emplace(m_items.back().socket, std::make_tuple(index, type, s)); - return index; +size_t ZeroMQPoller::register_socket(zmq::socket_t &socket, zmq::PollType type) +{ + zmq::socket_t *s = &socket; + auto it = m_sockets.find(s); + if (it != m_sockets.end()) { + return std::get<0>(it->second); + } + size_t index = m_free.empty() ? m_items.size() : m_free.front(); + if (!m_free.empty()) + m_free.pop_front(); + // NOTE: tis uses the conversion-to-void* operator of + // zmq::socket_t, which returns the wrapped object + m_items.push_back({socket, 0, type, 0}); + + // We need to lookup by the pointer to the object wrapped by zmq::socket_t + m_sockets.emplace(m_items.back().socket, std::make_tuple(index, type, s)); + return index; } -size_t ZeroMQPoller::register_socket(int fd, zmq::PollType type) { - auto it = m_fds.find(fd); - if (it != m_fds.end()) { - return std::get<0>(it->second); - } - size_t index = m_free.empty() ? m_items.size() : m_free.front(); - if (!m_free.empty()) m_free.pop_front(); - // NOTE: tis uses the conversion-to-void* operator of - // zmq::socket_t, which returns the wrapped object - m_items.push_back({nullptr, fd, type, 0}); - - // We need to lookup by the pointer to the object wrapped by zmq::socket_t - m_fds.emplace(fd, std::make_tuple(index, type)); - return index; +size_t ZeroMQPoller::register_socket(int fd, zmq::PollType type) +{ + auto it = m_fds.find(fd); + if (it != m_fds.end()) { + return std::get<0>(it->second); + } + size_t index = m_free.empty() ? m_items.size() : m_free.front(); + if (!m_free.empty()) + m_free.pop_front(); + // NOTE: tis uses the conversion-to-void* operator of + // zmq::socket_t, which returns the wrapped object + m_items.push_back({nullptr, fd, type, 0}); + + // We need to lookup by the pointer to the object wrapped by zmq::socket_t + m_fds.emplace(fd, std::make_tuple(index, type)); + return index; } - -size_t ZeroMQPoller::unregister_socket(zmq::socket_t& socket) { - if (!m_sockets.count(socket.operator void*())) { - throw std::out_of_range("Socket is not registered"); - } - // Remove from m_sockets - // Can't search by the key of m_sockets, as that is the wrapped - // object, but have to use the pointer to the wrapper - // (zmq::socket_t) - auto it = std::find_if(begin(m_sockets), end(m_sockets), - [&socket](const decltype(m_sockets)::value_type& entry) { - return &socket == std::get<2>(entry.second); - }); - auto index = std::get<0>(it->second); - m_free.push_back(index); - m_sockets.erase(it); - - // Remove from m_items - auto iit = std::find_if(begin(m_items), end(m_items), [&it](const zmq::pollitem_t& item) { - return it->first == item.socket; - }); - assert(iit != end(m_items)); - m_items.erase(iit); - - return index; +size_t ZeroMQPoller::unregister_socket(zmq::socket_t &socket) +{ + if (!m_sockets.count(socket.operator void *())) { + throw std::out_of_range("Socket is not registered"); + } + // Remove from m_sockets + // Can't search by the key of m_sockets, as that is the wrapped + // object, but have to use the pointer to the wrapper + // (zmq::socket_t) + auto it = std::find_if(begin(m_sockets), end(m_sockets), [&socket](const decltype(m_sockets)::value_type &entry) { + return &socket == std::get<2>(entry.second); + }); + auto index = std::get<0>(it->second); + m_free.push_back(index); + m_sockets.erase(it); + + // Remove from m_items + auto iit = std::find_if(begin(m_items), end(m_items), + [&it](const zmq::pollitem_t &item) { return it->first == item.socket; }); + assert(iit != end(m_items)); + m_items.erase(iit); + + return index; } -size_t ZeroMQPoller::unregister_socket(int fd) { - if (!m_fds.count(fd)) { - throw std::out_of_range("fileno is not registered"); - } - // Remove from m_fds - auto it = m_fds.find(fd); - auto index = std::get<0>(it->second); - m_free.push_back(index); - m_fds.erase(it); - - // Remove from m_items - auto iit = std::find_if(begin(m_items), end(m_items), [&it](const zmq::pollitem_t& item) { - return it->first == item.fd; - }); - assert(iit != end(m_items)); - m_items.erase(iit); - - return index; +size_t ZeroMQPoller::unregister_socket(int fd) +{ + if (!m_fds.count(fd)) { + throw std::out_of_range("fileno is not registered"); + } + // Remove from m_fds + auto it = m_fds.find(fd); + auto index = std::get<0>(it->second); + m_free.push_back(index); + m_fds.erase(it); + + // Remove from m_items + auto iit = + std::find_if(begin(m_items), end(m_items), [&it](const zmq::pollitem_t &item) { return it->first == item.fd; }); + assert(iit != end(m_items)); + m_items.erase(iit); + + return index; } diff --git a/roofit/roofitZMQ/src/ZeroMQSvc.cpp b/roofit/roofitZMQ/src/ZeroMQSvc.cpp index db6e0b7103bc5..9c37d57b5fc4d 100644 --- a/roofit/roofitZMQ/src/ZeroMQSvc.cpp +++ b/roofit/roofitZMQ/src/ZeroMQSvc.cpp @@ -1,9 +1,10 @@ #include "RooFit_ZMQ/ZeroMQSvc.h" -#include // std::ref +#include // std::ref #include -ZeroMQSvc& zmqSvc() { +ZeroMQSvc &zmqSvc() +{ static std::unique_ptr svc; if (!svc) { svc = std::make_unique(); @@ -11,66 +12,75 @@ ZeroMQSvc& zmqSvc() { return *svc; } -ZeroMQSvc::Encoding ZeroMQSvc::encoding() const { - return m_enc; +ZeroMQSvc::Encoding ZeroMQSvc::encoding() const +{ + return m_enc; } -void ZeroMQSvc::setEncoding(const ZeroMQSvc::Encoding &e) { - m_enc = e; +void ZeroMQSvc::setEncoding(const ZeroMQSvc::Encoding &e) +{ + m_enc = e; } -zmq::context_t& ZeroMQSvc::context() const { - if (!m_context) { - try { - m_context = new zmq::context_t; - } catch (zmq::error_t& e) { - std::cerr << "ERROR: Creating ZeroMQ context failed. This only happens when PGM initialization failed or when a nullptr was returned from zmq_ctx_new because the created context was invalid. Contact ZMQ experts when this happens, because it shouldn't.\n"; - throw; - } - } - return *m_context; +zmq::context_t &ZeroMQSvc::context() const +{ + if (!m_context) { + try { + m_context = new zmq::context_t; + } catch (zmq::error_t &e) { + std::cerr << "ERROR: Creating ZeroMQ context failed. This only happens when PGM initialization failed or when " + "a nullptr was returned from zmq_ctx_new because the created context was invalid. Contact ZMQ " + "experts when this happens, because it shouldn't.\n"; + throw; + } + } + return *m_context; } -zmq::socket_t ZeroMQSvc::socket(int type) const { - try { - // the actual work this function should do, all the rest is error handling: - return zmq::socket_t{context(), type}; - } catch (zmq::error_t& e) { - // all zmq errors not recoverable from here, only at call site - std::cerr << "ERROR in ZeroMQSvc::socket: " << e.what() << " (errno: " << e.num() << ")\n"; - throw; - } +zmq::socket_t ZeroMQSvc::socket(int type) const +{ + try { + // the actual work this function should do, all the rest is error handling: + return zmq::socket_t{context(), type}; + } catch (zmq::error_t &e) { + // all zmq errors not recoverable from here, only at call site + std::cerr << "ERROR in ZeroMQSvc::socket: " << e.what() << " (errno: " << e.num() << ")\n"; + throw; + } } -zmq::socket_t* ZeroMQSvc::socket_ptr(int type) const { - try { - // the actual work this function should do, all the rest is error handling: - return new zmq::socket_t(context(), type); - } catch (zmq::error_t& e) { - // all zmq errors not recoverable from here, only at call site - std::cerr << "ERROR in ZeroMQSvc::socket_ptr: " << e.what() << " (errno: " << e.num() << ")\n"; - throw; - } +zmq::socket_t *ZeroMQSvc::socket_ptr(int type) const +{ + try { + // the actual work this function should do, all the rest is error handling: + return new zmq::socket_t(context(), type); + } catch (zmq::error_t &e) { + // all zmq errors not recoverable from here, only at call site + std::cerr << "ERROR in ZeroMQSvc::socket_ptr: " << e.what() << " (errno: " << e.num() << ")\n"; + throw; + } } -void ZeroMQSvc::close_context() const { - if (m_context) { - delete m_context; - m_context = nullptr; - } +void ZeroMQSvc::close_context() const +{ + if (m_context) { + delete m_context; + m_context = nullptr; + } } - -zmq::message_t ZeroMQSvc::encode(const char* item) const { - std::function fun = ZMQ::stringLength; - return encode(*item, fun); +zmq::message_t ZeroMQSvc::encode(const char *item) const +{ + std::function fun = ZMQ::stringLength; + return encode(*item, fun); } -zmq::message_t ZeroMQSvc::encode(const std::string& item) const { - return encode(item.c_str()); +zmq::message_t ZeroMQSvc::encode(const std::string &item) const +{ + return encode(item.c_str()); } -//zmq::message_t ZeroMQSvc::encode(const TObject& item) const { +// zmq::message_t ZeroMQSvc::encode(const TObject& item) const { // auto deleteBuffer = []( void* data, void* /* hint */ ) -> void { // delete [] (char*)data; // }; @@ -88,15 +98,17 @@ zmq::message_t ZeroMQSvc::encode(const std::string& item) const { // return message; //} - -bool ZeroMQSvc::send(zmq::socket_t& socket, const char* item, int flags) const { - return retry_send(socket, 2, encode(item), flags); +bool ZeroMQSvc::send(zmq::socket_t &socket, const char *item, int flags) const +{ + return retry_send(socket, 2, encode(item), flags); } -bool ZeroMQSvc::send(zmq::socket_t& socket, zmq::message_t& msg, int flags) const { - return retry_send(socket, 2, std::ref(msg), flags); +bool ZeroMQSvc::send(zmq::socket_t &socket, zmq::message_t &msg, int flags) const +{ + return retry_send(socket, 2, std::ref(msg), flags); } -bool ZeroMQSvc::send(zmq::socket_t& socket, zmq::message_t&& msg, int flags) const { - return retry_send(socket, 2, std::move(msg), flags); +bool ZeroMQSvc::send(zmq::socket_t &socket, zmq::message_t &&msg, int flags) const +{ + return retry_send(socket, 2, std::move(msg), flags); } diff --git a/roofit/roofitZMQ/src/functions.cpp b/roofit/roofitZMQ/src/functions.cpp index eded4cebd33d5..d43e79b79624d 100644 --- a/roofit/roofitZMQ/src/functions.cpp +++ b/roofit/roofitZMQ/src/functions.cpp @@ -4,7 +4,8 @@ #include "RooFit_ZMQ/ZeroMQSvc.h" namespace ZMQ { - size_t stringLength(const char& cs) { - return strlen(&cs); - } +size_t stringLength(const char &cs) +{ + return strlen(&cs); } +} // namespace ZMQ diff --git a/roofit/roofitZMQ/src/ppoll.cpp b/roofit/roofitZMQ/src/ppoll.cpp index a753ed9628143..a82163b09837d 100644 --- a/roofit/roofitZMQ/src/ppoll.cpp +++ b/roofit/roofitZMQ/src/ppoll.cpp @@ -14,309 +14,287 @@ #include -#include // pselect -#include // timespec +#include // pselect +#include // timespec #include -#include "likely.hpp" // unlikely -#include "clock.hpp" // zmq::clock_t -#include "err.hpp" // zmq_assert -#include "polling_util.hpp" // zmq::optimized_fd_set_t, zmq::compute_timeout, zmq::timeout_t -#include "fd.hpp" // zmq::fd_t - +#include "likely.hpp" // unlikely +#include "clock.hpp" // zmq::clock_t +#include "err.hpp" // zmq_assert +#include "polling_util.hpp" // zmq::optimized_fd_set_t, zmq::compute_timeout, zmq::timeout_t +#include "fd.hpp" // zmq::fd_t namespace ZMQ { -int zmq_ppoll (zmq_pollitem_t *items_, int nitems_, long timeout_, - const sigset_t * sigmask_) +int zmq_ppoll(zmq_pollitem_t *items_, int nitems_, long timeout_, const sigset_t *sigmask_) { #if defined ZMQ_HAVE_POLLER // if poller is present, use that if there is at least 1 thread-safe socket, - // otherwise fall back to the previous implementation as it's faster. - for (int i = 0; i != nitems_; i++) { - if (items_[i].socket - && as_socket_base_t (items_[i].socket)->is_thread_safe ()) { - return zmq_poller_poll (items_, nitems_, timeout_); - } - } + // otherwise fall back to the previous implementation as it's faster. + for (int i = 0; i != nitems_; i++) { + if (items_[i].socket && as_socket_base_t(items_[i].socket)->is_thread_safe()) { + return zmq_poller_poll(items_, nitems_, timeout_); + } + } #endif // ZMQ_HAVE_POLLER #if defined ZMQ_POLL_BASED_ON_POLL || defined ZMQ_POLL_BASED_ON_SELECT - if (unlikely (nitems_ < 0)) { - errno = EINVAL; - return -1; - } - if (unlikely (nitems_ == 0)) { - if (timeout_ == 0) - return 0; + if (unlikely(nitems_ < 0)) { + errno = EINVAL; + return -1; + } + if (unlikely(nitems_ == 0)) { + if (timeout_ == 0) + return 0; #if defined ZMQ_HAVE_WINDOWS - Sleep (timeout_ > 0 ? timeout_ : INFINITE); - return 0; + Sleep(timeout_ > 0 ? timeout_ : INFINITE); + return 0; #elif defined ZMQ_HAVE_VXWORKS - struct timespec ns_; - ns_.tv_sec = timeout_ / 1000; - ns_.tv_nsec = timeout_ % 1000 * 1000000; - return nanosleep (&ns_, 0); + struct timespec ns_; + ns_.tv_sec = timeout_ / 1000; + ns_.tv_nsec = timeout_ % 1000 * 1000000; + return nanosleep(&ns_, 0); #else - return usleep (timeout_ * 1000); + return usleep(timeout_ * 1000); #endif - } - if (!items_) { - errno = EFAULT; - return -1; - } - - zmq::clock_t clock; - uint64_t now = 0; - uint64_t end = 0; + } + if (!items_) { + errno = EFAULT; + return -1; + } + + zmq::clock_t clock; + uint64_t now = 0; + uint64_t end = 0; #if defined ZMQ_POLL_BASED_ON_POLL - zmq::fast_vector_t pollfds (nitems_); - - // Build pollset for poll () system call. - for (int i = 0; i != nitems_; i++) { - // If the poll item is a 0MQ socket, we poll on the file descriptor - // retrieved by the ZMQ_FD socket option. - if (items_[i].socket) { - size_t zmq_fd_size = sizeof (zmq::fd_t); - if (zmq_getsockopt (items_[i].socket, ZMQ_FD, &pollfds[i].fd, - &zmq_fd_size) - == -1) { - return -1; - } - pollfds[i].events = items_[i].events ? POLLIN : 0; - } - // Else, the poll item is a raw file descriptor. Just convert the - // events to normal POLLIN/POLLOUT for poll (). - else { - pollfds[i].fd = items_[i].fd; - pollfds[i].events = - (items_[i].events & ZMQ_POLLIN ? POLLIN : 0) - | (items_[i].events & ZMQ_POLLOUT ? POLLOUT : 0) - | (items_[i].events & ZMQ_POLLPRI ? POLLPRI : 0); - } - } + zmq::fast_vector_t pollfds(nitems_); + + // Build pollset for poll () system call. + for (int i = 0; i != nitems_; i++) { + // If the poll item is a 0MQ socket, we poll on the file descriptor + // retrieved by the ZMQ_FD socket option. + if (items_[i].socket) { + size_t zmq_fd_size = sizeof(zmq::fd_t); + if (zmq_getsockopt(items_[i].socket, ZMQ_FD, &pollfds[i].fd, &zmq_fd_size) == -1) { + return -1; + } + pollfds[i].events = items_[i].events ? POLLIN : 0; + } + // Else, the poll item is a raw file descriptor. Just convert the + // events to normal POLLIN/POLLOUT for poll (). + else { + pollfds[i].fd = items_[i].fd; + pollfds[i].events = (items_[i].events & ZMQ_POLLIN ? POLLIN : 0) | + (items_[i].events & ZMQ_POLLOUT ? POLLOUT : 0) | + (items_[i].events & ZMQ_POLLPRI ? POLLPRI : 0); + } + } #else - // Ensure we do not attempt to select () on more than FD_SETSIZE - // file descriptors. - // TODO since this function is called by a client, we could return errno EINVAL/ENOMEM/... here - zmq_assert (nitems_ <= FD_SETSIZE); - - zmq::optimized_fd_set_t pollset_in (nitems_); - FD_ZERO (pollset_in.get ()); - zmq::optimized_fd_set_t pollset_out (nitems_); - FD_ZERO (pollset_out.get ()); - zmq::optimized_fd_set_t pollset_err (nitems_); - FD_ZERO (pollset_err.get ()); - - zmq::fd_t maxfd = 0; - - // Build the fd_sets for passing to select (). - for (int i = 0; i != nitems_; i++) { - // If the poll item is a 0MQ socket we are interested in input on the - // notification file descriptor retrieved by the ZMQ_FD socket option. - if (items_[i].socket) { - size_t zmq_fd_size = sizeof (zmq::fd_t); - zmq::fd_t notify_fd; - if (zmq_getsockopt (items_[i].socket, ZMQ_FD, ¬ify_fd, - &zmq_fd_size) - == -1) - return -1; - if (items_[i].events) { - FD_SET (notify_fd, pollset_in.get ()); - if (maxfd < notify_fd) - maxfd = notify_fd; - } - } - // Else, the poll item is a raw file descriptor. Convert the poll item - // events to the appropriate fd_sets. - else { - if (items_[i].events & ZMQ_POLLIN) - FD_SET (items_[i].fd, pollset_in.get ()); - if (items_[i].events & ZMQ_POLLOUT) - FD_SET (items_[i].fd, pollset_out.get ()); - if (items_[i].events & ZMQ_POLLERR) - FD_SET (items_[i].fd, pollset_err.get ()); - if (maxfd < items_[i].fd) - maxfd = items_[i].fd; - } - } - - zmq::optimized_fd_set_t inset (nitems_); - zmq::optimized_fd_set_t outset (nitems_); - zmq::optimized_fd_set_t errset (nitems_); + // Ensure we do not attempt to select () on more than FD_SETSIZE + // file descriptors. + // TODO since this function is called by a client, we could return errno EINVAL/ENOMEM/... here + zmq_assert(nitems_ <= FD_SETSIZE); + + zmq::optimized_fd_set_t pollset_in(nitems_); + FD_ZERO(pollset_in.get()); + zmq::optimized_fd_set_t pollset_out(nitems_); + FD_ZERO(pollset_out.get()); + zmq::optimized_fd_set_t pollset_err(nitems_); + FD_ZERO(pollset_err.get()); + + zmq::fd_t maxfd = 0; + + // Build the fd_sets for passing to select (). + for (int i = 0; i != nitems_; i++) { + // If the poll item is a 0MQ socket we are interested in input on the + // notification file descriptor retrieved by the ZMQ_FD socket option. + if (items_[i].socket) { + size_t zmq_fd_size = sizeof(zmq::fd_t); + zmq::fd_t notify_fd; + if (zmq_getsockopt(items_[i].socket, ZMQ_FD, ¬ify_fd, &zmq_fd_size) == -1) + return -1; + if (items_[i].events) { + FD_SET(notify_fd, pollset_in.get()); + if (maxfd < notify_fd) + maxfd = notify_fd; + } + } + // Else, the poll item is a raw file descriptor. Convert the poll item + // events to the appropriate fd_sets. + else { + if (items_[i].events & ZMQ_POLLIN) + FD_SET(items_[i].fd, pollset_in.get()); + if (items_[i].events & ZMQ_POLLOUT) + FD_SET(items_[i].fd, pollset_out.get()); + if (items_[i].events & ZMQ_POLLERR) + FD_SET(items_[i].fd, pollset_err.get()); + if (maxfd < items_[i].fd) + maxfd = items_[i].fd; + } + } + + zmq::optimized_fd_set_t inset(nitems_); + zmq::optimized_fd_set_t outset(nitems_); + zmq::optimized_fd_set_t errset(nitems_); #endif - bool first_pass = true; - int nevents = 0; + bool first_pass = true; + int nevents = 0; - while (true) { + while (true) { #if defined ZMQ_POLL_BASED_ON_POLL - // Compute the timeout for the subsequent poll. - zmq::timeout_t timeout = - zmq::compute_timeout (first_pass, timeout_, now, end); - - // Wait for events. - { - int rc = ppoll (&pollfds[0], nitems_, timeout); - if (rc == -1 && errno == EINTR) { - return -1; - } - errno_assert (rc >= 0); - } - // Check for the events. - for (int i = 0; i != nitems_; i++) { - items_[i].revents = 0; - - // The poll item is a 0MQ socket. Retrieve pending events - // using the ZMQ_EVENTS socket option. - if (items_[i].socket) { - size_t zmq_events_size = sizeof (uint32_t); - uint32_t zmq_events; - if (zmq_getsockopt (items_[i].socket, ZMQ_EVENTS, &zmq_events, - &zmq_events_size) - == -1) { - return -1; - } - if ((items_[i].events & ZMQ_POLLOUT) - && (zmq_events & ZMQ_POLLOUT)) - items_[i].revents |= ZMQ_POLLOUT; - if ((items_[i].events & ZMQ_POLLIN) - && (zmq_events & ZMQ_POLLIN)) - items_[i].revents |= ZMQ_POLLIN; - } - // Else, the poll item is a raw file descriptor, simply convert - // the events to zmq_pollitem_t-style format. - else { - if (pollfds[i].revents & POLLIN) - items_[i].revents |= ZMQ_POLLIN; - if (pollfds[i].revents & POLLOUT) - items_[i].revents |= ZMQ_POLLOUT; - if (pollfds[i].revents & POLLPRI) - items_[i].revents |= ZMQ_POLLPRI; - if (pollfds[i].revents & ~(POLLIN | POLLOUT | POLLPRI)) - items_[i].revents |= ZMQ_POLLERR; + // Compute the timeout for the subsequent poll. + zmq::timeout_t timeout = zmq::compute_timeout(first_pass, timeout_, now, end); + + // Wait for events. + { + int rc = ppoll(&pollfds[0], nitems_, timeout); + if (rc == -1 && errno == EINTR) { + return -1; + } + errno_assert(rc >= 0); + } + // Check for the events. + for (int i = 0; i != nitems_; i++) { + items_[i].revents = 0; + + // The poll item is a 0MQ socket. Retrieve pending events + // using the ZMQ_EVENTS socket option. + if (items_[i].socket) { + size_t zmq_events_size = sizeof(uint32_t); + uint32_t zmq_events; + if (zmq_getsockopt(items_[i].socket, ZMQ_EVENTS, &zmq_events, &zmq_events_size) == -1) { + return -1; } - - if (items_[i].revents) - nevents++; - } + if ((items_[i].events & ZMQ_POLLOUT) && (zmq_events & ZMQ_POLLOUT)) + items_[i].revents |= ZMQ_POLLOUT; + if ((items_[i].events & ZMQ_POLLIN) && (zmq_events & ZMQ_POLLIN)) + items_[i].revents |= ZMQ_POLLIN; + } + // Else, the poll item is a raw file descriptor, simply convert + // the events to zmq_pollitem_t-style format. + else { + if (pollfds[i].revents & POLLIN) + items_[i].revents |= ZMQ_POLLIN; + if (pollfds[i].revents & POLLOUT) + items_[i].revents |= ZMQ_POLLOUT; + if (pollfds[i].revents & POLLPRI) + items_[i].revents |= ZMQ_POLLPRI; + if (pollfds[i].revents & ~(POLLIN | POLLOUT | POLLPRI)) + items_[i].revents |= ZMQ_POLLERR; + } + + if (items_[i].revents) + nevents++; + } #else - // Compute the timeout for the subsequent poll. - struct timespec timeout; - struct timespec *ptimeout; - if (first_pass) { - timeout.tv_sec = 0; - timeout.tv_nsec = 0; - ptimeout = &timeout; - } else if (timeout_ < 0) - ptimeout = NULL; - else { - timeout.tv_sec = static_cast ((end - now) / 1000); - timeout.tv_nsec = static_cast ((end - now) % 1000 * 1000000); - ptimeout = &timeout; - } - - // Wait for events. Ignore interrupts if there's infinite timeout. - while (true) { - memcpy (inset.get (), pollset_in.get (), - zmq::valid_pollset_bytes (*pollset_in.get ())); - memcpy (outset.get (), pollset_out.get (), - zmq::valid_pollset_bytes (*pollset_out.get ())); - memcpy (errset.get (), pollset_err.get (), - zmq::valid_pollset_bytes (*pollset_err.get ())); + // Compute the timeout for the subsequent poll. + struct timespec timeout; + struct timespec *ptimeout; + if (first_pass) { + timeout.tv_sec = 0; + timeout.tv_nsec = 0; + ptimeout = &timeout; + } else if (timeout_ < 0) + ptimeout = NULL; + else { + timeout.tv_sec = static_cast((end - now) / 1000); + timeout.tv_nsec = static_cast((end - now) % 1000 * 1000000); + ptimeout = &timeout; + } + + // Wait for events. Ignore interrupts if there's infinite timeout. + while (true) { + memcpy(inset.get(), pollset_in.get(), zmq::valid_pollset_bytes(*pollset_in.get())); + memcpy(outset.get(), pollset_out.get(), zmq::valid_pollset_bytes(*pollset_out.get())); + memcpy(errset.get(), pollset_err.get(), zmq::valid_pollset_bytes(*pollset_err.get())); #if defined ZMQ_HAVE_WINDOWS - int rc = - select (0, inset.get (), outset.get (), errset.get (), ptimeout); - if (unlikely (rc == SOCKET_ERROR)) { - errno = zmq::wsa_error_to_errno (WSAGetLastError ()); - wsa_assert (errno == ENOTSOCK); - return -1; - } + int rc = select(0, inset.get(), outset.get(), errset.get(), ptimeout); + if (unlikely(rc == SOCKET_ERROR)) { + errno = zmq::wsa_error_to_errno(WSAGetLastError()); + wsa_assert(errno == ENOTSOCK); + return -1; + } #else - int rc = pselect (maxfd + 1, inset.get (), outset.get (), - errset.get (), ptimeout, sigmask_); - if (unlikely (rc == -1)) { - errno_assert (errno == EINTR || errno == EBADF); - return -1; - } + int rc = pselect(maxfd + 1, inset.get(), outset.get(), errset.get(), ptimeout, sigmask_); + if (unlikely(rc == -1)) { + errno_assert(errno == EINTR || errno == EBADF); + return -1; + } #endif - break; - } - - // Check for the events. - for (int i = 0; i != nitems_; i++) { - items_[i].revents = 0; - - // The poll item is a 0MQ socket. Retrieve pending events - // using the ZMQ_EVENTS socket option. - if (items_[i].socket) { - size_t zmq_events_size = sizeof (uint32_t); - uint32_t zmq_events; - if (zmq_getsockopt (items_[i].socket, ZMQ_EVENTS, &zmq_events, - &zmq_events_size) - == -1) - return -1; - if ((items_[i].events & ZMQ_POLLOUT) - && (zmq_events & ZMQ_POLLOUT)) - items_[i].revents |= ZMQ_POLLOUT; - if ((items_[i].events & ZMQ_POLLIN) - && (zmq_events & ZMQ_POLLIN)) - items_[i].revents |= ZMQ_POLLIN; - } - // Else, the poll item is a raw file descriptor, simply convert - // the events to zmq_pollitem_t-style format. - else { - if (FD_ISSET (items_[i].fd, inset.get ())) - items_[i].revents |= ZMQ_POLLIN; - if (FD_ISSET (items_[i].fd, outset.get ())) - items_[i].revents |= ZMQ_POLLOUT; - if (FD_ISSET (items_[i].fd, errset.get ())) - items_[i].revents |= ZMQ_POLLERR; - } - - if (items_[i].revents) - nevents++; - } + break; + } + + // Check for the events. + for (int i = 0; i != nitems_; i++) { + items_[i].revents = 0; + + // The poll item is a 0MQ socket. Retrieve pending events + // using the ZMQ_EVENTS socket option. + if (items_[i].socket) { + size_t zmq_events_size = sizeof(uint32_t); + uint32_t zmq_events; + if (zmq_getsockopt(items_[i].socket, ZMQ_EVENTS, &zmq_events, &zmq_events_size) == -1) + return -1; + if ((items_[i].events & ZMQ_POLLOUT) && (zmq_events & ZMQ_POLLOUT)) + items_[i].revents |= ZMQ_POLLOUT; + if ((items_[i].events & ZMQ_POLLIN) && (zmq_events & ZMQ_POLLIN)) + items_[i].revents |= ZMQ_POLLIN; + } + // Else, the poll item is a raw file descriptor, simply convert + // the events to zmq_pollitem_t-style format. + else { + if (FD_ISSET(items_[i].fd, inset.get())) + items_[i].revents |= ZMQ_POLLIN; + if (FD_ISSET(items_[i].fd, outset.get())) + items_[i].revents |= ZMQ_POLLOUT; + if (FD_ISSET(items_[i].fd, errset.get())) + items_[i].revents |= ZMQ_POLLERR; + } + + if (items_[i].revents) + nevents++; + } #endif - // If timeout is zero, exit immediately whether there are events or not. - if (timeout_ == 0) - break; + // If timeout is zero, exit immediately whether there are events or not. + if (timeout_ == 0) + break; - // If there are events to return, we can exit immediately. - if (nevents) - break; + // If there are events to return, we can exit immediately. + if (nevents) + break; - // At this point we are meant to wait for events but there are none. - // If timeout is infinite we can just loop until we get some events. - if (timeout_ < 0) { - if (first_pass) - first_pass = false; - continue; - } - - // The timeout is finite and there are no events. In the first pass - // we get a timestamp of when the polling have begun. (We assume that - // first pass have taken negligible time). We also compute the time - // when the polling should time out. - if (first_pass) { - now = clock.now_ms (); - end = now + timeout_; - if (now == end) - break; + // At this point we are meant to wait for events but there are none. + // If timeout is infinite we can just loop until we get some events. + if (timeout_ < 0) { + if (first_pass) first_pass = false; - continue; - } - - // Find out whether timeout have expired. - now = clock.now_ms (); - if (now >= end) + continue; + } + + // The timeout is finite and there are no events. In the first pass + // we get a timestamp of when the polling have begun. (We assume that + // first pass have taken negligible time). We also compute the time + // when the polling should time out. + if (first_pass) { + now = clock.now_ms(); + end = now + timeout_; + if (now == end) break; - } + first_pass = false; + continue; + } - return nevents; + // Find out whether timeout have expired. + now = clock.now_ms(); + if (now >= end) + break; + } + + return nevents; #else // Exotic platforms that support neither poll() nor select(). errno = ENOTSUP; @@ -324,9 +302,8 @@ int zmq_ppoll (zmq_pollitem_t *items_, int nitems_, long timeout_, #endif } - // This function can throw, so wrap in try-catch! -int ppoll(zmq_pollitem_t *items_, size_t nitems_, long timeout_, const sigset_t * sigmask_) +int ppoll(zmq_pollitem_t *items_, size_t nitems_, long timeout_, const sigset_t *sigmask_) { int rc = zmq_ppoll(items_, static_cast(nitems_), timeout_, sigmask_); if (rc < 0) @@ -334,12 +311,10 @@ int ppoll(zmq_pollitem_t *items_, size_t nitems_, long timeout_, const sigset_t return rc; } - // This function can throw, so wrap in try-catch! -int ppoll(std::vector &items, long timeout_, const sigset_t * sigmask_) +int ppoll(std::vector &items, long timeout_, const sigset_t *sigmask_) { return ppoll(items.data(), items.size(), timeout_, sigmask_); } -} - +} // namespace ZMQ diff --git a/roofit/roofitZMQ/test/test_HWM.cxx b/roofit/roofitZMQ/test/test_HWM.cxx index 7ef11b5c3f5ed..c6bfa7b29d4ec 100644 --- a/roofit/roofitZMQ/test/test_HWM.cxx +++ b/roofit/roofitZMQ/test/test_HWM.cxx @@ -20,19 +20,20 @@ class HighWaterMarkTest : public ::testing::Test { protected: - void SetUp() override { + void SetUp() override + { do { child_pid = fork(); - } while (child_pid == -1); // retry if fork fails + } while (child_pid == -1); // retry if fork fails - if (child_pid > 0) { // parent + if (child_pid > 0) { // parent pusher.reset(zmqSvc().socket_ptr(zmq::PUSH)); if (set_hwm) { auto rc = zmq_setsockopt(*pusher, ZMQ_SNDHWM, &hwm, sizeof hwm); assert(rc == 0); } pusher->bind("ipc:///tmp/ZMQ_test_fork_polling_P2C.ipc"); - } else { // child + } else { // child puller.reset(zmqSvc().socket_ptr(zmq::PULL)); if (set_hwm) { auto rc = zmq_setsockopt(*puller, ZMQ_RCVHWM, &hwm, sizeof hwm); @@ -42,8 +43,9 @@ class HighWaterMarkTest : public ::testing::Test { } } - void TearDown() override { - if (child_pid > 0) { // parent + void TearDown() override + { + if (child_pid > 0) { // parent // wait for child int status = 0; pid_t pid; @@ -51,23 +53,32 @@ class HighWaterMarkTest : public ::testing::Test { pid = waitpid(child_pid, &status, 0); } while (-1 == pid && EINTR == errno); // retry on interrupted system call if (0 != status) { - if (WIFEXITED(status)) { printf("exited, status=%d\n", WEXITSTATUS(status)); } else if (WIFSIGNALED(status)) { printf("killed by signal %d\n", WTERMSIG(status)); } else if (WIFSTOPPED(status)) { printf("stopped by signal %d\n", WSTOPSIG(status)); } else if (WIFCONTINUED(status)) { printf("continued\n"); } + if (WIFEXITED(status)) { + printf("exited, status=%d\n", WEXITSTATUS(status)); + } else if (WIFSIGNALED(status)) { + printf("killed by signal %d\n", WTERMSIG(status)); + } else if (WIFSTOPPED(status)) { + printf("stopped by signal %d\n", WSTOPSIG(status)); + } else if (WIFCONTINUED(status)) { + printf("continued\n"); + } } if (-1 == pid) { throw std::runtime_error(std::string("waitpid, errno ") + std::to_string(errno)); } pusher.reset(nullptr); zmqSvc().close_context(); - } else { // child + } else { // child puller.reset(nullptr); zmqSvc().close_context(); } } - void run_test() { + void run_test() + { std::size_t max_sends = 2000; - if (child_pid > 0) { // parent + if (child_pid > 0) { // parent // start test for (std::size_t ix = 0; ix < max_sends; ++ix) { zmqSvc().send(*pusher, 0.1f); @@ -77,7 +88,7 @@ class HighWaterMarkTest : public ::testing::Test { printf("parent at ix = %lu\n", ix); } } - } else { // child + } else { // child // wait a few seconds before reading to allow the parent to overflow the HWM printf("child waiting for 2 seconds...\n"); sleep(2); @@ -94,37 +105,41 @@ class HighWaterMarkTest : public ::testing::Test { } } - pid_t child_pid {0}; + pid_t child_pid{0}; ZmqLingeringSocketPtr<> pusher, puller; bool set_hwm = false; int hwm = 0; }; - -TEST_F(HighWaterMarkTest, demonstrateHittingDefaultHWM) { +TEST_F(HighWaterMarkTest, demonstrateHittingDefaultHWM) +{ run_test(); } class HighWaterMarkZeroTest : public HighWaterMarkTest { - void SetUp() override { + void SetUp() override + { set_hwm = true; hwm = 0; HighWaterMarkTest::SetUp(); } }; -TEST_F(HighWaterMarkZeroTest, zeroHWM) { +TEST_F(HighWaterMarkZeroTest, zeroHWM) +{ run_test(); } class HighWaterMark2kTest : public HighWaterMarkTest { - void SetUp() override { + void SetUp() override + { set_hwm = true; hwm = 2000; HighWaterMarkTest::SetUp(); } }; -TEST_F(HighWaterMark2kTest, HWM2k) { +TEST_F(HighWaterMark2kTest, HWM2k) +{ run_test(); } diff --git a/roofit/roofitZMQ/test/test_ZMQ.cpp b/roofit/roofitZMQ/test/test_ZMQ.cpp index 2cf6751968df9..d93d230f04a34 100644 --- a/roofit/roofitZMQ/test/test_ZMQ.cpp +++ b/roofit/roofitZMQ/test/test_ZMQ.cpp @@ -18,10 +18,9 @@ #include "RooFit_ZMQ/ZeroMQSvc.h" - // N.B.: wait_for_child is identically defined in RooFit::MultiProcess, but we copy it here to reduce module // interdependencies. It also requires an extra include: -#include // kill, SIGKILL +#include // kill, SIGKILL int wait_for_child(pid_t child_pid, bool may_throw, int retries_before_killing) { @@ -35,7 +34,7 @@ int wait_for_child(pid_t child_pid, bool may_throw, int retries_before_killing) tmp = waitpid(child_pid, &status, WNOHANG); } while (tmp == 0 // child has not yet changed state, try again || (-1 == tmp && EINTR == errno) // retry on interrupted system call - ); + ); if (patience < 1) { std::cout << "Had to send PID " << child_pid << " " << (-patience + 1) << " SIGKILLs"; @@ -59,248 +58,251 @@ int wait_for_child(pid_t child_pid, bool may_throw, int retries_before_killing) return status; } -void elaborate_bind(const ZmqLingeringSocketPtr<>& socket, std::string name) { - try { - socket->bind(name); - } catch (const zmq::error_t& e) { - if (e.num() == EADDRINUSE) { - std::cerr << "address already in use, retrying bind in 500ms\n"; - usleep(500000); - try { - socket->bind(name); - } catch (const zmq::error_t& e2) { - if (e2.num() == EADDRINUSE) { - std::cerr - << "again: address already in use, aborting; please check whether there are any remaining improperly exited processes (zombies) around or whether some other program is using port 6660\n"; - } - throw; +void elaborate_bind(const ZmqLingeringSocketPtr<> &socket, std::string name) +{ + try { + socket->bind(name); + } catch (const zmq::error_t &e) { + if (e.num() == EADDRINUSE) { + std::cerr << "address already in use, retrying bind in 500ms\n"; + usleep(500000); + try { + socket->bind(name); + } catch (const zmq::error_t &e2) { + if (e2.num() == EADDRINUSE) { + std::cerr + << "again: address already in use, aborting; please check whether there are any remaining improperly " + "exited processes (zombies) around or whether some other program is using port 6660\n"; + } + throw; + } + // Sometimes, the socket from the previous test needs some time to close, so + // we introduce a latency here. A more robust and fast approach might be to + // do the following on the bind side: + // 1. first try another port, e.g. increase by one + // 2. if that doesn't work, do the latency and retry the original port + // The connect side then also needs to change, because it doesn't know which + // port the bind side will bind to. The connect side could try connecting to + // both options asynchronously, and then in a loop check both for signs of + // life. If one comes alive, transfer ownership of that pointer to the pointer + // you want to eventually use (`socket`) and that's it. + } else { + throw; } - // Sometimes, the socket from the previous test needs some time to close, so - // we introduce a latency here. A more robust and fast approach might be to - // do the following on the bind side: - // 1. first try another port, e.g. increase by one - // 2. if that doesn't work, do the latency and retry the original port - // The connect side then also needs to change, because it doesn't know which - // port the bind side will bind to. The connect side could try connecting to - // both options asynchronously, and then in a loop check both for signs of - // life. If one comes alive, transfer ownership of that pointer to the pointer - // you want to eventually use (`socket`) and that's it. - } else { - throw; - } - } + } } -class AllSocketTypes : public ::testing::TestWithParam< std::tuple, std::pair /* socket_names */> > {}; - -TEST_P(AllSocketTypes, forkHandshake) { - auto socket_names = std::get<2>(GetParam()); - pid_t child_pid {0}; - do { - child_pid = fork(); - } while (child_pid == -1); // retry if fork fails - - if (child_pid > 0) { // master - ZmqLingeringSocketPtr<> socket; - socket.reset(zmqSvc().socket_ptr(std::get<1>(GetParam()).first)); - elaborate_bind(socket, socket_names.second); - // bind is on the master process to avoid zombie children to hold on to binds - - // start test - zmqSvc().send(*socket, std::string("breaker breaker")); - - auto receipt = zmqSvc().receive(*socket); - - EXPECT_EQ(receipt, 1212); - -// socket->close(); // this gives exception of type zmq::error_t: Socket operation on non-socket - socket.reset(nullptr); - zmqSvc().close_context(); // if you don't close context in parent process as well, the next repeat will hang - - wait_for_child(child_pid, true, 5); - } else { // child - ZmqLingeringSocketPtr<> socket; - socket.reset(zmqSvc().socket_ptr(std::get<1>(GetParam()).second)); - socket->connect(socket_names.first); - - // start test - auto receipt = zmqSvc().receive(*socket); - if (receipt == "breaker breaker") { - zmqSvc().send(*socket, 1212); - } - // take care, don't just use _exit, it will not cleanly destroy context etc! - // if you really need to, at least close and destroy everything properly -// socket->close(); // this gives exception of type zmq::error_t: Socket operation on non-socket - socket.reset(nullptr); - zmqSvc().close_context(); - _Exit(0); - } -} +class AllSocketTypes + : public ::testing::TestWithParam, + std::pair /* socket_names */>> { +}; + +TEST_P(AllSocketTypes, forkHandshake) +{ + auto socket_names = std::get<2>(GetParam()); + pid_t child_pid{0}; + do { + child_pid = fork(); + } while (child_pid == -1); // retry if fork fails + + if (child_pid > 0) { // master + ZmqLingeringSocketPtr<> socket; + socket.reset(zmqSvc().socket_ptr(std::get<1>(GetParam()).first)); + elaborate_bind(socket, socket_names.second); + // bind is on the master process to avoid zombie children to hold on to binds + + // start test + zmqSvc().send(*socket, std::string("breaker breaker")); + + auto receipt = zmqSvc().receive(*socket); + + EXPECT_EQ(receipt, 1212); + + // socket->close(); // this gives exception of type zmq::error_t: Socket operation on non-socket + socket.reset(nullptr); + zmqSvc().close_context(); // if you don't close context in parent process as well, the next repeat will hang -std::string ipc {"ipc:///tmp/ZMQ_test_fork.ipc"}; -std::string tcp_server {"tcp://127.0.0.1:6660"}; -std::string tcp_client {"tcp://*:6660"}; -auto socket_name_options = ::testing::Values( - std::make_pair(tcp_server, tcp_client), - std::make_pair(ipc, ipc) - ); + wait_for_child(child_pid, true, 5); + } else { // child + ZmqLingeringSocketPtr<> socket; + socket.reset(zmqSvc().socket_ptr(std::get<1>(GetParam()).second)); + socket->connect(socket_names.first); + + // start test + auto receipt = zmqSvc().receive(*socket); + if (receipt == "breaker breaker") { + zmqSvc().send(*socket, 1212); + } + // take care, don't just use _exit, it will not cleanly destroy context etc! + // if you really need to, at least close and destroy everything properly + // socket->close(); // this gives exception of type zmq::error_t: Socket operation on non-socket + socket.reset(nullptr); + zmqSvc().close_context(); + _Exit(0); + } +} +std::string ipc{"ipc:///tmp/ZMQ_test_fork.ipc"}; +std::string tcp_server{"tcp://127.0.0.1:6660"}; +std::string tcp_client{"tcp://*:6660"}; +auto socket_name_options = ::testing::Values(std::make_pair(tcp_server, tcp_client), std::make_pair(ipc, ipc)); INSTANTIATE_TEST_SUITE_P(REQREP, AllSocketTypes, - ::testing::Combine(::testing::Range(0, 10), // repeat to probe connection stability - ::testing::Values(std::make_pair(zmq::REQ, zmq::REP)), - socket_name_options - )); + ::testing::Combine(::testing::Range(0, 10), // repeat to probe connection stability + ::testing::Values(std::make_pair(zmq::REQ, zmq::REP)), + socket_name_options)); INSTANTIATE_TEST_SUITE_P(PAIRPAIR, AllSocketTypes, - ::testing::Combine(::testing::Range(0, 10), // repeat to probe connection stability - ::testing::Values(std::make_pair(zmq::PAIR, zmq::PAIR)), - socket_name_options - )); - - -class AsyncSocketTypes : public ::testing::TestWithParam< std::tuple, std::pair /* socket_names */, bool /* expect_throw */> > {}; - -TEST_P(AsyncSocketTypes, forkMultiSendReceive) { - bool expect_throw = std::get<3>(GetParam()); - ZmqLingeringSocketPtr<> socket; - auto socket_names = std::get<2>(GetParam()); - pid_t child_pid {0}; - do { - child_pid = fork(); - } while (child_pid == -1); // retry if fork fails - - if (child_pid > 0) { // master - socket.reset(zmqSvc().socket_ptr(std::get<1>(GetParam()).first)); - elaborate_bind(socket, socket_names.second); - // bind is on the master process to avoid zombie children to hold on to binds - - // start test: send 2 things, receive 1, send 1 more, finish - zmqSvc().send(*socket, std::string("breaker breaker")); - - if (expect_throw) { - EXPECT_ANY_THROW(zmqSvc().send(*socket, std::string("anybody out there?"))); - // NOTE: also in case of a throw, be sure to properly close down the connection! - // Otherwise, you may get zombies waiting for a reply. + ::testing::Combine(::testing::Range(0, 10), // repeat to probe connection stability + ::testing::Values(std::make_pair(zmq::PAIR, zmq::PAIR)), + socket_name_options)); + +class AsyncSocketTypes + : public ::testing::TestWithParam< + std::tuple, + std::pair /* socket_names */, bool /* expect_throw */>> { +}; + +TEST_P(AsyncSocketTypes, forkMultiSendReceive) +{ + bool expect_throw = std::get<3>(GetParam()); + ZmqLingeringSocketPtr<> socket; + auto socket_names = std::get<2>(GetParam()); + pid_t child_pid{0}; + do { + child_pid = fork(); + } while (child_pid == -1); // retry if fork fails + + if (child_pid > 0) { // master + socket.reset(zmqSvc().socket_ptr(std::get<1>(GetParam()).first)); + elaborate_bind(socket, socket_names.second); + // bind is on the master process to avoid zombie children to hold on to binds + + // start test: send 2 things, receive 1, send 1 more, finish + zmqSvc().send(*socket, std::string("breaker breaker")); + + if (expect_throw) { + EXPECT_ANY_THROW(zmqSvc().send(*socket, std::string("anybody out there?"))); + // NOTE: also in case of a throw, be sure to properly close down the connection! + // Otherwise, you may get zombies waiting for a reply. + socket.reset(nullptr); + zmqSvc().close_context(); // if you don't close context in parent process as well, the next repeat will hang + wait_for_child(child_pid, true, 5); + return; + } else { + EXPECT_NO_THROW(zmqSvc().send(*socket, std::string("anybody out there?"))); + } + + auto receipt = zmqSvc().receive(*socket); + + EXPECT_EQ(receipt, 1212); + + zmqSvc().send(*socket, std::string("kthxbye")); + + // socket->close(); // this gives exception of type zmq::error_t: Socket operation on non-socket socket.reset(nullptr); zmqSvc().close_context(); // if you don't close context in parent process as well, the next repeat will hang + wait_for_child(child_pid, true, 5); - return; - } else { - EXPECT_NO_THROW(zmqSvc().send(*socket, std::string("anybody out there?"))); - } - - auto receipt = zmqSvc().receive(*socket); - - EXPECT_EQ(receipt, 1212); - - zmqSvc().send(*socket, std::string("kthxbye")); - -// socket->close(); // this gives exception of type zmq::error_t: Socket operation on non-socket - socket.reset(nullptr); - zmqSvc().close_context(); // if you don't close context in parent process as well, the next repeat will hang - - wait_for_child(child_pid, true, 5); - } else { // child - socket.reset(zmqSvc().socket_ptr(std::get<1>(GetParam()).second)); - socket->connect(socket_names.first); - - // start test, receive something - auto receipt1 = zmqSvc().receive(*socket); - auto receipt2 = zmqSvc().receive(*socket); - if (receipt1 == "breaker breaker" && receipt2 == "anybody out there?") { - zmqSvc().send(*socket, 1212); - } - auto receipt3 = zmqSvc().receive(*socket); - if (receipt3 != "kthxbye") { - std::cerr << "did not receive final reply correctly\n"; - } - - // take care, don't just use _exit, it will not cleanly destroy context etc! - // if you really need to, at least close and destroy everything properly -// socket->close(); // this gives exception of type zmq::error_t: Socket operation on non-socket - socket.reset(nullptr); - zmqSvc().close_context(); - _Exit(0); - } + } else { // child + socket.reset(zmqSvc().socket_ptr(std::get<1>(GetParam()).second)); + socket->connect(socket_names.first); + + // start test, receive something + auto receipt1 = zmqSvc().receive(*socket); + auto receipt2 = zmqSvc().receive(*socket); + if (receipt1 == "breaker breaker" && receipt2 == "anybody out there?") { + zmqSvc().send(*socket, 1212); + } + auto receipt3 = zmqSvc().receive(*socket); + if (receipt3 != "kthxbye") { + std::cerr << "did not receive final reply correctly\n"; + } + + // take care, don't just use _exit, it will not cleanly destroy context etc! + // if you really need to, at least close and destroy everything properly + // socket->close(); // this gives exception of type zmq::error_t: Socket operation on non-socket + socket.reset(nullptr); + zmqSvc().close_context(); + _Exit(0); + } } -TEST_P(AsyncSocketTypes, forkIgnoreSomeMessages) { - bool expect_throw = std::get<3>(GetParam()); - ZmqLingeringSocketPtr<> socket; - auto socket_names = std::get<2>(GetParam()); - pid_t child_pid {0}; - do { - child_pid = fork(); - } while (child_pid == -1); // retry if fork fails - - if (child_pid > 0) { // master - socket.reset(zmqSvc().socket_ptr(std::get<1>(GetParam()).first)); - elaborate_bind(socket, socket_names.second); - // bind is on the master process to avoid zombie children to hold on to binds - - // start test: send 2 things, receive 1, send 1 more, finish - zmqSvc().send(*socket, std::string("breaker breaker")); - - if (expect_throw) { - EXPECT_ANY_THROW(zmqSvc().send(*socket, std::string("anybody out there?"))); - // NOTE: also in case of a throw, be sure to properly close down the connection! - // Otherwise, you may get zombies waiting for a reply. +TEST_P(AsyncSocketTypes, forkIgnoreSomeMessages) +{ + bool expect_throw = std::get<3>(GetParam()); + ZmqLingeringSocketPtr<> socket; + auto socket_names = std::get<2>(GetParam()); + pid_t child_pid{0}; + do { + child_pid = fork(); + } while (child_pid == -1); // retry if fork fails + + if (child_pid > 0) { // master + socket.reset(zmqSvc().socket_ptr(std::get<1>(GetParam()).first)); + elaborate_bind(socket, socket_names.second); + // bind is on the master process to avoid zombie children to hold on to binds + + // start test: send 2 things, receive 1, send 1 more, finish + zmqSvc().send(*socket, std::string("breaker breaker")); + + if (expect_throw) { + EXPECT_ANY_THROW(zmqSvc().send(*socket, std::string("anybody out there?"))); + // NOTE: also in case of a throw, be sure to properly close down the connection! + // Otherwise, you may get zombies waiting for a reply. + socket.reset(nullptr); + zmqSvc().close_context(); // if you don't close context in parent process as well, the next repeat will hang + wait_for_child(child_pid, true, 5); + return; + } else { + EXPECT_NO_THROW(zmqSvc().send(*socket, std::string("anybody out there?"))); + } + + auto receipt = zmqSvc().receive(*socket); + + EXPECT_EQ(receipt, 1212); + + zmqSvc().send(*socket, std::string("kthxbye")); + + // socket->close(); // this gives exception of type zmq::error_t: Socket operation on non-socket socket.reset(nullptr); zmqSvc().close_context(); // if you don't close context in parent process as well, the next repeat will hang + wait_for_child(child_pid, true, 5); - return; - } else { - EXPECT_NO_THROW(zmqSvc().send(*socket, std::string("anybody out there?"))); - } - - auto receipt = zmqSvc().receive(*socket); - - EXPECT_EQ(receipt, 1212); - - zmqSvc().send(*socket, std::string("kthxbye")); - -// socket->close(); // this gives exception of type zmq::error_t: Socket operation on non-socket - socket.reset(nullptr); - zmqSvc().close_context(); // if you don't close context in parent process as well, the next repeat will hang - - wait_for_child(child_pid, true, 5); - } else { // child - socket.reset(zmqSvc().socket_ptr(std::get<1>(GetParam()).second)); - socket->connect(socket_names.first); - - // start test, receive first thing - auto receipt = zmqSvc().receive(*socket); - if (receipt == "breaker breaker") { - zmqSvc().send(*socket, 1212); - } - - // ignore the rest of the sent messages, but give the other end a second to - // actually send its stuff, instead of hanging in retry_send because the - // connection has died; a better solution would be if retry_send (in - // ZeroMQSvc::send) had a callback mechanism that could be used to break - // out when a child has died, but ok - sleep(1); - - // take care, don't just use _exit, it will not cleanly destroy context etc! - // if you really need to, at least close and destroy everything properly -// socket->close(); // this gives exception of type zmq::error_t: Socket operation on non-socket - socket.reset(nullptr); - zmqSvc().close_context(); - _Exit(0); - } -} + } else { // child + socket.reset(zmqSvc().socket_ptr(std::get<1>(GetParam()).second)); + socket->connect(socket_names.first); + + // start test, receive first thing + auto receipt = zmqSvc().receive(*socket); + if (receipt == "breaker breaker") { + zmqSvc().send(*socket, 1212); + } + + // ignore the rest of the sent messages, but give the other end a second to + // actually send its stuff, instead of hanging in retry_send because the + // connection has died; a better solution would be if retry_send (in + // ZeroMQSvc::send) had a callback mechanism that could be used to break + // out when a child has died, but ok + sleep(1); + // take care, don't just use _exit, it will not cleanly destroy context etc! + // if you really need to, at least close and destroy everything properly + // socket->close(); // this gives exception of type zmq::error_t: Socket operation on non-socket + socket.reset(nullptr); + zmqSvc().close_context(); + _Exit(0); + } +} INSTANTIATE_TEST_SUITE_P(PAIRPAIR, AsyncSocketTypes, - ::testing::Combine(::testing::Range(0, 10), // repeat to probe connection stability - ::testing::Values(std::make_pair(zmq::PAIR, zmq::PAIR)), - socket_name_options, - ::testing::Values(false) // don't expect throw - )); + ::testing::Combine(::testing::Range(0, 10), // repeat to probe connection stability + ::testing::Values(std::make_pair(zmq::PAIR, zmq::PAIR)), + socket_name_options, + ::testing::Values(false) // don't expect throw + )); INSTANTIATE_TEST_SUITE_P(REQREP, AsyncSocketTypes, - ::testing::Combine(::testing::Values(0), // no repeats, we only care about the throw - ::testing::Values(std::make_pair(zmq::REQ, zmq::REP)), - socket_name_options, - ::testing::Values(true) // expect throw - )); + ::testing::Combine(::testing::Values(0), // no repeats, we only care about the throw + ::testing::Values(std::make_pair(zmq::REQ, zmq::REP)), socket_name_options, + ::testing::Values(true) // expect throw + )); diff --git a/roofit/roofitZMQ/test/test_ZMQ_load_balancing.cxx b/roofit/roofitZMQ/test/test_ZMQ_load_balancing.cxx index ebb4680357104..1acbd44117eab 100644 --- a/roofit/roofitZMQ/test/test_ZMQ_load_balancing.cxx +++ b/roofit/roofitZMQ/test/test_ZMQ_load_balancing.cxx @@ -24,12 +24,13 @@ class ZMQPushPullTest : public ::testing::Test { std::size_t N_children = 4; std::size_t max_sends = 20; - void SetUp() override { + void SetUp() override + { for (std::size_t i = 0; i < N_children; ++i) { do { child_pid = fork(); - } while (child_pid == -1); // retry if fork fails - if (child_pid == 0) { // child + } while (child_pid == -1); // retry if fork fails + if (child_pid == 0) { // child child_id = i; break; } else { @@ -37,10 +38,10 @@ class ZMQPushPullTest : public ::testing::Test { } } - if (child_pid > 0) { // parent + if (child_pid > 0) { // parent pusher.reset(zmqSvc().socket_ptr(zmq::PUSH)); pusher->bind("ipc:///tmp/ZMQ_test_push_pull_P2C.ipc"); - } else { // child + } else { // child puller.reset(zmqSvc().socket_ptr(zmq::PULL)); puller->connect("ipc:///tmp/ZMQ_test_push_pull_P2C.ipc"); @@ -48,8 +49,9 @@ class ZMQPushPullTest : public ::testing::Test { } } - void TearDown() override { - if (child_pid > 0) { // parent + void TearDown() override + { + if (child_pid > 0) { // parent // wait for children int status = 0; pid_t pid; @@ -58,7 +60,15 @@ class ZMQPushPullTest : public ::testing::Test { pid = waitpid(child_pid_i, &status, 0); } while (-1 == pid && EINTR == errno); // retry on interrupted system call if (0 != status) { - if (WIFEXITED(status)) { printf("exited, status=%d\n", WEXITSTATUS(status)); } else if (WIFSIGNALED(status)) { printf("killed by signal %d\n", WTERMSIG(status)); } else if (WIFSTOPPED(status)) { printf("stopped by signal %d\n", WSTOPSIG(status)); } else if (WIFCONTINUED(status)) { printf("continued\n"); } + if (WIFEXITED(status)) { + printf("exited, status=%d\n", WEXITSTATUS(status)); + } else if (WIFSIGNALED(status)) { + printf("killed by signal %d\n", WTERMSIG(status)); + } else if (WIFSTOPPED(status)) { + printf("stopped by signal %d\n", WSTOPSIG(status)); + } else if (WIFCONTINUED(status)) { + printf("continued\n"); + } } if (-1 == pid) { throw std::runtime_error(std::string("waitpid, errno ") + std::to_string(errno)); @@ -66,16 +76,18 @@ class ZMQPushPullTest : public ::testing::Test { } pusher.reset(nullptr); zmqSvc().close_context(); - } else { // child + } else { // child puller.reset(nullptr); zmqSvc().close_context(); } } - void run_parent() { + void run_parent() + { // start test usleep(1000); // wait a second so that all pull sockets are connected for round-robin distribution - // if you don't wait a second above, the push socket will "round-robin" all the messages to just one or two connected sockets + // if you don't wait a second above, the push socket will "round-robin" all the messages to just one or two + // connected sockets for (std::size_t ix = 0; ix < max_sends; ++ix) { zmqSvc().send(*pusher, 0); } @@ -85,7 +97,8 @@ class ZMQPushPullTest : public ::testing::Test { } } - void run_child() { + void run_child() + { std::size_t count = 0; for (std::size_t ix = 0; ix < max_sends; ++ix) { auto r = poller.poll(2000); @@ -105,16 +118,17 @@ class ZMQPushPullTest : public ::testing::Test { printf("child %d got %lu values\n", child_id, count); } - pid_t child_pid {0}; + pid_t child_pid{0}; int child_id = -1; std::vector child_pids; ZmqLingeringSocketPtr<> pusher, puller; ZeroMQPoller poller; }; - -/// This test shows how push-pull is unsuited for load balancing; messages are just sent to the first available pull socket without any dynamic load balancing -TEST_F(ZMQPushPullTest, demoRoundRobin) { +/// This test shows how push-pull is unsuited for load balancing; messages are just sent to the first available pull +/// socket without any dynamic load balancing +TEST_F(ZMQPushPullTest, demoRoundRobin) +{ if (child_pid > 0) { run_parent(); } else { @@ -122,8 +136,10 @@ TEST_F(ZMQPushPullTest, demoRoundRobin) { } } -/// This test tries to see whether push-pull can be made to work as a bit of a load balancer, using a low HWM at the receiver -TEST_F(ZMQPushPullTest, demoHWM1LoadBalancing) { +/// This test tries to see whether push-pull can be made to work as a bit of a load balancer, using a low HWM at the +/// receiver +TEST_F(ZMQPushPullTest, demoHWM1LoadBalancing) +{ if (child_pid > 0) { run_parent(); } else { diff --git a/roofit/roofitZMQ/test/test_polling.cxx b/roofit/roofitZMQ/test/test_polling.cxx index 9250b2ee6018b..9f6b889c85a9c 100644 --- a/roofit/roofitZMQ/test/test_polling.cxx +++ b/roofit/roofitZMQ/test/test_polling.cxx @@ -12,7 +12,6 @@ * http://www.apache.org/licenses/LICENSE-2.0 */ - #include "gtest/gtest.h" #include // fork, usleep @@ -24,28 +23,28 @@ #include "RooFit_ZMQ/ZeroMQSvc.h" #include "RooFit_ZMQ/ZeroMQPoller.h" - static bool terminated = false; -void handle_sigterm(int signum) { +void handle_sigterm(int signum) +{ terminated = true; std::cout << "handled signal " << strsignal(signum) << " on PID " << getpid() << std::endl; } - -TEST(Polling, doublePoll) { - pid_t child_pid {0}; +TEST(Polling, doublePoll) +{ + pid_t child_pid{0}; do { child_pid = fork(); - } while (child_pid == -1); // retry if fork fails + } while (child_pid == -1); // retry if fork fails - if (child_pid > 0) { // master + if (child_pid > 0) { // master sigset_t sigmask, sigmask_old; sigemptyset(&sigmask); sigaddset(&sigmask, SIGCHLD); sigprocmask(SIG_BLOCK, &sigmask, &sigmask_old); -// std::cout << "master PID: " << getpid() << std::endl; + // std::cout << "master PID: " << getpid() << std::endl; ZmqLingeringSocketPtr<> pusher, puller; pusher.reset(zmqSvc().socket_ptr(zmq::PUSH)); pusher->bind("ipc:///tmp/ZMQ_test_fork_polling_M2C.ipc"); @@ -84,7 +83,15 @@ TEST(Polling, doublePoll) { } while (-1 == pid && EINTR == errno); // retry on interrupted system call if (0 != status) { - if (WIFEXITED(status)) { printf("exited, status=%d\n", WEXITSTATUS(status)); } else if (WIFSIGNALED(status)) { printf("killed by signal %d\n", WTERMSIG(status)); } else if (WIFSTOPPED(status)) { printf("stopped by signal %d\n", WSTOPSIG(status)); } else if (WIFCONTINUED(status)) { printf("continued\n"); } + if (WIFEXITED(status)) { + printf("exited, status=%d\n", WEXITSTATUS(status)); + } else if (WIFSIGNALED(status)) { + printf("killed by signal %d\n", WTERMSIG(status)); + } else if (WIFSTOPPED(status)) { + printf("stopped by signal %d\n", WSTOPSIG(status)); + } else if (WIFCONTINUED(status)) { + printf("continued\n"); + } } if (-1 == pid) { @@ -92,8 +99,8 @@ TEST(Polling, doublePoll) { } sigprocmask(SIG_SETMASK, &sigmask_old, nullptr); - } else { // child -// std::cout << "child PID: " << getpid() << std::endl; + } else { // child + // std::cout << "child PID: " << getpid() << std::endl; sigset_t sigmask, sigmask_old; sigemptyset(&sigmask); @@ -101,7 +108,7 @@ TEST(Polling, doublePoll) { sigprocmask(SIG_BLOCK, &sigmask, &sigmask_old); struct sigaction sa; - memset (&sa, '\0', sizeof(sa)); + memset(&sa, '\0', sizeof(sa)); sa.sa_handler = handle_sigterm; if (sigaction(SIGTERM, &sa, NULL) < 0) { @@ -136,14 +143,14 @@ TEST(Polling, doublePoll) { sigprocmask(SIG_SETMASK, &sigmask_old, nullptr); - while (!terminated) {} + while (!terminated) { + } -// std::cout << "child terminated" << std::endl; + // std::cout << "child terminated" << std::endl; puller.reset(nullptr); pusher.reset(nullptr); zmqSvc().close_context(); _Exit(0); } - } From f8fb2b005a45dd0ce885535b359450958767e84c Mon Sep 17 00:00:00 2001 From: "E. G. Patrick Bos" Date: Wed, 23 Jun 2021 20:59:34 +0200 Subject: [PATCH 14/15] switch cppzmq to version 4.3.0 This gets rid of deprecation warnings. --- builtins/zeromq/CMakeLists.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/builtins/zeromq/CMakeLists.txt b/builtins/zeromq/CMakeLists.txt index 8b12714b9d9cf..11f9828297488 100644 --- a/builtins/zeromq/CMakeLists.txt +++ b/builtins/zeromq/CMakeLists.txt @@ -8,6 +8,7 @@ set(ZeroMQ_LIBRARIES ${ZeroMQ_LIBRARY} CACHE INTERNAL "" FORCE) ExternalProject_Add(BUILTIN_ZeroMQ URL https://github.com/zeromq/libzmq/archive/refs/tags/v4.3.2.tar.gz + URL_HASH SHA256=02ecc88466ae38cf2c8d79f09cfd2675ba299a439680b64ade733e26a349edeb CMAKE_ARGS -DCMAKE_INSTALL_PREFIX= -DWITH_PERF_TOOL=OFF @@ -41,7 +42,8 @@ set(ZeroMQ_SOURCE_DIR ${ZeroMQ_PREFIX}/src/BUILTIN_ZeroMQ/src CACHE INTERNAL "" set(ZeroMQ_BUILD_DIR ${ZeroMQ_PREFIX}/src/BUILTIN_ZeroMQ-build CACHE INTERNAL "" FORCE) ExternalProject_Add(BUILTIN_cppzmq - URL https://github.com/zeromq/cppzmq/archive/refs/tags/v4.7.1.tar.gz + URL https://github.com/zeromq/cppzmq/archive/refs/tags/v4.3.0.tar.gz + URL_HASH SHA256=27d1f56406ba94ee779e639203218820975cf68174f92fbeae0f645df0fcada4 CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${install_dir} -DCPPZMQ_BUILD_TESTS=OFF From 1fde1c3bef663d65836ac30f242744aa37da1167 Mon Sep 17 00:00:00 2001 From: "E. G. Patrick Bos" Date: Tue, 29 Jun 2021 12:11:52 +0200 Subject: [PATCH 15/15] remove ppoll copyright headers --- roofit/roofitZMQ/inc/RooFit_ZMQ/ppoll.h | 13 ------------- roofit/roofitZMQ/src/ppoll.cpp | 14 -------------- 2 files changed, 27 deletions(-) diff --git a/roofit/roofitZMQ/inc/RooFit_ZMQ/ppoll.h b/roofit/roofitZMQ/inc/RooFit_ZMQ/ppoll.h index 2f1edaf9ae78a..678dbe1d9ee04 100644 --- a/roofit/roofitZMQ/inc/RooFit_ZMQ/ppoll.h +++ b/roofit/roofitZMQ/inc/RooFit_ZMQ/ppoll.h @@ -1,16 +1,3 @@ -/* - * Project: RooFit - * Authors: - * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl - * - * Copyright (c) 2016-2019, Netherlands eScience Center - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - */ #ifndef ROOT_ROOFIT_ZMQ_ppoll #define ROOT_ROOFIT_ZMQ_ppoll diff --git a/roofit/roofitZMQ/src/ppoll.cpp b/roofit/roofitZMQ/src/ppoll.cpp index a82163b09837d..32372698dbc2b 100644 --- a/roofit/roofitZMQ/src/ppoll.cpp +++ b/roofit/roofitZMQ/src/ppoll.cpp @@ -1,17 +1,3 @@ -/* - * Project: RooFit - * Authors: - * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl - * - * Copyright (c) 2016-2019, Netherlands eScience Center - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - */ - #include #include // pselect