From faa4d0af19724e955f02de81e5c5e41b813d1b1c Mon Sep 17 00:00:00 2001 From: "E. G. Patrick Bos" Date: Wed, 9 Jun 2021 15:44:52 +0200 Subject: [PATCH 1/3] Add (builtin) ZeroMQ dependency for RooFit::MultiProcess --- .ci/copy_headers.sh | 2 +- builtins/zeromq/cppzmq/CMakeLists.txt | 35 +++++++++++ builtins/zeromq/libzmq/CMakeLists.txt | 47 ++++++++++++++ cmake/modules/FindZeroMQ.cmake | 60 ++++++++++++++++++ cmake/modules/Findcppzmq.cmake | 21 +++++++ cmake/modules/RootBuildOptions.cmake | 4 ++ cmake/modules/SearchInstalledSoftware.cmake | 69 +++++++++++++++++++++ 7 files changed, 237 insertions(+), 1 deletion(-) create mode 100644 builtins/zeromq/cppzmq/CMakeLists.txt create mode 100644 builtins/zeromq/libzmq/CMakeLists.txt create mode 100644 cmake/modules/FindZeroMQ.cmake create mode 100644 cmake/modules/Findcppzmq.cmake diff --git a/.ci/copy_headers.sh b/.ci/copy_headers.sh index fd4ec82b96e9d..a6587b6ee1dbb 100755 --- a/.ci/copy_headers.sh +++ b/.ci/copy_headers.sh @@ -10,6 +10,6 @@ cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -Dall=On -Dtesting=On -Dx11=Off -Dalien # We need to prebuild a minimal set of targets which are responsible for header copy # or generation. make -j4 move_headers intrinsics_gen clang-tablegen-targets ClangDriverOptions \ - googletest Dictgen BaseTROOT + googletest Dictgen BaseTROOT BUILTIN_cppzmq ln -s $PWD/compile_commands.json $PWD/../root/ diff --git a/builtins/zeromq/cppzmq/CMakeLists.txt b/builtins/zeromq/cppzmq/CMakeLists.txt new file mode 100644 index 0000000000000..186de3872d857 --- /dev/null +++ b/builtins/zeromq/cppzmq/CMakeLists.txt @@ -0,0 +1,35 @@ +include(ExternalProject) + +ExternalProject_Add(BUILTIN_cppzmq + URL https://github.com/zeromq/cppzmq/archive/refs/tags/v4.8.1.tar.gz + URL_HASH SHA256=7a23639a45f3a0049e11a188e29aaedd10b2f4845f0000cf3e22d6774ebde0af + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + DEPENDS libzmq +) + +# manually copy header (inspired by nlohmann builtin) +set(cppzmq_HEADER_PATH ${CMAKE_CURRENT_BINARY_DIR}/BUILTIN_cppzmq-prefix/src/BUILTIN_cppzmq) +add_custom_command( + OUTPUT ${CMAKE_BINARY_DIR}/include/zmq.hpp + COMMAND ${CMAKE_COMMAND} -E copy ${cppzmq_HEADER_PATH}/zmq.hpp ${CMAKE_BINARY_DIR}/include/zmq.hpp + COMMENT "Copying zmq.hpp header to ${CMAKE_BINARY_DIR}/include" + DEPENDS BUILTIN_cppzmq) + +add_custom_target(builtin_cppzmq_incl + DEPENDS ${CMAKE_BINARY_DIR}/include/zmq.hpp) + +set_property(GLOBAL APPEND PROPERTY ROOT_HEADER_TARGETS builtin_cppzmq_incl) + +install(FILES ${cppzmq_HEADER_PATH}/zmq.hpp + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/) + +# find_package emulation +set(cppzmq_FOUND TRUE CACHE BOOL "" FORCE) +set(cppzmq_INCLUDE_DIR ${CMAKE_BINARY_DIR}/include CACHE INTERNAL "" FORCE) +set(cppzmq_INCLUDE_DIRS ${cppzmq_INCLUDE_DIR} CACHE INTERNAL "" FORCE) + +add_library(cppzmq INTERFACE IMPORTED GLOBAL) +target_include_directories(cppzmq INTERFACE $) +add_dependencies(cppzmq BUILTIN_cppzmq builtin_cppzmq_incl) diff --git a/builtins/zeromq/libzmq/CMakeLists.txt b/builtins/zeromq/libzmq/CMakeLists.txt new file mode 100644 index 0000000000000..9ad57d202c070 --- /dev/null +++ b/builtins/zeromq/libzmq/CMakeLists.txt @@ -0,0 +1,47 @@ +include(ExternalProject) + +set(ZeroMQ_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/BUILTIN_ZeroMQ-prefix) +set(ZeroMQ_LIBNAME ${CMAKE_STATIC_LIBRARY_PREFIX}zmq${CMAKE_STATIC_LIBRARY_SUFFIX}) + +set(ZeroMQ_LIBRARY ${ZeroMQ_PREFIX}/lib/${ZeroMQ_LIBNAME} CACHE INTERNAL "" FORCE) +set(ZeroMQ_LIBRARIES ${ZeroMQ_LIBRARY} CACHE INTERNAL "" FORCE) + +if(NOT WIN32) + # 2021-10-07: libzmq's minimum CMake version does not yet support CXX_VISIBILITY_PRESET, so we need to pass the flag manually + set(cxx_visibility_flag "-fvisibility=hidden") +else() + set(cxx_visibility_flag "") +endif() + +ExternalProject_Add(BUILTIN_ZeroMQ + URL https://github.com/zeromq/libzmq/archive/7c2df78b49a3aa63e654b3f3526adf71ed091534.tar.gz + URL_HASH SHA256=fcc1b0648afa5d92e0ff0e6e93beb28cbbe008a5f98c228ff97144ba6e4a6c3e + CMAKE_ARGS + -DCMAKE_INSTALL_PREFIX= + -DCMAKE_INSTALL_LIBDIR=lib + -DWITH_PERF_TOOL=OFF + -DBUILD_TESTS=OFF + -DENABLE_DRAFTS=ON + -DENABLE_NO_EXPORT=ON + -DENABLE_CURVE=OFF + -DWITH_DOCS=OFF + -DENABLE_WS=OFF + -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} + -DCMAKE_CXX_FLAGS=${cxx_visibility_flag}\ ${CMAKE_CXX_FLAGS}\ -w + BUILD_BYPRODUCTS ${ZeroMQ_LIBRARIES} + ) + +set(ZeroMQ_FOUND TRUE CACHE BOOL "" FORCE) + +set(ZeroMQ_INCLUDE_DIR ${ZeroMQ_PREFIX}/include CACHE INTERNAL "" FORCE) +set(ZeroMQ_INCLUDE_DIRS ${ZeroMQ_PREFIX}/include CACHE INTERNAL "" FORCE) + +# Workaround to propagate INTERFACE_INCLUDE_DIRECTORIES, see https://stackoverflow.com/a/47358004/1199693 +file(MAKE_DIRECTORY ${ZeroMQ_INCLUDE_DIR}) + +add_library(libzmq INTERFACE IMPORTED GLOBAL) +target_include_directories(libzmq INTERFACE $) +target_link_libraries(libzmq INTERFACE $) +add_dependencies(libzmq BUILTIN_ZeroMQ) + +set(ZeroMQ_DIR ${ZeroMQ_PREFIX}/share/cmake/ZeroMQ CACHE INTERNAL "" FORCE) diff --git a/cmake/modules/FindZeroMQ.cmake b/cmake/modules/FindZeroMQ.cmake new file mode 100644 index 0000000000000..24a1c171f554e --- /dev/null +++ b/cmake/modules/FindZeroMQ.cmake @@ -0,0 +1,60 @@ +# Based on https://github.com/zeromq/cppzmq/blob/a98fa4a91d868a3844e5456741d6782cc1a8d98b/libzmq-pkg-config/FindZeroMQ.cmake +# MIT Licensed: +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +set(PKG_CONFIG_USE_CMAKE_PREFIX_PATH ON) +find_package(PkgConfig) +pkg_check_modules(PC_LIBZMQ QUIET libzmq) + +set(ZeroMQ_VERSION ${PC_LIBZMQ_VERSION}) + +find_path(ZeroMQ_INCLUDE_DIR zmq.h + PATHS ${ZeroMQ_DIR}/include + ${PC_LIBZMQ_INCLUDE_DIRS}) + +find_library(ZeroMQ_LIBRARY + NAMES zmq + PATHS ${ZeroMQ_DIR}/lib + ${PC_LIBZMQ_LIBDIR} + ${PC_LIBZMQ_LIBRARY_DIRS}) + +set ( ZeroMQ_LIBRARIES ${ZeroMQ_LIBRARY} ) +set ( ZeroMQ_INCLUDE_DIRS ${ZeroMQ_INCLUDE_DIR} ) + +# check for zmq_ppoll +if(ZeroMQ_FOUND) + SET(SAVE_CMAKE_REQUIRED_DEFINITIONS "${CMAKE_REQUIRED_DEFINITIONS}") + SET(CMAKE_REQUIRED_DEFINITIONS "-DZMQ_BUILD_DRAFT_API") + check_cxx_symbol_exists(zmq_ppoll zmq.h ZeroMQ_HAS_PPOLL) + SET(CMAKE_REQUIRED_DEFINITIONS "${SAVE_CMAKE_REQUIRED_DEFINITIONS}") +endif() + +include ( FindPackageHandleStandardArgs ) +# handle the QUIETLY and REQUIRED arguments and set ZeroMQ_FOUND to TRUE +# if all listed variables are TRUE +find_package_handle_standard_args ( ZeroMQ DEFAULT_MSG ZeroMQ_LIBRARIES ZeroMQ_INCLUDE_DIRS ZeroMQ_HAS_PPOLL) + +if(ZeroMQ_FOUND) + if(NOT TARGET libzmq) + add_library(libzmq UNKNOWN IMPORTED) + set_target_properties(libzmq PROPERTIES + IMPORTED_LOCATION ${ZeroMQ_LIBRARIES} + INTERFACE_INCLUDE_DIRECTORIES ${ZeroMQ_INCLUDE_DIRS}) + endif() +endif() \ No newline at end of file diff --git a/cmake/modules/Findcppzmq.cmake b/cmake/modules/Findcppzmq.cmake new file mode 100644 index 0000000000000..8b6d5f6cddae6 --- /dev/null +++ b/cmake/modules/Findcppzmq.cmake @@ -0,0 +1,21 @@ +if (NOT (ZeroMQ_FOUND OR TARGET libzmq)) + message(FATAL_ERROR "Search for libzmq first!") +endif() + +find_path(cppzmq_INCLUDE_DIRS "zmq.hpp" + HINTS "${ZeroMQ_INCLUDE_DIR}" + PATH_SUFFIXES "include" "cppzmq" + ) +mark_as_advanced(cppzmq_INCLUDE_DIRS) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args (cppzmq DEFAULT_MSG cppzmq_INCLUDE_DIRS) + +if(cppzmq_FOUND) + add_library(cppzmq INTERFACE IMPORTED) + set_target_properties(cppzmq PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${CPPZMQ_INCLUDE_DIRS}" + INTERFACE_LINK_LIBRARIES libzmq + ) + set(CPPZMQ_LIBRARIES cppzmq) +endif() \ No newline at end of file diff --git a/cmake/modules/RootBuildOptions.cmake b/cmake/modules/RootBuildOptions.cmake index 7c864b93289a6..62abd3c86618d 100644 --- a/cmake/modules/RootBuildOptions.cmake +++ b/cmake/modules/RootBuildOptions.cmake @@ -86,6 +86,7 @@ ROOT_BUILD_OPTION(builtin_cfitsio OFF "Build CFITSIO internally (requires networ ROOT_BUILD_OPTION(builtin_clang ON "Build bundled copy of Clang") ROOT_BUILD_OPTION(builtin_cling ON "Build bundled copy of Cling. Only build with an external cling if you know what you are doing: associating ROOT commits with cling commits is tricky.") MARK_AS_ADVANCED(builtin_cling) +ROOT_BUILD_OPTION(builtin_cppzmq OFF "Use ZeroMQ C++ bindings installed by ROOT (requires network)") ROOT_BUILD_OPTION(builtin_davix OFF "Build Davix internally (requires network)") ROOT_BUILD_OPTION(builtin_fftw3 OFF "Build FFTW3 internally (requires network)") ROOT_BUILD_OPTION(builtin_freetype OFF "Build bundled copy of freetype") @@ -107,6 +108,7 @@ ROOT_BUILD_OPTION(builtin_vdt OFF "Build VDT internally (requires network)") ROOT_BUILD_OPTION(builtin_veccore OFF "Build VecCore internally (requires network)") ROOT_BUILD_OPTION(builtin_xrootd OFF "Build XRootD internally (requires network)") ROOT_BUILD_OPTION(builtin_xxhash OFF "Build bundled copy of xxHash") +ROOT_BUILD_OPTION(builtin_zeromq OFF "Build ZeroMQ internally (requires network)") ROOT_BUILD_OPTION(builtin_zlib OFF "Build bundled copy of zlib") ROOT_BUILD_OPTION(builtin_zstd OFF "Build included libzstd, or use system libzstd") ROOT_BUILD_OPTION(ccache OFF "Enable ccache usage for speeding up builds") @@ -280,6 +282,7 @@ if(builtin_all) set(builtin_cfitsio_defvalue ON) set(builtin_clang_defvalue ON) set(builtin_cling_defvalue ON) + set(builtin_cppzmq_defvalue ON) set(builtin_davix_defvalue ON) set(builtin_fftw3_defvalue ON) set(builtin_freetype_defvalue ON) @@ -301,6 +304,7 @@ if(builtin_all) set(builtin_veccore_defvalue ON) set(builtin_xrootd_defvalue ON) set(builtin_xxhash_defvalue ON) + set(builtin_zeromq_defvalue ON) set(builtin_zlib_defvalue ON) set(builtin_zstd_defvalue ON) endif() diff --git a/cmake/modules/SearchInstalledSoftware.cmake b/cmake/modules/SearchInstalledSoftware.cmake index 19c3339390f37..744e8488f1f2b 100644 --- a/cmake/modules/SearchInstalledSoftware.cmake +++ b/cmake/modules/SearchInstalledSoftware.cmake @@ -1862,6 +1862,75 @@ if(testing AND NO_CONNECTION) endif() endif() +#---Check for ZeroMQ when building RooFit::MultiProcess-------------------------------------------- + +if (roofit_multiprocess) + if(NOT builtin_zeromq) + message(STATUS "Looking for ZeroMQ (libzmq)") + # Clear cache before calling find_package(ZeroMQ), + # necessary to be able to toggle builtin_zeromq and + # not have find_package(ZeroMQ) find builtin ZeroMQ. + foreach(suffix FOUND INCLUDE_DIR INCLUDE_DIRS LIBRARY LIBRARIES) + unset(ZeroMQ_${suffix} CACHE) + endforeach() + + # Temporarily prefer config mode over module mode, so that a CMake-installed system version + # gets detected before looking for an autotools-installed system version (which the + # FindZeroMQ.cmake module does). + set(CMAKE_FIND_PACKAGE_PREFER_CONFIG_ORIGINAL_VALUE ${CMAKE_FIND_PACKAGE_PREFER_CONFIG}) + set(CMAKE_FIND_PACKAGE_PREFER_CONFIG TRUE) + + if(fail-on-missing) + find_package(ZeroMQ REQUIRED) + else() + find_package(ZeroMQ) + if(NOT ZeroMQ_FOUND) + message(STATUS "ZeroMQ not found. Switching on builtin_zeromq option") + set(builtin_zeromq ON CACHE BOOL "Enabled because ZeroMQ not found (${builtin_zeromq_description})" FORCE) + endif() + endif() + + # Reset default find_package mode + set(CMAKE_FIND_PACKAGE_PREFER_CONFIG ${CMAKE_FIND_PACKAGE_PREFER_CONFIG_ORIGINAL_VALUE}) + unset(CMAKE_FIND_PACKAGE_PREFER_CONFIG_ORIGINAL_VALUE) + endif() + + if(builtin_zeromq) + list(APPEND ROOT_BUILTINS ZeroMQ) + add_subdirectory(builtins/zeromq/libzmq) + endif() + + if(NOT builtin_cppzmq) + message(STATUS "Looking for ZeroMQ C++ bindings (cppzmq)") + # Clear cache before calling find_package(cppzmq), + # necessary to be able to toggle builtin_cppzmq and + # not have find_package(cppzmq) find builtin cppzmq. + foreach(suffix FOUND INCLUDE_DIR INCLUDE_DIRS) + unset(cppzmq_${suffix} CACHE) + endforeach() + if(fail-on-missing) + find_package(cppzmq REQUIRED) + else() + find_package(cppzmq QUIET) + if(NOT cppzmq_FOUND) + message(STATUS "ZeroMQ C++ bindings not found. Switching on builtin_cppzmq option") + set(builtin_cppzmq ON CACHE BOOL "Enabled because ZeroMQ C++ bindings not found (${builtin_cppzmq_description})" FORCE) + endif() + endif() + endif() + + if(builtin_cppzmq) + list(APPEND ROOT_BUILTINS cppzmq) + add_subdirectory(builtins/zeromq/cppzmq) + endif() + + # zmq_ppoll is still in the draft API, so enable that transitively + target_compile_definitions(libzmq INTERFACE ZMQ_BUILD_DRAFT_API) + target_compile_definitions(libzmq INTERFACE ZMQ_NO_EXPORT) + target_compile_definitions(cppzmq INTERFACE ZMQ_BUILD_DRAFT_API) + target_compile_definitions(cppzmq INTERFACE ZMQ_NO_EXPORT) +endif (roofit_multiprocess) + #---Download googletest-------------------------------------------------------------- if (testing) # FIXME: Remove our version of gtest in roottest. We can reuse this one. From 12cb88d849fac2389e521028cce1b8876463ce53 Mon Sep 17 00:00:00 2001 From: "E. G. Patrick Bos" Date: Thu, 10 Jun 2021 15:55:06 +0200 Subject: [PATCH 2/3] add RooFitZMQ package The files in here are modified versions of the ZMQ convenience functions from https://gitlab.cern.ch/raaij/generate_and_sort, contributed by @roelaaij. --- roofit/CMakeLists.txt | 3 + roofit/roofitZMQ/CMakeLists.txt | 17 + roofit/roofitZMQ/res/RooFit_ZMQ/Utility.h | 47 +++ .../roofitZMQ/res/RooFit_ZMQ/ZeroMQPoller.h | 61 ++++ roofit/roofitZMQ/res/RooFit_ZMQ/ZeroMQSvc.h | 223 ++++++++++++ roofit/roofitZMQ/res/RooFit_ZMQ/functions.h | 31 ++ roofit/roofitZMQ/res/RooFit_ZMQ/ppoll.h | 28 ++ roofit/roofitZMQ/src/ZeroMQPoller.cpp | 244 ++++++++++++++ roofit/roofitZMQ/src/ZeroMQSvc.cpp | 173 ++++++++++ roofit/roofitZMQ/src/functions.cpp | 23 ++ roofit/roofitZMQ/src/ppoll.cpp | 34 ++ roofit/roofitZMQ/test/CMakeLists.txt | 14 + roofit/roofitZMQ/test/test_HWM.cxx | 144 ++++++++ roofit/roofitZMQ/test/test_ZMQ.cpp | 317 ++++++++++++++++++ .../test/test_ZMQ_load_balancing.cxx | 148 ++++++++ roofit/roofitZMQ/test/test_mkstemp.cxx | 33 ++ roofit/roofitZMQ/test/test_polling.cxx | 161 +++++++++ 17 files changed, 1701 insertions(+) create mode 100644 roofit/roofitZMQ/CMakeLists.txt create mode 100644 roofit/roofitZMQ/res/RooFit_ZMQ/Utility.h create mode 100644 roofit/roofitZMQ/res/RooFit_ZMQ/ZeroMQPoller.h create mode 100644 roofit/roofitZMQ/res/RooFit_ZMQ/ZeroMQSvc.h create mode 100644 roofit/roofitZMQ/res/RooFit_ZMQ/functions.h create mode 100644 roofit/roofitZMQ/res/RooFit_ZMQ/ppoll.h create mode 100644 roofit/roofitZMQ/src/ZeroMQPoller.cpp create mode 100644 roofit/roofitZMQ/src/ZeroMQSvc.cpp create mode 100644 roofit/roofitZMQ/src/functions.cpp create mode 100644 roofit/roofitZMQ/src/ppoll.cpp create mode 100644 roofit/roofitZMQ/test/CMakeLists.txt create mode 100644 roofit/roofitZMQ/test/test_HWM.cxx create mode 100644 roofit/roofitZMQ/test/test_ZMQ.cpp create mode 100644 roofit/roofitZMQ/test/test_ZMQ_load_balancing.cxx create mode 100644 roofit/roofitZMQ/test/test_mkstemp.cxx create mode 100644 roofit/roofitZMQ/test/test_polling.cxx diff --git a/roofit/CMakeLists.txt b/roofit/CMakeLists.txt index 890df0340dde2..cec391f22a510 100644 --- a/roofit/CMakeLists.txt +++ b/roofit/CMakeLists.txt @@ -5,6 +5,9 @@ # For the list of contributors see $ROOTSYS/README/CREDITS. add_subdirectory(batchcompute) +if (builtin_zeromq) + add_subdirectory(roofitZMQ) +endif(builtin_zeromq) add_subdirectory(roofitcore) add_subdirectory(roofit) if(mathmore) diff --git a/roofit/roofitZMQ/CMakeLists.txt b/roofit/roofitZMQ/CMakeLists.txt new file mode 100644 index 0000000000000..f22d3f3910b74 --- /dev/null +++ b/roofit/roofitZMQ/CMakeLists.txt @@ -0,0 +1,17 @@ +############################################################################ +# CMakeLists.txt file for building ROOT roofitcore/ZMQ package +# @author Patrick Bos, Netherlands eScience Center +############################################################################ + +ROOT_LINKER_LIBRARY(RooFitZMQ + src/ZeroMQSvc.cpp + src/ZeroMQPoller.cpp + src/functions.cpp + src/ppoll.cpp + ) + +target_link_libraries(RooFitZMQ PUBLIC libzmq cppzmq) +set(RooFitZMQ_INCLUDE_DIRS ${CMAKE_CURRENT_SOURCE_DIR}/res) +target_include_directories(RooFitZMQ PRIVATE ${RooFitZMQ_INCLUDE_DIRS}) + +ROOT_ADD_TEST_SUBDIRECTORY(test) diff --git a/roofit/roofitZMQ/res/RooFit_ZMQ/Utility.h b/roofit/roofitZMQ/res/RooFit_ZMQ/Utility.h new file mode 100644 index 0000000000000..77d18d6b139c9 --- /dev/null +++ b/roofit/roofitZMQ/res/RooFit_ZMQ/Utility.h @@ -0,0 +1,47 @@ +/* + * Project: RooFit + * Authors: + * RA, Roel Aaij, NIKHEF + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#ifndef SERIALIZE_UTILITY_H +#define SERIALIZE_UTILITY_H 1 + +#if defined(__clang__) && (__clang_major__ > 3 || (__clang_major__ == 3 && __clang_minor__ >= 10)) +#define HAVE_TRIVIALLY_COPYABLE 1 +#elif defined(__GNUC__) && __GNUC__ >= 5 +#define HAVE_TRIVIALLY_COPYABLE 1 +#else +#undef HAVE_TRIVIALLY_COPYABLE +#endif + +#include + +namespace ZMQ { +namespace Detail { + +#if defined(HAVE_TRIVIALLY_COPYABLE) +template +using simple_object = std::is_trivially_copyable; +#else +template +using simple_object = std::is_pod; +#endif + +// is trivial +template +struct is_trivial + : std::conditional::type>::value, std::true_type, std::false_type>::type { +}; + +} // namespace Detail +} // namespace ZMQ + +#endif // SERIALIZE_UTILITY_H diff --git a/roofit/roofitZMQ/res/RooFit_ZMQ/ZeroMQPoller.h b/roofit/roofitZMQ/res/RooFit_ZMQ/ZeroMQPoller.h new file mode 100644 index 0000000000000..fda44dc3b913a --- /dev/null +++ b/roofit/roofitZMQ/res/RooFit_ZMQ/ZeroMQPoller.h @@ -0,0 +1,61 @@ +/* + * Project: RooFit + * Authors: + * RA, Roel Aaij, NIKHEF + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#ifndef ZEROMQPOLLER_H +#define ZEROMQPOLLER_H 1 + +#include "RooFit_ZMQ/ZeroMQSvc.h" +#include "RooFit_ZMQ/functions.h" +#include + +#include +#include +#include +#include + +class ZeroMQPoller { +public: + using entry_t = std::tuple; + // The key is what zmq::socket_t stores inside, and what goes into + // pollitem_t through zmq::socket_t's conversion to void* operator + using sockets_t = std::unordered_map; + + using fd_entry_t = std::tuple; + using fds_t = std::unordered_map; + + using free_t = std::deque; + + ZeroMQPoller() = default; + + std::vector> poll(int timeo = -1); + std::vector> ppoll(int timeo, const sigset_t *sigmask); + + size_t size() const; + + size_t register_socket(zmq::socket_t &socket, zmq::event_flags type); + size_t register_socket(int fd, zmq::event_flags type); + + size_t unregister_socket(zmq::socket_t &socket); + size_t unregister_socket(int fd); + +private: + // Vector of (socket, flags) + std::vector m_items; + sockets_t m_sockets; + fds_t m_fds; + + // free slots in items + free_t m_free; +}; + +#endif // ZEROMQPOLLER_H diff --git a/roofit/roofitZMQ/res/RooFit_ZMQ/ZeroMQSvc.h b/roofit/roofitZMQ/res/RooFit_ZMQ/ZeroMQSvc.h new file mode 100644 index 0000000000000..77c3303216c39 --- /dev/null +++ b/roofit/roofitZMQ/res/RooFit_ZMQ/ZeroMQSvc.h @@ -0,0 +1,223 @@ +/* + * Project: RooFit + * Authors: + * RA, Roel Aaij, NIKHEF + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#ifndef ZEROMQ_IZEROMQSVC_H +#define ZEROMQ_IZEROMQSVC_H 1 + +#include +#include "RooFit_ZMQ/Utility.h" +#include "RooFit_ZMQ/functions.h" + +#include +#include +#include +#include +#include +#include // std::cerr +#include + +// debugging +#include // getpid + +namespace ZMQ { + +struct TimeOutException : std::exception { + TimeOutException() = default; + TimeOutException(const TimeOutException &) = default; + ~TimeOutException() = default; + TimeOutException &operator=(const TimeOutException &) = default; +}; + +struct MoreException : std::exception { + MoreException() = default; + MoreException(const MoreException &) = default; + ~MoreException() = default; + MoreException &operator=(const MoreException &) = default; +}; + +} // namespace ZMQ + +template +struct ZmqLingeringSocketPtrDeleter { + void operator()(zmq::socket_t *socket) + { + int tries = 0; + int max_tries = 3; + while (true) { + try { + // the actual work this function should do, plus the delete socket below: + if (socket) + socket->set(zmq::sockopt::linger, PERIOD); + break; + } catch (zmq::error_t &e) { + if (++tries == max_tries || e.num() == EINVAL || e.num() == ETERM || + e.num() == ENOTSOCK // not recoverable from here + ) { + std::cerr << "ERROR in ZeroMQSvc::socket: " << e.what() << " (errno: " << e.num() << ")\n"; + throw; + } + std::cerr << "RETRY " << tries << "/" << (max_tries - 1) + << " in ZmqLingeringSocketPtrDeleter: call interrupted (errno: " << e.num() << ")\n"; + } + } + + delete socket; + } +}; + +template +using ZmqLingeringSocketPtr = std::unique_ptr>; + +// We retry send and receive only on EINTR, all other errors are either fatal, or can only +// be handled at the caller. +template +auto retry_send(zmq::socket_t &socket, int max_tries, args_t... args) -> decltype(socket.send(args...)) +{ + int tries = 0; + while (true) { + try { + // the actual work this function should do, all the rest is error handling: + return socket.send(args...); + } catch (zmq::error_t &e) { + if (++tries == max_tries || e.num() != EINTR // only recoverable error + ) { + throw; + } + std::cerr << "RETRY " << tries << "/" << (max_tries - 1) << " in ZeroMQSvc::send (retry_send) on pid " + << getpid() << ": " << e.what() << ")\n"; + } + } +} + +template +auto retry_recv(zmq::socket_t &socket, int max_tries, args_t... args) -> decltype(socket.recv(args...)) +{ + int tries = 0; + while (true) { + try { + // the actual work this function should do, all the rest is error handling: + return socket.recv(args...); + } catch (zmq::error_t &e) { + if (++tries == max_tries || e.num() != EINTR // only recoverable error + ) { + throw; + } + std::cerr << "RETRY " << tries << "/" << (max_tries - 1) << " in ZeroMQSvc::recv (retry_recv) on pid " + << getpid() << ": " << e.what() << ")\n"; + } + } +} + +class ZeroMQSvc { + // Note on error handling: + // Creating message_t can throw, but only when memory ran out (errno ENOMEM), + // and that is something only the caller can fix, so we don't catch it here. + +public: + enum Encoding { Text = 0, Binary }; + + Encoding encoding() const; + void setEncoding(const Encoding &e); + zmq::context_t &context() const; + zmq::socket_t socket(zmq::socket_type type) const; + zmq::socket_t *socket_ptr(zmq::socket_type type) const; + void close_context() const; + + /// decode message with ZMQ, POD version + template ::value && ZMQ::Detail::is_trivial::value, T>::type + * = nullptr> + T decode(const zmq::message_t &msg) const + { + T object; + memcpy(&object, msg.data(), msg.size()); + return object; + } + + /// decode ZMQ message, string version + template ::value, T>::type * = nullptr> + std::string decode(const zmq::message_t &msg) const + { + std::string r(msg.size() + 1, char{}); + r.assign(static_cast(msg.data()), msg.size()); + return r; + } + + /// receive message with ZMQ, general version + // FIXME: what to do with flags=0.... more is a pointer, that might prevent conversion + template ::value), T>::type * = nullptr> + T receive(zmq::socket_t &socket, zmq::recv_flags flags = zmq::recv_flags::none, bool *more = nullptr) const + { + // receive message + zmq::message_t msg; + auto recv_result = retry_recv(socket, 2, std::ref(msg), flags); + if (!recv_result) { + throw ZMQ::TimeOutException{}; + } + if (more) + *more = msg.more(); + + // decode message + return decode(msg); + } + + /// receive message with ZMQ + template ::value, T>::type * = nullptr> + T receive(zmq::socket_t &socket, zmq::recv_flags flags = zmq::recv_flags::none, bool *more = nullptr) const + { + // receive message + zmq::message_t msg; + auto recv_result = retry_recv(socket, 2, std::ref(msg), flags); + if (!recv_result) { + throw ZMQ::TimeOutException{}; + } + if (more) + *more = msg.more(); + return msg; + } + + /// encode message to ZMQ + template ::value && ZMQ::Detail::is_trivial::value, T>::type + * = nullptr> + zmq::message_t encode(const T &item, std::function sizeFun = ZMQ::defaultSizeOf) const + { + size_t s = sizeFun(item); + zmq::message_t msg{s}; + memcpy((void *)msg.data(), &item, s); + return msg; + } + + zmq::message_t encode(const char *item) const; + zmq::message_t encode(const std::string &item) const; + + /// Send message with ZMQ + template ::value, T>::type * = nullptr> + zmq::send_result_t send(zmq::socket_t &socket, const T &item, zmq::send_flags flags = zmq::send_flags::none) const + { + return retry_send(socket, 1, encode(item), flags); + } + + zmq::send_result_t + send(zmq::socket_t &socket, const char *item, zmq::send_flags flags = zmq::send_flags::none) const; + zmq::send_result_t + send(zmq::socket_t &socket, zmq::message_t &msg, zmq::send_flags flags = zmq::send_flags::none) const; + zmq::send_result_t + send(zmq::socket_t &socket, zmq::message_t &&msg, zmq::send_flags flags = zmq::send_flags::none) const; + +private: + Encoding m_enc = Text; + mutable zmq::context_t *m_context = nullptr; +}; + +ZeroMQSvc &zmqSvc(); + +#endif // ZEROMQ_IZEROMQSVC_H diff --git a/roofit/roofitZMQ/res/RooFit_ZMQ/functions.h b/roofit/roofitZMQ/res/RooFit_ZMQ/functions.h new file mode 100644 index 0000000000000..2c4175ca27be1 --- /dev/null +++ b/roofit/roofitZMQ/res/RooFit_ZMQ/functions.h @@ -0,0 +1,31 @@ +/* + * Project: RooFit + * Authors: + * RA, Roel Aaij, NIKHEF + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#ifndef ZEROMQ_FUNCTIONS_H +#define ZEROMQ_FUNCTIONS_H 1 + +#include // std::size_t + +namespace ZMQ { + +template +std::size_t defaultSizeOf(const T &) +{ + return sizeof(T); +} + +std::size_t stringLength(const char &cs); + +} // namespace ZMQ + +#endif // ZEROMQ_FUNCTIONS_H diff --git a/roofit/roofitZMQ/res/RooFit_ZMQ/ppoll.h b/roofit/roofitZMQ/res/RooFit_ZMQ/ppoll.h new file mode 100644 index 0000000000000..4eef9da3c9039 --- /dev/null +++ b/roofit/roofitZMQ/res/RooFit_ZMQ/ppoll.h @@ -0,0 +1,28 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#ifndef ROOT_ROOFIT_ZMQ_ppoll +#define ROOT_ROOFIT_ZMQ_ppoll + +#include +#include + +namespace ZMQ { + +int ppoll(zmq_pollitem_t *items_, size_t nitems_, long timeout_, const sigset_t *sigmask_); +int ppoll(std::vector &items, long timeout_, const sigset_t *sigmask_); +class ppoll_error_t : public zmq::error_t { +}; + +} // namespace ZMQ + +#endif // ROOT_ROOFIT_ZMQ_ppoll diff --git a/roofit/roofitZMQ/src/ZeroMQPoller.cpp b/roofit/roofitZMQ/src/ZeroMQPoller.cpp new file mode 100644 index 0000000000000..28909a294a4cc --- /dev/null +++ b/roofit/roofitZMQ/src/ZeroMQPoller.cpp @@ -0,0 +1,244 @@ +/* + * Project: RooFit + * Authors: + * RA, Roel Aaij, NIKHEF + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#include "RooFit_ZMQ/ZeroMQPoller.h" + +#include "RooFit_ZMQ/ppoll.h" +#include + +/** \class ZeroMQPoller + * \brief Wrapper class for polling ZeroMQ sockets + * + * This class simplifies calls to poll or ppoll ZeroMQ sockets. It stores the + * list of sockets to be polled, which means they don't have to be separately + * carried around by the user. It also parses output and returns an easily + * digestible vector of events. + */ + +/** + * \brief Poll the sockets + * + * \param[in] timeo Timeout in milliseconds. 0 means return immediately. -1 means wait for an event indefinitely. + * \return A vector of pairs of index and flags; index is the index of the registered fd or socket and flags are 0 (no + * events), ZMQ_POLLIN or ZMQ_POLLOUT. + * + * \note This function can throw (from inside zmq::poll), so wrap in try-catch! + */ +std::vector> ZeroMQPoller::poll(int timeo) +{ + std::vector> r; + if (m_items.empty()) { + throw std::runtime_error("No sockets registered"); + } + int n = 0; + while (true) { + try { + n = zmq::poll(m_items, std::chrono::milliseconds{timeo}); + if (n == 0) + return r; + break; + } catch (const zmq::error_t &e) { + std::cerr << "in ZeroMQPoller::poll on PID " << getpid() << ": " << e.what() << std::endl; + if (e.num() != EINTR) { + throw; + } + } + } + // TODO: replace this with ranges::v3::zip + for (size_t i = 0; i < m_items.size(); ++i) { + void *socket = m_items[i].socket; + size_t index = 0; + zmq::event_flags flags = zmq::event_flags::none; + if (socket == nullptr) { + // an fd was registered + std::tie(index, flags) = m_fds[m_items[i].fd]; + } else { + // a socket was registered + const zmq::socket_t *s; + std::tie(index, flags, s) = m_sockets[socket]; + } + if (m_items[i].revents & short(flags)) { + r.emplace_back(index, flags); + } + } + return r; +} + +/** + * \brief Poll the sockets with ppoll + * + * By polling with ppoll instead of poll, one can pass along a signal mask to + * handle POSIX signals properly. See the zmq_ppoll documentation for examples + * of when this is useful: http://api.zeromq.org/ + * + * \param[in] timeo Timeout in milliseconds. 0 means return immediately. -1 means wait for an event indefinitely. + * \param[in] sigmask A non-NULL pointer to a signal mask must be constructed and passed to 'sigmask'. See the man page + * of sigprocmask(2) for more details on this. \return A vector of pairs of index and flags; index is the index of the + * registered fd or socket and flags are 0 (no events), ZMQ_POLLIN or ZMQ_POLLOUT. + * + * \note This function can throw (from inside ZMQ::ppoll), so wrap in try-catch! + */ +std::vector> ZeroMQPoller::ppoll(int timeo, const sigset_t *sigmask) +{ + if (m_items.empty()) { + throw std::runtime_error("No sockets registered"); + } + + std::vector> r; + + auto n = ZMQ::ppoll(m_items, timeo, sigmask); + if (n == 0) + return r; + + for (auto &m_item : m_items) { + size_t index = 0; + zmq::event_flags flags = zmq::event_flags::none; + if (m_item.socket == nullptr) { + // an fd was registered + std::tie(index, flags) = m_fds[m_item.fd]; + } else { + // a socket was registered + const zmq::socket_t *s; + std::tie(index, flags, s) = m_sockets[m_item.socket]; + } + if (m_item.revents & short(flags)) { + r.emplace_back(index, flags); + } + } + return r; +} + +size_t ZeroMQPoller::size() const +{ + return m_items.size(); +} + +/** + * \brief Register socket to poll + * + * Adds the socket to the internal list of sockets to poll. + * + * \param[in] socket Socket to register. + * \param[in] type Type of events to poll for. Can be ZMQ_POLLIN, ZMQ_POLLOUT or a bit-wise combination of the two. + * \return The index of the socket in the poller's internal list. Can be used to match with indices returned from + * (p)poll. + */ +size_t ZeroMQPoller::register_socket(zmq::socket_t &socket, zmq::event_flags type) +{ + zmq::socket_t *s = &socket; + auto it = m_sockets.find(s); + if (it != m_sockets.end()) { + return std::get<0>(it->second); + } + size_t index = m_free.empty() ? m_items.size() : m_free.front(); + if (!m_free.empty()) + m_free.pop_front(); + // NOTE: this uses the conversion-to-void* operator of + // zmq::socket_t, which returns the wrapped object + m_items.push_back({socket, 0, static_cast(type), 0}); + + // We need to lookup by the pointer to the object wrapped by zmq::socket_t + m_sockets.emplace(m_items.back().socket, std::make_tuple(index, type, s)); + return index; +} + +/** + * \brief Register socket to poll + * + * Adds the socket to the internal list of sockets to poll. + * + * \param[in] fd File descriptor of socket to register. + * \param[in] type Type of events to poll for. Can be ZMQ_POLLIN, ZMQ_POLLOUT or a bit-wise combination of the two. + * \return The index of the socket in the poller's internal list. Can be used to match with indices returned from + * (p)poll. + */ +size_t ZeroMQPoller::register_socket(int fd, zmq::event_flags type) +{ + auto it = m_fds.find(fd); + if (it != m_fds.end()) { + return std::get<0>(it->second); + } + size_t index = m_free.empty() ? m_items.size() : m_free.front(); + if (!m_free.empty()) + m_free.pop_front(); + // NOTE: this uses the conversion-to-void* operator of + // zmq::socket_t, which returns the wrapped object + m_items.push_back({nullptr, fd, static_cast(type), 0}); + + // We need to lookup by the pointer to the object wrapped by zmq::socket_t + m_fds.emplace(fd, std::make_tuple(index, type)); + return index; +} + +/** + * \brief Unregister socket from poller + * + * Removes the socket from the internal list of sockets to poll. + * + * \param[in] socket Socket to unregister. + * \return The index of the socket in the poller's internal list before removal. + */ +size_t ZeroMQPoller::unregister_socket(zmq::socket_t &socket) +{ + if (!m_sockets.count(socket.operator void *())) { + throw std::out_of_range("Socket is not registered"); + } + // Remove from m_sockets + // Can't search by the key of m_sockets, as that is the wrapped + // object, but have to use the pointer to the wrapper + // (zmq::socket_t) + auto it = std::find_if(begin(m_sockets), end(m_sockets), [&socket](const decltype(m_sockets)::value_type &entry) { + return &socket == std::get<2>(entry.second); + }); + auto index = std::get<0>(it->second); + m_free.push_back(index); + void *it_first = it->first; + m_sockets.erase(it); + + // Remove from m_items + auto iit = std::find_if(begin(m_items), end(m_items), + [&it_first](const zmq::pollitem_t &item) { return it_first == item.socket; }); + assert(iit != end(m_items)); + m_items.erase(iit); + + return index; +} + +/** + * \brief Unregister socket from poller + * + * Removes the socket from the internal list of sockets to poll. + * + * \param[in] fd File descriptor of socket to unregister. + * \return The index of the socket in the poller's internal list before removal. + */ +size_t ZeroMQPoller::unregister_socket(int fd) +{ + if (!m_fds.count(fd)) { + throw std::out_of_range("fileno is not registered"); + } + // Remove from m_fds + auto it = m_fds.find(fd); + auto index = std::get<0>(it->second); + m_free.push_back(index); + int it_first = it->first; + m_fds.erase(it); + + // Remove from m_items + auto iit = std::find_if(begin(m_items), end(m_items), + [&it_first](const zmq::pollitem_t &item) { return it_first == item.fd; }); + assert(iit != end(m_items)); + m_items.erase(iit); + + return index; +} diff --git a/roofit/roofitZMQ/src/ZeroMQSvc.cpp b/roofit/roofitZMQ/src/ZeroMQSvc.cpp new file mode 100644 index 0000000000000..63c66bb0a6c60 --- /dev/null +++ b/roofit/roofitZMQ/src/ZeroMQSvc.cpp @@ -0,0 +1,173 @@ +/* + * Project: RooFit + * Authors: + * RA, Roel Aaij, NIKHEF + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#include "RooFit_ZMQ/ZeroMQSvc.h" + +#include // std::ref + +/** \class ZeroMQSvc + * \brief Wrapper class for basic ZeroMQ context and socket management + * + * This singleton class wraps a couple of basic ZeroMQ tasks: + * + * 1. Creating, storing and eventually closing a ZeroMQ context. + * 2. Creating new sockets in the context. + * 3. Sending, receiving, encoding and decoding messages over sockets. + * + * For convenience, it offers a number of template overloads that automatically + * encode all kinds of data types in ZeroMQ message objects. + */ + +/** + * \brief Get singleton object of this class + */ +ZeroMQSvc &zmqSvc() +{ + static std::unique_ptr svc; + if (!svc) { + svc = std::make_unique(); + } + return *svc; +} + +ZeroMQSvc::Encoding ZeroMQSvc::encoding() const +{ + return m_enc; +} + +/** + * \brief Set encoding mode + * + * \param[in] e Encoding mode; either Text or Binary. + */ +void ZeroMQSvc::setEncoding(const ZeroMQSvc::Encoding &e) +{ + m_enc = e; +} + +/** + * \brief Get context + * + * Creates a context if it has not yet been created and returns a reference to it. + */ +zmq::context_t &ZeroMQSvc::context() const +{ + if (!m_context) { + try { + m_context = new zmq::context_t; + } catch (zmq::error_t &e) { + std::cerr << "ERROR: Creating ZeroMQ context failed. This only happens when PGM initialization failed or when " + "a nullptr was returned from zmq_ctx_new because the created context was invalid. Contact ZMQ " + "experts when this happens, because it shouldn't.\n"; + throw; + } + } + return *m_context; +} + +/** + * \brief Create and return a new socket + * + * \param[in] type Type of the socket. See http://api.zeromq.org/master:zmq-socket for possible values. + * \return The socket object. + */ +zmq::socket_t ZeroMQSvc::socket(zmq::socket_type type) const +{ + try { + // the actual work this function should do, all the rest is error handling: + return zmq::socket_t{context(), type}; + } catch (zmq::error_t &e) { + // all zmq errors not recoverable from here, only at call site + std::cerr << "ERROR in ZeroMQSvc::socket: " << e.what() << " (errno: " << e.num() << ")\n"; + throw; + } +} + +/** + * \brief Create and return a new socket by pointer + * + * \param[in] type Type of the socket. See http://api.zeromq.org/master:zmq-socket for possible values. + * \return A raw pointer to the socket object. Note: the caller must take ownership! + */ +zmq::socket_t *ZeroMQSvc::socket_ptr(zmq::socket_type type) const +{ + try { + // the actual work this function should do, all the rest is error handling: + return new zmq::socket_t(context(), type); + } catch (zmq::error_t &e) { + // all zmq errors not recoverable from here, only at call site + std::cerr << "ERROR in ZeroMQSvc::socket_ptr: " << e.what() << " (errno: " << e.num() << ")\n"; + throw; + } +} + +void ZeroMQSvc::close_context() const +{ + if (m_context) { + delete m_context; + m_context = nullptr; + } +} + +/** + * \fn zmq::message_t ZeroMQSvc::encode(const char *item) const + * \brief Encode string as a ZeroMQ message object + * + * \param[in] item String. + */ +zmq::message_t ZeroMQSvc::encode(const char *item) const +{ + std::function fun = ZMQ::stringLength; + return encode(*item, fun); +} + +/** + * \overload zmq::message_t ZeroMQSvc::encode(const std::string &item) const + */ +zmq::message_t ZeroMQSvc::encode(const std::string &item) const +{ + return encode(item.c_str()); +} + +/** + * \fn bool ZeroMQSvc::send(zmq::socket_t &socket, const char *item, zmq::send_flags flags) const + * \brief Send message over a socket + * + * \param[in] socket Socket. + * \param[in] item Message to send over. + * \param[in] flags Flags to send. See http://api.zeromq.org/master:zmq-send for possible flags and the cppzmq API for + * the type-safe equivalents in the zmq::send_flags enum class. + * \return An optional of type zmq::send_result_t that contains the number of bytes sent if successful, and is empty if + * EAGAIN was received, which probably means you should try again. + */ +zmq::send_result_t ZeroMQSvc::send(zmq::socket_t &socket, const char *item, zmq::send_flags flags) const +{ + return retry_send(socket, 2, encode(item), flags); +} + +/** + * \overload zmq::send_result_t ZeroMQSvc::send(zmq::socket_t &socket, zmq::message_t &msg, zmq::send_flags flags) const + */ +zmq::send_result_t ZeroMQSvc::send(zmq::socket_t &socket, zmq::message_t &msg, zmq::send_flags flags) const +{ + return retry_send(socket, 2, std::ref(msg), flags); +} + +/** + * \overload zmq::send_result_t ZeroMQSvc::send(zmq::socket_t &socket, zmq::message_t &&msg, zmq::send_flags flags) + * const + */ +zmq::send_result_t ZeroMQSvc::send(zmq::socket_t &socket, zmq::message_t &&msg, zmq::send_flags flags) const +{ + return retry_send(socket, 2, std::move(msg), flags); +} diff --git a/roofit/roofitZMQ/src/functions.cpp b/roofit/roofitZMQ/src/functions.cpp new file mode 100644 index 0000000000000..6f7a0f0d72f23 --- /dev/null +++ b/roofit/roofitZMQ/src/functions.cpp @@ -0,0 +1,23 @@ +/* + * Project: RooFit + * Authors: + * RA, Roel Aaij, NIKHEF + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#include "RooFit_ZMQ/functions.h" + +#include + +namespace ZMQ { +std::size_t stringLength(const char &cs) +{ + return strlen(&cs); +} +} // namespace ZMQ diff --git a/roofit/roofitZMQ/src/ppoll.cpp b/roofit/roofitZMQ/src/ppoll.cpp new file mode 100644 index 0000000000000..c6b4c75d346b3 --- /dev/null +++ b/roofit/roofitZMQ/src/ppoll.cpp @@ -0,0 +1,34 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#include "RooFit_ZMQ/ppoll.h" + +namespace ZMQ { + +/// Wrapper around zmq_ppoll +/// This function can throw, so wrap in try-catch! +int ppoll(zmq_pollitem_t *items_, size_t nitems_, long timeout_, const sigset_t *sigmask_) +{ + int rc = zmq_ppoll(items_, static_cast(nitems_), timeout_, sigmask_); + if (rc < 0) + throw ppoll_error_t(); + return rc; +} + +/// Wrapper around zmq_ppoll +/// This function can throw, so wrap in try-catch! +int ppoll(std::vector &items, long timeout_, const sigset_t *sigmask_) +{ + return ppoll(items.data(), items.size(), timeout_, sigmask_); +} + +} // namespace ZMQ diff --git a/roofit/roofitZMQ/test/CMakeLists.txt b/roofit/roofitZMQ/test/CMakeLists.txt new file mode 100644 index 0000000000000..5ce5cb7d185d9 --- /dev/null +++ b/roofit/roofitZMQ/test/CMakeLists.txt @@ -0,0 +1,14 @@ +ROOT_ADD_GTEST(test_RooFitZMQ test_ZMQ.cpp LIBRARIES RooFitZMQ) +target_include_directories(test_RooFitZMQ PRIVATE ${RooFitZMQ_INCLUDE_DIRS}) + +ROOT_ADD_GTEST(test_RooFitZMQ_polling test_polling.cxx LIBRARIES RooFitZMQ) +target_include_directories(test_RooFitZMQ_polling PRIVATE ${RooFitZMQ_INCLUDE_DIRS}) + +ROOT_ADD_GTEST(test_RooFitZMQ_HWM test_HWM.cxx LIBRARIES RooFitZMQ) +target_include_directories(test_RooFitZMQ_HWM PRIVATE ${RooFitZMQ_INCLUDE_DIRS}) + +ROOT_ADD_GTEST(test_RooFitZMQ_load_balancing test_ZMQ_load_balancing.cxx LIBRARIES RooFitZMQ) +target_include_directories(test_RooFitZMQ_load_balancing PRIVATE ${RooFitZMQ_INCLUDE_DIRS}) + +ROOT_ADD_GTEST(test_RooFitZMQ_mkstemp test_mkstemp.cxx LIBRARIES RooFitZMQ) +target_include_directories(test_RooFitZMQ_mkstemp PRIVATE ${RooFitZMQ_INCLUDE_DIRS}) diff --git a/roofit/roofitZMQ/test/test_HWM.cxx b/roofit/roofitZMQ/test/test_HWM.cxx new file mode 100644 index 0000000000000..bda6ae9bf3d1d --- /dev/null +++ b/roofit/roofitZMQ/test/test_HWM.cxx @@ -0,0 +1,144 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#include "RooFit_ZMQ/ZeroMQSvc.h" + +#include "gtest/gtest.h" + +#include // fork, usleep + +class HighWaterMarkTest : public ::testing::Test { +protected: + void SetUp() override + { + do { + child_pid = fork(); + } while (child_pid == -1); // retry if fork fails + + if (child_pid > 0) { // parent + pusher.reset(zmqSvc().socket_ptr(zmq::socket_type::push)); + if (set_hwm) { + auto rc = zmq_setsockopt(*pusher, ZMQ_SNDHWM, &hwm, sizeof hwm); + assert(rc == 0); + } + pusher->bind("ipc:///tmp/ZMQ_test_fork_polling_P2C.ipc"); + } else { // child + puller.reset(zmqSvc().socket_ptr(zmq::socket_type::pull)); + if (set_hwm) { + auto rc = zmq_setsockopt(*puller, ZMQ_RCVHWM, &hwm, sizeof hwm); + assert(rc == 0); + } + puller->connect("ipc:///tmp/ZMQ_test_fork_polling_P2C.ipc"); + } + } + + void TearDown() override + { + if (child_pid > 0) { // parent + // wait for child + int status = 0; + pid_t pid; + do { + pid = waitpid(child_pid, &status, 0); + } while (-1 == pid && EINTR == errno); // retry on interrupted system call + if (0 != status) { + if (WIFEXITED(status)) { + printf("exited, status=%d\n", WEXITSTATUS(status)); + } else if (WIFSIGNALED(status)) { + printf("killed by signal %d\n", WTERMSIG(status)); + } else if (WIFSTOPPED(status)) { + printf("stopped by signal %d\n", WSTOPSIG(status)); + } else if (WIFCONTINUED(status)) { + printf("continued\n"); + } + } + if (-1 == pid) { + throw std::runtime_error(std::string("waitpid, errno ") + std::to_string(errno)); + } + pusher.reset(nullptr); + zmqSvc().close_context(); + } else { // child + puller.reset(nullptr); + zmqSvc().close_context(); + _Exit(0); + } + } + + void run_test() + { + std::size_t max_sends = 2000; + + if (child_pid > 0) { // parent + // start test + for (std::size_t ix = 0; ix < max_sends; ++ix) { + zmqSvc().send(*pusher, 0.1f); + zmqSvc().send(*pusher, 1); + zmqSvc().send(*pusher, true); + if (ix % 100 == 0) { + printf("parent at ix = %zu\n", ix); + } + } + } else { // child + // wait a few seconds before reading to allow the parent to overflow the HWM + printf("child waiting for 2 seconds...\n"); + sleep(2); + printf("child starts receiving\n"); + + for (std::size_t ix = 0; ix < max_sends; ++ix) { + zmqSvc().receive(*puller); + zmqSvc().receive(*puller); + zmqSvc().receive(*puller); + if (ix % 100 == 0) { + printf("child at ix = %zu\n", ix); + } + } + } + } + + pid_t child_pid{0}; + ZmqLingeringSocketPtr<> pusher, puller; + bool set_hwm = false; + int hwm = 0; +}; + +TEST_F(HighWaterMarkTest, demonstrateHittingDefaultHWM) +{ + run_test(); +} + +class HighWaterMarkZeroTest : public HighWaterMarkTest { + void SetUp() override + { + set_hwm = true; + hwm = 0; + HighWaterMarkTest::SetUp(); + } +}; + +TEST_F(HighWaterMarkZeroTest, zeroHWM) +{ + run_test(); +} + +class HighWaterMark2kTest : public HighWaterMarkTest { + void SetUp() override + { + set_hwm = true; + hwm = 2000; + HighWaterMarkTest::SetUp(); + } +}; + +TEST_F(HighWaterMark2kTest, HWM2k) +{ + run_test(); +} diff --git a/roofit/roofitZMQ/test/test_ZMQ.cpp b/roofit/roofitZMQ/test/test_ZMQ.cpp new file mode 100644 index 0000000000000..da87822c184cd --- /dev/null +++ b/roofit/roofitZMQ/test/test_ZMQ.cpp @@ -0,0 +1,317 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#include "RooFit_ZMQ/ZeroMQSvc.h" + +#include "gtest/gtest.h" + +#include // fork, usleep + +#include + +// N.B.: wait_for_child is identically defined in RooFit::MultiProcess, but we copy it here to reduce module +// interdependencies. It also requires an extra include: +#include // kill, SIGKILL + +int wait_for_child(pid_t child_pid, bool may_throw, int retries_before_killing) +{ + int status = 0; + int patience = retries_before_killing; + pid_t tmp; + do { + if (patience-- < 1) { + ::kill(child_pid, SIGKILL); + } + tmp = waitpid(child_pid, &status, WNOHANG); + } while (tmp == 0 // child has not yet changed state, try again + || (-1 == tmp && EINTR == errno) // retry on interrupted system call + ); + + if (0 != status) { + if (WIFEXITED(status)) { + printf("exited, status=%d\n", WEXITSTATUS(status)); + } else if (WIFSIGNALED(status)) { + if (WTERMSIG(status) != SIGKILL) { // SIGKILL is expected, so needn't be printed + printf("killed by signal %d\n", WTERMSIG(status)); + } + } else if (WIFSTOPPED(status)) { + printf("stopped by signal %d\n", WSTOPSIG(status)); + } else if (WIFCONTINUED(status)) { + printf("continued\n"); + } + } + + if (-1 == tmp && may_throw) { + throw std::runtime_error(std::string("waitpid, errno ") + std::to_string(errno)); + } + + return status; +} + +void elaborate_bind(const ZmqLingeringSocketPtr<> &socket, std::string name) +{ + try { + socket->bind(name); + } catch (const zmq::error_t &e) { + if (e.num() == EADDRINUSE) { + std::cerr << "address already in use, retrying bind in 500ms\n"; + usleep(500000); + try { + socket->bind(name); + } catch (const zmq::error_t &e2) { + if (e2.num() == EADDRINUSE) { + std::cerr + << "again: address already in use, aborting; please check whether there are any remaining improperly " + "exited processes (zombies) around or whether some other program is using port 6660\n"; + } + throw; + } + // Sometimes, the socket from the previous test needs some time to close, so + // we introduce a latency here. A more robust and fast approach might be to + // do the following on the bind side: + // 1. first try another port, e.g. increase by one + // 2. if that doesn't work, do the latency and retry the original port + // The connect side then also needs to change, because it doesn't know which + // port the bind side will bind to. The connect side could try connecting to + // both options asynchronously, and then in a loop check both for signs of + // life. If one comes alive, transfer ownership of that pointer to the pointer + // you want to eventually use (`socket`) and that's it. + } else { + throw; + } + } +} + +class AllSocketTypes + : public ::testing::TestWithParam, + std::pair /* socket_names */>> { +}; + +TEST_P(AllSocketTypes, forkHandshake) +{ + auto socket_names = std::get<2>(GetParam()); + pid_t child_pid{0}; + do { + child_pid = fork(); + } while (child_pid == -1); // retry if fork fails + + if (child_pid > 0) { // master + ZmqLingeringSocketPtr<> socket; + socket.reset(zmqSvc().socket_ptr(std::get<1>(GetParam()).first)); + elaborate_bind(socket, socket_names.second); + // bind is on the master process to avoid zombie children to hold on to binds + + // start test + zmqSvc().send(*socket, std::string("breaker breaker")); + + auto receipt = zmqSvc().receive(*socket); + + EXPECT_EQ(receipt, 1212); + + socket.reset(nullptr); + zmqSvc().close_context(); // if you don't close context in parent process as well, the next repeat will hang + + wait_for_child(child_pid, true, 5); + } else { // child + ZmqLingeringSocketPtr<> socket; + socket.reset(zmqSvc().socket_ptr(std::get<1>(GetParam()).second)); + socket->connect(socket_names.first); + + // start test + auto receipt = zmqSvc().receive(*socket); + if (receipt == "breaker breaker") { + zmqSvc().send(*socket, 1212); + } + // take care, don't just use _exit, it will not cleanly destroy context etc! + // if you really need to, at least close and destroy everything properly + socket.reset(nullptr); + zmqSvc().close_context(); + _Exit(0); + } +} + +std::string ipc{"ipc:///tmp/ZMQ_test_fork.ipc"}; +std::string tcp_server{"tcp://127.0.0.1:6660"}; +std::string tcp_client{"tcp://*:6660"}; +auto socket_name_options = ::testing::Values(std::make_pair(tcp_server, tcp_client), std::make_pair(ipc, ipc)); + +INSTANTIATE_TEST_SUITE_P(REQREP, AllSocketTypes, + ::testing::Combine(::testing::Range(0, 10), // repeat to probe connection stability + ::testing::Values(std::make_pair(zmq::socket_type::req, + zmq::socket_type::rep)), + socket_name_options)); +INSTANTIATE_TEST_SUITE_P(PAIRPAIR, AllSocketTypes, + ::testing::Combine(::testing::Range(0, 10), // repeat to probe connection stability + ::testing::Values(std::make_pair(zmq::socket_type::pair, + zmq::socket_type::pair)), + socket_name_options)); + +class AsyncSocketTypes + : public ::testing::TestWithParam< + std::tuple, + std::pair /* socket_names */, bool /* expect_throw */>> { +}; + +TEST_P(AsyncSocketTypes, forkMultiSendReceive) +{ + bool expect_throw = std::get<3>(GetParam()); + ZmqLingeringSocketPtr<> socket; + auto socket_names = std::get<2>(GetParam()); + pid_t child_pid{0}; + do { + child_pid = fork(); + } while (child_pid == -1); // retry if fork fails + + if (child_pid > 0) { // master + socket.reset(zmqSvc().socket_ptr(std::get<1>(GetParam()).first)); + elaborate_bind(socket, socket_names.second); + // bind is on the master process to avoid zombie children to hold on to binds + + // start test: send 2 things, receive 1, send 1 more, finish + zmqSvc().send(*socket, std::string("breaker breaker")); + + if (expect_throw) { + EXPECT_ANY_THROW(zmqSvc().send(*socket, std::string("anybody out there?"))); + // NOTE: also in case of a throw, be sure to properly close down the connection! + // Otherwise, you may get zombies waiting for a reply. + socket.reset(nullptr); + zmqSvc().close_context(); // if you don't close context in parent process as well, the next repeat will hang + wait_for_child(child_pid, true, 5); + return; + } else { + EXPECT_NO_THROW(zmqSvc().send(*socket, std::string("anybody out there?"))); + } + + auto receipt = zmqSvc().receive(*socket); + + EXPECT_EQ(receipt, 1212); + + zmqSvc().send(*socket, std::string("kthxbye")); + + socket.reset(nullptr); + zmqSvc().close_context(); // if you don't close context in parent process as well, the next repeat will hang + + wait_for_child(child_pid, true, 5); + } else { // child + socket.reset(zmqSvc().socket_ptr(std::get<1>(GetParam()).second)); + socket->connect(socket_names.first); + + // start test, receive something + auto receipt1 = zmqSvc().receive(*socket); + std::string receipt2; + if (expect_throw) { + EXPECT_ANY_THROW(zmqSvc().receive(*socket)); + // NOTE: also in case of a throw, be sure to properly close down the connection! + // Otherwise, you may get zombies waiting for a reply. + socket.reset(nullptr); + zmqSvc().close_context(); + _Exit(0); + } else { + receipt2 = zmqSvc().receive(*socket); + } + + if (receipt1 == "breaker breaker" && receipt2 == "anybody out there?") { + zmqSvc().send(*socket, 1212); + } + auto receipt3 = zmqSvc().receive(*socket); + if (receipt3 != "kthxbye") { + std::cerr << "did not receive final reply correctly\n"; + } + + // take care, don't just use _exit, it will not cleanly destroy context etc! + // if you really need to, at least close and destroy everything properly + socket.reset(nullptr); + zmqSvc().close_context(); + _Exit(0); + } +} + +TEST_P(AsyncSocketTypes, forkIgnoreSomeMessages) +{ + bool expect_throw = std::get<3>(GetParam()); + ZmqLingeringSocketPtr<> socket; + auto socket_names = std::get<2>(GetParam()); + pid_t child_pid{0}; + do { + child_pid = fork(); + } while (child_pid == -1); // retry if fork fails + + if (child_pid > 0) { // master + socket.reset(zmqSvc().socket_ptr(std::get<1>(GetParam()).first)); + elaborate_bind(socket, socket_names.second); + // bind is on the master process to avoid zombie children to hold on to binds + + // start test: send 2 things, receive 1, send 1 more, finish + zmqSvc().send(*socket, std::string("breaker breaker")); + + if (expect_throw) { + EXPECT_ANY_THROW(zmqSvc().send(*socket, std::string("anybody out there?"))); + // NOTE: also in case of a throw, be sure to properly close down the connection! + // Otherwise, you may get zombies waiting for a reply. + socket.reset(nullptr); + zmqSvc().close_context(); // if you don't close context in parent process as well, the next repeat will hang + wait_for_child(child_pid, true, 5); + return; + } else { + EXPECT_NO_THROW(zmqSvc().send(*socket, std::string("anybody out there?"))); + } + + auto receipt = zmqSvc().receive(*socket); + + EXPECT_EQ(receipt, 1212); + + zmqSvc().send(*socket, std::string("kthxbye")); + + socket.reset(nullptr); + zmqSvc().close_context(); // if you don't close context in parent process as well, the next repeat will hang + + wait_for_child(child_pid, true, 5); + } else { // child + socket.reset(zmqSvc().socket_ptr(std::get<1>(GetParam()).second)); + socket->connect(socket_names.first); + + // start test, receive first thing + auto receipt = zmqSvc().receive(*socket); + if (receipt == "breaker breaker") { + zmqSvc().send(*socket, 1212); + } + + // ignore the rest of the sent messages, but give the other end a second to + // actually send its stuff, instead of hanging in retry_send because the + // connection has died; a better solution would be if retry_send (in + // ZeroMQSvc::send) had a callback mechanism that could be used to break + // out when a child has died, but ok + sleep(1); + + // take care, don't just use _exit, it will not cleanly destroy context etc! + // if you really need to, at least close and destroy everything properly + socket.reset(nullptr); + zmqSvc().close_context(); + _Exit(0); + } +} + +INSTANTIATE_TEST_SUITE_P(PAIRPAIR, AsyncSocketTypes, + ::testing::Combine(::testing::Range(0, 10), // repeat to probe connection stability + ::testing::Values(std::make_pair(zmq::socket_type::pair, + zmq::socket_type::pair)), + socket_name_options, + ::testing::Values(false) // don't expect throw + )); + +INSTANTIATE_TEST_SUITE_P(REQREP, AsyncSocketTypes, + ::testing::Combine(::testing::Values(0), // no repeats, we only care about the throw + ::testing::Values(std::make_pair(zmq::socket_type::req, + zmq::socket_type::rep)), + socket_name_options, + ::testing::Values(true) // expect throw + )); diff --git a/roofit/roofitZMQ/test/test_ZMQ_load_balancing.cxx b/roofit/roofitZMQ/test/test_ZMQ_load_balancing.cxx new file mode 100644 index 0000000000000..934885909e420 --- /dev/null +++ b/roofit/roofitZMQ/test/test_ZMQ_load_balancing.cxx @@ -0,0 +1,148 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#include "RooFit_ZMQ/ZeroMQSvc.h" +#include "RooFit_ZMQ/ZeroMQPoller.h" + +#include "gtest/gtest.h" + +#include // fork, usleep + +class ZMQPushPullTest : public ::testing::Test { +protected: + std::size_t N_children = 4; + std::size_t max_sends = 20; + + void SetUp() override + { + for (std::size_t i = 0; i < N_children; ++i) { + do { + child_pid = fork(); + } while (child_pid == -1); // retry if fork fails + if (child_pid == 0) { // child + child_id = i; + break; + } else { + child_pids.push_back(child_pid); + } + } + + if (child_pid > 0) { // parent + pusher.reset(zmqSvc().socket_ptr(zmq::socket_type::push)); + pusher->bind("ipc:///tmp/ZMQ_test_push_pull_P2C.ipc"); + } else { // child + puller.reset(zmqSvc().socket_ptr(zmq::socket_type::pull)); + puller->connect("ipc:///tmp/ZMQ_test_push_pull_P2C.ipc"); + + poller.register_socket(*puller, zmq::event_flags::pollin); + } + } + + void TearDown() override + { + if (child_pid > 0) { // parent + // wait for children + int status = 0; + pid_t pid; + for (pid_t child_pid_i : child_pids) { + do { + pid = waitpid(child_pid_i, &status, 0); + } while (-1 == pid && EINTR == errno); // retry on interrupted system call + if (0 != status) { + if (WIFEXITED(status)) { + printf("exited, status=%d\n", WEXITSTATUS(status)); + } else if (WIFSIGNALED(status)) { + printf("killed by signal %d\n", WTERMSIG(status)); + } else if (WIFSTOPPED(status)) { + printf("stopped by signal %d\n", WSTOPSIG(status)); + } else if (WIFCONTINUED(status)) { + printf("continued\n"); + } + } + if (-1 == pid) { + throw std::runtime_error(std::string("waitpid, errno ") + std::to_string(errno)); + } + } + pusher.reset(nullptr); + zmqSvc().close_context(); + } else { // child + puller.reset(nullptr); + zmqSvc().close_context(); + _Exit(0); + } + } + + void run_parent() + { + // start test + usleep(1000); // wait a second so that all pull sockets are connected for round-robin distribution + // if you don't wait a second above, the push socket will "round-robin" all the messages to just one or two + // connected sockets + for (std::size_t ix = 0; ix < max_sends; ++ix) { + zmqSvc().send(*pusher, 0); + } + for (std::size_t ix = 0; ix < N_children; ++ix) { + // end by sending some 1's to all children, to let them know the sending is over + zmqSvc().send(*pusher, 1); + } + } + + void run_child() + { + std::size_t count = 0; + for (std::size_t ix = 0; ix < max_sends; ++ix) { + auto r = poller.poll(2000); + if (r.empty()) { + printf("poller of child %d timed out after 2 seconds\n", child_id); + break; + } + auto value = zmqSvc().receive(*puller, zmq::recv_flags::dontwait); + usleep(200); // "do some work" + printf("value on child %d: %d\n", child_id, value); + if (value == 1) { + printf("child %d got value %d, done here\n", child_id, value); + break; + } + ++count; + } + printf("child %d got %zu values\n", child_id, count); + } + + pid_t child_pid{0}; + int child_id = -1; + std::vector child_pids; + ZmqLingeringSocketPtr<> pusher, puller; + ZeroMQPoller poller; +}; + +/// This test shows how push-pull is unsuited for load balancing; messages are just sent to the first available pull +/// socket without any dynamic load balancing +TEST_F(ZMQPushPullTest, demoRoundRobin) +{ + if (child_pid > 0) { + run_parent(); + } else { + run_child(); + } +} + +/// This test tries to see whether push-pull can be made to work as a bit of a load balancer, using a low HWM at the +/// receiver +TEST_F(ZMQPushPullTest, demoHWM1LoadBalancing) +{ + if (child_pid > 0) { + run_parent(); + } else { + puller->set(zmq::sockopt::rcvhwm, 1); + run_child(); + } +} diff --git a/roofit/roofitZMQ/test/test_mkstemp.cxx b/roofit/roofitZMQ/test/test_mkstemp.cxx new file mode 100644 index 0000000000000..a914192c1b61f --- /dev/null +++ b/roofit/roofitZMQ/test/test_mkstemp.cxx @@ -0,0 +1,33 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#include "RooFit_ZMQ/ZeroMQSvc.h" + +#include "gtest/gtest.h" + +#include // mkstemp +#include // for "legacy" systems + +TEST(BindToTmpFile, mkstemp) +{ + char filename[] = "/tmp/roofitMP_XXXXXX"; + while (mkstemp(filename) < 0) { + } + auto socket = zmqSvc().socket(zmq::socket_type::push); + char address[50]; + sprintf(address, "ipc://%s", filename); + try { + socket.bind(address); + } catch (const zmq::error_t &) { + printf("caught an exception\n"); + } +} \ No newline at end of file diff --git a/roofit/roofitZMQ/test/test_polling.cxx b/roofit/roofitZMQ/test/test_polling.cxx new file mode 100644 index 0000000000000..43dd5c0759b6c --- /dev/null +++ b/roofit/roofitZMQ/test/test_polling.cxx @@ -0,0 +1,161 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#include "RooFit_ZMQ/ZeroMQSvc.h" +#include "RooFit_ZMQ/ZeroMQPoller.h" + +#include "gtest/gtest.h" + +#include // fork, usleep +#include // signal blocking +#include // strsignal() + +#include + +static volatile sig_atomic_t terminated = 0; + +void handle_sigterm(int signum) +{ + terminated = 1; + std::cout << "handled signal " << strsignal(signum) << " on PID " << getpid() << std::endl; +} + +std::string unique_tmp_ipc_address(const char *filename_template) +{ + assert(strlen(filename_template) < 256); + char filename_template_mutable[256]; + strcpy(filename_template_mutable, filename_template); + while (mkstemp(filename_template_mutable) >= 0) { + } + std::stringstream ss; + ss << "ipc:///tmp/" << filename_template_mutable << ".ipc"; + return ss.str(); +} + +TEST(Polling, doublePoll) +{ + auto M2C_address = unique_tmp_ipc_address("ZMQ_test_fork_polling_M2C_XXXXXX"); + auto C2M_address = unique_tmp_ipc_address("ZMQ_test_fork_polling_C2M_XXXXXX"); + pid_t child_pid{0}; + do { + child_pid = fork(); + } while (child_pid == -1); // retry if fork fails + + if (child_pid > 0) { // master + sigset_t sigmask, sigmask_old; + sigemptyset(&sigmask); + sigaddset(&sigmask, SIGCHLD); + sigprocmask(SIG_BLOCK, &sigmask, &sigmask_old); + + ZmqLingeringSocketPtr<> pusher, puller; + pusher.reset(zmqSvc().socket_ptr(zmq::socket_type::push)); + pusher->bind(M2C_address); + puller.reset(zmqSvc().socket_ptr(zmq::socket_type::pull)); + puller->bind(C2M_address); + + ZeroMQPoller poller1, poller2; + poller1.register_socket(*puller, zmq::event_flags::pollin); + poller2.register_socket(*puller, zmq::event_flags::pollin); + + // start test + zmqSvc().send(*pusher, std::string("breaker breaker")); + + auto result1a = poller1.poll(-1); + auto result1b = poller1.poll(-1); + auto result2 = poller2.poll(-1); + EXPECT_EQ(result1a.size(), result1b.size()); + EXPECT_EQ(result1a.size(), result2.size()); + + auto receipt = zmqSvc().receive(*puller, zmq::recv_flags::dontwait); + + EXPECT_EQ(receipt, 1212); + + kill(child_pid, SIGTERM); + + pusher.reset(nullptr); + puller.reset(nullptr); + zmqSvc().close_context(); // if you don't close context in parent process as well, the next repeat will hang + + // wait for child + int status = 0; + pid_t pid; + do { + pid = waitpid(child_pid, &status, 0); + } while (-1 == pid && EINTR == errno); // retry on interrupted system call + + if (0 != status) { + if (WIFEXITED(status)) { + printf("exited, status=%d\n", WEXITSTATUS(status)); + } else if (WIFSIGNALED(status)) { + printf("killed by signal %d\n", WTERMSIG(status)); + } else if (WIFSTOPPED(status)) { + printf("stopped by signal %d\n", WSTOPSIG(status)); + } else if (WIFCONTINUED(status)) { + printf("continued\n"); + } + } + + if (-1 == pid) { + throw std::runtime_error(std::string("waitpid, errno ") + std::to_string(errno)); + } + + sigprocmask(SIG_SETMASK, &sigmask_old, nullptr); + } else { // child + sigset_t sigmask, sigmask_old; + sigemptyset(&sigmask); + sigaddset(&sigmask, SIGTERM); + sigprocmask(SIG_BLOCK, &sigmask, &sigmask_old); + + struct sigaction sa; + memset(&sa, '\0', sizeof(sa)); + sa.sa_handler = handle_sigterm; + + if (sigaction(SIGTERM, &sa, NULL) < 0) { + std::perror("sigaction failed"); + std::exit(1); + } + + ZmqLingeringSocketPtr<> puller, pusher; + puller.reset(zmqSvc().socket_ptr(zmq::socket_type::pull)); + puller->connect(M2C_address); + pusher.reset(zmqSvc().socket_ptr(zmq::socket_type::push)); + pusher->connect(C2M_address); + + ZeroMQPoller poller1, poller2; + poller1.register_socket(*puller, zmq::event_flags::pollin); + poller2.register_socket(*puller, zmq::event_flags::pollin); + + // start test + auto result1a = poller1.poll(-1); + auto result1b = poller1.poll(-1); + auto result2 = poller2.poll(-1); + EXPECT_EQ(result1a.size(), result1b.size()); + EXPECT_EQ(result1a.size(), result2.size()); + + auto receipt = zmqSvc().receive(*puller, zmq::recv_flags::dontwait); + if (receipt == "breaker breaker") { + zmqSvc().send(*pusher, 1212); + } + // take care, don't just use _exit, it will not cleanly destroy context etc! + // if you really need to, at least close and destroy everything properly + + sigprocmask(SIG_SETMASK, &sigmask_old, nullptr); + + while (!terminated) { + } + + puller.reset(nullptr); + pusher.reset(nullptr); + zmqSvc().close_context(); + _Exit(0); + } +} From d526af98f6bee9e80014546f226187f567cfc6ba Mon Sep 17 00:00:00 2001 From: "E. G. Patrick Bos" Date: Fri, 11 Jun 2021 12:16:52 +0200 Subject: [PATCH 3/3] add RooFit::MultiProcess RooFit::MultiProcess is a task-based parallelization framework that will be used to parallelize gradients in RooFit/Minuit2 fits. It uses forked processes for parallelization, as opposed to threads. We chose this approach because A) the existing RooRealMPFE parallelization framework already made use of forks as well, so we had something to build on and B) it was at the time deemed infeasible to check the entire RooFit code for thread-safety. Moreover, we use MultiProcess to parallelize gradients -- i.e. the tasks to be executed in parallel are partial derivatives -- and these are sufficiently large tasks that communication in between tasks is not a big concern in the big fits that we aimed to parallelize. The communication between processes is done using ZeroMQ, accessed through convenience functions in RooFitZMQ. Co-authored-by: Jonas Rembser --- cmake/modules/RootBuildOptions.cmake | 2 + cmake/modules/SearchInstalledSoftware.cmake | 4 +- roofit/CMakeLists.txt | 5 +- roofit/multiprocess/CMakeLists.txt | 22 + .../res/RooFit/MultiProcess/Job.h | 56 ++ .../res/RooFit/MultiProcess/JobManager.h | 69 +++ .../res/RooFit/MultiProcess/Messenger.h | 214 +++++++ .../res/RooFit/MultiProcess/Messenger_decl.h | 166 ++++++ .../res/RooFit/MultiProcess/ProcessManager.h | 75 +++ .../res/RooFit/MultiProcess/Queue.h | 43 ++ .../res/RooFit/MultiProcess/types.h | 34 ++ .../res/RooFit/MultiProcess/util.h | 34 ++ .../res/RooFit/MultiProcess/worker.h | 25 + roofit/multiprocess/src/Job.cxx | 151 +++++ roofit/multiprocess/src/JobManager.cxx | 218 +++++++ roofit/multiprocess/src/Messenger.cxx | 544 ++++++++++++++++++ roofit/multiprocess/src/ProcessManager.cxx | 306 ++++++++++ roofit/multiprocess/src/Queue.cxx | 175 ++++++ roofit/multiprocess/src/util.cxx | 130 +++++ roofit/multiprocess/src/worker.cxx | 150 +++++ roofit/multiprocess/test/CMakeLists.txt | 16 + roofit/multiprocess/test/test_Job.cxx | 247 ++++++++ roofit/multiprocess/test/test_Messenger.cxx | 238 ++++++++ .../multiprocess/test/test_ProcessManager.cxx | 54 ++ roofit/multiprocess/test/test_worker.cxx | 27 + roofit/multiprocess/test/utils.h | 169 ++++++ 26 files changed, 3170 insertions(+), 4 deletions(-) create mode 100644 roofit/multiprocess/CMakeLists.txt create mode 100644 roofit/multiprocess/res/RooFit/MultiProcess/Job.h create mode 100644 roofit/multiprocess/res/RooFit/MultiProcess/JobManager.h create mode 100644 roofit/multiprocess/res/RooFit/MultiProcess/Messenger.h create mode 100644 roofit/multiprocess/res/RooFit/MultiProcess/Messenger_decl.h create mode 100644 roofit/multiprocess/res/RooFit/MultiProcess/ProcessManager.h create mode 100644 roofit/multiprocess/res/RooFit/MultiProcess/Queue.h create mode 100644 roofit/multiprocess/res/RooFit/MultiProcess/types.h create mode 100644 roofit/multiprocess/res/RooFit/MultiProcess/util.h create mode 100644 roofit/multiprocess/res/RooFit/MultiProcess/worker.h create mode 100644 roofit/multiprocess/src/Job.cxx create mode 100644 roofit/multiprocess/src/JobManager.cxx create mode 100644 roofit/multiprocess/src/Messenger.cxx create mode 100644 roofit/multiprocess/src/ProcessManager.cxx create mode 100644 roofit/multiprocess/src/Queue.cxx create mode 100644 roofit/multiprocess/src/util.cxx create mode 100644 roofit/multiprocess/src/worker.cxx create mode 100644 roofit/multiprocess/test/CMakeLists.txt create mode 100644 roofit/multiprocess/test/test_Job.cxx create mode 100644 roofit/multiprocess/test/test_Messenger.cxx create mode 100644 roofit/multiprocess/test/test_ProcessManager.cxx create mode 100644 roofit/multiprocess/test/test_worker.cxx create mode 100644 roofit/multiprocess/test/utils.h diff --git a/cmake/modules/RootBuildOptions.cmake b/cmake/modules/RootBuildOptions.cmake index 62abd3c86618d..0baea571b3aff 100644 --- a/cmake/modules/RootBuildOptions.cmake +++ b/cmake/modules/RootBuildOptions.cmake @@ -162,6 +162,7 @@ ROOT_BUILD_OPTION(qt5web OFF "Enable support for Qt5 web-based display (requires ROOT_BUILD_OPTION(qt6web OFF "Enable support for Qt6 web-based display (requires Qt6::WebEngineCore and Qt6::WebEngineWidgets)") ROOT_BUILD_OPTION(r OFF "Enable support for R bindings (requires R, Rcpp, and RInside)") ROOT_BUILD_OPTION(roofit ON "Build the advanced fitting package RooFit, and RooStats for statistical tests. If xml is available, also build HistFactory.") +ROOT_BUILD_OPTION(roofit_multiprocess ON "Build RooFit::MultiProcess and multi-process RooFit::TestStatistics classes (requires ZeroMQ with zmq_ppoll and cppzmq).") ROOT_BUILD_OPTION(webgui ON "Build Web-based UI components of ROOT (requires C++17 standard or higher)") ROOT_BUILD_OPTION(root7 ON "Build ROOT 7 components of ROOT (requires C++17 standard or higher)") ROOT_BUILD_OPTION(rpath ON "Link libraries with built-in RPATH (run-time search path)") @@ -255,6 +256,7 @@ if(all) set(qt5web_defvalue ON) set(r_defvalue ON) set(roofit_defvalue ON) + set(roofit_multiprocess_defvalue ON) set(webgui_defvalue ON) set(root7_defvalue ON) set(shadowpw_defvalue ON) diff --git a/cmake/modules/SearchInstalledSoftware.cmake b/cmake/modules/SearchInstalledSoftware.cmake index 744e8488f1f2b..95b7a397a6066 100644 --- a/cmake/modules/SearchInstalledSoftware.cmake +++ b/cmake/modules/SearchInstalledSoftware.cmake @@ -1864,7 +1864,7 @@ endif() #---Check for ZeroMQ when building RooFit::MultiProcess-------------------------------------------- -if (roofit_multiprocess) +if (roofit_multiprocess AND NOT MSVC) if(NOT builtin_zeromq) message(STATUS "Looking for ZeroMQ (libzmq)") # Clear cache before calling find_package(ZeroMQ), @@ -1929,7 +1929,7 @@ if (roofit_multiprocess) target_compile_definitions(libzmq INTERFACE ZMQ_NO_EXPORT) target_compile_definitions(cppzmq INTERFACE ZMQ_BUILD_DRAFT_API) target_compile_definitions(cppzmq INTERFACE ZMQ_NO_EXPORT) -endif (roofit_multiprocess) +endif (roofit_multiprocess AND NOT MSVC) #---Download googletest-------------------------------------------------------------- if (testing) diff --git a/roofit/CMakeLists.txt b/roofit/CMakeLists.txt index cec391f22a510..7e9230d9a5d1c 100644 --- a/roofit/CMakeLists.txt +++ b/roofit/CMakeLists.txt @@ -5,9 +5,10 @@ # For the list of contributors see $ROOTSYS/README/CREDITS. add_subdirectory(batchcompute) -if (builtin_zeromq) +if (roofit_multiprocess AND NOT MSVC) add_subdirectory(roofitZMQ) -endif(builtin_zeromq) + add_subdirectory(multiprocess) +endif(roofit_multiprocess AND NOT MSVC) add_subdirectory(roofitcore) add_subdirectory(roofit) if(mathmore) diff --git a/roofit/multiprocess/CMakeLists.txt b/roofit/multiprocess/CMakeLists.txt new file mode 100644 index 0000000000000..c6ff00f3d80e6 --- /dev/null +++ b/roofit/multiprocess/CMakeLists.txt @@ -0,0 +1,22 @@ +############################################################################ +# CMakeLists.txt file for building ROOT RooFitMultiProcess package +# @author Patrick Bos, Netherlands eScience Center +############################################################################ + +ROOT_LINKER_LIBRARY(RooFitMultiProcess + src/worker.cxx + src/Messenger.cxx + src/ProcessManager.cxx + src/util.cxx + src/Queue.cxx + src/JobManager.cxx + src/Job.cxx + DEPENDENCIES + RooFitZMQ +) + +target_link_libraries(RooFitMultiProcess PUBLIC RooFitZMQ) +set(RooFitMultiProcess_INCLUDE_DIRS ${CMAKE_CURRENT_SOURCE_DIR}/res ${RooFitZMQ_INCLUDE_DIRS}) +target_include_directories(RooFitMultiProcess PRIVATE ${RooFitMultiProcess_INCLUDE_DIRS}) + +ROOT_ADD_TEST_SUBDIRECTORY(test) diff --git a/roofit/multiprocess/res/RooFit/MultiProcess/Job.h b/roofit/multiprocess/res/RooFit/MultiProcess/Job.h new file mode 100644 index 0000000000000..33d4492ba93d1 --- /dev/null +++ b/roofit/multiprocess/res/RooFit/MultiProcess/Job.h @@ -0,0 +1,56 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ +#ifndef ROOT_ROOFIT_MultiProcess_Job_decl +#define ROOT_ROOFIT_MultiProcess_Job_decl + +#include +#include + +namespace RooFit { +namespace MultiProcess { + +// forward declaration +class JobManager; + +class Job { +public: + explicit Job(); + Job(const Job &other); + + ~Job(); + + virtual void evaluate_task(std::size_t task) = 0; + virtual void update_state(); + + virtual void send_back_task_result_from_worker(std::size_t task) = 0; + virtual bool receive_task_result_on_master(const zmq::message_t &message) = 0; + + void gather_worker_results(); + + std::size_t get_state_id(); + +protected: + JobManager *get_manager(); + + std::size_t id_; + std::size_t state_id_ = 0; + +private: + // do not use _manager directly, it must first be initialized! use get_manager() + JobManager *_manager = nullptr; +}; + +} // namespace MultiProcess +} // namespace RooFit + +#endif // ROOT_ROOFIT_MultiProcess_Job_decl diff --git a/roofit/multiprocess/res/RooFit/MultiProcess/JobManager.h b/roofit/multiprocess/res/RooFit/MultiProcess/JobManager.h new file mode 100644 index 0000000000000..3853b0f62f380 --- /dev/null +++ b/roofit/multiprocess/res/RooFit/MultiProcess/JobManager.h @@ -0,0 +1,69 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ +#ifndef ROOT_ROOFIT_MultiProcess_JobManager +#define ROOT_ROOFIT_MultiProcess_JobManager + +#include "RooFit/MultiProcess/types.h" + +#include // unique_ptr +#include + +namespace RooFit { +namespace MultiProcess { + +// forward definitions +class ProcessManager; +class Messenger; +class Queue; +class Job; + +class JobManager { +public: + static JobManager *instance(); + static bool is_instantiated(); + + ~JobManager(); + + static std::size_t add_job_object(Job *job_object); + static Job *get_job_object(std::size_t job_object_id); + static bool remove_job_object(std::size_t job_object_id); + + ProcessManager &process_manager() const; + Messenger &messenger() const; + Queue &queue() const; + + void retrieve(std::size_t requesting_job_id); + + void activate(); + bool is_activated() const; + +private: + explicit JobManager(std::size_t N_workers); + + std::unique_ptr process_manager_ptr_; + std::unique_ptr messenger_ptr_; + std::unique_ptr queue_ptr_; + bool activated_ = false; + + static std::map job_objects_; + static std::size_t job_counter_; + static std::unique_ptr instance_; + +public: + static unsigned int default_N_workers; // no need for getters/setters, just public +}; + +} // namespace MultiProcess +} // namespace RooFit + +#endif // ROOT_ROOFIT_MultiProcess_JobManager diff --git a/roofit/multiprocess/res/RooFit/MultiProcess/Messenger.h b/roofit/multiprocess/res/RooFit/MultiProcess/Messenger.h new file mode 100644 index 0000000000000..5174e1ad1c2e8 --- /dev/null +++ b/roofit/multiprocess/res/RooFit/MultiProcess/Messenger.h @@ -0,0 +1,214 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ +#ifndef ROOT_ROOFIT_MultiProcess_Messenger +#define ROOT_ROOFIT_MultiProcess_Messenger + +#include "RooFit/MultiProcess/Messenger_decl.h" + +#ifdef NDEBUG +#undef NDEBUG +#define turn_NDEBUG_back_on +#endif + +namespace RooFit { +namespace MultiProcess { + +// -- WORKER - QUEUE COMMUNICATION -- + +template +void Messenger::send_from_worker_to_queue(T item, Ts... items) +{ +#ifndef NDEBUG + std::stringstream ss; + ss << "PID " << getpid() << " sends W2Q " << item; + debug_print(ss.str()); +#endif + + zmqSvc().send(*this_worker_qw_push_, item, send_flag_); + // if (sizeof...(items) > 0) { // this will only work with if constexpr, c++17 + send_from_worker_to_queue(items...); +} + +template +value_t Messenger::receive_from_worker_on_queue(std::size_t this_worker_id) +{ + qw_pull_poller_[this_worker_id].ppoll(-1, &ppoll_sigmask); + auto value = zmqSvc().receive(*qw_pull_[this_worker_id], zmq::recv_flags::dontwait); + +#ifndef NDEBUG + std::stringstream ss; + ss << "PID " << getpid() << " receives W(" << this_worker_id << ")2Q " << value; + debug_print(ss.str()); +#endif + + return value; +} + +template +void Messenger::send_from_queue_to_worker(std::size_t this_worker_id, T item, Ts... items) +{ +#ifndef NDEBUG + std::stringstream ss; + ss << "PID " << getpid() << " sends Q2W(" << this_worker_id << ") " << item; + debug_print(ss.str()); +#endif + + zmqSvc().send(*qw_push_[this_worker_id], item, send_flag_); + // if (sizeof...(items) > 0) { // this will only work with if constexpr, c++17 + send_from_queue_to_worker(this_worker_id, items...); +} + +template +value_t Messenger::receive_from_queue_on_worker() +{ + qw_pull_poller_[0].ppoll(-1, &ppoll_sigmask); + auto value = zmqSvc().receive(*this_worker_qw_pull_, zmq::recv_flags::dontwait); + +#ifndef NDEBUG + std::stringstream ss; + ss << "PID " << getpid() << " receives Q2W " << value; + debug_print(ss.str()); +#endif + + return value; +} + +// -- QUEUE - MASTER COMMUNICATION -- + +template +void Messenger::send_from_queue_to_master(T item, Ts... items) +{ +#ifndef NDEBUG + std::stringstream ss; + ss << "PID " << getpid() << " sends Q2M " << item; + debug_print(ss.str()); +#endif + + zmqSvc().send(*mq_push_, item, send_flag_); + // if (sizeof...(items) > 0) { // this will only work with if constexpr, c++17 + send_from_queue_to_master(items...); +} + +template +value_t Messenger::receive_from_queue_on_master() +{ + mq_pull_poller_.ppoll(-1, &ppoll_sigmask); + auto value = zmqSvc().receive(*mq_pull_, zmq::recv_flags::dontwait); + +#ifndef NDEBUG + std::stringstream ss; + ss << "PID " << getpid() << " receives Q2M " << value; + debug_print(ss.str()); +#endif + + return value; +} + +template +void Messenger::send_from_master_to_queue(T item, Ts... items) +{ +#ifndef NDEBUG + std::stringstream ss; + ss << "PID " << getpid() << " sends M2Q " << item; + debug_print(ss.str()); +#endif + + zmqSvc().send(*mq_push_, item, send_flag_); + // if (sizeof...(items) > 0) { // this will only work with if constexpr, c++17 + send_from_master_to_queue(items...); +} + +template +value_t Messenger::receive_from_master_on_queue() +{ + mq_pull_poller_.ppoll(-1, &ppoll_sigmask); + auto value = zmqSvc().receive(*mq_pull_, zmq::recv_flags::dontwait); + +#ifndef NDEBUG + std::stringstream ss; + ss << "PID " << getpid() << " receives M2Q " << value; + debug_print(ss.str()); +#endif + + return value; +} + +// -- MASTER - WORKER COMMUNICATION -- + +template +void Messenger::publish_from_master_to_workers(T item, Ts... items) +{ +#ifndef NDEBUG + std::stringstream ss; + ss << "PID " << getpid() << " sends M2W " << item; + debug_print(ss.str()); +#endif + + zmqSvc().send(*mw_pub_, item, send_flag_); + // if (sizeof...(items) > 0) { // this will only work with if constexpr, c++17 + publish_from_master_to_workers(items...); +} + +template +value_t Messenger::receive_from_master_on_worker() +{ + mw_sub_poller_.ppoll(-1, &ppoll_sigmask); + auto value = zmqSvc().receive(*mw_sub_, zmq::recv_flags::dontwait); + +#ifndef NDEBUG + std::stringstream ss; + ss << "PID " << getpid() << " receives M2W " << value; + debug_print(ss.str()); +#endif + + return value; +} + +template +void Messenger::send_from_worker_to_master(T item, Ts... items) +{ +#ifndef NDEBUG + std::stringstream ss; + ss << "PID " << getpid() << " sends M2W " << item; + debug_print(ss.str()); +#endif + + zmqSvc().send(*wm_push_, item, send_flag_); + // if (sizeof...(items) > 0) { // this will only work with if constexpr, c++17 + send_from_worker_to_master(items...); +} + +template +value_t Messenger::receive_from_worker_on_master() +{ + wm_pull_poller_.ppoll(-1, &ppoll_sigmask); + auto value = zmqSvc().receive(*wm_pull_, zmq::recv_flags::dontwait); + +#ifndef NDEBUG + std::stringstream ss; + ss << "PID " << getpid() << " receives M2W " << value; + debug_print(ss.str()); +#endif + + return value; +} + +} // namespace MultiProcess +} // namespace RooFit + +#ifdef turn_NDEBUG_back_on +#define NDEBUG +#undef turn_NDEBUG_back_on +#endif + +#endif // ROOT_ROOFIT_MultiProcess_Messenger diff --git a/roofit/multiprocess/res/RooFit/MultiProcess/Messenger_decl.h b/roofit/multiprocess/res/RooFit/MultiProcess/Messenger_decl.h new file mode 100644 index 0000000000000..6ab49d9ea1694 --- /dev/null +++ b/roofit/multiprocess/res/RooFit/MultiProcess/Messenger_decl.h @@ -0,0 +1,166 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ +#ifndef ROOT_ROOFIT_MultiProcess_Messenger_decl +#define ROOT_ROOFIT_MultiProcess_Messenger_decl + +#include "RooFit/MultiProcess/ProcessManager.h" +#include "RooFit_ZMQ/ZeroMQSvc.h" +#include "RooFit_ZMQ/ZeroMQPoller.h" + +#include +#include +#include // sigprocmask, sigset_t, etc +#include + +namespace RooFit { +namespace MultiProcess { + +void set_socket_immediate(ZmqLingeringSocketPtr<> &socket); + +// test messages +enum class X2X : int { ping = -1, pong = -2, initial_value = 0 }; + +class Messenger { +public: + explicit Messenger(const ProcessManager &process_manager); + ~Messenger(); + + void test_connections(const ProcessManager &process_manager); + + enum class test_snd_pipes { + M2Q, + Q2M, + Q2W, + W2Q, + }; + + enum class test_rcv_pipes { + fromQonM, + fromMonQ, + fromWonQ, + fromQonW, + }; + + std::pair create_queue_poller(); + std::pair create_worker_poller(); + + // -- WORKER - QUEUE COMMUNICATION -- + + void send_from_worker_to_queue(); + template + void send_from_worker_to_queue(T item, Ts... items); + template + value_t receive_from_worker_on_queue(std::size_t this_worker_id); + void send_from_queue_to_worker(std::size_t this_worker_id); + template + void send_from_queue_to_worker(std::size_t this_worker_id, T item, Ts... items); + template + value_t receive_from_queue_on_worker(); + + // -- QUEUE - MASTER COMMUNICATION -- + + void send_from_queue_to_master(); + + template + void send_from_queue_to_master(T item, Ts... items); + template + value_t receive_from_queue_on_master(); + void send_from_master_to_queue(); + + template + void send_from_master_to_queue(T item, Ts... items); + template + value_t receive_from_master_on_queue(); + + // -- MASTER - WORKER COMMUNICATION -- + + void publish_from_master_to_workers(); + template + void publish_from_master_to_workers(T item, Ts... items); + template + value_t receive_from_master_on_worker(); + + void send_from_worker_to_master(); + template + void send_from_worker_to_master(T item, Ts... items); + template + value_t receive_from_worker_on_master(); + + void test_receive(X2X expected_ping_value, test_rcv_pipes rcv_pipe, std::size_t worker_id); + void test_send(X2X ping_value, test_snd_pipes snd_pipe, std::size_t worker_id); + + sigset_t ppoll_sigmask; + + void set_send_flag(zmq::send_flags flag); + +private: + void debug_print(std::string s); + + // push + std::vector> qw_push_; + ZmqLingeringSocketPtr<> this_worker_qw_push_; + ZmqLingeringSocketPtr<> mq_push_; + // pollers for all push sockets + std::vector qw_push_poller_; + ZeroMQPoller mq_push_poller_; + // pull + std::vector> qw_pull_; + ZmqLingeringSocketPtr<> this_worker_qw_pull_; + ZmqLingeringSocketPtr<> mq_pull_; + // pollers for all pull sockets + std::vector qw_pull_poller_; + ZeroMQPoller mq_pull_poller_; + + // publish/subscribe sockets for parameter updating from master to workers + ZmqLingeringSocketPtr<> mw_pub_; + ZmqLingeringSocketPtr<> mw_sub_; + ZeroMQPoller mw_sub_poller_; + // push/pull sockets for result retrieving from workers on master + ZmqLingeringSocketPtr<> wm_push_; + ZmqLingeringSocketPtr<> wm_pull_; + ZeroMQPoller wm_pull_poller_; + + // destruction flags to distinguish between different process-type setups: + bool close_MQ_on_destruct_ = false; + bool close_this_QW_on_destruct_ = false; + bool close_QW_container_on_destruct_ = false; + + zmq::send_flags send_flag_ = zmq::send_flags::none; + + std::vector bound_ipc_addresses_; +}; + +// Messages from master to queue +enum class M2Q : int { + enqueue = 10, +}; + +// Messages from worker to queue +enum class W2Q : int { dequeue = 30 }; + +// Messages from queue to worker +enum class Q2W : int { + dequeue_rejected = 40, + dequeue_accepted = 41, +}; + +// stream output operators for debugging +std::ostream &operator<<(std::ostream &out, const M2Q value); +std::ostream &operator<<(std::ostream &out, const Q2W value); +std::ostream &operator<<(std::ostream &out, const W2Q value); +std::ostream &operator<<(std::ostream &out, const X2X value); + +} // namespace MultiProcess +} // namespace RooFit + +#endif // ROOT_ROOFIT_MultiProcess_Messenger_decl diff --git a/roofit/multiprocess/res/RooFit/MultiProcess/ProcessManager.h b/roofit/multiprocess/res/RooFit/MultiProcess/ProcessManager.h new file mode 100644 index 0000000000000..39927d48bafcd --- /dev/null +++ b/roofit/multiprocess/res/RooFit/MultiProcess/ProcessManager.h @@ -0,0 +1,75 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ +#ifndef ROOT_ROOFIT_MultiProcess_ProcessManager +#define ROOT_ROOFIT_MultiProcess_ProcessManager + +#include // pid_t +#include // sig_atomic_t and for sigterm handling on child processes (in ProcessManager.cxx) +#include + +// forward declaration +class Queue; + +namespace RooFit { +namespace MultiProcess { + +class ProcessManager { + friend Queue; + +public: + explicit ProcessManager(std::size_t N_workers); + ~ProcessManager(); + + bool is_initialized() const; + + void terminate() noexcept; + void wait_for_sigterm_then_exit(); + + bool is_master() const; + bool is_queue() const; + bool is_worker() const; + std::size_t worker_id() const; + std::size_t N_workers() const; + + void identify_processes() const; + + static void handle_sigterm(int signum); + static bool sigterm_received(); + + // for debugging/testing: + pid_t get_queue_pid() const { return queue_pid_; } + std::vector get_worker_pids() { return worker_pids_; } + +private: + void initialize_processes(bool cpu_pinning = true); + void shutdown_processes(); + + bool is_master_ = false; + bool is_queue_ = false; + bool is_worker_ = false; + std::size_t worker_id_; + std::size_t N_workers_; + + // master must wait for workers after completion, for which it needs their PIDs + std::vector worker_pids_; + pid_t queue_pid_; + + bool initialized_ = false; + + static volatile sig_atomic_t sigterm_received_; +}; + +} // namespace MultiProcess +} // namespace RooFit + +#endif // ROOT_ROOFIT_MultiProcess_ProcessManager diff --git a/roofit/multiprocess/res/RooFit/MultiProcess/Queue.h b/roofit/multiprocess/res/RooFit/MultiProcess/Queue.h new file mode 100644 index 0000000000000..042de395b1b11 --- /dev/null +++ b/roofit/multiprocess/res/RooFit/MultiProcess/Queue.h @@ -0,0 +1,43 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ +#ifndef ROOT_ROOFIT_MultiProcess_Queue +#define ROOT_ROOFIT_MultiProcess_Queue + +#include "RooFit/MultiProcess/types.h" +#include "RooFit/MultiProcess/Messenger.h" + +#include + +namespace RooFit { +namespace MultiProcess { + +class Queue { +public: + bool pop(JobTask &job_task); + void add(JobTask job_task); + + void loop(); + + void process_master_message(M2Q message); + void process_worker_message(std::size_t this_worker_id, W2Q message); + +private: + std::queue queue_; + std::size_t N_tasks_ = 0; // total number of received tasks + std::size_t N_tasks_at_workers_ = 0; +}; + +} // namespace MultiProcess +} // namespace RooFit + +#endif // ROOT_ROOFIT_MultiProcess_Queue diff --git a/roofit/multiprocess/res/RooFit/MultiProcess/types.h b/roofit/multiprocess/res/RooFit/MultiProcess/types.h new file mode 100644 index 0000000000000..0bd1b25efd23d --- /dev/null +++ b/roofit/multiprocess/res/RooFit/MultiProcess/types.h @@ -0,0 +1,34 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ +#ifndef ROOT_ROOFIT_MultiProcess_types +#define ROOT_ROOFIT_MultiProcess_types + +#include // pair + +namespace RooFit { +namespace MultiProcess { + +// some helper types +using Task = std::size_t; +using State = std::size_t; +/// combined job_object, state and task identifier type +struct JobTask { + std::size_t job_id; + State state_id; + Task task_id; +}; + +} // namespace MultiProcess +} // namespace RooFit + +#endif // ROOT_ROOFIT_MultiProcess_types diff --git a/roofit/multiprocess/res/RooFit/MultiProcess/util.h b/roofit/multiprocess/res/RooFit/MultiProcess/util.h new file mode 100644 index 0000000000000..2893d39eea37f --- /dev/null +++ b/roofit/multiprocess/res/RooFit/MultiProcess/util.h @@ -0,0 +1,34 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#ifndef ROOT_ROOFIT_MultiProcess_util +#define ROOT_ROOFIT_MultiProcess_util + +#include "RooFit_ZMQ/ppoll.h" // for ZMQ::ppoll_error_t +#include "RooFit_ZMQ/ZeroMQPoller.h" + +#include // getpid, pid_t + +namespace RooFit { +namespace MultiProcess { + +int wait_for_child(pid_t child_pid, bool may_throw, int retries_before_killing); + +enum class zmq_ppoll_error_response { abort, unknown_eintr, retry }; +zmq_ppoll_error_response handle_zmq_ppoll_error(ZMQ::ppoll_error_t &e); +std::tuple>, bool> +careful_ppoll(ZeroMQPoller &poller, const sigset_t &ppoll_sigmask, std::size_t max_tries = 2); + +} // namespace MultiProcess +} // namespace RooFit +#endif // ROOT_ROOFIT_MultiProcess_util diff --git a/roofit/multiprocess/res/RooFit/MultiProcess/worker.h b/roofit/multiprocess/res/RooFit/MultiProcess/worker.h new file mode 100644 index 0000000000000..7244ab83e6190 --- /dev/null +++ b/roofit/multiprocess/res/RooFit/MultiProcess/worker.h @@ -0,0 +1,25 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ +#ifndef ROOT_ROOFIT_MultiProcess_worker +#define ROOT_ROOFIT_MultiProcess_worker + +namespace RooFit { +namespace MultiProcess { + +void worker_loop(); +bool is_worker_loop_running(); + +} // namespace MultiProcess +} // namespace RooFit + +#endif // ROOT_ROOFIT_MultiProcess_worker diff --git a/roofit/multiprocess/src/Job.cxx b/roofit/multiprocess/src/Job.cxx new file mode 100644 index 0000000000000..5fc04ee0a94d1 --- /dev/null +++ b/roofit/multiprocess/src/Job.cxx @@ -0,0 +1,151 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#include "RooFit/MultiProcess/JobManager.h" +#include "RooFit/MultiProcess/Messenger.h" +#include "RooFit/MultiProcess/Job.h" + +namespace RooFit { +namespace MultiProcess { + +/** @class Job + * + * @brief interface class for defining the actual work that must be done + * + * Think of "job" as in "employment", e.g. the job of a baker, which + * involves *tasks* like baking and selling bread. The Job must define the + * tasks through its execution (evaluate_task), based on a task index argument. + * + * Classes inheriting from Job must implement the pure virtual methods: + * - void evaluate_task(std::size_t task) + * - void send_back_task_result_from_worker(std::size_t task) + * - void receive_task_result_on_master(const zmq::message_t & message) + * + * An example/reference implementation can be found in test_Job.cxx. + * + * Most Jobs will also want to override the virtual update_state() function. + * This function can be used to send and receive state from master to worker. + * In the worker loop, when something is received over the ZeroMQ "SUB" socket, + * update_state() is called to put the received data into the right places, + * thus updating for instance parameter values on the worker that were updated + * since the last call on the master side. + * + * ## Message protocol + * + * One simple rule must be upheld for the messages that the implementer will + * send with 'send_back_task_result_from_worker' and 'update_state': the first + * part of the message must always be the 'Job''s ID, stored in 'Job::id'. + * The rest of the message, i.e. the actual data to be sent, is completely up + * to the implementation. Note that on the receiving end, i.e. in the + * implementation of 'receive_task_result_on_master', one will get the whole + * message, but the 'Job' ID part will already have been identified in the + * 'JobManager', so one needn't worry about it further inside + * 'Job::receive_task_result_on_master' (it is already routed to the correct + * 'Job'). The same goes for the receiving end of 'update_state', except that + * update_state is routed from the 'worker_loop', not the 'JobManager'. + * + * A second rule applies to 'update_state' messages: the second part must be + * a state identifier. This identifier will also be sent along with tasks to + * the queue. When a worker then takes a task from the queue, it can check + * whether it has already updated its state to what is expected to be there + * for the task at hand. If not, it should wait for the new state to arrive + * over the state subscription socket. Note: it is the implementer's task to + * actually update 'Job::state_id_' inside 'Job::update_state()'! + * + * ## Implementers notes + * + * The type of result from each task is strongly dependent on the Job at hand + * and so Job does not provide a default results member. It is up to the + * inheriting class to implement this in the above functions. We would have + * liked a template parameter task_result_t, so that we could also provide a + * default "boilerplate" calculate function to show a typical Job use-case of + * all the above infrastructure. This is not trivial, because the JobManager + * has to keep a list of Job pointers, so if there would be different template + * instantiations of Jobs, this would complicate this list. + * + * A typical Job implementation will have an evaluation function that is + * called from the master process, like RooAbsArg::getVal calls evaluate(). + * This function will have three purposes: 1. send updated parameter values + * to the workers (possibly through update_state() or in a dedicated + * function), 2. queue tasks and 3. wait for the results to be retrieved. + * 'Job::gather_worker_results()' is provided for convenience to wait for + * all tasks to be retrieved for the current Job. Implementers can also + * choose to have the master process perform other tasks in between any of + * these three steps, or even skip steps completely. + * + * Child classes should refrain from direct access to the JobManager instance + * (through JobManager::instance), but rather use the here provided + * Job::get_manager(). This function starts the worker_loop on the worker when + * first called, meaning that the workers will not be running before they + * are needed. + */ + +Job::Job() +{ + id_ = JobManager::add_job_object(this); +} + +Job::Job(const Job &other) : _manager(other._manager) +{ + id_ = JobManager::add_job_object(this); +} + +Job::~Job() +{ + JobManager::remove_job_object(id_); +} + +/** \brief Get JobManager instance; create and activate if necessary + * + * Child classes should refrain from direct access to the JobManager instance + * (through JobManager::instance), but rather use the here provided + * Job::get_manager(). This function starts the worker_loop on the worker when + * first called, meaning that the workers will not be running before they + * are needed. + */ +JobManager *Job::get_manager() +{ + if (!_manager) { + _manager = JobManager::instance(); + } + + if (!_manager->is_activated()) { + _manager->activate(); + } + + return _manager; +} + +/// Wait for all tasks to be retrieved for the current Job. +void Job::gather_worker_results() +{ + get_manager()->retrieve(id_); +} + +/// \brief Virtual function to update any necessary state on workers +/// +/// This function is called from the worker loop when something is received +/// over the ZeroMQ "SUB" socket. The master process sends messages to workers +/// on its "PUB" socket. Thus, we can update, for instance, parameter values +/// on the worker that were updated since the last call on the master side. +/// \note Implementers: make sure to also update the state_id_ member. +void Job::update_state() {} + +/// Get the current state identifier +std::size_t Job::get_state_id() +{ + return state_id_; +} + +} // namespace MultiProcess +} // namespace RooFit diff --git a/roofit/multiprocess/src/JobManager.cxx b/roofit/multiprocess/src/JobManager.cxx new file mode 100644 index 0000000000000..ea9fcab5d06b9 --- /dev/null +++ b/roofit/multiprocess/src/JobManager.cxx @@ -0,0 +1,218 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#include "RooFit/MultiProcess/JobManager.h" +#include "RooFit/MultiProcess/ProcessManager.h" +#include "RooFit/MultiProcess/Messenger.h" +#include "RooFit/MultiProcess/Job.h" +#include "RooFit/MultiProcess/Queue.h" // complete type for JobManager::queue() +#include "RooFit/MultiProcess/worker.h" +#include "RooFit/MultiProcess/util.h" + +#include // std::thread::hardware_concurrency() + +namespace RooFit { +namespace MultiProcess { + +/** \class JobManager + * + * \brief Main point of access for all MultiProcess infrastructure + * + * This class mainly serves as the access point to the multi-process infrastructure + * for 'Job's. It is meant to be used as a singleton that holds and connects the other + * infrastructural classes: the messenger, process manager, worker and queue loops. + * + * It is important that the user of this class, particularly the one that calls + * 'instance()' first, calls 'activate()' soon after, because everything that is + * done in between 'instance()' and 'activate()' will be executed on all processes. + * This may be useful in some cases, but in general, one will probably want to always + * use the 'JobManager' in its full capacity, including the queue and worker loops. + * This is the way the Job class uses this class, see 'Job::get_manager()'. + * + * The default number of processes is set using 'std::thread::hardware_concurrency()'. + * To change it, simply set 'JobManager::default_N_workers' to a different value + * before creation of a new instance. + */ + +// static function +JobManager *JobManager::instance() +{ + if (!JobManager::is_instantiated()) { + assert(default_N_workers != 0); + instance_.reset(new JobManager(default_N_workers)); // can't use make_unique, because ctor is private + instance_->messenger().test_connections(instance_->process_manager()); + // set send to non blocking on all processes after checking the connections are working: + instance_->messenger().set_send_flag(zmq::send_flags::dontwait); + } + return instance_.get(); +} + +// static function +bool JobManager::is_instantiated() +{ + return static_cast(instance_); +} + +// (private) constructor +/// Don't construct JobManager objects manually, use the static instance if +/// you need to run multiple jobs. +JobManager::JobManager(std::size_t N_workers) +{ + queue_ptr_ = std::make_unique(); + process_manager_ptr_ = std::make_unique(N_workers); + messenger_ptr_ = std::make_unique(*process_manager_ptr_); +} + +JobManager::~JobManager() +{ + // The instance typically gets created by some Job. Once all Jobs are gone, the + // JM will get destroyed. In this case, the job_objects map should have + // been emptied. This check makes sure: + assert(JobManager::job_objects_.empty()); + messenger_ptr_.reset(nullptr); + process_manager_ptr_.reset(nullptr); + queue_ptr_.reset(nullptr); +} + +// static function +/// \return job_id for added job_object +std::size_t JobManager::add_job_object(Job *job_object) +{ + if (JobManager::is_instantiated()) { + if (instance_->process_manager().is_initialized()) { + std::stringstream ss; + ss << "Cannot add Job to JobManager instantiation, forking has already taken place! Instance object at raw " + "ptr " + << instance_.get(); + throw std::logic_error("Cannot add Job to JobManager instantiation, forking has already taken place! Call " + "terminate() on the instance before adding new Jobs."); + } + } + std::size_t job_id = job_counter_++; + job_objects_[job_id] = job_object; + return job_id; +} + +// static function +Job *JobManager::get_job_object(std::size_t job_object_id) +{ + return job_objects_[job_object_id]; +} + +// static function +/// \return Returns 'true' when removed successfully, 'false' otherwise. +bool JobManager::remove_job_object(std::size_t job_object_id) +{ + bool removed_succesfully = job_objects_.erase(job_object_id) == 1; + if (job_objects_.empty()) { + instance_.reset(nullptr); + } + return removed_succesfully; +} + +ProcessManager &JobManager::process_manager() const +{ + return *process_manager_ptr_; +} + +Messenger &JobManager::messenger() const +{ + return *messenger_ptr_; +} + +Queue &JobManager::queue() const +{ + return *queue_ptr_; +} + +/// Retrieve results for a Job +/// +/// \param requesting_job_id ID number of the Job in the JobManager's Job list +void JobManager::retrieve(std::size_t requesting_job_id) +{ + if (process_manager().is_master()) { + bool job_fully_retrieved = false; + while (not job_fully_retrieved) { + try { + auto task_result_message = messenger().receive_from_worker_on_master(); + auto job_object_id = *reinterpret_cast( + task_result_message.data()); // job_id must always be the first element of the result message! + bool this_job_fully_retrieved = + JobManager::get_job_object(job_object_id)->receive_task_result_on_master(task_result_message); + if (requesting_job_id == job_object_id) { + job_fully_retrieved = this_job_fully_retrieved; + } + } catch (ZMQ::ppoll_error_t &e) { + zmq_ppoll_error_response response; + try { + response = handle_zmq_ppoll_error(e); + } catch (std::logic_error &) { + printf("JobManager::retrieve got unhandleable ZMQ::ppoll_error_t\n"); + throw; + } + if (response == zmq_ppoll_error_response::abort) { + throw std::logic_error("in JobManager::retrieve: master received a SIGTERM, aborting"); + } else if (response == zmq_ppoll_error_response::unknown_eintr) { + printf("EINTR in JobManager::retrieve, continuing\n"); + continue; + } else if (response == zmq_ppoll_error_response::retry) { + printf("EAGAIN from ppoll in JobManager::retrieve, continuing\n"); + continue; + } + } catch (zmq::error_t &e) { + printf("unhandled zmq::error_t (not a ppoll_error_t) in JobManager::retrieve with errno %d: %s\n", e.num(), + e.what()); + throw; + } + } + } +} + +/// \brief Start queue and worker loops on child processes +/// +/// This function exists purely because activation from the constructor is +/// impossible; the constructor must return a constructed instance, which it +/// can't do if it's stuck in an infinite loop. This means the Job that first +/// creates the JobManager instance must also activate it (or any other user +/// of this class). +/// This should be called soon after creation of instance, because everything +/// between construction and activation gets executed both on the master +/// process and on the slaves. +void JobManager::activate() +{ + activated_ = true; + + if (process_manager().is_queue()) { + queue().loop(); + std::_Exit(0); + } + + if (!is_worker_loop_running() && process_manager().is_worker()) { + RooFit::MultiProcess::worker_loop(); + std::_Exit(0); + } +} + +bool JobManager::is_activated() const +{ + return activated_; +} + +// initialize static members +std::map JobManager::job_objects_; +std::size_t JobManager::job_counter_ = 0; +std::unique_ptr JobManager::instance_{nullptr}; +unsigned int JobManager::default_N_workers = std::thread::hardware_concurrency(); + +} // namespace MultiProcess +} // namespace RooFit diff --git a/roofit/multiprocess/src/Messenger.cxx b/roofit/multiprocess/src/Messenger.cxx new file mode 100644 index 0000000000000..6dc748cc4aab6 --- /dev/null +++ b/roofit/multiprocess/src/Messenger.cxx @@ -0,0 +1,544 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#include "RooFit/MultiProcess/Messenger.h" +#include "RooFit/MultiProcess/util.h" + +#include // sigprocmask etc +#include // sprintf + +namespace RooFit { +namespace MultiProcess { + +void set_socket_immediate(ZmqLingeringSocketPtr<> &socket) +{ + int optval = 1; + socket->set(zmq::sockopt::immediate, optval); +} + +/** \class Messenger + * + * \brief Manages ZeroMQ sockets and wraps send and receive calls + * + * This class is used for all interprocess communication between the master, + * queue and worker processes. It sets up ZeroMQ sockets between all processes + * over IPC socket files stored in /tmp on the filesystem. + * + * Several sockets are used for communication between different places for + * different purposes: + * - Master and queue processes each have a PUSH-PULL socket pair to directly + * send/receive data between only the master and queue processes. This is + * currently used mainly for sending tasks to the queue from master. The + * socket from queue back to master is used only to test connections and may + * be removed in the future. + * - The queue process also has a PUSH-PULL socket pair with each worker + * process. These are used by the workers to obtain tasks from the queue. + * - The master has a PUB socket that the workers subscribe to with SUB + * sockets. These are used to update state. Note that to ensure robust + * reception of all messages on the SUB socket, it's important to send over + * state in as little messages as possible. For instance, it's best to send + * arrays over in a single big message instead of sending over each element + * separately. This also improves performance, since each message has some + * fixed overhead. + * - Each worker has a PUSH socket connected to a PULL socket on master that + * is used to send back task results from workers to master in + * 'JobManager::retrieve()'. + * + * @param process_manager ProcessManager instance which manages the master, + * queue and worker processes that we want to set up + * communication for in this Messenger. + */ + +Messenger::Messenger(const ProcessManager &process_manager) +{ + sigemptyset(&ppoll_sigmask); + + char addr_prefix_template[30] = "ipc:///tmp/%d_roofitMP"; + char addr_prefix[50]; + char addr[100]; + + // high water mark for master-queue sending, which can be quite a busy channel, especially at the start of a run + int hwm = 0; + // create zmq connections (zmq context is automatically created in the ZeroMQSvc class and maintained as singleton) + // and pollers where necessary + try { + if (process_manager.is_master()) { + sprintf(addr_prefix, addr_prefix_template, getpid()); + + mq_push_.reset(zmqSvc().socket_ptr(zmq::socket_type::push)); + auto rc = zmq_setsockopt(*mq_push_, ZMQ_SNDHWM, &hwm, sizeof hwm); + assert(rc == 0); + sprintf(addr, "%s_%s", addr_prefix, "from_master_to_queue"); + mq_push_->bind(addr); + bound_ipc_addresses_.emplace_back(addr); + + mq_push_poller_.register_socket(*mq_push_, zmq::event_flags::pollout); + + mq_pull_.reset(zmqSvc().socket_ptr(zmq::socket_type::pull)); + rc = zmq_setsockopt(*mq_pull_, ZMQ_RCVHWM, &hwm, sizeof hwm); + assert(rc == 0); + sprintf(addr, "%s_%s", addr_prefix, "from_queue_to_master"); + mq_pull_->bind(addr); + bound_ipc_addresses_.emplace_back(addr); + + mq_pull_poller_.register_socket(*mq_pull_, zmq::event_flags::pollin); + + mw_pub_.reset(zmqSvc().socket_ptr(zmq::socket_type::pub)); + rc = zmq_setsockopt(*mw_pub_, ZMQ_SNDHWM, &hwm, sizeof hwm); + assert(rc == 0); + sprintf(addr, "%s_%s", addr_prefix, "from_master_to_workers"); + mw_pub_->bind(addr); + bound_ipc_addresses_.emplace_back(addr); + + wm_pull_.reset(zmqSvc().socket_ptr(zmq::socket_type::pull)); + rc = zmq_setsockopt(*wm_pull_, ZMQ_RCVHWM, &hwm, sizeof hwm); + assert(rc == 0); + sprintf(addr, "%s_%s", addr_prefix, "from_workers_to_master"); + wm_pull_->bind(addr); + bound_ipc_addresses_.emplace_back(addr); + wm_pull_poller_.register_socket(*wm_pull_, zmq::event_flags::pollin); + + close_MQ_on_destruct_ = true; + + // make sure all subscribers are connected + ZmqLingeringSocketPtr<> subscriber_ping_socket{zmqSvc().socket_ptr(zmq::socket_type::pull)}; + sprintf(addr, "%s_%s", addr_prefix, "subscriber_ping_socket"); + subscriber_ping_socket->bind(addr); + bound_ipc_addresses_.emplace_back(addr); + ZeroMQPoller subscriber_ping_poller; + subscriber_ping_poller.register_socket(*subscriber_ping_socket, zmq::event_flags::pollin); + std::size_t N_subscribers_confirmed = 0; + while (N_subscribers_confirmed < process_manager.N_workers()) { + zmqSvc().send(*mw_pub_, false); + auto poll_results = subscriber_ping_poller.poll(0); + for (std::size_t ix = 0; ix < poll_results.size(); ++ix) { + auto request = zmqSvc().receive(*subscriber_ping_socket, zmq::recv_flags::dontwait); + assert(request == "present"); + ++N_subscribers_confirmed; + } + } + zmqSvc().send(*mw_pub_, true); + + } else if (process_manager.is_queue()) { + sprintf(addr_prefix, addr_prefix_template, getppid()); + + // first the queue-worker sockets + // do resize instead of reserve so that the unique_ptrs are initialized + // (to nullptr) so that we can do reset below, alternatively you can do + // push/emplace_back with move or something + qw_push_.resize(process_manager.N_workers()); + qw_pull_.resize(process_manager.N_workers()); + qw_push_poller_.resize(process_manager.N_workers()); + qw_pull_poller_.resize(process_manager.N_workers()); + for (std::size_t ix = 0; ix < process_manager.N_workers(); ++ix) { + // push + qw_push_[ix].reset(zmqSvc().socket_ptr(zmq::socket_type::push)); + sprintf(addr, "%s_%s_%zu", addr_prefix, "from_queue_to_worker", ix); + qw_push_[ix]->bind(addr); + bound_ipc_addresses_.emplace_back(addr); + + qw_push_poller_[ix].register_socket(*qw_push_[ix], zmq::event_flags::pollout); + + // pull + qw_pull_[ix].reset(zmqSvc().socket_ptr(zmq::socket_type::pull)); + sprintf(addr, "%s_%s_%zu_%s", addr_prefix, "from_worker", ix, "to_queue"); + qw_pull_[ix]->bind(addr); + bound_ipc_addresses_.emplace_back(addr); + + qw_pull_poller_[ix].register_socket(*qw_pull_[ix], zmq::event_flags::pollin); + } + + // then the master-queue sockets + mq_push_.reset(zmqSvc().socket_ptr(zmq::socket_type::push)); + auto rc = zmq_setsockopt(*mq_push_, ZMQ_SNDHWM, &hwm, sizeof hwm); + assert(rc == 0); + sprintf(addr, "%s_%s", addr_prefix, "from_queue_to_master"); + mq_push_->connect(addr); + + mq_push_poller_.register_socket(*mq_push_, zmq::event_flags::pollout); + + mq_pull_.reset(zmqSvc().socket_ptr(zmq::socket_type::pull)); + rc = zmq_setsockopt(*mq_pull_, ZMQ_RCVHWM, &hwm, sizeof hwm); + assert(rc == 0); + sprintf(addr, "%s_%s", addr_prefix, "from_master_to_queue"); + mq_pull_->connect(addr); + + mq_pull_poller_.register_socket(*mq_pull_, zmq::event_flags::pollin); + + close_MQ_on_destruct_ = true; + close_QW_container_on_destruct_ = true; + } else if (process_manager.is_worker()) { + sprintf(addr_prefix, addr_prefix_template, getppid()); + + // we only need one queue-worker pipe on the worker + qw_push_poller_.resize(1); + qw_pull_poller_.resize(1); + + // push + this_worker_qw_push_.reset(zmqSvc().socket_ptr(zmq::socket_type::push)); + sprintf(addr, "%s_%s_%zu_%s", addr_prefix, "from_worker", process_manager.worker_id(), "to_queue"); + this_worker_qw_push_->connect(addr); + + qw_push_poller_[0].register_socket(*this_worker_qw_push_, zmq::event_flags::pollout); + + // pull + this_worker_qw_pull_.reset(zmqSvc().socket_ptr(zmq::socket_type::pull)); + sprintf(addr, "%s_%s_%zu", addr_prefix, "from_queue_to_worker", process_manager.worker_id()); + this_worker_qw_pull_->connect(addr); + + qw_pull_poller_[0].register_socket(*this_worker_qw_pull_, zmq::event_flags::pollin); + + mw_sub_.reset(zmqSvc().socket_ptr(zmq::socket_type::sub)); + auto rc = zmq_setsockopt(*mw_sub_, ZMQ_RCVHWM, &hwm, sizeof hwm); + assert(rc == 0); + rc = zmq_setsockopt(*mw_sub_, ZMQ_SUBSCRIBE, "", 0); + assert(rc == 0); + sprintf(addr, "%s_%s", addr_prefix, "from_master_to_workers"); + mw_sub_->connect(addr); + mw_sub_poller_.register_socket(*mw_sub_, zmq::event_flags::pollin); + + wm_push_.reset(zmqSvc().socket_ptr(zmq::socket_type::push)); + rc = zmq_setsockopt(*wm_push_, ZMQ_SNDHWM, &hwm, sizeof hwm); + assert(rc == 0); + sprintf(addr, "%s_%s", addr_prefix, "from_workers_to_master"); + wm_push_->connect(addr); + + // check publisher connection and then wait until all subscribers are connected + ZmqLingeringSocketPtr<> subscriber_ping_socket{zmqSvc().socket_ptr(zmq::socket_type::push)}; + sprintf(addr, "%s_%s", addr_prefix, "subscriber_ping_socket"); + subscriber_ping_socket->connect(addr); + auto all_connected = zmqSvc().receive(*mw_sub_); + zmqSvc().send(*subscriber_ping_socket, "present"); + + while (!all_connected) { + all_connected = zmqSvc().receive(*mw_sub_); + } + + close_this_QW_on_destruct_ = true; + } else { + // should never get here + throw std::runtime_error("Messenger ctor: I'm neither master, nor queue, nor a worker"); + } + } catch (zmq::error_t &e) { + std::cerr << e.what() << " -- errnum: " << e.num() << std::endl; + throw; + }; +} + +Messenger::~Messenger() +{ + if (close_MQ_on_destruct_) { + try { + mq_push_.reset(nullptr); + mq_pull_.reset(nullptr); + mw_pub_.reset(nullptr); + wm_pull_.reset(nullptr); + // remove bound files + for (const auto &address : bound_ipc_addresses_) { + // no need to check return value, they are only zero byte /tmp files, the OS should eventually clean them up + remove(address.substr(6).c_str()); + } + } catch (const std::exception &e) { + std::cerr << "WARNING: something in Messenger dtor threw an exception! Original exception message:\n" + << e.what() << std::endl; + } + } + if (close_this_QW_on_destruct_) { + this_worker_qw_push_.reset(nullptr); + this_worker_qw_pull_.reset(nullptr); + mw_sub_.reset(nullptr); + wm_push_.reset(nullptr); + } + if (close_QW_container_on_destruct_) { + for (auto &socket : qw_push_) { + socket.reset(nullptr); + } + for (auto &socket : qw_pull_) { + socket.reset(nullptr); + } + } + zmqSvc().close_context(); +} + +void Messenger::test_send(X2X ping_value, test_snd_pipes snd_pipe, std::size_t worker_id) +{ + try { + switch (snd_pipe) { + case test_snd_pipes::M2Q: { + send_from_master_to_queue(ping_value); + break; + } + case test_snd_pipes::Q2M: { + send_from_queue_to_master(ping_value); + break; + } + case test_snd_pipes::Q2W: { + send_from_queue_to_worker(worker_id, ping_value); + break; + } + case test_snd_pipes::W2Q: { + send_from_worker_to_queue(ping_value); + break; + } + } + } catch (zmq::error_t &e) { + if (e.num() == EAGAIN) { + throw std::runtime_error("Messenger::test_connections: SEND over master-queue connection timed out!"); + } else { + throw; + } + } +} + +void Messenger::test_receive(X2X expected_ping_value, test_rcv_pipes rcv_pipe, std::size_t worker_id) +{ + X2X handshake = X2X::initial_value; + + std::size_t max_tries = 3, tries = 0; + bool carry_on = true; + while (carry_on && (tries++ < max_tries)) { + try { + switch (rcv_pipe) { + case test_rcv_pipes::fromMonQ: { + handshake = receive_from_master_on_queue(); + break; + } + case test_rcv_pipes::fromQonM: { + handshake = receive_from_queue_on_master(); + break; + } + case test_rcv_pipes::fromQonW: { + handshake = receive_from_queue_on_worker(); + break; + } + case test_rcv_pipes::fromWonQ: { + handshake = receive_from_worker_on_queue(worker_id); + break; + } + } + carry_on = false; + } catch (ZMQ::ppoll_error_t &e) { + auto response = handle_zmq_ppoll_error(e); + if (response == zmq_ppoll_error_response::abort) { + throw std::runtime_error("EINTR in test_receive and SIGTERM received, aborting\n"); + } else if (response == zmq_ppoll_error_response::unknown_eintr) { + printf("EINTR in test_receive but no SIGTERM received, try %zu\n", tries); + continue; + } else if (response == zmq_ppoll_error_response::retry) { + printf("EAGAIN in test_receive, try %zu\n", tries); + continue; + } + } catch (zmq::error_t &e) { + if (e.num() == EAGAIN) { + throw std::runtime_error("Messenger::test_connections: RECEIVE over master-queue connection timed out!"); + } else { + printf("unhandled zmq::error_t (not a ppoll_error_t) in Messenger::test_receive with errno %d: %s\n", + e.num(), e.what()); + throw; + } + } + } + + if (handshake != expected_ping_value) { + throw std::runtime_error( + "Messenger::test_connections: RECEIVE over master-queue connection failed, did not receive expected value!"); + } +} + +/// \brief Test whether push-pull sockets are working +/// +/// \note This function tests the PUSH-PULL socket pairs only. The PUB-SUB sockets are already tested in the +/// constructor. +/// +/// \param process_manager ProcessManager object used to instantiate this object. Used to identify which process we are +/// running on and hence which sockets need to be tested. +void Messenger::test_connections(const ProcessManager &process_manager) +{ + if (process_manager.is_queue() || process_manager.is_worker()) { + // Before blocking SIGTERM, set the signal handler, so we can also check after blocking whether a signal occurred + // In our case, we already set it in the ProcessManager after forking to the queue and worker processes. + sigset_t sigmask; + sigemptyset(&sigmask); + sigaddset(&sigmask, SIGTERM); + int rc = sigprocmask(SIG_BLOCK, &sigmask, &ppoll_sigmask); + if (rc < 0) { + throw std::runtime_error("sigprocmask failed in test_connections"); + } + } + + if (process_manager.is_master()) { + test_receive(X2X::ping, test_rcv_pipes::fromQonM, -1); + test_send(X2X::pong, test_snd_pipes::M2Q, -1); + test_send(X2X::ping, test_snd_pipes::M2Q, -1); + // make sure to always receive last on master, so that master knows when queue is done, + // which means workers are done as well, so if master is done everything is done: + test_receive(X2X::pong, test_rcv_pipes::fromQonM, -1); + } else if (process_manager.is_queue()) { + ZeroMQPoller poller; + std::size_t mq_index; + std::tie(poller, mq_index) = create_queue_poller(); + + for (std::size_t ix = 0; ix < process_manager.N_workers(); ++ix) { + test_send(X2X::ping, test_snd_pipes::Q2W, ix); + } + test_send(X2X::ping, test_snd_pipes::Q2M, -1); + + while (!process_manager.sigterm_received() && (poller.size() > 0)) { + // poll: wait until status change (-1: infinite timeout) + std::vector> poll_result; + bool abort; + std::tie(poll_result, abort) = careful_ppoll(poller, ppoll_sigmask); + if (abort) + break; + + // then process incoming messages from sockets + for (auto readable_socket : poll_result) { + // message comes from the master/queue socket (first element): + if (readable_socket.first == mq_index) { + test_receive(X2X::pong, test_rcv_pipes::fromMonQ, -1); + test_receive(X2X::ping, test_rcv_pipes::fromMonQ, -1); + poller.unregister_socket(*mq_pull_); + } else { // from a worker socket + // TODO: dangerous assumption for this_worker_id, may become invalid if we allow multiple queue_loops on + // the same process! + auto this_worker_id = readable_socket.first - 1; // TODO: replace with a more reliable lookup + + test_receive(X2X::pong, test_rcv_pipes::fromWonQ, this_worker_id); + test_receive(X2X::ping, test_rcv_pipes::fromWonQ, this_worker_id); + test_send(X2X::pong, test_snd_pipes::Q2W, this_worker_id); + + poller.unregister_socket(*qw_pull_[this_worker_id]); + } + } + } + test_send(X2X::pong, test_snd_pipes::Q2M, -1); + + } else if (process_manager.is_worker()) { + test_receive(X2X::ping, test_rcv_pipes::fromQonW, -1); + test_send(X2X::pong, test_snd_pipes::W2Q, -1); + test_send(X2X::ping, test_snd_pipes::W2Q, -1); + test_receive(X2X::pong, test_rcv_pipes::fromQonW, -1); + } else { + // should never get here + throw std::runtime_error("Messenger::test_connections: I'm neither master, nor queue, nor a worker"); + } + + if (process_manager.is_queue() || process_manager.is_worker()) { + // clean up signal management modifications + sigprocmask(SIG_SETMASK, &ppoll_sigmask, nullptr); + } +} + +/// Helper function that creates a poller for Queue::loop() +std::pair Messenger::create_queue_poller() +{ + ZeroMQPoller poller; + std::size_t mq_index = poller.register_socket(*mq_pull_, zmq::event_flags::pollin); + for (auto &s : qw_pull_) { + poller.register_socket(*s, zmq::event_flags::pollin); + } + return {std::move(poller), mq_index}; +} + +/// Helper function that creates a poller for worker_loop() +std::pair Messenger::create_worker_poller() +{ + ZeroMQPoller poller; + poller.register_socket(*this_worker_qw_pull_, zmq::event_flags::pollin); + std::size_t mw_sub_index = poller.register_socket(*mw_sub_, zmq::event_flags::pollin); + return {std::move(poller), mw_sub_index}; +} + +// -- WORKER - QUEUE COMMUNICATION -- + +void Messenger::send_from_worker_to_queue() {} + +void Messenger::send_from_queue_to_worker(std::size_t /*this_worker_id*/) {} + +// -- QUEUE - MASTER COMMUNICATION -- + +void Messenger::send_from_queue_to_master() {} + +void Messenger::send_from_master_to_queue() {} + +/// Set the flag used in all send functions; 0, ZMQ_DONTWAIT, ZMQ_SNDMORE or bitwise combination +void Messenger::set_send_flag(zmq::send_flags flag) +{ + send_flag_ = flag; +} + +// -- MASTER - WORKER COMMUNICATION -- + +void Messenger::publish_from_master_to_workers() {} + +void Messenger::send_from_worker_to_master() {} + +// for debugging +#define PROCESS_VAL(p) \ + case (p): s = #p; break; + +std::ostream &operator<<(std::ostream &out, const M2Q value) +{ + std::string s; + switch (value) { + PROCESS_VAL(M2Q::enqueue); + default: s = std::to_string(static_cast(value)); + } + return out << s; +} + +std::ostream &operator<<(std::ostream &out, const W2Q value) +{ + std::string s; + switch (value) { + PROCESS_VAL(W2Q::dequeue); + default: s = std::to_string(static_cast(value)); + } + return out << s; +} + +std::ostream &operator<<(std::ostream &out, const Q2W value) +{ + std::string s; + switch (value) { + PROCESS_VAL(Q2W::dequeue_rejected); + PROCESS_VAL(Q2W::dequeue_accepted); + default: s = std::to_string(static_cast(value)); + } + return out << s; +} + +std::ostream &operator<<(std::ostream &out, const X2X value) +{ + std::string s; + switch (value) { + PROCESS_VAL(X2X::ping); + PROCESS_VAL(X2X::pong); + default: s = std::to_string(static_cast(value)); + } + return out << s; +} + +#undef PROCESS_VAL + +/// Function called from send and receive template functions in debug builds +/// used to monitor the messages that are going to be sent or are received. +/// By defining this in the implementation file, compilation is a lot faster +/// during debugging of Messenger or communication protocols. +void Messenger::debug_print(std::string /*s*/) +{ + // print 's' when debugging +} + +} // namespace MultiProcess +} // namespace RooFit diff --git a/roofit/multiprocess/src/ProcessManager.cxx b/roofit/multiprocess/src/ProcessManager.cxx new file mode 100644 index 0000000000000..e9a00bff766c3 --- /dev/null +++ b/roofit/multiprocess/src/ProcessManager.cxx @@ -0,0 +1,306 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#include "RooFit/MultiProcess/ProcessManager.h" +#include "RooFit/MultiProcess/JobManager.h" +#include "RooFit/MultiProcess/util.h" + +#include // for strsignal +#include // for wait +#include +#include + +namespace RooFit { +namespace MultiProcess { + +/// \class ProcessManager +/// \brief Fork processes for queue and workers +/// +/// This class manages three types of processes: +/// 1. master: the initial main process. It defines and enqueues tasks +/// and processes results. +/// 2. workers: a pool of processes that will try to take tasks from the +/// queue. These are forked from master. +/// 3. queue: This process runs the queue_loop and maintains the queue of +/// tasks. It is also forked from master. +/// +/// \param N_workers Number of worker processes to spawn. +ProcessManager::ProcessManager(std::size_t N_workers) : N_workers_(N_workers) +{ + initialize_processes(); +} + +ProcessManager::~ProcessManager() +{ + if (is_master()) { + terminate(); + } else { + wait_for_sigterm_then_exit(); + } +} + +// static member initialization +volatile sig_atomic_t ProcessManager::sigterm_received_ = 0; + +// static function +/// We need this to tell the children to die, because we can't talk +/// to them anymore during JobManager destruction, because that kills +/// the Messenger first. We do that with SIGTERMs. The sigterm_received() +/// should be checked in message loops to stop them when it's true. +void ProcessManager::handle_sigterm(int /*signum*/) +{ + sigterm_received_ = 1; +} + +// static function +bool ProcessManager::sigterm_received() +{ + if (sigterm_received_ > 0) { + return true; + } else { + return false; + } +} + +pid_t fork_and_handle_errors() +{ + pid_t child_pid = fork(); + int retries = 0; + while (child_pid == -1) { + if (retries < 3) { + ++retries; + printf("fork returned with error number %d, retrying after 1 second...\n", errno); + sleep(1); + child_pid = fork(); + } else { + printf("fork returned with error number %d\n", errno); + throw std::runtime_error("fork returned with error 3 times, aborting!"); + } + } + return child_pid; +} + +/// \brief Fork processes and activate CPU pinning +/// +/// \param cpu_pinning Activate CPU pinning if true. Effective on Linux only. +void ProcessManager::initialize_processes(bool cpu_pinning) +{ + // Initialize processes; + // ... first workers: + worker_pids_.resize(N_workers_); + pid_t child_pid{}; + for (std::size_t ix = 0; ix < N_workers_; ++ix) { + child_pid = fork_and_handle_errors(); + if (!child_pid) { // we're on the worker + is_worker_ = true; + worker_id_ = ix; + break; + } else { // we're on master + worker_pids_[ix] = child_pid; + } + } + + // ... then queue: + if (child_pid) { // we're on master + queue_pid_ = fork_and_handle_errors(); + if (!queue_pid_) { // we're now on queue + is_queue_ = true; + } else { + is_master_ = true; + } + } + + // set the sigterm handler on the child processes + if (!is_master_) { + struct sigaction sa; + memset(&sa, '\0', sizeof(sa)); + sa.sa_handler = ProcessManager::handle_sigterm; + + if (sigaction(SIGTERM, &sa, NULL) < 0) { + std::perror("sigaction failed"); + std::exit(1); + } + } + + if (cpu_pinning) { +#if defined(__APPLE__) +#ifndef NDEBUG + static bool affinity_warned = false; + if (is_master() & !affinity_warned) { + std::cout << "CPU affinity cannot be set on macOS" << std::endl; + affinity_warned = true; + } +#endif // NDEBUG +#elif defined(_WIN32) +#ifndef NDEBUG + if (is_master()) + std::cerr << "WARNING: CPU affinity setting not implemented on Windows, continuing..." << std::endl; +#endif // NDEBUG +#else + cpu_set_t mask; + // zero all bits in mask + CPU_ZERO(&mask); + // set correct bit + std::size_t set_cpu; + if (is_master()) { + set_cpu = N_workers() + 1; + } else if (is_queue()) { + set_cpu = N_workers(); + } else { + set_cpu = worker_id(); + } + CPU_SET(set_cpu, &mask); +#ifndef NDEBUG + // sched_setaffinity returns 0 on success + if (sched_setaffinity(0, sizeof(mask), &mask) == -1) { + std::cerr << "WARNING: Could not set CPU affinity, continuing..." << std::endl; + } else { + std::cerr << "CPU affinity set to cpu " << set_cpu << " in process " << getpid() << std::endl; + } +#endif // NDEBUG +#endif + } + +#ifndef NDEBUG + identify_processes(); +#endif // NDEBUG + + initialized_ = true; +} + +bool ProcessManager::is_initialized() const +{ + return initialized_; +} + +/// Shutdown forked processes if on master and if this process manager is initialized +void ProcessManager::terminate() noexcept +{ + try { + if (is_master() && is_initialized()) { + shutdown_processes(); + } + } catch (const std::exception &e) { + std::cerr << "WARNING: something in ProcessManager::terminate threw an exception! Original exception message:\n" + << e.what() << std::endl; + } +} + +void ProcessManager::wait_for_sigterm_then_exit() +{ + if (!is_master()) { + while (!sigterm_received()) { + } + std::_Exit(0); + } +} + +int chill_wait() +{ + int status = 0; + pid_t pid; + do { + pid = wait(&status); + } while (-1 == pid && EINTR == errno); // retry on interrupted system call + + if (0 != status) { + if (WIFEXITED(status)) { + printf("exited, status=%d\n", WEXITSTATUS(status)); + } else if (WIFSIGNALED(status)) { + if (WTERMSIG(status) != SIGTERM) { + printf("killed by signal %d\n", WTERMSIG(status)); + } + } else if (WIFSTOPPED(status)) { + printf("stopped by signal %d\n", WSTOPSIG(status)); + } else if (WIFCONTINUED(status)) { + printf("continued\n"); + } + } + + if (-1 == pid) { + if (errno == ECHILD) { + printf("chill_wait: no children (got ECHILD error code from wait call), done\n"); + } else { + throw std::runtime_error(std::string("chill_wait: error in wait call: ") + strerror(errno) + + std::string(", errno ") + std::to_string(errno)); + } + } + + return pid; +} + +/// Shutdown forked processes if on master +void ProcessManager::shutdown_processes() +{ + if (is_master()) { + // terminate all children + std::unordered_set children; + children.insert(queue_pid_); + kill(queue_pid_, SIGTERM); + for (auto pid : worker_pids_) { + kill(pid, SIGTERM); + children.insert(pid); + } + // then wait for them to actually die and clean out the zombies + while (!children.empty()) { + pid_t pid = chill_wait(); + children.erase(pid); + } + } + + initialized_ = false; +} + +// Getters + +bool ProcessManager::is_master() const +{ + return is_master_; +} + +bool ProcessManager::is_queue() const +{ + return is_queue_; +} + +bool ProcessManager::is_worker() const +{ + return is_worker_; +} + +std::size_t ProcessManager::worker_id() const +{ + return worker_id_; +} + +std::size_t ProcessManager::N_workers() const +{ + return N_workers_; +} + +/// Print to stdout which type of process we are on and what its PID is (for debugging) +void ProcessManager::identify_processes() const +{ + if (is_worker_) { + printf("I'm a worker, PID %d\n", getpid()); + } else if (is_master_) { + printf("I'm master, PID %d\n", getpid()); + } else if (is_queue_) { + printf("I'm queue, PID %d\n", getpid()); + } else { + printf("I'm not master, queue or worker, weird! PID %d\n", getpid()); + } +} + +} // namespace MultiProcess +} // namespace RooFit diff --git a/roofit/multiprocess/src/Queue.cxx b/roofit/multiprocess/src/Queue.cxx new file mode 100644 index 0000000000000..f73d543e26522 --- /dev/null +++ b/roofit/multiprocess/src/Queue.cxx @@ -0,0 +1,175 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#include "RooFit/MultiProcess/Queue.h" +#include "RooFit/MultiProcess/JobManager.h" +#include "RooFit/MultiProcess/ProcessManager.h" +#include "RooFit/MultiProcess/Job.h" // complete Job object for JobManager::get_job_object() +#include "RooFit/MultiProcess/util.h" + +namespace RooFit { +namespace MultiProcess { + +/** \class Queue + * \brief Keeps a queue of tasks for workers and manages the queue process through its event loop + * + * The Queue maintains a set of tasks on the queue process by receiving them + * from the master process. Worker processes can request to pop them off the + * queue. The communication between these processes is handled inside + * 'Queue::loop()', the queue process's event loop that polls the Messenger's + * sockets for incoming messages and handles them when they come. + * + * The reason for this class is to get automatic load balancing between + * workers. By allowing workers to request tasks whenever they are ready to + * do work, we don't need to manually distribute work over workers and they + * will always have something to do until all tasks have been completed. + * The alternative simple strategy of just distributing all tasks evenly over + * workers will be suboptimal when tasks have different or even varying + * runtimes (this simple strategy could be implemented with a PUSH-PULL + * ZeroMQ socket from master to workers, which would distribute tasks in a + * round-robin fashion, which, indeed, does not do load balancing). + */ + +/// Have a worker ask for a task-message from the queue +/// +/// \param[out] job_task JobTask reference to put the Job ID and the task index into. +/// \return true if a task was popped from the queue successfully, false if the queue was empty. +bool Queue::pop(JobTask &job_task) +{ + if (queue_.empty()) { + return false; + } else { + job_task = queue_.front(); + queue_.pop(); + return true; + } +} + +/// Enqueue a task +/// +/// \param[in] job_task JobTask object that contains the Job ID and the task index. +void Queue::add(JobTask job_task) +{ + if (JobManager::instance()->process_manager().is_master()) { + JobManager::instance()->messenger().send_from_master_to_queue(M2Q::enqueue, job_task.job_id, job_task.state_id, + job_task.task_id); + } else if (JobManager::instance()->process_manager().is_queue()) { + queue_.push(job_task); + } else { + throw std::logic_error("calling Communicator::to_master_queue from slave process"); + } +} + +/// Helper function for 'Queue::loop()' +void Queue::process_master_message(M2Q message) +{ + switch (message) { + case M2Q::enqueue: { + // enqueue task + auto job_object_id = JobManager::instance()->messenger().receive_from_master_on_queue(); + auto state_id = JobManager::instance()->messenger().receive_from_master_on_queue(); + auto task_id = JobManager::instance()->messenger().receive_from_master_on_queue(); + JobTask job_task{job_object_id, state_id, task_id}; + add(job_task); + N_tasks_++; + break; + } + } +} + +/// Helper function for 'Queue::loop()' +void Queue::process_worker_message(std::size_t this_worker_id, W2Q message) +{ + switch (message) { + case W2Q::dequeue: { + // dequeue task + JobTask job_task; + bool popped = pop(job_task); + if (popped) { + // Note: below two commands should be run atomically for thread safety (if that ever becomes an issue) + JobManager::instance()->messenger().send_from_queue_to_worker( + this_worker_id, Q2W::dequeue_accepted, job_task.job_id, job_task.state_id, job_task.task_id); + ++N_tasks_at_workers_; + } else { + JobManager::instance()->messenger().send_from_queue_to_worker(this_worker_id, Q2W::dequeue_rejected); + } + break; + } + } +} + +/// \brief The queue process's event loop +/// +/// Polls for incoming messages from other processes and handles them. +void Queue::loop() +{ + assert(JobManager::instance()->process_manager().is_queue()); + ZeroMQPoller poller; + std::size_t mq_index; + std::tie(poller, mq_index) = JobManager::instance()->messenger().create_queue_poller(); + + // Before blocking SIGTERM, set the signal handler, so we can also check after blocking whether a signal occurred + // In our case, we already set it in the ProcessManager after forking to the queue and worker processes. + + sigset_t sigmask; + sigemptyset(&sigmask); + sigaddset(&sigmask, SIGTERM); + sigprocmask(SIG_BLOCK, &sigmask, &JobManager::instance()->messenger().ppoll_sigmask); + + // Before doing anything, check whether we have received a terminate signal while blocking signals! + // In this case, we also do that in the while condition. + while (!ProcessManager::sigterm_received()) { + try { // watch for zmq_error from ppoll caused by SIGTERM from master + // poll: wait until status change (-1: infinite timeout) + auto poll_result = poller.ppoll(-1, &JobManager::instance()->messenger().ppoll_sigmask); + // then process incoming messages from sockets + for (auto readable_socket : poll_result) { + // message comes from the master/queue socket (first element): + if (readable_socket.first == mq_index) { + auto message = JobManager::instance()->messenger().receive_from_master_on_queue(); + process_master_message(message); + } else { // from a worker socket + auto this_worker_id = readable_socket.first - 1; + auto message = JobManager::instance()->messenger().receive_from_worker_on_queue(this_worker_id); + process_worker_message(this_worker_id, message); + } + } + } catch (ZMQ::ppoll_error_t &e) { + zmq_ppoll_error_response response; + try { + response = handle_zmq_ppoll_error(e); + } catch (std::logic_error &) { + printf("queue loop got unhandleable ZMQ::ppoll_error_t\n"); + throw; + } + if (response == zmq_ppoll_error_response::abort) { + break; + } else if (response == zmq_ppoll_error_response::unknown_eintr) { + printf("EINTR in queue loop but no SIGTERM received, continuing\n"); + continue; + } else if (response == zmq_ppoll_error_response::retry) { + printf("EAGAIN from ppoll in queue loop, continuing\n"); + continue; + } + } catch (zmq::error_t &e) { + printf("unhandled zmq::error_t (not a ppoll_error_t) in queue loop with errno %d: %s\n", e.num(), e.what()); + throw; + } + } + + // clean up signal management modifications + sigprocmask(SIG_SETMASK, &JobManager::instance()->messenger().ppoll_sigmask, nullptr); +} + +} // namespace MultiProcess +} // namespace RooFit diff --git a/roofit/multiprocess/src/util.cxx b/roofit/multiprocess/src/util.cxx new file mode 100644 index 0000000000000..8a361ba688229 --- /dev/null +++ b/roofit/multiprocess/src/util.cxx @@ -0,0 +1,130 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#include "RooFit/MultiProcess/util.h" +#include "RooFit/MultiProcess/ProcessManager.h" + +#include // kill, SIGKILL +#include // cerr, and indirectly WNOHANG, EINTR, W* macros +#include // runtime_error +#include // waitpid +#include + +namespace RooFit { +namespace MultiProcess { + +int wait_for_child(pid_t child_pid, bool may_throw, int retries_before_killing) +{ + int status = 0; + int patience = retries_before_killing; + pid_t tmp; + do { + if (patience-- < 1) { + ::kill(child_pid, SIGKILL); + } + tmp = waitpid(child_pid, &status, WNOHANG); + } while (tmp == 0 // child has not yet changed state, try again + || (-1 == tmp && EINTR == errno) // retry on interrupted system call + ); + + if (patience < 1) { + std::cout << "Had to send PID " << child_pid << " " << (-patience + 1) << " SIGKILLs"; + } + + if (0 != status) { + if (WIFEXITED(status)) { + printf("exited, status=%d\n", WEXITSTATUS(status)); + } else if (WIFSIGNALED(status)) { + if (WTERMSIG(status) != SIGTERM) { + printf("killed by signal %d\n", WTERMSIG(status)); + } + } else if (WIFSTOPPED(status)) { + printf("stopped by signal %d\n", WSTOPSIG(status)); + } else if (WIFCONTINUED(status)) { + printf("continued\n"); + } + } + + if (-1 == tmp && may_throw) + throw std::runtime_error(std::string("waitpid, errno ") + std::to_string(errno)); + + return status; +} + +zmq_ppoll_error_response handle_zmq_ppoll_error(ZMQ::ppoll_error_t &e) +{ + if ((e.num() == EINTR) && (ProcessManager::sigterm_received())) { + // valid EINTR, because we want to exit and kill the processes on SIGTERM + return zmq_ppoll_error_response::abort; + } else if (e.num() == EINTR) { + // on other EINTRs, we retry (mostly this happens in debuggers) + return zmq_ppoll_error_response::unknown_eintr; + } else if (e.num() == EAGAIN) { + // This can happen from recv if ppoll initially gets a read-ready signal for a socket, + // but the received data does not pass the checksum test, so the socket becomes unreadable + // again or from non-blocking send if the socket becomes unwritable either due to the HWM + // being reached or the socket not being connected (anymore). The latter case usually means + // the connection has been severed from the other side, meaning it has probably been killed + // and in that case the next ppoll call will probably also receive a SIGTERM, ending the + // loop. In case something else is wrong, this message will print multiple times, which + // should be taken as a cue for writing a bug report :) + return zmq_ppoll_error_response::retry; + } else { + char buffer[512]; + snprintf(buffer, 512, + "handle_zmq_ppoll_error is out of options to handle exception, caught ZMQ::ppoll_error_t had errno %d " + "and text: %s\n", + e.num(), e.what()); + throw std::logic_error(buffer); + } +} + +// returns a tuple containing first the poll result and second a boolean flag that tells the caller whether it should +// abort the enclosing loop +std::tuple>, bool> +careful_ppoll(ZeroMQPoller &poller, const sigset_t &ppoll_sigmask, std::size_t max_tries) +{ + std::size_t tries = 0; + std::vector> poll_result; + bool abort = true; + bool carry_on = true; + while (carry_on && (tries++ < max_tries)) { + if (tries > 1) { + printf("careful_ppoll try %zu\n", tries); + } + try { // watch for zmq_error from ppoll caused by SIGTERM from master + poll_result = poller.ppoll(-1, &ppoll_sigmask); + abort = false; + carry_on = false; + } catch (ZMQ::ppoll_error_t &e) { + auto response = handle_zmq_ppoll_error(e); + if (response == zmq_ppoll_error_response::abort) { + break; + } else if (response == zmq_ppoll_error_response::unknown_eintr) { + printf("EINTR in careful_ppoll but no SIGTERM received, try %zu\n", tries); + continue; + } else if (response == zmq_ppoll_error_response::retry) { + printf("EAGAIN in careful_ppoll (from either send or receive), try %zu\n", tries); + continue; + } + } + } + + if (tries == max_tries) { + printf("careful_ppoll reached maximum number of tries, %zu, please report as a bug\n", tries); + } + return std::make_tuple(poll_result, abort); +} + +} // namespace MultiProcess +} // namespace RooFit \ No newline at end of file diff --git a/roofit/multiprocess/src/worker.cxx b/roofit/multiprocess/src/worker.cxx new file mode 100644 index 0000000000000..fb85e3078e31c --- /dev/null +++ b/roofit/multiprocess/src/worker.cxx @@ -0,0 +1,150 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#include "RooFit/MultiProcess/worker.h" + +#include "RooFit/MultiProcess/JobManager.h" +#include "RooFit/MultiProcess/types.h" +#include "RooFit/MultiProcess/Messenger.h" +#include "RooFit/MultiProcess/Job.h" +#include "RooFit/MultiProcess/util.h" + +#include // getpid, pid_t +#include // EINTR +#include // sigprocmask etc + +namespace RooFit { +namespace MultiProcess { + +static bool worker_loop_running = false; + +bool is_worker_loop_running() +{ + return worker_loop_running; +} + +/// \brief The worker processes' event loop +/// +/// Asks the queue process for tasks, polls for incoming messages from other +/// processes and handles them. +void worker_loop() +{ + assert(JobManager::instance()->process_manager().is_worker()); + worker_loop_running = true; + Q2W message_q2w; + + // use a flag to not ask twice + bool dequeue_acknowledged = true; + + ZeroMQPoller poller; + std::size_t mw_sub_index; + + std::tie(poller, mw_sub_index) = JobManager::instance()->messenger().create_worker_poller(); + + // Before blocking SIGTERM, set the signal handler, so we can also check after blocking whether a signal occurred + // In our case, we already set it in the ProcessManager after forking to the queue and worker processes. + + sigset_t sigmask; + sigemptyset(&sigmask); + sigaddset(&sigmask, SIGTERM); + sigprocmask(SIG_BLOCK, &sigmask, &JobManager::instance()->messenger().ppoll_sigmask); + + // Before doing anything, check whether we have received a terminate signal while blocking signals! + // In this case, we also do that in the while condition. + while (!ProcessManager::sigterm_received()) { + try { // watch for error from ppoll (which is called inside receive functions) caused by SIGTERM from master + + // try to dequeue a task + if (dequeue_acknowledged) { // don't ask twice + JobManager::instance()->messenger().send_from_worker_to_queue(W2Q::dequeue); + dequeue_acknowledged = false; + } + + // wait for handshake from queue or update from SUB socket + auto poll_result = poller.ppoll(-1, &JobManager::instance()->messenger().ppoll_sigmask); + // because the poller may now have a waiting update from master over the SUB socket, + // but the queue socket could be first in the poll_result vector, and during handling + // of a new task it is possible we need to already receive the updated state over SUB, + // we have to then flip this boolean so that in the for loop when we reach the SUB + // socket's result, we can skip it (otherwise we will hang there, because no more + // updated state will be coming): + bool skip_sub = false; + // then process incoming messages from sockets + for (auto readable_socket : poll_result) { + // message comes from the master-worker SUB socket (first element): + if (readable_socket.first == mw_sub_index) { + if (!skip_sub) { + auto job_id = JobManager::instance()->messenger().receive_from_master_on_worker(); + JobManager::get_job_object(job_id)->update_state(); + } + } else { // from queue socket + message_q2w = JobManager::instance()->messenger().receive_from_queue_on_worker(); + switch (message_q2w) { + case Q2W::dequeue_rejected: { + dequeue_acknowledged = true; + break; + } + case Q2W::dequeue_accepted: { + dequeue_acknowledged = true; + auto job_id = JobManager::instance()->messenger().receive_from_queue_on_worker(); + auto state_id = JobManager::instance()->messenger().receive_from_queue_on_worker(); + auto task_id = JobManager::instance()->messenger().receive_from_queue_on_worker(); + + // while loop, because multiple jobs may have updated state coming + while (state_id != JobManager::get_job_object(job_id)->get_state_id()) { + skip_sub = true; + auto job_id_for_state = + JobManager::instance()->messenger().receive_from_master_on_worker(); + JobManager::get_job_object(job_id_for_state)->update_state(); + } + + JobManager::get_job_object(job_id)->evaluate_task(task_id); + JobManager::get_job_object(job_id)->send_back_task_result_from_worker(task_id); + + break; + } + } + } + } + + } catch (ZMQ::ppoll_error_t &e) { + zmq_ppoll_error_response response; + try { + response = handle_zmq_ppoll_error(e); + } catch (std::logic_error &) { + printf("worker loop at PID %d got unhandleable ZMQ::ppoll_error_t\n", getpid()); + throw; + } + if (response == zmq_ppoll_error_response::abort) { + break; + } else if (response == zmq_ppoll_error_response::unknown_eintr) { + printf("EINTR in worker loop at PID %d but no SIGTERM received, continuing\n", getpid()); + continue; + } else if (response == zmq_ppoll_error_response::retry) { + printf("EAGAIN from ppoll in worker loop at PID %d, continuing\n", getpid()); + continue; + } + } catch (zmq::error_t &e) { + printf("unhandled zmq::error_t (not a ppoll_error_t) in worker loop at PID %d with errno %d: %s\n", getpid(), + e.num(), e.what()); + throw; + } + } + // clean up signal management modifications + sigprocmask(SIG_SETMASK, &JobManager::instance()->messenger().ppoll_sigmask, nullptr); + + worker_loop_running = false; +} + +} // namespace MultiProcess +} // namespace RooFit diff --git a/roofit/multiprocess/test/CMakeLists.txt b/roofit/multiprocess/test/CMakeLists.txt new file mode 100644 index 0000000000000..5e8392bcd06db --- /dev/null +++ b/roofit/multiprocess/test/CMakeLists.txt @@ -0,0 +1,16 @@ +# @author Patrick Bos, NL eScience Center, 2019-2021 + +add_library(RooFit_multiprocess_testing_utils INTERFACE) +target_link_libraries(RooFit_multiprocess_testing_utils INTERFACE RooFitCore RooBatchCompute) +target_include_directories(RooFit_multiprocess_testing_utils PRIVATE ${RooFitMultiProcess_INCLUDE_DIRS}) + +ROOT_ADD_GTEST(test_RooFit_MultiProcess_Job test_Job.cxx LIBRARIES RooFitMultiProcess) +# link to the INTERFACE library separately, ROOT_EXECUTABLE cannot handle INTERFACE library properties: +target_link_libraries(test_RooFit_MultiProcess_Job RooFit_multiprocess_testing_utils) +target_include_directories(test_RooFit_MultiProcess_Job PRIVATE ${RooFitMultiProcess_INCLUDE_DIRS}) + +ROOT_ADD_GTEST(test_RooFit_MultiProcess_ProcessManager test_ProcessManager.cxx LIBRARIES RooFitMultiProcess) +target_include_directories(test_RooFit_MultiProcess_ProcessManager PRIVATE ${RooFitMultiProcess_INCLUDE_DIRS}) + +ROOT_ADD_GTEST(test_RooFit_MultiProcess_Messenger test_Messenger.cxx LIBRARIES RooFitMultiProcess) +target_include_directories(test_RooFit_MultiProcess_Messenger PRIVATE ${RooFitMultiProcess_INCLUDE_DIRS}) diff --git a/roofit/multiprocess/test/test_Job.cxx b/roofit/multiprocess/test/test_Job.cxx new file mode 100644 index 0000000000000..017dd87a83562 --- /dev/null +++ b/roofit/multiprocess/test/test_Job.cxx @@ -0,0 +1,247 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * VC, Vince Croft, DIANA / NYU, vincent.croft@cern.ch + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#include +#include + +#include "RooFit/MultiProcess/Job.h" +#include "RooFit/MultiProcess/types.h" // JobTask +// needed to complete type returned from... +#include "RooFit/MultiProcess/JobManager.h" // ... Job::get_manager() +#include "RooFit/MultiProcess/ProcessManager.h" // ... JobManager::process_manager() +#include "RooFit/MultiProcess/Queue.h" // ... JobManager::queue() + +#include "gtest/gtest.h" +#include "utils.h" + +class xSquaredPlusBVectorSerial { +public: + xSquaredPlusBVectorSerial(double b, const std::vector &x) : b_(b), x_(x), result_(x.size()) {} + + void evaluate() + { + // call evaluate_task for each task + for (std::size_t ix = 0; ix < x_.size(); ++ix) { + result_[ix] = std::pow(x_[ix], 2) + b_; + } + } + + std::vector get_result() + { + evaluate(); + return result_; + } + + // for simplicity of the example (avoiding getters/setters) we make data members public as well + double b_; + std::vector x_; + std::vector result_; +}; + +class xSquaredPlusBVectorParallel : public RooFit::MultiProcess::Job { +public: + explicit xSquaredPlusBVectorParallel(xSquaredPlusBVectorSerial *serial, bool update_state = false) + : serial_(serial), update_state_(update_state) + { + } + + void evaluate() + { + if (get_manager()->process_manager().is_master()) { + if (update_state_) { + update_state(); + } + // master fills queue with tasks + for (std::size_t task_id = 0; task_id < serial_->x_.size(); ++task_id) { + RooFit::MultiProcess::JobTask job_task{id_, state_id_, task_id}; + get_manager()->queue().add(job_task); + ++N_tasks_at_workers_; + } + + // wait for task results back from workers to master + gather_worker_results(); + } + } + + std::vector get_result() + { + evaluate(); + return serial_->result_; + } + + // -- BEGIN plumbing -- + + // typically, on master, this would be called inside evaluate, before queuing tasks; on workers it's called + // automatically when a published message is received from master + void update_state() override + { + if (get_manager()->process_manager().is_master()) { + ++state_id_; + get_manager()->messenger().publish_from_master_to_workers( + id_, state_id_, serial_->b_); // always send Job id first! This is used in worker_loop to route the + // update_state call to the correct Job. + } else if (get_manager()->process_manager().is_worker()) { + state_id_ = get_manager()->messenger().receive_from_master_on_worker(); + serial_->b_ = get_manager()->messenger().receive_from_master_on_worker(); + } + } + + struct task_result_t { + std::size_t job_id; // job ID must always be the first part of any result message/type + std::size_t task_id; + double value; + }; + + void send_back_task_result_from_worker(std::size_t task) override + { + task_result_t task_result{id_, task, serial_->result_[task]}; + zmq::message_t message(sizeof(task_result_t)); + memcpy(message.data(), &task_result, sizeof(task_result_t)); + get_manager()->messenger().send_from_worker_to_master(std::move(message)); + } + + bool receive_task_result_on_master(const zmq::message_t &message) override + { + auto result = message.data(); + serial_->result_[result->task_id] = result->value; + --N_tasks_at_workers_; + bool job_completed = (N_tasks_at_workers_ == 0); + return job_completed; + } + + // -- END plumbing -- + +private: + void evaluate_task(std::size_t task) override + { + assert(get_manager()->process_manager().is_worker()); + serial_->result_[task] = std::pow(serial_->x_[task], 2) + serial_->b_; + } + + xSquaredPlusBVectorSerial *serial_; + bool update_state_ = false; + std::size_t N_tasks_at_workers_ = 0; +}; + +class TestMPJob : public ::testing::TestWithParam { + // You can implement all the usual fixture class members here. + // To access the test parameter, call GetParam() from class + // TestWithParam. +}; + +TEST_P(TestMPJob, singleJobGetResult) +{ + // Simple test case: calculate x^2 + b, where x is a vector. This case does + // both a simple calculation (squaring the input vector x) and represents + // handling of state updates in b. + std::vector x{0, 1, 2, 3}; + double b_initial = 3.; + + // start serial test + + xSquaredPlusBVectorSerial x_sq_plus_b(b_initial, x); + + auto y = x_sq_plus_b.get_result(); + std::vector y_expected{3, 4, 7, 12}; + + EXPECT_EQ(Hex(y[0]), Hex(y_expected[0])); + EXPECT_EQ(Hex(y[1]), Hex(y_expected[1])); + EXPECT_EQ(Hex(y[2]), Hex(y_expected[2])); + EXPECT_EQ(Hex(y[3]), Hex(y_expected[3])); + + std::size_t NumCPU = GetParam(); + + // start parallel test + + xSquaredPlusBVectorParallel x_sq_plus_b_parallel(&x_sq_plus_b); + RooFit::MultiProcess::JobManager::default_N_workers = NumCPU; + + auto y_parallel = x_sq_plus_b_parallel.get_result(); + + EXPECT_EQ(Hex(y_parallel[0]), Hex(y_expected[0])); + EXPECT_EQ(Hex(y_parallel[1]), Hex(y_expected[1])); + EXPECT_EQ(Hex(y_parallel[2]), Hex(y_expected[2])); + EXPECT_EQ(Hex(y_parallel[3]), Hex(y_expected[3])); +} + +TEST_P(TestMPJob, multiJobGetResult) +{ + // Simple test case: calculate x^2 + b, where x is a vector. This case does + // both a simple calculation (squaring the input vector x) and represents + // handling of state updates in b. + std::vector x{0, 1, 2, 3}; + double b_initial = 3.; + + std::vector y_expected{3, 4, 7, 12}; + + std::size_t NumCPU = GetParam(); + + // define jobs + xSquaredPlusBVectorSerial x_sq_plus_b(b_initial, x); + xSquaredPlusBVectorSerial x_sq_plus_b2(b_initial + 1, x); + xSquaredPlusBVectorParallel x_sq_plus_b_parallel(&x_sq_plus_b); + xSquaredPlusBVectorParallel x_sq_plus_b_parallel2(&x_sq_plus_b2); + RooFit::MultiProcess::JobManager::default_N_workers = NumCPU; + + // do stuff + auto y_parallel = x_sq_plus_b_parallel.get_result(); + auto y_parallel2 = x_sq_plus_b_parallel2.get_result(); + + EXPECT_EQ(Hex(y_parallel[0]), Hex(y_expected[0])); + EXPECT_EQ(Hex(y_parallel[1]), Hex(y_expected[1])); + EXPECT_EQ(Hex(y_parallel[2]), Hex(y_expected[2])); + EXPECT_EQ(Hex(y_parallel[3]), Hex(y_expected[3])); + + EXPECT_EQ(Hex(y_parallel2[0]), Hex(y_expected[0] + 1)); + EXPECT_EQ(Hex(y_parallel2[1]), Hex(y_expected[1] + 1)); + EXPECT_EQ(Hex(y_parallel2[2]), Hex(y_expected[2] + 1)); + EXPECT_EQ(Hex(y_parallel2[3]), Hex(y_expected[3] + 1)); +} + +TEST_P(TestMPJob, singleJobUpdateState) +{ + // Simple test case: calculate x^2 + b, where x is a vector. This case does + // both a simple calculation (squaring the input vector x) and represents + // handling of state updates in b. + std::vector x{0, 1, 2, 3}; + double b_initial = 1.; + xSquaredPlusBVectorSerial x_sq_plus_b(b_initial, x); + + std::vector y_expected{3, 4, 7, 12}; + + std::size_t NumCPU = GetParam(); + + // start parallel test + bool update_state = true; + xSquaredPlusBVectorParallel x_sq_plus_b_parallel(&x_sq_plus_b, update_state); + RooFit::MultiProcess::JobManager::default_N_workers = NumCPU; + + auto y_parallel_before_change = x_sq_plus_b_parallel.get_result(); + + x_sq_plus_b.b_ = 3.; + + auto y_parallel_after_change = x_sq_plus_b_parallel.get_result(); + + EXPECT_NE(Hex(y_parallel_before_change[0]), Hex(y_expected[0])); + EXPECT_NE(Hex(y_parallel_before_change[1]), Hex(y_expected[1])); + EXPECT_NE(Hex(y_parallel_before_change[2]), Hex(y_expected[2])); + EXPECT_NE(Hex(y_parallel_before_change[3]), Hex(y_expected[3])); + + EXPECT_EQ(Hex(y_parallel_after_change[0]), Hex(y_expected[0])); + EXPECT_EQ(Hex(y_parallel_after_change[1]), Hex(y_expected[1])); + EXPECT_EQ(Hex(y_parallel_after_change[2]), Hex(y_expected[2])); + EXPECT_EQ(Hex(y_parallel_after_change[3]), Hex(y_expected[3])); +} + +INSTANTIATE_TEST_SUITE_P(NumberOfWorkerProcesses, TestMPJob, ::testing::Values(1, 2, 3)); diff --git a/roofit/multiprocess/test/test_Messenger.cxx b/roofit/multiprocess/test/test_Messenger.cxx new file mode 100644 index 0000000000000..0db05100328c6 --- /dev/null +++ b/roofit/multiprocess/test/test_Messenger.cxx @@ -0,0 +1,238 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#include "RooFit/MultiProcess/Messenger.h" +#include "RooFit/MultiProcess/ProcessManager.h" // ... JobManager::process_manager() + +#include "gtest/gtest.h" + +void handle_sigchld(int signum) +{ + printf("handled %s on PID %d\n", strsignal(signum), getpid()); +} + +TEST(TestMPMessenger, Connections) +{ + struct sigaction sa; + RooFit::MultiProcess::ProcessManager pm(4); + if (pm.is_master()) { + // on master, we have to handle SIGCHLD + memset(&sa, '\0', sizeof(sa)); + sa.sa_handler = handle_sigchld; + if (sigaction(SIGCHLD, &sa, NULL) < 0) { + std::perror("sigaction failed"); + std::exit(1); + } + } + RooFit::MultiProcess::Messenger messenger(pm); + if (pm.is_master()) { + // more SIGCHLD handling + sigset_t sigmask; + sigemptyset(&sigmask); + sigaddset(&sigmask, SIGCHLD); + int rc = sigprocmask(SIG_BLOCK, &sigmask, &messenger.ppoll_sigmask); + if (rc < 0) { + throw std::runtime_error("sigprocmask failed in TestMPMessenger.Connections"); + } + } + messenger.test_connections(pm); + if (pm.is_master()) { + // clean up signal management modifications + sigprocmask(SIG_SETMASK, &messenger.ppoll_sigmask, nullptr); + sa.sa_handler = SIG_DFL; + if (sigaction(SIGCHLD, &sa, NULL) < 0) { + std::perror("sigaction failed"); + std::exit(1); + } + } +} + +TEST(TestMPMessenger, ConnectionsManualExit) +{ + // the point of this test is to see whether clean-up of ZeroMQ resources is done properly without calling any + // destructors (which is what happens when you call _Exit() instead of regularly ending the program by reaching the + // end of main()). + + struct sigaction sa; + + RooFit::MultiProcess::ProcessManager pm(4); + if (pm.is_master()) { + // on master, we have to handle SIGCHLD + memset(&sa, '\0', sizeof(sa)); + sa.sa_handler = handle_sigchld; + if (sigaction(SIGCHLD, &sa, NULL) < 0) { + std::perror("sigaction failed"); + std::exit(1); + } + } + RooFit::MultiProcess::Messenger messenger(pm); + if (pm.is_master()) { + // more SIGCHLD handling + sigset_t sigmask; + sigemptyset(&sigmask); + sigaddset(&sigmask, SIGCHLD); + int rc = sigprocmask(SIG_BLOCK, &sigmask, &messenger.ppoll_sigmask); + if (rc < 0) { + throw std::runtime_error("sigprocmask failed in TestMPMessenger.Connections"); + } + } + messenger.test_connections(pm); + if (!pm.is_master()) { + // just wait until we get terminated + while (!RooFit::MultiProcess::ProcessManager::sigterm_received()) { + }; + std::_Exit(0); + } else { + pm.terminate(); + } + if (pm.is_master()) { + // clean up signal management modifications + sigprocmask(SIG_SETMASK, &messenger.ppoll_sigmask, nullptr); + sa.sa_handler = SIG_DFL; + if (sigaction(SIGCHLD, &sa, NULL) < 0) { + std::perror("sigaction failed"); + std::exit(1); + } + } +} + +#include + +TEST(TestMPMessenger, SigStop) +{ + // Originally, we could not get the debugger (lldb) to play nicely with ZeroMQ; every time we paused a process, it + // said it crashed because of SIGSTOP, but SIGSTOP should just pause it... this test was meant to reproduce this + // behavior and fix it, which was done by more carefully handling exceptions. + RooFit::MultiProcess::ProcessManager pm(2); + RooFit::MultiProcess::Messenger messenger(pm); + + if (pm.is_master()) { + printf("first send message to queue...\n"); + messenger.send_from_master_to_queue(1); + printf("sleep for 2 seconds...\n"); + sleep(2); + printf("SIGSTOPping all children...\n"); + kill(pm.get_queue_pid(), SIGSTOP); + kill(pm.get_worker_pids()[0], SIGSTOP); + kill(pm.get_worker_pids()[1], SIGSTOP); + printf("send another message to queue...\n"); + messenger.send_from_master_to_queue(2); + printf("sleep for 2 seconds...\n"); + sleep(2); + printf("SIGCONT queue and worker 1...\n"); + kill(pm.get_queue_pid(), SIGCONT); + kill(pm.get_worker_pids()[0], SIGCONT); + printf("sleep for 2 seconds...\n"); + sleep(2); + printf("SIGCONT worker 2...\n"); + kill(pm.get_worker_pids()[1], SIGCONT); + EXPECT_EQ(messenger.receive_from_queue_on_master(), 3); + } + + if (pm.is_queue()) { + auto number = messenger.receive_from_master_on_queue(); + printf("received %d on queue\n", number); + number = messenger.receive_from_master_on_queue(); + printf("received %d on queue\n", number); + messenger.send_from_queue_to_master(3); + } + + if (!pm.is_master()) { + while (!pm.sigterm_received()) { + } + } +} + +TEST(TestMPMessenger, DISABLED_StressSigStop) +{ + // The SIGSTOP test failed spuriously on CI at some point. We suspected this was due to some + // improbable race condition caused in some place where SIGSTOP/SIGCONT crashes a process. + // To find this crash, we bombard the processes with signals in this test. + // We were not able to trigger the crash, so we disabled the test, but leave it in for when + // the spurious test resurfaces. + RooFit::MultiProcess::ProcessManager pm(2); + RooFit::MultiProcess::Messenger messenger(pm); + + std::size_t runs = 100; + std::size_t sig_repeats = 10000; + + if (pm.is_master()) { + for (std::size_t ix = 0; ix < runs; ++ix) { + // printf("first send message to queue...\n"); + messenger.send_from_master_to_queue(1); + // printf("SIGSTOPping all children...\n"); + for (std::size_t jx = 0; jx < sig_repeats; ++jx) { + kill(pm.get_queue_pid(), SIGCONT); + kill(pm.get_worker_pids()[0], SIGCONT); + kill(pm.get_worker_pids()[1], SIGCONT); + kill(pm.get_queue_pid(), SIGSTOP); + kill(pm.get_worker_pids()[0], SIGSTOP); + kill(pm.get_worker_pids()[1], SIGSTOP); + } + // printf("send another message to queue...\n"); + messenger.send_from_master_to_queue(2); + + for (std::size_t jx = 0; jx < sig_repeats; ++jx) { + kill(pm.get_queue_pid(), SIGSTOP); + kill(pm.get_worker_pids()[0], SIGSTOP); + kill(pm.get_worker_pids()[1], SIGSTOP); + kill(pm.get_queue_pid(), SIGCONT); + kill(pm.get_worker_pids()[0], SIGCONT); + kill(pm.get_worker_pids()[1], SIGCONT); + } + EXPECT_EQ(messenger.receive_from_queue_on_master(), 3); + } + } + + if (pm.is_queue()) { + for (std::size_t ix = 0; ix < runs; ++ix) { + /*auto number =*/messenger.receive_from_master_on_queue(); + // printf("received %d on queue\n", number); + /*number =*/messenger.receive_from_master_on_queue(); + // printf("received %d on queue\n", number); + messenger.send_from_queue_to_master(3); + } + } + + if (!pm.is_master()) { + while (!pm.sigterm_received()) { + } + } +} + +// This test has been disabled in favor of the pub-sub connection check in the Messenger ctor, which is more robust +TEST(TestMPMessenger, DISABLED_StressPubSub) +{ + std::size_t N_workers = 64; + RooFit::MultiProcess::ProcessManager pm(N_workers); + RooFit::MultiProcess::Messenger messenger(pm); + + if (pm.is_master()) { + messenger.publish_from_master_to_workers("bert"); + std::size_t ernies = 0; + for (; ernies < N_workers; ++ernies) { + auto receipt = messenger.receive_from_worker_on_master(); + if (receipt != "ernie") { + printf("whoops, got %s instead of ernie!\n", receipt.c_str()); + FAIL(); + } + } + EXPECT_EQ(ernies, N_workers); + } else if (pm.is_worker()) { + auto receipt = messenger.receive_from_master_on_worker(); + if (receipt == "bert") { + messenger.send_from_worker_to_master("ernie"); + } else { + printf("no bert on worker %zu\n", pm.worker_id()); + } + } +} diff --git a/roofit/multiprocess/test/test_ProcessManager.cxx b/roofit/multiprocess/test/test_ProcessManager.cxx new file mode 100644 index 0000000000000..7bf34ffa691b3 --- /dev/null +++ b/roofit/multiprocess/test/test_ProcessManager.cxx @@ -0,0 +1,54 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#include "RooFit/MultiProcess/ProcessManager.h" // ... JobManager::process_manager() + +#include "gtest/gtest.h" + +TEST(TestMPProcessManager, birthAndDeath) +{ + auto pm = RooFit::MultiProcess::ProcessManager(2); +} + +// disabled: it is not working anymore (16 March 2021), due to changes introduced to fix Messenger::test_connections +TEST(TestMPProcessManager, DISABLED_multiBirth) +{ + // This test doesn't actually represent a current usecase in RooFit, but let's + // showcase the possibility anyway for future reference. + + // first create a regular pm like in the birthAndDeath test + auto pm = RooFit::MultiProcess::ProcessManager(2); + // then from each forked node, spin up another set of workers+queue! + auto pm2 = RooFit::MultiProcess::ProcessManager(2); +} + +TEST(TestMPProcessManager, checkState) +{ + std::size_t N_workers = 2; + auto master_pid = getpid(); + auto pm = RooFit::MultiProcess::ProcessManager(N_workers); + ASSERT_TRUE(pm.is_initialized()); + ASSERT_EQ(pm.N_workers(), N_workers); + + if (pm.is_master()) { + EXPECT_EQ(getpid(), master_pid); + EXPECT_FALSE(pm.is_queue()); + EXPECT_FALSE(pm.is_worker()); + } else if (pm.is_queue()) { + EXPECT_FALSE(pm.is_master()); + EXPECT_FALSE(pm.is_worker()); + } else if (pm.is_worker()) { + EXPECT_FALSE(pm.is_master()); + EXPECT_FALSE(pm.is_queue()); + EXPECT_TRUE(pm.worker_id() == 0 || pm.worker_id() == 1); + } +} \ No newline at end of file diff --git a/roofit/multiprocess/test/test_worker.cxx b/roofit/multiprocess/test/test_worker.cxx new file mode 100644 index 0000000000000..0b2dd54027572 --- /dev/null +++ b/roofit/multiprocess/test/test_worker.cxx @@ -0,0 +1,27 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#include "RooFit/MultiProcess/worker.h" + +#include "gtest/gtest.h" + +// TODO: implement something along these lines (the following is pseudocode) +// TEST(worker, loop) +//{ +// do_fork(); +// if (on_master()) { +// wait(); +// terminate_worker(); +// } else if (on_worker()) { +// worker_loop(); +// } +//} diff --git a/roofit/multiprocess/test/utils.h b/roofit/multiprocess/test/utils.h new file mode 100644 index 0000000000000..431b7c98b6499 --- /dev/null +++ b/roofit/multiprocess/test/utils.h @@ -0,0 +1,169 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * VC, Vince Croft, DIANA / NYU, vincent.croft@cern.ch + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#ifndef ROOT_ROOFIT_MultiProcess_tests_utils_h +#define ROOT_ROOFIT_MultiProcess_tests_utils_h + +#include + +#include "RooWorkspace.h" +#include "RooRandom.h" +#include "RooAddPdf.h" +#include "RooDataSet.h" +#include "RooRealVar.h" // for the dynamic cast to have a complete type + +std::tuple, std::unique_ptr> +generate_1D_gaussian_pdf_nll(RooWorkspace &w, unsigned long N_events) +{ + w.factory("Gaussian::g(x[-5,5],mu[0,-3,3],sigma[1])"); + + RooAbsPdf *pdf = w.pdf("g"); + RooRealVar *mu = w.var("mu"); + + RooDataSet *data = pdf->generate(RooArgSet(*w.var("x")), N_events); + mu->setVal(-2.9); + + std::unique_ptr nll{pdf->createNLL(*data)}; + + // save initial values for the start of all minimizations + std::unique_ptr values = std::make_unique(*mu, *pdf, *nll, "values"); + + return std::make_tuple(std::move(nll), std::move(values)); +} + +// return two unique_ptrs, the first because nll is a pointer, +// the second because RooArgSet doesn't have a move ctor +std::tuple, std::unique_ptr> +generate_ND_gaussian_pdf_nll(RooWorkspace &w, unsigned int n, unsigned long N_events) +{ + RooArgSet obs_set; + + // create gaussian parameters + double mean[n], sigma[n]; + for (unsigned ix = 0; ix < n; ++ix) { + mean[ix] = RooRandom::randomGenerator()->Gaus(0, 2); + sigma[ix] = 0.1 + abs(RooRandom::randomGenerator()->Gaus(0, 2)); + } + + // create gaussians and also the observables and parameters they depend on + for (unsigned ix = 0; ix < n; ++ix) { + std::ostringstream os; + os << "Gaussian::g" << ix << "(x" << ix << "[-10,10]," + << "m" << ix << "[" << mean[ix] << ",-10,10]," + << "s" << ix << "[" << sigma[ix] << ",0.1,10])"; + w.factory(os.str().c_str()); + } + + // create uniform background signals on each observable + for (unsigned ix = 0; ix < n; ++ix) { + { + std::ostringstream os; + os << "Uniform::u" << ix << "(x" << ix << ")"; + w.factory(os.str().c_str()); + } + + // gather the observables in a list for data generation below + { + std::ostringstream os; + os << "x" << ix; + obs_set.add(*w.arg(os.str().c_str())); + } + } + + RooArgSet pdf_set = w.allPdfs(); + + // create event counts for all pdfs + RooArgSet count_set; + + // ... for the gaussians + for (unsigned ix = 0; ix < n; ++ix) { + std::stringstream os, os2; + os << "Nsig" << ix; + os2 << "#signal events comp " << ix; + RooRealVar a(os.str().c_str(), os2.str().c_str(), N_events / 10, 0., 10 * N_events); + w.import(a); + // gather in count_set + count_set.add(*w.arg(os.str().c_str())); + } + // ... and for the uniform background components + for (unsigned ix = 0; ix < n; ++ix) { + std::stringstream os, os2; + os << "Nbkg" << ix; + os2 << "#background events comp " << ix; + RooRealVar a(os.str().c_str(), os2.str().c_str(), N_events / 10, 0., 10 * N_events); + w.import(a); + // gather in count_set + count_set.add(*w.arg(os.str().c_str())); + } + + RooAddPdf *sum = new RooAddPdf("sum", "gaussians+uniforms", pdf_set, count_set); + w.import(*sum); // keep sum around after returning + + // --- Generate a toyMC sample from composite PDF --- + RooDataSet *data = sum->generate(obs_set, N_events); + + std::unique_ptr nll{sum->createNLL(*data)}; + + // set values randomly so that they actually need to do some fitting + for (unsigned ix = 0; ix < n; ++ix) { + { + std::ostringstream os; + os << "m" << ix; + dynamic_cast(w.arg(os.str().c_str()))->setVal(RooRandom::randomGenerator()->Gaus(0, 2)); + } + { + std::ostringstream os; + os << "s" << ix; + dynamic_cast(w.arg(os.str().c_str())) + ->setVal(0.1 + abs(RooRandom::randomGenerator()->Gaus(0, 2))); + } + } + + // gather all values of parameters, pdfs and nll here for easy + // saving and restoring + std::unique_ptr all_values = std::make_unique(pdf_set, count_set, "all_values"); + all_values->add(*nll); + all_values->add(*sum); + for (unsigned ix = 0; ix < n; ++ix) { + { + std::ostringstream os; + os << "m" << ix; + all_values->add(*w.arg(os.str().c_str())); + } + { + std::ostringstream os; + os << "s" << ix; + all_values->add(*w.arg(os.str().c_str())); + } + } + + return std::make_tuple(std::move(nll), std::move(all_values)); +} + +class Hex { +public: + explicit Hex(double n) : number_(n) {} + operator double() const { return number_; } + bool operator==(const Hex &other) { return double(*this) == double(other); } + +private: + double number_; +}; + +::std::ostream &operator<<(::std::ostream &os, const Hex &hex) +{ + return os << std::hexfloat << double(hex) << std::defaultfloat; // whatever needed to print bar to os +} + +#endif // ROOT_ROOFIT_MultiProcess_tests_utils_h