diff --git a/.ci/copy_headers.sh b/.ci/copy_headers.sh index fd4ec82b96e9d..a6587b6ee1dbb 100755 --- a/.ci/copy_headers.sh +++ b/.ci/copy_headers.sh @@ -10,6 +10,6 @@ cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -Dall=On -Dtesting=On -Dx11=Off -Dalien # We need to prebuild a minimal set of targets which are responsible for header copy # or generation. make -j4 move_headers intrinsics_gen clang-tablegen-targets ClangDriverOptions \ - googletest Dictgen BaseTROOT + googletest Dictgen BaseTROOT BUILTIN_cppzmq ln -s $PWD/compile_commands.json $PWD/../root/ diff --git a/builtins/zeromq/cppzmq/CMakeLists.txt b/builtins/zeromq/cppzmq/CMakeLists.txt new file mode 100644 index 0000000000000..186de3872d857 --- /dev/null +++ b/builtins/zeromq/cppzmq/CMakeLists.txt @@ -0,0 +1,35 @@ +include(ExternalProject) + +ExternalProject_Add(BUILTIN_cppzmq + URL https://github.com/zeromq/cppzmq/archive/refs/tags/v4.8.1.tar.gz + URL_HASH SHA256=7a23639a45f3a0049e11a188e29aaedd10b2f4845f0000cf3e22d6774ebde0af + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + DEPENDS libzmq +) + +# manually copy header (inspired by nlohmann builtin) +set(cppzmq_HEADER_PATH ${CMAKE_CURRENT_BINARY_DIR}/BUILTIN_cppzmq-prefix/src/BUILTIN_cppzmq) +add_custom_command( + OUTPUT ${CMAKE_BINARY_DIR}/include/zmq.hpp + COMMAND ${CMAKE_COMMAND} -E copy ${cppzmq_HEADER_PATH}/zmq.hpp ${CMAKE_BINARY_DIR}/include/zmq.hpp + COMMENT "Copying zmq.hpp header to ${CMAKE_BINARY_DIR}/include" + DEPENDS BUILTIN_cppzmq) + +add_custom_target(builtin_cppzmq_incl + DEPENDS ${CMAKE_BINARY_DIR}/include/zmq.hpp) + +set_property(GLOBAL APPEND PROPERTY ROOT_HEADER_TARGETS builtin_cppzmq_incl) + +install(FILES ${cppzmq_HEADER_PATH}/zmq.hpp + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/) + +# find_package emulation +set(cppzmq_FOUND TRUE CACHE BOOL "" FORCE) +set(cppzmq_INCLUDE_DIR ${CMAKE_BINARY_DIR}/include CACHE INTERNAL "" FORCE) +set(cppzmq_INCLUDE_DIRS ${cppzmq_INCLUDE_DIR} CACHE INTERNAL "" FORCE) + +add_library(cppzmq INTERFACE IMPORTED GLOBAL) +target_include_directories(cppzmq INTERFACE $) +add_dependencies(cppzmq BUILTIN_cppzmq builtin_cppzmq_incl) diff --git a/builtins/zeromq/libzmq/CMakeLists.txt b/builtins/zeromq/libzmq/CMakeLists.txt new file mode 100644 index 0000000000000..9ad57d202c070 --- /dev/null +++ b/builtins/zeromq/libzmq/CMakeLists.txt @@ -0,0 +1,47 @@ +include(ExternalProject) + +set(ZeroMQ_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/BUILTIN_ZeroMQ-prefix) +set(ZeroMQ_LIBNAME ${CMAKE_STATIC_LIBRARY_PREFIX}zmq${CMAKE_STATIC_LIBRARY_SUFFIX}) + +set(ZeroMQ_LIBRARY ${ZeroMQ_PREFIX}/lib/${ZeroMQ_LIBNAME} CACHE INTERNAL "" FORCE) +set(ZeroMQ_LIBRARIES ${ZeroMQ_LIBRARY} CACHE INTERNAL "" FORCE) + +if(NOT WIN32) + # 2021-10-07: libzmq's minimum CMake version does not yet support CXX_VISIBILITY_PRESET, so we need to pass the flag manually + set(cxx_visibility_flag "-fvisibility=hidden") +else() + set(cxx_visibility_flag "") +endif() + +ExternalProject_Add(BUILTIN_ZeroMQ + URL https://github.com/zeromq/libzmq/archive/7c2df78b49a3aa63e654b3f3526adf71ed091534.tar.gz + URL_HASH SHA256=fcc1b0648afa5d92e0ff0e6e93beb28cbbe008a5f98c228ff97144ba6e4a6c3e + CMAKE_ARGS + -DCMAKE_INSTALL_PREFIX= + -DCMAKE_INSTALL_LIBDIR=lib + -DWITH_PERF_TOOL=OFF + -DBUILD_TESTS=OFF + -DENABLE_DRAFTS=ON + -DENABLE_NO_EXPORT=ON + -DENABLE_CURVE=OFF + -DWITH_DOCS=OFF + -DENABLE_WS=OFF + -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} + -DCMAKE_CXX_FLAGS=${cxx_visibility_flag}\ ${CMAKE_CXX_FLAGS}\ -w + BUILD_BYPRODUCTS ${ZeroMQ_LIBRARIES} + ) + +set(ZeroMQ_FOUND TRUE CACHE BOOL "" FORCE) + +set(ZeroMQ_INCLUDE_DIR ${ZeroMQ_PREFIX}/include CACHE INTERNAL "" FORCE) +set(ZeroMQ_INCLUDE_DIRS ${ZeroMQ_PREFIX}/include CACHE INTERNAL "" FORCE) + +# Workaround to propagate INTERFACE_INCLUDE_DIRECTORIES, see https://stackoverflow.com/a/47358004/1199693 +file(MAKE_DIRECTORY ${ZeroMQ_INCLUDE_DIR}) + +add_library(libzmq INTERFACE IMPORTED GLOBAL) +target_include_directories(libzmq INTERFACE $) +target_link_libraries(libzmq INTERFACE $) +add_dependencies(libzmq BUILTIN_ZeroMQ) + +set(ZeroMQ_DIR ${ZeroMQ_PREFIX}/share/cmake/ZeroMQ CACHE INTERNAL "" FORCE) diff --git a/cmake/modules/FindZeroMQ.cmake b/cmake/modules/FindZeroMQ.cmake new file mode 100644 index 0000000000000..24a1c171f554e --- /dev/null +++ b/cmake/modules/FindZeroMQ.cmake @@ -0,0 +1,60 @@ +# Based on https://github.com/zeromq/cppzmq/blob/a98fa4a91d868a3844e5456741d6782cc1a8d98b/libzmq-pkg-config/FindZeroMQ.cmake +# MIT Licensed: +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +set(PKG_CONFIG_USE_CMAKE_PREFIX_PATH ON) +find_package(PkgConfig) +pkg_check_modules(PC_LIBZMQ QUIET libzmq) + +set(ZeroMQ_VERSION ${PC_LIBZMQ_VERSION}) + +find_path(ZeroMQ_INCLUDE_DIR zmq.h + PATHS ${ZeroMQ_DIR}/include + ${PC_LIBZMQ_INCLUDE_DIRS}) + +find_library(ZeroMQ_LIBRARY + NAMES zmq + PATHS ${ZeroMQ_DIR}/lib + ${PC_LIBZMQ_LIBDIR} + ${PC_LIBZMQ_LIBRARY_DIRS}) + +set ( ZeroMQ_LIBRARIES ${ZeroMQ_LIBRARY} ) +set ( ZeroMQ_INCLUDE_DIRS ${ZeroMQ_INCLUDE_DIR} ) + +# check for zmq_ppoll +if(ZeroMQ_FOUND) + SET(SAVE_CMAKE_REQUIRED_DEFINITIONS "${CMAKE_REQUIRED_DEFINITIONS}") + SET(CMAKE_REQUIRED_DEFINITIONS "-DZMQ_BUILD_DRAFT_API") + check_cxx_symbol_exists(zmq_ppoll zmq.h ZeroMQ_HAS_PPOLL) + SET(CMAKE_REQUIRED_DEFINITIONS "${SAVE_CMAKE_REQUIRED_DEFINITIONS}") +endif() + +include ( FindPackageHandleStandardArgs ) +# handle the QUIETLY and REQUIRED arguments and set ZeroMQ_FOUND to TRUE +# if all listed variables are TRUE +find_package_handle_standard_args ( ZeroMQ DEFAULT_MSG ZeroMQ_LIBRARIES ZeroMQ_INCLUDE_DIRS ZeroMQ_HAS_PPOLL) + +if(ZeroMQ_FOUND) + if(NOT TARGET libzmq) + add_library(libzmq UNKNOWN IMPORTED) + set_target_properties(libzmq PROPERTIES + IMPORTED_LOCATION ${ZeroMQ_LIBRARIES} + INTERFACE_INCLUDE_DIRECTORIES ${ZeroMQ_INCLUDE_DIRS}) + endif() +endif() \ No newline at end of file diff --git a/cmake/modules/Findcppzmq.cmake b/cmake/modules/Findcppzmq.cmake new file mode 100644 index 0000000000000..8b6d5f6cddae6 --- /dev/null +++ b/cmake/modules/Findcppzmq.cmake @@ -0,0 +1,21 @@ +if (NOT (ZeroMQ_FOUND OR TARGET libzmq)) + message(FATAL_ERROR "Search for libzmq first!") +endif() + +find_path(cppzmq_INCLUDE_DIRS "zmq.hpp" + HINTS "${ZeroMQ_INCLUDE_DIR}" + PATH_SUFFIXES "include" "cppzmq" + ) +mark_as_advanced(cppzmq_INCLUDE_DIRS) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args (cppzmq DEFAULT_MSG cppzmq_INCLUDE_DIRS) + +if(cppzmq_FOUND) + add_library(cppzmq INTERFACE IMPORTED) + set_target_properties(cppzmq PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${CPPZMQ_INCLUDE_DIRS}" + INTERFACE_LINK_LIBRARIES libzmq + ) + set(CPPZMQ_LIBRARIES cppzmq) +endif() \ No newline at end of file diff --git a/cmake/modules/RootBuildOptions.cmake b/cmake/modules/RootBuildOptions.cmake index 7c864b93289a6..0baea571b3aff 100644 --- a/cmake/modules/RootBuildOptions.cmake +++ b/cmake/modules/RootBuildOptions.cmake @@ -86,6 +86,7 @@ ROOT_BUILD_OPTION(builtin_cfitsio OFF "Build CFITSIO internally (requires networ ROOT_BUILD_OPTION(builtin_clang ON "Build bundled copy of Clang") ROOT_BUILD_OPTION(builtin_cling ON "Build bundled copy of Cling. Only build with an external cling if you know what you are doing: associating ROOT commits with cling commits is tricky.") MARK_AS_ADVANCED(builtin_cling) +ROOT_BUILD_OPTION(builtin_cppzmq OFF "Use ZeroMQ C++ bindings installed by ROOT (requires network)") ROOT_BUILD_OPTION(builtin_davix OFF "Build Davix internally (requires network)") ROOT_BUILD_OPTION(builtin_fftw3 OFF "Build FFTW3 internally (requires network)") ROOT_BUILD_OPTION(builtin_freetype OFF "Build bundled copy of freetype") @@ -107,6 +108,7 @@ ROOT_BUILD_OPTION(builtin_vdt OFF "Build VDT internally (requires network)") ROOT_BUILD_OPTION(builtin_veccore OFF "Build VecCore internally (requires network)") ROOT_BUILD_OPTION(builtin_xrootd OFF "Build XRootD internally (requires network)") ROOT_BUILD_OPTION(builtin_xxhash OFF "Build bundled copy of xxHash") +ROOT_BUILD_OPTION(builtin_zeromq OFF "Build ZeroMQ internally (requires network)") ROOT_BUILD_OPTION(builtin_zlib OFF "Build bundled copy of zlib") ROOT_BUILD_OPTION(builtin_zstd OFF "Build included libzstd, or use system libzstd") ROOT_BUILD_OPTION(ccache OFF "Enable ccache usage for speeding up builds") @@ -160,6 +162,7 @@ ROOT_BUILD_OPTION(qt5web OFF "Enable support for Qt5 web-based display (requires ROOT_BUILD_OPTION(qt6web OFF "Enable support for Qt6 web-based display (requires Qt6::WebEngineCore and Qt6::WebEngineWidgets)") ROOT_BUILD_OPTION(r OFF "Enable support for R bindings (requires R, Rcpp, and RInside)") ROOT_BUILD_OPTION(roofit ON "Build the advanced fitting package RooFit, and RooStats for statistical tests. If xml is available, also build HistFactory.") +ROOT_BUILD_OPTION(roofit_multiprocess ON "Build RooFit::MultiProcess and multi-process RooFit::TestStatistics classes (requires ZeroMQ with zmq_ppoll and cppzmq).") ROOT_BUILD_OPTION(webgui ON "Build Web-based UI components of ROOT (requires C++17 standard or higher)") ROOT_BUILD_OPTION(root7 ON "Build ROOT 7 components of ROOT (requires C++17 standard or higher)") ROOT_BUILD_OPTION(rpath ON "Link libraries with built-in RPATH (run-time search path)") @@ -253,6 +256,7 @@ if(all) set(qt5web_defvalue ON) set(r_defvalue ON) set(roofit_defvalue ON) + set(roofit_multiprocess_defvalue ON) set(webgui_defvalue ON) set(root7_defvalue ON) set(shadowpw_defvalue ON) @@ -280,6 +284,7 @@ if(builtin_all) set(builtin_cfitsio_defvalue ON) set(builtin_clang_defvalue ON) set(builtin_cling_defvalue ON) + set(builtin_cppzmq_defvalue ON) set(builtin_davix_defvalue ON) set(builtin_fftw3_defvalue ON) set(builtin_freetype_defvalue ON) @@ -301,6 +306,7 @@ if(builtin_all) set(builtin_veccore_defvalue ON) set(builtin_xrootd_defvalue ON) set(builtin_xxhash_defvalue ON) + set(builtin_zeromq_defvalue ON) set(builtin_zlib_defvalue ON) set(builtin_zstd_defvalue ON) endif() diff --git a/cmake/modules/SearchInstalledSoftware.cmake b/cmake/modules/SearchInstalledSoftware.cmake index 19c3339390f37..95b7a397a6066 100644 --- a/cmake/modules/SearchInstalledSoftware.cmake +++ b/cmake/modules/SearchInstalledSoftware.cmake @@ -1862,6 +1862,75 @@ if(testing AND NO_CONNECTION) endif() endif() +#---Check for ZeroMQ when building RooFit::MultiProcess-------------------------------------------- + +if (roofit_multiprocess AND NOT MSVC) + if(NOT builtin_zeromq) + message(STATUS "Looking for ZeroMQ (libzmq)") + # Clear cache before calling find_package(ZeroMQ), + # necessary to be able to toggle builtin_zeromq and + # not have find_package(ZeroMQ) find builtin ZeroMQ. + foreach(suffix FOUND INCLUDE_DIR INCLUDE_DIRS LIBRARY LIBRARIES) + unset(ZeroMQ_${suffix} CACHE) + endforeach() + + # Temporarily prefer config mode over module mode, so that a CMake-installed system version + # gets detected before looking for an autotools-installed system version (which the + # FindZeroMQ.cmake module does). + set(CMAKE_FIND_PACKAGE_PREFER_CONFIG_ORIGINAL_VALUE ${CMAKE_FIND_PACKAGE_PREFER_CONFIG}) + set(CMAKE_FIND_PACKAGE_PREFER_CONFIG TRUE) + + if(fail-on-missing) + find_package(ZeroMQ REQUIRED) + else() + find_package(ZeroMQ) + if(NOT ZeroMQ_FOUND) + message(STATUS "ZeroMQ not found. Switching on builtin_zeromq option") + set(builtin_zeromq ON CACHE BOOL "Enabled because ZeroMQ not found (${builtin_zeromq_description})" FORCE) + endif() + endif() + + # Reset default find_package mode + set(CMAKE_FIND_PACKAGE_PREFER_CONFIG ${CMAKE_FIND_PACKAGE_PREFER_CONFIG_ORIGINAL_VALUE}) + unset(CMAKE_FIND_PACKAGE_PREFER_CONFIG_ORIGINAL_VALUE) + endif() + + if(builtin_zeromq) + list(APPEND ROOT_BUILTINS ZeroMQ) + add_subdirectory(builtins/zeromq/libzmq) + endif() + + if(NOT builtin_cppzmq) + message(STATUS "Looking for ZeroMQ C++ bindings (cppzmq)") + # Clear cache before calling find_package(cppzmq), + # necessary to be able to toggle builtin_cppzmq and + # not have find_package(cppzmq) find builtin cppzmq. + foreach(suffix FOUND INCLUDE_DIR INCLUDE_DIRS) + unset(cppzmq_${suffix} CACHE) + endforeach() + if(fail-on-missing) + find_package(cppzmq REQUIRED) + else() + find_package(cppzmq QUIET) + if(NOT cppzmq_FOUND) + message(STATUS "ZeroMQ C++ bindings not found. Switching on builtin_cppzmq option") + set(builtin_cppzmq ON CACHE BOOL "Enabled because ZeroMQ C++ bindings not found (${builtin_cppzmq_description})" FORCE) + endif() + endif() + endif() + + if(builtin_cppzmq) + list(APPEND ROOT_BUILTINS cppzmq) + add_subdirectory(builtins/zeromq/cppzmq) + endif() + + # zmq_ppoll is still in the draft API, so enable that transitively + target_compile_definitions(libzmq INTERFACE ZMQ_BUILD_DRAFT_API) + target_compile_definitions(libzmq INTERFACE ZMQ_NO_EXPORT) + target_compile_definitions(cppzmq INTERFACE ZMQ_BUILD_DRAFT_API) + target_compile_definitions(cppzmq INTERFACE ZMQ_NO_EXPORT) +endif (roofit_multiprocess AND NOT MSVC) + #---Download googletest-------------------------------------------------------------- if (testing) # FIXME: Remove our version of gtest in roottest. We can reuse this one. diff --git a/roofit/CMakeLists.txt b/roofit/CMakeLists.txt index 890df0340dde2..7e9230d9a5d1c 100644 --- a/roofit/CMakeLists.txt +++ b/roofit/CMakeLists.txt @@ -5,6 +5,10 @@ # For the list of contributors see $ROOTSYS/README/CREDITS. add_subdirectory(batchcompute) +if (roofit_multiprocess AND NOT MSVC) + add_subdirectory(roofitZMQ) + add_subdirectory(multiprocess) +endif(roofit_multiprocess AND NOT MSVC) add_subdirectory(roofitcore) add_subdirectory(roofit) if(mathmore) diff --git a/roofit/multiprocess/CMakeLists.txt b/roofit/multiprocess/CMakeLists.txt new file mode 100644 index 0000000000000..c6ff00f3d80e6 --- /dev/null +++ b/roofit/multiprocess/CMakeLists.txt @@ -0,0 +1,22 @@ +############################################################################ +# CMakeLists.txt file for building ROOT RooFitMultiProcess package +# @author Patrick Bos, Netherlands eScience Center +############################################################################ + +ROOT_LINKER_LIBRARY(RooFitMultiProcess + src/worker.cxx + src/Messenger.cxx + src/ProcessManager.cxx + src/util.cxx + src/Queue.cxx + src/JobManager.cxx + src/Job.cxx + DEPENDENCIES + RooFitZMQ +) + +target_link_libraries(RooFitMultiProcess PUBLIC RooFitZMQ) +set(RooFitMultiProcess_INCLUDE_DIRS ${CMAKE_CURRENT_SOURCE_DIR}/res ${RooFitZMQ_INCLUDE_DIRS}) +target_include_directories(RooFitMultiProcess PRIVATE ${RooFitMultiProcess_INCLUDE_DIRS}) + +ROOT_ADD_TEST_SUBDIRECTORY(test) diff --git a/roofit/multiprocess/res/RooFit/MultiProcess/Job.h b/roofit/multiprocess/res/RooFit/MultiProcess/Job.h new file mode 100644 index 0000000000000..33d4492ba93d1 --- /dev/null +++ b/roofit/multiprocess/res/RooFit/MultiProcess/Job.h @@ -0,0 +1,56 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ +#ifndef ROOT_ROOFIT_MultiProcess_Job_decl +#define ROOT_ROOFIT_MultiProcess_Job_decl + +#include +#include + +namespace RooFit { +namespace MultiProcess { + +// forward declaration +class JobManager; + +class Job { +public: + explicit Job(); + Job(const Job &other); + + ~Job(); + + virtual void evaluate_task(std::size_t task) = 0; + virtual void update_state(); + + virtual void send_back_task_result_from_worker(std::size_t task) = 0; + virtual bool receive_task_result_on_master(const zmq::message_t &message) = 0; + + void gather_worker_results(); + + std::size_t get_state_id(); + +protected: + JobManager *get_manager(); + + std::size_t id_; + std::size_t state_id_ = 0; + +private: + // do not use _manager directly, it must first be initialized! use get_manager() + JobManager *_manager = nullptr; +}; + +} // namespace MultiProcess +} // namespace RooFit + +#endif // ROOT_ROOFIT_MultiProcess_Job_decl diff --git a/roofit/multiprocess/res/RooFit/MultiProcess/JobManager.h b/roofit/multiprocess/res/RooFit/MultiProcess/JobManager.h new file mode 100644 index 0000000000000..3853b0f62f380 --- /dev/null +++ b/roofit/multiprocess/res/RooFit/MultiProcess/JobManager.h @@ -0,0 +1,69 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ +#ifndef ROOT_ROOFIT_MultiProcess_JobManager +#define ROOT_ROOFIT_MultiProcess_JobManager + +#include "RooFit/MultiProcess/types.h" + +#include // unique_ptr +#include + +namespace RooFit { +namespace MultiProcess { + +// forward definitions +class ProcessManager; +class Messenger; +class Queue; +class Job; + +class JobManager { +public: + static JobManager *instance(); + static bool is_instantiated(); + + ~JobManager(); + + static std::size_t add_job_object(Job *job_object); + static Job *get_job_object(std::size_t job_object_id); + static bool remove_job_object(std::size_t job_object_id); + + ProcessManager &process_manager() const; + Messenger &messenger() const; + Queue &queue() const; + + void retrieve(std::size_t requesting_job_id); + + void activate(); + bool is_activated() const; + +private: + explicit JobManager(std::size_t N_workers); + + std::unique_ptr process_manager_ptr_; + std::unique_ptr messenger_ptr_; + std::unique_ptr queue_ptr_; + bool activated_ = false; + + static std::map job_objects_; + static std::size_t job_counter_; + static std::unique_ptr instance_; + +public: + static unsigned int default_N_workers; // no need for getters/setters, just public +}; + +} // namespace MultiProcess +} // namespace RooFit + +#endif // ROOT_ROOFIT_MultiProcess_JobManager diff --git a/roofit/multiprocess/res/RooFit/MultiProcess/Messenger.h b/roofit/multiprocess/res/RooFit/MultiProcess/Messenger.h new file mode 100644 index 0000000000000..5174e1ad1c2e8 --- /dev/null +++ b/roofit/multiprocess/res/RooFit/MultiProcess/Messenger.h @@ -0,0 +1,214 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ +#ifndef ROOT_ROOFIT_MultiProcess_Messenger +#define ROOT_ROOFIT_MultiProcess_Messenger + +#include "RooFit/MultiProcess/Messenger_decl.h" + +#ifdef NDEBUG +#undef NDEBUG +#define turn_NDEBUG_back_on +#endif + +namespace RooFit { +namespace MultiProcess { + +// -- WORKER - QUEUE COMMUNICATION -- + +template +void Messenger::send_from_worker_to_queue(T item, Ts... items) +{ +#ifndef NDEBUG + std::stringstream ss; + ss << "PID " << getpid() << " sends W2Q " << item; + debug_print(ss.str()); +#endif + + zmqSvc().send(*this_worker_qw_push_, item, send_flag_); + // if (sizeof...(items) > 0) { // this will only work with if constexpr, c++17 + send_from_worker_to_queue(items...); +} + +template +value_t Messenger::receive_from_worker_on_queue(std::size_t this_worker_id) +{ + qw_pull_poller_[this_worker_id].ppoll(-1, &ppoll_sigmask); + auto value = zmqSvc().receive(*qw_pull_[this_worker_id], zmq::recv_flags::dontwait); + +#ifndef NDEBUG + std::stringstream ss; + ss << "PID " << getpid() << " receives W(" << this_worker_id << ")2Q " << value; + debug_print(ss.str()); +#endif + + return value; +} + +template +void Messenger::send_from_queue_to_worker(std::size_t this_worker_id, T item, Ts... items) +{ +#ifndef NDEBUG + std::stringstream ss; + ss << "PID " << getpid() << " sends Q2W(" << this_worker_id << ") " << item; + debug_print(ss.str()); +#endif + + zmqSvc().send(*qw_push_[this_worker_id], item, send_flag_); + // if (sizeof...(items) > 0) { // this will only work with if constexpr, c++17 + send_from_queue_to_worker(this_worker_id, items...); +} + +template +value_t Messenger::receive_from_queue_on_worker() +{ + qw_pull_poller_[0].ppoll(-1, &ppoll_sigmask); + auto value = zmqSvc().receive(*this_worker_qw_pull_, zmq::recv_flags::dontwait); + +#ifndef NDEBUG + std::stringstream ss; + ss << "PID " << getpid() << " receives Q2W " << value; + debug_print(ss.str()); +#endif + + return value; +} + +// -- QUEUE - MASTER COMMUNICATION -- + +template +void Messenger::send_from_queue_to_master(T item, Ts... items) +{ +#ifndef NDEBUG + std::stringstream ss; + ss << "PID " << getpid() << " sends Q2M " << item; + debug_print(ss.str()); +#endif + + zmqSvc().send(*mq_push_, item, send_flag_); + // if (sizeof...(items) > 0) { // this will only work with if constexpr, c++17 + send_from_queue_to_master(items...); +} + +template +value_t Messenger::receive_from_queue_on_master() +{ + mq_pull_poller_.ppoll(-1, &ppoll_sigmask); + auto value = zmqSvc().receive(*mq_pull_, zmq::recv_flags::dontwait); + +#ifndef NDEBUG + std::stringstream ss; + ss << "PID " << getpid() << " receives Q2M " << value; + debug_print(ss.str()); +#endif + + return value; +} + +template +void Messenger::send_from_master_to_queue(T item, Ts... items) +{ +#ifndef NDEBUG + std::stringstream ss; + ss << "PID " << getpid() << " sends M2Q " << item; + debug_print(ss.str()); +#endif + + zmqSvc().send(*mq_push_, item, send_flag_); + // if (sizeof...(items) > 0) { // this will only work with if constexpr, c++17 + send_from_master_to_queue(items...); +} + +template +value_t Messenger::receive_from_master_on_queue() +{ + mq_pull_poller_.ppoll(-1, &ppoll_sigmask); + auto value = zmqSvc().receive(*mq_pull_, zmq::recv_flags::dontwait); + +#ifndef NDEBUG + std::stringstream ss; + ss << "PID " << getpid() << " receives M2Q " << value; + debug_print(ss.str()); +#endif + + return value; +} + +// -- MASTER - WORKER COMMUNICATION -- + +template +void Messenger::publish_from_master_to_workers(T item, Ts... items) +{ +#ifndef NDEBUG + std::stringstream ss; + ss << "PID " << getpid() << " sends M2W " << item; + debug_print(ss.str()); +#endif + + zmqSvc().send(*mw_pub_, item, send_flag_); + // if (sizeof...(items) > 0) { // this will only work with if constexpr, c++17 + publish_from_master_to_workers(items...); +} + +template +value_t Messenger::receive_from_master_on_worker() +{ + mw_sub_poller_.ppoll(-1, &ppoll_sigmask); + auto value = zmqSvc().receive(*mw_sub_, zmq::recv_flags::dontwait); + +#ifndef NDEBUG + std::stringstream ss; + ss << "PID " << getpid() << " receives M2W " << value; + debug_print(ss.str()); +#endif + + return value; +} + +template +void Messenger::send_from_worker_to_master(T item, Ts... items) +{ +#ifndef NDEBUG + std::stringstream ss; + ss << "PID " << getpid() << " sends M2W " << item; + debug_print(ss.str()); +#endif + + zmqSvc().send(*wm_push_, item, send_flag_); + // if (sizeof...(items) > 0) { // this will only work with if constexpr, c++17 + send_from_worker_to_master(items...); +} + +template +value_t Messenger::receive_from_worker_on_master() +{ + wm_pull_poller_.ppoll(-1, &ppoll_sigmask); + auto value = zmqSvc().receive(*wm_pull_, zmq::recv_flags::dontwait); + +#ifndef NDEBUG + std::stringstream ss; + ss << "PID " << getpid() << " receives M2W " << value; + debug_print(ss.str()); +#endif + + return value; +} + +} // namespace MultiProcess +} // namespace RooFit + +#ifdef turn_NDEBUG_back_on +#define NDEBUG +#undef turn_NDEBUG_back_on +#endif + +#endif // ROOT_ROOFIT_MultiProcess_Messenger diff --git a/roofit/multiprocess/res/RooFit/MultiProcess/Messenger_decl.h b/roofit/multiprocess/res/RooFit/MultiProcess/Messenger_decl.h new file mode 100644 index 0000000000000..6ab49d9ea1694 --- /dev/null +++ b/roofit/multiprocess/res/RooFit/MultiProcess/Messenger_decl.h @@ -0,0 +1,166 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ +#ifndef ROOT_ROOFIT_MultiProcess_Messenger_decl +#define ROOT_ROOFIT_MultiProcess_Messenger_decl + +#include "RooFit/MultiProcess/ProcessManager.h" +#include "RooFit_ZMQ/ZeroMQSvc.h" +#include "RooFit_ZMQ/ZeroMQPoller.h" + +#include +#include +#include // sigprocmask, sigset_t, etc +#include + +namespace RooFit { +namespace MultiProcess { + +void set_socket_immediate(ZmqLingeringSocketPtr<> &socket); + +// test messages +enum class X2X : int { ping = -1, pong = -2, initial_value = 0 }; + +class Messenger { +public: + explicit Messenger(const ProcessManager &process_manager); + ~Messenger(); + + void test_connections(const ProcessManager &process_manager); + + enum class test_snd_pipes { + M2Q, + Q2M, + Q2W, + W2Q, + }; + + enum class test_rcv_pipes { + fromQonM, + fromMonQ, + fromWonQ, + fromQonW, + }; + + std::pair create_queue_poller(); + std::pair create_worker_poller(); + + // -- WORKER - QUEUE COMMUNICATION -- + + void send_from_worker_to_queue(); + template + void send_from_worker_to_queue(T item, Ts... items); + template + value_t receive_from_worker_on_queue(std::size_t this_worker_id); + void send_from_queue_to_worker(std::size_t this_worker_id); + template + void send_from_queue_to_worker(std::size_t this_worker_id, T item, Ts... items); + template + value_t receive_from_queue_on_worker(); + + // -- QUEUE - MASTER COMMUNICATION -- + + void send_from_queue_to_master(); + + template + void send_from_queue_to_master(T item, Ts... items); + template + value_t receive_from_queue_on_master(); + void send_from_master_to_queue(); + + template + void send_from_master_to_queue(T item, Ts... items); + template + value_t receive_from_master_on_queue(); + + // -- MASTER - WORKER COMMUNICATION -- + + void publish_from_master_to_workers(); + template + void publish_from_master_to_workers(T item, Ts... items); + template + value_t receive_from_master_on_worker(); + + void send_from_worker_to_master(); + template + void send_from_worker_to_master(T item, Ts... items); + template + value_t receive_from_worker_on_master(); + + void test_receive(X2X expected_ping_value, test_rcv_pipes rcv_pipe, std::size_t worker_id); + void test_send(X2X ping_value, test_snd_pipes snd_pipe, std::size_t worker_id); + + sigset_t ppoll_sigmask; + + void set_send_flag(zmq::send_flags flag); + +private: + void debug_print(std::string s); + + // push + std::vector> qw_push_; + ZmqLingeringSocketPtr<> this_worker_qw_push_; + ZmqLingeringSocketPtr<> mq_push_; + // pollers for all push sockets + std::vector qw_push_poller_; + ZeroMQPoller mq_push_poller_; + // pull + std::vector> qw_pull_; + ZmqLingeringSocketPtr<> this_worker_qw_pull_; + ZmqLingeringSocketPtr<> mq_pull_; + // pollers for all pull sockets + std::vector qw_pull_poller_; + ZeroMQPoller mq_pull_poller_; + + // publish/subscribe sockets for parameter updating from master to workers + ZmqLingeringSocketPtr<> mw_pub_; + ZmqLingeringSocketPtr<> mw_sub_; + ZeroMQPoller mw_sub_poller_; + // push/pull sockets for result retrieving from workers on master + ZmqLingeringSocketPtr<> wm_push_; + ZmqLingeringSocketPtr<> wm_pull_; + ZeroMQPoller wm_pull_poller_; + + // destruction flags to distinguish between different process-type setups: + bool close_MQ_on_destruct_ = false; + bool close_this_QW_on_destruct_ = false; + bool close_QW_container_on_destruct_ = false; + + zmq::send_flags send_flag_ = zmq::send_flags::none; + + std::vector bound_ipc_addresses_; +}; + +// Messages from master to queue +enum class M2Q : int { + enqueue = 10, +}; + +// Messages from worker to queue +enum class W2Q : int { dequeue = 30 }; + +// Messages from queue to worker +enum class Q2W : int { + dequeue_rejected = 40, + dequeue_accepted = 41, +}; + +// stream output operators for debugging +std::ostream &operator<<(std::ostream &out, const M2Q value); +std::ostream &operator<<(std::ostream &out, const Q2W value); +std::ostream &operator<<(std::ostream &out, const W2Q value); +std::ostream &operator<<(std::ostream &out, const X2X value); + +} // namespace MultiProcess +} // namespace RooFit + +#endif // ROOT_ROOFIT_MultiProcess_Messenger_decl diff --git a/roofit/multiprocess/res/RooFit/MultiProcess/ProcessManager.h b/roofit/multiprocess/res/RooFit/MultiProcess/ProcessManager.h new file mode 100644 index 0000000000000..39927d48bafcd --- /dev/null +++ b/roofit/multiprocess/res/RooFit/MultiProcess/ProcessManager.h @@ -0,0 +1,75 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ +#ifndef ROOT_ROOFIT_MultiProcess_ProcessManager +#define ROOT_ROOFIT_MultiProcess_ProcessManager + +#include // pid_t +#include // sig_atomic_t and for sigterm handling on child processes (in ProcessManager.cxx) +#include + +// forward declaration +class Queue; + +namespace RooFit { +namespace MultiProcess { + +class ProcessManager { + friend Queue; + +public: + explicit ProcessManager(std::size_t N_workers); + ~ProcessManager(); + + bool is_initialized() const; + + void terminate() noexcept; + void wait_for_sigterm_then_exit(); + + bool is_master() const; + bool is_queue() const; + bool is_worker() const; + std::size_t worker_id() const; + std::size_t N_workers() const; + + void identify_processes() const; + + static void handle_sigterm(int signum); + static bool sigterm_received(); + + // for debugging/testing: + pid_t get_queue_pid() const { return queue_pid_; } + std::vector get_worker_pids() { return worker_pids_; } + +private: + void initialize_processes(bool cpu_pinning = true); + void shutdown_processes(); + + bool is_master_ = false; + bool is_queue_ = false; + bool is_worker_ = false; + std::size_t worker_id_; + std::size_t N_workers_; + + // master must wait for workers after completion, for which it needs their PIDs + std::vector worker_pids_; + pid_t queue_pid_; + + bool initialized_ = false; + + static volatile sig_atomic_t sigterm_received_; +}; + +} // namespace MultiProcess +} // namespace RooFit + +#endif // ROOT_ROOFIT_MultiProcess_ProcessManager diff --git a/roofit/multiprocess/res/RooFit/MultiProcess/Queue.h b/roofit/multiprocess/res/RooFit/MultiProcess/Queue.h new file mode 100644 index 0000000000000..042de395b1b11 --- /dev/null +++ b/roofit/multiprocess/res/RooFit/MultiProcess/Queue.h @@ -0,0 +1,43 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ +#ifndef ROOT_ROOFIT_MultiProcess_Queue +#define ROOT_ROOFIT_MultiProcess_Queue + +#include "RooFit/MultiProcess/types.h" +#include "RooFit/MultiProcess/Messenger.h" + +#include + +namespace RooFit { +namespace MultiProcess { + +class Queue { +public: + bool pop(JobTask &job_task); + void add(JobTask job_task); + + void loop(); + + void process_master_message(M2Q message); + void process_worker_message(std::size_t this_worker_id, W2Q message); + +private: + std::queue queue_; + std::size_t N_tasks_ = 0; // total number of received tasks + std::size_t N_tasks_at_workers_ = 0; +}; + +} // namespace MultiProcess +} // namespace RooFit + +#endif // ROOT_ROOFIT_MultiProcess_Queue diff --git a/roofit/multiprocess/res/RooFit/MultiProcess/types.h b/roofit/multiprocess/res/RooFit/MultiProcess/types.h new file mode 100644 index 0000000000000..0bd1b25efd23d --- /dev/null +++ b/roofit/multiprocess/res/RooFit/MultiProcess/types.h @@ -0,0 +1,34 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ +#ifndef ROOT_ROOFIT_MultiProcess_types +#define ROOT_ROOFIT_MultiProcess_types + +#include // pair + +namespace RooFit { +namespace MultiProcess { + +// some helper types +using Task = std::size_t; +using State = std::size_t; +/// combined job_object, state and task identifier type +struct JobTask { + std::size_t job_id; + State state_id; + Task task_id; +}; + +} // namespace MultiProcess +} // namespace RooFit + +#endif // ROOT_ROOFIT_MultiProcess_types diff --git a/roofit/multiprocess/res/RooFit/MultiProcess/util.h b/roofit/multiprocess/res/RooFit/MultiProcess/util.h new file mode 100644 index 0000000000000..2893d39eea37f --- /dev/null +++ b/roofit/multiprocess/res/RooFit/MultiProcess/util.h @@ -0,0 +1,34 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#ifndef ROOT_ROOFIT_MultiProcess_util +#define ROOT_ROOFIT_MultiProcess_util + +#include "RooFit_ZMQ/ppoll.h" // for ZMQ::ppoll_error_t +#include "RooFit_ZMQ/ZeroMQPoller.h" + +#include // getpid, pid_t + +namespace RooFit { +namespace MultiProcess { + +int wait_for_child(pid_t child_pid, bool may_throw, int retries_before_killing); + +enum class zmq_ppoll_error_response { abort, unknown_eintr, retry }; +zmq_ppoll_error_response handle_zmq_ppoll_error(ZMQ::ppoll_error_t &e); +std::tuple>, bool> +careful_ppoll(ZeroMQPoller &poller, const sigset_t &ppoll_sigmask, std::size_t max_tries = 2); + +} // namespace MultiProcess +} // namespace RooFit +#endif // ROOT_ROOFIT_MultiProcess_util diff --git a/roofit/multiprocess/res/RooFit/MultiProcess/worker.h b/roofit/multiprocess/res/RooFit/MultiProcess/worker.h new file mode 100644 index 0000000000000..7244ab83e6190 --- /dev/null +++ b/roofit/multiprocess/res/RooFit/MultiProcess/worker.h @@ -0,0 +1,25 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ +#ifndef ROOT_ROOFIT_MultiProcess_worker +#define ROOT_ROOFIT_MultiProcess_worker + +namespace RooFit { +namespace MultiProcess { + +void worker_loop(); +bool is_worker_loop_running(); + +} // namespace MultiProcess +} // namespace RooFit + +#endif // ROOT_ROOFIT_MultiProcess_worker diff --git a/roofit/multiprocess/src/Job.cxx b/roofit/multiprocess/src/Job.cxx new file mode 100644 index 0000000000000..5fc04ee0a94d1 --- /dev/null +++ b/roofit/multiprocess/src/Job.cxx @@ -0,0 +1,151 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#include "RooFit/MultiProcess/JobManager.h" +#include "RooFit/MultiProcess/Messenger.h" +#include "RooFit/MultiProcess/Job.h" + +namespace RooFit { +namespace MultiProcess { + +/** @class Job + * + * @brief interface class for defining the actual work that must be done + * + * Think of "job" as in "employment", e.g. the job of a baker, which + * involves *tasks* like baking and selling bread. The Job must define the + * tasks through its execution (evaluate_task), based on a task index argument. + * + * Classes inheriting from Job must implement the pure virtual methods: + * - void evaluate_task(std::size_t task) + * - void send_back_task_result_from_worker(std::size_t task) + * - void receive_task_result_on_master(const zmq::message_t & message) + * + * An example/reference implementation can be found in test_Job.cxx. + * + * Most Jobs will also want to override the virtual update_state() function. + * This function can be used to send and receive state from master to worker. + * In the worker loop, when something is received over the ZeroMQ "SUB" socket, + * update_state() is called to put the received data into the right places, + * thus updating for instance parameter values on the worker that were updated + * since the last call on the master side. + * + * ## Message protocol + * + * One simple rule must be upheld for the messages that the implementer will + * send with 'send_back_task_result_from_worker' and 'update_state': the first + * part of the message must always be the 'Job''s ID, stored in 'Job::id'. + * The rest of the message, i.e. the actual data to be sent, is completely up + * to the implementation. Note that on the receiving end, i.e. in the + * implementation of 'receive_task_result_on_master', one will get the whole + * message, but the 'Job' ID part will already have been identified in the + * 'JobManager', so one needn't worry about it further inside + * 'Job::receive_task_result_on_master' (it is already routed to the correct + * 'Job'). The same goes for the receiving end of 'update_state', except that + * update_state is routed from the 'worker_loop', not the 'JobManager'. + * + * A second rule applies to 'update_state' messages: the second part must be + * a state identifier. This identifier will also be sent along with tasks to + * the queue. When a worker then takes a task from the queue, it can check + * whether it has already updated its state to what is expected to be there + * for the task at hand. If not, it should wait for the new state to arrive + * over the state subscription socket. Note: it is the implementer's task to + * actually update 'Job::state_id_' inside 'Job::update_state()'! + * + * ## Implementers notes + * + * The type of result from each task is strongly dependent on the Job at hand + * and so Job does not provide a default results member. It is up to the + * inheriting class to implement this in the above functions. We would have + * liked a template parameter task_result_t, so that we could also provide a + * default "boilerplate" calculate function to show a typical Job use-case of + * all the above infrastructure. This is not trivial, because the JobManager + * has to keep a list of Job pointers, so if there would be different template + * instantiations of Jobs, this would complicate this list. + * + * A typical Job implementation will have an evaluation function that is + * called from the master process, like RooAbsArg::getVal calls evaluate(). + * This function will have three purposes: 1. send updated parameter values + * to the workers (possibly through update_state() or in a dedicated + * function), 2. queue tasks and 3. wait for the results to be retrieved. + * 'Job::gather_worker_results()' is provided for convenience to wait for + * all tasks to be retrieved for the current Job. Implementers can also + * choose to have the master process perform other tasks in between any of + * these three steps, or even skip steps completely. + * + * Child classes should refrain from direct access to the JobManager instance + * (through JobManager::instance), but rather use the here provided + * Job::get_manager(). This function starts the worker_loop on the worker when + * first called, meaning that the workers will not be running before they + * are needed. + */ + +Job::Job() +{ + id_ = JobManager::add_job_object(this); +} + +Job::Job(const Job &other) : _manager(other._manager) +{ + id_ = JobManager::add_job_object(this); +} + +Job::~Job() +{ + JobManager::remove_job_object(id_); +} + +/** \brief Get JobManager instance; create and activate if necessary + * + * Child classes should refrain from direct access to the JobManager instance + * (through JobManager::instance), but rather use the here provided + * Job::get_manager(). This function starts the worker_loop on the worker when + * first called, meaning that the workers will not be running before they + * are needed. + */ +JobManager *Job::get_manager() +{ + if (!_manager) { + _manager = JobManager::instance(); + } + + if (!_manager->is_activated()) { + _manager->activate(); + } + + return _manager; +} + +/// Wait for all tasks to be retrieved for the current Job. +void Job::gather_worker_results() +{ + get_manager()->retrieve(id_); +} + +/// \brief Virtual function to update any necessary state on workers +/// +/// This function is called from the worker loop when something is received +/// over the ZeroMQ "SUB" socket. The master process sends messages to workers +/// on its "PUB" socket. Thus, we can update, for instance, parameter values +/// on the worker that were updated since the last call on the master side. +/// \note Implementers: make sure to also update the state_id_ member. +void Job::update_state() {} + +/// Get the current state identifier +std::size_t Job::get_state_id() +{ + return state_id_; +} + +} // namespace MultiProcess +} // namespace RooFit diff --git a/roofit/multiprocess/src/JobManager.cxx b/roofit/multiprocess/src/JobManager.cxx new file mode 100644 index 0000000000000..ea9fcab5d06b9 --- /dev/null +++ b/roofit/multiprocess/src/JobManager.cxx @@ -0,0 +1,218 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#include "RooFit/MultiProcess/JobManager.h" +#include "RooFit/MultiProcess/ProcessManager.h" +#include "RooFit/MultiProcess/Messenger.h" +#include "RooFit/MultiProcess/Job.h" +#include "RooFit/MultiProcess/Queue.h" // complete type for JobManager::queue() +#include "RooFit/MultiProcess/worker.h" +#include "RooFit/MultiProcess/util.h" + +#include // std::thread::hardware_concurrency() + +namespace RooFit { +namespace MultiProcess { + +/** \class JobManager + * + * \brief Main point of access for all MultiProcess infrastructure + * + * This class mainly serves as the access point to the multi-process infrastructure + * for 'Job's. It is meant to be used as a singleton that holds and connects the other + * infrastructural classes: the messenger, process manager, worker and queue loops. + * + * It is important that the user of this class, particularly the one that calls + * 'instance()' first, calls 'activate()' soon after, because everything that is + * done in between 'instance()' and 'activate()' will be executed on all processes. + * This may be useful in some cases, but in general, one will probably want to always + * use the 'JobManager' in its full capacity, including the queue and worker loops. + * This is the way the Job class uses this class, see 'Job::get_manager()'. + * + * The default number of processes is set using 'std::thread::hardware_concurrency()'. + * To change it, simply set 'JobManager::default_N_workers' to a different value + * before creation of a new instance. + */ + +// static function +JobManager *JobManager::instance() +{ + if (!JobManager::is_instantiated()) { + assert(default_N_workers != 0); + instance_.reset(new JobManager(default_N_workers)); // can't use make_unique, because ctor is private + instance_->messenger().test_connections(instance_->process_manager()); + // set send to non blocking on all processes after checking the connections are working: + instance_->messenger().set_send_flag(zmq::send_flags::dontwait); + } + return instance_.get(); +} + +// static function +bool JobManager::is_instantiated() +{ + return static_cast(instance_); +} + +// (private) constructor +/// Don't construct JobManager objects manually, use the static instance if +/// you need to run multiple jobs. +JobManager::JobManager(std::size_t N_workers) +{ + queue_ptr_ = std::make_unique(); + process_manager_ptr_ = std::make_unique(N_workers); + messenger_ptr_ = std::make_unique(*process_manager_ptr_); +} + +JobManager::~JobManager() +{ + // The instance typically gets created by some Job. Once all Jobs are gone, the + // JM will get destroyed. In this case, the job_objects map should have + // been emptied. This check makes sure: + assert(JobManager::job_objects_.empty()); + messenger_ptr_.reset(nullptr); + process_manager_ptr_.reset(nullptr); + queue_ptr_.reset(nullptr); +} + +// static function +/// \return job_id for added job_object +std::size_t JobManager::add_job_object(Job *job_object) +{ + if (JobManager::is_instantiated()) { + if (instance_->process_manager().is_initialized()) { + std::stringstream ss; + ss << "Cannot add Job to JobManager instantiation, forking has already taken place! Instance object at raw " + "ptr " + << instance_.get(); + throw std::logic_error("Cannot add Job to JobManager instantiation, forking has already taken place! Call " + "terminate() on the instance before adding new Jobs."); + } + } + std::size_t job_id = job_counter_++; + job_objects_[job_id] = job_object; + return job_id; +} + +// static function +Job *JobManager::get_job_object(std::size_t job_object_id) +{ + return job_objects_[job_object_id]; +} + +// static function +/// \return Returns 'true' when removed successfully, 'false' otherwise. +bool JobManager::remove_job_object(std::size_t job_object_id) +{ + bool removed_succesfully = job_objects_.erase(job_object_id) == 1; + if (job_objects_.empty()) { + instance_.reset(nullptr); + } + return removed_succesfully; +} + +ProcessManager &JobManager::process_manager() const +{ + return *process_manager_ptr_; +} + +Messenger &JobManager::messenger() const +{ + return *messenger_ptr_; +} + +Queue &JobManager::queue() const +{ + return *queue_ptr_; +} + +/// Retrieve results for a Job +/// +/// \param requesting_job_id ID number of the Job in the JobManager's Job list +void JobManager::retrieve(std::size_t requesting_job_id) +{ + if (process_manager().is_master()) { + bool job_fully_retrieved = false; + while (not job_fully_retrieved) { + try { + auto task_result_message = messenger().receive_from_worker_on_master(); + auto job_object_id = *reinterpret_cast( + task_result_message.data()); // job_id must always be the first element of the result message! + bool this_job_fully_retrieved = + JobManager::get_job_object(job_object_id)->receive_task_result_on_master(task_result_message); + if (requesting_job_id == job_object_id) { + job_fully_retrieved = this_job_fully_retrieved; + } + } catch (ZMQ::ppoll_error_t &e) { + zmq_ppoll_error_response response; + try { + response = handle_zmq_ppoll_error(e); + } catch (std::logic_error &) { + printf("JobManager::retrieve got unhandleable ZMQ::ppoll_error_t\n"); + throw; + } + if (response == zmq_ppoll_error_response::abort) { + throw std::logic_error("in JobManager::retrieve: master received a SIGTERM, aborting"); + } else if (response == zmq_ppoll_error_response::unknown_eintr) { + printf("EINTR in JobManager::retrieve, continuing\n"); + continue; + } else if (response == zmq_ppoll_error_response::retry) { + printf("EAGAIN from ppoll in JobManager::retrieve, continuing\n"); + continue; + } + } catch (zmq::error_t &e) { + printf("unhandled zmq::error_t (not a ppoll_error_t) in JobManager::retrieve with errno %d: %s\n", e.num(), + e.what()); + throw; + } + } + } +} + +/// \brief Start queue and worker loops on child processes +/// +/// This function exists purely because activation from the constructor is +/// impossible; the constructor must return a constructed instance, which it +/// can't do if it's stuck in an infinite loop. This means the Job that first +/// creates the JobManager instance must also activate it (or any other user +/// of this class). +/// This should be called soon after creation of instance, because everything +/// between construction and activation gets executed both on the master +/// process and on the slaves. +void JobManager::activate() +{ + activated_ = true; + + if (process_manager().is_queue()) { + queue().loop(); + std::_Exit(0); + } + + if (!is_worker_loop_running() && process_manager().is_worker()) { + RooFit::MultiProcess::worker_loop(); + std::_Exit(0); + } +} + +bool JobManager::is_activated() const +{ + return activated_; +} + +// initialize static members +std::map JobManager::job_objects_; +std::size_t JobManager::job_counter_ = 0; +std::unique_ptr JobManager::instance_{nullptr}; +unsigned int JobManager::default_N_workers = std::thread::hardware_concurrency(); + +} // namespace MultiProcess +} // namespace RooFit diff --git a/roofit/multiprocess/src/Messenger.cxx b/roofit/multiprocess/src/Messenger.cxx new file mode 100644 index 0000000000000..6dc748cc4aab6 --- /dev/null +++ b/roofit/multiprocess/src/Messenger.cxx @@ -0,0 +1,544 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#include "RooFit/MultiProcess/Messenger.h" +#include "RooFit/MultiProcess/util.h" + +#include // sigprocmask etc +#include // sprintf + +namespace RooFit { +namespace MultiProcess { + +void set_socket_immediate(ZmqLingeringSocketPtr<> &socket) +{ + int optval = 1; + socket->set(zmq::sockopt::immediate, optval); +} + +/** \class Messenger + * + * \brief Manages ZeroMQ sockets and wraps send and receive calls + * + * This class is used for all interprocess communication between the master, + * queue and worker processes. It sets up ZeroMQ sockets between all processes + * over IPC socket files stored in /tmp on the filesystem. + * + * Several sockets are used for communication between different places for + * different purposes: + * - Master and queue processes each have a PUSH-PULL socket pair to directly + * send/receive data between only the master and queue processes. This is + * currently used mainly for sending tasks to the queue from master. The + * socket from queue back to master is used only to test connections and may + * be removed in the future. + * - The queue process also has a PUSH-PULL socket pair with each worker + * process. These are used by the workers to obtain tasks from the queue. + * - The master has a PUB socket that the workers subscribe to with SUB + * sockets. These are used to update state. Note that to ensure robust + * reception of all messages on the SUB socket, it's important to send over + * state in as little messages as possible. For instance, it's best to send + * arrays over in a single big message instead of sending over each element + * separately. This also improves performance, since each message has some + * fixed overhead. + * - Each worker has a PUSH socket connected to a PULL socket on master that + * is used to send back task results from workers to master in + * 'JobManager::retrieve()'. + * + * @param process_manager ProcessManager instance which manages the master, + * queue and worker processes that we want to set up + * communication for in this Messenger. + */ + +Messenger::Messenger(const ProcessManager &process_manager) +{ + sigemptyset(&ppoll_sigmask); + + char addr_prefix_template[30] = "ipc:///tmp/%d_roofitMP"; + char addr_prefix[50]; + char addr[100]; + + // high water mark for master-queue sending, which can be quite a busy channel, especially at the start of a run + int hwm = 0; + // create zmq connections (zmq context is automatically created in the ZeroMQSvc class and maintained as singleton) + // and pollers where necessary + try { + if (process_manager.is_master()) { + sprintf(addr_prefix, addr_prefix_template, getpid()); + + mq_push_.reset(zmqSvc().socket_ptr(zmq::socket_type::push)); + auto rc = zmq_setsockopt(*mq_push_, ZMQ_SNDHWM, &hwm, sizeof hwm); + assert(rc == 0); + sprintf(addr, "%s_%s", addr_prefix, "from_master_to_queue"); + mq_push_->bind(addr); + bound_ipc_addresses_.emplace_back(addr); + + mq_push_poller_.register_socket(*mq_push_, zmq::event_flags::pollout); + + mq_pull_.reset(zmqSvc().socket_ptr(zmq::socket_type::pull)); + rc = zmq_setsockopt(*mq_pull_, ZMQ_RCVHWM, &hwm, sizeof hwm); + assert(rc == 0); + sprintf(addr, "%s_%s", addr_prefix, "from_queue_to_master"); + mq_pull_->bind(addr); + bound_ipc_addresses_.emplace_back(addr); + + mq_pull_poller_.register_socket(*mq_pull_, zmq::event_flags::pollin); + + mw_pub_.reset(zmqSvc().socket_ptr(zmq::socket_type::pub)); + rc = zmq_setsockopt(*mw_pub_, ZMQ_SNDHWM, &hwm, sizeof hwm); + assert(rc == 0); + sprintf(addr, "%s_%s", addr_prefix, "from_master_to_workers"); + mw_pub_->bind(addr); + bound_ipc_addresses_.emplace_back(addr); + + wm_pull_.reset(zmqSvc().socket_ptr(zmq::socket_type::pull)); + rc = zmq_setsockopt(*wm_pull_, ZMQ_RCVHWM, &hwm, sizeof hwm); + assert(rc == 0); + sprintf(addr, "%s_%s", addr_prefix, "from_workers_to_master"); + wm_pull_->bind(addr); + bound_ipc_addresses_.emplace_back(addr); + wm_pull_poller_.register_socket(*wm_pull_, zmq::event_flags::pollin); + + close_MQ_on_destruct_ = true; + + // make sure all subscribers are connected + ZmqLingeringSocketPtr<> subscriber_ping_socket{zmqSvc().socket_ptr(zmq::socket_type::pull)}; + sprintf(addr, "%s_%s", addr_prefix, "subscriber_ping_socket"); + subscriber_ping_socket->bind(addr); + bound_ipc_addresses_.emplace_back(addr); + ZeroMQPoller subscriber_ping_poller; + subscriber_ping_poller.register_socket(*subscriber_ping_socket, zmq::event_flags::pollin); + std::size_t N_subscribers_confirmed = 0; + while (N_subscribers_confirmed < process_manager.N_workers()) { + zmqSvc().send(*mw_pub_, false); + auto poll_results = subscriber_ping_poller.poll(0); + for (std::size_t ix = 0; ix < poll_results.size(); ++ix) { + auto request = zmqSvc().receive(*subscriber_ping_socket, zmq::recv_flags::dontwait); + assert(request == "present"); + ++N_subscribers_confirmed; + } + } + zmqSvc().send(*mw_pub_, true); + + } else if (process_manager.is_queue()) { + sprintf(addr_prefix, addr_prefix_template, getppid()); + + // first the queue-worker sockets + // do resize instead of reserve so that the unique_ptrs are initialized + // (to nullptr) so that we can do reset below, alternatively you can do + // push/emplace_back with move or something + qw_push_.resize(process_manager.N_workers()); + qw_pull_.resize(process_manager.N_workers()); + qw_push_poller_.resize(process_manager.N_workers()); + qw_pull_poller_.resize(process_manager.N_workers()); + for (std::size_t ix = 0; ix < process_manager.N_workers(); ++ix) { + // push + qw_push_[ix].reset(zmqSvc().socket_ptr(zmq::socket_type::push)); + sprintf(addr, "%s_%s_%zu", addr_prefix, "from_queue_to_worker", ix); + qw_push_[ix]->bind(addr); + bound_ipc_addresses_.emplace_back(addr); + + qw_push_poller_[ix].register_socket(*qw_push_[ix], zmq::event_flags::pollout); + + // pull + qw_pull_[ix].reset(zmqSvc().socket_ptr(zmq::socket_type::pull)); + sprintf(addr, "%s_%s_%zu_%s", addr_prefix, "from_worker", ix, "to_queue"); + qw_pull_[ix]->bind(addr); + bound_ipc_addresses_.emplace_back(addr); + + qw_pull_poller_[ix].register_socket(*qw_pull_[ix], zmq::event_flags::pollin); + } + + // then the master-queue sockets + mq_push_.reset(zmqSvc().socket_ptr(zmq::socket_type::push)); + auto rc = zmq_setsockopt(*mq_push_, ZMQ_SNDHWM, &hwm, sizeof hwm); + assert(rc == 0); + sprintf(addr, "%s_%s", addr_prefix, "from_queue_to_master"); + mq_push_->connect(addr); + + mq_push_poller_.register_socket(*mq_push_, zmq::event_flags::pollout); + + mq_pull_.reset(zmqSvc().socket_ptr(zmq::socket_type::pull)); + rc = zmq_setsockopt(*mq_pull_, ZMQ_RCVHWM, &hwm, sizeof hwm); + assert(rc == 0); + sprintf(addr, "%s_%s", addr_prefix, "from_master_to_queue"); + mq_pull_->connect(addr); + + mq_pull_poller_.register_socket(*mq_pull_, zmq::event_flags::pollin); + + close_MQ_on_destruct_ = true; + close_QW_container_on_destruct_ = true; + } else if (process_manager.is_worker()) { + sprintf(addr_prefix, addr_prefix_template, getppid()); + + // we only need one queue-worker pipe on the worker + qw_push_poller_.resize(1); + qw_pull_poller_.resize(1); + + // push + this_worker_qw_push_.reset(zmqSvc().socket_ptr(zmq::socket_type::push)); + sprintf(addr, "%s_%s_%zu_%s", addr_prefix, "from_worker", process_manager.worker_id(), "to_queue"); + this_worker_qw_push_->connect(addr); + + qw_push_poller_[0].register_socket(*this_worker_qw_push_, zmq::event_flags::pollout); + + // pull + this_worker_qw_pull_.reset(zmqSvc().socket_ptr(zmq::socket_type::pull)); + sprintf(addr, "%s_%s_%zu", addr_prefix, "from_queue_to_worker", process_manager.worker_id()); + this_worker_qw_pull_->connect(addr); + + qw_pull_poller_[0].register_socket(*this_worker_qw_pull_, zmq::event_flags::pollin); + + mw_sub_.reset(zmqSvc().socket_ptr(zmq::socket_type::sub)); + auto rc = zmq_setsockopt(*mw_sub_, ZMQ_RCVHWM, &hwm, sizeof hwm); + assert(rc == 0); + rc = zmq_setsockopt(*mw_sub_, ZMQ_SUBSCRIBE, "", 0); + assert(rc == 0); + sprintf(addr, "%s_%s", addr_prefix, "from_master_to_workers"); + mw_sub_->connect(addr); + mw_sub_poller_.register_socket(*mw_sub_, zmq::event_flags::pollin); + + wm_push_.reset(zmqSvc().socket_ptr(zmq::socket_type::push)); + rc = zmq_setsockopt(*wm_push_, ZMQ_SNDHWM, &hwm, sizeof hwm); + assert(rc == 0); + sprintf(addr, "%s_%s", addr_prefix, "from_workers_to_master"); + wm_push_->connect(addr); + + // check publisher connection and then wait until all subscribers are connected + ZmqLingeringSocketPtr<> subscriber_ping_socket{zmqSvc().socket_ptr(zmq::socket_type::push)}; + sprintf(addr, "%s_%s", addr_prefix, "subscriber_ping_socket"); + subscriber_ping_socket->connect(addr); + auto all_connected = zmqSvc().receive(*mw_sub_); + zmqSvc().send(*subscriber_ping_socket, "present"); + + while (!all_connected) { + all_connected = zmqSvc().receive(*mw_sub_); + } + + close_this_QW_on_destruct_ = true; + } else { + // should never get here + throw std::runtime_error("Messenger ctor: I'm neither master, nor queue, nor a worker"); + } + } catch (zmq::error_t &e) { + std::cerr << e.what() << " -- errnum: " << e.num() << std::endl; + throw; + }; +} + +Messenger::~Messenger() +{ + if (close_MQ_on_destruct_) { + try { + mq_push_.reset(nullptr); + mq_pull_.reset(nullptr); + mw_pub_.reset(nullptr); + wm_pull_.reset(nullptr); + // remove bound files + for (const auto &address : bound_ipc_addresses_) { + // no need to check return value, they are only zero byte /tmp files, the OS should eventually clean them up + remove(address.substr(6).c_str()); + } + } catch (const std::exception &e) { + std::cerr << "WARNING: something in Messenger dtor threw an exception! Original exception message:\n" + << e.what() << std::endl; + } + } + if (close_this_QW_on_destruct_) { + this_worker_qw_push_.reset(nullptr); + this_worker_qw_pull_.reset(nullptr); + mw_sub_.reset(nullptr); + wm_push_.reset(nullptr); + } + if (close_QW_container_on_destruct_) { + for (auto &socket : qw_push_) { + socket.reset(nullptr); + } + for (auto &socket : qw_pull_) { + socket.reset(nullptr); + } + } + zmqSvc().close_context(); +} + +void Messenger::test_send(X2X ping_value, test_snd_pipes snd_pipe, std::size_t worker_id) +{ + try { + switch (snd_pipe) { + case test_snd_pipes::M2Q: { + send_from_master_to_queue(ping_value); + break; + } + case test_snd_pipes::Q2M: { + send_from_queue_to_master(ping_value); + break; + } + case test_snd_pipes::Q2W: { + send_from_queue_to_worker(worker_id, ping_value); + break; + } + case test_snd_pipes::W2Q: { + send_from_worker_to_queue(ping_value); + break; + } + } + } catch (zmq::error_t &e) { + if (e.num() == EAGAIN) { + throw std::runtime_error("Messenger::test_connections: SEND over master-queue connection timed out!"); + } else { + throw; + } + } +} + +void Messenger::test_receive(X2X expected_ping_value, test_rcv_pipes rcv_pipe, std::size_t worker_id) +{ + X2X handshake = X2X::initial_value; + + std::size_t max_tries = 3, tries = 0; + bool carry_on = true; + while (carry_on && (tries++ < max_tries)) { + try { + switch (rcv_pipe) { + case test_rcv_pipes::fromMonQ: { + handshake = receive_from_master_on_queue(); + break; + } + case test_rcv_pipes::fromQonM: { + handshake = receive_from_queue_on_master(); + break; + } + case test_rcv_pipes::fromQonW: { + handshake = receive_from_queue_on_worker(); + break; + } + case test_rcv_pipes::fromWonQ: { + handshake = receive_from_worker_on_queue(worker_id); + break; + } + } + carry_on = false; + } catch (ZMQ::ppoll_error_t &e) { + auto response = handle_zmq_ppoll_error(e); + if (response == zmq_ppoll_error_response::abort) { + throw std::runtime_error("EINTR in test_receive and SIGTERM received, aborting\n"); + } else if (response == zmq_ppoll_error_response::unknown_eintr) { + printf("EINTR in test_receive but no SIGTERM received, try %zu\n", tries); + continue; + } else if (response == zmq_ppoll_error_response::retry) { + printf("EAGAIN in test_receive, try %zu\n", tries); + continue; + } + } catch (zmq::error_t &e) { + if (e.num() == EAGAIN) { + throw std::runtime_error("Messenger::test_connections: RECEIVE over master-queue connection timed out!"); + } else { + printf("unhandled zmq::error_t (not a ppoll_error_t) in Messenger::test_receive with errno %d: %s\n", + e.num(), e.what()); + throw; + } + } + } + + if (handshake != expected_ping_value) { + throw std::runtime_error( + "Messenger::test_connections: RECEIVE over master-queue connection failed, did not receive expected value!"); + } +} + +/// \brief Test whether push-pull sockets are working +/// +/// \note This function tests the PUSH-PULL socket pairs only. The PUB-SUB sockets are already tested in the +/// constructor. +/// +/// \param process_manager ProcessManager object used to instantiate this object. Used to identify which process we are +/// running on and hence which sockets need to be tested. +void Messenger::test_connections(const ProcessManager &process_manager) +{ + if (process_manager.is_queue() || process_manager.is_worker()) { + // Before blocking SIGTERM, set the signal handler, so we can also check after blocking whether a signal occurred + // In our case, we already set it in the ProcessManager after forking to the queue and worker processes. + sigset_t sigmask; + sigemptyset(&sigmask); + sigaddset(&sigmask, SIGTERM); + int rc = sigprocmask(SIG_BLOCK, &sigmask, &ppoll_sigmask); + if (rc < 0) { + throw std::runtime_error("sigprocmask failed in test_connections"); + } + } + + if (process_manager.is_master()) { + test_receive(X2X::ping, test_rcv_pipes::fromQonM, -1); + test_send(X2X::pong, test_snd_pipes::M2Q, -1); + test_send(X2X::ping, test_snd_pipes::M2Q, -1); + // make sure to always receive last on master, so that master knows when queue is done, + // which means workers are done as well, so if master is done everything is done: + test_receive(X2X::pong, test_rcv_pipes::fromQonM, -1); + } else if (process_manager.is_queue()) { + ZeroMQPoller poller; + std::size_t mq_index; + std::tie(poller, mq_index) = create_queue_poller(); + + for (std::size_t ix = 0; ix < process_manager.N_workers(); ++ix) { + test_send(X2X::ping, test_snd_pipes::Q2W, ix); + } + test_send(X2X::ping, test_snd_pipes::Q2M, -1); + + while (!process_manager.sigterm_received() && (poller.size() > 0)) { + // poll: wait until status change (-1: infinite timeout) + std::vector> poll_result; + bool abort; + std::tie(poll_result, abort) = careful_ppoll(poller, ppoll_sigmask); + if (abort) + break; + + // then process incoming messages from sockets + for (auto readable_socket : poll_result) { + // message comes from the master/queue socket (first element): + if (readable_socket.first == mq_index) { + test_receive(X2X::pong, test_rcv_pipes::fromMonQ, -1); + test_receive(X2X::ping, test_rcv_pipes::fromMonQ, -1); + poller.unregister_socket(*mq_pull_); + } else { // from a worker socket + // TODO: dangerous assumption for this_worker_id, may become invalid if we allow multiple queue_loops on + // the same process! + auto this_worker_id = readable_socket.first - 1; // TODO: replace with a more reliable lookup + + test_receive(X2X::pong, test_rcv_pipes::fromWonQ, this_worker_id); + test_receive(X2X::ping, test_rcv_pipes::fromWonQ, this_worker_id); + test_send(X2X::pong, test_snd_pipes::Q2W, this_worker_id); + + poller.unregister_socket(*qw_pull_[this_worker_id]); + } + } + } + test_send(X2X::pong, test_snd_pipes::Q2M, -1); + + } else if (process_manager.is_worker()) { + test_receive(X2X::ping, test_rcv_pipes::fromQonW, -1); + test_send(X2X::pong, test_snd_pipes::W2Q, -1); + test_send(X2X::ping, test_snd_pipes::W2Q, -1); + test_receive(X2X::pong, test_rcv_pipes::fromQonW, -1); + } else { + // should never get here + throw std::runtime_error("Messenger::test_connections: I'm neither master, nor queue, nor a worker"); + } + + if (process_manager.is_queue() || process_manager.is_worker()) { + // clean up signal management modifications + sigprocmask(SIG_SETMASK, &ppoll_sigmask, nullptr); + } +} + +/// Helper function that creates a poller for Queue::loop() +std::pair Messenger::create_queue_poller() +{ + ZeroMQPoller poller; + std::size_t mq_index = poller.register_socket(*mq_pull_, zmq::event_flags::pollin); + for (auto &s : qw_pull_) { + poller.register_socket(*s, zmq::event_flags::pollin); + } + return {std::move(poller), mq_index}; +} + +/// Helper function that creates a poller for worker_loop() +std::pair Messenger::create_worker_poller() +{ + ZeroMQPoller poller; + poller.register_socket(*this_worker_qw_pull_, zmq::event_flags::pollin); + std::size_t mw_sub_index = poller.register_socket(*mw_sub_, zmq::event_flags::pollin); + return {std::move(poller), mw_sub_index}; +} + +// -- WORKER - QUEUE COMMUNICATION -- + +void Messenger::send_from_worker_to_queue() {} + +void Messenger::send_from_queue_to_worker(std::size_t /*this_worker_id*/) {} + +// -- QUEUE - MASTER COMMUNICATION -- + +void Messenger::send_from_queue_to_master() {} + +void Messenger::send_from_master_to_queue() {} + +/// Set the flag used in all send functions; 0, ZMQ_DONTWAIT, ZMQ_SNDMORE or bitwise combination +void Messenger::set_send_flag(zmq::send_flags flag) +{ + send_flag_ = flag; +} + +// -- MASTER - WORKER COMMUNICATION -- + +void Messenger::publish_from_master_to_workers() {} + +void Messenger::send_from_worker_to_master() {} + +// for debugging +#define PROCESS_VAL(p) \ + case (p): s = #p; break; + +std::ostream &operator<<(std::ostream &out, const M2Q value) +{ + std::string s; + switch (value) { + PROCESS_VAL(M2Q::enqueue); + default: s = std::to_string(static_cast(value)); + } + return out << s; +} + +std::ostream &operator<<(std::ostream &out, const W2Q value) +{ + std::string s; + switch (value) { + PROCESS_VAL(W2Q::dequeue); + default: s = std::to_string(static_cast(value)); + } + return out << s; +} + +std::ostream &operator<<(std::ostream &out, const Q2W value) +{ + std::string s; + switch (value) { + PROCESS_VAL(Q2W::dequeue_rejected); + PROCESS_VAL(Q2W::dequeue_accepted); + default: s = std::to_string(static_cast(value)); + } + return out << s; +} + +std::ostream &operator<<(std::ostream &out, const X2X value) +{ + std::string s; + switch (value) { + PROCESS_VAL(X2X::ping); + PROCESS_VAL(X2X::pong); + default: s = std::to_string(static_cast(value)); + } + return out << s; +} + +#undef PROCESS_VAL + +/// Function called from send and receive template functions in debug builds +/// used to monitor the messages that are going to be sent or are received. +/// By defining this in the implementation file, compilation is a lot faster +/// during debugging of Messenger or communication protocols. +void Messenger::debug_print(std::string /*s*/) +{ + // print 's' when debugging +} + +} // namespace MultiProcess +} // namespace RooFit diff --git a/roofit/multiprocess/src/ProcessManager.cxx b/roofit/multiprocess/src/ProcessManager.cxx new file mode 100644 index 0000000000000..e9a00bff766c3 --- /dev/null +++ b/roofit/multiprocess/src/ProcessManager.cxx @@ -0,0 +1,306 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#include "RooFit/MultiProcess/ProcessManager.h" +#include "RooFit/MultiProcess/JobManager.h" +#include "RooFit/MultiProcess/util.h" + +#include // for strsignal +#include // for wait +#include +#include + +namespace RooFit { +namespace MultiProcess { + +/// \class ProcessManager +/// \brief Fork processes for queue and workers +/// +/// This class manages three types of processes: +/// 1. master: the initial main process. It defines and enqueues tasks +/// and processes results. +/// 2. workers: a pool of processes that will try to take tasks from the +/// queue. These are forked from master. +/// 3. queue: This process runs the queue_loop and maintains the queue of +/// tasks. It is also forked from master. +/// +/// \param N_workers Number of worker processes to spawn. +ProcessManager::ProcessManager(std::size_t N_workers) : N_workers_(N_workers) +{ + initialize_processes(); +} + +ProcessManager::~ProcessManager() +{ + if (is_master()) { + terminate(); + } else { + wait_for_sigterm_then_exit(); + } +} + +// static member initialization +volatile sig_atomic_t ProcessManager::sigterm_received_ = 0; + +// static function +/// We need this to tell the children to die, because we can't talk +/// to them anymore during JobManager destruction, because that kills +/// the Messenger first. We do that with SIGTERMs. The sigterm_received() +/// should be checked in message loops to stop them when it's true. +void ProcessManager::handle_sigterm(int /*signum*/) +{ + sigterm_received_ = 1; +} + +// static function +bool ProcessManager::sigterm_received() +{ + if (sigterm_received_ > 0) { + return true; + } else { + return false; + } +} + +pid_t fork_and_handle_errors() +{ + pid_t child_pid = fork(); + int retries = 0; + while (child_pid == -1) { + if (retries < 3) { + ++retries; + printf("fork returned with error number %d, retrying after 1 second...\n", errno); + sleep(1); + child_pid = fork(); + } else { + printf("fork returned with error number %d\n", errno); + throw std::runtime_error("fork returned with error 3 times, aborting!"); + } + } + return child_pid; +} + +/// \brief Fork processes and activate CPU pinning +/// +/// \param cpu_pinning Activate CPU pinning if true. Effective on Linux only. +void ProcessManager::initialize_processes(bool cpu_pinning) +{ + // Initialize processes; + // ... first workers: + worker_pids_.resize(N_workers_); + pid_t child_pid{}; + for (std::size_t ix = 0; ix < N_workers_; ++ix) { + child_pid = fork_and_handle_errors(); + if (!child_pid) { // we're on the worker + is_worker_ = true; + worker_id_ = ix; + break; + } else { // we're on master + worker_pids_[ix] = child_pid; + } + } + + // ... then queue: + if (child_pid) { // we're on master + queue_pid_ = fork_and_handle_errors(); + if (!queue_pid_) { // we're now on queue + is_queue_ = true; + } else { + is_master_ = true; + } + } + + // set the sigterm handler on the child processes + if (!is_master_) { + struct sigaction sa; + memset(&sa, '\0', sizeof(sa)); + sa.sa_handler = ProcessManager::handle_sigterm; + + if (sigaction(SIGTERM, &sa, NULL) < 0) { + std::perror("sigaction failed"); + std::exit(1); + } + } + + if (cpu_pinning) { +#if defined(__APPLE__) +#ifndef NDEBUG + static bool affinity_warned = false; + if (is_master() & !affinity_warned) { + std::cout << "CPU affinity cannot be set on macOS" << std::endl; + affinity_warned = true; + } +#endif // NDEBUG +#elif defined(_WIN32) +#ifndef NDEBUG + if (is_master()) + std::cerr << "WARNING: CPU affinity setting not implemented on Windows, continuing..." << std::endl; +#endif // NDEBUG +#else + cpu_set_t mask; + // zero all bits in mask + CPU_ZERO(&mask); + // set correct bit + std::size_t set_cpu; + if (is_master()) { + set_cpu = N_workers() + 1; + } else if (is_queue()) { + set_cpu = N_workers(); + } else { + set_cpu = worker_id(); + } + CPU_SET(set_cpu, &mask); +#ifndef NDEBUG + // sched_setaffinity returns 0 on success + if (sched_setaffinity(0, sizeof(mask), &mask) == -1) { + std::cerr << "WARNING: Could not set CPU affinity, continuing..." << std::endl; + } else { + std::cerr << "CPU affinity set to cpu " << set_cpu << " in process " << getpid() << std::endl; + } +#endif // NDEBUG +#endif + } + +#ifndef NDEBUG + identify_processes(); +#endif // NDEBUG + + initialized_ = true; +} + +bool ProcessManager::is_initialized() const +{ + return initialized_; +} + +/// Shutdown forked processes if on master and if this process manager is initialized +void ProcessManager::terminate() noexcept +{ + try { + if (is_master() && is_initialized()) { + shutdown_processes(); + } + } catch (const std::exception &e) { + std::cerr << "WARNING: something in ProcessManager::terminate threw an exception! Original exception message:\n" + << e.what() << std::endl; + } +} + +void ProcessManager::wait_for_sigterm_then_exit() +{ + if (!is_master()) { + while (!sigterm_received()) { + } + std::_Exit(0); + } +} + +int chill_wait() +{ + int status = 0; + pid_t pid; + do { + pid = wait(&status); + } while (-1 == pid && EINTR == errno); // retry on interrupted system call + + if (0 != status) { + if (WIFEXITED(status)) { + printf("exited, status=%d\n", WEXITSTATUS(status)); + } else if (WIFSIGNALED(status)) { + if (WTERMSIG(status) != SIGTERM) { + printf("killed by signal %d\n", WTERMSIG(status)); + } + } else if (WIFSTOPPED(status)) { + printf("stopped by signal %d\n", WSTOPSIG(status)); + } else if (WIFCONTINUED(status)) { + printf("continued\n"); + } + } + + if (-1 == pid) { + if (errno == ECHILD) { + printf("chill_wait: no children (got ECHILD error code from wait call), done\n"); + } else { + throw std::runtime_error(std::string("chill_wait: error in wait call: ") + strerror(errno) + + std::string(", errno ") + std::to_string(errno)); + } + } + + return pid; +} + +/// Shutdown forked processes if on master +void ProcessManager::shutdown_processes() +{ + if (is_master()) { + // terminate all children + std::unordered_set children; + children.insert(queue_pid_); + kill(queue_pid_, SIGTERM); + for (auto pid : worker_pids_) { + kill(pid, SIGTERM); + children.insert(pid); + } + // then wait for them to actually die and clean out the zombies + while (!children.empty()) { + pid_t pid = chill_wait(); + children.erase(pid); + } + } + + initialized_ = false; +} + +// Getters + +bool ProcessManager::is_master() const +{ + return is_master_; +} + +bool ProcessManager::is_queue() const +{ + return is_queue_; +} + +bool ProcessManager::is_worker() const +{ + return is_worker_; +} + +std::size_t ProcessManager::worker_id() const +{ + return worker_id_; +} + +std::size_t ProcessManager::N_workers() const +{ + return N_workers_; +} + +/// Print to stdout which type of process we are on and what its PID is (for debugging) +void ProcessManager::identify_processes() const +{ + if (is_worker_) { + printf("I'm a worker, PID %d\n", getpid()); + } else if (is_master_) { + printf("I'm master, PID %d\n", getpid()); + } else if (is_queue_) { + printf("I'm queue, PID %d\n", getpid()); + } else { + printf("I'm not master, queue or worker, weird! PID %d\n", getpid()); + } +} + +} // namespace MultiProcess +} // namespace RooFit diff --git a/roofit/multiprocess/src/Queue.cxx b/roofit/multiprocess/src/Queue.cxx new file mode 100644 index 0000000000000..f73d543e26522 --- /dev/null +++ b/roofit/multiprocess/src/Queue.cxx @@ -0,0 +1,175 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#include "RooFit/MultiProcess/Queue.h" +#include "RooFit/MultiProcess/JobManager.h" +#include "RooFit/MultiProcess/ProcessManager.h" +#include "RooFit/MultiProcess/Job.h" // complete Job object for JobManager::get_job_object() +#include "RooFit/MultiProcess/util.h" + +namespace RooFit { +namespace MultiProcess { + +/** \class Queue + * \brief Keeps a queue of tasks for workers and manages the queue process through its event loop + * + * The Queue maintains a set of tasks on the queue process by receiving them + * from the master process. Worker processes can request to pop them off the + * queue. The communication between these processes is handled inside + * 'Queue::loop()', the queue process's event loop that polls the Messenger's + * sockets for incoming messages and handles them when they come. + * + * The reason for this class is to get automatic load balancing between + * workers. By allowing workers to request tasks whenever they are ready to + * do work, we don't need to manually distribute work over workers and they + * will always have something to do until all tasks have been completed. + * The alternative simple strategy of just distributing all tasks evenly over + * workers will be suboptimal when tasks have different or even varying + * runtimes (this simple strategy could be implemented with a PUSH-PULL + * ZeroMQ socket from master to workers, which would distribute tasks in a + * round-robin fashion, which, indeed, does not do load balancing). + */ + +/// Have a worker ask for a task-message from the queue +/// +/// \param[out] job_task JobTask reference to put the Job ID and the task index into. +/// \return true if a task was popped from the queue successfully, false if the queue was empty. +bool Queue::pop(JobTask &job_task) +{ + if (queue_.empty()) { + return false; + } else { + job_task = queue_.front(); + queue_.pop(); + return true; + } +} + +/// Enqueue a task +/// +/// \param[in] job_task JobTask object that contains the Job ID and the task index. +void Queue::add(JobTask job_task) +{ + if (JobManager::instance()->process_manager().is_master()) { + JobManager::instance()->messenger().send_from_master_to_queue(M2Q::enqueue, job_task.job_id, job_task.state_id, + job_task.task_id); + } else if (JobManager::instance()->process_manager().is_queue()) { + queue_.push(job_task); + } else { + throw std::logic_error("calling Communicator::to_master_queue from slave process"); + } +} + +/// Helper function for 'Queue::loop()' +void Queue::process_master_message(M2Q message) +{ + switch (message) { + case M2Q::enqueue: { + // enqueue task + auto job_object_id = JobManager::instance()->messenger().receive_from_master_on_queue(); + auto state_id = JobManager::instance()->messenger().receive_from_master_on_queue(); + auto task_id = JobManager::instance()->messenger().receive_from_master_on_queue(); + JobTask job_task{job_object_id, state_id, task_id}; + add(job_task); + N_tasks_++; + break; + } + } +} + +/// Helper function for 'Queue::loop()' +void Queue::process_worker_message(std::size_t this_worker_id, W2Q message) +{ + switch (message) { + case W2Q::dequeue: { + // dequeue task + JobTask job_task; + bool popped = pop(job_task); + if (popped) { + // Note: below two commands should be run atomically for thread safety (if that ever becomes an issue) + JobManager::instance()->messenger().send_from_queue_to_worker( + this_worker_id, Q2W::dequeue_accepted, job_task.job_id, job_task.state_id, job_task.task_id); + ++N_tasks_at_workers_; + } else { + JobManager::instance()->messenger().send_from_queue_to_worker(this_worker_id, Q2W::dequeue_rejected); + } + break; + } + } +} + +/// \brief The queue process's event loop +/// +/// Polls for incoming messages from other processes and handles them. +void Queue::loop() +{ + assert(JobManager::instance()->process_manager().is_queue()); + ZeroMQPoller poller; + std::size_t mq_index; + std::tie(poller, mq_index) = JobManager::instance()->messenger().create_queue_poller(); + + // Before blocking SIGTERM, set the signal handler, so we can also check after blocking whether a signal occurred + // In our case, we already set it in the ProcessManager after forking to the queue and worker processes. + + sigset_t sigmask; + sigemptyset(&sigmask); + sigaddset(&sigmask, SIGTERM); + sigprocmask(SIG_BLOCK, &sigmask, &JobManager::instance()->messenger().ppoll_sigmask); + + // Before doing anything, check whether we have received a terminate signal while blocking signals! + // In this case, we also do that in the while condition. + while (!ProcessManager::sigterm_received()) { + try { // watch for zmq_error from ppoll caused by SIGTERM from master + // poll: wait until status change (-1: infinite timeout) + auto poll_result = poller.ppoll(-1, &JobManager::instance()->messenger().ppoll_sigmask); + // then process incoming messages from sockets + for (auto readable_socket : poll_result) { + // message comes from the master/queue socket (first element): + if (readable_socket.first == mq_index) { + auto message = JobManager::instance()->messenger().receive_from_master_on_queue(); + process_master_message(message); + } else { // from a worker socket + auto this_worker_id = readable_socket.first - 1; + auto message = JobManager::instance()->messenger().receive_from_worker_on_queue(this_worker_id); + process_worker_message(this_worker_id, message); + } + } + } catch (ZMQ::ppoll_error_t &e) { + zmq_ppoll_error_response response; + try { + response = handle_zmq_ppoll_error(e); + } catch (std::logic_error &) { + printf("queue loop got unhandleable ZMQ::ppoll_error_t\n"); + throw; + } + if (response == zmq_ppoll_error_response::abort) { + break; + } else if (response == zmq_ppoll_error_response::unknown_eintr) { + printf("EINTR in queue loop but no SIGTERM received, continuing\n"); + continue; + } else if (response == zmq_ppoll_error_response::retry) { + printf("EAGAIN from ppoll in queue loop, continuing\n"); + continue; + } + } catch (zmq::error_t &e) { + printf("unhandled zmq::error_t (not a ppoll_error_t) in queue loop with errno %d: %s\n", e.num(), e.what()); + throw; + } + } + + // clean up signal management modifications + sigprocmask(SIG_SETMASK, &JobManager::instance()->messenger().ppoll_sigmask, nullptr); +} + +} // namespace MultiProcess +} // namespace RooFit diff --git a/roofit/multiprocess/src/util.cxx b/roofit/multiprocess/src/util.cxx new file mode 100644 index 0000000000000..8a361ba688229 --- /dev/null +++ b/roofit/multiprocess/src/util.cxx @@ -0,0 +1,130 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#include "RooFit/MultiProcess/util.h" +#include "RooFit/MultiProcess/ProcessManager.h" + +#include // kill, SIGKILL +#include // cerr, and indirectly WNOHANG, EINTR, W* macros +#include // runtime_error +#include // waitpid +#include + +namespace RooFit { +namespace MultiProcess { + +int wait_for_child(pid_t child_pid, bool may_throw, int retries_before_killing) +{ + int status = 0; + int patience = retries_before_killing; + pid_t tmp; + do { + if (patience-- < 1) { + ::kill(child_pid, SIGKILL); + } + tmp = waitpid(child_pid, &status, WNOHANG); + } while (tmp == 0 // child has not yet changed state, try again + || (-1 == tmp && EINTR == errno) // retry on interrupted system call + ); + + if (patience < 1) { + std::cout << "Had to send PID " << child_pid << " " << (-patience + 1) << " SIGKILLs"; + } + + if (0 != status) { + if (WIFEXITED(status)) { + printf("exited, status=%d\n", WEXITSTATUS(status)); + } else if (WIFSIGNALED(status)) { + if (WTERMSIG(status) != SIGTERM) { + printf("killed by signal %d\n", WTERMSIG(status)); + } + } else if (WIFSTOPPED(status)) { + printf("stopped by signal %d\n", WSTOPSIG(status)); + } else if (WIFCONTINUED(status)) { + printf("continued\n"); + } + } + + if (-1 == tmp && may_throw) + throw std::runtime_error(std::string("waitpid, errno ") + std::to_string(errno)); + + return status; +} + +zmq_ppoll_error_response handle_zmq_ppoll_error(ZMQ::ppoll_error_t &e) +{ + if ((e.num() == EINTR) && (ProcessManager::sigterm_received())) { + // valid EINTR, because we want to exit and kill the processes on SIGTERM + return zmq_ppoll_error_response::abort; + } else if (e.num() == EINTR) { + // on other EINTRs, we retry (mostly this happens in debuggers) + return zmq_ppoll_error_response::unknown_eintr; + } else if (e.num() == EAGAIN) { + // This can happen from recv if ppoll initially gets a read-ready signal for a socket, + // but the received data does not pass the checksum test, so the socket becomes unreadable + // again or from non-blocking send if the socket becomes unwritable either due to the HWM + // being reached or the socket not being connected (anymore). The latter case usually means + // the connection has been severed from the other side, meaning it has probably been killed + // and in that case the next ppoll call will probably also receive a SIGTERM, ending the + // loop. In case something else is wrong, this message will print multiple times, which + // should be taken as a cue for writing a bug report :) + return zmq_ppoll_error_response::retry; + } else { + char buffer[512]; + snprintf(buffer, 512, + "handle_zmq_ppoll_error is out of options to handle exception, caught ZMQ::ppoll_error_t had errno %d " + "and text: %s\n", + e.num(), e.what()); + throw std::logic_error(buffer); + } +} + +// returns a tuple containing first the poll result and second a boolean flag that tells the caller whether it should +// abort the enclosing loop +std::tuple>, bool> +careful_ppoll(ZeroMQPoller &poller, const sigset_t &ppoll_sigmask, std::size_t max_tries) +{ + std::size_t tries = 0; + std::vector> poll_result; + bool abort = true; + bool carry_on = true; + while (carry_on && (tries++ < max_tries)) { + if (tries > 1) { + printf("careful_ppoll try %zu\n", tries); + } + try { // watch for zmq_error from ppoll caused by SIGTERM from master + poll_result = poller.ppoll(-1, &ppoll_sigmask); + abort = false; + carry_on = false; + } catch (ZMQ::ppoll_error_t &e) { + auto response = handle_zmq_ppoll_error(e); + if (response == zmq_ppoll_error_response::abort) { + break; + } else if (response == zmq_ppoll_error_response::unknown_eintr) { + printf("EINTR in careful_ppoll but no SIGTERM received, try %zu\n", tries); + continue; + } else if (response == zmq_ppoll_error_response::retry) { + printf("EAGAIN in careful_ppoll (from either send or receive), try %zu\n", tries); + continue; + } + } + } + + if (tries == max_tries) { + printf("careful_ppoll reached maximum number of tries, %zu, please report as a bug\n", tries); + } + return std::make_tuple(poll_result, abort); +} + +} // namespace MultiProcess +} // namespace RooFit \ No newline at end of file diff --git a/roofit/multiprocess/src/worker.cxx b/roofit/multiprocess/src/worker.cxx new file mode 100644 index 0000000000000..fb85e3078e31c --- /dev/null +++ b/roofit/multiprocess/src/worker.cxx @@ -0,0 +1,150 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#include "RooFit/MultiProcess/worker.h" + +#include "RooFit/MultiProcess/JobManager.h" +#include "RooFit/MultiProcess/types.h" +#include "RooFit/MultiProcess/Messenger.h" +#include "RooFit/MultiProcess/Job.h" +#include "RooFit/MultiProcess/util.h" + +#include // getpid, pid_t +#include // EINTR +#include // sigprocmask etc + +namespace RooFit { +namespace MultiProcess { + +static bool worker_loop_running = false; + +bool is_worker_loop_running() +{ + return worker_loop_running; +} + +/// \brief The worker processes' event loop +/// +/// Asks the queue process for tasks, polls for incoming messages from other +/// processes and handles them. +void worker_loop() +{ + assert(JobManager::instance()->process_manager().is_worker()); + worker_loop_running = true; + Q2W message_q2w; + + // use a flag to not ask twice + bool dequeue_acknowledged = true; + + ZeroMQPoller poller; + std::size_t mw_sub_index; + + std::tie(poller, mw_sub_index) = JobManager::instance()->messenger().create_worker_poller(); + + // Before blocking SIGTERM, set the signal handler, so we can also check after blocking whether a signal occurred + // In our case, we already set it in the ProcessManager after forking to the queue and worker processes. + + sigset_t sigmask; + sigemptyset(&sigmask); + sigaddset(&sigmask, SIGTERM); + sigprocmask(SIG_BLOCK, &sigmask, &JobManager::instance()->messenger().ppoll_sigmask); + + // Before doing anything, check whether we have received a terminate signal while blocking signals! + // In this case, we also do that in the while condition. + while (!ProcessManager::sigterm_received()) { + try { // watch for error from ppoll (which is called inside receive functions) caused by SIGTERM from master + + // try to dequeue a task + if (dequeue_acknowledged) { // don't ask twice + JobManager::instance()->messenger().send_from_worker_to_queue(W2Q::dequeue); + dequeue_acknowledged = false; + } + + // wait for handshake from queue or update from SUB socket + auto poll_result = poller.ppoll(-1, &JobManager::instance()->messenger().ppoll_sigmask); + // because the poller may now have a waiting update from master over the SUB socket, + // but the queue socket could be first in the poll_result vector, and during handling + // of a new task it is possible we need to already receive the updated state over SUB, + // we have to then flip this boolean so that in the for loop when we reach the SUB + // socket's result, we can skip it (otherwise we will hang there, because no more + // updated state will be coming): + bool skip_sub = false; + // then process incoming messages from sockets + for (auto readable_socket : poll_result) { + // message comes from the master-worker SUB socket (first element): + if (readable_socket.first == mw_sub_index) { + if (!skip_sub) { + auto job_id = JobManager::instance()->messenger().receive_from_master_on_worker(); + JobManager::get_job_object(job_id)->update_state(); + } + } else { // from queue socket + message_q2w = JobManager::instance()->messenger().receive_from_queue_on_worker(); + switch (message_q2w) { + case Q2W::dequeue_rejected: { + dequeue_acknowledged = true; + break; + } + case Q2W::dequeue_accepted: { + dequeue_acknowledged = true; + auto job_id = JobManager::instance()->messenger().receive_from_queue_on_worker(); + auto state_id = JobManager::instance()->messenger().receive_from_queue_on_worker(); + auto task_id = JobManager::instance()->messenger().receive_from_queue_on_worker(); + + // while loop, because multiple jobs may have updated state coming + while (state_id != JobManager::get_job_object(job_id)->get_state_id()) { + skip_sub = true; + auto job_id_for_state = + JobManager::instance()->messenger().receive_from_master_on_worker(); + JobManager::get_job_object(job_id_for_state)->update_state(); + } + + JobManager::get_job_object(job_id)->evaluate_task(task_id); + JobManager::get_job_object(job_id)->send_back_task_result_from_worker(task_id); + + break; + } + } + } + } + + } catch (ZMQ::ppoll_error_t &e) { + zmq_ppoll_error_response response; + try { + response = handle_zmq_ppoll_error(e); + } catch (std::logic_error &) { + printf("worker loop at PID %d got unhandleable ZMQ::ppoll_error_t\n", getpid()); + throw; + } + if (response == zmq_ppoll_error_response::abort) { + break; + } else if (response == zmq_ppoll_error_response::unknown_eintr) { + printf("EINTR in worker loop at PID %d but no SIGTERM received, continuing\n", getpid()); + continue; + } else if (response == zmq_ppoll_error_response::retry) { + printf("EAGAIN from ppoll in worker loop at PID %d, continuing\n", getpid()); + continue; + } + } catch (zmq::error_t &e) { + printf("unhandled zmq::error_t (not a ppoll_error_t) in worker loop at PID %d with errno %d: %s\n", getpid(), + e.num(), e.what()); + throw; + } + } + // clean up signal management modifications + sigprocmask(SIG_SETMASK, &JobManager::instance()->messenger().ppoll_sigmask, nullptr); + + worker_loop_running = false; +} + +} // namespace MultiProcess +} // namespace RooFit diff --git a/roofit/multiprocess/test/CMakeLists.txt b/roofit/multiprocess/test/CMakeLists.txt new file mode 100644 index 0000000000000..5e8392bcd06db --- /dev/null +++ b/roofit/multiprocess/test/CMakeLists.txt @@ -0,0 +1,16 @@ +# @author Patrick Bos, NL eScience Center, 2019-2021 + +add_library(RooFit_multiprocess_testing_utils INTERFACE) +target_link_libraries(RooFit_multiprocess_testing_utils INTERFACE RooFitCore RooBatchCompute) +target_include_directories(RooFit_multiprocess_testing_utils PRIVATE ${RooFitMultiProcess_INCLUDE_DIRS}) + +ROOT_ADD_GTEST(test_RooFit_MultiProcess_Job test_Job.cxx LIBRARIES RooFitMultiProcess) +# link to the INTERFACE library separately, ROOT_EXECUTABLE cannot handle INTERFACE library properties: +target_link_libraries(test_RooFit_MultiProcess_Job RooFit_multiprocess_testing_utils) +target_include_directories(test_RooFit_MultiProcess_Job PRIVATE ${RooFitMultiProcess_INCLUDE_DIRS}) + +ROOT_ADD_GTEST(test_RooFit_MultiProcess_ProcessManager test_ProcessManager.cxx LIBRARIES RooFitMultiProcess) +target_include_directories(test_RooFit_MultiProcess_ProcessManager PRIVATE ${RooFitMultiProcess_INCLUDE_DIRS}) + +ROOT_ADD_GTEST(test_RooFit_MultiProcess_Messenger test_Messenger.cxx LIBRARIES RooFitMultiProcess) +target_include_directories(test_RooFit_MultiProcess_Messenger PRIVATE ${RooFitMultiProcess_INCLUDE_DIRS}) diff --git a/roofit/multiprocess/test/test_Job.cxx b/roofit/multiprocess/test/test_Job.cxx new file mode 100644 index 0000000000000..017dd87a83562 --- /dev/null +++ b/roofit/multiprocess/test/test_Job.cxx @@ -0,0 +1,247 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * VC, Vince Croft, DIANA / NYU, vincent.croft@cern.ch + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#include +#include + +#include "RooFit/MultiProcess/Job.h" +#include "RooFit/MultiProcess/types.h" // JobTask +// needed to complete type returned from... +#include "RooFit/MultiProcess/JobManager.h" // ... Job::get_manager() +#include "RooFit/MultiProcess/ProcessManager.h" // ... JobManager::process_manager() +#include "RooFit/MultiProcess/Queue.h" // ... JobManager::queue() + +#include "gtest/gtest.h" +#include "utils.h" + +class xSquaredPlusBVectorSerial { +public: + xSquaredPlusBVectorSerial(double b, const std::vector &x) : b_(b), x_(x), result_(x.size()) {} + + void evaluate() + { + // call evaluate_task for each task + for (std::size_t ix = 0; ix < x_.size(); ++ix) { + result_[ix] = std::pow(x_[ix], 2) + b_; + } + } + + std::vector get_result() + { + evaluate(); + return result_; + } + + // for simplicity of the example (avoiding getters/setters) we make data members public as well + double b_; + std::vector x_; + std::vector result_; +}; + +class xSquaredPlusBVectorParallel : public RooFit::MultiProcess::Job { +public: + explicit xSquaredPlusBVectorParallel(xSquaredPlusBVectorSerial *serial, bool update_state = false) + : serial_(serial), update_state_(update_state) + { + } + + void evaluate() + { + if (get_manager()->process_manager().is_master()) { + if (update_state_) { + update_state(); + } + // master fills queue with tasks + for (std::size_t task_id = 0; task_id < serial_->x_.size(); ++task_id) { + RooFit::MultiProcess::JobTask job_task{id_, state_id_, task_id}; + get_manager()->queue().add(job_task); + ++N_tasks_at_workers_; + } + + // wait for task results back from workers to master + gather_worker_results(); + } + } + + std::vector get_result() + { + evaluate(); + return serial_->result_; + } + + // -- BEGIN plumbing -- + + // typically, on master, this would be called inside evaluate, before queuing tasks; on workers it's called + // automatically when a published message is received from master + void update_state() override + { + if (get_manager()->process_manager().is_master()) { + ++state_id_; + get_manager()->messenger().publish_from_master_to_workers( + id_, state_id_, serial_->b_); // always send Job id first! This is used in worker_loop to route the + // update_state call to the correct Job. + } else if (get_manager()->process_manager().is_worker()) { + state_id_ = get_manager()->messenger().receive_from_master_on_worker(); + serial_->b_ = get_manager()->messenger().receive_from_master_on_worker(); + } + } + + struct task_result_t { + std::size_t job_id; // job ID must always be the first part of any result message/type + std::size_t task_id; + double value; + }; + + void send_back_task_result_from_worker(std::size_t task) override + { + task_result_t task_result{id_, task, serial_->result_[task]}; + zmq::message_t message(sizeof(task_result_t)); + memcpy(message.data(), &task_result, sizeof(task_result_t)); + get_manager()->messenger().send_from_worker_to_master(std::move(message)); + } + + bool receive_task_result_on_master(const zmq::message_t &message) override + { + auto result = message.data(); + serial_->result_[result->task_id] = result->value; + --N_tasks_at_workers_; + bool job_completed = (N_tasks_at_workers_ == 0); + return job_completed; + } + + // -- END plumbing -- + +private: + void evaluate_task(std::size_t task) override + { + assert(get_manager()->process_manager().is_worker()); + serial_->result_[task] = std::pow(serial_->x_[task], 2) + serial_->b_; + } + + xSquaredPlusBVectorSerial *serial_; + bool update_state_ = false; + std::size_t N_tasks_at_workers_ = 0; +}; + +class TestMPJob : public ::testing::TestWithParam { + // You can implement all the usual fixture class members here. + // To access the test parameter, call GetParam() from class + // TestWithParam. +}; + +TEST_P(TestMPJob, singleJobGetResult) +{ + // Simple test case: calculate x^2 + b, where x is a vector. This case does + // both a simple calculation (squaring the input vector x) and represents + // handling of state updates in b. + std::vector x{0, 1, 2, 3}; + double b_initial = 3.; + + // start serial test + + xSquaredPlusBVectorSerial x_sq_plus_b(b_initial, x); + + auto y = x_sq_plus_b.get_result(); + std::vector y_expected{3, 4, 7, 12}; + + EXPECT_EQ(Hex(y[0]), Hex(y_expected[0])); + EXPECT_EQ(Hex(y[1]), Hex(y_expected[1])); + EXPECT_EQ(Hex(y[2]), Hex(y_expected[2])); + EXPECT_EQ(Hex(y[3]), Hex(y_expected[3])); + + std::size_t NumCPU = GetParam(); + + // start parallel test + + xSquaredPlusBVectorParallel x_sq_plus_b_parallel(&x_sq_plus_b); + RooFit::MultiProcess::JobManager::default_N_workers = NumCPU; + + auto y_parallel = x_sq_plus_b_parallel.get_result(); + + EXPECT_EQ(Hex(y_parallel[0]), Hex(y_expected[0])); + EXPECT_EQ(Hex(y_parallel[1]), Hex(y_expected[1])); + EXPECT_EQ(Hex(y_parallel[2]), Hex(y_expected[2])); + EXPECT_EQ(Hex(y_parallel[3]), Hex(y_expected[3])); +} + +TEST_P(TestMPJob, multiJobGetResult) +{ + // Simple test case: calculate x^2 + b, where x is a vector. This case does + // both a simple calculation (squaring the input vector x) and represents + // handling of state updates in b. + std::vector x{0, 1, 2, 3}; + double b_initial = 3.; + + std::vector y_expected{3, 4, 7, 12}; + + std::size_t NumCPU = GetParam(); + + // define jobs + xSquaredPlusBVectorSerial x_sq_plus_b(b_initial, x); + xSquaredPlusBVectorSerial x_sq_plus_b2(b_initial + 1, x); + xSquaredPlusBVectorParallel x_sq_plus_b_parallel(&x_sq_plus_b); + xSquaredPlusBVectorParallel x_sq_plus_b_parallel2(&x_sq_plus_b2); + RooFit::MultiProcess::JobManager::default_N_workers = NumCPU; + + // do stuff + auto y_parallel = x_sq_plus_b_parallel.get_result(); + auto y_parallel2 = x_sq_plus_b_parallel2.get_result(); + + EXPECT_EQ(Hex(y_parallel[0]), Hex(y_expected[0])); + EXPECT_EQ(Hex(y_parallel[1]), Hex(y_expected[1])); + EXPECT_EQ(Hex(y_parallel[2]), Hex(y_expected[2])); + EXPECT_EQ(Hex(y_parallel[3]), Hex(y_expected[3])); + + EXPECT_EQ(Hex(y_parallel2[0]), Hex(y_expected[0] + 1)); + EXPECT_EQ(Hex(y_parallel2[1]), Hex(y_expected[1] + 1)); + EXPECT_EQ(Hex(y_parallel2[2]), Hex(y_expected[2] + 1)); + EXPECT_EQ(Hex(y_parallel2[3]), Hex(y_expected[3] + 1)); +} + +TEST_P(TestMPJob, singleJobUpdateState) +{ + // Simple test case: calculate x^2 + b, where x is a vector. This case does + // both a simple calculation (squaring the input vector x) and represents + // handling of state updates in b. + std::vector x{0, 1, 2, 3}; + double b_initial = 1.; + xSquaredPlusBVectorSerial x_sq_plus_b(b_initial, x); + + std::vector y_expected{3, 4, 7, 12}; + + std::size_t NumCPU = GetParam(); + + // start parallel test + bool update_state = true; + xSquaredPlusBVectorParallel x_sq_plus_b_parallel(&x_sq_plus_b, update_state); + RooFit::MultiProcess::JobManager::default_N_workers = NumCPU; + + auto y_parallel_before_change = x_sq_plus_b_parallel.get_result(); + + x_sq_plus_b.b_ = 3.; + + auto y_parallel_after_change = x_sq_plus_b_parallel.get_result(); + + EXPECT_NE(Hex(y_parallel_before_change[0]), Hex(y_expected[0])); + EXPECT_NE(Hex(y_parallel_before_change[1]), Hex(y_expected[1])); + EXPECT_NE(Hex(y_parallel_before_change[2]), Hex(y_expected[2])); + EXPECT_NE(Hex(y_parallel_before_change[3]), Hex(y_expected[3])); + + EXPECT_EQ(Hex(y_parallel_after_change[0]), Hex(y_expected[0])); + EXPECT_EQ(Hex(y_parallel_after_change[1]), Hex(y_expected[1])); + EXPECT_EQ(Hex(y_parallel_after_change[2]), Hex(y_expected[2])); + EXPECT_EQ(Hex(y_parallel_after_change[3]), Hex(y_expected[3])); +} + +INSTANTIATE_TEST_SUITE_P(NumberOfWorkerProcesses, TestMPJob, ::testing::Values(1, 2, 3)); diff --git a/roofit/multiprocess/test/test_Messenger.cxx b/roofit/multiprocess/test/test_Messenger.cxx new file mode 100644 index 0000000000000..0db05100328c6 --- /dev/null +++ b/roofit/multiprocess/test/test_Messenger.cxx @@ -0,0 +1,238 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#include "RooFit/MultiProcess/Messenger.h" +#include "RooFit/MultiProcess/ProcessManager.h" // ... JobManager::process_manager() + +#include "gtest/gtest.h" + +void handle_sigchld(int signum) +{ + printf("handled %s on PID %d\n", strsignal(signum), getpid()); +} + +TEST(TestMPMessenger, Connections) +{ + struct sigaction sa; + RooFit::MultiProcess::ProcessManager pm(4); + if (pm.is_master()) { + // on master, we have to handle SIGCHLD + memset(&sa, '\0', sizeof(sa)); + sa.sa_handler = handle_sigchld; + if (sigaction(SIGCHLD, &sa, NULL) < 0) { + std::perror("sigaction failed"); + std::exit(1); + } + } + RooFit::MultiProcess::Messenger messenger(pm); + if (pm.is_master()) { + // more SIGCHLD handling + sigset_t sigmask; + sigemptyset(&sigmask); + sigaddset(&sigmask, SIGCHLD); + int rc = sigprocmask(SIG_BLOCK, &sigmask, &messenger.ppoll_sigmask); + if (rc < 0) { + throw std::runtime_error("sigprocmask failed in TestMPMessenger.Connections"); + } + } + messenger.test_connections(pm); + if (pm.is_master()) { + // clean up signal management modifications + sigprocmask(SIG_SETMASK, &messenger.ppoll_sigmask, nullptr); + sa.sa_handler = SIG_DFL; + if (sigaction(SIGCHLD, &sa, NULL) < 0) { + std::perror("sigaction failed"); + std::exit(1); + } + } +} + +TEST(TestMPMessenger, ConnectionsManualExit) +{ + // the point of this test is to see whether clean-up of ZeroMQ resources is done properly without calling any + // destructors (which is what happens when you call _Exit() instead of regularly ending the program by reaching the + // end of main()). + + struct sigaction sa; + + RooFit::MultiProcess::ProcessManager pm(4); + if (pm.is_master()) { + // on master, we have to handle SIGCHLD + memset(&sa, '\0', sizeof(sa)); + sa.sa_handler = handle_sigchld; + if (sigaction(SIGCHLD, &sa, NULL) < 0) { + std::perror("sigaction failed"); + std::exit(1); + } + } + RooFit::MultiProcess::Messenger messenger(pm); + if (pm.is_master()) { + // more SIGCHLD handling + sigset_t sigmask; + sigemptyset(&sigmask); + sigaddset(&sigmask, SIGCHLD); + int rc = sigprocmask(SIG_BLOCK, &sigmask, &messenger.ppoll_sigmask); + if (rc < 0) { + throw std::runtime_error("sigprocmask failed in TestMPMessenger.Connections"); + } + } + messenger.test_connections(pm); + if (!pm.is_master()) { + // just wait until we get terminated + while (!RooFit::MultiProcess::ProcessManager::sigterm_received()) { + }; + std::_Exit(0); + } else { + pm.terminate(); + } + if (pm.is_master()) { + // clean up signal management modifications + sigprocmask(SIG_SETMASK, &messenger.ppoll_sigmask, nullptr); + sa.sa_handler = SIG_DFL; + if (sigaction(SIGCHLD, &sa, NULL) < 0) { + std::perror("sigaction failed"); + std::exit(1); + } + } +} + +#include + +TEST(TestMPMessenger, SigStop) +{ + // Originally, we could not get the debugger (lldb) to play nicely with ZeroMQ; every time we paused a process, it + // said it crashed because of SIGSTOP, but SIGSTOP should just pause it... this test was meant to reproduce this + // behavior and fix it, which was done by more carefully handling exceptions. + RooFit::MultiProcess::ProcessManager pm(2); + RooFit::MultiProcess::Messenger messenger(pm); + + if (pm.is_master()) { + printf("first send message to queue...\n"); + messenger.send_from_master_to_queue(1); + printf("sleep for 2 seconds...\n"); + sleep(2); + printf("SIGSTOPping all children...\n"); + kill(pm.get_queue_pid(), SIGSTOP); + kill(pm.get_worker_pids()[0], SIGSTOP); + kill(pm.get_worker_pids()[1], SIGSTOP); + printf("send another message to queue...\n"); + messenger.send_from_master_to_queue(2); + printf("sleep for 2 seconds...\n"); + sleep(2); + printf("SIGCONT queue and worker 1...\n"); + kill(pm.get_queue_pid(), SIGCONT); + kill(pm.get_worker_pids()[0], SIGCONT); + printf("sleep for 2 seconds...\n"); + sleep(2); + printf("SIGCONT worker 2...\n"); + kill(pm.get_worker_pids()[1], SIGCONT); + EXPECT_EQ(messenger.receive_from_queue_on_master(), 3); + } + + if (pm.is_queue()) { + auto number = messenger.receive_from_master_on_queue(); + printf("received %d on queue\n", number); + number = messenger.receive_from_master_on_queue(); + printf("received %d on queue\n", number); + messenger.send_from_queue_to_master(3); + } + + if (!pm.is_master()) { + while (!pm.sigterm_received()) { + } + } +} + +TEST(TestMPMessenger, DISABLED_StressSigStop) +{ + // The SIGSTOP test failed spuriously on CI at some point. We suspected this was due to some + // improbable race condition caused in some place where SIGSTOP/SIGCONT crashes a process. + // To find this crash, we bombard the processes with signals in this test. + // We were not able to trigger the crash, so we disabled the test, but leave it in for when + // the spurious test resurfaces. + RooFit::MultiProcess::ProcessManager pm(2); + RooFit::MultiProcess::Messenger messenger(pm); + + std::size_t runs = 100; + std::size_t sig_repeats = 10000; + + if (pm.is_master()) { + for (std::size_t ix = 0; ix < runs; ++ix) { + // printf("first send message to queue...\n"); + messenger.send_from_master_to_queue(1); + // printf("SIGSTOPping all children...\n"); + for (std::size_t jx = 0; jx < sig_repeats; ++jx) { + kill(pm.get_queue_pid(), SIGCONT); + kill(pm.get_worker_pids()[0], SIGCONT); + kill(pm.get_worker_pids()[1], SIGCONT); + kill(pm.get_queue_pid(), SIGSTOP); + kill(pm.get_worker_pids()[0], SIGSTOP); + kill(pm.get_worker_pids()[1], SIGSTOP); + } + // printf("send another message to queue...\n"); + messenger.send_from_master_to_queue(2); + + for (std::size_t jx = 0; jx < sig_repeats; ++jx) { + kill(pm.get_queue_pid(), SIGSTOP); + kill(pm.get_worker_pids()[0], SIGSTOP); + kill(pm.get_worker_pids()[1], SIGSTOP); + kill(pm.get_queue_pid(), SIGCONT); + kill(pm.get_worker_pids()[0], SIGCONT); + kill(pm.get_worker_pids()[1], SIGCONT); + } + EXPECT_EQ(messenger.receive_from_queue_on_master(), 3); + } + } + + if (pm.is_queue()) { + for (std::size_t ix = 0; ix < runs; ++ix) { + /*auto number =*/messenger.receive_from_master_on_queue(); + // printf("received %d on queue\n", number); + /*number =*/messenger.receive_from_master_on_queue(); + // printf("received %d on queue\n", number); + messenger.send_from_queue_to_master(3); + } + } + + if (!pm.is_master()) { + while (!pm.sigterm_received()) { + } + } +} + +// This test has been disabled in favor of the pub-sub connection check in the Messenger ctor, which is more robust +TEST(TestMPMessenger, DISABLED_StressPubSub) +{ + std::size_t N_workers = 64; + RooFit::MultiProcess::ProcessManager pm(N_workers); + RooFit::MultiProcess::Messenger messenger(pm); + + if (pm.is_master()) { + messenger.publish_from_master_to_workers("bert"); + std::size_t ernies = 0; + for (; ernies < N_workers; ++ernies) { + auto receipt = messenger.receive_from_worker_on_master(); + if (receipt != "ernie") { + printf("whoops, got %s instead of ernie!\n", receipt.c_str()); + FAIL(); + } + } + EXPECT_EQ(ernies, N_workers); + } else if (pm.is_worker()) { + auto receipt = messenger.receive_from_master_on_worker(); + if (receipt == "bert") { + messenger.send_from_worker_to_master("ernie"); + } else { + printf("no bert on worker %zu\n", pm.worker_id()); + } + } +} diff --git a/roofit/multiprocess/test/test_ProcessManager.cxx b/roofit/multiprocess/test/test_ProcessManager.cxx new file mode 100644 index 0000000000000..7bf34ffa691b3 --- /dev/null +++ b/roofit/multiprocess/test/test_ProcessManager.cxx @@ -0,0 +1,54 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#include "RooFit/MultiProcess/ProcessManager.h" // ... JobManager::process_manager() + +#include "gtest/gtest.h" + +TEST(TestMPProcessManager, birthAndDeath) +{ + auto pm = RooFit::MultiProcess::ProcessManager(2); +} + +// disabled: it is not working anymore (16 March 2021), due to changes introduced to fix Messenger::test_connections +TEST(TestMPProcessManager, DISABLED_multiBirth) +{ + // This test doesn't actually represent a current usecase in RooFit, but let's + // showcase the possibility anyway for future reference. + + // first create a regular pm like in the birthAndDeath test + auto pm = RooFit::MultiProcess::ProcessManager(2); + // then from each forked node, spin up another set of workers+queue! + auto pm2 = RooFit::MultiProcess::ProcessManager(2); +} + +TEST(TestMPProcessManager, checkState) +{ + std::size_t N_workers = 2; + auto master_pid = getpid(); + auto pm = RooFit::MultiProcess::ProcessManager(N_workers); + ASSERT_TRUE(pm.is_initialized()); + ASSERT_EQ(pm.N_workers(), N_workers); + + if (pm.is_master()) { + EXPECT_EQ(getpid(), master_pid); + EXPECT_FALSE(pm.is_queue()); + EXPECT_FALSE(pm.is_worker()); + } else if (pm.is_queue()) { + EXPECT_FALSE(pm.is_master()); + EXPECT_FALSE(pm.is_worker()); + } else if (pm.is_worker()) { + EXPECT_FALSE(pm.is_master()); + EXPECT_FALSE(pm.is_queue()); + EXPECT_TRUE(pm.worker_id() == 0 || pm.worker_id() == 1); + } +} \ No newline at end of file diff --git a/roofit/multiprocess/test/test_worker.cxx b/roofit/multiprocess/test/test_worker.cxx new file mode 100644 index 0000000000000..0b2dd54027572 --- /dev/null +++ b/roofit/multiprocess/test/test_worker.cxx @@ -0,0 +1,27 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#include "RooFit/MultiProcess/worker.h" + +#include "gtest/gtest.h" + +// TODO: implement something along these lines (the following is pseudocode) +// TEST(worker, loop) +//{ +// do_fork(); +// if (on_master()) { +// wait(); +// terminate_worker(); +// } else if (on_worker()) { +// worker_loop(); +// } +//} diff --git a/roofit/multiprocess/test/utils.h b/roofit/multiprocess/test/utils.h new file mode 100644 index 0000000000000..431b7c98b6499 --- /dev/null +++ b/roofit/multiprocess/test/utils.h @@ -0,0 +1,169 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * VC, Vince Croft, DIANA / NYU, vincent.croft@cern.ch + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#ifndef ROOT_ROOFIT_MultiProcess_tests_utils_h +#define ROOT_ROOFIT_MultiProcess_tests_utils_h + +#include + +#include "RooWorkspace.h" +#include "RooRandom.h" +#include "RooAddPdf.h" +#include "RooDataSet.h" +#include "RooRealVar.h" // for the dynamic cast to have a complete type + +std::tuple, std::unique_ptr> +generate_1D_gaussian_pdf_nll(RooWorkspace &w, unsigned long N_events) +{ + w.factory("Gaussian::g(x[-5,5],mu[0,-3,3],sigma[1])"); + + RooAbsPdf *pdf = w.pdf("g"); + RooRealVar *mu = w.var("mu"); + + RooDataSet *data = pdf->generate(RooArgSet(*w.var("x")), N_events); + mu->setVal(-2.9); + + std::unique_ptr nll{pdf->createNLL(*data)}; + + // save initial values for the start of all minimizations + std::unique_ptr values = std::make_unique(*mu, *pdf, *nll, "values"); + + return std::make_tuple(std::move(nll), std::move(values)); +} + +// return two unique_ptrs, the first because nll is a pointer, +// the second because RooArgSet doesn't have a move ctor +std::tuple, std::unique_ptr> +generate_ND_gaussian_pdf_nll(RooWorkspace &w, unsigned int n, unsigned long N_events) +{ + RooArgSet obs_set; + + // create gaussian parameters + double mean[n], sigma[n]; + for (unsigned ix = 0; ix < n; ++ix) { + mean[ix] = RooRandom::randomGenerator()->Gaus(0, 2); + sigma[ix] = 0.1 + abs(RooRandom::randomGenerator()->Gaus(0, 2)); + } + + // create gaussians and also the observables and parameters they depend on + for (unsigned ix = 0; ix < n; ++ix) { + std::ostringstream os; + os << "Gaussian::g" << ix << "(x" << ix << "[-10,10]," + << "m" << ix << "[" << mean[ix] << ",-10,10]," + << "s" << ix << "[" << sigma[ix] << ",0.1,10])"; + w.factory(os.str().c_str()); + } + + // create uniform background signals on each observable + for (unsigned ix = 0; ix < n; ++ix) { + { + std::ostringstream os; + os << "Uniform::u" << ix << "(x" << ix << ")"; + w.factory(os.str().c_str()); + } + + // gather the observables in a list for data generation below + { + std::ostringstream os; + os << "x" << ix; + obs_set.add(*w.arg(os.str().c_str())); + } + } + + RooArgSet pdf_set = w.allPdfs(); + + // create event counts for all pdfs + RooArgSet count_set; + + // ... for the gaussians + for (unsigned ix = 0; ix < n; ++ix) { + std::stringstream os, os2; + os << "Nsig" << ix; + os2 << "#signal events comp " << ix; + RooRealVar a(os.str().c_str(), os2.str().c_str(), N_events / 10, 0., 10 * N_events); + w.import(a); + // gather in count_set + count_set.add(*w.arg(os.str().c_str())); + } + // ... and for the uniform background components + for (unsigned ix = 0; ix < n; ++ix) { + std::stringstream os, os2; + os << "Nbkg" << ix; + os2 << "#background events comp " << ix; + RooRealVar a(os.str().c_str(), os2.str().c_str(), N_events / 10, 0., 10 * N_events); + w.import(a); + // gather in count_set + count_set.add(*w.arg(os.str().c_str())); + } + + RooAddPdf *sum = new RooAddPdf("sum", "gaussians+uniforms", pdf_set, count_set); + w.import(*sum); // keep sum around after returning + + // --- Generate a toyMC sample from composite PDF --- + RooDataSet *data = sum->generate(obs_set, N_events); + + std::unique_ptr nll{sum->createNLL(*data)}; + + // set values randomly so that they actually need to do some fitting + for (unsigned ix = 0; ix < n; ++ix) { + { + std::ostringstream os; + os << "m" << ix; + dynamic_cast(w.arg(os.str().c_str()))->setVal(RooRandom::randomGenerator()->Gaus(0, 2)); + } + { + std::ostringstream os; + os << "s" << ix; + dynamic_cast(w.arg(os.str().c_str())) + ->setVal(0.1 + abs(RooRandom::randomGenerator()->Gaus(0, 2))); + } + } + + // gather all values of parameters, pdfs and nll here for easy + // saving and restoring + std::unique_ptr all_values = std::make_unique(pdf_set, count_set, "all_values"); + all_values->add(*nll); + all_values->add(*sum); + for (unsigned ix = 0; ix < n; ++ix) { + { + std::ostringstream os; + os << "m" << ix; + all_values->add(*w.arg(os.str().c_str())); + } + { + std::ostringstream os; + os << "s" << ix; + all_values->add(*w.arg(os.str().c_str())); + } + } + + return std::make_tuple(std::move(nll), std::move(all_values)); +} + +class Hex { +public: + explicit Hex(double n) : number_(n) {} + operator double() const { return number_; } + bool operator==(const Hex &other) { return double(*this) == double(other); } + +private: + double number_; +}; + +::std::ostream &operator<<(::std::ostream &os, const Hex &hex) +{ + return os << std::hexfloat << double(hex) << std::defaultfloat; // whatever needed to print bar to os +} + +#endif // ROOT_ROOFIT_MultiProcess_tests_utils_h diff --git a/roofit/roofitZMQ/CMakeLists.txt b/roofit/roofitZMQ/CMakeLists.txt new file mode 100644 index 0000000000000..f22d3f3910b74 --- /dev/null +++ b/roofit/roofitZMQ/CMakeLists.txt @@ -0,0 +1,17 @@ +############################################################################ +# CMakeLists.txt file for building ROOT roofitcore/ZMQ package +# @author Patrick Bos, Netherlands eScience Center +############################################################################ + +ROOT_LINKER_LIBRARY(RooFitZMQ + src/ZeroMQSvc.cpp + src/ZeroMQPoller.cpp + src/functions.cpp + src/ppoll.cpp + ) + +target_link_libraries(RooFitZMQ PUBLIC libzmq cppzmq) +set(RooFitZMQ_INCLUDE_DIRS ${CMAKE_CURRENT_SOURCE_DIR}/res) +target_include_directories(RooFitZMQ PRIVATE ${RooFitZMQ_INCLUDE_DIRS}) + +ROOT_ADD_TEST_SUBDIRECTORY(test) diff --git a/roofit/roofitZMQ/res/RooFit_ZMQ/Utility.h b/roofit/roofitZMQ/res/RooFit_ZMQ/Utility.h new file mode 100644 index 0000000000000..77d18d6b139c9 --- /dev/null +++ b/roofit/roofitZMQ/res/RooFit_ZMQ/Utility.h @@ -0,0 +1,47 @@ +/* + * Project: RooFit + * Authors: + * RA, Roel Aaij, NIKHEF + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#ifndef SERIALIZE_UTILITY_H +#define SERIALIZE_UTILITY_H 1 + +#if defined(__clang__) && (__clang_major__ > 3 || (__clang_major__ == 3 && __clang_minor__ >= 10)) +#define HAVE_TRIVIALLY_COPYABLE 1 +#elif defined(__GNUC__) && __GNUC__ >= 5 +#define HAVE_TRIVIALLY_COPYABLE 1 +#else +#undef HAVE_TRIVIALLY_COPYABLE +#endif + +#include + +namespace ZMQ { +namespace Detail { + +#if defined(HAVE_TRIVIALLY_COPYABLE) +template +using simple_object = std::is_trivially_copyable; +#else +template +using simple_object = std::is_pod; +#endif + +// is trivial +template +struct is_trivial + : std::conditional::type>::value, std::true_type, std::false_type>::type { +}; + +} // namespace Detail +} // namespace ZMQ + +#endif // SERIALIZE_UTILITY_H diff --git a/roofit/roofitZMQ/res/RooFit_ZMQ/ZeroMQPoller.h b/roofit/roofitZMQ/res/RooFit_ZMQ/ZeroMQPoller.h new file mode 100644 index 0000000000000..fda44dc3b913a --- /dev/null +++ b/roofit/roofitZMQ/res/RooFit_ZMQ/ZeroMQPoller.h @@ -0,0 +1,61 @@ +/* + * Project: RooFit + * Authors: + * RA, Roel Aaij, NIKHEF + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#ifndef ZEROMQPOLLER_H +#define ZEROMQPOLLER_H 1 + +#include "RooFit_ZMQ/ZeroMQSvc.h" +#include "RooFit_ZMQ/functions.h" +#include + +#include +#include +#include +#include + +class ZeroMQPoller { +public: + using entry_t = std::tuple; + // The key is what zmq::socket_t stores inside, and what goes into + // pollitem_t through zmq::socket_t's conversion to void* operator + using sockets_t = std::unordered_map; + + using fd_entry_t = std::tuple; + using fds_t = std::unordered_map; + + using free_t = std::deque; + + ZeroMQPoller() = default; + + std::vector> poll(int timeo = -1); + std::vector> ppoll(int timeo, const sigset_t *sigmask); + + size_t size() const; + + size_t register_socket(zmq::socket_t &socket, zmq::event_flags type); + size_t register_socket(int fd, zmq::event_flags type); + + size_t unregister_socket(zmq::socket_t &socket); + size_t unregister_socket(int fd); + +private: + // Vector of (socket, flags) + std::vector m_items; + sockets_t m_sockets; + fds_t m_fds; + + // free slots in items + free_t m_free; +}; + +#endif // ZEROMQPOLLER_H diff --git a/roofit/roofitZMQ/res/RooFit_ZMQ/ZeroMQSvc.h b/roofit/roofitZMQ/res/RooFit_ZMQ/ZeroMQSvc.h new file mode 100644 index 0000000000000..77c3303216c39 --- /dev/null +++ b/roofit/roofitZMQ/res/RooFit_ZMQ/ZeroMQSvc.h @@ -0,0 +1,223 @@ +/* + * Project: RooFit + * Authors: + * RA, Roel Aaij, NIKHEF + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#ifndef ZEROMQ_IZEROMQSVC_H +#define ZEROMQ_IZEROMQSVC_H 1 + +#include +#include "RooFit_ZMQ/Utility.h" +#include "RooFit_ZMQ/functions.h" + +#include +#include +#include +#include +#include +#include // std::cerr +#include + +// debugging +#include // getpid + +namespace ZMQ { + +struct TimeOutException : std::exception { + TimeOutException() = default; + TimeOutException(const TimeOutException &) = default; + ~TimeOutException() = default; + TimeOutException &operator=(const TimeOutException &) = default; +}; + +struct MoreException : std::exception { + MoreException() = default; + MoreException(const MoreException &) = default; + ~MoreException() = default; + MoreException &operator=(const MoreException &) = default; +}; + +} // namespace ZMQ + +template +struct ZmqLingeringSocketPtrDeleter { + void operator()(zmq::socket_t *socket) + { + int tries = 0; + int max_tries = 3; + while (true) { + try { + // the actual work this function should do, plus the delete socket below: + if (socket) + socket->set(zmq::sockopt::linger, PERIOD); + break; + } catch (zmq::error_t &e) { + if (++tries == max_tries || e.num() == EINVAL || e.num() == ETERM || + e.num() == ENOTSOCK // not recoverable from here + ) { + std::cerr << "ERROR in ZeroMQSvc::socket: " << e.what() << " (errno: " << e.num() << ")\n"; + throw; + } + std::cerr << "RETRY " << tries << "/" << (max_tries - 1) + << " in ZmqLingeringSocketPtrDeleter: call interrupted (errno: " << e.num() << ")\n"; + } + } + + delete socket; + } +}; + +template +using ZmqLingeringSocketPtr = std::unique_ptr>; + +// We retry send and receive only on EINTR, all other errors are either fatal, or can only +// be handled at the caller. +template +auto retry_send(zmq::socket_t &socket, int max_tries, args_t... args) -> decltype(socket.send(args...)) +{ + int tries = 0; + while (true) { + try { + // the actual work this function should do, all the rest is error handling: + return socket.send(args...); + } catch (zmq::error_t &e) { + if (++tries == max_tries || e.num() != EINTR // only recoverable error + ) { + throw; + } + std::cerr << "RETRY " << tries << "/" << (max_tries - 1) << " in ZeroMQSvc::send (retry_send) on pid " + << getpid() << ": " << e.what() << ")\n"; + } + } +} + +template +auto retry_recv(zmq::socket_t &socket, int max_tries, args_t... args) -> decltype(socket.recv(args...)) +{ + int tries = 0; + while (true) { + try { + // the actual work this function should do, all the rest is error handling: + return socket.recv(args...); + } catch (zmq::error_t &e) { + if (++tries == max_tries || e.num() != EINTR // only recoverable error + ) { + throw; + } + std::cerr << "RETRY " << tries << "/" << (max_tries - 1) << " in ZeroMQSvc::recv (retry_recv) on pid " + << getpid() << ": " << e.what() << ")\n"; + } + } +} + +class ZeroMQSvc { + // Note on error handling: + // Creating message_t can throw, but only when memory ran out (errno ENOMEM), + // and that is something only the caller can fix, so we don't catch it here. + +public: + enum Encoding { Text = 0, Binary }; + + Encoding encoding() const; + void setEncoding(const Encoding &e); + zmq::context_t &context() const; + zmq::socket_t socket(zmq::socket_type type) const; + zmq::socket_t *socket_ptr(zmq::socket_type type) const; + void close_context() const; + + /// decode message with ZMQ, POD version + template ::value && ZMQ::Detail::is_trivial::value, T>::type + * = nullptr> + T decode(const zmq::message_t &msg) const + { + T object; + memcpy(&object, msg.data(), msg.size()); + return object; + } + + /// decode ZMQ message, string version + template ::value, T>::type * = nullptr> + std::string decode(const zmq::message_t &msg) const + { + std::string r(msg.size() + 1, char{}); + r.assign(static_cast(msg.data()), msg.size()); + return r; + } + + /// receive message with ZMQ, general version + // FIXME: what to do with flags=0.... more is a pointer, that might prevent conversion + template ::value), T>::type * = nullptr> + T receive(zmq::socket_t &socket, zmq::recv_flags flags = zmq::recv_flags::none, bool *more = nullptr) const + { + // receive message + zmq::message_t msg; + auto recv_result = retry_recv(socket, 2, std::ref(msg), flags); + if (!recv_result) { + throw ZMQ::TimeOutException{}; + } + if (more) + *more = msg.more(); + + // decode message + return decode(msg); + } + + /// receive message with ZMQ + template ::value, T>::type * = nullptr> + T receive(zmq::socket_t &socket, zmq::recv_flags flags = zmq::recv_flags::none, bool *more = nullptr) const + { + // receive message + zmq::message_t msg; + auto recv_result = retry_recv(socket, 2, std::ref(msg), flags); + if (!recv_result) { + throw ZMQ::TimeOutException{}; + } + if (more) + *more = msg.more(); + return msg; + } + + /// encode message to ZMQ + template ::value && ZMQ::Detail::is_trivial::value, T>::type + * = nullptr> + zmq::message_t encode(const T &item, std::function sizeFun = ZMQ::defaultSizeOf) const + { + size_t s = sizeFun(item); + zmq::message_t msg{s}; + memcpy((void *)msg.data(), &item, s); + return msg; + } + + zmq::message_t encode(const char *item) const; + zmq::message_t encode(const std::string &item) const; + + /// Send message with ZMQ + template ::value, T>::type * = nullptr> + zmq::send_result_t send(zmq::socket_t &socket, const T &item, zmq::send_flags flags = zmq::send_flags::none) const + { + return retry_send(socket, 1, encode(item), flags); + } + + zmq::send_result_t + send(zmq::socket_t &socket, const char *item, zmq::send_flags flags = zmq::send_flags::none) const; + zmq::send_result_t + send(zmq::socket_t &socket, zmq::message_t &msg, zmq::send_flags flags = zmq::send_flags::none) const; + zmq::send_result_t + send(zmq::socket_t &socket, zmq::message_t &&msg, zmq::send_flags flags = zmq::send_flags::none) const; + +private: + Encoding m_enc = Text; + mutable zmq::context_t *m_context = nullptr; +}; + +ZeroMQSvc &zmqSvc(); + +#endif // ZEROMQ_IZEROMQSVC_H diff --git a/roofit/roofitZMQ/res/RooFit_ZMQ/functions.h b/roofit/roofitZMQ/res/RooFit_ZMQ/functions.h new file mode 100644 index 0000000000000..2c4175ca27be1 --- /dev/null +++ b/roofit/roofitZMQ/res/RooFit_ZMQ/functions.h @@ -0,0 +1,31 @@ +/* + * Project: RooFit + * Authors: + * RA, Roel Aaij, NIKHEF + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#ifndef ZEROMQ_FUNCTIONS_H +#define ZEROMQ_FUNCTIONS_H 1 + +#include // std::size_t + +namespace ZMQ { + +template +std::size_t defaultSizeOf(const T &) +{ + return sizeof(T); +} + +std::size_t stringLength(const char &cs); + +} // namespace ZMQ + +#endif // ZEROMQ_FUNCTIONS_H diff --git a/roofit/roofitZMQ/res/RooFit_ZMQ/ppoll.h b/roofit/roofitZMQ/res/RooFit_ZMQ/ppoll.h new file mode 100644 index 0000000000000..4eef9da3c9039 --- /dev/null +++ b/roofit/roofitZMQ/res/RooFit_ZMQ/ppoll.h @@ -0,0 +1,28 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#ifndef ROOT_ROOFIT_ZMQ_ppoll +#define ROOT_ROOFIT_ZMQ_ppoll + +#include +#include + +namespace ZMQ { + +int ppoll(zmq_pollitem_t *items_, size_t nitems_, long timeout_, const sigset_t *sigmask_); +int ppoll(std::vector &items, long timeout_, const sigset_t *sigmask_); +class ppoll_error_t : public zmq::error_t { +}; + +} // namespace ZMQ + +#endif // ROOT_ROOFIT_ZMQ_ppoll diff --git a/roofit/roofitZMQ/src/ZeroMQPoller.cpp b/roofit/roofitZMQ/src/ZeroMQPoller.cpp new file mode 100644 index 0000000000000..28909a294a4cc --- /dev/null +++ b/roofit/roofitZMQ/src/ZeroMQPoller.cpp @@ -0,0 +1,244 @@ +/* + * Project: RooFit + * Authors: + * RA, Roel Aaij, NIKHEF + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#include "RooFit_ZMQ/ZeroMQPoller.h" + +#include "RooFit_ZMQ/ppoll.h" +#include + +/** \class ZeroMQPoller + * \brief Wrapper class for polling ZeroMQ sockets + * + * This class simplifies calls to poll or ppoll ZeroMQ sockets. It stores the + * list of sockets to be polled, which means they don't have to be separately + * carried around by the user. It also parses output and returns an easily + * digestible vector of events. + */ + +/** + * \brief Poll the sockets + * + * \param[in] timeo Timeout in milliseconds. 0 means return immediately. -1 means wait for an event indefinitely. + * \return A vector of pairs of index and flags; index is the index of the registered fd or socket and flags are 0 (no + * events), ZMQ_POLLIN or ZMQ_POLLOUT. + * + * \note This function can throw (from inside zmq::poll), so wrap in try-catch! + */ +std::vector> ZeroMQPoller::poll(int timeo) +{ + std::vector> r; + if (m_items.empty()) { + throw std::runtime_error("No sockets registered"); + } + int n = 0; + while (true) { + try { + n = zmq::poll(m_items, std::chrono::milliseconds{timeo}); + if (n == 0) + return r; + break; + } catch (const zmq::error_t &e) { + std::cerr << "in ZeroMQPoller::poll on PID " << getpid() << ": " << e.what() << std::endl; + if (e.num() != EINTR) { + throw; + } + } + } + // TODO: replace this with ranges::v3::zip + for (size_t i = 0; i < m_items.size(); ++i) { + void *socket = m_items[i].socket; + size_t index = 0; + zmq::event_flags flags = zmq::event_flags::none; + if (socket == nullptr) { + // an fd was registered + std::tie(index, flags) = m_fds[m_items[i].fd]; + } else { + // a socket was registered + const zmq::socket_t *s; + std::tie(index, flags, s) = m_sockets[socket]; + } + if (m_items[i].revents & short(flags)) { + r.emplace_back(index, flags); + } + } + return r; +} + +/** + * \brief Poll the sockets with ppoll + * + * By polling with ppoll instead of poll, one can pass along a signal mask to + * handle POSIX signals properly. See the zmq_ppoll documentation for examples + * of when this is useful: http://api.zeromq.org/ + * + * \param[in] timeo Timeout in milliseconds. 0 means return immediately. -1 means wait for an event indefinitely. + * \param[in] sigmask A non-NULL pointer to a signal mask must be constructed and passed to 'sigmask'. See the man page + * of sigprocmask(2) for more details on this. \return A vector of pairs of index and flags; index is the index of the + * registered fd or socket and flags are 0 (no events), ZMQ_POLLIN or ZMQ_POLLOUT. + * + * \note This function can throw (from inside ZMQ::ppoll), so wrap in try-catch! + */ +std::vector> ZeroMQPoller::ppoll(int timeo, const sigset_t *sigmask) +{ + if (m_items.empty()) { + throw std::runtime_error("No sockets registered"); + } + + std::vector> r; + + auto n = ZMQ::ppoll(m_items, timeo, sigmask); + if (n == 0) + return r; + + for (auto &m_item : m_items) { + size_t index = 0; + zmq::event_flags flags = zmq::event_flags::none; + if (m_item.socket == nullptr) { + // an fd was registered + std::tie(index, flags) = m_fds[m_item.fd]; + } else { + // a socket was registered + const zmq::socket_t *s; + std::tie(index, flags, s) = m_sockets[m_item.socket]; + } + if (m_item.revents & short(flags)) { + r.emplace_back(index, flags); + } + } + return r; +} + +size_t ZeroMQPoller::size() const +{ + return m_items.size(); +} + +/** + * \brief Register socket to poll + * + * Adds the socket to the internal list of sockets to poll. + * + * \param[in] socket Socket to register. + * \param[in] type Type of events to poll for. Can be ZMQ_POLLIN, ZMQ_POLLOUT or a bit-wise combination of the two. + * \return The index of the socket in the poller's internal list. Can be used to match with indices returned from + * (p)poll. + */ +size_t ZeroMQPoller::register_socket(zmq::socket_t &socket, zmq::event_flags type) +{ + zmq::socket_t *s = &socket; + auto it = m_sockets.find(s); + if (it != m_sockets.end()) { + return std::get<0>(it->second); + } + size_t index = m_free.empty() ? m_items.size() : m_free.front(); + if (!m_free.empty()) + m_free.pop_front(); + // NOTE: this uses the conversion-to-void* operator of + // zmq::socket_t, which returns the wrapped object + m_items.push_back({socket, 0, static_cast(type), 0}); + + // We need to lookup by the pointer to the object wrapped by zmq::socket_t + m_sockets.emplace(m_items.back().socket, std::make_tuple(index, type, s)); + return index; +} + +/** + * \brief Register socket to poll + * + * Adds the socket to the internal list of sockets to poll. + * + * \param[in] fd File descriptor of socket to register. + * \param[in] type Type of events to poll for. Can be ZMQ_POLLIN, ZMQ_POLLOUT or a bit-wise combination of the two. + * \return The index of the socket in the poller's internal list. Can be used to match with indices returned from + * (p)poll. + */ +size_t ZeroMQPoller::register_socket(int fd, zmq::event_flags type) +{ + auto it = m_fds.find(fd); + if (it != m_fds.end()) { + return std::get<0>(it->second); + } + size_t index = m_free.empty() ? m_items.size() : m_free.front(); + if (!m_free.empty()) + m_free.pop_front(); + // NOTE: this uses the conversion-to-void* operator of + // zmq::socket_t, which returns the wrapped object + m_items.push_back({nullptr, fd, static_cast(type), 0}); + + // We need to lookup by the pointer to the object wrapped by zmq::socket_t + m_fds.emplace(fd, std::make_tuple(index, type)); + return index; +} + +/** + * \brief Unregister socket from poller + * + * Removes the socket from the internal list of sockets to poll. + * + * \param[in] socket Socket to unregister. + * \return The index of the socket in the poller's internal list before removal. + */ +size_t ZeroMQPoller::unregister_socket(zmq::socket_t &socket) +{ + if (!m_sockets.count(socket.operator void *())) { + throw std::out_of_range("Socket is not registered"); + } + // Remove from m_sockets + // Can't search by the key of m_sockets, as that is the wrapped + // object, but have to use the pointer to the wrapper + // (zmq::socket_t) + auto it = std::find_if(begin(m_sockets), end(m_sockets), [&socket](const decltype(m_sockets)::value_type &entry) { + return &socket == std::get<2>(entry.second); + }); + auto index = std::get<0>(it->second); + m_free.push_back(index); + void *it_first = it->first; + m_sockets.erase(it); + + // Remove from m_items + auto iit = std::find_if(begin(m_items), end(m_items), + [&it_first](const zmq::pollitem_t &item) { return it_first == item.socket; }); + assert(iit != end(m_items)); + m_items.erase(iit); + + return index; +} + +/** + * \brief Unregister socket from poller + * + * Removes the socket from the internal list of sockets to poll. + * + * \param[in] fd File descriptor of socket to unregister. + * \return The index of the socket in the poller's internal list before removal. + */ +size_t ZeroMQPoller::unregister_socket(int fd) +{ + if (!m_fds.count(fd)) { + throw std::out_of_range("fileno is not registered"); + } + // Remove from m_fds + auto it = m_fds.find(fd); + auto index = std::get<0>(it->second); + m_free.push_back(index); + int it_first = it->first; + m_fds.erase(it); + + // Remove from m_items + auto iit = std::find_if(begin(m_items), end(m_items), + [&it_first](const zmq::pollitem_t &item) { return it_first == item.fd; }); + assert(iit != end(m_items)); + m_items.erase(iit); + + return index; +} diff --git a/roofit/roofitZMQ/src/ZeroMQSvc.cpp b/roofit/roofitZMQ/src/ZeroMQSvc.cpp new file mode 100644 index 0000000000000..63c66bb0a6c60 --- /dev/null +++ b/roofit/roofitZMQ/src/ZeroMQSvc.cpp @@ -0,0 +1,173 @@ +/* + * Project: RooFit + * Authors: + * RA, Roel Aaij, NIKHEF + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#include "RooFit_ZMQ/ZeroMQSvc.h" + +#include // std::ref + +/** \class ZeroMQSvc + * \brief Wrapper class for basic ZeroMQ context and socket management + * + * This singleton class wraps a couple of basic ZeroMQ tasks: + * + * 1. Creating, storing and eventually closing a ZeroMQ context. + * 2. Creating new sockets in the context. + * 3. Sending, receiving, encoding and decoding messages over sockets. + * + * For convenience, it offers a number of template overloads that automatically + * encode all kinds of data types in ZeroMQ message objects. + */ + +/** + * \brief Get singleton object of this class + */ +ZeroMQSvc &zmqSvc() +{ + static std::unique_ptr svc; + if (!svc) { + svc = std::make_unique(); + } + return *svc; +} + +ZeroMQSvc::Encoding ZeroMQSvc::encoding() const +{ + return m_enc; +} + +/** + * \brief Set encoding mode + * + * \param[in] e Encoding mode; either Text or Binary. + */ +void ZeroMQSvc::setEncoding(const ZeroMQSvc::Encoding &e) +{ + m_enc = e; +} + +/** + * \brief Get context + * + * Creates a context if it has not yet been created and returns a reference to it. + */ +zmq::context_t &ZeroMQSvc::context() const +{ + if (!m_context) { + try { + m_context = new zmq::context_t; + } catch (zmq::error_t &e) { + std::cerr << "ERROR: Creating ZeroMQ context failed. This only happens when PGM initialization failed or when " + "a nullptr was returned from zmq_ctx_new because the created context was invalid. Contact ZMQ " + "experts when this happens, because it shouldn't.\n"; + throw; + } + } + return *m_context; +} + +/** + * \brief Create and return a new socket + * + * \param[in] type Type of the socket. See http://api.zeromq.org/master:zmq-socket for possible values. + * \return The socket object. + */ +zmq::socket_t ZeroMQSvc::socket(zmq::socket_type type) const +{ + try { + // the actual work this function should do, all the rest is error handling: + return zmq::socket_t{context(), type}; + } catch (zmq::error_t &e) { + // all zmq errors not recoverable from here, only at call site + std::cerr << "ERROR in ZeroMQSvc::socket: " << e.what() << " (errno: " << e.num() << ")\n"; + throw; + } +} + +/** + * \brief Create and return a new socket by pointer + * + * \param[in] type Type of the socket. See http://api.zeromq.org/master:zmq-socket for possible values. + * \return A raw pointer to the socket object. Note: the caller must take ownership! + */ +zmq::socket_t *ZeroMQSvc::socket_ptr(zmq::socket_type type) const +{ + try { + // the actual work this function should do, all the rest is error handling: + return new zmq::socket_t(context(), type); + } catch (zmq::error_t &e) { + // all zmq errors not recoverable from here, only at call site + std::cerr << "ERROR in ZeroMQSvc::socket_ptr: " << e.what() << " (errno: " << e.num() << ")\n"; + throw; + } +} + +void ZeroMQSvc::close_context() const +{ + if (m_context) { + delete m_context; + m_context = nullptr; + } +} + +/** + * \fn zmq::message_t ZeroMQSvc::encode(const char *item) const + * \brief Encode string as a ZeroMQ message object + * + * \param[in] item String. + */ +zmq::message_t ZeroMQSvc::encode(const char *item) const +{ + std::function fun = ZMQ::stringLength; + return encode(*item, fun); +} + +/** + * \overload zmq::message_t ZeroMQSvc::encode(const std::string &item) const + */ +zmq::message_t ZeroMQSvc::encode(const std::string &item) const +{ + return encode(item.c_str()); +} + +/** + * \fn bool ZeroMQSvc::send(zmq::socket_t &socket, const char *item, zmq::send_flags flags) const + * \brief Send message over a socket + * + * \param[in] socket Socket. + * \param[in] item Message to send over. + * \param[in] flags Flags to send. See http://api.zeromq.org/master:zmq-send for possible flags and the cppzmq API for + * the type-safe equivalents in the zmq::send_flags enum class. + * \return An optional of type zmq::send_result_t that contains the number of bytes sent if successful, and is empty if + * EAGAIN was received, which probably means you should try again. + */ +zmq::send_result_t ZeroMQSvc::send(zmq::socket_t &socket, const char *item, zmq::send_flags flags) const +{ + return retry_send(socket, 2, encode(item), flags); +} + +/** + * \overload zmq::send_result_t ZeroMQSvc::send(zmq::socket_t &socket, zmq::message_t &msg, zmq::send_flags flags) const + */ +zmq::send_result_t ZeroMQSvc::send(zmq::socket_t &socket, zmq::message_t &msg, zmq::send_flags flags) const +{ + return retry_send(socket, 2, std::ref(msg), flags); +} + +/** + * \overload zmq::send_result_t ZeroMQSvc::send(zmq::socket_t &socket, zmq::message_t &&msg, zmq::send_flags flags) + * const + */ +zmq::send_result_t ZeroMQSvc::send(zmq::socket_t &socket, zmq::message_t &&msg, zmq::send_flags flags) const +{ + return retry_send(socket, 2, std::move(msg), flags); +} diff --git a/roofit/roofitZMQ/src/functions.cpp b/roofit/roofitZMQ/src/functions.cpp new file mode 100644 index 0000000000000..6f7a0f0d72f23 --- /dev/null +++ b/roofit/roofitZMQ/src/functions.cpp @@ -0,0 +1,23 @@ +/* + * Project: RooFit + * Authors: + * RA, Roel Aaij, NIKHEF + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#include "RooFit_ZMQ/functions.h" + +#include + +namespace ZMQ { +std::size_t stringLength(const char &cs) +{ + return strlen(&cs); +} +} // namespace ZMQ diff --git a/roofit/roofitZMQ/src/ppoll.cpp b/roofit/roofitZMQ/src/ppoll.cpp new file mode 100644 index 0000000000000..c6b4c75d346b3 --- /dev/null +++ b/roofit/roofitZMQ/src/ppoll.cpp @@ -0,0 +1,34 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#include "RooFit_ZMQ/ppoll.h" + +namespace ZMQ { + +/// Wrapper around zmq_ppoll +/// This function can throw, so wrap in try-catch! +int ppoll(zmq_pollitem_t *items_, size_t nitems_, long timeout_, const sigset_t *sigmask_) +{ + int rc = zmq_ppoll(items_, static_cast(nitems_), timeout_, sigmask_); + if (rc < 0) + throw ppoll_error_t(); + return rc; +} + +/// Wrapper around zmq_ppoll +/// This function can throw, so wrap in try-catch! +int ppoll(std::vector &items, long timeout_, const sigset_t *sigmask_) +{ + return ppoll(items.data(), items.size(), timeout_, sigmask_); +} + +} // namespace ZMQ diff --git a/roofit/roofitZMQ/test/CMakeLists.txt b/roofit/roofitZMQ/test/CMakeLists.txt new file mode 100644 index 0000000000000..5ce5cb7d185d9 --- /dev/null +++ b/roofit/roofitZMQ/test/CMakeLists.txt @@ -0,0 +1,14 @@ +ROOT_ADD_GTEST(test_RooFitZMQ test_ZMQ.cpp LIBRARIES RooFitZMQ) +target_include_directories(test_RooFitZMQ PRIVATE ${RooFitZMQ_INCLUDE_DIRS}) + +ROOT_ADD_GTEST(test_RooFitZMQ_polling test_polling.cxx LIBRARIES RooFitZMQ) +target_include_directories(test_RooFitZMQ_polling PRIVATE ${RooFitZMQ_INCLUDE_DIRS}) + +ROOT_ADD_GTEST(test_RooFitZMQ_HWM test_HWM.cxx LIBRARIES RooFitZMQ) +target_include_directories(test_RooFitZMQ_HWM PRIVATE ${RooFitZMQ_INCLUDE_DIRS}) + +ROOT_ADD_GTEST(test_RooFitZMQ_load_balancing test_ZMQ_load_balancing.cxx LIBRARIES RooFitZMQ) +target_include_directories(test_RooFitZMQ_load_balancing PRIVATE ${RooFitZMQ_INCLUDE_DIRS}) + +ROOT_ADD_GTEST(test_RooFitZMQ_mkstemp test_mkstemp.cxx LIBRARIES RooFitZMQ) +target_include_directories(test_RooFitZMQ_mkstemp PRIVATE ${RooFitZMQ_INCLUDE_DIRS}) diff --git a/roofit/roofitZMQ/test/test_HWM.cxx b/roofit/roofitZMQ/test/test_HWM.cxx new file mode 100644 index 0000000000000..bda6ae9bf3d1d --- /dev/null +++ b/roofit/roofitZMQ/test/test_HWM.cxx @@ -0,0 +1,144 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#include "RooFit_ZMQ/ZeroMQSvc.h" + +#include "gtest/gtest.h" + +#include // fork, usleep + +class HighWaterMarkTest : public ::testing::Test { +protected: + void SetUp() override + { + do { + child_pid = fork(); + } while (child_pid == -1); // retry if fork fails + + if (child_pid > 0) { // parent + pusher.reset(zmqSvc().socket_ptr(zmq::socket_type::push)); + if (set_hwm) { + auto rc = zmq_setsockopt(*pusher, ZMQ_SNDHWM, &hwm, sizeof hwm); + assert(rc == 0); + } + pusher->bind("ipc:///tmp/ZMQ_test_fork_polling_P2C.ipc"); + } else { // child + puller.reset(zmqSvc().socket_ptr(zmq::socket_type::pull)); + if (set_hwm) { + auto rc = zmq_setsockopt(*puller, ZMQ_RCVHWM, &hwm, sizeof hwm); + assert(rc == 0); + } + puller->connect("ipc:///tmp/ZMQ_test_fork_polling_P2C.ipc"); + } + } + + void TearDown() override + { + if (child_pid > 0) { // parent + // wait for child + int status = 0; + pid_t pid; + do { + pid = waitpid(child_pid, &status, 0); + } while (-1 == pid && EINTR == errno); // retry on interrupted system call + if (0 != status) { + if (WIFEXITED(status)) { + printf("exited, status=%d\n", WEXITSTATUS(status)); + } else if (WIFSIGNALED(status)) { + printf("killed by signal %d\n", WTERMSIG(status)); + } else if (WIFSTOPPED(status)) { + printf("stopped by signal %d\n", WSTOPSIG(status)); + } else if (WIFCONTINUED(status)) { + printf("continued\n"); + } + } + if (-1 == pid) { + throw std::runtime_error(std::string("waitpid, errno ") + std::to_string(errno)); + } + pusher.reset(nullptr); + zmqSvc().close_context(); + } else { // child + puller.reset(nullptr); + zmqSvc().close_context(); + _Exit(0); + } + } + + void run_test() + { + std::size_t max_sends = 2000; + + if (child_pid > 0) { // parent + // start test + for (std::size_t ix = 0; ix < max_sends; ++ix) { + zmqSvc().send(*pusher, 0.1f); + zmqSvc().send(*pusher, 1); + zmqSvc().send(*pusher, true); + if (ix % 100 == 0) { + printf("parent at ix = %zu\n", ix); + } + } + } else { // child + // wait a few seconds before reading to allow the parent to overflow the HWM + printf("child waiting for 2 seconds...\n"); + sleep(2); + printf("child starts receiving\n"); + + for (std::size_t ix = 0; ix < max_sends; ++ix) { + zmqSvc().receive(*puller); + zmqSvc().receive(*puller); + zmqSvc().receive(*puller); + if (ix % 100 == 0) { + printf("child at ix = %zu\n", ix); + } + } + } + } + + pid_t child_pid{0}; + ZmqLingeringSocketPtr<> pusher, puller; + bool set_hwm = false; + int hwm = 0; +}; + +TEST_F(HighWaterMarkTest, demonstrateHittingDefaultHWM) +{ + run_test(); +} + +class HighWaterMarkZeroTest : public HighWaterMarkTest { + void SetUp() override + { + set_hwm = true; + hwm = 0; + HighWaterMarkTest::SetUp(); + } +}; + +TEST_F(HighWaterMarkZeroTest, zeroHWM) +{ + run_test(); +} + +class HighWaterMark2kTest : public HighWaterMarkTest { + void SetUp() override + { + set_hwm = true; + hwm = 2000; + HighWaterMarkTest::SetUp(); + } +}; + +TEST_F(HighWaterMark2kTest, HWM2k) +{ + run_test(); +} diff --git a/roofit/roofitZMQ/test/test_ZMQ.cpp b/roofit/roofitZMQ/test/test_ZMQ.cpp new file mode 100644 index 0000000000000..da87822c184cd --- /dev/null +++ b/roofit/roofitZMQ/test/test_ZMQ.cpp @@ -0,0 +1,317 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#include "RooFit_ZMQ/ZeroMQSvc.h" + +#include "gtest/gtest.h" + +#include // fork, usleep + +#include + +// N.B.: wait_for_child is identically defined in RooFit::MultiProcess, but we copy it here to reduce module +// interdependencies. It also requires an extra include: +#include // kill, SIGKILL + +int wait_for_child(pid_t child_pid, bool may_throw, int retries_before_killing) +{ + int status = 0; + int patience = retries_before_killing; + pid_t tmp; + do { + if (patience-- < 1) { + ::kill(child_pid, SIGKILL); + } + tmp = waitpid(child_pid, &status, WNOHANG); + } while (tmp == 0 // child has not yet changed state, try again + || (-1 == tmp && EINTR == errno) // retry on interrupted system call + ); + + if (0 != status) { + if (WIFEXITED(status)) { + printf("exited, status=%d\n", WEXITSTATUS(status)); + } else if (WIFSIGNALED(status)) { + if (WTERMSIG(status) != SIGKILL) { // SIGKILL is expected, so needn't be printed + printf("killed by signal %d\n", WTERMSIG(status)); + } + } else if (WIFSTOPPED(status)) { + printf("stopped by signal %d\n", WSTOPSIG(status)); + } else if (WIFCONTINUED(status)) { + printf("continued\n"); + } + } + + if (-1 == tmp && may_throw) { + throw std::runtime_error(std::string("waitpid, errno ") + std::to_string(errno)); + } + + return status; +} + +void elaborate_bind(const ZmqLingeringSocketPtr<> &socket, std::string name) +{ + try { + socket->bind(name); + } catch (const zmq::error_t &e) { + if (e.num() == EADDRINUSE) { + std::cerr << "address already in use, retrying bind in 500ms\n"; + usleep(500000); + try { + socket->bind(name); + } catch (const zmq::error_t &e2) { + if (e2.num() == EADDRINUSE) { + std::cerr + << "again: address already in use, aborting; please check whether there are any remaining improperly " + "exited processes (zombies) around or whether some other program is using port 6660\n"; + } + throw; + } + // Sometimes, the socket from the previous test needs some time to close, so + // we introduce a latency here. A more robust and fast approach might be to + // do the following on the bind side: + // 1. first try another port, e.g. increase by one + // 2. if that doesn't work, do the latency and retry the original port + // The connect side then also needs to change, because it doesn't know which + // port the bind side will bind to. The connect side could try connecting to + // both options asynchronously, and then in a loop check both for signs of + // life. If one comes alive, transfer ownership of that pointer to the pointer + // you want to eventually use (`socket`) and that's it. + } else { + throw; + } + } +} + +class AllSocketTypes + : public ::testing::TestWithParam, + std::pair /* socket_names */>> { +}; + +TEST_P(AllSocketTypes, forkHandshake) +{ + auto socket_names = std::get<2>(GetParam()); + pid_t child_pid{0}; + do { + child_pid = fork(); + } while (child_pid == -1); // retry if fork fails + + if (child_pid > 0) { // master + ZmqLingeringSocketPtr<> socket; + socket.reset(zmqSvc().socket_ptr(std::get<1>(GetParam()).first)); + elaborate_bind(socket, socket_names.second); + // bind is on the master process to avoid zombie children to hold on to binds + + // start test + zmqSvc().send(*socket, std::string("breaker breaker")); + + auto receipt = zmqSvc().receive(*socket); + + EXPECT_EQ(receipt, 1212); + + socket.reset(nullptr); + zmqSvc().close_context(); // if you don't close context in parent process as well, the next repeat will hang + + wait_for_child(child_pid, true, 5); + } else { // child + ZmqLingeringSocketPtr<> socket; + socket.reset(zmqSvc().socket_ptr(std::get<1>(GetParam()).second)); + socket->connect(socket_names.first); + + // start test + auto receipt = zmqSvc().receive(*socket); + if (receipt == "breaker breaker") { + zmqSvc().send(*socket, 1212); + } + // take care, don't just use _exit, it will not cleanly destroy context etc! + // if you really need to, at least close and destroy everything properly + socket.reset(nullptr); + zmqSvc().close_context(); + _Exit(0); + } +} + +std::string ipc{"ipc:///tmp/ZMQ_test_fork.ipc"}; +std::string tcp_server{"tcp://127.0.0.1:6660"}; +std::string tcp_client{"tcp://*:6660"}; +auto socket_name_options = ::testing::Values(std::make_pair(tcp_server, tcp_client), std::make_pair(ipc, ipc)); + +INSTANTIATE_TEST_SUITE_P(REQREP, AllSocketTypes, + ::testing::Combine(::testing::Range(0, 10), // repeat to probe connection stability + ::testing::Values(std::make_pair(zmq::socket_type::req, + zmq::socket_type::rep)), + socket_name_options)); +INSTANTIATE_TEST_SUITE_P(PAIRPAIR, AllSocketTypes, + ::testing::Combine(::testing::Range(0, 10), // repeat to probe connection stability + ::testing::Values(std::make_pair(zmq::socket_type::pair, + zmq::socket_type::pair)), + socket_name_options)); + +class AsyncSocketTypes + : public ::testing::TestWithParam< + std::tuple, + std::pair /* socket_names */, bool /* expect_throw */>> { +}; + +TEST_P(AsyncSocketTypes, forkMultiSendReceive) +{ + bool expect_throw = std::get<3>(GetParam()); + ZmqLingeringSocketPtr<> socket; + auto socket_names = std::get<2>(GetParam()); + pid_t child_pid{0}; + do { + child_pid = fork(); + } while (child_pid == -1); // retry if fork fails + + if (child_pid > 0) { // master + socket.reset(zmqSvc().socket_ptr(std::get<1>(GetParam()).first)); + elaborate_bind(socket, socket_names.second); + // bind is on the master process to avoid zombie children to hold on to binds + + // start test: send 2 things, receive 1, send 1 more, finish + zmqSvc().send(*socket, std::string("breaker breaker")); + + if (expect_throw) { + EXPECT_ANY_THROW(zmqSvc().send(*socket, std::string("anybody out there?"))); + // NOTE: also in case of a throw, be sure to properly close down the connection! + // Otherwise, you may get zombies waiting for a reply. + socket.reset(nullptr); + zmqSvc().close_context(); // if you don't close context in parent process as well, the next repeat will hang + wait_for_child(child_pid, true, 5); + return; + } else { + EXPECT_NO_THROW(zmqSvc().send(*socket, std::string("anybody out there?"))); + } + + auto receipt = zmqSvc().receive(*socket); + + EXPECT_EQ(receipt, 1212); + + zmqSvc().send(*socket, std::string("kthxbye")); + + socket.reset(nullptr); + zmqSvc().close_context(); // if you don't close context in parent process as well, the next repeat will hang + + wait_for_child(child_pid, true, 5); + } else { // child + socket.reset(zmqSvc().socket_ptr(std::get<1>(GetParam()).second)); + socket->connect(socket_names.first); + + // start test, receive something + auto receipt1 = zmqSvc().receive(*socket); + std::string receipt2; + if (expect_throw) { + EXPECT_ANY_THROW(zmqSvc().receive(*socket)); + // NOTE: also in case of a throw, be sure to properly close down the connection! + // Otherwise, you may get zombies waiting for a reply. + socket.reset(nullptr); + zmqSvc().close_context(); + _Exit(0); + } else { + receipt2 = zmqSvc().receive(*socket); + } + + if (receipt1 == "breaker breaker" && receipt2 == "anybody out there?") { + zmqSvc().send(*socket, 1212); + } + auto receipt3 = zmqSvc().receive(*socket); + if (receipt3 != "kthxbye") { + std::cerr << "did not receive final reply correctly\n"; + } + + // take care, don't just use _exit, it will not cleanly destroy context etc! + // if you really need to, at least close and destroy everything properly + socket.reset(nullptr); + zmqSvc().close_context(); + _Exit(0); + } +} + +TEST_P(AsyncSocketTypes, forkIgnoreSomeMessages) +{ + bool expect_throw = std::get<3>(GetParam()); + ZmqLingeringSocketPtr<> socket; + auto socket_names = std::get<2>(GetParam()); + pid_t child_pid{0}; + do { + child_pid = fork(); + } while (child_pid == -1); // retry if fork fails + + if (child_pid > 0) { // master + socket.reset(zmqSvc().socket_ptr(std::get<1>(GetParam()).first)); + elaborate_bind(socket, socket_names.second); + // bind is on the master process to avoid zombie children to hold on to binds + + // start test: send 2 things, receive 1, send 1 more, finish + zmqSvc().send(*socket, std::string("breaker breaker")); + + if (expect_throw) { + EXPECT_ANY_THROW(zmqSvc().send(*socket, std::string("anybody out there?"))); + // NOTE: also in case of a throw, be sure to properly close down the connection! + // Otherwise, you may get zombies waiting for a reply. + socket.reset(nullptr); + zmqSvc().close_context(); // if you don't close context in parent process as well, the next repeat will hang + wait_for_child(child_pid, true, 5); + return; + } else { + EXPECT_NO_THROW(zmqSvc().send(*socket, std::string("anybody out there?"))); + } + + auto receipt = zmqSvc().receive(*socket); + + EXPECT_EQ(receipt, 1212); + + zmqSvc().send(*socket, std::string("kthxbye")); + + socket.reset(nullptr); + zmqSvc().close_context(); // if you don't close context in parent process as well, the next repeat will hang + + wait_for_child(child_pid, true, 5); + } else { // child + socket.reset(zmqSvc().socket_ptr(std::get<1>(GetParam()).second)); + socket->connect(socket_names.first); + + // start test, receive first thing + auto receipt = zmqSvc().receive(*socket); + if (receipt == "breaker breaker") { + zmqSvc().send(*socket, 1212); + } + + // ignore the rest of the sent messages, but give the other end a second to + // actually send its stuff, instead of hanging in retry_send because the + // connection has died; a better solution would be if retry_send (in + // ZeroMQSvc::send) had a callback mechanism that could be used to break + // out when a child has died, but ok + sleep(1); + + // take care, don't just use _exit, it will not cleanly destroy context etc! + // if you really need to, at least close and destroy everything properly + socket.reset(nullptr); + zmqSvc().close_context(); + _Exit(0); + } +} + +INSTANTIATE_TEST_SUITE_P(PAIRPAIR, AsyncSocketTypes, + ::testing::Combine(::testing::Range(0, 10), // repeat to probe connection stability + ::testing::Values(std::make_pair(zmq::socket_type::pair, + zmq::socket_type::pair)), + socket_name_options, + ::testing::Values(false) // don't expect throw + )); + +INSTANTIATE_TEST_SUITE_P(REQREP, AsyncSocketTypes, + ::testing::Combine(::testing::Values(0), // no repeats, we only care about the throw + ::testing::Values(std::make_pair(zmq::socket_type::req, + zmq::socket_type::rep)), + socket_name_options, + ::testing::Values(true) // expect throw + )); diff --git a/roofit/roofitZMQ/test/test_ZMQ_load_balancing.cxx b/roofit/roofitZMQ/test/test_ZMQ_load_balancing.cxx new file mode 100644 index 0000000000000..934885909e420 --- /dev/null +++ b/roofit/roofitZMQ/test/test_ZMQ_load_balancing.cxx @@ -0,0 +1,148 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#include "RooFit_ZMQ/ZeroMQSvc.h" +#include "RooFit_ZMQ/ZeroMQPoller.h" + +#include "gtest/gtest.h" + +#include // fork, usleep + +class ZMQPushPullTest : public ::testing::Test { +protected: + std::size_t N_children = 4; + std::size_t max_sends = 20; + + void SetUp() override + { + for (std::size_t i = 0; i < N_children; ++i) { + do { + child_pid = fork(); + } while (child_pid == -1); // retry if fork fails + if (child_pid == 0) { // child + child_id = i; + break; + } else { + child_pids.push_back(child_pid); + } + } + + if (child_pid > 0) { // parent + pusher.reset(zmqSvc().socket_ptr(zmq::socket_type::push)); + pusher->bind("ipc:///tmp/ZMQ_test_push_pull_P2C.ipc"); + } else { // child + puller.reset(zmqSvc().socket_ptr(zmq::socket_type::pull)); + puller->connect("ipc:///tmp/ZMQ_test_push_pull_P2C.ipc"); + + poller.register_socket(*puller, zmq::event_flags::pollin); + } + } + + void TearDown() override + { + if (child_pid > 0) { // parent + // wait for children + int status = 0; + pid_t pid; + for (pid_t child_pid_i : child_pids) { + do { + pid = waitpid(child_pid_i, &status, 0); + } while (-1 == pid && EINTR == errno); // retry on interrupted system call + if (0 != status) { + if (WIFEXITED(status)) { + printf("exited, status=%d\n", WEXITSTATUS(status)); + } else if (WIFSIGNALED(status)) { + printf("killed by signal %d\n", WTERMSIG(status)); + } else if (WIFSTOPPED(status)) { + printf("stopped by signal %d\n", WSTOPSIG(status)); + } else if (WIFCONTINUED(status)) { + printf("continued\n"); + } + } + if (-1 == pid) { + throw std::runtime_error(std::string("waitpid, errno ") + std::to_string(errno)); + } + } + pusher.reset(nullptr); + zmqSvc().close_context(); + } else { // child + puller.reset(nullptr); + zmqSvc().close_context(); + _Exit(0); + } + } + + void run_parent() + { + // start test + usleep(1000); // wait a second so that all pull sockets are connected for round-robin distribution + // if you don't wait a second above, the push socket will "round-robin" all the messages to just one or two + // connected sockets + for (std::size_t ix = 0; ix < max_sends; ++ix) { + zmqSvc().send(*pusher, 0); + } + for (std::size_t ix = 0; ix < N_children; ++ix) { + // end by sending some 1's to all children, to let them know the sending is over + zmqSvc().send(*pusher, 1); + } + } + + void run_child() + { + std::size_t count = 0; + for (std::size_t ix = 0; ix < max_sends; ++ix) { + auto r = poller.poll(2000); + if (r.empty()) { + printf("poller of child %d timed out after 2 seconds\n", child_id); + break; + } + auto value = zmqSvc().receive(*puller, zmq::recv_flags::dontwait); + usleep(200); // "do some work" + printf("value on child %d: %d\n", child_id, value); + if (value == 1) { + printf("child %d got value %d, done here\n", child_id, value); + break; + } + ++count; + } + printf("child %d got %zu values\n", child_id, count); + } + + pid_t child_pid{0}; + int child_id = -1; + std::vector child_pids; + ZmqLingeringSocketPtr<> pusher, puller; + ZeroMQPoller poller; +}; + +/// This test shows how push-pull is unsuited for load balancing; messages are just sent to the first available pull +/// socket without any dynamic load balancing +TEST_F(ZMQPushPullTest, demoRoundRobin) +{ + if (child_pid > 0) { + run_parent(); + } else { + run_child(); + } +} + +/// This test tries to see whether push-pull can be made to work as a bit of a load balancer, using a low HWM at the +/// receiver +TEST_F(ZMQPushPullTest, demoHWM1LoadBalancing) +{ + if (child_pid > 0) { + run_parent(); + } else { + puller->set(zmq::sockopt::rcvhwm, 1); + run_child(); + } +} diff --git a/roofit/roofitZMQ/test/test_mkstemp.cxx b/roofit/roofitZMQ/test/test_mkstemp.cxx new file mode 100644 index 0000000000000..a914192c1b61f --- /dev/null +++ b/roofit/roofitZMQ/test/test_mkstemp.cxx @@ -0,0 +1,33 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#include "RooFit_ZMQ/ZeroMQSvc.h" + +#include "gtest/gtest.h" + +#include // mkstemp +#include // for "legacy" systems + +TEST(BindToTmpFile, mkstemp) +{ + char filename[] = "/tmp/roofitMP_XXXXXX"; + while (mkstemp(filename) < 0) { + } + auto socket = zmqSvc().socket(zmq::socket_type::push); + char address[50]; + sprintf(address, "ipc://%s", filename); + try { + socket.bind(address); + } catch (const zmq::error_t &) { + printf("caught an exception\n"); + } +} \ No newline at end of file diff --git a/roofit/roofitZMQ/test/test_polling.cxx b/roofit/roofitZMQ/test/test_polling.cxx new file mode 100644 index 0000000000000..43dd5c0759b6c --- /dev/null +++ b/roofit/roofitZMQ/test/test_polling.cxx @@ -0,0 +1,161 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2021, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#include "RooFit_ZMQ/ZeroMQSvc.h" +#include "RooFit_ZMQ/ZeroMQPoller.h" + +#include "gtest/gtest.h" + +#include // fork, usleep +#include // signal blocking +#include // strsignal() + +#include + +static volatile sig_atomic_t terminated = 0; + +void handle_sigterm(int signum) +{ + terminated = 1; + std::cout << "handled signal " << strsignal(signum) << " on PID " << getpid() << std::endl; +} + +std::string unique_tmp_ipc_address(const char *filename_template) +{ + assert(strlen(filename_template) < 256); + char filename_template_mutable[256]; + strcpy(filename_template_mutable, filename_template); + while (mkstemp(filename_template_mutable) >= 0) { + } + std::stringstream ss; + ss << "ipc:///tmp/" << filename_template_mutable << ".ipc"; + return ss.str(); +} + +TEST(Polling, doublePoll) +{ + auto M2C_address = unique_tmp_ipc_address("ZMQ_test_fork_polling_M2C_XXXXXX"); + auto C2M_address = unique_tmp_ipc_address("ZMQ_test_fork_polling_C2M_XXXXXX"); + pid_t child_pid{0}; + do { + child_pid = fork(); + } while (child_pid == -1); // retry if fork fails + + if (child_pid > 0) { // master + sigset_t sigmask, sigmask_old; + sigemptyset(&sigmask); + sigaddset(&sigmask, SIGCHLD); + sigprocmask(SIG_BLOCK, &sigmask, &sigmask_old); + + ZmqLingeringSocketPtr<> pusher, puller; + pusher.reset(zmqSvc().socket_ptr(zmq::socket_type::push)); + pusher->bind(M2C_address); + puller.reset(zmqSvc().socket_ptr(zmq::socket_type::pull)); + puller->bind(C2M_address); + + ZeroMQPoller poller1, poller2; + poller1.register_socket(*puller, zmq::event_flags::pollin); + poller2.register_socket(*puller, zmq::event_flags::pollin); + + // start test + zmqSvc().send(*pusher, std::string("breaker breaker")); + + auto result1a = poller1.poll(-1); + auto result1b = poller1.poll(-1); + auto result2 = poller2.poll(-1); + EXPECT_EQ(result1a.size(), result1b.size()); + EXPECT_EQ(result1a.size(), result2.size()); + + auto receipt = zmqSvc().receive(*puller, zmq::recv_flags::dontwait); + + EXPECT_EQ(receipt, 1212); + + kill(child_pid, SIGTERM); + + pusher.reset(nullptr); + puller.reset(nullptr); + zmqSvc().close_context(); // if you don't close context in parent process as well, the next repeat will hang + + // wait for child + int status = 0; + pid_t pid; + do { + pid = waitpid(child_pid, &status, 0); + } while (-1 == pid && EINTR == errno); // retry on interrupted system call + + if (0 != status) { + if (WIFEXITED(status)) { + printf("exited, status=%d\n", WEXITSTATUS(status)); + } else if (WIFSIGNALED(status)) { + printf("killed by signal %d\n", WTERMSIG(status)); + } else if (WIFSTOPPED(status)) { + printf("stopped by signal %d\n", WSTOPSIG(status)); + } else if (WIFCONTINUED(status)) { + printf("continued\n"); + } + } + + if (-1 == pid) { + throw std::runtime_error(std::string("waitpid, errno ") + std::to_string(errno)); + } + + sigprocmask(SIG_SETMASK, &sigmask_old, nullptr); + } else { // child + sigset_t sigmask, sigmask_old; + sigemptyset(&sigmask); + sigaddset(&sigmask, SIGTERM); + sigprocmask(SIG_BLOCK, &sigmask, &sigmask_old); + + struct sigaction sa; + memset(&sa, '\0', sizeof(sa)); + sa.sa_handler = handle_sigterm; + + if (sigaction(SIGTERM, &sa, NULL) < 0) { + std::perror("sigaction failed"); + std::exit(1); + } + + ZmqLingeringSocketPtr<> puller, pusher; + puller.reset(zmqSvc().socket_ptr(zmq::socket_type::pull)); + puller->connect(M2C_address); + pusher.reset(zmqSvc().socket_ptr(zmq::socket_type::push)); + pusher->connect(C2M_address); + + ZeroMQPoller poller1, poller2; + poller1.register_socket(*puller, zmq::event_flags::pollin); + poller2.register_socket(*puller, zmq::event_flags::pollin); + + // start test + auto result1a = poller1.poll(-1); + auto result1b = poller1.poll(-1); + auto result2 = poller2.poll(-1); + EXPECT_EQ(result1a.size(), result1b.size()); + EXPECT_EQ(result1a.size(), result2.size()); + + auto receipt = zmqSvc().receive(*puller, zmq::recv_flags::dontwait); + if (receipt == "breaker breaker") { + zmqSvc().send(*pusher, 1212); + } + // take care, don't just use _exit, it will not cleanly destroy context etc! + // if you really need to, at least close and destroy everything properly + + sigprocmask(SIG_SETMASK, &sigmask_old, nullptr); + + while (!terminated) { + } + + puller.reset(nullptr); + pusher.reset(nullptr); + zmqSvc().close_context(); + _Exit(0); + } +}