diff --git a/builtins/zeromq/CMakeLists.txt b/builtins/zeromq/CMakeLists.txt new file mode 100644 index 0000000000000..5ec3b60f59a22 --- /dev/null +++ b/builtins/zeromq/CMakeLists.txt @@ -0,0 +1,48 @@ +include(ExternalProject) + +set(ZeroMQ_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/BUILTIN_ZeroMQ-prefix) +set(ZeroMQ_LIBNAME ${CMAKE_STATIC_LIBRARY_PREFIX}zmq${CMAKE_STATIC_LIBRARY_SUFFIX}) + +set(ZeroMQ_LIBRARY ${ZeroMQ_PREFIX}/lib/${ZeroMQ_LIBNAME} CACHE INTERNAL "" FORCE) +set(ZeroMQ_LIBRARIES ${ZeroMQ_LIBRARY} CACHE INTERNAL "" FORCE) + +ExternalProject_Add(BUILTIN_ZeroMQ + URL https://github.com/zeromq/libzmq/archive/refs/tags/v4.3.2.tar.gz + CMAKE_ARGS + -DCMAKE_INSTALL_PREFIX= + -DWITH_PERF_TOOL=OFF + -DZMQ_BUILD_TESTS=OFF + -DPOLLER=select + # TODO: remove -w flag when updating to v4.3.5 or higher, current warnings should be gone by then + -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}\ -w + BUILD_BYPRODUCTS ${ZeroMQ_LIBRARIES} + ) + +ExternalProject_Get_Property(BUILTIN_ZeroMQ install_dir) + +set(ZeroMQ_FOUND TRUE CACHE BOOL "" FORCE) + +set(ZeroMQ_INCLUDE_DIR ${ZeroMQ_PREFIX}/include CACHE INTERNAL "" FORCE) +set(ZeroMQ_INCLUDE_DIRS ${ZeroMQ_PREFIX}/include CACHE INTERNAL "" FORCE) + +add_library(ZeroMQ INTERFACE) +target_include_directories(ZeroMQ INTERFACE $) +target_link_libraries(ZeroMQ INTERFACE $) +add_dependencies(ZeroMQ BUILTIN_ZeroMQ) + +add_library(ZeroMQ::ZeroMQ ALIAS ZeroMQ) +add_library(libzmq ALIAS ZeroMQ) + +set(ZeroMQ_DIR ${ZeroMQ_PREFIX}/share/cmake/ZeroMQ CACHE INTERNAL "" FORCE) + +# also need source and build directories for ppoll in RooFitZMQ +set(ZeroMQ_SOURCE_DIR ${ZeroMQ_PREFIX}/src/BUILTIN_ZeroMQ/src CACHE INTERNAL "" FORCE) +set(ZeroMQ_BUILD_DIR ${ZeroMQ_PREFIX}/src/BUILTIN_ZeroMQ-build CACHE INTERNAL "" FORCE) + +ExternalProject_Add(BUILTIN_cppzmq + URL https://github.com/zeromq/cppzmq/archive/refs/tags/v4.7.1.tar.gz + CMAKE_ARGS + -DCMAKE_INSTALL_PREFIX=${install_dir} + -DCPPZMQ_BUILD_TESTS=OFF + DEPENDS BUILTIN_ZeroMQ + ) diff --git a/cmake/modules/RootBuildOptions.cmake b/cmake/modules/RootBuildOptions.cmake index 1cc9f48926367..cd157236e552e 100644 --- a/cmake/modules/RootBuildOptions.cmake +++ b/cmake/modules/RootBuildOptions.cmake @@ -107,6 +107,7 @@ ROOT_BUILD_OPTION(builtin_vdt OFF "Build VDT internally (requires network)") ROOT_BUILD_OPTION(builtin_veccore OFF "Build VecCore internally (requires network)") ROOT_BUILD_OPTION(builtin_xrootd OFF "Build XRootD internally (requires network)") ROOT_BUILD_OPTION(builtin_xxhash OFF "Build bundled copy of xxHash") +ROOT_BUILD_OPTION(builtin_zeromq ON "Build ZeroMQ internally (requires network)") ROOT_BUILD_OPTION(builtin_zlib OFF "Build bundled copy of zlib") ROOT_BUILD_OPTION(builtin_zstd OFF "Build included libzstd, or use system libzstd") ROOT_BUILD_OPTION(ccache OFF "Enable ccache usage for speeding up builds") @@ -299,6 +300,7 @@ if(builtin_all) set(builtin_veccore_defvalue ON) set(builtin_xrootd_defvalue ON) set(builtin_xxhash_defvalue ON) + set(builtin_zeromq_defvalue ON) set(builtin_zlib_defvalue ON) set(builtin_zstd_defvalue ON) endif() diff --git a/cmake/modules/SearchInstalledSoftware.cmake b/cmake/modules/SearchInstalledSoftware.cmake index c64ab05414e21..0624a00fbb3a3 100644 --- a/cmake/modules/SearchInstalledSoftware.cmake +++ b/cmake/modules/SearchInstalledSoftware.cmake @@ -1767,6 +1767,13 @@ if (mpi) endif() endif() +#---Check for ZeroMQ----------------------------------------------------------- + +if(builtin_zeromq) + list(APPEND ROOT_BUILTINS ZeroMQ) + add_subdirectory(builtins/zeromq) +endif() + #---Download googletest-------------------------------------------------------------- if (testing) # FIXME: Remove our version of gtest in roottest. We can reuse this one. diff --git a/roofit/CMakeLists.txt b/roofit/CMakeLists.txt index 890df0340dde2..6715e07edd8a9 100644 --- a/roofit/CMakeLists.txt +++ b/roofit/CMakeLists.txt @@ -5,6 +5,10 @@ # For the list of contributors see $ROOTSYS/README/CREDITS. add_subdirectory(batchcompute) +if (builtin_zeromq) + add_subdirectory(roofitZMQ) + add_subdirectory(multiprocess) +endif(builtin_zeromq) add_subdirectory(roofitcore) add_subdirectory(roofit) if(mathmore) diff --git a/roofit/multiprocess/CMakeLists.txt b/roofit/multiprocess/CMakeLists.txt new file mode 100644 index 0000000000000..702e4e611f45e --- /dev/null +++ b/roofit/multiprocess/CMakeLists.txt @@ -0,0 +1,40 @@ +############################################################################ +# CMakeLists.txt file for building ROOT RooFitMultiProcess package +# @author Patrick Bos, Netherlands eScience Center +############################################################################ + +#FIND_PACKAGE(ZeroMQ REQUIRED) + +ROOT_STANDARD_LIBRARY_PACKAGE(RooFitMultiProcess + HEADERS + RooFit/MultiProcess/worker.h + RooFit/MultiProcess/Messenger_decl.h + RooFit/MultiProcess/Messenger.h + RooFit/MultiProcess/ProcessManager.h + RooFit/MultiProcess/JobManager.h + RooFit/MultiProcess/util.h + RooFit/MultiProcess/Queue.h + RooFit/MultiProcess/types.h + RooFit/MultiProcess/Job.h + SOURCES + src/worker.cxx + src/Messenger.cxx + src/ProcessManager.cxx + src/util.cxx + src/Queue.cxx + src/JobManager.cxx + src/Job.cxx + DICTIONARY_OPTIONS + "-writeEmptyRootPCM" + DEPENDENCIES + RooFitZMQ + LIBRARIES + ZeroMQ +) + +target_link_libraries(RooFitMultiProcess PRIVATE ZeroMQ) +target_include_directories(RooFitMultiProcess PUBLIC $) + +if(testing) + ROOT_ADD_TEST_SUBDIRECTORY(test) +endif() diff --git a/roofit/multiprocess/inc/LinkDef.h b/roofit/multiprocess/inc/LinkDef.h new file mode 100644 index 0000000000000..6a9ddbee0696a --- /dev/null +++ b/roofit/multiprocess/inc/LinkDef.h @@ -0,0 +1 @@ +// ceci n'est pas un fichier \ No newline at end of file diff --git a/roofit/multiprocess/inc/RooFit/MultiProcess/Job.h b/roofit/multiprocess/inc/RooFit/MultiProcess/Job.h new file mode 100644 index 0000000000000..423071fabfeb1 --- /dev/null +++ b/roofit/multiprocess/inc/RooFit/MultiProcess/Job.h @@ -0,0 +1,130 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2016-2019, Netherlands eScience Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ +#ifndef ROOT_ROOFIT_MultiProcess_Job_decl +#define ROOT_ROOFIT_MultiProcess_Job_decl + +#include +#include + +namespace RooFit { +namespace MultiProcess { + +// forward declaration +class JobManager; + +/* + * @brief interface class for defining the actual work that must be done + * + * Think of "job" as in "employment", e.g. the job of a baker, which + * involves *tasks* like baking and selling bread. The Job must define the + * tasks through its execution (evaluate_task), based on a task index argument. + * + * Classes inheriting from Job must implement the pure virtual methods: + * - void evaluate_task(std::size_t task) + * - void update_real(std::size_t ix, double val, bool is_constant) + * - void update_bool(std::size_t ix, bool value) + * - void receive_task_result_on_queue(std::size_t task, std::size_t worker_id) + * - void send_back_results_from_queue_to_master() + * - void clear_results() + * - void receive_results_on_master() + * - void send_back_task_result_from_worker(std::size_t task) + * + * The latter five deal with sending results back from workers to master. + * update_real is used to sync parameters that changed on master to the workers + * before the next set of tasks is queued; it must have a part that updates the + * actual parameters on the worker process, but can optionally also be used to + * send them from the master to the queue (Q2W is handled by the Queue::loop). + * An example implementation: + + + + * + * The type of result from each task is strongly dependent on the Job at hand + * and so Job does not provide a default results vector or anything like this. + * It is up to the inheriting class to implement this. We would have liked a + * template parameter task_result_t, so that we could also provide a default + * "boilerplate" calculate function to show a typical Job use-case of all the + * above infrastructure. This is not trivial, because the JobManager has to + * keep a list of Job pointers, so if there would be different template + * instantiations of Jobs, this would complicate this list. As an example, + * a boilerplate calculation could look something like this: + * +return_type CertainJob::evaluate() { + if (get_manager()->process_manager().is_master()) { + // update parameters that changed since last calculation (or creation if first time) + for (size_t ix = 0; ix < changed_parameters.size(); ++ix) { // changed_parameters would be a member of CertainJob + auto msg = RooFit::MultiProcess::M2Q::update_real; + get_manager()->messenger().send_from_master_to_queue(msg, id, ix, + changed_parameters[ix].getVal(), + changed_parameters[ix].isConstant()); + } + + // master fills queue with tasks + for (std::size_t ix = 0; ix < N_tasks; ++ix) { // N_tasks would also be a member of CertainJob + JobTask job_task(id, ix); + get_manager()->queue().add(job_task); + } + + // wait for task results back from workers to master + gather_worker_results(); + + // possibly do some final manipulation of the results that were gathered + } + return result; // or not, it could be stored and accessed later; also here, result would be a member of CertainJob +} + * + * Child classes should refrain from direct access to the JobManager instance + * (through JobManager::instance), but rather use the here provided + * Job::get_manager(). This function starts the worker_loop on the worker when + * first called, meaning that the workers will not + */ +class Job { +public: + explicit Job(); + Job(const Job &other); + + ~Job(); + + virtual void evaluate_task(std::size_t task) = 0; + virtual void update_real(std::size_t ix, double val, bool is_constant) = 0; + virtual void update_bool(std::size_t ix, bool value) = 0; + virtual void update_state(); + + virtual void send_back_task_result_from_worker(std::size_t task) = 0; + virtual void receive_task_result_on_queue(std::size_t task, std::size_t worker_id) = 0; + virtual void send_back_results_from_queue_to_master() = 0; + // after results have been retrieved, they may need to be cleared to ensure + // they won't be retrieved the next time again, e.g. when using a map to + // collect results; if not needed it can just be left empty + virtual void clear_results() = 0; + virtual void receive_results_on_master() = 0; + virtual bool receive_task_result_on_master(const zmq::message_t & message) = 0; + + void gather_worker_results(); + +protected: + JobManager *get_manager(); + + std::size_t id; + +private: + // do not use _manager directly, it must first be initialized! use get_manager() + JobManager *_manager = nullptr; +}; + +} // namespace MultiProcess +} // namespace RooFit + +#endif // ROOT_ROOFIT_MultiProcess_Job_decl diff --git a/roofit/multiprocess/inc/RooFit/MultiProcess/JobManager.h b/roofit/multiprocess/inc/RooFit/MultiProcess/JobManager.h new file mode 100644 index 0000000000000..c3524138ae50a --- /dev/null +++ b/roofit/multiprocess/inc/RooFit/MultiProcess/JobManager.h @@ -0,0 +1,91 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2016-2019, Netherlands eScience Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ +#ifndef ROOT_ROOFIT_MultiProcess_JobManager +#define ROOT_ROOFIT_MultiProcess_JobManager + +#include // unique_ptr +#include + +#include "RooFit/MultiProcess/types.h" + +namespace RooFit { +namespace MultiProcess { + +// forward definitions +class ProcessManager; +class Messenger; +class Queue; +class Job; + +/* + * @brief Main point of access for all MultiProcess infrastructure + * + * This class mainly serves as the access point to the multi-process infrastructure + * for Jobs. It is meant to be used as a singleton that holds and connects the other + * infrastructural classes: the messenger, process manager, worker and queue loops. + * + * It is important that the user of this class, particularly the one that calls + * instance() first, calls activate_loops() soon after, because everything that is + * done in between instance() and activate_loops() will be executed on all processes. + * This may be useful in some cases, but in general, one will probably want to always + * use the JobManager in its full capacity, including the queue and worker loops. + * This is the way the Job class uses this class, see + * + * The default number of processes is set using std::thread::hardware_concurrency(). + * To change it, simply set JobManager::default_N_workers to a different value + * before creation of a new instance. + */ + +class JobManager { +public: + static JobManager *instance(); + static bool is_instantiated(); + + ~JobManager(); + + static std::size_t add_job_object(Job *job_object); + static Job *get_job_object(std::size_t job_object_id); + static bool remove_job_object(std::size_t job_object_id); + + ProcessManager & process_manager() const; + Messenger & messenger() const; + Queue & queue() const; + + void retrieve(std::size_t requesting_job_id); + void results_from_queue_to_master(); + + void activate(); + bool is_activated() const; + +private: + explicit JobManager(std::size_t N_workers); + + std::unique_ptr process_manager_ptr; + std::unique_ptr messenger_ptr; + std::unique_ptr queue_ptr; + bool activated = false; + + static std::map job_objects; + static std::size_t job_counter; + static std::unique_ptr _instance; + +public: + static unsigned int default_N_workers; // no need for getters/setters, just public +}; + +} // namespace MultiProcess +} // namespace RooFit + +#endif // ROOT_ROOFIT_MultiProcess_JobManager diff --git a/roofit/multiprocess/inc/RooFit/MultiProcess/Messenger.h b/roofit/multiprocess/inc/RooFit/MultiProcess/Messenger.h new file mode 100644 index 0000000000000..69163933b0db0 --- /dev/null +++ b/roofit/multiprocess/inc/RooFit/MultiProcess/Messenger.h @@ -0,0 +1,208 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2016-2019, Netherlands eScience Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ +#ifndef ROOT_ROOFIT_MultiProcess_Messenger +#define ROOT_ROOFIT_MultiProcess_Messenger + +#include "RooFit/MultiProcess/Messenger_decl.h" + +namespace RooFit { +namespace MultiProcess { + +// -- WORKER - QUEUE COMMUNICATION -- + +template +void Messenger::send_from_worker_to_queue(T item, Ts... items) +{ + #ifndef NDEBUG + std::stringstream ss; + ss << "PID " << getpid() << " sends W2Q " << item; + debug_print(ss.str()); + #endif + + zmqSvc().send(*this_worker_qw_push, item, send_flag); + // if (sizeof...(items) > 0) { // this will only work with if constexpr, c++17 + send_from_worker_to_queue(items...); +} + +template +value_t Messenger::receive_from_worker_on_queue(std::size_t this_worker_id) +{ + qw_pull_poller[this_worker_id].ppoll(-1, &ppoll_sigmask); + auto value = zmqSvc().receive(*qw_pull[this_worker_id], ZMQ_DONTWAIT); + + #ifndef NDEBUG + std::stringstream ss; + ss << "PID " << getpid() << " receives W(" << this_worker_id << ")2Q " << value; + debug_print(ss.str()); + #endif + + return value; +} + +template +void Messenger::send_from_queue_to_worker(std::size_t this_worker_id, T item, Ts... items) +{ +#ifndef NDEBUG + std::stringstream ss; + ss << "PID " << getpid() << " sends Q2W(" << this_worker_id << ") " << item; + debug_print(ss.str()); +#endif + + zmqSvc().send(*qw_push[this_worker_id], item, send_flag); + // if (sizeof...(items) > 0) { // this will only work with if constexpr, c++17 + send_from_queue_to_worker(this_worker_id, items...); +} + +template +value_t Messenger::receive_from_queue_on_worker() +{ + qw_pull_poller[0].ppoll(-1, &ppoll_sigmask); + auto value = zmqSvc().receive(*this_worker_qw_pull, ZMQ_DONTWAIT); + + #ifndef NDEBUG + std::stringstream ss; + ss << "PID " << getpid() << " receives Q2W " << value; + debug_print(ss.str()); + #endif + + return value; +} + + +// -- QUEUE - MASTER COMMUNICATION -- + +template +void Messenger::send_from_queue_to_master(T item, Ts... items) +{ + #ifndef NDEBUG + std::stringstream ss; + ss << "PID " << getpid() << " sends Q2M " << item; + debug_print(ss.str()); + #endif + + zmqSvc().send(*mq_push, item, send_flag); + // if (sizeof...(items) > 0) { // this will only work with if constexpr, c++17 + send_from_queue_to_master(items...); +} + +template +value_t Messenger::receive_from_queue_on_master() +{ + mq_pull_poller.ppoll(-1, &ppoll_sigmask); + auto value = zmqSvc().receive(*mq_pull, ZMQ_DONTWAIT); + + #ifndef NDEBUG + std::stringstream ss; + ss << "PID " << getpid() << " receives Q2M " << value; + debug_print(ss.str()); + #endif + + return value; +} + +template +void Messenger::send_from_master_to_queue(T item, Ts... items) +{ + #ifndef NDEBUG + std::stringstream ss; + ss << "PID " << getpid() << " sends M2Q " << item; + debug_print(ss.str()); + #endif + + zmqSvc().send(*mq_push, item, send_flag); + // if (sizeof...(items) > 0) { // this will only work with if constexpr, c++17 + send_from_master_to_queue(items...); +} + +template +value_t Messenger::receive_from_master_on_queue() +{ + mq_pull_poller.ppoll(-1, &ppoll_sigmask); + auto value = zmqSvc().receive(*mq_pull, ZMQ_DONTWAIT); + + #ifndef NDEBUG + std::stringstream ss; + ss << "PID " << getpid() << " receives M2Q " << value; + debug_print(ss.str()); + #endif + + return value; +} + + +// -- MASTER - WORKER COMMUNICATION -- + +template +void Messenger::publish_from_master_to_workers(T item, Ts... items) +{ +#ifndef NDEBUG + std::stringstream ss; + ss << "PID " << getpid() << " sends M2W " << item; + debug_print(ss.str()); +#endif + + zmqSvc().send(*mw_pub, item, send_flag); + // if (sizeof...(items) > 0) { // this will only work with if constexpr, c++17 + publish_from_master_to_workers(items...); +} + +template +value_t Messenger::receive_from_master_on_worker() +{ + mw_sub_poller.ppoll(-1, &ppoll_sigmask); + auto value = zmqSvc().receive(*mw_sub, ZMQ_DONTWAIT); + +#ifndef NDEBUG + std::stringstream ss; + ss << "PID " << getpid() << " receives M2W " << value; + debug_print(ss.str()); +#endif + + return value; +} + +template +void Messenger::send_from_worker_to_master(T item, Ts... items) +{ +#ifndef NDEBUG + std::stringstream ss; + ss << "PID " << getpid() << " sends M2W " << item; + debug_print(ss.str()); +#endif + + zmqSvc().send(*wm_push, item, send_flag); + // if (sizeof...(items) > 0) { // this will only work with if constexpr, c++17 + send_from_worker_to_master(items...); +} + +template +value_t Messenger::receive_from_worker_on_master() +{ + wm_pull_poller.ppoll(-1, &ppoll_sigmask); + auto value = zmqSvc().receive(*wm_pull, ZMQ_DONTWAIT); + +#ifndef NDEBUG + std::stringstream ss; + ss << "PID " << getpid() << " receives M2W " << value; + debug_print(ss.str()); +#endif + + return value; +} + +} // namespace MultiProcess +} // namespace RooFit + +#endif // ROOT_ROOFIT_MultiProcess_Messenger diff --git a/roofit/multiprocess/inc/RooFit/MultiProcess/Messenger_decl.h b/roofit/multiprocess/inc/RooFit/MultiProcess/Messenger_decl.h new file mode 100644 index 0000000000000..80d690658b241 --- /dev/null +++ b/roofit/multiprocess/inc/RooFit/MultiProcess/Messenger_decl.h @@ -0,0 +1,185 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2016-2019, Netherlands eScience Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ +#ifndef ROOT_ROOFIT_MultiProcess_Messenger_decl +#define ROOT_ROOFIT_MultiProcess_Messenger_decl + +#include +#include +#include // sigprocmask, sigset_t, etc + +#include "RooFit_ZMQ/ZeroMQSvc.h" +#include "RooFit_ZMQ/ZeroMQPoller.h" +#include "RooFit/MultiProcess/ProcessManager.h" + +namespace RooFit { +namespace MultiProcess { + +void set_socket_immediate(ZmqLingeringSocketPtr<> &socket); + +// test messages +enum class X2X : int { + ping = -1, + pong = -2, + initial_value = 0 +}; + +class Messenger { +public: + explicit Messenger(const ProcessManager &process_manager); + ~Messenger(); + + void test_connections(const ProcessManager &process_manager); + + enum class test_snd_pipes { + M2Q, + Q2M, + Q2W, + W2Q + }; + + enum class test_rcv_pipes { + fromQonM, + fromMonQ, + fromWonQ, + fromQonW, + }; + + std::pair create_queue_poller(); + std::pair create_worker_poller(); + + // -- WORKER - QUEUE COMMUNICATION -- + + void send_from_worker_to_queue(); + template + void send_from_worker_to_queue(T item, Ts... items); + template + value_t receive_from_worker_on_queue(std::size_t this_worker_id); + void send_from_queue_to_worker(std::size_t this_worker_id); + template + void send_from_queue_to_worker(std::size_t this_worker_id, T item, Ts... items); + template + value_t receive_from_queue_on_worker(); + + // -- QUEUE - MASTER COMMUNICATION -- + + void send_from_queue_to_master(); + + template + void send_from_queue_to_master(T item, Ts... items); + template + value_t receive_from_queue_on_master(); + void send_from_master_to_queue(); + + template + void send_from_master_to_queue(T item, Ts... items); + template + value_t receive_from_master_on_queue(); + + // -- MASTER - WORKER COMMUNICATION -- + + void publish_from_master_to_workers(); + template + void publish_from_master_to_workers(T item, Ts... items); + template + value_t receive_from_master_on_worker(); + + void send_from_worker_to_master(); + template + void send_from_worker_to_master(T item, Ts... items); + template + value_t receive_from_worker_on_master(); + + void test_receive(X2X expected_ping_value, test_rcv_pipes rcv_pipe, std::size_t worker_id); + void test_send(X2X ping_value, test_snd_pipes snd_pipe, std::size_t worker_id); + + sigset_t ppoll_sigmask; +// std::size_t N_available_polled_results = 0; + + void set_send_flag(int flag); + +private: + void debug_print(std::string s); + + // push + std::vector> qw_push; + ZmqLingeringSocketPtr<> this_worker_qw_push; + ZmqLingeringSocketPtr<> mq_push; + // pollers for all push sockets + std::vector qw_push_poller; + ZeroMQPoller mq_push_poller; + // pull + std::vector> qw_pull; + ZmqLingeringSocketPtr<> this_worker_qw_pull; + ZmqLingeringSocketPtr<> mq_pull; + // pollers for all pull sockets + std::vector qw_pull_poller; + ZeroMQPoller mq_pull_poller; + + // test to circumvent queue for parameter updating + ZmqLingeringSocketPtr<> mw_pub; + ZmqLingeringSocketPtr<> mw_sub; + ZeroMQPoller mw_sub_poller; + // test to circumvent queue for result retrieving + ZmqLingeringSocketPtr<> wm_push; + ZmqLingeringSocketPtr<> wm_pull; + ZeroMQPoller wm_pull_poller; + + // destruction flags to distinguish between different process-type setups: + bool close_MQ_on_destruct_ = false; + bool close_this_QW_on_destruct_ = false; + bool close_QW_container_on_destruct_ = false; + + int send_flag = 0; +}; + + +// Messages from master to queue +enum class M2Q : int { + terminate = 100, + enqueue = 10, + retrieve = 11, + update_real = 12, + // update_cat = 13, + update_bool = 14, +}; + +// Messages from queue to master +enum class Q2M : int { retrieve_rejected = 20, retrieve_accepted = 21, retrieve_later = 22 }; + +// Messages from worker to queue +enum class W2Q : int { dequeue = 30, send_result = 31 }; + +// Messages from queue to worker +enum class Q2W : int { + terminate = 400, + dequeue_rejected = 40, + dequeue_accepted = 41, + result_received = 43, + update_real = 44, + // update_cat = 45 + update_bool = 46, +}; + +// stream output operators for debugging +std::ostream &operator<<(std::ostream &out, const M2Q value); +std::ostream &operator<<(std::ostream &out, const Q2M value); +std::ostream &operator<<(std::ostream &out, const Q2W value); +std::ostream &operator<<(std::ostream &out, const W2Q value); +std::ostream &operator<<(std::ostream &out, const X2X value); + +} // namespace MultiProcess +} // namespace RooFit + +#endif // ROOT_ROOFIT_MultiProcess_Messenger_decl diff --git a/roofit/multiprocess/inc/RooFit/MultiProcess/ProcessManager.h b/roofit/multiprocess/inc/RooFit/MultiProcess/ProcessManager.h new file mode 100644 index 0000000000000..7c83546edecf0 --- /dev/null +++ b/roofit/multiprocess/inc/RooFit/MultiProcess/ProcessManager.h @@ -0,0 +1,78 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2016-2019, Netherlands eScience Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ +#ifndef ROOT_ROOFIT_MultiProcess_ProcessManager +#define ROOT_ROOFIT_MultiProcess_ProcessManager + +#include // pid_t +#include // sig_atomic_t and for sigterm handling on child processes (in ProcessManager.cxx) +#include + +// forward declaration +class Queue; + +namespace RooFit { +namespace MultiProcess { + +class ProcessManager { + friend Queue; + +public: + explicit ProcessManager(std::size_t N_workers); + ~ProcessManager(); + + bool is_initialized() const; + + void terminate() noexcept; +// void terminate_workers(); + void wait_for_sigterm_then_exit(); + + bool is_master() const; + bool is_queue() const; + bool is_worker() const; + std::size_t worker_id() const; + std::size_t N_workers() const; + + void identify_processes() const; + + static void handle_sigterm(int signum); + static bool sigterm_received(); + + // for debugging/testing: + pid_t get_queue_pid() {return queue_pid;} + std::vector get_worker_pids() {return worker_pids;} + +private: + void initialize_processes(bool cpu_pinning = true); + void shutdown_processes(); + + bool _is_master = false; + bool _is_queue = false; + bool _is_worker = false; + std::size_t _worker_id; + std::size_t _N_workers; + + // master must wait for workers after completion, for which it needs their PIDs + std::vector worker_pids; + pid_t queue_pid; + + bool initialized = false; + + static volatile sig_atomic_t _sigterm_received; +}; + +} // namespace MultiProcess +} // namespace RooFit + +#endif // ROOT_ROOFIT_MultiProcess_ProcessManager diff --git a/roofit/multiprocess/inc/RooFit/MultiProcess/Queue.h b/roofit/multiprocess/inc/RooFit/MultiProcess/Queue.h new file mode 100644 index 0000000000000..bd44ac1c36a96 --- /dev/null +++ b/roofit/multiprocess/inc/RooFit/MultiProcess/Queue.h @@ -0,0 +1,46 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2016-2019, Netherlands eScience Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ +#ifndef ROOT_ROOFIT_MultiProcess_Queue +#define ROOT_ROOFIT_MultiProcess_Queue + +#include + +#include "RooFit/MultiProcess/types.h" +#include "RooFit/MultiProcess/Messenger.h" + +namespace RooFit { +namespace MultiProcess { + +class Queue { +public: + bool pop(JobTask &job_task); + void add(JobTask job_task); + + void loop(); + + bool process_master_message(M2Q message); + void process_worker_message(std::size_t this_worker_id, W2Q message); + +private: + std::queue _queue; + std::size_t N_tasks = 0; // total number of received tasks + std::size_t N_tasks_completed = 0; + std::size_t N_tasks_at_workers = 0; +}; + +} // namespace MultiProcess +} // namespace RooFit + +#endif // ROOT_ROOFIT_MultiProcess_Queue diff --git a/roofit/multiprocess/inc/RooFit/MultiProcess/types.h b/roofit/multiprocess/inc/RooFit/MultiProcess/types.h new file mode 100644 index 0000000000000..6b0f0cab87ece --- /dev/null +++ b/roofit/multiprocess/inc/RooFit/MultiProcess/types.h @@ -0,0 +1,29 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2016-2019, Netherlands eScience Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ +#ifndef ROOT_ROOFIT_MultiProcess_types +#define ROOT_ROOFIT_MultiProcess_types + +#include // pair + +namespace RooFit { +namespace MultiProcess { + +// some helper types +using Task = std::size_t; +using JobTask = std::pair; // combined job_object and task identifier type + +} // namespace MultiProcess +} // namespace RooFit + +#endif // ROOT_ROOFIT_MultiProcess_types diff --git a/roofit/multiprocess/inc/RooFit/MultiProcess/util.h b/roofit/multiprocess/inc/RooFit/MultiProcess/util.h new file mode 100644 index 0000000000000..4f5f5f287857a --- /dev/null +++ b/roofit/multiprocess/inc/RooFit/MultiProcess/util.h @@ -0,0 +1,34 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2016-2019, Netherlands eScience Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef ROOT_ROOFIT_MultiProcess_util +#define ROOT_ROOFIT_MultiProcess_util + +#include // getpid, pid_t + +#include // for ZMQ::ppoll_error_t +#include + +namespace RooFit { +namespace MultiProcess { + +int wait_for_child(pid_t child_pid, bool may_throw, int retries_before_killing); + +enum class zmq_ppoll_error_response { abort, unknown_eintr, retry }; +zmq_ppoll_error_response handle_zmq_ppoll_error(ZMQ::ppoll_error_t &e); +std::tuple>, bool> careful_ppoll(ZeroMQPoller &poller, const sigset_t &ppoll_sigmask, std::size_t max_tries = 2); + +} // namespace MultiProcess +} // namespace RooFit +#endif // ROOT_ROOFIT_MultiProcess_util diff --git a/roofit/multiprocess/inc/RooFit/MultiProcess/worker.h b/roofit/multiprocess/inc/RooFit/MultiProcess/worker.h new file mode 100644 index 0000000000000..2fc2eaa7ef5ae --- /dev/null +++ b/roofit/multiprocess/inc/RooFit/MultiProcess/worker.h @@ -0,0 +1,27 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2016-2019, Netherlands eScience Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ +#ifndef ROOT_ROOFIT_MultiProcess_worker +#define ROOT_ROOFIT_MultiProcess_worker + +namespace RooFit { +namespace MultiProcess { + +void worker_loop(); +bool is_worker_loop_running(); + +} // namespace MultiProcess +} // namespace RooFit + +#endif // ROOT_ROOFIT_MultiProcess_worker diff --git a/roofit/multiprocess/src/Job.cxx b/roofit/multiprocess/src/Job.cxx new file mode 100644 index 0000000000000..9f3cd7aacf1a1 --- /dev/null +++ b/roofit/multiprocess/src/Job.cxx @@ -0,0 +1,65 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2016-2019, Netherlands eScience Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "RooFit/MultiProcess/JobManager.h" +#include "RooFit/MultiProcess/Messenger.h" +#include "RooFit/MultiProcess/worker.h" +#include "RooFit/MultiProcess/Job.h" +#include "RooFit/MultiProcess/Queue.h" + +namespace RooFit { +namespace MultiProcess { + +Job::Job() +{ + id = JobManager::add_job_object(this); +} + +Job::Job(const Job &other) + : _manager(other._manager) +{ + id = JobManager::add_job_object(this); +} + +Job::~Job() +{ + JobManager::remove_job_object(id); +} + +// This function is necessary here, because the Job knows about the number +// of workers, so only from the Job can the JobManager be instantiated. +JobManager *Job::get_manager() +{ + if (!_manager) { + _manager = JobManager::instance(); + } + + if (!_manager->is_activated()) { + _manager->activate(); + } + + return _manager; +} + +void Job::gather_worker_results() +{ + get_manager()->retrieve(id); +} + +void Job::update_state() {} + + +} // namespace MultiProcess +} // namespace RooFit diff --git a/roofit/multiprocess/src/JobManager.cxx b/roofit/multiprocess/src/JobManager.cxx new file mode 100644 index 0000000000000..d65a821d39c66 --- /dev/null +++ b/roofit/multiprocess/src/JobManager.cxx @@ -0,0 +1,264 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2016-2019, Netherlands eScience Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ +#include // getpid + +#include // std::thread::hardware_concurrency() + +#include "ROOT/RMakeUnique.hxx" // make_unique in C++11 + +#include "RooFit/MultiProcess/ProcessManager.h" +#include "RooFit/MultiProcess/Messenger.h" +#include "RooFit/MultiProcess/JobManager.h" +#include "RooFit/MultiProcess/Job.h" +#include "RooFit/MultiProcess/Queue.h" // complete type for JobManager::queue() +#include "RooFit/MultiProcess/worker.h" +#include "RooFit/MultiProcess/util.h" + + +namespace RooFit { +namespace MultiProcess { + +// static function +JobManager* JobManager::instance() { + if (!JobManager::is_instantiated()) { + assert(default_N_workers != 0); + _instance.reset(new JobManager(default_N_workers)); // can't use make_unique, because ctor is private + _instance->messenger().test_connections(_instance->process_manager()); + // set send to non blocking on all processes after checking the connections are working: + _instance->messenger().set_send_flag(ZMQ_DONTWAIT); + } + return _instance.get(); +} + + +// static function +bool JobManager::is_instantiated() { + return static_cast(_instance); +} + +// (private) constructor +// Don't construct JobManager objects manually, use the static instance if +// you need to run multiple jobs. +JobManager::JobManager(std::size_t N_workers) { + queue_ptr = std::make_unique(); + process_manager_ptr = std::make_unique(N_workers); + messenger_ptr = std::make_unique(*process_manager_ptr); +} + +JobManager::~JobManager() { + // The instance gets created by some Job. Once all Jobs are gone, the + // JM will get destroyed. In this case, the job_objects map should have + // been emptied. This check makes sure: + assert(JobManager::job_objects.empty()); + messenger_ptr.reset(nullptr); + process_manager_ptr.reset(nullptr); + queue_ptr.reset(nullptr); +} + + +// static function +// returns job_id for added job_object +std::size_t JobManager::add_job_object(Job *job_object) { + if (JobManager::is_instantiated()) { + if (_instance->process_manager().is_initialized()) { + std::stringstream ss; + ss << "Cannot add Job to JobManager instantiation, forking has already taken place! Instance object at raw ptr " << _instance.get(); + throw std::logic_error("Cannot add Job to JobManager instantiation, forking has already taken place! Call terminate() on the instance before adding new Jobs."); + } + } + std::size_t job_id = job_counter++; + job_objects[job_id] = job_object; + return job_id; +} + +// static function +Job* JobManager::get_job_object(std::size_t job_object_id) { + return job_objects[job_object_id]; +} + +// static function +bool JobManager::remove_job_object(std::size_t job_object_id) { + bool removed_succesfully = job_objects.erase(job_object_id) == 1; + if (job_objects.empty()) { + _instance.reset(nullptr); + } + return removed_succesfully; +} + + +ProcessManager & JobManager::process_manager() const { + return *process_manager_ptr; +} + + +Messenger & JobManager::messenger() const { + return *messenger_ptr; +} + + +Queue & JobManager::queue() const { + return *queue_ptr; +} + + +//void JobManager::retrieve() { +// if (process_manager().is_master()) { +// bool carry_on = true; +// while (carry_on) { +// messenger().send_from_master_to_queue(M2Q::retrieve); +// try { +// auto handshake = messenger().receive_from_queue_on_master(); +// switch (handshake) { +// case Q2M::retrieve_accepted: { +// carry_on = false; +// auto N_jobs = messenger().receive_from_queue_on_master(); +// for (std::size_t job_ix = 0; job_ix < N_jobs; ++job_ix) { +// auto job_object_id = messenger().receive_from_queue_on_master(); +// JobManager::get_job_object(job_object_id)->receive_results_on_master(); +// } +// } break; +// case Q2M::retrieve_later: { +// carry_on = true; +// } break; +// case Q2M::retrieve_rejected: { +// carry_on = false; +// throw std::logic_error( +// "Master sent M2Q::retrieve, but queue had no tasks yet: Q2M::retrieve_rejected. Aborting!"); +// } break; +// } +// } catch (ZMQ::ppoll_error_t &e) { +// zmq_ppoll_error_response response; +// try { +// response = handle_zmq_ppoll_error(e); +// } catch (std::logic_error& e) { +// printf("JobManager::retrieve got unhandleable ZMQ::ppoll_error_t\n"); +// throw; +// } +// if (response == zmq_ppoll_error_response::abort) { +// throw std::logic_error("in JobManager::retrieve: master received a SIGTERM, aborting"); +// } else if (response == zmq_ppoll_error_response::unknown_eintr) { +// printf("EINTR in JobManager::retrieve, continuing\n"); +// continue; +// } else if (response == zmq_ppoll_error_response::retry) { +// printf("EAGAIN from ppoll in JobManager::retrieve, continuing\n"); +// continue; +// } +// } catch (zmq::error_t& e) { +// printf("unhandled zmq::error_t (not a ppoll_error_t) in JobManager::retrieve with errno %d: %s\n", e.num(), e.what()); +// throw; +// } +// +// } +// } +//} + + +void JobManager::retrieve(std::size_t requesting_job_id) { + if (process_manager().is_master()) { +// auto get_time = []() { +// return std::chrono::duration_cast( +// std::chrono::high_resolution_clock::now().time_since_epoch()) +// .count(); +// }; +// decltype(get_time()) t1, t2; + + bool job_fully_retrieved = false; + while (not job_fully_retrieved) { + try { +// t1 = get_time(); + auto task_result_message = messenger().receive_from_worker_on_master(); + auto job_object_id = *reinterpret_cast(task_result_message.data()); // job_id must always be the first element of the result message! + bool this_job_fully_retrieved = JobManager::get_job_object(job_object_id)->receive_task_result_on_master(task_result_message); + if (requesting_job_id == job_object_id) { + job_fully_retrieved = this_job_fully_retrieved; + } +// t2 = get_time(); +// printf("wallclock [master] retrieve got task %lu: %f\n", *(reinterpret_cast(task_result_message.data()) + 1), (t2 - t1) / 1.e9); + } catch (ZMQ::ppoll_error_t &e) { + zmq_ppoll_error_response response; + try { + response = handle_zmq_ppoll_error(e); + } catch (std::logic_error& e) { + printf("JobManager::retrieve got unhandleable ZMQ::ppoll_error_t\n"); + throw; + } + if (response == zmq_ppoll_error_response::abort) { + throw std::logic_error("in JobManager::retrieve: master received a SIGTERM, aborting"); + } else if (response == zmq_ppoll_error_response::unknown_eintr) { + printf("EINTR in JobManager::retrieve, continuing\n"); + continue; + } else if (response == zmq_ppoll_error_response::retry) { + printf("EAGAIN from ppoll in JobManager::retrieve, continuing\n"); + continue; + } + } catch (zmq::error_t& e) { + printf("unhandled zmq::error_t (not a ppoll_error_t) in JobManager::retrieve with errno %d: %s\n", e.num(), e.what()); + throw; + } + } + } +} + + +void JobManager::results_from_queue_to_master() +{ + assert(process_manager().is_queue()); + messenger().send_from_queue_to_master(Q2M::retrieve_accepted, JobManager::job_objects.size()); + for (auto job_tuple : JobManager::job_objects) { + messenger().send_from_queue_to_master(job_tuple.first); // job id + job_tuple.second->send_back_results_from_queue_to_master(); // N_job_tasks, task_ids and results + job_tuple.second->clear_results(); + } +} + + +void JobManager::activate() +{ + // This function exists purely because activation from the constructor is + // impossible; the constructor must return a constructed instance, which it + // can't do if it's stuck in an infinite loop. This means the Job that first + // creates the JobManager instance must also activate it (or any other user + // of this class). + // This should be called soon after creation of instance, because everything + // between construction and activation gets executed both on the master + // process and on the slaves. + + activated = true; + + if (process_manager().is_queue()) { + queue().loop(); + std::_Exit(0); + } + + if (!is_worker_loop_running() && process_manager().is_worker()) { + RooFit::MultiProcess::worker_loop(); + std::_Exit(0); + } +} + + +bool JobManager::is_activated() const +{ + return activated; +} + +// initialize static members +std::map JobManager::job_objects; +std::size_t JobManager::job_counter = 0; +std::unique_ptr JobManager::_instance {nullptr}; +unsigned int JobManager::default_N_workers = std::thread::hardware_concurrency(); + +} // namespace MultiProcess +} // namespace RooFit diff --git a/roofit/multiprocess/src/Messenger.cxx b/roofit/multiprocess/src/Messenger.cxx new file mode 100644 index 0000000000000..911535506034d --- /dev/null +++ b/roofit/multiprocess/src/Messenger.cxx @@ -0,0 +1,481 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2016-2019, Netherlands eScience Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include // sigprocmask etc +#include // sprintf + +#include +#include "RooFit/MultiProcess/Messenger.h" + +namespace RooFit { +namespace MultiProcess { + +void set_socket_immediate(ZmqLingeringSocketPtr<> &socket) +{ + int optval = 1; + socket->setsockopt(ZMQ_IMMEDIATE, &optval, sizeof(optval)); +} + +Messenger::Messenger(const ProcessManager &process_manager) +{ + sigemptyset(&ppoll_sigmask); + + // high water mark for master-queue sending, which can be quite a busy channel, especially at the start of a run + int hwm = 0; + // create zmq connections (zmq context is automatically created in the ZeroMQSvc class and maintained as singleton) + // and pollers where necessary + try { + if (process_manager.is_master()) { + mq_push.reset(zmqSvc().socket_ptr(zmq::PUSH)); + auto rc = zmq_setsockopt (*mq_push, ZMQ_SNDHWM, &hwm, sizeof hwm); + assert (rc == 0); + mq_push->bind("ipc:///tmp/roofitMP_from_master_to_queue"); + + mq_push_poller.register_socket(*mq_push, zmq::POLLOUT); + + mq_pull.reset(zmqSvc().socket_ptr(zmq::PULL)); + rc = zmq_setsockopt (*mq_pull, ZMQ_RCVHWM, &hwm, sizeof hwm); + assert (rc == 0); + mq_pull->bind("ipc:///tmp/roofitMP_from_queue_to_master"); + + mq_pull_poller.register_socket(*mq_pull, zmq::POLLIN); + + mw_pub.reset(zmqSvc().socket_ptr(zmq::PUB)); + rc = zmq_setsockopt (*mw_pub, ZMQ_SNDHWM, &hwm, sizeof hwm); + assert (rc == 0); + mw_pub->bind("ipc:///tmp/roofitMP_from_master_to_workers"); + + wm_pull.reset(zmqSvc().socket_ptr(zmq::PULL)); + rc = zmq_setsockopt (*wm_pull, ZMQ_RCVHWM, &hwm, sizeof hwm); + assert (rc == 0); + wm_pull->bind("ipc:///tmp/roofitMP_from_workers_to_master"); + wm_pull_poller.register_socket(*wm_pull, zmq::POLLIN); + + close_MQ_on_destruct_ = true; + + // make sure all subscribers are connected + ZmqLingeringSocketPtr<> subscriber_ping_socket {zmqSvc().socket_ptr(zmq::REP)}; + subscriber_ping_socket->bind("ipc:///tmp/roofitMP_subscriber_ping_socket"); + ZeroMQPoller subscriber_ping_poller; + subscriber_ping_poller.register_socket(*subscriber_ping_socket, zmq::POLLIN); + std::size_t N_subscribers_confirmed = 0; + while (N_subscribers_confirmed < process_manager.N_workers()) { + zmqSvc().send(*mw_pub, false); + auto poll_results = subscriber_ping_poller.poll(0); +// for (auto& poll_result : poll_results) { + for (std::size_t ix = 0; ix < poll_results.size(); ++ix) { + auto request = zmqSvc().receive(*subscriber_ping_socket, zmq::DONTWAIT); + assert(request == "present"); + zmqSvc().send(*subscriber_ping_socket, "roger"); + ++N_subscribers_confirmed; + } + } + zmqSvc().send(*mw_pub, true); + + } else if (process_manager.is_queue()) { + // first the queue-worker sockets + // do resize instead of reserve so that the unique_ptrs are initialized + // (to nullptr) so that we can do reset below, alternatively you can do + // push/emplace_back with move or something + qw_push.resize(process_manager.N_workers()); + qw_pull.resize(process_manager.N_workers()); + qw_push_poller.resize(process_manager.N_workers()); + qw_pull_poller.resize(process_manager.N_workers()); + for (std::size_t ix = 0; ix < process_manager.N_workers(); ++ix) { + std::stringstream push_name, pull_name; + // push + qw_push[ix].reset(zmqSvc().socket_ptr(zmq::PUSH)); + push_name << "ipc:///tmp/roofitMP_from_queue_to_worker_" << ix; + qw_push[ix]->bind(push_name.str()); + + qw_push_poller[ix].register_socket(*qw_push[ix], zmq::POLLOUT); + + // pull + qw_pull[ix].reset(zmqSvc().socket_ptr(zmq::PULL)); + pull_name << "ipc:///tmp/roofitMP_from_worker_" << ix << "_to_queue"; + qw_pull[ix]->bind(pull_name.str()); + + qw_pull_poller[ix].register_socket(*qw_pull[ix], zmq::POLLIN); + } + + // then the master-queue sockets + mq_push.reset(zmqSvc().socket_ptr(zmq::PUSH)); + auto rc = zmq_setsockopt (*mq_push, ZMQ_SNDHWM, &hwm, sizeof hwm); + assert (rc == 0); + mq_push->connect("ipc:///tmp/roofitMP_from_queue_to_master"); + + mq_push_poller.register_socket(*mq_push, zmq::POLLOUT); + + mq_pull.reset(zmqSvc().socket_ptr(zmq::PULL)); + rc = zmq_setsockopt (*mq_pull, ZMQ_RCVHWM, &hwm, sizeof hwm); + assert (rc == 0); + mq_pull->connect("ipc:///tmp/roofitMP_from_master_to_queue"); + + mq_pull_poller.register_socket(*mq_pull, zmq::POLLIN); + + close_MQ_on_destruct_ = true; + close_QW_container_on_destruct_ = true; + } else if (process_manager.is_worker()) { + // we only need one queue-worker pipe on the worker + qw_push_poller.resize(1); + qw_pull_poller.resize(1); + + std::stringstream push_name, pull_name; + // push + this_worker_qw_push.reset(zmqSvc().socket_ptr(zmq::PUSH)); + push_name << "ipc:///tmp/roofitMP_from_worker_" << process_manager.worker_id() << "_to_queue"; + this_worker_qw_push->connect(push_name.str()); + + qw_push_poller[0].register_socket(*this_worker_qw_push, zmq::POLLOUT); + + // pull + this_worker_qw_pull.reset(zmqSvc().socket_ptr(zmq::PULL)); + pull_name << "ipc:///tmp/roofitMP_from_queue_to_worker_" << process_manager.worker_id(); + this_worker_qw_pull->connect(pull_name.str()); + + qw_pull_poller[0].register_socket(*this_worker_qw_pull, zmq::POLLIN); + + mw_sub.reset(zmqSvc().socket_ptr(zmq::SUB)); + auto rc = zmq_setsockopt (*mw_sub, ZMQ_RCVHWM, &hwm, sizeof hwm); + assert (rc == 0); + rc = zmq_setsockopt(*mw_sub, ZMQ_SUBSCRIBE, "", 0); + assert (rc == 0); + mw_sub->connect("ipc:///tmp/roofitMP_from_master_to_workers"); + mw_sub_poller.register_socket(*mw_sub, zmq::POLLIN); + + wm_push.reset(zmqSvc().socket_ptr(zmq::PUSH)); + rc = zmq_setsockopt (*wm_push, ZMQ_SNDHWM, &hwm, sizeof hwm); + assert (rc == 0); + wm_push->connect("ipc:///tmp/roofitMP_from_workers_to_master"); + + // check publisher connection and then wait until all subscribers are connected + ZmqLingeringSocketPtr<> subscriber_ping_socket {zmqSvc().socket_ptr(zmq::REQ)}; + subscriber_ping_socket->connect("ipc:///tmp/roofitMP_subscriber_ping_socket"); + auto all_connected = zmqSvc().receive(*mw_sub); + zmqSvc().send(*subscriber_ping_socket, "present"); + auto reply = zmqSvc().receive(*subscriber_ping_socket); + assert(reply == "roger"); + + while (!all_connected) { + all_connected = zmqSvc().receive(*mw_sub); + } + + close_this_QW_on_destruct_ = true; + } else { + // should never get here + throw std::runtime_error("Messenger ctor: I'm neither master, nor queue, nor a worker"); + } + } catch (zmq::error_t &e) { + std::cerr << e.what() << " -- errnum: " << e.num() << std::endl; + throw; + }; +} + +Messenger::~Messenger() { + printf("Messenger dtor on PID %d\n", getpid()); + if (close_MQ_on_destruct_) { + try { + mq_push.reset(nullptr); + mq_pull.reset(nullptr); + mw_pub.reset(nullptr); + wm_pull.reset(nullptr); + } catch (const std::exception& e) { + std::cerr << "WARNING: something in Messenger dtor threw an exception! Original exception message:\n" << e.what() << std::endl; + } + } + if (close_this_QW_on_destruct_) { + this_worker_qw_push.reset(nullptr); + this_worker_qw_pull.reset(nullptr); + mw_sub.reset(nullptr); + wm_push.reset(nullptr); + } + if (close_QW_container_on_destruct_) { + for (auto& socket : qw_push) { + socket.reset(nullptr); + } + for (auto& socket : qw_pull) { + socket.reset(nullptr); + } + } + zmqSvc().close_context(); +} + + +void Messenger::test_send(X2X ping_value, test_snd_pipes snd_pipe, std::size_t worker_id) { + try { + switch (snd_pipe) { + case test_snd_pipes::M2Q: { + send_from_master_to_queue(ping_value); + break; + } + case test_snd_pipes::Q2M : { + send_from_queue_to_master(ping_value); + break; + } + case test_snd_pipes::Q2W: { + send_from_queue_to_worker(worker_id, ping_value); + break; + } + case test_snd_pipes::W2Q: { + send_from_worker_to_queue(ping_value); + break; + } + } + } catch (zmq::error_t &e) { + if (e.num() == EAGAIN) { + throw std::runtime_error("Messenger::test_connections: SEND over master-queue connection timed out!"); + } else { + throw; + } + } +} + + +void Messenger::test_receive(X2X expected_ping_value, test_rcv_pipes rcv_pipe, std::size_t worker_id) { + X2X handshake = X2X::initial_value; + + std::size_t max_tries = 3, tries = 0; + bool carry_on = true; + while (carry_on && (tries++ < max_tries)) { + try { + switch (rcv_pipe) { + case test_rcv_pipes::fromMonQ: { + handshake = receive_from_master_on_queue(); + break; + } + case test_rcv_pipes::fromQonM: { + handshake = receive_from_queue_on_master(); + break; + } + case test_rcv_pipes::fromQonW: { + handshake = receive_from_queue_on_worker(); + break; + } + case test_rcv_pipes::fromWonQ: { + handshake = receive_from_worker_on_queue(worker_id); + break; + } + } + carry_on = false; + } catch (ZMQ::ppoll_error_t &e) { + auto response = handle_zmq_ppoll_error(e); + if (response == zmq_ppoll_error_response::abort) { + throw std::runtime_error("EINTR in test_receive and SIGTERM received, aborting\n"); + } else if (response == zmq_ppoll_error_response::unknown_eintr) { + printf("EINTR in test_receive but no SIGTERM received, try %lu\n", tries); + continue; + } else if (response == zmq_ppoll_error_response::retry) { + printf("EAGAIN in test_receive, try %lu\n", tries); + continue; + } + } catch (zmq::error_t &e) { + if (e.num() == EAGAIN) { + throw std::runtime_error("Messenger::test_connections: RECEIVE over master-queue connection timed out!"); + } else { + printf("unhandled zmq::error_t (not a ppoll_error_t) in Messenger::test_receive with errno %d: %s\n", e.num(), e.what()); + throw; + } + } + } + + if (handshake != expected_ping_value) { + throw std::runtime_error("Messenger::test_connections: RECEIVE over master-queue connection failed, did not receive pong!"); + } +} + + +void Messenger::test_connections(const ProcessManager &process_manager) { + // Before blocking SIGTERM, set the signal handler, so we can also check after blocking whether a signal occurred + // In our case, we already set it in the ProcessManager after forking to the queue and worker processes. + sigset_t sigmask; + sigemptyset(&sigmask); + sigaddset(&sigmask, SIGTERM); + sigprocmask(SIG_BLOCK, &sigmask, &ppoll_sigmask); + + if (process_manager.is_master()) { + test_send(X2X::ping, test_snd_pipes::M2Q, -1); + test_receive(X2X::pong, test_rcv_pipes::fromQonM, -1); + test_receive(X2X::ping, test_rcv_pipes::fromQonM, -1); + test_send(X2X::pong, test_snd_pipes::M2Q, -1); + } else if (process_manager.is_queue()) { + ZeroMQPoller poller; + std::size_t mq_index; + std::tie(poller, mq_index) = create_queue_poller(); + + for (std::size_t ix = 0; ix < process_manager.N_workers(); ++ix) { + test_send(X2X::ping, test_snd_pipes::Q2W, ix); + } + + while (!process_manager.sigterm_received() && (poller.size() > 0)) { + // poll: wait until status change (-1: infinite timeout) + std::vector> poll_result; + bool abort; + std::tie(poll_result, abort) = careful_ppoll(poller, ppoll_sigmask); + if (abort) break; + + // then process incoming messages from sockets + for (auto readable_socket : poll_result) { + // message comes from the master/queue socket (first element): + if (readable_socket.first == mq_index) { + test_receive(X2X::ping, test_rcv_pipes::fromMonQ, -1); + test_send(X2X::pong, test_snd_pipes::Q2M, -1); + test_send(X2X::ping, test_snd_pipes::Q2M, -1); + test_receive(X2X::pong, test_rcv_pipes::fromMonQ, -1); + poller.unregister_socket(*mq_pull); + } else { // from a worker socket + // TODO: dangerous assumption for this_worker_id, may become invalid if we allow multiple queue_loops on the same process! + auto this_worker_id = readable_socket.first - 1; // TODO: replace with a more reliable lookup + + test_receive(X2X::pong, test_rcv_pipes::fromWonQ, this_worker_id); + test_receive(X2X::ping, test_rcv_pipes::fromWonQ, this_worker_id); + test_send(X2X::pong, test_snd_pipes::Q2W, this_worker_id); + + poller.unregister_socket(*qw_pull[this_worker_id]); + } + } + } + + } else if (process_manager.is_worker()) { + test_receive(X2X::ping, test_rcv_pipes::fromQonW, -1); + test_send(X2X::pong, test_snd_pipes::W2Q, -1); + test_send(X2X::ping, test_snd_pipes::W2Q, -1); + test_receive(X2X::pong, test_rcv_pipes::fromQonW, -1); + } else { + // should never get here + throw std::runtime_error("Messenger::test_connections: I'm neither master, nor queue, nor a worker"); + } + + // clean up signal management modifications + sigprocmask(SIG_SETMASK, &ppoll_sigmask, nullptr); + printf("done with test_connections on PID %d\n", getpid()); +} + + +std::pair Messenger::create_queue_poller() { + ZeroMQPoller poller; + std::size_t mq_index = poller.register_socket(*mq_pull, zmq::POLLIN); + for (auto &s : qw_pull) { + poller.register_socket(*s, zmq::POLLIN); + } + return {std::move(poller), mq_index}; +} + + +std::pair Messenger::create_worker_poller() { + ZeroMQPoller poller; + poller.register_socket(*this_worker_qw_pull, zmq::POLLIN); + std::size_t mw_sub_index = poller.register_socket(*mw_sub, zmq::POLLIN); + return {std::move(poller), mw_sub_index}; +} + + +// -- WORKER - QUEUE COMMUNICATION -- + +void Messenger::send_from_worker_to_queue() {} + +void Messenger::send_from_queue_to_worker(std::size_t /*this_worker_id*/) {} + +// -- QUEUE - MASTER COMMUNICATION -- + +void Messenger::send_from_queue_to_master() {} + +void Messenger::send_from_master_to_queue() {} + +void Messenger::set_send_flag(int flag) { + if (flag == 0 || flag == ZMQ_DONTWAIT || flag == ZMQ_SNDMORE || flag == (ZMQ_DONTWAIT | ZMQ_SNDMORE)) { + send_flag = flag; + } else { + throw std::runtime_error("in Messenger::set_send_flag: trying to set illegal flag, see zmq_send API for allowed flags"); + } +} + +// -- MASTER - WORKER COMMUNICATION -- + +void Messenger::publish_from_master_to_workers() {} + +void Messenger::send_from_worker_to_master() {} + + +// for debugging +#define PROCESS_VAL(p) case(p): s = #p; break; + +std::ostream& operator<<(std::ostream& out, const M2Q value){ + std::string s; + switch(value){ + PROCESS_VAL(M2Q::terminate); + PROCESS_VAL(M2Q::enqueue); + PROCESS_VAL(M2Q::retrieve); + PROCESS_VAL(M2Q::update_real); + PROCESS_VAL(M2Q::update_bool); + default: s = std::to_string(static_cast(value)); + } + return out << s; +} + +std::ostream& operator<<(std::ostream& out, const Q2M value){ + std::string s; + switch(value){ + PROCESS_VAL(Q2M::retrieve_rejected); + PROCESS_VAL(Q2M::retrieve_accepted); + PROCESS_VAL(Q2M::retrieve_later); + default: s = std::to_string(static_cast(value)); + } + return out << s; +} + +std::ostream& operator<<(std::ostream& out, const W2Q value){ + std::string s; + switch(value){ + PROCESS_VAL(W2Q::dequeue); + PROCESS_VAL(W2Q::send_result); + default: s = std::to_string(static_cast(value)); + } + return out << s; +} + +std::ostream& operator<<(std::ostream& out, const Q2W value){ + std::string s; + switch(value){ + PROCESS_VAL(Q2W::terminate); + PROCESS_VAL(Q2W::dequeue_rejected); + PROCESS_VAL(Q2W::dequeue_accepted); + PROCESS_VAL(Q2W::update_real); + PROCESS_VAL(Q2W::result_received); + PROCESS_VAL(Q2W::update_bool); + default: s = std::to_string(static_cast(value)); + } + return out << s; +} + +std::ostream& operator<<(std::ostream& out, const X2X value){ + std::string s; + switch(value){ + PROCESS_VAL(X2X::ping); + PROCESS_VAL(X2X::pong); + default: s = std::to_string(static_cast(value)); + } + return out << s; +} + +#undef PROCESS_VAL + +void Messenger::debug_print(std::string /*s*/) +{ +// printf("%s\n", s.c_str()); +// std::cerr << s << std::endl; +} + +} // namespace MultiProcess +} // namespace RooFit diff --git a/roofit/multiprocess/src/ProcessManager.cxx b/roofit/multiprocess/src/ProcessManager.cxx new file mode 100644 index 0000000000000..6720443b9dddd --- /dev/null +++ b/roofit/multiprocess/src/ProcessManager.cxx @@ -0,0 +1,286 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2016-2019, Netherlands eScience Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include // for strsignal +#include // for wait +#include +#include + +#include "RooFit/MultiProcess/util.h" +#include "RooFit/MultiProcess/JobManager.h" +#include "RooFit/MultiProcess/Messenger.h" +#include "RooFit/MultiProcess/ProcessManager.h" + +namespace RooFit { +namespace MultiProcess { + + +ProcessManager::ProcessManager(std::size_t N_workers) : _N_workers(N_workers) { + // This class defines three types of processes: + // 1. master: the initial main process. It defines and enqueues tasks + // and processes results. + // 2. workers: a pool of processes that will try to take tasks from the + // queue. These are forked from master. + // 3. queue: communication between the other types (necessarily) goes + // through this process. This process runs the queue_loop and + // maintains the queue of tasks. + + initialize_processes(); +} + +ProcessManager::~ProcessManager() +{ + if (is_master()) { + terminate(); + } else { + wait_for_sigterm_then_exit(); + } +} + + +// static member initialization +volatile sig_atomic_t ProcessManager::_sigterm_received = 0; + +// static function +void ProcessManager::handle_sigterm(int signum) { + // We need this to tell the children to die, because we can't talk + // to them anymore during JobManager destruction, because that kills + // the Messenger first. We do that with SIGTERMs. The sigterm_received() + // should be checked in message loops to stop them when it's true. + _sigterm_received = 1; + printf("handled %s on PID %d\n", strsignal(signum), getpid()); +} + +// static function +bool ProcessManager::sigterm_received() { + if (_sigterm_received > 0) { + return true; + } else { + return false; + } +} + +void ProcessManager::initialize_processes(bool cpu_pinning) { + // Initialize processes; + // ... first workers: + worker_pids.resize(_N_workers); + pid_t child_pid {}; + for (std::size_t ix = 0; ix < _N_workers; ++ix) { + child_pid = fork(); + if (!child_pid) { // we're on the worker + _is_worker = true; + _worker_id = ix; + break; + } else { // we're on master + worker_pids[ix] = child_pid; + } + } + + // ... then queue: + if (child_pid) { // we're on master + queue_pid = fork(); + if (!queue_pid) { // we're now on queue + _is_queue = true; + } else { + _is_master = true; + } + } + +// printf("ATTACH NOOOOOOW PID %d\n", getpid()); +// sleep(30); + + // set the sigterm handler on the child processes + if (!_is_master) { + struct sigaction sa; + memset (&sa, '\0', sizeof(sa)); + sa.sa_handler = ProcessManager::handle_sigterm; + + if (sigaction(SIGTERM, &sa, NULL) < 0) { + std::perror("sigaction failed"); + std::exit(1); + } + } + + if (cpu_pinning) { +#if defined(__APPLE__) + #ifndef NDEBUG + static bool affinity_warned = false; + if (is_master() & !affinity_warned) { + std::cout << "CPU affinity cannot be set on macOS" << std::endl; + affinity_warned = true; + } + #endif // NDEBUG +#elif defined(_WIN32) + #ifndef NDEBUG + if (is_master()) std::cerr << "WARNING: CPU affinity setting not implemented on Windows, continuing..." << std::endl; + #endif // NDEBUG +#else + cpu_set_t mask; + // zero all bits in mask + CPU_ZERO(&mask); + // set correct bit + std::size_t set_cpu; + if (is_master()) { + set_cpu = _N_workers + 1; + } else if (is_queue()) { + set_cpu = _N_workers; + } else { + set_cpu = _worker_id; + } + CPU_SET(set_cpu, &mask); + #ifndef NDEBUG + // sched_setaffinity returns 0 on success + if (sched_setaffinity(0, sizeof(mask), &mask) == -1) { + std::cerr << "WARNING: Could not set CPU affinity, continuing..." << std::endl; + } else { + std::cerr << "CPU affinity set to cpu " << set_cpu << " in process " << getpid() << std::endl; + } + #endif // NDEBUG +#endif + } + +#ifndef NDEBUG + identify_processes(); +#endif // NDEBUG + + initialized = true; +} + +bool ProcessManager::is_initialized() const { + return initialized; +} + + +void ProcessManager::terminate() noexcept { + try { + if (is_master() && is_initialized()) { +// JobManager::instance()->messenger().send_from_master_to_queue(M2Q::terminate); + shutdown_processes(); + } + } catch (const std::exception& e) { + std::cerr << "WARNING: something in ProcessManager::terminate threw an exception! Original exception message:\n" << e.what() << std::endl; + } +} + + +void ProcessManager::wait_for_sigterm_then_exit() { + if (!is_master()) { + while(!sigterm_received()) {} + std::_Exit(0); + } +} + + +//void ProcessManager::terminate_workers() { +// if (is_queue()) { +// for(std::size_t worker_ix = 0; worker_ix < _N_workers; ++worker_ix) { +// JobManager::instance()->messenger().send_from_queue_to_worker(worker_ix, Q2W::terminate); +// } +// JobManager::instance()->messenger().close_queue_worker_connections(); +// } +//} + + +int chill_wait() { + int status = 0; + pid_t pid; + do { + pid = wait(&status); + } while (-1 == pid && EINTR == errno); // retry on interrupted system call + + if (0 != status) { + if (WIFEXITED(status)) { + printf("exited, status=%d\n", WEXITSTATUS(status)); + } else if (WIFSIGNALED(status)) { + printf("killed by signal %d\n", WTERMSIG(status)); + } else if (WIFSTOPPED(status)) { + printf("stopped by signal %d\n", WSTOPSIG(status)); + } else if (WIFCONTINUED(status)) { + printf("continued\n"); + } + } + + if (-1 == pid) { + if (errno == ECHILD) { + printf("chill_wait: no children (got ECHILD error code from wait call), done\n"); + } else { + throw std::runtime_error(std::string("chill_wait: error in wait call: ") + strerror(errno) + std::string(", errno ") + std::to_string(errno)); + } + } + + return pid; +} + +void ProcessManager::shutdown_processes() { + if (is_master()) { + // terminate all children + std::unordered_set children; + children.insert(queue_pid); + kill(queue_pid, SIGTERM); + for (auto pid : worker_pids) { + kill(pid, SIGTERM); + children.insert(pid); + } + // then wait for them to actually die and clean out the zombies + while (!children.empty()) { + pid_t pid = chill_wait(); + children.erase(pid); + } + } + + initialized = false; +} + + +// Getters + +bool ProcessManager::is_master() const { + return _is_master; +} + +bool ProcessManager::is_queue() const { + return _is_queue; +} + +bool ProcessManager::is_worker() const { + return _is_worker; +} + +std::size_t ProcessManager::worker_id() const { + return _worker_id; +} + +std::size_t ProcessManager::N_workers() const { + return _N_workers; +} + + +// Debugging + +void ProcessManager::identify_processes() const { + // identify yourselves (for debugging) + if (_is_worker) { + printf("I'm a worker, PID %d\n", getpid()); + } else if (_is_master) { + printf("I'm master, PID %d\n", getpid()); + } else if (_is_queue) { + printf("I'm queue, PID %d\n", getpid()); + } else { + printf("I'm not master, queue or worker, weird! PID %d\n", getpid()); + } +} + +} // namespace MultiProcess +} // namespace RooFit diff --git a/roofit/multiprocess/src/Queue.cxx b/roofit/multiprocess/src/Queue.cxx new file mode 100644 index 0000000000000..5ea167812de59 --- /dev/null +++ b/roofit/multiprocess/src/Queue.cxx @@ -0,0 +1,224 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2016-2019, Netherlands eScience Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "RooFit/MultiProcess/JobManager.h" +#include "RooFit/MultiProcess/ProcessManager.h" +#include "RooFit/MultiProcess/Job.h" // complete Job object for JobManager::get_job_object() +#include "RooFit/MultiProcess/Queue.h" +#include "RooFit/MultiProcess/util.h" + +namespace RooFit { +namespace MultiProcess { + +// Have a worker ask for a task-message from the queue +bool Queue::pop(JobTask &job_task) +{ + if (_queue.empty()) { + return false; + } else { + job_task = _queue.front(); + _queue.pop(); + return true; + } +} + +// Enqueue a task +void Queue::add(JobTask job_task) +{ + if (JobManager::instance()->process_manager().is_master()) { +// not necessary, the job adds tasks, so it is already activated there; TODO: remove commented out lines +// if (!JobManager::instance()->is_activated()) { +// JobManager::instance()->activate(); +// } + JobManager::instance()->messenger().send_from_master_to_queue(M2Q::enqueue, job_task.first, job_task.second); + } else if (JobManager::instance()->process_manager().is_queue()) { + _queue.push(job_task); + } else { + throw std::logic_error("calling Communicator::to_master_queue from slave process"); + } +} + + +bool Queue::process_master_message(M2Q message) +{ + bool carry_on = true; + + switch (message) { + case M2Q::terminate: { + carry_on = false; + break; + } + + case M2Q::enqueue: { + // enqueue task + auto job_object_id = JobManager::instance()->messenger().receive_from_master_on_queue(); + auto task = JobManager::instance()->messenger().receive_from_master_on_queue(); + JobTask job_task(job_object_id, task); + add(job_task); + N_tasks++; + break; + } + + case M2Q::retrieve: { + // retrieve task results after queue is empty and all + // tasks have been completed + if (_queue.empty() && N_tasks_at_workers == 0 && N_tasks_completed == 0) { + JobManager::instance()->messenger().send_from_queue_to_master(Q2M::retrieve_rejected); // handshake message: no tasks enqueued, premature retrieve! + } else if (_queue.empty() && N_tasks_completed == N_tasks) { + JobManager::instance()->results_from_queue_to_master(); + // reset number of received and completed tasks + N_tasks = 0; + N_tasks_completed = 0; + } else { + JobManager::instance()->messenger().send_from_queue_to_master(Q2M::retrieve_later); // handshake message: tasks not done yet, try again + } + break; + } + + case M2Q::update_real: { +// auto get_time = []() { +// return std::chrono::duration_cast( +// std::chrono::high_resolution_clock::now().time_since_epoch()) +// .count(); +// }; +// auto t1 = get_time(); + auto job_id = JobManager::instance()->messenger().receive_from_master_on_queue(); + auto ix = JobManager::instance()->messenger().receive_from_master_on_queue(); + auto val = JobManager::instance()->messenger().receive_from_master_on_queue(); + auto is_constant = JobManager::instance()->messenger().receive_from_master_on_queue(); +// std::cout << "\ngot ix = " << ix << ", job_id = " << job_id << ", val = " << val << ", is_constant = " << is_constant << " in queue loop from M2Q::update_real" << std::endl; + for (std::size_t worker_ix = 0; worker_ix < JobManager::instance()->process_manager().N_workers(); ++worker_ix) { + JobManager::instance()->messenger().send_from_queue_to_worker(worker_ix, Q2W::update_real, job_id, ix, val, is_constant); + } +// auto t2 = get_time(); +// std::cout << "update_real on queue: " << (t2 - t1) / 1.e9 << "s (from " << t1 << " to " << t2 << "ns)" << std::endl; + break; + } + + case M2Q::update_bool: { + auto job_id = JobManager::instance()->messenger().receive_from_master_on_queue(); + auto ix = JobManager::instance()->messenger().receive_from_master_on_queue(); + auto value = JobManager::instance()->messenger().receive_from_master_on_queue(); + for (std::size_t worker_ix = 0; worker_ix < JobManager::instance()->process_manager().N_workers(); ++worker_ix) { + JobManager::instance()->messenger().send_from_queue_to_worker(worker_ix, Q2W::update_bool, job_id, ix, value); + } + } + } + + return carry_on; +} + + +void Queue::process_worker_message(std::size_t this_worker_id, W2Q message) +{ + switch (message) { + case W2Q::dequeue: { + // dequeue task + JobTask job_task; + bool popped = pop(job_task); + if (popped) { + // Note: below two commands should be run atomically for thread safety (if that ever becomes an issue) + JobManager::instance()->messenger().send_from_queue_to_worker(this_worker_id, Q2W::dequeue_accepted, job_task.first, job_task.second); + ++N_tasks_at_workers; + } else { + JobManager::instance()->messenger().send_from_queue_to_worker(this_worker_id, Q2W::dequeue_rejected); + } + break; + } + + case W2Q::send_result: { + // receive back task result + auto job_object_id = JobManager::instance()->messenger().receive_from_worker_on_queue(this_worker_id); + auto task = JobManager::instance()->messenger().receive_from_worker_on_queue(this_worker_id); + JobManager::get_job_object(job_object_id)->receive_task_result_on_queue(task, this_worker_id); + JobManager::instance()->messenger().send_from_queue_to_worker(this_worker_id, Q2W::result_received); + N_tasks_completed++; + --N_tasks_at_workers; + break; + } + } +} + + +void Queue::loop() +{ + assert(JobManager::instance()->process_manager().is_queue()); + bool carry_on = true; + ZeroMQPoller poller; + std::size_t mq_index; + std::tie(poller, mq_index) = JobManager::instance()->messenger().create_queue_poller(); + + // Before blocking SIGTERM, set the signal handler, so we can also check after blocking whether a signal occurred + // In our case, we already set it in the ProcessManager after forking to the queue and worker processes. + + sigset_t sigmask; + sigemptyset(&sigmask); + sigaddset(&sigmask, SIGTERM); + sigprocmask(SIG_BLOCK, &sigmask, &JobManager::instance()->messenger().ppoll_sigmask); + + // Before doing anything, check whether we have received a terminate signal while blocking signals! + // In this case, we also do that in the while condition. + while (!ProcessManager::sigterm_received() && carry_on) { + try { // watch for zmq_error from ppoll caused by SIGTERM from master + // poll: wait until status change (-1: infinite timeout) + auto poll_result = poller.ppoll(-1, &JobManager::instance()->messenger().ppoll_sigmask); +// JobManager::instance()->messenger().N_available_polled_results = poll_result.size(); + // then process incoming messages from sockets + for (auto readable_socket : poll_result) { + // message comes from the master/queue socket (first element): + if (readable_socket.first == mq_index) { + auto message = JobManager::instance()->messenger().receive_from_master_on_queue(); + carry_on = process_master_message(message); + // on terminate, also stop for-loop, no need to check other + // sockets anymore: + if (!carry_on) { + break; + } + } else { // from a worker socket + // TODO: dangerous assumption for this_worker_id, may become invalid if we allow multiple queue_loops on the same process! + auto this_worker_id = readable_socket.first - 1; // TODO: replace with a more reliable lookup + auto message = JobManager::instance()->messenger().receive_from_worker_on_queue(this_worker_id); + process_worker_message(this_worker_id, message); + } + } + } catch (ZMQ::ppoll_error_t& e) { + zmq_ppoll_error_response response; + try { + response = handle_zmq_ppoll_error(e); + } catch (std::logic_error& e) { + printf("queue loop got unhandleable ZMQ::ppoll_error_t\n"); + throw; + } + if (response == zmq_ppoll_error_response::abort) { + break; + } else if (response == zmq_ppoll_error_response::unknown_eintr) { + printf("EINTR in queue loop but no SIGTERM received, continuing\n"); + continue; + } else if (response == zmq_ppoll_error_response::retry) { + printf("EAGAIN from ppoll in queue loop, continuing\n"); + continue; + } + } catch (zmq::error_t& e) { + printf("unhandled zmq::error_t (not a ppoll_error_t) in queue loop with errno %d: %s\n", e.num(), e.what()); + throw; + } + } + + // clean up signal management modifications + sigprocmask(SIG_SETMASK, &JobManager::instance()->messenger().ppoll_sigmask, nullptr); +} + +} // namespace MultiProcess +} // namespace RooFit diff --git a/roofit/multiprocess/src/util.cxx b/roofit/multiprocess/src/util.cxx new file mode 100644 index 0000000000000..ff9aa14120eac --- /dev/null +++ b/roofit/multiprocess/src/util.cxx @@ -0,0 +1,130 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2016-2019, Netherlands eScience Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include // kill, SIGKILL +#include // cerr, and indirectly WNOHANG, EINTR, W* macros +#include // runtime_error +#include // waitpid +#include + +#include +#include + +namespace RooFit { +namespace MultiProcess { + +int wait_for_child(pid_t child_pid, bool may_throw, int retries_before_killing) +{ + int status = 0; + int patience = retries_before_killing; + pid_t tmp; + do { + if (patience-- < 1) { + ::kill(child_pid, SIGKILL); + } + tmp = waitpid(child_pid, &status, WNOHANG); + } while (tmp == 0 // child has not yet changed state, try again + || (-1 == tmp && EINTR == errno) // retry on interrupted system call + ); + + if (patience < 1) { + std::cout << "Had to send PID " << child_pid << " " << (-patience + 1) << " SIGKILLs"; + } + + if (0 != status) { + if (WIFEXITED(status)) { + printf("exited, status=%d\n", WEXITSTATUS(status)); + } else if (WIFSIGNALED(status)) { + printf("killed by signal %d\n", WTERMSIG(status)); + } else if (WIFSTOPPED(status)) { + printf("stopped by signal %d\n", WSTOPSIG(status)); + } else if (WIFCONTINUED(status)) { + printf("continued\n"); + } + } + + if (-1 == tmp && may_throw) + throw std::runtime_error(std::string("waitpid, errno ") + std::to_string(errno)); + + return status; +} + +zmq_ppoll_error_response handle_zmq_ppoll_error(ZMQ::ppoll_error_t &e) +{ + if ((e.num() == EINTR) && (ProcessManager::sigterm_received())) { + // valid EINTR, because we want to exit and kill the processes on SIGTERM + return zmq_ppoll_error_response::abort; + } else if (e.num() == EINTR) { + // on other EINTRs, we retry (mostly this happens in debuggers) + // printf("got EINTR but no SIGTERM received"); + return zmq_ppoll_error_response::unknown_eintr; + } else if (e.num() == EAGAIN) { + // This can happen from recv if ppoll initially gets a read-ready signal for a socket, + // but the received data does not pass the checksum test, so the socket becomes unreadable + // again or from non-blocking send if the socket becomes unwritable either due to the HWM + // being reached or the socket not being connected (anymore). The latter case usually means + // the connection has been severed from the other side, meaning it has probably been killed + // and in that case the next ppoll call will probably also receive a SIGTERM, ending the + // loop. In case something else is wrong, this message will print multiple times, which + // should be taken as a cue for writing a bug report :) + // TODO: handle this more rigorously + // printf("EAGAIN (from either send or receive), try %d\n", tries); + return zmq_ppoll_error_response::retry; + } else { + char buffer[512]; + snprintf(buffer, 512, "handle_zmq_ppoll_error is out of options to handle exception, caught ZMQ::ppoll_error_t had errno %d and text: %s\n", + e.num(), e.what()); + throw std::logic_error(buffer); + } +} + +// returns a tuple containing first the poll result and second a boolean flag that tells the caller whether it should +// abort the enclosing loop +std::tuple>, bool> +careful_ppoll(ZeroMQPoller &poller, const sigset_t &ppoll_sigmask, std::size_t max_tries) +{ + std::size_t tries = 0; + std::vector> poll_result; + bool abort = true; + bool carry_on = true; + while (carry_on && (tries++ < max_tries)) { + if (tries > 1) { + printf("careful_ppoll try %lu\n", tries); + } + try { // watch for zmq_error from ppoll caused by SIGTERM from master + poll_result = poller.ppoll(-1, &ppoll_sigmask); + abort = false; + carry_on = false; + } catch (ZMQ::ppoll_error_t &e) { + auto response = handle_zmq_ppoll_error(e); + if (response == zmq_ppoll_error_response::abort) { + break; + } else if (response == zmq_ppoll_error_response::unknown_eintr) { + printf("EINTR in careful_ppoll but no SIGTERM received, try %lu\n", tries); + continue; + } else if (response == zmq_ppoll_error_response::retry) { + printf("EAGAIN in careful_ppoll (from either send or receive), try %lu\n", tries); + continue; + } + } + } + + if (tries == max_tries) { + printf("careful_ppoll reached maximum number of tries, %lu, please report as a bug\n", tries); + } + return {poll_result, abort}; +} + +} // namespace MultiProcess +} // namespace RooFit \ No newline at end of file diff --git a/roofit/multiprocess/src/worker.cxx b/roofit/multiprocess/src/worker.cxx new file mode 100644 index 0000000000000..86191dfd694a4 --- /dev/null +++ b/roofit/multiprocess/src/worker.cxx @@ -0,0 +1,228 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl + * + * Copyright (c) 2016-2019, Netherlands eScience Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include // getpid, pid_t +#include // EINTR +#include +#include // sigprocmask etc + +#include "RooFit/MultiProcess/JobManager.h" +#include "RooFit/MultiProcess/types.h" +#include "RooFit/MultiProcess/Messenger.h" +#include "RooFit/MultiProcess/Job.h" +#include "RooFit/MultiProcess/util.h" + +#include "RooFit/MultiProcess/worker.h" + +namespace RooFit { +namespace MultiProcess { + +static bool worker_loop_running = false; + +bool is_worker_loop_running() +{ + return worker_loop_running; +} + +bool process_queue_message(Q2W message_q2w, bool &dequeue_acknowledged) +{ + bool carry_on = true; + + auto get_time = []() { + return std::chrono::duration_cast( + std::chrono::high_resolution_clock::now().time_since_epoch()) + .count(); + }; + decltype(get_time()) t1 = 0, t2 = 0; + + + switch (message_q2w) { + case Q2W::terminate: { + carry_on = false; + break; + } + + case Q2W::dequeue_rejected: { + t2 = get_time(); +// std::cout +// << "no work: worker " << JobManager::instance()->process_manager().worker_id() << " asked at " << t1 +// << " and got rejected at " << t2 << std::endl; + + dequeue_acknowledged = true; + break; + } + case Q2W::dequeue_accepted: { + dequeue_acknowledged = true; + auto job_id = JobManager::instance()->messenger().receive_from_queue_on_worker(); + auto task = JobManager::instance()->messenger().receive_from_queue_on_worker(); + + JobManager::get_job_object(job_id)->evaluate_task(task); + + // std::cout +// << "task done: worker " << JobManager::instance()->process_manager().worker_id() << " asked at " << t1 << ", started at " +// << t2 << " and finished at " << t3 << std::endl; + +// { +// // old +// JobManager::instance()->messenger().send_from_worker_to_queue(W2Q::send_result, job_id, task); +// JobManager::get_job_object(job_id)->send_back_task_result_from_worker(task); +// +// message_q2w = JobManager::instance()->messenger().receive_from_queue_on_worker(); +// if (message_q2w != Q2W::result_received) { +// std::cerr << "worker " << getpid() +// << " sent result, but did not receive Q2W::result_received handshake! Got " << message_q2w +// << " instead." << std::endl; +// throw std::runtime_error(""); +// } + +// } + { + // new +// JobManager::instance()->messenger().send_from_worker_to_master(job_id, task); + JobManager::get_job_object(job_id)->send_back_task_result_from_worker(task); + } + + // std::cout << "task done: worker " << JobManager::instance()->process_manager().worker_id() << " sent back results" << std::endl; + +// std::cout << "task done: worker " << JobManager::instance()->process_manager().worker_id() << " finished dequeue_accepted" << std::endl; + break; + } + + // previously this was non-work mode + + case Q2W::update_real: { + t1 = get_time(); + + auto job_id = JobManager::instance()->messenger().receive_from_queue_on_worker(); + auto ix = JobManager::instance()->messenger().receive_from_queue_on_worker(); + auto val = JobManager::instance()->messenger().receive_from_queue_on_worker(); + bool is_constant = JobManager::instance()->messenger().receive_from_queue_on_worker(); + JobManager::get_job_object(job_id)->update_real(ix, val, is_constant); + + t2 = get_time(); +// std::cout +// << "update_real on worker " << JobManager::instance()->process_manager().worker_id() << ": " << (t2 - t1) / 1.e9 +// << "s (from " << t1 << " to " << t2 << "ns)" << std::endl; + + break; + } + + case Q2W::update_bool: { + auto job_id = JobManager::instance()->messenger().receive_from_queue_on_worker(); + auto ix = JobManager::instance()->messenger().receive_from_queue_on_worker(); + auto value = JobManager::instance()->messenger().receive_from_queue_on_worker(); + JobManager::get_job_object(job_id)->update_bool(ix, value); + + break; + } + + case Q2W::result_received: { + std::cerr << "In worker_loop: " << message_q2w + << " message received, but should only be received as handshake!" << std::endl; + break; + } + } + + return carry_on; +} + +void worker_loop() +{ + assert(JobManager::instance()->process_manager().is_worker()); + worker_loop_running = true; + bool carry_on = true; +// Task task; + std::size_t job_id; + Q2W message_q2w; + + // use a flag to not ask twice + bool dequeue_acknowledged = true; + + ZeroMQPoller poller; + std::size_t mw_sub_index; + + std::tie(poller, mw_sub_index) = JobManager::instance()->messenger().create_worker_poller(); + + // Before blocking SIGTERM, set the signal handler, so we can also check after blocking whether a signal occurred + // In our case, we already set it in the ProcessManager after forking to the queue and worker processes. + + sigset_t sigmask; + sigemptyset(&sigmask); + sigaddset(&sigmask, SIGTERM); + sigprocmask(SIG_BLOCK, &sigmask, &JobManager::instance()->messenger().ppoll_sigmask); + + // Before doing anything, check whether we have received a terminate signal while blocking signals! + // In this case, we also do that in the while condition. + while (!ProcessManager::sigterm_received() && carry_on) { + try { // watch for error from ppoll (which is called inside receive functions) caused by SIGTERM from master + + // try to dequeue a task + if (dequeue_acknowledged) { // don't ask twice + JobManager::instance()->messenger().send_from_worker_to_queue(W2Q::dequeue); + dequeue_acknowledged = false; + } + +// // receive handshake +// message_q2w = JobManager::instance()->messenger().receive_from_queue_on_worker(); + // receive handshake, other message or update from sub socket + auto poll_result = poller.ppoll(-1, &JobManager::instance()->messenger().ppoll_sigmask); +// JobManager::instance()->messenger().N_available_polled_results = poll_result.size(); + // then process incoming messages from sockets + for (auto readable_socket : poll_result) { + // message comes from the master/queue socket (first element): + if (readable_socket.first == mw_sub_index) { + job_id = JobManager::instance()->messenger().receive_from_master_on_worker(); + JobManager::get_job_object(job_id)->update_state(); + } else { // from queue socket + message_q2w = JobManager::instance()->messenger().receive_from_queue_on_worker(); + carry_on = process_queue_message(message_q2w, dequeue_acknowledged); + // on terminate, also stop for-loop, no need to check other + // sockets anymore: + if (!carry_on) { + break; + } + } + } + + } catch (ZMQ::ppoll_error_t& e) { + zmq_ppoll_error_response response; + try { + response = handle_zmq_ppoll_error(e); + } catch (std::logic_error& e) { + printf("worker loop at PID %d got unhandleable ZMQ::ppoll_error_t\n", getpid()); + throw; + } + if (response == zmq_ppoll_error_response::abort) { + break; + } else if (response == zmq_ppoll_error_response::unknown_eintr) { + printf("EINTR in worker loop at PID %d but no SIGTERM received, continuing\n", getpid()); + continue; + } else if (response == zmq_ppoll_error_response::retry) { + printf("EAGAIN from ppoll in worker loop at PID %d, continuing\n", getpid()); + continue; + } + } catch (zmq::error_t& e) { + printf("unhandled zmq::error_t (not a ppoll_error_t) in worker loop at PID %d with errno %d: %s\n", getpid(), e.num(), e.what()); + throw; + } + } + // clean up signal management modifications + sigprocmask(SIG_SETMASK, &JobManager::instance()->messenger().ppoll_sigmask, nullptr); + + worker_loop_running = false; +} + +} // namespace MultiProcess +} // namespace RooFit diff --git a/roofit/multiprocess/test/CMakeLists.txt b/roofit/multiprocess/test/CMakeLists.txt new file mode 100644 index 0000000000000..7cc22c78e4c7d --- /dev/null +++ b/roofit/multiprocess/test/CMakeLists.txt @@ -0,0 +1,13 @@ +# @author Patrick Bos, NL eScience Center, 2019 + +#find_package(ZeroMQ REQUIRED) + +add_library(RooFit_multiprocess_testing_utils INTERFACE) +target_link_libraries(RooFit_multiprocess_testing_utils INTERFACE RooFitCore RooBatchCompute) + +ROOT_ADD_GTEST(test_RooFitMultiProcess_Job test_Job.cxx LIBRARIES RooFitMultiProcess ${ZeroMQ_LIBRARY} RooFit_multiprocess_testing_utils) +ROOT_ADD_GTEST(test_RooFitMultiProcess_ProcessManager test_ProcessManager.cxx LIBRARIES RooFitMultiProcess ${ZeroMQ_LIBRARY}) +ROOT_ADD_GTEST(test_RooFitMultiProcess_Messenger test_Messenger.cxx LIBRARIES RooFitMultiProcess ${ZeroMQ_LIBRARY}) + +#target_include_directories(test_RooFitMultiProcess_Job PUBLIC ${ZeroMQ_INCLUDE_DIR}) +#target_include_directories(test_RooFitMultiProcess_Messenger PUBLIC ${ZeroMQ_INCLUDE_DIR}) diff --git a/roofit/multiprocess/test/test_Job.cxx b/roofit/multiprocess/test/test_Job.cxx new file mode 100644 index 0000000000000..6b71f7e3369fa --- /dev/null +++ b/roofit/multiprocess/test/test_Job.cxx @@ -0,0 +1,252 @@ +/***************************************************************************** + * Project: RooFit * + * Package: RooFitCore * + * @(#)root/roofitcore:$Id$ + * Authors: * + * PB, Patrick Bos, NL eScience Center, p.bos@esciencecenter.nl * + * IP, Inti Pelupessy, NL eScience Center, i.pelupessy@esciencecenter.nl * + * VC, Vince Croft, DIANA / NYU, vincent.croft@cern.ch * + * * + * Redistribution and use in source and binary forms, * + * with or without modification, are permitted according to the terms * + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) * + *****************************************************************************/ + +#include +#include + +#include +#include // JobTask +// needed to complete type returned from... +#include // ... Job::get_manager() +#include // ... JobManager::process_manager() +#include // ... JobManager::queue() + +#include "gtest/gtest.h" +#include "utils.h" + +class xSquaredPlusBVectorSerial { +public: + xSquaredPlusBVectorSerial(double b, std::vector x_init) : _b(b), x(std::move(x_init)), result(x.size()) {} + + void evaluate() + { + // call evaluate_task for each task + for (std::size_t ix = 0; ix < x.size(); ++ix) { + result[ix] = std::pow(x[ix], 2) + _b; + } + } + + std::vector get_result() + { + evaluate(); + return result; + } + + // for simplicity of the example (avoiding getters/setters) we make data members public as well + double _b; + std::vector x; + std::vector result; +}; + +class xSquaredPlusBVectorParallel : public RooFit::MultiProcess::Job { +public: + xSquaredPlusBVectorParallel(xSquaredPlusBVectorSerial* serial) + : serial(serial) + { + } + + void evaluate() + { + if (get_manager()->process_manager().is_master()) { + // master fills queue with tasks + for (std::size_t task_id = 0; task_id < serial->x.size(); ++task_id) { + RooFit::MultiProcess::JobTask job_task(id, task_id); + get_manager()->queue().add(job_task); + } + + // wait for task results back from workers to master + gather_worker_results(); + } + } + + std::vector get_result() + { + evaluate(); + return serial->result; + } + + void update_real(std::size_t /*ix*/, double val, bool /*is_constant*/) override { + if (get_manager()->process_manager().is_worker()) { + serial->_b = val; + } + } + + void update_bool(std::size_t /*ix*/, bool /*value*/) override { + // pass + } + + // -- BEGIN plumbing -- + + void receive_task_result_on_queue(std::size_t task, std::size_t worker_id) override { + double result = get_manager()->messenger().template receive_from_worker_on_queue(worker_id); + serial->result[task] = result; + } + + void send_back_task_result_from_worker(std::size_t task) override { + get_manager()->messenger().template send_from_worker_to_queue(serial->result[task]); + } + + void send_back_results_from_queue_to_master() override { +// get_manager()->messenger().send_from_queue_to_master(serial->result.size()); + for (std::size_t task_ix = 0ul; task_ix < serial->result.size(); ++task_ix) { + get_manager()->messenger().send_from_queue_to_master(task_ix, serial->result[task_ix]); + } + } + + void clear_results() override { + // no need to clear any results cache since we just reuse the result vector on the queue + } + + void receive_results_on_master() override { +// std::size_t N_job_tasks = get_manager()->messenger().template receive_from_queue_on_master(); +// for (std::size_t task_ix = 0ul; task_ix < N_job_tasks; ++task_ix) { + for (std::size_t task_ix = 0ul; task_ix < serial->result.size(); ++task_ix) { + std::size_t task_id = get_manager()->messenger().template receive_from_queue_on_master(); + serial->result[task_id] = get_manager()->messenger().template receive_from_queue_on_master(); + } + } + + bool receive_task_result_on_master(const zmq::message_t & /*message*/) override { + // TODO: implement; this no-op placeholder is just to make everything compile first so I can check whether merge was successful + return true; + } + + // -- END plumbing -- + + +private: + void evaluate_task(std::size_t task) override + { + assert(get_manager()->process_manager().is_worker()); + serial->result[task] = std::pow(serial->x[task], 2) + serial->_b; + } + + xSquaredPlusBVectorSerial* serial; +}; + +class TestMPJob : public ::testing::TestWithParam { + // You can implement all the usual fixture class members here. + // To access the test parameter, call GetParam() from class + // TestWithParam. +}; + +TEST_P(TestMPJob, singleJobGetResult) +{ + // Simple test case: calculate x^2 + b, where x is a vector. This case does + // both a simple calculation (squaring the input vector x) and represents + // handling of state updates in b. + std::vector x{0, 1, 2, 3}; + double b_initial = 3.; + + // start serial test + + xSquaredPlusBVectorSerial x_sq_plus_b(b_initial, x); + + auto y = x_sq_plus_b.get_result(); + std::vector y_expected{3, 4, 7, 12}; + + EXPECT_EQ(Hex(y[0]), Hex(y_expected[0])); + EXPECT_EQ(Hex(y[1]), Hex(y_expected[1])); + EXPECT_EQ(Hex(y[2]), Hex(y_expected[2])); + EXPECT_EQ(Hex(y[3]), Hex(y_expected[3])); + + std::size_t NumCPU = GetParam(); + + // start parallel test + + xSquaredPlusBVectorParallel x_sq_plus_b_parallel(&x_sq_plus_b); + RooFit::MultiProcess::JobManager::default_N_workers = NumCPU; + + auto y_parallel = x_sq_plus_b_parallel.get_result(); + + EXPECT_EQ(Hex(y_parallel[0]), Hex(y_expected[0])); + EXPECT_EQ(Hex(y_parallel[1]), Hex(y_expected[1])); + EXPECT_EQ(Hex(y_parallel[2]), Hex(y_expected[2])); + EXPECT_EQ(Hex(y_parallel[3]), Hex(y_expected[3])); +} + + +TEST_P(TestMPJob, multiJobGetResult) +{ + // Simple test case: calculate x^2 + b, where x is a vector. This case does + // both a simple calculation (squaring the input vector x) and represents + // handling of state updates in b. + std::vector x{0, 1, 2, 3}; + double b_initial = 3.; + + std::vector y_expected{3, 4, 7, 12}; + + std::size_t NumCPU = GetParam(); + + // define jobs + xSquaredPlusBVectorSerial x_sq_plus_b(b_initial, x); + xSquaredPlusBVectorSerial x_sq_plus_b2(b_initial + 1, x); + xSquaredPlusBVectorParallel x_sq_plus_b_parallel(&x_sq_plus_b); + xSquaredPlusBVectorParallel x_sq_plus_b_parallel2(&x_sq_plus_b2); + RooFit::MultiProcess::JobManager::default_N_workers = NumCPU; + + // do stuff + auto y_parallel = x_sq_plus_b_parallel.get_result(); + auto y_parallel2 = x_sq_plus_b_parallel2.get_result(); + + EXPECT_EQ(Hex(y_parallel[0]), Hex(y_expected[0])); + EXPECT_EQ(Hex(y_parallel[1]), Hex(y_expected[1])); + EXPECT_EQ(Hex(y_parallel[2]), Hex(y_expected[2])); + EXPECT_EQ(Hex(y_parallel[3]), Hex(y_expected[3])); + + EXPECT_EQ(Hex(y_parallel2[0]), Hex(y_expected[0] + 1)); + EXPECT_EQ(Hex(y_parallel2[1]), Hex(y_expected[1] + 1)); + EXPECT_EQ(Hex(y_parallel2[2]), Hex(y_expected[2] + 1)); + EXPECT_EQ(Hex(y_parallel2[3]), Hex(y_expected[3] + 1)); +} + + +// TODO: implement update_real test! + +//TEST_P(TestMPJob, singleJobUpdateReal) +//{ +//// Simple test case: calculate x^2 + b, where x is a vector. This case does +//// both a simple calculation (squaring the input vector x) and represents +//// handling of state updates in b. +//std::vector x{0, 1, 2, 3}; +//double b_initial = 3.; +// +//// start serial test +// +//xSquaredPlusBVectorSerial x_sq_plus_b(b_initial, x); +// +//auto y = x_sq_plus_b.get_result(); +//std::vector y_expected{3, 4, 7, 12}; +// +//EXPECT_EQ(Hex(y[0]), Hex(y_expected[0])); +//EXPECT_EQ(Hex(y[1]), Hex(y_expected[1])); +//EXPECT_EQ(Hex(y[2]), Hex(y_expected[2])); +//EXPECT_EQ(Hex(y[3]), Hex(y_expected[3])); +// +//std::size_t NumCPU = GetParam(); +// +//// start parallel test +// +//xSquaredPlusBVectorParallel x_sq_plus_b_parallel(NumCPU, &x_sq_plus_b); +// +//auto y_parallel = x_sq_plus_b_parallel.get_result(); +// +//EXPECT_EQ(Hex(y_parallel[0]), Hex(y_expected[0])); +//EXPECT_EQ(Hex(y_parallel[1]), Hex(y_expected[1])); +//EXPECT_EQ(Hex(y_parallel[2]), Hex(y_expected[2])); +//EXPECT_EQ(Hex(y_parallel[3]), Hex(y_expected[3])); +//} + + +INSTANTIATE_TEST_SUITE_P(NumberOfWorkerProcesses, TestMPJob, ::testing::Values(1, 2, 3)); diff --git a/roofit/multiprocess/test/test_Messenger.cxx b/roofit/multiprocess/test/test_Messenger.cxx new file mode 100644 index 0000000000000..49b9936f4e30f --- /dev/null +++ b/roofit/multiprocess/test/test_Messenger.cxx @@ -0,0 +1,116 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2016-2019, Netherlands eScience Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "RooFit/MultiProcess/Messenger.h" +#include // ... JobManager::process_manager() + +#include "gtest/gtest.h" + +TEST(TestMPMessenger, Connections) +{ + RooFit::MultiProcess::ProcessManager pm(2); + RooFit::MultiProcess::Messenger messenger(pm); + messenger.test_connections(pm); +} + +TEST(TestMPMessenger, ConnectionsManualExit) +{ + // the point of this test is to see whether clean-up of ZeroMQ resources is done properly without calling any + // destructors (which is what happens when you call _Exit() instead of regularly ending the program by reaching the + // end of main()). + RooFit::MultiProcess::ProcessManager pm( 2); + RooFit::MultiProcess::Messenger messenger(pm); + messenger.test_connections(pm); + if (!pm.is_master()) { + std::_Exit(0); + } +} + + +#include + +TEST(TestMPMessenger, SigStop) +{ + // For some reason, I cannot get my debugger (lldb) to play nicely with ZeroMQ; every time I pause a process, it + // says it crashed because of SIGSTOP, but SIGSTOP should just pause it... let's see whether we can reproduce this + // behavior here and, if possible, fix it. + RooFit::MultiProcess::ProcessManager pm(2); + RooFit::MultiProcess::Messenger messenger(pm); +// messenger.test_connections(pm); + + // all this works fine without a sigmask, maybe the sigmask is the issue? let's try: + + + if (pm.is_master()) { + printf("first send message to queue...\n"); + messenger.send_from_master_to_queue(1); + printf("sleep for 2 seconds...\n"); + sleep(2); + printf("SIGSTOPping all children...\n"); + kill(pm.get_queue_pid(), SIGSTOP); + kill(pm.get_worker_pids()[0], SIGSTOP); + kill(pm.get_worker_pids()[1], SIGSTOP); + printf("send another message to queue...\n"); + messenger.send_from_master_to_queue(2); + printf("sleep for 2 seconds...\n"); + sleep(2); + printf("SIGCONT queue and worker 1...\n"); + kill(pm.get_queue_pid(), SIGCONT); + kill(pm.get_worker_pids()[0], SIGCONT); + printf("sleep for 2 seconds...\n"); + sleep(2); + printf("SIGCONT worker 2...\n"); + kill(pm.get_worker_pids()[1], SIGCONT); + EXPECT_EQ(messenger.receive_from_queue_on_master(), 3); + } + + if (pm.is_queue()) { + auto number = messenger.receive_from_master_on_queue(); + printf("received %d on queue\n", number); + number = messenger.receive_from_master_on_queue(); + printf("received %d on queue\n", number); + messenger.send_from_queue_to_master(3); + } + + if (!pm.is_master()) { + while (!pm.sigterm_received()) {} + } +} + +TEST(TestMPMessenger, StressPubSub) +{ + std::size_t N_workers = 64; + RooFit::MultiProcess::ProcessManager pm(N_workers); + RooFit::MultiProcess::Messenger messenger(pm); + + if (pm.is_master()) { + messenger.publish_from_master_to_workers("bert"); + std::size_t ernies = 0; + for (; ernies < N_workers; ++ernies) { + auto receipt = messenger.receive_from_worker_on_master(); + if (receipt != "ernie") { + printf("whoops, got %s instead of ernie!\n", receipt.c_str()); + FAIL(); + } + } + EXPECT_EQ(ernies, N_workers); + } else if (pm.is_worker()) { + auto receipt = messenger.receive_from_master_on_worker(); + if (receipt == "bert") { + messenger.send_from_worker_to_master("ernie"); + } else { + printf("no bert on worker %lu\n", pm.worker_id()); + } + } +} diff --git a/roofit/multiprocess/test/test_ProcessManager.cxx b/roofit/multiprocess/test/test_ProcessManager.cxx new file mode 100644 index 0000000000000..4660657e4c720 --- /dev/null +++ b/roofit/multiprocess/test/test_ProcessManager.cxx @@ -0,0 +1,57 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2016-2019, Netherlands eScience Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include // ... JobManager::process_manager() + +#include "gtest/gtest.h" + +TEST(TestMPProcessManager, birthAndDeath) +{ + auto pm = RooFit::MultiProcess::ProcessManager(2); +} + +// disabled: it is not working anymore (16 March 2021), due to changes introduced to fix Messenger::test_connections +TEST(TestMPProcessManager, DISABLED_multiBirth) +{ + // This test doesn't actually represent a current usecase in RooFit, but let's + // showcase the possibility anyway for future reference. + + // first create a regular pm like in the birthAndDeath test + auto pm = RooFit::MultiProcess::ProcessManager(2); + // then from each forked node, spin up another set of workers+queue! + auto pm2 = RooFit::MultiProcess::ProcessManager(2); +} + + +TEST(TestMPProcessManager, checkState) +{ + std::size_t N_workers = 2; + auto master_pid = getpid(); + auto pm = RooFit::MultiProcess::ProcessManager(N_workers); + ASSERT_TRUE(pm.is_initialized()); + ASSERT_EQ(pm.N_workers(), N_workers); + + if (pm.is_master()) { + EXPECT_EQ(getpid(), master_pid); + EXPECT_FALSE(pm.is_queue()); + EXPECT_FALSE(pm.is_worker()); + } else if (pm.is_queue()) { + EXPECT_FALSE(pm.is_master()); + EXPECT_FALSE(pm.is_worker()); + } else if (pm.is_worker()) { + EXPECT_FALSE(pm.is_master()); + EXPECT_FALSE(pm.is_queue()); + EXPECT_TRUE(pm.worker_id() == 0 || pm.worker_id() == 1); + } +} \ No newline at end of file diff --git a/roofit/multiprocess/test/test_worker.cxx b/roofit/multiprocess/test/test_worker.cxx new file mode 100644 index 0000000000000..949527a89bb87 --- /dev/null +++ b/roofit/multiprocess/test/test_worker.cxx @@ -0,0 +1,29 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2016-2019, Netherlands eScience Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "RooFit/MultiProcess/worker.h" + +#include "gtest/gtest.h" + +// TODO: implement something along these lines (the following is pseudocode) +//TEST(worker, loop) +//{ +// do_fork(); +// if (on_master()) { +// wait(); +// terminate_worker(); +// } else if (on_worker()) { +// worker_loop(); +// } +//} diff --git a/roofit/multiprocess/test/utils.h b/roofit/multiprocess/test/utils.h new file mode 100644 index 0000000000000..f4f72976e7226 --- /dev/null +++ b/roofit/multiprocess/test/utils.h @@ -0,0 +1,171 @@ +/***************************************************************************** + * Project: RooFit * + * Package: RooFitCore * + * @(#)root/roofitcore:$Id$ + * Authors: * + * PB, Patrick Bos, NL eScience Center, p.bos@esciencecenter.nl * + * * + * Redistribution and use in source and binary forms, * + * with or without modification, are permitted according to the terms * + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) * + *****************************************************************************/ + +#ifndef ROOT_ROOFIT_MultiProcess_tests_utils_h +#define ROOT_ROOFIT_MultiProcess_tests_utils_h + +#include + +#include // make_unique + +#include "RooWorkspace.h" +#include "RooRandom.h" +#include "RooAddPdf.h" +#include "RooDataSet.h" +#include "RooRealVar.h" // for the dynamic cast to have a complete type + + +std::tuple, std::unique_ptr> +generate_1D_gaussian_pdf_nll(RooWorkspace &w, unsigned long N_events) { + w.factory("Gaussian::g(x[-5,5],mu[0,-3,3],sigma[1])"); + + RooAbsPdf *pdf = w.pdf("g"); + RooRealVar *mu = w.var("mu"); + + RooDataSet *data = pdf->generate(RooArgSet(*w.var("x")), N_events); + mu->setVal(-2.9); + + std::unique_ptr nll {pdf->createNLL(*data)}; + + // save initial values for the start of all minimizations + std::unique_ptr values = std::make_unique(*mu, *pdf, *nll, "values"); + + return std::make_tuple(std::move(nll), std::move(values)); +} + +// return two unique_ptrs, the first because nll is a pointer, +// the second because RooArgSet doesn't have a move ctor +std::tuple, std::unique_ptr> +generate_ND_gaussian_pdf_nll(RooWorkspace &w, unsigned int n, unsigned long N_events) { + RooArgSet obs_set; + + // create gaussian parameters + double mean[n], sigma[n]; + for (unsigned ix = 0; ix < n; ++ix) { + mean[ix] = RooRandom::randomGenerator()->Gaus(0, 2); + sigma[ix] = 0.1 + abs(RooRandom::randomGenerator()->Gaus(0, 2)); + } + + // create gaussians and also the observables and parameters they depend on + for (unsigned ix = 0; ix < n; ++ix) { + std::ostringstream os; + os << "Gaussian::g" << ix + << "(x" << ix << "[-10,10]," + << "m" << ix << "[" << mean[ix] << ",-10,10]," + << "s" << ix << "[" << sigma[ix] << ",0.1,10])"; + w.factory(os.str().c_str()); + } + + // create uniform background signals on each observable + for (unsigned ix = 0; ix < n; ++ix) { + { + std::ostringstream os; + os << "Uniform::u" << ix << "(x" << ix << ")"; + w.factory(os.str().c_str()); + } + + // gather the observables in a list for data generation below + { + std::ostringstream os; + os << "x" << ix; + obs_set.add(*w.arg(os.str().c_str())); + } + } + + RooArgSet pdf_set = w.allPdfs(); + + // create event counts for all pdfs + RooArgSet count_set; + + // ... for the gaussians + for (unsigned ix = 0; ix < n; ++ix) { + std::stringstream os, os2; + os << "Nsig" << ix; + os2 << "#signal events comp " << ix; + RooRealVar a(os.str().c_str(), os2.str().c_str(), N_events/10, 0., 10*N_events); + w.import(a); + // gather in count_set + count_set.add(*w.arg(os.str().c_str())); + } + // ... and for the uniform background components + for (unsigned ix = 0; ix < n; ++ix) { + std::stringstream os, os2; + os << "Nbkg" << ix; + os2 << "#background events comp " << ix; + RooRealVar a(os.str().c_str(), os2.str().c_str(), N_events/10, 0., 10*N_events); + w.import(a); + // gather in count_set + count_set.add(*w.arg(os.str().c_str())); + } + + RooAddPdf* sum = new RooAddPdf("sum", "gaussians+uniforms", pdf_set, count_set); + w.import(*sum); // keep sum around after returning + + // --- Generate a toyMC sample from composite PDF --- + RooDataSet *data = sum->generate(obs_set, N_events); + + std::unique_ptr nll {sum->createNLL(*data)}; + + // set values randomly so that they actually need to do some fitting + for (unsigned ix = 0; ix < n; ++ix) { + { + std::ostringstream os; + os << "m" << ix; + dynamic_cast(w.arg(os.str().c_str()))->setVal(RooRandom::randomGenerator()->Gaus(0, 2)); + } + { + std::ostringstream os; + os << "s" << ix; + dynamic_cast(w.arg(os.str().c_str()))->setVal(0.1 + abs(RooRandom::randomGenerator()->Gaus(0, 2))); + } + } + + // gather all values of parameters, pdfs and nll here for easy + // saving and restoring + std::unique_ptr all_values = std::make_unique(pdf_set, count_set, "all_values"); + all_values->add(*nll); + all_values->add(*sum); + for (unsigned ix = 0; ix < n; ++ix) { + { + std::ostringstream os; + os << "m" << ix; + all_values->add(*w.arg(os.str().c_str())); + } + { + std::ostringstream os; + os << "s" << ix; + all_values->add(*w.arg(os.str().c_str())); + } + } + + return std::make_tuple(std::move(nll), std::move(all_values)); +} + + +class Hex { +public: + explicit Hex(double n) : number_(n) {} + operator double() const { return number_; } + bool operator==(const Hex& other) { + return double(*this) == double(other); + } + +private: + double number_; +}; + +::std::ostream& operator<<(::std::ostream& os, const Hex& hex) { + return os << std::hexfloat << double(hex) << std::defaultfloat; // whatever needed to print bar to os +} + + +#endif //ROOT_ROOFIT_MultiProcess_tests_utils_h diff --git a/roofit/roofitZMQ/CMakeLists.txt b/roofit/roofitZMQ/CMakeLists.txt new file mode 100644 index 0000000000000..81fee8d871a92 --- /dev/null +++ b/roofit/roofitZMQ/CMakeLists.txt @@ -0,0 +1,38 @@ +############################################################################ +# CMakeLists.txt file for building ROOT roofitcore/ZMQ package +# @author Patrick Bos, NL eScience Center +############################################################################ + +#if(NOT ZeroMQ_FOUND) +# find_package(ZeroMQ REQUIRED) +#endif() + +ROOT_STANDARD_LIBRARY_PACKAGE(RooFitZMQ + HEADERS + RooFit_ZMQ/Utility.h + RooFit_ZMQ/ZeroMQSvc.h + RooFit_ZMQ/functions.h + RooFit_ZMQ/ZeroMQPoller.h + RooFit_ZMQ/ppoll.h + SOURCES + src/ZeroMQSvc.cpp + src/ZeroMQPoller.cpp + src/functions.cpp + src/ppoll.cpp + DICTIONARY_OPTIONS + -writeEmptyRootPCM + DEPENDENCIES + Core # TVersionCheck, RegisterModule, these are always called from G__RooFitZMQ.cxx.o + LIBRARIES + ZeroMQ + ) + + +target_link_libraries(RooFitZMQ PRIVATE ZeroMQ) +target_include_directories(RooFitZMQ PUBLIC $) +target_include_directories(RooFitZMQ PUBLIC $) +target_include_directories(RooFitZMQ PUBLIC $) + +if(testing) + ROOT_ADD_TEST_SUBDIRECTORY(test) +endif() diff --git a/roofit/roofitZMQ/inc/LinkDef.h b/roofit/roofitZMQ/inc/LinkDef.h new file mode 100644 index 0000000000000..5495493c90b99 --- /dev/null +++ b/roofit/roofitZMQ/inc/LinkDef.h @@ -0,0 +1 @@ +// something \ No newline at end of file diff --git a/roofit/roofitZMQ/inc/RooFit_ZMQ/Utility.h b/roofit/roofitZMQ/inc/RooFit_ZMQ/Utility.h new file mode 100644 index 0000000000000..a329805967df6 --- /dev/null +++ b/roofit/roofitZMQ/inc/RooFit_ZMQ/Utility.h @@ -0,0 +1,39 @@ +#ifndef SERIALIZE_UTILITY_H +#define SERIALIZE_UTILITY_H 1 + +#if defined(__clang__) && (__clang_major__ > 3 || (__clang_major__ == 3 && __clang_minor__ >= 10)) +#define HAVE_TRIVIALLY_COPYABLE 1 +#elif defined(__GNUC__) && __GNUC__ >= 5 +#define HAVE_TRIVIALLY_COPYABLE 1 +#else +#undef HAVE_TRIVIALLY_COPYABLE +#endif + +#include + +namespace ZMQ { + namespace Detail { + +#if defined(HAVE_TRIVIALLY_COPYABLE) + template + using simple_object = std::is_trivially_copyable; +#else + template + using simple_object = std::is_pod; +#endif + +// is trivial + template + struct is_trivial : std::conditional< + simple_object< + typename std::decay::type + >::value, + std::true_type, + std::false_type + >::type { + }; + + } +} + +#endif // SERIALIZE_UTILITY_H diff --git a/roofit/roofitZMQ/inc/RooFit_ZMQ/ZeroMQPoller.h b/roofit/roofitZMQ/inc/RooFit_ZMQ/ZeroMQPoller.h new file mode 100644 index 0000000000000..4d14444c01553 --- /dev/null +++ b/roofit/roofitZMQ/inc/RooFit_ZMQ/ZeroMQPoller.h @@ -0,0 +1,48 @@ +#ifndef ZEROMQPOLLER_H +#define ZEROMQPOLLER_H 1 +#include +#include +#include +#include + +#include "RooFit_ZMQ/ZeroMQSvc.h" +#include "RooFit_ZMQ/functions.h" + +class ZeroMQPoller { +public: + + using entry_t = std::tuple; + // The key is what zmq::socket_t stores inside, and what goes into + // pollitem_t through zmq::socket_t's conversion to void* operator + using sockets_t = std::unordered_map; + + using fd_entry_t = std::tuple; + using fds_t = std::unordered_map; + + using free_t = std::deque; + + ZeroMQPoller() = default; + + std::vector> poll(int timeo = -1); + std::vector> ppoll(int timeo, const sigset_t * sigmask_); + + size_t size() const; + + size_t register_socket(zmq::socket_t& socket, zmq::PollType type); + size_t register_socket(int fd, zmq::PollType type); + + size_t unregister_socket(zmq::socket_t& socket); + size_t unregister_socket(int fd); + +private: + + // Vector of (socket, flags) + std::vector m_items; + sockets_t m_sockets; + fds_t m_fds; + + // free slots in items + free_t m_free; +}; + +#endif // ZEROMQPOLLER_H diff --git a/roofit/roofitZMQ/inc/RooFit_ZMQ/ZeroMQSvc.h b/roofit/roofitZMQ/inc/RooFit_ZMQ/ZeroMQSvc.h new file mode 100644 index 0000000000000..0eace37b7f6c4 --- /dev/null +++ b/roofit/roofitZMQ/inc/RooFit_ZMQ/ZeroMQSvc.h @@ -0,0 +1,214 @@ +#ifndef ZEROMQ_IZEROMQSVC_H +#define ZEROMQ_IZEROMQSVC_H 1 + +// Include files +// from STL +#include +#include +#include +#include +#include +#include // std::cerr + +// ZeroMQ +#include +#include "RooFit_ZMQ/Utility.h" +#include "RooFit_ZMQ/functions.h" + +// debugging +#include // getpid + +namespace ZMQ { + +struct TimeOutException : std::exception { + TimeOutException() = default; + TimeOutException(const TimeOutException&) = default; + ~TimeOutException() = default; + TimeOutException& operator=(const TimeOutException&) = default; +}; + +struct MoreException : std::exception { + MoreException() = default; + MoreException(const MoreException&) = default; + ~MoreException() = default; + MoreException& operator=(const MoreException&) = default; +}; + +} + +template +struct ZmqLingeringSocketPtrDeleter { + void operator()(zmq::socket_t* socket) { + auto period = PERIOD; + + int tries = 0; + int max_tries = 3; + while(true) { + try { + // the actual work this function should do, plus the delete socket below: + if (socket) socket->setsockopt(ZMQ_LINGER, &period, sizeof(period)); + break; + } catch (zmq::error_t& e) { + if (++tries == max_tries + || e.num() == EINVAL || e.num() == ETERM || e.num() == ENOTSOCK // not recoverable from here + ) { + std::cerr << "ERROR in ZeroMQSvc::socket: " << e.what() << " (errno: " << e.num() << ")\n"; + throw; + } + std::cerr << "RETRY " << tries << "/" << (max_tries - 1) << " in ZmqLingeringSocketPtrDeleter: call interrupted (errno: " << e.num() << ")\n"; + } + } + + delete socket; + } +}; + +template +using ZmqLingeringSocketPtr = std::unique_ptr>; + + +// We retry send and receive only on EINTR, all other errors are either fatal, or can only +// be handled at the caller. +template +auto retry_send(zmq::socket_t& socket, int max_tries, args_t ...args) -> decltype(socket.send(args...)) { + int tries = 0; + while(true) { + try { + // the actual work this function should do, all the rest is error handling: + return socket.send(args...); + } catch (zmq::error_t& e) { + if (++tries == max_tries + || e.num() != EINTR // only recoverable error + ) { +// std::cerr << "ERROR in ZeroMQSvc::send (retry_send) on pid " << getpid() << ": " << e.what() << " (errno: " << e.num() << ")\n"; + throw; + } + std::cerr << "RETRY " << tries << "/" << (max_tries - 1) << " in ZeroMQSvc::send (retry_send) on pid " << getpid() << ": " << e.what() << ")\n"; + } + } +} + +template +auto retry_recv(zmq::socket_t& socket, int max_tries, args_t ...args) -> decltype(socket.recv(args...)) { + int tries = 0; + while(true) { + try { + // the actual work this function should do, all the rest is error handling: + return socket.recv(args...); + } catch (zmq::error_t& e) { + if (++tries == max_tries + || e.num() != EINTR // only recoverable error + ) { +// std::cerr << "ERROR in ZeroMQSvc::recv (retry_recv) on pid " << getpid() << ": " << e.what() << " (errno: " << e.num() << ")\n"; + throw; + } + std::cerr << "RETRY " << tries << "/" << (max_tries - 1) << " in ZeroMQSvc::recv (retry_recv) on pid " << getpid() << ": " << e.what() << ")\n"; + } + } +} + + +/** @class IZeroMQSvc IZeroMQSvc.h ZeroMQ/IZeroMQSvc.h + * + * + * @author + * @date 2015-06-22 + */ +class ZeroMQSvc { + // Note on error handling: + // Creating message_t can throw, but only when memory ran out (errno ENOMEM), + // and that is something only the caller can fix, so we don't catch it here. + +public: + + enum Encoding { + Text = 0, + Binary + }; + + Encoding encoding() const; + void setEncoding(const Encoding& e); + zmq::context_t& context() const; + zmq::socket_t socket(int type) const; + zmq::socket_t* socket_ptr(int type) const; + void close_context() const; + + // decode message with ZMQ, POD version + template ::value + && ZMQ::Detail::is_trivial::value, T>::type* = nullptr> + T decode(const zmq::message_t& msg) const { + T object; + memcpy(&object, msg.data(), msg.size()); + return object; + } + + // decode ZMQ message, string version + template ::value, T>::type* = nullptr> + std::string decode(const zmq::message_t& msg) const { + std::string r(msg.size() + 1, char{}); + r.assign(static_cast(msg.data()), msg.size()); + return r; + } + + // receive message with ZMQ, general version + // FIXME: what to do with flags=0.... more is a pointer, that might prevent conversion + template ::value), T>::type* = nullptr> + T receive(zmq::socket_t& socket, int flags = 0, bool* more = nullptr) const { + // receive message + zmq::message_t msg; + auto nbytes = retry_recv(socket, 2, &msg, flags); + if (0 == nbytes) { + throw ZMQ::TimeOutException{}; + } + if (more) *more = msg.more(); + + // decode message + return decode(msg); + } + + // receive message with ZMQ + template ::value, T>::type* = nullptr> + T receive(zmq::socket_t& socket, int flags = 0, bool* more = nullptr) const { + // receive message + zmq::message_t msg; + auto nbytes = retry_recv(socket, 2, &msg, flags); + if (0 == nbytes) { + throw ZMQ::TimeOutException{}; + } + if (more) *more = msg.more(); + return msg; + } + + // encode message to ZMQ + template ::value + && ZMQ::Detail::is_trivial::value, T>::type* = nullptr> + zmq::message_t encode(const T& item, std::function sizeFun = ZMQ::defaultSizeOf) const { + size_t s = sizeFun(item); + zmq::message_t msg{s}; + memcpy((void *)msg.data(), &item, s); + return msg; + } + + zmq::message_t encode(const char* item) const; + zmq::message_t encode(const std::string& item) const; + + // Send message with ZMQ + template ::value, T>::type* = nullptr> + bool send(zmq::socket_t& socket, const T& item, int flags = 0) const { + return retry_send(socket, 1, encode(item), flags); + } + + bool send(zmq::socket_t& socket, const char* item, int flags = 0) const; + bool send(zmq::socket_t& socket, zmq::message_t& msg, int flags = 0) const; + bool send(zmq::socket_t& socket, zmq::message_t&& msg, int flags = 0) const; + +private: + + Encoding m_enc = Text; + mutable zmq::context_t* m_context = nullptr; + +}; + +ZeroMQSvc& zmqSvc(); + +#endif // ZEROMQ_IZEROMQSVC_H diff --git a/roofit/roofitZMQ/inc/RooFit_ZMQ/functions.h b/roofit/roofitZMQ/inc/RooFit_ZMQ/functions.h new file mode 100644 index 0000000000000..2199275b1d272 --- /dev/null +++ b/roofit/roofitZMQ/inc/RooFit_ZMQ/functions.h @@ -0,0 +1,111 @@ +#ifndef ZEROMQ_FUNCTIONS_H +#define ZEROMQ_FUNCTIONS_H 1 + +#include + +namespace zmq { + + enum SocketTypes { + PAIR = ZMQ_PAIR, + PUB = ZMQ_PUB, + SUB = ZMQ_SUB, + REQ = ZMQ_REQ, + REP = ZMQ_REP, + DEALER = ZMQ_DEALER, + ROUTER = ZMQ_ROUTER, + PULL = ZMQ_PULL, + PUSH = ZMQ_PUSH, + XPUB = ZMQ_XPUB, + XSUB = ZMQ_XSUB, + STREAM = ZMQ_STREAM + }; + + enum PollType : short { + POLLIN = ZMQ_POLLIN, + POLLOUT = ZMQ_POLLOUT + }; + + enum SocketOptions { + AFFINITY = ZMQ_AFFINITY, + IDENTITY = ZMQ_IDENTITY, + SUBSCRIBE = ZMQ_SUBSCRIBE, + UNSUBSCRIBE = ZMQ_UNSUBSCRIBE, + RATE = ZMQ_RATE, + RECOVERY_IVL = ZMQ_RECOVERY_IVL, + SNDBUF = ZMQ_SNDBUF, + RCVBUF = ZMQ_RCVBUF, + RCVMORE = ZMQ_RCVMORE, + FD = ZMQ_FD, + EVENTS = ZMQ_EVENTS, + TYPE = ZMQ_TYPE, + LINGER = ZMQ_LINGER, + RECONNECT_IVL = ZMQ_RECONNECT_IVL, + BACKLOG = ZMQ_BACKLOG, + RECONNECT_IVL_MAX = ZMQ_RECONNECT_IVL_MAX, + MAXMSGSIZE = ZMQ_MAXMSGSIZE, + SNDHWM = ZMQ_SNDHWM, + RCVHWM = ZMQ_RCVHWM, + MULTICAST_HOPS = ZMQ_MULTICAST_HOPS, + RCVTIMEO = ZMQ_RCVTIMEO, + SNDTIMEO = ZMQ_SNDTIMEO, + LAST_ENDPOINT = ZMQ_LAST_ENDPOINT, + ROUTER_MANDATORY = ZMQ_ROUTER_MANDATORY, + TCP_KEEPALIVE = ZMQ_TCP_KEEPALIVE, + TCP_KEEPALIVE_CNT = ZMQ_TCP_KEEPALIVE_CNT, + TCP_KEEPALIVE_IDLE = ZMQ_TCP_KEEPALIVE_IDLE, + TCP_KEEPALIVE_INTVL = ZMQ_TCP_KEEPALIVE_INTVL, + IMMEDIATE = ZMQ_IMMEDIATE, + XPUB_VERBOSE = ZMQ_XPUB_VERBOSE, + ROUTER_RAW = ZMQ_ROUTER_RAW, + IPV6 = ZMQ_IPV6, + MECHANISM = ZMQ_MECHANISM, + PLAIN_SERVER = ZMQ_PLAIN_SERVER, + PLAIN_USERNAME = ZMQ_PLAIN_USERNAME, + PLAIN_PASSWORD = ZMQ_PLAIN_PASSWORD, + CURVE_SERVER = ZMQ_CURVE_SERVER, + CURVE_PUBLICKEY = ZMQ_CURVE_PUBLICKEY, + CURVE_SECRETKEY = ZMQ_CURVE_SECRETKEY, + CURVE_SERVERKEY = ZMQ_CURVE_SERVERKEY, + PROBE_ROUTER = ZMQ_PROBE_ROUTER, + REQ_CORRELATE = ZMQ_REQ_CORRELATE, + REQ_RELAXED = ZMQ_REQ_RELAXED, + CONFLATE = ZMQ_CONFLATE, + ZAP_DOMAIN = ZMQ_ZAP_DOMAIN, + ROUTER_HANDOVER = ZMQ_ROUTER_HANDOVER, + TOS = ZMQ_TOS, + CONNECT_RID = ZMQ_CONNECT_RID, + GSSAPI_SERVER = ZMQ_GSSAPI_SERVER, + GSSAPI_PRINCIPAL = ZMQ_GSSAPI_PRINCIPAL, + GSSAPI_SERVICE_PRINCIPAL = ZMQ_GSSAPI_SERVICE_PRINCIPAL, + GSSAPI_PLAINTEXT = ZMQ_GSSAPI_PLAINTEXT, + HANDSHAKE_IVL = ZMQ_HANDSHAKE_IVL, + SOCKS_PROXY = ZMQ_SOCKS_PROXY, + XPUB_NODROP = ZMQ_XPUB_NODROP + }; + + // Message options + enum MessageOptions { + MORE = ZMQ_MORE, + SRCFD = ZMQ_SRCFD, + SHARED = ZMQ_SHARED + }; + + // Send/recv options. + enum SendRecvOptions { + DONTWAIT = ZMQ_DONTWAIT, + SNDMORE = ZMQ_SNDMORE + }; +} + +namespace ZMQ { + +template +size_t defaultSizeOf(const T&) { + return sizeof(T); +} + +size_t stringLength(const char& cs); + +} + +#endif // ZEROMQ_FUNCTIONS_H diff --git a/roofit/roofitZMQ/inc/RooFit_ZMQ/ppoll.h b/roofit/roofitZMQ/inc/RooFit_ZMQ/ppoll.h new file mode 100644 index 0000000000000..c00a1ccfebd7f --- /dev/null +++ b/roofit/roofitZMQ/inc/RooFit_ZMQ/ppoll.h @@ -0,0 +1,29 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2016-2019, Netherlands eScience Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ +#ifndef ROOT_ROOFIT_ZMQ_ppoll +#define ROOT_ROOFIT_ZMQ_ppoll + +#include +#include + +namespace ZMQ { + +int zmq_ppoll (zmq_pollitem_t *items_, int nitems_, long timeout_, const sigset_t * sigmask_); +int ppoll(zmq_pollitem_t *items_, size_t nitems_, long timeout_, const sigset_t * sigmask_); +int ppoll(std::vector &items, long timeout_, const sigset_t * sigmask_); +class ppoll_error_t : public zmq::error_t {}; + +} + +#endif // ROOT_ROOFIT_ZMQ_ppoll diff --git a/roofit/roofitZMQ/src/ZeroMQPoller.cpp b/roofit/roofitZMQ/src/ZeroMQPoller.cpp new file mode 100644 index 0000000000000..29a0dd7d610c7 --- /dev/null +++ b/roofit/roofitZMQ/src/ZeroMQPoller.cpp @@ -0,0 +1,164 @@ +/* + * ROOT + * Copyright (c) 2018, Patrick Bos + * Distributed under the terms of the BSD 3-Clause License. + * + * The full license is in the file LICENSE, distributed with this software. + */ + +#include + +#include "RooFit_ZMQ/ppoll.h" +#include "RooFit_ZMQ/ZeroMQPoller.h" + +std::vector> ZeroMQPoller::poll(int timeo) { + std::vector> r; + if (m_items.empty()) { + throw std::runtime_error("No sockets registered"); + } + int n = 0; + while (true) { + try { + n = zmq::poll(&m_items[0], m_items.size(), timeo); + if (n == 0) return r; + break; + } catch (const zmq::error_t& e) { + std::cerr << "in ZeroMQPoller::poll on PID " << getpid() << ": " << e.what() << std::endl; + if (e.num() != EINTR) { + throw; + } + } + } + // TODO: replace this with ranges::v3::zip + for (size_t i = 0; i < m_items.size(); ++i) { + void* socket = m_items[i].socket; + size_t index = 0; + int flags = 0; + if (socket == nullptr) { + // an fd was registered + std::tie(index, flags) = m_fds[m_items[i].fd]; + } else { + // a socket was registered + const zmq::socket_t* s; + std::tie(index, flags, s) = m_sockets[socket]; + } + if (m_items[i].revents & short(flags)) { + r.emplace_back(index, flags); + } + } + return r; +} + +// This function can throw (from inside ZMQ::ppoll), so wrap in try-catch! +std::vector> ZeroMQPoller::ppoll(int timeo, const sigset_t * sigmask_) { + if (m_items.empty()) { + throw std::runtime_error("No sockets registered"); + } + + std::vector> r; + + auto n = ZMQ::ppoll(m_items, timeo, sigmask_); + if (n == 0) return r; + + for (auto& m_item : m_items) { + size_t index = 0; + int flags = 0; + if (m_item.socket == nullptr) { + // an fd was registered + std::tie(index, flags) = m_fds[m_item.fd]; + } else { + // a socket was registered + const zmq::socket_t* s; + std::tie(index, flags, s) = m_sockets[m_item.socket]; + } + if (m_item.revents & short(flags)) { + r.emplace_back(index, flags); + } + } + return r; +} + + +size_t ZeroMQPoller::size() const { + return m_items.size(); +} + +size_t ZeroMQPoller::register_socket(zmq::socket_t& socket, zmq::PollType type) { + zmq::socket_t* s = &socket; + auto it = m_sockets.find(s); + if (it != m_sockets.end()) { + return std::get<0>(it->second); + } + size_t index = m_free.empty() ? m_items.size() : m_free.front(); + if (!m_free.empty()) m_free.pop_front(); + // NOTE: tis uses the conversion-to-void* operator of + // zmq::socket_t, which returns the wrapped object + m_items.push_back({socket, 0, type, 0}); + + // We need to lookup by the pointer to the object wrapped by zmq::socket_t + m_sockets.emplace(m_items.back().socket, std::make_tuple(index, type, s)); + return index; +} + +size_t ZeroMQPoller::register_socket(int fd, zmq::PollType type) { + auto it = m_fds.find(fd); + if (it != m_fds.end()) { + return std::get<0>(it->second); + } + size_t index = m_free.empty() ? m_items.size() : m_free.front(); + if (!m_free.empty()) m_free.pop_front(); + // NOTE: tis uses the conversion-to-void* operator of + // zmq::socket_t, which returns the wrapped object + m_items.push_back({nullptr, fd, type, 0}); + + // We need to lookup by the pointer to the object wrapped by zmq::socket_t + m_fds.emplace(fd, std::make_tuple(index, type)); + return index; +} + + +size_t ZeroMQPoller::unregister_socket(zmq::socket_t& socket) { + if (!m_sockets.count(socket.operator void*())) { + throw std::out_of_range("Socket is not registered"); + } + // Remove from m_sockets + // Can't search by the key of m_sockets, as that is the wrapped + // object, but have to use the pointer to the wrapper + // (zmq::socket_t) + auto it = std::find_if(begin(m_sockets), end(m_sockets), + [&socket](const decltype(m_sockets)::value_type& entry) { + return &socket == std::get<2>(entry.second); + }); + auto index = std::get<0>(it->second); + m_free.push_back(index); + m_sockets.erase(it); + + // Remove from m_items + auto iit = std::find_if(begin(m_items), end(m_items), [&it](const zmq::pollitem_t& item) { + return it->first == item.socket; + }); + assert(iit != end(m_items)); + m_items.erase(iit); + + return index; +} + +size_t ZeroMQPoller::unregister_socket(int fd) { + if (!m_fds.count(fd)) { + throw std::out_of_range("fileno is not registered"); + } + // Remove from m_fds + auto it = m_fds.find(fd); + auto index = std::get<0>(it->second); + m_free.push_back(index); + m_fds.erase(it); + + // Remove from m_items + auto iit = std::find_if(begin(m_items), end(m_items), [&it](const zmq::pollitem_t& item) { + return it->first == item.fd; + }); + assert(iit != end(m_items)); + m_items.erase(iit); + + return index; +} diff --git a/roofit/roofitZMQ/src/ZeroMQSvc.cpp b/roofit/roofitZMQ/src/ZeroMQSvc.cpp new file mode 100644 index 0000000000000..b69695349e0c3 --- /dev/null +++ b/roofit/roofitZMQ/src/ZeroMQSvc.cpp @@ -0,0 +1,103 @@ +//#include +#include // std::ref +#include + +#include "RooFit_ZMQ/ZeroMQSvc.h" + +ZeroMQSvc& zmqSvc() { + static std::unique_ptr svc; + if (!svc) { + svc = std::make_unique(); + } + return *svc; +} + +ZeroMQSvc::Encoding ZeroMQSvc::encoding() const { + return m_enc; +} + +void ZeroMQSvc::setEncoding(const ZeroMQSvc::Encoding &e) { + m_enc = e; +} + +zmq::context_t& ZeroMQSvc::context() const { + if (!m_context) { + try { + m_context = new zmq::context_t; + } catch (zmq::error_t& e) { + std::cerr << "ERROR: Creating ZeroMQ context failed. This only happens when PGM initialization failed or when a nullptr was returned from zmq_ctx_new because the created context was invalid. Contact ZMQ experts when this happens, because it shouldn't.\n"; + throw e; + } + } + return *m_context; +} + +zmq::socket_t ZeroMQSvc::socket(int type) const { + try { + // the actual work this function should do, all the rest is error handling: + return zmq::socket_t{context(), type}; + } catch (zmq::error_t& e) { + // all zmq errors not recoverable from here, only at call site + std::cerr << "ERROR in ZeroMQSvc::socket: " << e.what() << " (errno: " << e.num() << ")\n"; + throw e; + } +} + +zmq::socket_t* ZeroMQSvc::socket_ptr(int type) const { + try { + // the actual work this function should do, all the rest is error handling: + return new zmq::socket_t(context(), type); + } catch (zmq::error_t& e) { + // all zmq errors not recoverable from here, only at call site + std::cerr << "ERROR in ZeroMQSvc::socket_ptr: " << e.what() << " (errno: " << e.num() << ")\n"; + throw e; + } +} + +void ZeroMQSvc::close_context() const { + if (m_context) { + delete m_context; + m_context = nullptr; + } +} + + +zmq::message_t ZeroMQSvc::encode(const char* item) const { + std::function fun = ZMQ::stringLength; + return encode(*item, fun); +} + +zmq::message_t ZeroMQSvc::encode(const std::string& item) const { + return encode(item.c_str()); +} + +//zmq::message_t ZeroMQSvc::encode(const TObject& item) const { +// auto deleteBuffer = []( void* data, void* /* hint */ ) -> void { +// delete [] (char*)data; +// }; +// +// TBufferFile buffer(TBuffer::kWrite); +// buffer.WriteObject(&item); +// +// // Create message and detach buffer +// // This is the only ZMQ thing that can throw, and only when memory ran out +// // (errno ENOMEM), and that is something only the caller can fix, so we don't +// // catch it here: +// zmq::message_t message(buffer.Buffer(), buffer.Length(), deleteBuffer); +// buffer.DetachBuffer(); +// +// return message; +//} + + +bool ZeroMQSvc::send(zmq::socket_t& socket, const char* item, int flags) const { + return retry_send(socket, 2, encode(item), flags); +} + +bool ZeroMQSvc::send(zmq::socket_t& socket, zmq::message_t& msg, int flags) const { + return retry_send(socket, 2, std::ref(msg), flags); +} + +bool ZeroMQSvc::send(zmq::socket_t& socket, zmq::message_t&& msg, int flags) const { + return retry_send(socket, 2, std::move(msg), flags); +} diff --git a/roofit/roofitZMQ/src/functions.cpp b/roofit/roofitZMQ/src/functions.cpp new file mode 100644 index 0000000000000..ebb6199bfdb5d --- /dev/null +++ b/roofit/roofitZMQ/src/functions.cpp @@ -0,0 +1,9 @@ +#include +#include "RooFit_ZMQ/ZeroMQSvc.h" +#include "RooFit_ZMQ/functions.h" + +namespace ZMQ { + size_t stringLength(const char& cs) { + return strlen(&cs); + } +} diff --git a/roofit/roofitZMQ/src/ppoll.cpp b/roofit/roofitZMQ/src/ppoll.cpp new file mode 100644 index 0000000000000..5cfcda16222e1 --- /dev/null +++ b/roofit/roofitZMQ/src/ppoll.cpp @@ -0,0 +1,345 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2016-2019, Netherlands eScience Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include // pselect +#include // timespec +#include + +#include "likely.hpp" // unlikely +#include "clock.hpp" // zmq::clock_t +#include "err.hpp" // zmq_assert +#include "polling_util.hpp" // zmq::optimized_fd_set_t, zmq::compute_timeout, zmq::timeout_t +#include "fd.hpp" // zmq::fd_t + +#include + + +namespace ZMQ { + +int zmq_ppoll (zmq_pollitem_t *items_, int nitems_, long timeout_, + const sigset_t * sigmask_) +{ +#if defined ZMQ_HAVE_POLLER + // if poller is present, use that if there is at least 1 thread-safe socket, + // otherwise fall back to the previous implementation as it's faster. + for (int i = 0; i != nitems_; i++) { + if (items_[i].socket + && as_socket_base_t (items_[i].socket)->is_thread_safe ()) { + return zmq_poller_poll (items_, nitems_, timeout_); + } + } +#endif // ZMQ_HAVE_POLLER +#if defined ZMQ_POLL_BASED_ON_POLL || defined ZMQ_POLL_BASED_ON_SELECT + if (unlikely (nitems_ < 0)) { + errno = EINVAL; + return -1; + } + if (unlikely (nitems_ == 0)) { + if (timeout_ == 0) + return 0; +#if defined ZMQ_HAVE_WINDOWS + Sleep (timeout_ > 0 ? timeout_ : INFINITE); + return 0; +#elif defined ZMQ_HAVE_VXWORKS + struct timespec ns_; + ns_.tv_sec = timeout_ / 1000; + ns_.tv_nsec = timeout_ % 1000 * 1000000; + return nanosleep (&ns_, 0); +#else + return usleep (timeout_ * 1000); +#endif + } + if (!items_) { + errno = EFAULT; + return -1; + } + + zmq::clock_t clock; + uint64_t now = 0; + uint64_t end = 0; +#if defined ZMQ_POLL_BASED_ON_POLL + zmq::fast_vector_t pollfds (nitems_); + + // Build pollset for poll () system call. + for (int i = 0; i != nitems_; i++) { + // If the poll item is a 0MQ socket, we poll on the file descriptor + // retrieved by the ZMQ_FD socket option. + if (items_[i].socket) { + size_t zmq_fd_size = sizeof (zmq::fd_t); + if (zmq_getsockopt (items_[i].socket, ZMQ_FD, &pollfds[i].fd, + &zmq_fd_size) + == -1) { + return -1; + } + pollfds[i].events = items_[i].events ? POLLIN : 0; + } + // Else, the poll item is a raw file descriptor. Just convert the + // events to normal POLLIN/POLLOUT for poll (). + else { + pollfds[i].fd = items_[i].fd; + pollfds[i].events = + (items_[i].events & ZMQ_POLLIN ? POLLIN : 0) + | (items_[i].events & ZMQ_POLLOUT ? POLLOUT : 0) + | (items_[i].events & ZMQ_POLLPRI ? POLLPRI : 0); + } + } +#else + // Ensure we do not attempt to select () on more than FD_SETSIZE + // file descriptors. + // TODO since this function is called by a client, we could return errno EINVAL/ENOMEM/... here + zmq_assert (nitems_ <= FD_SETSIZE); + + zmq::optimized_fd_set_t pollset_in (nitems_); + FD_ZERO (pollset_in.get ()); + zmq::optimized_fd_set_t pollset_out (nitems_); + FD_ZERO (pollset_out.get ()); + zmq::optimized_fd_set_t pollset_err (nitems_); + FD_ZERO (pollset_err.get ()); + + zmq::fd_t maxfd = 0; + + // Build the fd_sets for passing to select (). + for (int i = 0; i != nitems_; i++) { + // If the poll item is a 0MQ socket we are interested in input on the + // notification file descriptor retrieved by the ZMQ_FD socket option. + if (items_[i].socket) { + size_t zmq_fd_size = sizeof (zmq::fd_t); + zmq::fd_t notify_fd; + if (zmq_getsockopt (items_[i].socket, ZMQ_FD, ¬ify_fd, + &zmq_fd_size) + == -1) + return -1; + if (items_[i].events) { + FD_SET (notify_fd, pollset_in.get ()); + if (maxfd < notify_fd) + maxfd = notify_fd; + } + } + // Else, the poll item is a raw file descriptor. Convert the poll item + // events to the appropriate fd_sets. + else { + if (items_[i].events & ZMQ_POLLIN) + FD_SET (items_[i].fd, pollset_in.get ()); + if (items_[i].events & ZMQ_POLLOUT) + FD_SET (items_[i].fd, pollset_out.get ()); + if (items_[i].events & ZMQ_POLLERR) + FD_SET (items_[i].fd, pollset_err.get ()); + if (maxfd < items_[i].fd) + maxfd = items_[i].fd; + } + } + + zmq::optimized_fd_set_t inset (nitems_); + zmq::optimized_fd_set_t outset (nitems_); + zmq::optimized_fd_set_t errset (nitems_); +#endif + + bool first_pass = true; + int nevents = 0; + + while (true) { +#if defined ZMQ_POLL_BASED_ON_POLL + + // Compute the timeout for the subsequent poll. + zmq::timeout_t timeout = + zmq::compute_timeout (first_pass, timeout_, now, end); + + // Wait for events. + { + int rc = ppoll (&pollfds[0], nitems_, timeout); + if (rc == -1 && errno == EINTR) { + return -1; + } + errno_assert (rc >= 0); + } + // Check for the events. + for (int i = 0; i != nitems_; i++) { + items_[i].revents = 0; + + // The poll item is a 0MQ socket. Retrieve pending events + // using the ZMQ_EVENTS socket option. + if (items_[i].socket) { + size_t zmq_events_size = sizeof (uint32_t); + uint32_t zmq_events; + if (zmq_getsockopt (items_[i].socket, ZMQ_EVENTS, &zmq_events, + &zmq_events_size) + == -1) { + return -1; + } + if ((items_[i].events & ZMQ_POLLOUT) + && (zmq_events & ZMQ_POLLOUT)) + items_[i].revents |= ZMQ_POLLOUT; + if ((items_[i].events & ZMQ_POLLIN) + && (zmq_events & ZMQ_POLLIN)) + items_[i].revents |= ZMQ_POLLIN; + } + // Else, the poll item is a raw file descriptor, simply convert + // the events to zmq_pollitem_t-style format. + else { + if (pollfds[i].revents & POLLIN) + items_[i].revents |= ZMQ_POLLIN; + if (pollfds[i].revents & POLLOUT) + items_[i].revents |= ZMQ_POLLOUT; + if (pollfds[i].revents & POLLPRI) + items_[i].revents |= ZMQ_POLLPRI; + if (pollfds[i].revents & ~(POLLIN | POLLOUT | POLLPRI)) + items_[i].revents |= ZMQ_POLLERR; + } + + if (items_[i].revents) + nevents++; + } + +#else + + // Compute the timeout for the subsequent poll. + struct timespec timeout; + struct timespec *ptimeout; + if (first_pass) { + timeout.tv_sec = 0; + timeout.tv_nsec = 0; + ptimeout = &timeout; + } else if (timeout_ < 0) + ptimeout = NULL; + else { + timeout.tv_sec = static_cast ((end - now) / 1000); + timeout.tv_nsec = static_cast ((end - now) % 1000 * 1000000); + ptimeout = &timeout; + } + + // Wait for events. Ignore interrupts if there's infinite timeout. + while (true) { + memcpy (inset.get (), pollset_in.get (), + zmq::valid_pollset_bytes (*pollset_in.get ())); + memcpy (outset.get (), pollset_out.get (), + zmq::valid_pollset_bytes (*pollset_out.get ())); + memcpy (errset.get (), pollset_err.get (), + zmq::valid_pollset_bytes (*pollset_err.get ())); +#if defined ZMQ_HAVE_WINDOWS + int rc = + select (0, inset.get (), outset.get (), errset.get (), ptimeout); + if (unlikely (rc == SOCKET_ERROR)) { + errno = zmq::wsa_error_to_errno (WSAGetLastError ()); + wsa_assert (errno == ENOTSOCK); + return -1; + } +#else + int rc = pselect (maxfd + 1, inset.get (), outset.get (), + errset.get (), ptimeout, sigmask_); + if (unlikely (rc == -1)) { + errno_assert (errno == EINTR || errno == EBADF); + return -1; + } +#endif + break; + } + + // Check for the events. + for (int i = 0; i != nitems_; i++) { + items_[i].revents = 0; + + // The poll item is a 0MQ socket. Retrieve pending events + // using the ZMQ_EVENTS socket option. + if (items_[i].socket) { + size_t zmq_events_size = sizeof (uint32_t); + uint32_t zmq_events; + if (zmq_getsockopt (items_[i].socket, ZMQ_EVENTS, &zmq_events, + &zmq_events_size) + == -1) + return -1; + if ((items_[i].events & ZMQ_POLLOUT) + && (zmq_events & ZMQ_POLLOUT)) + items_[i].revents |= ZMQ_POLLOUT; + if ((items_[i].events & ZMQ_POLLIN) + && (zmq_events & ZMQ_POLLIN)) + items_[i].revents |= ZMQ_POLLIN; + } + // Else, the poll item is a raw file descriptor, simply convert + // the events to zmq_pollitem_t-style format. + else { + if (FD_ISSET (items_[i].fd, inset.get ())) + items_[i].revents |= ZMQ_POLLIN; + if (FD_ISSET (items_[i].fd, outset.get ())) + items_[i].revents |= ZMQ_POLLOUT; + if (FD_ISSET (items_[i].fd, errset.get ())) + items_[i].revents |= ZMQ_POLLERR; + } + + if (items_[i].revents) + nevents++; + } +#endif + + // If timeout is zero, exit immediately whether there are events or not. + if (timeout_ == 0) + break; + + // If there are events to return, we can exit immediately. + if (nevents) + break; + + // At this point we are meant to wait for events but there are none. + // If timeout is infinite we can just loop until we get some events. + if (timeout_ < 0) { + if (first_pass) + first_pass = false; + continue; + } + + // The timeout is finite and there are no events. In the first pass + // we get a timestamp of when the polling have begun. (We assume that + // first pass have taken negligible time). We also compute the time + // when the polling should time out. + if (first_pass) { + now = clock.now_ms (); + end = now + timeout_; + if (now == end) + break; + first_pass = false; + continue; + } + + // Find out whether timeout have expired. + now = clock.now_ms (); + if (now >= end) + break; + } + + return nevents; +#else + // Exotic platforms that support neither poll() nor select(). + errno = ENOTSUP; + return -1; +#endif +} + + +// This function can throw, so wrap in try-catch! +int ppoll(zmq_pollitem_t *items_, size_t nitems_, long timeout_, const sigset_t * sigmask_) +{ + int rc = zmq_ppoll(items_, static_cast(nitems_), timeout_, sigmask_); + if (rc < 0) + throw ppoll_error_t(); + return rc; +} + + +// This function can throw, so wrap in try-catch! +int ppoll(std::vector &items, long timeout_, const sigset_t * sigmask_) +{ + return ppoll(items.data(), items.size(), timeout_, sigmask_); +} + +} + diff --git a/roofit/roofitZMQ/test/CMakeLists.txt b/roofit/roofitZMQ/test/CMakeLists.txt new file mode 100644 index 0000000000000..b1da06f39da1f --- /dev/null +++ b/roofit/roofitZMQ/test/CMakeLists.txt @@ -0,0 +1,6 @@ +#find_package(ZeroMQ REQUIRED) + +ROOT_ADD_GTEST(test_RooFitZMQ test_ZMQ.cpp LIBRARIES RooFitZMQ ${ZeroMQ_LIBRARY}) +ROOT_ADD_GTEST(test_RooFitZMQ_polling test_polling.cxx LIBRARIES RooFitZMQ ${ZeroMQ_LIBRARY}) +ROOT_ADD_GTEST(test_RooFitZMQ_HWM test_HWM.cxx LIBRARIES RooFitZMQ ${ZeroMQ_LIBRARY}) +ROOT_ADD_GTEST(test_RooFitZMQ_load_balancing test_ZMQ_load_balancing.cxx LIBRARIES RooFitZMQ ${ZeroMQ_LIBRARY}) diff --git a/roofit/roofitZMQ/test/test_HWM.cxx b/roofit/roofitZMQ/test/test_HWM.cxx new file mode 100644 index 0000000000000..7ef11b5c3f5ed --- /dev/null +++ b/roofit/roofitZMQ/test/test_HWM.cxx @@ -0,0 +1,130 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2016-2021, Netherlands eScience Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "gtest/gtest.h" + +#include // fork, usleep + +#include "RooFit_ZMQ/ZeroMQSvc.h" + +class HighWaterMarkTest : public ::testing::Test { +protected: + void SetUp() override { + do { + child_pid = fork(); + } while (child_pid == -1); // retry if fork fails + + if (child_pid > 0) { // parent + pusher.reset(zmqSvc().socket_ptr(zmq::PUSH)); + if (set_hwm) { + auto rc = zmq_setsockopt(*pusher, ZMQ_SNDHWM, &hwm, sizeof hwm); + assert(rc == 0); + } + pusher->bind("ipc:///tmp/ZMQ_test_fork_polling_P2C.ipc"); + } else { // child + puller.reset(zmqSvc().socket_ptr(zmq::PULL)); + if (set_hwm) { + auto rc = zmq_setsockopt(*puller, ZMQ_RCVHWM, &hwm, sizeof hwm); + assert(rc == 0); + } + puller->connect("ipc:///tmp/ZMQ_test_fork_polling_P2C.ipc"); + } + } + + void TearDown() override { + if (child_pid > 0) { // parent + // wait for child + int status = 0; + pid_t pid; + do { + pid = waitpid(child_pid, &status, 0); + } while (-1 == pid && EINTR == errno); // retry on interrupted system call + if (0 != status) { + if (WIFEXITED(status)) { printf("exited, status=%d\n", WEXITSTATUS(status)); } else if (WIFSIGNALED(status)) { printf("killed by signal %d\n", WTERMSIG(status)); } else if (WIFSTOPPED(status)) { printf("stopped by signal %d\n", WSTOPSIG(status)); } else if (WIFCONTINUED(status)) { printf("continued\n"); } + } + if (-1 == pid) { + throw std::runtime_error(std::string("waitpid, errno ") + std::to_string(errno)); + } + pusher.reset(nullptr); + zmqSvc().close_context(); + } else { // child + puller.reset(nullptr); + zmqSvc().close_context(); + } + } + + void run_test() { + std::size_t max_sends = 2000; + + if (child_pid > 0) { // parent + // start test + for (std::size_t ix = 0; ix < max_sends; ++ix) { + zmqSvc().send(*pusher, 0.1f); + zmqSvc().send(*pusher, 1); + zmqSvc().send(*pusher, true); + if (ix % 100 == 0) { + printf("parent at ix = %lu\n", ix); + } + } + } else { // child + // wait a few seconds before reading to allow the parent to overflow the HWM + printf("child waiting for 2 seconds...\n"); + sleep(2); + printf("child starts receiving\n"); + + for (std::size_t ix = 0; ix < max_sends; ++ix) { + zmqSvc().receive(*puller); + zmqSvc().receive(*puller); + zmqSvc().receive(*puller); + if (ix % 100 == 0) { + printf("child at ix = %lu\n", ix); + } + } + } + } + + pid_t child_pid {0}; + ZmqLingeringSocketPtr<> pusher, puller; + bool set_hwm = false; + int hwm = 0; +}; + + +TEST_F(HighWaterMarkTest, demonstrateHittingDefaultHWM) { + run_test(); +} + +class HighWaterMarkZeroTest : public HighWaterMarkTest { + void SetUp() override { + set_hwm = true; + hwm = 0; + HighWaterMarkTest::SetUp(); + } +}; + +TEST_F(HighWaterMarkZeroTest, zeroHWM) { + run_test(); +} + +class HighWaterMark2kTest : public HighWaterMarkTest { + void SetUp() override { + set_hwm = true; + hwm = 2000; + HighWaterMarkTest::SetUp(); + } +}; + +TEST_F(HighWaterMark2kTest, HWM2k) { + run_test(); +} diff --git a/roofit/roofitZMQ/test/test_ZMQ.cpp b/roofit/roofitZMQ/test/test_ZMQ.cpp new file mode 100644 index 0000000000000..985d1b91c5902 --- /dev/null +++ b/roofit/roofitZMQ/test/test_ZMQ.cpp @@ -0,0 +1,306 @@ +/***************************************************************************** + * Project: RooFit * + * Package: RooFitCore * + * @(#)root/roofitcore:$Id$ + * Authors: * + * PB, Patrick Bos, NL eScience Center, p.bos@esciencecenter.nl * + * * + * Redistribution and use in source and binary forms, * + * with or without modification, are permitted according to the terms * + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) * + *****************************************************************************/ + +#include "gtest/gtest.h" + +#include // fork, usleep + +#include + +#include "RooFit_ZMQ/ZeroMQSvc.h" + + +// N.B.: wait_for_child is identically defined in RooFit::MultiProcess, but we copy it here to reduce module +// interdependencies. It also requires an extra include: +#include // kill, SIGKILL + +int wait_for_child(pid_t child_pid, bool may_throw, int retries_before_killing) +{ + int status = 0; + int patience = retries_before_killing; + pid_t tmp; + do { + if (patience-- < 1) { + ::kill(child_pid, SIGKILL); + } + tmp = waitpid(child_pid, &status, WNOHANG); + } while (tmp == 0 // child has not yet changed state, try again + || (-1 == tmp && EINTR == errno) // retry on interrupted system call + ); + + if (patience < 1) { + std::cout << "Had to send PID " << child_pid << " " << (-patience + 1) << " SIGKILLs"; + } + + if (0 != status) { + if (WIFEXITED(status)) { + printf("exited, status=%d\n", WEXITSTATUS(status)); + } else if (WIFSIGNALED(status)) { + printf("killed by signal %d\n", WTERMSIG(status)); + } else if (WIFSTOPPED(status)) { + printf("stopped by signal %d\n", WSTOPSIG(status)); + } else if (WIFCONTINUED(status)) { + printf("continued\n"); + } + } + + if (-1 == tmp && may_throw) + throw std::runtime_error(std::string("waitpid, errno ") + std::to_string(errno)); + + return status; +} + +void elaborate_bind(const ZmqLingeringSocketPtr<>& socket, std::string name) { + try { + socket->bind(name); + } catch (const zmq::error_t& e) { + if (e.num() == EADDRINUSE) { + std::cerr << "address already in use, retrying bind in 500ms\n"; + usleep(500000); + try { + socket->bind(name); + } catch (const zmq::error_t& e2) { + if (e2.num() == EADDRINUSE) { + std::cerr + << "again: address already in use, aborting; please check whether there are any remaining improperly exited processes (zombies) around or whether some other program is using port 6660\n"; + } + throw e2; + } + // Sometimes, the socket from the previous test needs some time to close, so + // we introduce a latency here. A more robust and fast approach might be to + // do the following on the bind side: + // 1. first try another port, e.g. increase by one + // 2. if that doesn't work, do the latency and retry the original port + // The connect side then also needs to change, because it doesn't know which + // port the bind side will bind to. The connect side could try connecting to + // both options asynchronously, and then in a loop check both for signs of + // life. If one comes alive, transfer ownership of that pointer to the pointer + // you want to eventually use (`socket`) and that's it. + } else { + throw e; + } + } +} + +class AllSocketTypes : public ::testing::TestWithParam< std::tuple, std::pair /* socket_names */> > {}; + +TEST_P(AllSocketTypes, forkHandshake) { + auto socket_names = std::get<2>(GetParam()); + pid_t child_pid {0}; + do { + child_pid = fork(); + } while (child_pid == -1); // retry if fork fails + + if (child_pid > 0) { // master + ZmqLingeringSocketPtr<> socket; + socket.reset(zmqSvc().socket_ptr(std::get<1>(GetParam()).first)); + elaborate_bind(socket, socket_names.second); + // bind is on the master process to avoid zombie children to hold on to binds + + // start test + zmqSvc().send(*socket, std::string("breaker breaker")); + + auto receipt = zmqSvc().receive(*socket); + + EXPECT_EQ(receipt, 1212); + +// socket->close(); // this gives exception of type zmq::error_t: Socket operation on non-socket + socket.reset(nullptr); + zmqSvc().close_context(); // if you don't close context in parent process as well, the next repeat will hang + + wait_for_child(child_pid, true, 5); + } else { // child + ZmqLingeringSocketPtr<> socket; + socket.reset(zmqSvc().socket_ptr(std::get<1>(GetParam()).second)); + socket->connect(socket_names.first); + + // start test + auto receipt = zmqSvc().receive(*socket); + if (receipt == "breaker breaker") { + zmqSvc().send(*socket, 1212); + } + // take care, don't just use _exit, it will not cleanly destroy context etc! + // if you really need to, at least close and destroy everything properly +// socket->close(); // this gives exception of type zmq::error_t: Socket operation on non-socket + socket.reset(nullptr); + zmqSvc().close_context(); + _Exit(0); + } +} + +std::string ipc {"ipc:///tmp/ZMQ_test_fork.ipc"}; +std::string tcp_server {"tcp://127.0.0.1:6660"}; +std::string tcp_client {"tcp://*:6660"}; +auto socket_name_options = ::testing::Values( + std::make_pair(tcp_server, tcp_client), + std::make_pair(ipc, ipc) + ); + + +INSTANTIATE_TEST_SUITE_P(REQREP, AllSocketTypes, + ::testing::Combine(::testing::Range(0, 10), // repeat to probe connection stability + ::testing::Values(std::make_pair(zmq::REQ, zmq::REP)), + socket_name_options + )); +INSTANTIATE_TEST_SUITE_P(PAIRPAIR, AllSocketTypes, + ::testing::Combine(::testing::Range(0, 10), // repeat to probe connection stability + ::testing::Values(std::make_pair(zmq::PAIR, zmq::PAIR)), + socket_name_options + )); + + +class AsyncSocketTypes : public ::testing::TestWithParam< std::tuple, std::pair /* socket_names */, bool /* expect_throw */> > {}; + +TEST_P(AsyncSocketTypes, forkMultiSendReceive) { + bool expect_throw = std::get<3>(GetParam()); + ZmqLingeringSocketPtr<> socket; + auto socket_names = std::get<2>(GetParam()); + pid_t child_pid {0}; + do { + child_pid = fork(); + } while (child_pid == -1); // retry if fork fails + + if (child_pid > 0) { // master + socket.reset(zmqSvc().socket_ptr(std::get<1>(GetParam()).first)); + elaborate_bind(socket, socket_names.second); + // bind is on the master process to avoid zombie children to hold on to binds + + // start test: send 2 things, receive 1, send 1 more, finish + zmqSvc().send(*socket, std::string("breaker breaker")); + + if (expect_throw) { + EXPECT_ANY_THROW(zmqSvc().send(*socket, std::string("anybody out there?"))); + // NOTE: also in case of a throw, be sure to properly close down the connection! + // Otherwise, you may get zombies waiting for a reply. + socket.reset(nullptr); + zmqSvc().close_context(); // if you don't close context in parent process as well, the next repeat will hang + wait_for_child(child_pid, true, 5); + return; + } else { + EXPECT_NO_THROW(zmqSvc().send(*socket, std::string("anybody out there?"))); + } + + auto receipt = zmqSvc().receive(*socket); + + EXPECT_EQ(receipt, 1212); + + zmqSvc().send(*socket, std::string("kthxbye")); + +// socket->close(); // this gives exception of type zmq::error_t: Socket operation on non-socket + socket.reset(nullptr); + zmqSvc().close_context(); // if you don't close context in parent process as well, the next repeat will hang + + wait_for_child(child_pid, true, 5); + } else { // child + socket.reset(zmqSvc().socket_ptr(std::get<1>(GetParam()).second)); + socket->connect(socket_names.first); + + // start test, receive something + auto receipt1 = zmqSvc().receive(*socket); + auto receipt2 = zmqSvc().receive(*socket); + if (receipt1 == "breaker breaker" && receipt2 == "anybody out there?") { + zmqSvc().send(*socket, 1212); + } + auto receipt3 = zmqSvc().receive(*socket); + if (receipt3 != "kthxbye") { + std::cerr << "did not receive final reply correctly\n"; + } + + // take care, don't just use _exit, it will not cleanly destroy context etc! + // if you really need to, at least close and destroy everything properly +// socket->close(); // this gives exception of type zmq::error_t: Socket operation on non-socket + socket.reset(nullptr); + zmqSvc().close_context(); + _Exit(0); + } +} + +TEST_P(AsyncSocketTypes, forkIgnoreSomeMessages) { + bool expect_throw = std::get<3>(GetParam()); + ZmqLingeringSocketPtr<> socket; + auto socket_names = std::get<2>(GetParam()); + pid_t child_pid {0}; + do { + child_pid = fork(); + } while (child_pid == -1); // retry if fork fails + + if (child_pid > 0) { // master + socket.reset(zmqSvc().socket_ptr(std::get<1>(GetParam()).first)); + elaborate_bind(socket, socket_names.second); + // bind is on the master process to avoid zombie children to hold on to binds + + // start test: send 2 things, receive 1, send 1 more, finish + zmqSvc().send(*socket, std::string("breaker breaker")); + + if (expect_throw) { + EXPECT_ANY_THROW(zmqSvc().send(*socket, std::string("anybody out there?"))); + // NOTE: also in case of a throw, be sure to properly close down the connection! + // Otherwise, you may get zombies waiting for a reply. + socket.reset(nullptr); + zmqSvc().close_context(); // if you don't close context in parent process as well, the next repeat will hang + wait_for_child(child_pid, true, 5); + return; + } else { + EXPECT_NO_THROW(zmqSvc().send(*socket, std::string("anybody out there?"))); + } + + auto receipt = zmqSvc().receive(*socket); + + EXPECT_EQ(receipt, 1212); + + zmqSvc().send(*socket, std::string("kthxbye")); + +// socket->close(); // this gives exception of type zmq::error_t: Socket operation on non-socket + socket.reset(nullptr); + zmqSvc().close_context(); // if you don't close context in parent process as well, the next repeat will hang + + wait_for_child(child_pid, true, 5); + } else { // child + socket.reset(zmqSvc().socket_ptr(std::get<1>(GetParam()).second)); + socket->connect(socket_names.first); + + // start test, receive first thing + auto receipt = zmqSvc().receive(*socket); + if (receipt == "breaker breaker") { + zmqSvc().send(*socket, 1212); + } + + // ignore the rest of the sent messages, but give the other end a second to + // actually send its stuff, instead of hanging in retry_send because the + // connection has died; a better solution would be if retry_send (in + // ZeroMQSvc::send) had a callback mechanism that could be used to break + // out when a child has died, but ok + sleep(1); + + // take care, don't just use _exit, it will not cleanly destroy context etc! + // if you really need to, at least close and destroy everything properly +// socket->close(); // this gives exception of type zmq::error_t: Socket operation on non-socket + socket.reset(nullptr); + zmqSvc().close_context(); + _Exit(0); + } +} + + +INSTANTIATE_TEST_SUITE_P(PAIRPAIR, AsyncSocketTypes, + ::testing::Combine(::testing::Range(0, 10), // repeat to probe connection stability + ::testing::Values(std::make_pair(zmq::PAIR, zmq::PAIR)), + socket_name_options, + ::testing::Values(false) // don't expect throw + )); + +INSTANTIATE_TEST_SUITE_P(REQREP, AsyncSocketTypes, + ::testing::Combine(::testing::Values(0), // no repeats, we only care about the throw + ::testing::Values(std::make_pair(zmq::REQ, zmq::REP)), + socket_name_options, + ::testing::Values(true) // expect throw + )); diff --git a/roofit/roofitZMQ/test/test_ZMQ_load_balancing.cxx b/roofit/roofitZMQ/test/test_ZMQ_load_balancing.cxx new file mode 100644 index 0000000000000..ebb4680357104 --- /dev/null +++ b/roofit/roofitZMQ/test/test_ZMQ_load_balancing.cxx @@ -0,0 +1,133 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2016-2021, Netherlands eScience Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "gtest/gtest.h" + +#include // fork, usleep + +#include "RooFit_ZMQ/ZeroMQSvc.h" +#include "RooFit_ZMQ/ZeroMQPoller.h" + +class ZMQPushPullTest : public ::testing::Test { +protected: + std::size_t N_children = 4; + std::size_t max_sends = 20; + + void SetUp() override { + for (std::size_t i = 0; i < N_children; ++i) { + do { + child_pid = fork(); + } while (child_pid == -1); // retry if fork fails + if (child_pid == 0) { // child + child_id = i; + break; + } else { + child_pids.push_back(child_pid); + } + } + + if (child_pid > 0) { // parent + pusher.reset(zmqSvc().socket_ptr(zmq::PUSH)); + pusher->bind("ipc:///tmp/ZMQ_test_push_pull_P2C.ipc"); + } else { // child + puller.reset(zmqSvc().socket_ptr(zmq::PULL)); + puller->connect("ipc:///tmp/ZMQ_test_push_pull_P2C.ipc"); + + poller.register_socket(*puller, zmq::POLLIN); + } + } + + void TearDown() override { + if (child_pid > 0) { // parent + // wait for children + int status = 0; + pid_t pid; + for (pid_t child_pid_i : child_pids) { + do { + pid = waitpid(child_pid_i, &status, 0); + } while (-1 == pid && EINTR == errno); // retry on interrupted system call + if (0 != status) { + if (WIFEXITED(status)) { printf("exited, status=%d\n", WEXITSTATUS(status)); } else if (WIFSIGNALED(status)) { printf("killed by signal %d\n", WTERMSIG(status)); } else if (WIFSTOPPED(status)) { printf("stopped by signal %d\n", WSTOPSIG(status)); } else if (WIFCONTINUED(status)) { printf("continued\n"); } + } + if (-1 == pid) { + throw std::runtime_error(std::string("waitpid, errno ") + std::to_string(errno)); + } + } + pusher.reset(nullptr); + zmqSvc().close_context(); + } else { // child + puller.reset(nullptr); + zmqSvc().close_context(); + } + } + + void run_parent() { + // start test + usleep(1000); // wait a second so that all pull sockets are connected for round-robin distribution + // if you don't wait a second above, the push socket will "round-robin" all the messages to just one or two connected sockets + for (std::size_t ix = 0; ix < max_sends; ++ix) { + zmqSvc().send(*pusher, 0); + } + for (std::size_t ix = 0; ix < N_children; ++ix) { + // end by sending some 1's to all children, to let them know the sending is over + zmqSvc().send(*pusher, 1); + } + } + + void run_child() { + std::size_t count = 0; + for (std::size_t ix = 0; ix < max_sends; ++ix) { + auto r = poller.poll(2000); + if (r.empty()) { + printf("poller of child %d timed out after 2 seconds\n", child_id); + break; + } + auto value = zmqSvc().receive(*puller, ZMQ_DONTWAIT); + usleep(200); // "do some work" + printf("value on child %d: %d\n", child_id, value); + if (value == 1) { + printf("child %d got value %d, done here\n", child_id, value); + break; + } + ++count; + } + printf("child %d got %lu values\n", child_id, count); + } + + pid_t child_pid {0}; + int child_id = -1; + std::vector child_pids; + ZmqLingeringSocketPtr<> pusher, puller; + ZeroMQPoller poller; +}; + + +/// This test shows how push-pull is unsuited for load balancing; messages are just sent to the first available pull socket without any dynamic load balancing +TEST_F(ZMQPushPullTest, demoRoundRobin) { + if (child_pid > 0) { + run_parent(); + } else { + run_child(); + } +} + +/// This test tries to see whether push-pull can be made to work as a bit of a load balancer, using a low HWM at the receiver +TEST_F(ZMQPushPullTest, demoHWM1LoadBalancing) { + if (child_pid > 0) { + run_parent(); + } else { + puller->setsockopt(ZMQ_RCVHWM, 1); + run_child(); + } +} diff --git a/roofit/roofitZMQ/test/test_polling.cxx b/roofit/roofitZMQ/test/test_polling.cxx new file mode 100644 index 0000000000000..9250b2ee6018b --- /dev/null +++ b/roofit/roofitZMQ/test/test_polling.cxx @@ -0,0 +1,149 @@ +/* + * Project: RooFit + * Authors: + * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl + * + * Copyright (c) 2016-2019, Netherlands eScience Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ + + +#include "gtest/gtest.h" + +#include // fork, usleep +#include // signal blocking +#include // strsignal() + +#include + +#include "RooFit_ZMQ/ZeroMQSvc.h" +#include "RooFit_ZMQ/ZeroMQPoller.h" + + +static bool terminated = false; + +void handle_sigterm(int signum) { + terminated = true; + std::cout << "handled signal " << strsignal(signum) << " on PID " << getpid() << std::endl; +} + + +TEST(Polling, doublePoll) { + pid_t child_pid {0}; + do { + child_pid = fork(); + } while (child_pid == -1); // retry if fork fails + + if (child_pid > 0) { // master + sigset_t sigmask, sigmask_old; + sigemptyset(&sigmask); + sigaddset(&sigmask, SIGCHLD); + sigprocmask(SIG_BLOCK, &sigmask, &sigmask_old); + +// std::cout << "master PID: " << getpid() << std::endl; + ZmqLingeringSocketPtr<> pusher, puller; + pusher.reset(zmqSvc().socket_ptr(zmq::PUSH)); + pusher->bind("ipc:///tmp/ZMQ_test_fork_polling_M2C.ipc"); + puller.reset(zmqSvc().socket_ptr(zmq::PULL)); + puller->bind("ipc:///tmp/ZMQ_test_fork_polling_C2M.ipc"); + + ZeroMQPoller poller1, poller2; + poller1.register_socket(*puller, zmq::POLLIN); + poller2.register_socket(*puller, zmq::POLLIN); + + // start test + zmqSvc().send(*pusher, std::string("breaker breaker")); + + auto result1a = poller1.poll(-1); + auto result1b = poller1.poll(-1); + auto result2 = poller2.poll(-1); + EXPECT_EQ(result1a.size(), result1b.size()); + EXPECT_EQ(result1a.size(), result2.size()); + + auto receipt = zmqSvc().receive(*puller, ZMQ_DONTWAIT); + + EXPECT_EQ(receipt, 1212); + + kill(child_pid, SIGTERM); + + // socket->close(); // this gives exception of type zmq::error_t: Socket operation on non-socket + pusher.reset(nullptr); + puller.reset(nullptr); + zmqSvc().close_context(); // if you don't close context in parent process as well, the next repeat will hang + + // wait for child + int status = 0; + pid_t pid; + do { + pid = waitpid(child_pid, &status, 0); + } while (-1 == pid && EINTR == errno); // retry on interrupted system call + + if (0 != status) { + if (WIFEXITED(status)) { printf("exited, status=%d\n", WEXITSTATUS(status)); } else if (WIFSIGNALED(status)) { printf("killed by signal %d\n", WTERMSIG(status)); } else if (WIFSTOPPED(status)) { printf("stopped by signal %d\n", WSTOPSIG(status)); } else if (WIFCONTINUED(status)) { printf("continued\n"); } + } + + if (-1 == pid) { + throw std::runtime_error(std::string("waitpid, errno ") + std::to_string(errno)); + } + + sigprocmask(SIG_SETMASK, &sigmask_old, nullptr); + } else { // child +// std::cout << "child PID: " << getpid() << std::endl; + + sigset_t sigmask, sigmask_old; + sigemptyset(&sigmask); + sigaddset(&sigmask, SIGTERM); + sigprocmask(SIG_BLOCK, &sigmask, &sigmask_old); + + struct sigaction sa; + memset (&sa, '\0', sizeof(sa)); + sa.sa_handler = handle_sigterm; + + if (sigaction(SIGTERM, &sa, NULL) < 0) { + std::perror("sigaction failed"); + std::exit(1); + } + + ZmqLingeringSocketPtr<> puller, pusher; + puller.reset(zmqSvc().socket_ptr(zmq::PULL)); + puller->connect("ipc:///tmp/ZMQ_test_fork_polling_M2C.ipc"); + pusher.reset(zmqSvc().socket_ptr(zmq::PUSH)); + pusher->connect("ipc:///tmp/ZMQ_test_fork_polling_C2M.ipc"); + + ZeroMQPoller poller1, poller2; + poller1.register_socket(*puller, zmq::POLLIN); + poller2.register_socket(*puller, zmq::POLLIN); + + // start test + auto result1a = poller1.poll(-1); + auto result1b = poller1.poll(-1); + auto result2 = poller2.poll(-1); + EXPECT_EQ(result1a.size(), result1b.size()); + EXPECT_EQ(result1a.size(), result2.size()); + + auto receipt = zmqSvc().receive(*puller, ZMQ_DONTWAIT); + if (receipt == "breaker breaker") { + zmqSvc().send(*pusher, 1212); + } + // take care, don't just use _exit, it will not cleanly destroy context etc! + // if you really need to, at least close and destroy everything properly + // socket->close(); // this gives exception of type zmq::error_t: Socket operation on non-socket + + sigprocmask(SIG_SETMASK, &sigmask_old, nullptr); + + while (!terminated) {} + +// std::cout << "child terminated" << std::endl; + + puller.reset(nullptr); + pusher.reset(nullptr); + zmqSvc().close_context(); + _Exit(0); + } + +}