RooFit::MultiProcess & TestStatistics part 2 redo: RooFitZMQ & MultiProcess

.ci/copy_headers.sh

@@ -10,6 +10,6 @@ cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -Dall=On -Dtesting=On -Dx11=Off -Dalien
 # We need to prebuild a minimal set of targets which are responsible for header copy
 # or generation.
 make -j4 move_headers intrinsics_gen clang-tablegen-targets ClangDriverOptions \
-         googletest Dictgen BaseTROOT
+         googletest Dictgen BaseTROOT BUILTIN_cppzmq
 ln -s $PWD/compile_commands.json $PWD/../root/
 

builtins/zeromq/cppzmq/CMakeLists.txt

@@ -0,0 +1,35 @@
+include(ExternalProject)
+
+ExternalProject_Add(BUILTIN_cppzmq
+    URL https://github.com/zeromq/cppzmq/archive/refs/tags/v4.8.1.tar.gz
+    URL_HASH SHA256=7a23639a45f3a0049e11a188e29aaedd10b2f4845f0000cf3e22d6774ebde0af
+    CONFIGURE_COMMAND ""
+    BUILD_COMMAND ""
+    INSTALL_COMMAND ""
+    DEPENDS libzmq
+)
+
+# manually copy header (inspired by nlohmann builtin)
+set(cppzmq_HEADER_PATH ${CMAKE_CURRENT_BINARY_DIR}/BUILTIN_cppzmq-prefix/src/BUILTIN_cppzmq)
+add_custom_command(
+        OUTPUT ${CMAKE_BINARY_DIR}/include/zmq.hpp
+        COMMAND ${CMAKE_COMMAND} -E copy ${cppzmq_HEADER_PATH}/zmq.hpp ${CMAKE_BINARY_DIR}/include/zmq.hpp
+        COMMENT "Copying zmq.hpp header to ${CMAKE_BINARY_DIR}/include"
+        DEPENDS BUILTIN_cppzmq)
+
+add_custom_target(builtin_cppzmq_incl
+        DEPENDS ${CMAKE_BINARY_DIR}/include/zmq.hpp)
+
+set_property(GLOBAL APPEND PROPERTY ROOT_HEADER_TARGETS builtin_cppzmq_incl)
+
+install(FILES ${cppzmq_HEADER_PATH}/zmq.hpp
+        DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/)
+
+# find_package emulation
+set(cppzmq_FOUND TRUE CACHE BOOL "" FORCE)
+set(cppzmq_INCLUDE_DIR ${CMAKE_BINARY_DIR}/include CACHE INTERNAL "" FORCE)
+set(cppzmq_INCLUDE_DIRS ${cppzmq_INCLUDE_DIR} CACHE INTERNAL "" FORCE)
+
+add_library(cppzmq INTERFACE IMPORTED GLOBAL)
+target_include_directories(cppzmq INTERFACE $<BUILD_INTERFACE:${cppzmq_INCLUDE_DIR}>)
+add_dependencies(cppzmq BUILTIN_cppzmq builtin_cppzmq_incl)

builtins/zeromq/libzmq/CMakeLists.txt

@@ -0,0 +1,47 @@
+include(ExternalProject)
+
+set(ZeroMQ_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/BUILTIN_ZeroMQ-prefix)
+set(ZeroMQ_LIBNAME ${CMAKE_STATIC_LIBRARY_PREFIX}zmq${CMAKE_STATIC_LIBRARY_SUFFIX})
+
+set(ZeroMQ_LIBRARY ${ZeroMQ_PREFIX}/lib/${ZeroMQ_LIBNAME} CACHE INTERNAL "" FORCE)
+set(ZeroMQ_LIBRARIES ${ZeroMQ_LIBRARY} CACHE INTERNAL "" FORCE)
+
+if(NOT WIN32)
+    # 2021-10-07: libzmq's minimum CMake version does not yet support CXX_VISIBILITY_PRESET, so we need to pass the flag manually
+    set(cxx_visibility_flag "-fvisibility=hidden")
+else()
+    set(cxx_visibility_flag "")
+endif()
+
+ExternalProject_Add(BUILTIN_ZeroMQ
+    URL https://github.com/zeromq/libzmq/archive/7c2df78b49a3aa63e654b3f3526adf71ed091534.tar.gz
+    URL_HASH SHA256=fcc1b0648afa5d92e0ff0e6e93beb28cbbe008a5f98c228ff97144ba6e4a6c3e
+    CMAKE_ARGS
+        -DCMAKE_INSTALL_PREFIX=<INSTALL_DIR>
+        -DCMAKE_INSTALL_LIBDIR=lib
+        -DWITH_PERF_TOOL=OFF
+        -DBUILD_TESTS=OFF
+        -DENABLE_DRAFTS=ON
+        -DENABLE_NO_EXPORT=ON
+        -DENABLE_CURVE=OFF
+        -DWITH_DOCS=OFF
+        -DENABLE_WS=OFF
+        -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
+        -DCMAKE_CXX_FLAGS=${cxx_visibility_flag}\ ${CMAKE_CXX_FLAGS}\ -w
+    BUILD_BYPRODUCTS ${ZeroMQ_LIBRARIES}
+    )
+
+set(ZeroMQ_FOUND TRUE CACHE BOOL "" FORCE)
+
+set(ZeroMQ_INCLUDE_DIR  ${ZeroMQ_PREFIX}/include CACHE INTERNAL "" FORCE)
+set(ZeroMQ_INCLUDE_DIRS ${ZeroMQ_PREFIX}/include CACHE INTERNAL "" FORCE)
+
+# Workaround to propagate INTERFACE_INCLUDE_DIRECTORIES, see https://stackoverflow.com/a/47358004/1199693
+file(MAKE_DIRECTORY ${ZeroMQ_INCLUDE_DIR})
+
+add_library(libzmq INTERFACE IMPORTED GLOBAL)
+target_include_directories(libzmq INTERFACE $<BUILD_INTERFACE:${ZeroMQ_INCLUDE_DIR}>)
+target_link_libraries(libzmq INTERFACE $<BUILD_INTERFACE:${ZeroMQ_LIBRARIES}>)
+add_dependencies(libzmq BUILTIN_ZeroMQ)
+
+set(ZeroMQ_DIR ${ZeroMQ_PREFIX}/share/cmake/ZeroMQ CACHE INTERNAL "" FORCE)

cmake/modules/FindZeroMQ.cmake

@@ -0,0 +1,60 @@
+# Based on https://github.com/zeromq/cppzmq/blob/a98fa4a91d868a3844e5456741d6782cc1a8d98b/libzmq-pkg-config/FindZeroMQ.cmake
+# MIT Licensed:
+#    Permission is hereby granted, free of charge, to any person obtaining a copy
+#    of this software and associated documentation files (the "Software"), to
+#    deal in the Software without restriction, including without limitation the
+#    rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+#    sell copies of the Software, and to permit persons to whom the Software is
+#    furnished to do so, subject to the following conditions:
+#
+#    The above copyright notice and this permission notice shall be included in
+#    all copies or substantial portions of the Software.
+#
+#    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+#    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+#    IN THE SOFTWARE.
+
+set(PKG_CONFIG_USE_CMAKE_PREFIX_PATH ON)
+find_package(PkgConfig)
+pkg_check_modules(PC_LIBZMQ QUIET libzmq)
+
+set(ZeroMQ_VERSION ${PC_LIBZMQ_VERSION})
+
+find_path(ZeroMQ_INCLUDE_DIR zmq.h
+        PATHS ${ZeroMQ_DIR}/include
+        ${PC_LIBZMQ_INCLUDE_DIRS})
+
+find_library(ZeroMQ_LIBRARY
+        NAMES zmq
+        PATHS ${ZeroMQ_DIR}/lib
+        ${PC_LIBZMQ_LIBDIR}
+        ${PC_LIBZMQ_LIBRARY_DIRS})
+
+set ( ZeroMQ_LIBRARIES ${ZeroMQ_LIBRARY} )
+set ( ZeroMQ_INCLUDE_DIRS ${ZeroMQ_INCLUDE_DIR} )
+
+# check for zmq_ppoll
+if(ZeroMQ_FOUND)
+    SET(SAVE_CMAKE_REQUIRED_DEFINITIONS "${CMAKE_REQUIRED_DEFINITIONS}")
+    SET(CMAKE_REQUIRED_DEFINITIONS "-DZMQ_BUILD_DRAFT_API")
+    check_cxx_symbol_exists(zmq_ppoll zmq.h ZeroMQ_HAS_PPOLL)
+    SET(CMAKE_REQUIRED_DEFINITIONS "${SAVE_CMAKE_REQUIRED_DEFINITIONS}")
+endif()
+
+include ( FindPackageHandleStandardArgs )
+# handle the QUIETLY and REQUIRED arguments and set ZeroMQ_FOUND to TRUE
+# if all listed variables are TRUE
+find_package_handle_standard_args ( ZeroMQ DEFAULT_MSG ZeroMQ_LIBRARIES ZeroMQ_INCLUDE_DIRS ZeroMQ_HAS_PPOLL)
+
+if(ZeroMQ_FOUND)
+    if(NOT TARGET libzmq)
+        add_library(libzmq UNKNOWN IMPORTED)
+        set_target_properties(libzmq PROPERTIES
+                IMPORTED_LOCATION ${ZeroMQ_LIBRARIES}
+                INTERFACE_INCLUDE_DIRECTORIES ${ZeroMQ_INCLUDE_DIRS})
+    endif()
+endif()

cmake/modules/Findcppzmq.cmake

@@ -0,0 +1,21 @@
+if (NOT (ZeroMQ_FOUND OR TARGET libzmq))
+    message(FATAL_ERROR "Search for libzmq first!")
+endif()
+
+find_path(cppzmq_INCLUDE_DIRS "zmq.hpp"
+        HINTS "${ZeroMQ_INCLUDE_DIR}"
+        PATH_SUFFIXES "include" "cppzmq"
+        )
+mark_as_advanced(cppzmq_INCLUDE_DIRS)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args (cppzmq DEFAULT_MSG cppzmq_INCLUDE_DIRS)
+
+if(cppzmq_FOUND)
+    add_library(cppzmq INTERFACE IMPORTED)
+    set_target_properties(cppzmq PROPERTIES
+            INTERFACE_INCLUDE_DIRECTORIES "${CPPZMQ_INCLUDE_DIRS}"
+            INTERFACE_LINK_LIBRARIES libzmq
+            )
+    set(CPPZMQ_LIBRARIES cppzmq)
+endif()

cmake/modules/RootBuildOptions.cmake

@@ -86,6 +86,7 @@ ROOT_BUILD_OPTION(builtin_cfitsio OFF "Build CFITSIO internally (requires networ
 ROOT_BUILD_OPTION(builtin_clang ON "Build bundled copy of Clang")
 ROOT_BUILD_OPTION(builtin_cling ON "Build bundled copy of Cling. Only build with an external cling if you know what you are doing: associating ROOT commits with cling commits is tricky.")
 MARK_AS_ADVANCED(builtin_cling)
+ROOT_BUILD_OPTION(builtin_cppzmq OFF "Use ZeroMQ C++ bindings installed by ROOT (requires network)")
 ROOT_BUILD_OPTION(builtin_davix OFF "Build Davix internally (requires network)")
 ROOT_BUILD_OPTION(builtin_fftw3 OFF "Build FFTW3 internally (requires network)")
 ROOT_BUILD_OPTION(builtin_freetype OFF "Build bundled copy of freetype")
@@ -107,6 +108,7 @@ ROOT_BUILD_OPTION(builtin_vdt OFF "Build VDT internally (requires network)")
 ROOT_BUILD_OPTION(builtin_veccore OFF "Build VecCore internally (requires network)")
 ROOT_BUILD_OPTION(builtin_xrootd OFF "Build XRootD internally (requires network)")
 ROOT_BUILD_OPTION(builtin_xxhash OFF "Build bundled copy of xxHash")
+ROOT_BUILD_OPTION(builtin_zeromq OFF "Build ZeroMQ internally (requires network)")
 ROOT_BUILD_OPTION(builtin_zlib OFF "Build bundled copy of zlib")
 ROOT_BUILD_OPTION(builtin_zstd OFF "Build included libzstd, or use system libzstd")
 ROOT_BUILD_OPTION(ccache OFF "Enable ccache usage for speeding up builds")
@@ -160,6 +162,7 @@ ROOT_BUILD_OPTION(qt5web OFF "Enable support for Qt5 web-based display (requires
 ROOT_BUILD_OPTION(qt6web OFF "Enable support for Qt6 web-based display (requires Qt6::WebEngineCore and Qt6::WebEngineWidgets)")
 ROOT_BUILD_OPTION(r OFF "Enable support for R bindings (requires R, Rcpp, and RInside)")
 ROOT_BUILD_OPTION(roofit ON "Build the advanced fitting package RooFit, and RooStats for statistical tests. If xml is available, also build HistFactory.")
+ROOT_BUILD_OPTION(roofit_multiprocess ON "Build RooFit::MultiProcess and multi-process RooFit::TestStatistics classes (requires ZeroMQ with zmq_ppoll and cppzmq).")
 ROOT_BUILD_OPTION(webgui ON "Build Web-based UI components of ROOT (requires C++17 standard or higher)")
 ROOT_BUILD_OPTION(root7 ON "Build ROOT 7 components of ROOT (requires C++17 standard or higher)")
 ROOT_BUILD_OPTION(rpath ON "Link libraries with built-in RPATH (run-time search path)")
@@ -253,6 +256,7 @@ if(all)
  set(qt5web_defvalue ON)
  set(r_defvalue ON)
  set(roofit_defvalue ON)
+ set(roofit_multiprocess_defvalue ON)
  set(webgui_defvalue ON)
  set(root7_defvalue ON)
  set(shadowpw_defvalue ON)
@@ -280,6 +284,7 @@ if(builtin_all)
   set(builtin_cfitsio_defvalue ON)
   set(builtin_clang_defvalue ON)
   set(builtin_cling_defvalue ON)
+  set(builtin_cppzmq_defvalue ON)
   set(builtin_davix_defvalue ON)
   set(builtin_fftw3_defvalue ON)
   set(builtin_freetype_defvalue ON)
@@ -301,6 +306,7 @@ if(builtin_all)
   set(builtin_veccore_defvalue ON)
   set(builtin_xrootd_defvalue ON)
   set(builtin_xxhash_defvalue ON)
+  set(builtin_zeromq_defvalue ON)
   set(builtin_zlib_defvalue ON)
   set(builtin_zstd_defvalue ON)
 endif()

cmake/modules/SearchInstalledSoftware.cmake

@@ -1862,6 +1862,75 @@ if(testing AND NO_CONNECTION)
   endif()
 endif()
 
+#---Check for ZeroMQ when building RooFit::MultiProcess--------------------------------------------
+
+if (roofit_multiprocess AND NOT MSVC)
+  if(NOT builtin_zeromq)
+    message(STATUS "Looking for ZeroMQ (libzmq)")
+    # Clear cache before calling find_package(ZeroMQ),
+    # necessary to be able to toggle builtin_zeromq and
+    # not have find_package(ZeroMQ) find builtin ZeroMQ.
+    foreach(suffix FOUND INCLUDE_DIR INCLUDE_DIRS LIBRARY LIBRARIES)
+      unset(ZeroMQ_${suffix} CACHE)
+    endforeach()
+
+    # Temporarily prefer config mode over module mode, so that a CMake-installed system version
+    # gets detected before looking for an autotools-installed system version (which the
+    # FindZeroMQ.cmake module does).
+    set(CMAKE_FIND_PACKAGE_PREFER_CONFIG_ORIGINAL_VALUE ${CMAKE_FIND_PACKAGE_PREFER_CONFIG})
+    set(CMAKE_FIND_PACKAGE_PREFER_CONFIG TRUE)
+
+    if(fail-on-missing)
+      find_package(ZeroMQ REQUIRED)
+    else()
+      find_package(ZeroMQ)
+      if(NOT ZeroMQ_FOUND)
+        message(STATUS "ZeroMQ not found. Switching on builtin_zeromq option")
+        set(builtin_zeromq ON CACHE BOOL "Enabled because ZeroMQ not found (${builtin_zeromq_description})" FORCE)
+      endif()
+    endif()
+
+    # Reset default find_package mode
+    set(CMAKE_FIND_PACKAGE_PREFER_CONFIG ${CMAKE_FIND_PACKAGE_PREFER_CONFIG_ORIGINAL_VALUE})
+    unset(CMAKE_FIND_PACKAGE_PREFER_CONFIG_ORIGINAL_VALUE)
+  endif()
+
+  if(builtin_zeromq)
+    list(APPEND ROOT_BUILTINS ZeroMQ)
+    add_subdirectory(builtins/zeromq/libzmq)
+  endif()
+
+  if(NOT builtin_cppzmq)
+    message(STATUS "Looking for ZeroMQ C++ bindings (cppzmq)")
+    # Clear cache before calling find_package(cppzmq),
+    # necessary to be able to toggle builtin_cppzmq and
+    # not have find_package(cppzmq) find builtin cppzmq.
+    foreach(suffix FOUND INCLUDE_DIR INCLUDE_DIRS)
+      unset(cppzmq_${suffix} CACHE)
+    endforeach()
+    if(fail-on-missing)
+      find_package(cppzmq REQUIRED)
+    else()
+      find_package(cppzmq QUIET)
+      if(NOT cppzmq_FOUND)
+        message(STATUS "ZeroMQ C++ bindings not found. Switching on builtin_cppzmq option")
+        set(builtin_cppzmq ON CACHE BOOL "Enabled because ZeroMQ C++ bindings not found (${builtin_cppzmq_description})" FORCE)
+      endif()
+    endif()
+  endif()
+
+  if(builtin_cppzmq)
+    list(APPEND ROOT_BUILTINS cppzmq)
+    add_subdirectory(builtins/zeromq/cppzmq)
+  endif()
+
+  # zmq_ppoll is still in the draft API, so enable that transitively
+  target_compile_definitions(libzmq INTERFACE ZMQ_BUILD_DRAFT_API)
+  target_compile_definitions(libzmq INTERFACE ZMQ_NO_EXPORT)
+  target_compile_definitions(cppzmq INTERFACE ZMQ_BUILD_DRAFT_API)
+  target_compile_definitions(cppzmq INTERFACE ZMQ_NO_EXPORT)
+endif (roofit_multiprocess AND NOT MSVC)
+
 #---Download googletest--------------------------------------------------------------
 if (testing)
   # FIXME: Remove our version of gtest in roottest. We can reuse this one.

roofit/CMakeLists.txt

@@ -5,6 +5,10 @@
 # For the list of contributors see $ROOTSYS/README/CREDITS.
 
 add_subdirectory(batchcompute)
+if (roofit_multiprocess AND NOT MSVC)
+  add_subdirectory(roofitZMQ)
+  add_subdirectory(multiprocess)
+endif(roofit_multiprocess AND NOT MSVC)
 add_subdirectory(roofitcore)
 add_subdirectory(roofit)
 if(mathmore)

roofit/multiprocess/CMakeLists.txt

@@ -0,0 +1,22 @@
+############################################################################
+# CMakeLists.txt file for building ROOT RooFitMultiProcess package
+# @author Patrick Bos, Netherlands eScience Center
+############################################################################
+
+ROOT_LINKER_LIBRARY(RooFitMultiProcess
+        src/worker.cxx
+        src/Messenger.cxx
+        src/ProcessManager.cxx
+        src/util.cxx
+        src/Queue.cxx
+        src/JobManager.cxx
+        src/Job.cxx
+    DEPENDENCIES
+        RooFitZMQ
+)
+
+target_link_libraries(RooFitMultiProcess PUBLIC RooFitZMQ)
+set(RooFitMultiProcess_INCLUDE_DIRS ${CMAKE_CURRENT_SOURCE_DIR}/res ${RooFitZMQ_INCLUDE_DIRS})
+target_include_directories(RooFitMultiProcess PRIVATE ${RooFitMultiProcess_INCLUDE_DIRS})
+
+ROOT_ADD_TEST_SUBDIRECTORY(test)

roofit/multiprocess/res/RooFit/MultiProcess/Job.h

@@ -0,0 +1,56 @@
+/*
+ * Project: RooFit
+ * Authors:
+ *   PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl
+ *   IP, Inti Pelupessy, Netherlands eScience Center, i.pelupessy@esciencecenter.nl
+ *
+ * Copyright (c) 2021, CERN
+ *
+ * Redistribution and use in source and binary forms,
+ * with or without modification, are permitted according to the terms
+ * listed in LICENSE (http://roofit.sourceforge.net/license.txt)
+ */
+#ifndef ROOT_ROOFIT_MultiProcess_Job_decl
+#define ROOT_ROOFIT_MultiProcess_Job_decl
+
+#include <string>
+#include <zmq.hpp>
+
+namespace RooFit {
+namespace MultiProcess {
+
+// forward declaration
+class JobManager;
+
+class Job {
+public:
+   explicit Job();
+   Job(const Job &other);
+
+   ~Job();
+
+   virtual void evaluate_task(std::size_t task) = 0;
+   virtual void update_state();
+
+   virtual void send_back_task_result_from_worker(std::size_t task) = 0;
+   virtual bool receive_task_result_on_master(const zmq::message_t &message) = 0;
+
+   void gather_worker_results();
+
+   std::size_t get_state_id();
+
+protected:
+   JobManager *get_manager();
+
+   std::size_t id_;
+   std::size_t state_id_ = 0;
+
+private:
+   // do not use _manager directly, it must first be initialized! use get_manager()
+   JobManager *_manager = nullptr;
+};
+
+} // namespace MultiProcess
+} // namespace RooFit
+
+#endif // ROOT_ROOFIT_MultiProcess_Job_decl