RooFit::MultiProcess & TestStatistics part 6: add RooFit::TestStatistics

roofit/roofitcore/CMakeLists.txt

@@ -233,6 +233,19 @@ ROOT_STANDARD_LIBRARY_PACKAGE(RooFitCore
     RooFitLegacy/RooNameSet.h
     RooFitLegacy/RooSetPair.h
     RooFitLegacy/RooTreeData.h
+    TestStatistics/ConstantTermsOptimizer.h
+    TestStatistics/LikelihoodGradientWrapper.h
+    TestStatistics/LikelihoodWrapper.h
+    TestStatistics/LikelihoodSerial.h
+    TestStatistics/MinuitFcnGrad.h
+    TestStatistics/RooAbsL.h
+    TestStatistics/RooBinnedL.h
+    TestStatistics/RooSubsidiaryL.h
+    TestStatistics/RooSumL.h
+    TestStatistics/RooRealL.h
+    TestStatistics/RooUnbinnedL.h
+    TestStatistics/optional_parameter_types.h
+    TestStatistics/buildLikelihood.h
   SOURCES
     src/BidirMMapPipe.cxx
     src/BidirMMapPipe.h
@@ -449,6 +462,19 @@ ROOT_STANDARD_LIBRARY_PACKAGE(RooFitCore
     src/RooFitLegacy/RooMultiCatIter.cxx
     src/RooFitLegacy/RooNameSet.cxx
     src/RooFitLegacy/RooSetPair.cxx
+    src/TestStatistics/ConstantTermsOptimizer.cxx
+    src/TestStatistics/LikelihoodGradientWrapper.cxx
+    src/TestStatistics/LikelihoodWrapper.cxx
+    src/TestStatistics/LikelihoodSerial.cxx
+    src/TestStatistics/MinuitFcnGrad.cxx
+    src/TestStatistics/RooAbsL.cxx
+    src/TestStatistics/RooBinnedL.cxx
+    src/TestStatistics/RooSubsidiaryL.cxx
+    src/TestStatistics/RooSumL.cxx
+    src/TestStatistics/RooRealL.cxx
+    src/TestStatistics/RooUnbinnedL.cxx
+    src/TestStatistics/optional_parameter_types.cxx
+    src/TestStatistics/buildLikelihood.cxx
   DICTIONARY_OPTIONS
     "-writeEmptyRootPCM"
   LIBRARIES

roofit/roofitcore/inc/LinkDef.h

@@ -338,6 +338,7 @@
 #pragma link C++ class RooUnitTest+ ;
 #pragma link C++ class RooMinimizer+ ;
 #pragma link C++ class RooMinimizerFcn+ ;
+#pragma link C++ class RooFit::TestStatistics::RooRealL+ ;
 #pragma link C++ class RooAbsMoment+ ;
 #pragma link C++ class RooMoment+ ;
 #pragma link C++ class RooFirstMoment+ ;

roofit/roofitcore/inc/RooAbsData.h

@@ -47,6 +47,12 @@ class RooFormulaVar;
 namespace RooBatchCompute{
 struct RunContext;
 }
+namespace RooFit {
+namespace TestStatistics {
+class RooAbsL;
+struct ConstantTermsOptimizer;
+}
+}
 
 
 // Writes a templated constructor for compatibility with ROOT builds using the
@@ -321,6 +327,9 @@ class RooAbsData : public TNamed, public RooPrintable {
   friend class RooAbsReal ;
   friend class RooAbsOptTestStatistic ;
   friend class RooAbsCachedPdf ;
+  friend struct RooFit::TestStatistics::ConstantTermsOptimizer;
+  // for access into copied dataset:
+  friend class RooFit::TestStatistics::RooAbsL;
 
   virtual void cacheArgs(const RooAbsArg* owner, RooArgSet& varSet, const RooArgSet* nset=0, Bool_t skipZeroWeights=kFALSE) ;
   virtual void resetCache() ;

roofit/roofitcore/inc/RooDataHist.h

@@ -35,6 +35,11 @@ class RooAbsArg;
 class RooCategory ;
 class RooPlot;
 class RooAbsLValue ;
+namespace RooFit {
+namespace TestStatistics {
+class RooAbsL;
+}
+}
 
 class RooDataHist : public RooAbsData, public RooDirItem {
 public:
@@ -222,6 +227,8 @@ class RooDataHist : public RooAbsData, public RooDirItem {
   friend class RooAbsCachedPdf ;
   friend class RooAbsCachedReal ;
   friend class RooDataHistSliceIter ;
+  // for access into copied dataset:
+  friend class RooFit::TestStatistics::RooAbsL;
 
   std::size_t calcTreeIndex(const RooAbsCollection& coords, bool fast) const;
   /// Legacy overload to calculate the tree index from the current value of `_vars`.

roofit/roofitcore/inc/RooMinimizer.h

@@ -29,8 +29,8 @@
 
 #include "RooArgList.h" // cannot just use forward decl due to default argument in lastMinuitFit
 
-#include "RooMinimizerFcn.h"
-#include "RooGradMinimizerFcn.h"
+#include "TestStatistics/RooAbsL.h"
+#include "TestStatistics/MinuitFcnGrad.h"
 
 #include "RooSentinel.h"
 #include "RooMsgService.h"
@@ -45,13 +45,23 @@ class RooRealVar ;
 class RooArgSet ;
 class TH2F ;
 class RooPlot ;
+namespace RooFit {
+namespace TestStatistics {
+class LikelihoodSerial;
+class LikelihoodGradientSerial;
+}
+} // namespace RooFit
 
 class RooMinimizer : public TObject {
 public:
-  enum class FcnMode { classic, gradient };
+  enum class FcnMode { classic, gradient, generic_wrapper };
 
   explicit RooMinimizer(RooAbsReal &function, FcnMode fcnMode = FcnMode::classic);
   static std::unique_ptr<RooMinimizer> create(RooAbsReal &function, FcnMode fcnMode = FcnMode::classic);
+  template <typename LikelihoodWrapperT = RooFit::TestStatistics::LikelihoodSerial,
+            typename LikelihoodGradientWrapperT = RooFit::TestStatistics::LikelihoodGradientSerial>
+  static std::unique_ptr<RooMinimizer> create(std::shared_ptr<RooFit::TestStatistics::RooAbsL> likelihood);
+
   ~RooMinimizer() override;
 
   enum Strategy { Speed=0, Balance=1, Robustness=2 } ;
@@ -126,6 +136,11 @@ class RooMinimizer : public TObject {
   bool fitFcn() const;
 
 private:
+  template <typename LikelihoodWrapperT = RooFit::TestStatistics::LikelihoodSerial, typename LikelihoodGradientWrapperT = RooFit::TestStatistics::LikelihoodGradientSerial>
+  RooMinimizer(std::shared_ptr<RooFit::TestStatistics::RooAbsL> likelihood,
+               LikelihoodWrapperT* /* used only for template deduction */ = static_cast<RooFit::TestStatistics::LikelihoodSerial*>(nullptr),
+               LikelihoodGradientWrapperT* /* used only for template deduction */ = static_cast<RooFit::TestStatistics::LikelihoodGradientSerial*>(nullptr));
+
   Int_t _printLevel = 1;
   Int_t _status = -99;
   Bool_t _profile = kFALSE;
@@ -150,4 +165,48 @@ class RooMinimizer : public TObject {
   ClassDefOverride(RooMinimizer,0) // RooFit interface to ROOT::Fit::Fitter
 } ;
 
+
+template <typename LikelihoodWrapperT, typename LikelihoodGradientWrapperT>
+RooMinimizer::RooMinimizer(std::shared_ptr<RooFit::TestStatistics::RooAbsL> likelihood, LikelihoodWrapperT* /* value unused */,
+                           LikelihoodGradientWrapperT* /* value unused */) :
+   _fcnMode(FcnMode::generic_wrapper)
+{
+   RooSentinel::activate();
+
+   if (_theFitter)
+      delete _theFitter;
+   _theFitter = new ROOT::Fit::Fitter;
+   _theFitter->Config().SetMinimizer(_minimizerType.c_str());
+   setEps(1.0); // default tolerance
+
+   _fcn = RooFit::TestStatistics::MinuitFcnGrad::create<LikelihoodWrapperT, LikelihoodGradientWrapperT>(likelihood, this, _theFitter->Config().ParamsSettings(), _verbose);
+
+   // default max number of calls
+   _theFitter->Config().MinimizerOptions().SetMaxIterations(500 * _fcn->getNDim());
+   _theFitter->Config().MinimizerOptions().SetMaxFunctionCalls(500 * _fcn->getNDim());
+
+   // Shut up for now
+   setPrintLevel(-1);
+
+   // Use +0.5 for 1-sigma errors
+   setErrorLevel(likelihood->defaultErrorLevel());
+
+   // Declare our parameters to MINUIT
+   _fcn->Synchronize(_theFitter->Config().ParamsSettings(), _fcn->getOptConst(), _verbose);
+
+   // Now set default verbosity
+   if (RooMsgService::instance().silentMode()) {
+      setPrintLevel(-1);
+   } else {
+      setPrintLevel(1);
+   }
+}
+
+// static function
+template <typename LikelihoodWrapperT, typename LikelihoodGradientWrapperT>
+std::unique_ptr<RooMinimizer> RooMinimizer::create(std::shared_ptr<RooFit::TestStatistics::RooAbsL> likelihood) {
+   return std::make_unique<RooMinimizer>(likelihood, static_cast<LikelihoodWrapperT*>(nullptr),
+                                         static_cast<LikelihoodGradientWrapperT*>(nullptr));
+}
+
 #endif

roofit/roofitcore/inc/RooNLLVar.h

@@ -62,6 +62,14 @@ class RooNLLVar : public RooAbsOptTestStatistic {
 
   using ComputeResult = std::pair<ROOT::Math::KahanSum<double>, double>;
 
+  static RooNLLVar::ComputeResult computeBatchedFunc(const RooAbsPdf *pdfClone, RooAbsData *dataClone,
+                                                     std::unique_ptr<RooBatchCompute::RunContext> &evalData,
+                                                 RooArgSet *normSet, bool weightSq, std::size_t stepSize,
+                                                 std::size_t firstEvent, std::size_t lastEvent);
+  static RooNLLVar::ComputeResult computeScalarFunc(const RooAbsPdf *pdfClone, RooAbsData *dataClone, RooArgSet *normSet,
+                                                bool weightSq, std::size_t stepSize, std::size_t firstEvent,
+                                                std::size_t lastEvent);
+
 protected:
 
   virtual Bool_t processEmptyDataSets() const { return _extended ; }

roofit/roofitcore/inc/TestStatistics/ConstantTermsOptimizer.h

@@ -0,0 +1,40 @@
+// Author: Patrick Bos, Netherlands eScience Center / NIKHEF 2021
+
+/*****************************************************************************
+ * RooFit
+ * Authors:                                                                  *
+ *   WV, Wouter Verkerke, UC Santa Barbara, verkerke@slac.stanford.edu       *
+ *   DK, David Kirkby,    UC Irvine,         dkirkby@uci.edu                 *
+ *                                                                           *
+ * Copyright (c) 2000-2021, Regents of the University of California          *
+ *                          and Stanford University. All rights reserved.    *
+ *                                                                           *
+ * Redistribution and use in source and binary forms,                        *
+ * with or without modification, are permitted according to the terms        *
+ * listed in LICENSE (http://roofit.sourceforge.net/license.txt)             *
+ *****************************************************************************/
+
+#ifndef ROOT_ROOFIT_TESTSTATISTICS_ConstantTermsOptimizer
+#define ROOT_ROOFIT_TESTSTATISTICS_ConstantTermsOptimizer
+
+// forward declarations
+class RooAbsReal;
+class RooArgSet;
+class RooAbsData;
+
+namespace RooFit {
+namespace TestStatistics {
+
+// this is a class only for convenience: it saves multiple friend definitions in RooAbsData for otherwise free functions
+struct ConstantTermsOptimizer {
+   static void enableConstantTermsOptimization(RooAbsReal *function, RooArgSet *norm_set, RooAbsData *dataset,
+                                           bool applyTrackingOpt);
+   static void optimizeCaching(RooAbsReal *function, RooArgSet *norm_set, RooArgSet* observables, RooAbsData *dataset);
+   static void disableConstantTermsOptimization(RooAbsReal *function, RooArgSet *norm_set, RooArgSet* observables, RooAbsData *dataset);
+   static RooArgSet requiredExtraObservables();
+};
+
+}
+}
+
+#endif // ROOT_ROOFIT_TESTSTATISTICS_ConstantTermsOptimizer

roofit/roofitcore/inc/TestStatistics/LikelihoodGradientWrapper.h

@@ -0,0 +1,73 @@
+// Author: Patrick Bos, Netherlands eScience Center / NIKHEF 2021
+
+/*****************************************************************************
+ * RooFit
+ * Authors:                                                                  *
+ *   WV, Wouter Verkerke, UC Santa Barbara, verkerke@slac.stanford.edu       *
+ *   DK, David Kirkby,    UC Irvine,         dkirkby@uci.edu                 *
+ *                                                                           *
+ * Copyright (c) 2000-2021, Regents of the University of California          *
+ *                          and Stanford University. All rights reserved.    *
+ *                                                                           *
+ * Redistribution and use in source and binary forms,                        *
+ * with or without modification, are permitted according to the terms        *
+ * listed in LICENSE (http://roofit.sourceforge.net/license.txt)             *
+ *****************************************************************************/
+
+#ifndef ROOT_ROOFIT_TESTSTATISTICS_LikelihoodGradientWrapper
+#define ROOT_ROOFIT_TESTSTATISTICS_LikelihoodGradientWrapper
+
+#include <Fit/ParameterSettings.h>
+#include <Math/IFunctionfwd.h>
+#include "Math/MinimizerOptions.h"
+
+#include <vector>
+#include <memory> // shared_ptr
+
+// forward declaration
+class RooMinimizer;
+
+namespace RooFit {
+namespace TestStatistics {
+
+// forward declaration
+class RooAbsL;
+struct WrapperCalculationCleanFlags;
+
+class LikelihoodGradientWrapper {
+public:
+   LikelihoodGradientWrapper(std::shared_ptr<RooAbsL> likelihood, std::shared_ptr<WrapperCalculationCleanFlags> calculation_is_clean, std::size_t N_dim, RooMinimizer* minimizer);
+   virtual ~LikelihoodGradientWrapper() = default;
+   virtual LikelihoodGradientWrapper* clone() const = 0;
+
+   virtual void fillGradient(double *grad) = 0;
+
+   /// Synchronize minimizer settings with calculators in child classes.
+   virtual void synchronizeWithMinimizer(const ROOT::Math::MinimizerOptions &options);
+   virtual void synchronizeParameterSettings(const std::vector<ROOT::Fit::ParameterSettings> &parameter_settings);
+   virtual void synchronizeParameterSettings(ROOT::Math::IMultiGenFunction* function, const std::vector<ROOT::Fit::ParameterSettings> &parameter_settings) = 0;
+   /// Minuit passes in parameter values that may not conform to RooFit internal standards (like applying range clipping),
+   /// but that the specific calculator does need. This function can be implemented to receive these Minuit-internal values.
+   virtual void updateMinuitInternalParameterValues(const std::vector<double>& minuit_internal_x);
+   virtual void updateMinuitExternalParameterValues(const std::vector<double>& minuit_external_x);
+
+   /// \brief Implement usesMinuitInternalValues to return true when you want Minuit to send this class Minuit-internal values, or return false when you want "regular" Minuit-external values.
+   ///
+   /// Minuit internally uses a transformed parameter space to graciously handle externally mandated parameter range
+   /// boundaries. Transformation from Minuit-internal to external (i.e. "regular") parameters is done using
+   /// trigonometric functions that in some cases can cause a few bits of precision loss with respect to the original
+   /// parameter values. To circumvent this, Minuit also allows external gradient providers (like LikelihoodGradientWrapper)
+   /// to take the Minuit-internal parameter values directly, without transformation. This way, the gradient provider
+   /// (e.g. the implementation of this class) can handle transformation manually, possibly with higher precision.
+   virtual bool usesMinuitInternalValues() = 0;
+
+protected:
+   std::shared_ptr<RooAbsL> likelihood_;
+   RooMinimizer * minimizer_;
+   std::shared_ptr<WrapperCalculationCleanFlags> calculation_is_clean_;
+};
+
+} // namespace TestStatistics
+} // namespace RooFit
+
+#endif // ROOT_ROOFIT_TESTSTATISTICS_LikelihoodGradientWrapper

roofit/roofitcore/inc/TestStatistics/LikelihoodSerial.h

@@ -0,0 +1,52 @@
+// Author: Patrick Bos, Netherlands eScience Center / NIKHEF 2021
+
+/*****************************************************************************
+ * RooFit
+ * Authors:                                                                  *
+ *   WV, Wouter Verkerke, UC Santa Barbara, verkerke@slac.stanford.edu       *
+ *   DK, David Kirkby,    UC Irvine,         dkirkby@uci.edu                 *
+ *                                                                           *
+ * Copyright (c) 2000-2021, Regents of the University of California          *
+ *                          and Stanford University. All rights reserved.    *
+ *                                                                           *
+ * Redistribution and use in source and binary forms,                        *
+ * with or without modification, are permitted according to the terms        *
+ * listed in LICENSE (http://roofit.sourceforge.net/license.txt)             *
+ *****************************************************************************/
+
+#ifndef ROOT_ROOFIT_LikelihoodSerial
+#define ROOT_ROOFIT_LikelihoodSerial
+
+#include <TestStatistics/LikelihoodWrapper.h>
+#include "RooArgList.h"
+
+#include "Math/MinimizerOptions.h"
+
+#include <map>
+
+namespace RooFit {
+namespace TestStatistics {
+
+class LikelihoodSerial : public LikelihoodWrapper {
+public:
+   LikelihoodSerial(std::shared_ptr<RooAbsL> likelihood, std::shared_ptr<WrapperCalculationCleanFlags> calculation_is_clean);
+   inline LikelihoodSerial *clone() const override { return new LikelihoodSerial(*this); }
+
+   void initVars();
+
+   void evaluate() override;
+   inline ROOT::Math::KahanSum<double> getResult() const override { return result; }
+
+private:
+   ROOT::Math::KahanSum<double> result;
+
+   RooArgList _vars;      // Variables
+   RooArgList _saveVars;  // Copy of variables
+
+   LikelihoodType likelihood_type;
+};
+
+}
+}
+
+#endif // ROOT_ROOFIT_LikelihoodSerial