Heterogeneous MadGraph: parallel CPU+GPU executions

epoch1/cuda/ee_mumu/SubProcesses/Makefile

@@ -39,6 +39,7 @@ ifneq ($(wildcard $(CUDA_HOME)/bin/nvcc),)
   ###CUFLAGS+= --maxrregcount 64 # degrades throughput: 1.7E8 (16384 32 12) flat at 1.7E8 (65536 128 12)
   cu_main     = gcheck.exe
   cu_objects  = gCPPProcess.o
+  het_main    = hcheck.exe
 else
   # No cuda. Switch cuda compilation off and go to common random numbers in C++
   NVCC       := $(warning CUDA_HOME is not set or is invalid. Export CUDA_HOME to compile with cuda)
@@ -62,7 +63,7 @@ ifeq ($(UNAME_P),ppc64le)
     CUFLAGS+= -Xcompiler -mno-float128
 endif
 
-all: ../../src $(cu_main) $(cxx_main) runTest.exe
+all: ../../src $(cu_main) $(cxx_main) $(het_main) runTest.exe
 
 debug: OPTFLAGS   = -g -O0 -DDEBUG2
 debug: CUOPTFLAGS = -G
@@ -81,11 +82,17 @@ gcheck.o: gcheck.cu *.h ../../src/*.h ../../src/*.cu
 %.o : %.cc *.h ../../src/*.h
 	$(CXX) $(CPPFLAGS) $(CXXFLAGS) $(CUINC) -c $< -o $@
 
-$(cu_main): gcheck.o $(LIBDIR)/lib$(MODELLIB).a $(cu_objects)
-	$(NVCC) $< -o $@ $(cu_objects) $(CUARCHFLAGS) $(LIBFLAGS) $(CULIBFLAGS)
+# This is built with nvcc and linked with objects compiled with nvcc or g++
+$(cu_main): gcheck.o $(LIBDIR)/lib$(MODELLIB).a $(cu_objects) gmain.cc
+	$(NVCC) $< -o $@ $(cu_objects) $(CUARCHFLAGS) $(LIBFLAGS) $(CULIBFLAGS) gmain.cc
 
-$(cxx_main): check.o $(LIBDIR)/lib$(MODELLIB).a $(cxx_objects)
-	$(CXX) $< -o $@ $(cxx_objects) $(CPPFLAGS) $(CXXFLAGS) -ldl -pthread $(LIBFLAGS) $(CULIBFLAGS)
+# This is built with g++ and linked with objects compiled with g++
+$(cxx_main): check.o $(LIBDIR)/lib$(MODELLIB).a $(cxx_objects) cmain.cc
+	$(CXX) $< -o $@ $(cxx_objects) $(CPPFLAGS) $(CXXFLAGS) -ldl -pthread $(LIBFLAGS) $(CULIBFLAGS) cmain.cc
+
+# This is built with nvcc and linked with objects compiled with nvcc or g++
+$(het_main): gcheck.o check.o $(LIBDIR)/lib$(MODELLIB).a $(cu_objects) $(cxx_objects) hmain.cc
+	$(NVCC) gcheck.o check.o -o $@ $(cu_objects) $(cxx_objects) $(CUARCHFLAGS) -lgomp $(LIBFLAGS) $(CULIBFLAGS) hmain.cc
 
 
 runTest.o:   $(GTESTLIBS)

epoch1/cuda/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/check.h

@@ -0,0 +1,29 @@
+#ifndef CHECK_H 
+#define CHECK_H 1
+
+#include <iostream>
+#include <string>
+#include <vector>
+
+// from gcheck.cu (compiled with nvcc)
+int gcheck( int argc,
+            char **argv,
+            std::string& out,
+            std::vector<double>& stats,
+            const std::string& tag = "",
+            const int niter_multiplier = 1 ); // only for the GPU
+
+// from check.cc (compiled with g++)
+int check( int argc,
+           char **argv,
+           std::string& out,
+           std::vector<double>& stats,
+           const std::string& tag = "" );
+
+// from check.cc (compiled with g++)
+int check_omp_threads( bool debug = false ); // returns the number of OMP threads
+
+// from check.cc (compiled with g++)
+const std::string check_nprocall(); // returns the output of `nproc --all`
+
+#endif // CHECK_H

epoch1/cuda/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/cmain.cc

@@ -0,0 +1,12 @@
+#include "check.h"
+
+// This is built with g++ and linked with objects compiled with g++
+int main( int argc, char **argv )
+{
+  std::string cpuOut;
+  std::vector<double> cpuStats;
+  int cpuStatus = check( argc, argv, cpuOut, cpuStats, "(CPU) " );
+  std::cout << cpuOut;
+  //for ( auto& stat : cpuStats ) std::cout << stat << std::endl;
+  return cpuStatus;
+}

epoch1/cuda/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/gmain.cc

@@ -0,0 +1,12 @@
+#include "check.h"
+
+// This is built with nvcc and linked with objects compiled with nvcc or g++
+int main( int argc, char **argv )
+{
+  std::string gpuOut;
+  std::vector<double> gpuStats;
+  int gpuStatus = gcheck( argc, argv, gpuOut, gpuStats, "(GPU) " );
+  std::cout << gpuOut;
+  //for ( auto& stat : gpuStats ) std::cout << stat << std::endl;
+  return gpuStatus;
+}

epoch1/cuda/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/hmain.cc

@@ -0,0 +1,107 @@
+#include "check.h"
+
+#include <iomanip>
+#include <thread>
+
+void dumptime( const std::string& tag,
+               const int nevtALL,
+               const double sumgtim,
+               const double sumrtim,
+               const double sumwtim )
+{
+  std::cout << "-----------------------------------------------------------------------------" << std::endl
+            << tag << "TotalEventsComputed         = " << nevtALL << std::endl
+            << std::scientific // fixed format: affects all floats (default precision: 6)
+            << tag << "TotalTime[Rnd+Rmb+ME] (123) = ( "
+            << sumgtim+sumrtim+sumwtim << std::string(16, ' ') << " )  sec" << std::endl
+            << tag << "TotalTime[Rambo+ME]    (23) = ( "
+            << sumrtim+sumwtim << std::string(16, ' ') << " )  sec" << std::endl
+            << tag << "TotalTime[RndNumGen]    (1) = ( " << sumgtim << std::string(16, ' ') << " )  sec" << std::endl
+            << tag << "TotalTime[Rambo]        (2) = ( " << sumrtim << std::string(16, ' ') << " )  sec" << std::endl
+            << tag << "TotalTime[MatrixElems]  (3) = ( " << sumwtim << std::string(16, ' ') << " )  sec" << std::endl
+            << std::defaultfloat // default format: affects all floats
+            << "-----------------------------------------------------------------------------" << std::endl;
+}
+
+void dumptput( const std::string& tag,
+               const int nevtALL,
+               const double tputgrw,
+               const double tputrw,
+               const double tputw,
+               const int nthreadsomp ) // >0 for CPU, 0 for GPU and HET
+{
+  if ( nthreadsomp > 0 )
+  {
+    std::string nprocall = check_nprocall();
+    std::cout << tag << "OMP threads / `nproc --all` = " << nthreadsomp << " / " << nprocall; // includes a newline
+  }
+  std::cout << tag << "TotalEventsComputed         = " << nevtALL << std::endl
+            << std::scientific // fixed format: affects all floats (default precision: 6)
+            << tag << "EvtsPerSec[Rnd+Rmb+ME](123) = ( " << tputgrw
+            << std::string(16, ' ') << " )  sec^-1" << std::endl
+            << tag << "EvtsPerSec[Rmb+ME]     (23) = ( " << tputrw
+            << std::string(16, ' ') << " )  sec^-1" << std::endl
+            << tag << "EvtsPerSec[MatrixElems] (3) = ( " << tputw
+            << std::string(16, ' ') << " )  sec^-1" << std::endl
+            << std::defaultfloat // default format: affects all floats
+            << "*****************************************************************************" << std::endl;
+}
+
+// This is compiled with g++ and linked with objects compiled with nvcc or g++
+int main( int argc, char **argv )
+{
+  std::string gpuOut;
+  std::vector<double> gpuStats;
+  int gpuStatus;
+  //int gpuMult = 1; // GPU processes same #events as CPU, with same random seeds
+  int gpuMult = 72; // GPU processes 72x #events as CPU, with different random seeds
+  int nthreadsomp = check_omp_threads(); // this is always > 0
+  gpuMult /= nthreadsomp; // reduce the number of GPU iterations if several OMP threads are used on the CPU
+  std::string gpuTag = "(GPU) ";
+  std::thread gpuThread( [&]{ gpuStatus = gcheck( argc, argv, gpuOut, gpuStats, gpuTag, gpuMult ); });
+
+  std::string cpuOut;
+  std::vector<double> cpuStats;
+  int cpuStatus;
+  std::string cpuTag = "(CPU) ";
+  std::thread cpuThread( [&]{ cpuStatus = check( argc, argv, cpuOut, cpuStats, cpuTag ); });
+
+  gpuThread.join();
+  std::cout << gpuOut;
+
+  cpuThread.join();
+  std::cout << cpuOut;
+
+  int gpuNevtALL = (int)(gpuStats[0]);
+  double gpuSumgtim = gpuStats[1];
+  double gpuSumrtim = gpuStats[2];
+  double gpuSumwtim = gpuStats[3];
+
+  dumptime( gpuTag, gpuNevtALL, gpuSumgtim, gpuSumrtim, gpuSumwtim );
+  double gpuTputgrw = gpuNevtALL/(gpuSumgtim+gpuSumrtim+gpuSumwtim);
+  double gpuTputrw  = gpuNevtALL/(gpuSumrtim+gpuSumwtim);
+  double gpuTputw   = gpuNevtALL/(gpuSumwtim);
+  dumptput( gpuTag, gpuNevtALL, gpuTputgrw, gpuTputrw, gpuTputw, 0 );
+
+  int cpuNevtALL = (int)(cpuStats[0]);
+  double cpuSumgtim = cpuStats[1];
+  double cpuSumrtim = cpuStats[2];
+  double cpuSumwtim = cpuStats[3];
+
+  dumptime( cpuTag, cpuNevtALL, cpuSumgtim, cpuSumrtim, cpuSumwtim );
+  double cpuTputgrw = cpuNevtALL/(cpuSumgtim+cpuSumrtim+cpuSumwtim);
+  double cpuTputrw  = cpuNevtALL/(cpuSumrtim+cpuSumwtim);
+  double cpuTputw   = cpuNevtALL/(cpuSumwtim);
+  dumptput( cpuTag, cpuNevtALL, cpuTputgrw, cpuTputrw, cpuTputw, nthreadsomp );
+
+  std::string hetTag = "(HET) ";
+  int hetNevtALL = gpuNevtALL+cpuNevtALL;
+  double hetTputgrw = gpuTputgrw+cpuTputgrw;
+  double hetTputrw  = gpuTputrw+cpuTputrw;
+  double hetTputw   = gpuTputw+cpuTputw;
+  dumptput( hetTag, hetNevtALL, hetTputgrw, hetTputrw, hetTputw, 0 );
+
+  if ( gpuStatus != 0 ) return 1;
+  if ( cpuStatus != 0 ) return 2;
+  return 0;
+}

epoch1/cuda/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/timermap.h

@@ -21,7 +21,7 @@ namespace mgOnGpu
 
   public:
 
-    TimerMap() : m_timer(), m_active(""), m_partitionTimers(), m_partitionIds() {}
+    TimerMap( const std::string& tag="" ) : m_timer(), m_tag(tag), m_active(""), m_partitionTimers(), m_partitionIds() {}
     virtual ~TimerMap() {}
 
     // Start the timer for a specific partition (key must be a non-empty string)
@@ -115,19 +115,19 @@ namespace mgOnGpu
         ostr << std::setprecision(6); // set precision (default=6): affects all floats
         ostr << std::fixed; // fixed format: affects all floats
         for ( auto ip : m_partitionTimers )
-          ostr << std::setw(maxsize) << ip.first << " : "
+          ostr << m_tag << std::setw(maxsize) << ip.first << " : "
                << std::setw(12) << ip.second << " sec" << std::endl;
-        ostr << std::setw(maxsize) << totalKey << " : "
+        ostr << m_tag << std::setw(maxsize) << totalKey << " : "
              << std::setw(12) << total << " sec" << std::endl
-             << std::setw(maxsize) << total123Key << " : "
+             << m_tag << std::setw(maxsize) << total123Key << " : "
              << std::setw(12) << total123 << " sec" << std::endl
-             << std::setw(maxsize) << total23Key << " : "
+             << m_tag << std::setw(maxsize) << total23Key << " : "
              << std::setw(12) << total23 << " sec" << std::endl
-             << std::setw(maxsize) << total1Key << " : "
+             << m_tag << std::setw(maxsize) << total1Key << " : "
              << std::setw(12) << total1 << " sec" << std::endl
-             << std::setw(maxsize) << total2Key << " : "
+             << m_tag << std::setw(maxsize) << total2Key << " : "
              << std::setw(12) << total2 << " sec" << std::endl
-             << std::setw(maxsize) << total3Key << " : "
+             << m_tag << std::setw(maxsize) << total3Key << " : "
              << std::setw(12) << total3 << " sec" << std::endl;
         ostr << std::defaultfloat; // default format: affects all floats
       }
@@ -136,6 +136,7 @@ namespace mgOnGpu
   private:
 
     Timer<TIMERTYPE> m_timer;
+    std::string m_tag;
     std::string m_active;
     std::map< std::string, float > m_partitionTimers;
     std::map< std::string, uint32_t > m_partitionIds;