Fixing condition to setup env for EPNs

Author: chiarazampolli

DATA/production/common/getStat.sh

@@ -14,6 +14,7 @@ nCTFsProcessed=$((nCTFsProcessed + 1))
 
 if [[ $nCTFsFilesInspected != $((nCTFsFilesFailed + nCTFsFilesOK)) ]]; then
   echo "Something went wrong with parsing the log file: CTF files inspected ($nCTFsFilesInspected) is not the sum of those successfully processed ($nCTFsFilesOK) and those that failed ($nCTFsFilesFailed)"
+  exit 8
 fi
 while read -r line; do
   currentPVs=`echo $line | sed 's/^.*Found \([0-9]*\) PVs.*/\1/'`

DATA/production/configurations/asyncReco/async_pass.sh

@@ -353,7 +353,7 @@ fi
 echo "SETTING_ROOT_OUTPUT = $SETTING_ROOT_OUTPUT"
 
 # Enabling GPUs
-if [[ -n "$ALIEN_JDL_USEGPUS" && $ALIEN_JDL_USEGPUS != 0 ]]; then
+if [[ -n "$ALIEN_JDL_USEGPUS" && $ALIEN_JDL_USEGPUS != 0 ]] ; then
   echo "Enabling GPUS"
   export GPUTYPE="HIP"
   export GPUMEMSIZE=$((25 << 30))
@@ -456,6 +456,13 @@ if [[ $ALIEN_JDL_SPLITWF != "1" ]]; then
     fi
     mv latest.log latest_reco_1.log
     $STATSCRIPT latest_reco_1.log
+    exitcode=$?
+    echo "exit code is $exitcode"
+    if [[ $exitcode -ne 0 ]]; then
+      echo "exit code from processing is " $exitcode > validation_error.message
+      echo "exit code from processing is " $exitcode
+      exit $exitcode
+    fi
   fi
 else
   # running the wf in split mode
@@ -486,7 +493,16 @@ else
 	exit $exitcode
       fi
       mv latest.log latest_reco_1.log
+      if [[ -f performanceMetrics.json ]]; then
+	mv performanceMetrics.json performanceMetrics_1.json
+      fi
       $STATSCRIPT latest_reco_1.log reco_1
+      exitcode=$?
+      if [[ $exitcode -ne 0 ]]; then
+	echo "exit code from processing is " $exitcode > validation_error.message
+	echo "exit code from processing is " $exitcode
+	exit $exitcode
+      fi
     fi
   fi
 
@@ -515,7 +531,16 @@ else
 	exit $exitcode
       fi
       mv latest.log latest_reco_2.log
+      if [[ -f performanceMetrics.json ]]; then
+	mv performanceMetrics.json performanceMetrics_2.json
+      fi
       $STATSCRIPT latest_reco_2.log reco_2
+      exitcode=$?
+      if [[ $exitcode -ne 0 ]]; then
+	echo "exit code from processing is " $exitcode > validation_error.message
+	echo "exit code from processing is " $exitcode
+	exit $exitcode
+      fi
       # let's compare to previous step
       if [[ -f latest_reco_1.log ]]; then
 	nCTFsFilesInspected_step1=`ls [0-9]*_[0-9]*_[0-9]*_[0-9]*_[0-9]*_reco_1.stat | sed 's/\(^[0-9]*\)_.*/\1/'`
@@ -569,23 +594,29 @@ else
 	exit $exitcode
       fi
       mv latest.log latest_reco_3.log
+      if [[ -f performanceMetrics.json ]]; then
+	mv performanceMetrics.json performanceMetrics_3.json
+      fi
     fi
   fi
 fi
 
 # now extract all performance metrics
 IFS=$'\n'
-if [[ -f "performanceMetrics.json" ]]; then
-    timeStart=`date +%s`
-    for workflow in `grep ': {' performanceMetrics.json`; do
-	strippedWorkflow=`echo $workflow | cut -d\" -f2`
-	cat performanceMetrics.json | jq '.'\"${strippedWorkflow}\"'' > ${strippedWorkflow}_metrics.json
+timeStart=`date +%s`
+for perfMetricsFiles in performanceMetrics.json performanceMetrics_1.json performanceMetrics_2.json performanceMetrics_3.json ; do
+  suffix=`echo $perfMetricsFiles | sed 's/performanceMetrics\(.*\).json/\1/'`
+  if [[ -f "performanceMetrics.json" ]]; then
+    for workflow in `grep ': {' $perfMetricsFiles`; do
+      strippedWorkflow=`echo $workflow | cut -d\" -f2`
+      cat $perfMetricsFiles | jq '.'\"${strippedWorkflow}\"'' > ${strippedWorkflow}_metrics${suffix}.json
     done
-    timeEnd=`date +%s`
-    timeUsed=$(( $timeUsed+$timeEnd-$timeStart ))
-    delta=$(( $timeEnd-$timeStart ))
-    echo "Time spent in splitting the metrics files = $delta s"
-fi
+  fi
+done
+timeEnd=`date +%s`
+timeUsed=$(( $timeUsed+$timeEnd-$timeStart ))
+delta=$(( $timeEnd-$timeStart ))
+echo "Time spent in splitting the metrics files = $delta s"
 
 if [[ $ALIEN_JDL_AODOFF != 1 ]]; then
   # flag to possibly enable Analysis QC

DATA/production/configurations/asyncReco/setenv_extra.sh

@@ -240,7 +240,7 @@ fi
 
 echo "BeamType = $BEAMTYPE"
 
-if [[ $ALIEN_JDL_ENABLEMONITORING == "1" ]]; then
+if [[ $ALIEN_JDL_ENABLEMONITORING != "0" ]]; then
   # add the performance metrics
   export ENABLE_METRICS=1
   export ARGS_ALL_EXTRA="$ARGS_ALL_EXTRA --resources-monitoring 50 --resources-monitoring-dump-interval 50"

DATA/production/workflow-multiplicities.sh

@@ -39,18 +39,18 @@ N_TPCTRK=$NGPUS
 if [[ ! -z ${OPTIMIZED_PARALLEL_ASYNC:-} ]]; then
   # Tuned multiplicities for async processing
   if [[ $OPTIMIZED_PARALLEL_ASYNC == "pp_8cpu" ]]; then
-    [[ -z $TIMEFRAME_RATE_LIMIT ]] && TIMEFRAME_RATE_LIMIT=3
+    [[ -z ${TIMEFRAME_RATE_LIMIT:-} ]] && TIMEFRAME_RATE_LIMIT=3
     NGPURECOTHREADS=5
   elif [[ $OPTIMIZED_PARALLEL_ASYNC == "pp_16cpu" ]]; then
-    [[ -z $TIMEFRAME_RATE_LIMIT ]] && TIMEFRAME_RATE_LIMIT=8
-    [[ -z $SHMSIZE ]] && SHMSIZE=22000000000
+    [[ -z ${TIMEFRAME_RATE_LIMIT:-} ]] && TIMEFRAME_RATE_LIMIT=8
+    [[ -z ${SHMSIZE:-} ]] && SHMSIZE=22000000000
     NGPURECOTHREADS=9
     NTRDTRKTHREADS=3
     ITSTRK_THREADS=3
     ITSTPC_THREADS=3
   elif [[ $OPTIMIZED_PARALLEL_ASYNC == "pp_64cpu" ]]; then
-    [[ -z $TIMEFRAME_RATE_LIMIT ]] && TIMEFRAME_RATE_LIMIT=32
-    [[ -z $SHMSIZE ]] && SHMSIZE=90000000000
+    [[ -z ${TIMEFRAME_RATE_LIMIT:-} ]] && TIMEFRAME_RATE_LIMIT=32
+    [[ -z ${SHMSIZE:-} ]] && SHMSIZE=90000000000
     NGPURECOTHREADS=12
     NTRDTRKTHREADS=3
     ITSTRK_THREADS=3
@@ -63,8 +63,8 @@ if [[ ! -z ${OPTIMIZED_PARALLEL_ASYNC:-} ]]; then
     N_TOFMATCH=2
     N_TPCENTDEC=3
   elif [[ $OPTIMIZED_PARALLEL_ASYNC == "pp_1gpu" ]]; then
-    [[ -z $TIMEFRAME_RATE_LIMIT ]] && TIMEFRAME_RATE_LIMIT=8
-    [[ -z $SHMSIZE ]] && SHMSIZE=20000000000
+    [[ -z ${TIMEFRAME_RATE_LIMIT:-} ]] && TIMEFRAME_RATE_LIMIT=8
+    [[ -z ${SHMSIZE:-} ]] && SHMSIZE=20000000000
     N_TOFMATCH=2
     N_MCHCL=3
     N_TPCENTDEC=2
@@ -76,8 +76,8 @@ if [[ ! -z ${OPTIMIZED_PARALLEL_ASYNC:-} ]]; then
     ITSTRK_THREADS=2
     ITSTPC_THREADS=2
   elif [[ $OPTIMIZED_PARALLEL_ASYNC == "pp_4gpu" ]]; then
-    [[ -z $TIMEFRAME_RATE_LIMIT ]] && TIMEFRAME_RATE_LIMIT=45
-    [[ -z $SHMSIZE ]] && SHMSIZE=100000000000
+    [[ -z ${TIMEFRAME_RATE_LIMIT:-} ]] && TIMEFRAME_RATE_LIMIT=45
+    [[ -z ${SHMSIZE:-} ]] && SHMSIZE=100000000000
     NGPURECOTHREADS=8
     NTRDTRKTHREADS=2
     ITSTRK_THREADS=2
@@ -98,9 +98,9 @@ if [[ ! -z ${OPTIMIZED_PARALLEL_ASYNC:-} ]]; then
     N_TPCITS=12
     N_ITSTRK=12
   elif [[ $OPTIMIZED_PARALLEL_ASYNC == "PbPb_4gpu" ]]; then
-    [[ -z $TIMEFRAME_RATE_LIMIT ]] && TIMEFRAME_RATE_LIMIT=20
-    [[ -z $SHMSIZE ]] && SHMSIZE=128000000000 # SHM_LIMIT 3/4
-    [[ -z $TIMEFRAME_SHM_LIMIT ]] && TIMEFRAME_SHM_LIMIT=$(($SHMSIZE / 3))
+    [[ -z ${TIMEFRAME_RATE_LIMIT:-} ]] && TIMEFRAME_RATE_LIMIT=20
+    [[ -z ${SHMSIZE:-} ]] && SHMSIZE=128000000000 # SHM_LIMIT 3/4
+    [[ -z ${TIMEFRAME_SHM_LIMIT:-} ]] && TIMEFRAME_SHM_LIMIT=$(($SHMSIZE / 3))
     NGPURECOTHREADS=8
     NTRDTRKTHREADS=4
     ITSTRK_THREADS=6