MUON: improve handling of QC for MCH-MID matching and add MFT-MCH matching

aphecetche · aphecetche · commit 2cac15c9674d · 2022-11-15T14:44:26.000+01:00
diff --git a/DATA/production/qc-sync/mch-mid.json b/DATA/production/qc-sync/mch-mid.json
@@ -0,0 +1,55 @@
+{
+    "qc": {
+        "config": {
+            "database": {
+                "implementation": "CCDB",
+                "host": "ali-qcdb.cern.ch:8083",
+                "username": "not_applicable",
+                "password": "not_applicable",
+                "name": "not_applicable"
+            },
+            "Activity": {
+                "number": "42",
+                "type": "2"
+            },
+            "monitoring": {
+                "url": "influxdb-unix:///tmp/telegraf.sock"
+            },
+            "consul": {
+                "url": "null:8500"
+            },
+            "conditionDB": {
+                "url": "null:8083"
+            }
+        },
+        "tasks": {
+            "FwdTracks": {
+                "active": "true",
+                "className": "o2::quality_control_modules::muon::TracksTask",
+                "moduleName": "QcMUONCommon",
+                "detectorName": "GLO",
+                "cycleDurationSeconds": "180",
+                "maxNumberCycles": "-1",
+                "dataSource": {
+                    "type": "direct",
+                    "query": "trackMCH:MCH/TRACKS;trackMCHROF:MCH/TRACKROFS;trackMCHTRACKCLUSTERS:MCH/TRACKCLUSTERS;mchtrackdigits:MCH/CLUSTERDIGITS;trackMID:MID/TRACKS;trackMIDROF:MID/TRACKROFS;trackMIDTRACKCLUSTERS:MID/TRACKCLUSTERS;trackClMIDROF:MID/TRCLUSROFS;matchMCHMID:GLO/MTC_MCHMID"
+                },
+                "taskParameters": {
+                    "maxTracksPerTF": "600",
+                    "GID": "MCH,MID,MCH-MID"
+                },
+                "location": "local",
+                "mergingMode": "delta",
+                "localControl": "odc",
+                "localMachines": [
+                    "localhost",
+                    "epn"
+                ],
+                "remotePort": "47789",
+                "remoteMachine": "alio2-cr1-qc01.cern.ch"
+            }
+        },
+        "checks": {}
+    },
+    "dataSamplingPolicies": []
+}
diff --git a/DATA/production/qc-sync/mft-mch-mid.json b/DATA/production/qc-sync/mft-mch-mid.json
@@ -0,0 +1,55 @@
+{
+    "qc": {
+        "config": {
+            "database": {
+                "implementation": "CCDB",
+                "host": "ali-qcdb.cern.ch:8083",
+                "username": "not_applicable",
+                "password": "not_applicable",
+                "name": "not_applicable"
+            },
+            "Activity": {
+                "number": "42",
+                "type": "2"
+            },
+            "monitoring": {
+                "url": "influxdb-unix:///tmp/telegraf.sock"
+            },
+            "consul": {
+                "url": "null:8500"
+            },
+            "conditionDB": {
+                "url": "null:8083"
+            }
+        },
+        "tasks": {
+            "FwdTracks": {
+                "active": "true",
+                "className": "o2::quality_control_modules::muon::TracksTask",
+                "moduleName": "QcMUONCommon",
+                "detectorName": "GLO",
+                "cycleDurationSeconds": "180",
+                "maxNumberCycles": "-1",
+                "dataSource": {
+                    "type": "direct",
+                    "query": "trackMCH:MCH/TRACKS;trackMCHROF:MCH/TRACKROFS;trackMCHTRACKCLUSTERS:MCH/TRACKCLUSTERS;mchtrackdigits:MCH/CLUSTERDIGITS;trackMID:MID/TRACKS;trackMIDROF:MID/TRACKROFS;trackMIDTRACKCLUSTERS:MID/TRACKCLUSTERS;trackClMIDROF:MID/TRCLUSROFS;fwdtracks:GLO/GLFWD;matchMCHMID:GLO/MTC_MCHMID;matchMFTMCH:GLO/MTC_MFTMCH"
+                },
+                "taskParameters": {
+                    "maxTracksPerTF": "600",
+                    "GID": "MFT,MCH,MID,MFT-MCH,MCH-MID"
+                },
+                "location": "local",
+                "mergingMode": "delta",
+                "localControl": "odc",
+                "localMachines": [
+                    "localhost",
+                    "epn"
+                ],
+                "remotePort": "47789",
+                "remoteMachine": "alio2-cr1-qc01.cern.ch"
+            }
+        },
+        "checks": {}
+    },
+    "dataSamplingPolicies": []
+}
diff --git a/DATA/production/qc-sync/mft-mch.json b/DATA/production/qc-sync/mft-mch.json
@@ -0,0 +1,55 @@
+{
+    "qc": {
+        "config": {
+            "database": {
+                "implementation": "CCDB",
+                "host": "ali-qcdb.cern.ch:8083",
+                "username": "not_applicable",
+                "password": "not_applicable",
+                "name": "not_applicable"
+            },
+            "Activity": {
+                "number": "42",
+                "type": "2"
+            },
+            "monitoring": {
+                "url": "influxdb-unix:///tmp/telegraf.sock"
+            },
+            "consul": {
+                "url": "null:8500"
+            },
+            "conditionDB": {
+                "url": "null:8083"
+            }
+        },
+        "tasks": {
+            "FwdTracks": {
+                "active": "true",
+                "className": "o2::quality_control_modules::muon::TracksTask",
+                "moduleName": "QcMUONCommon",
+                "detectorName": "GLO",
+                "cycleDurationSeconds": "180",
+                "maxNumberCycles": "-1",
+                "dataSource": {
+                    "type": "direct",
+                    "query": "trackMCH:MCH/TRACKS;trackMCHROF:MCH/TRACKROFS;trackMCHTRACKCLUSTERS:MCH/TRACKCLUSTERS;mchtrackdigits:MCH/CLUSTERDIGITS;trackMID:MID/TRACKS;trackMIDROF:MID/TRACKROFS;fwdtracks:GLO/GLFWD;matchMFTMCH:GLO/MTC_MFTMCH"
+                },
+                "taskParameters": {
+                    "maxTracksPerTF": "600",
+                    "GID": "MFT,MCH,MFT-MCH"
+                },
+                "location": "local",
+                "mergingMode": "delta",
+                "localControl": "odc",
+                "localMachines": [
+                    "localhost",
+                    "epn"
+                ],
+                "remotePort": "47789",
+                "remoteMachine": "alio2-cr1-qc01.cern.ch"
+            }
+        },
+        "checks": {}
+    },
+    "dataSamplingPolicies": []
+}
diff --git a/DATA/production/qc-workflow.sh b/DATA/production/qc-workflow.sh
@@ -10,6 +10,8 @@ if [[ ! -z $GEN_TOPO_QC_JSON_FILE ]]; then
   flock 101 || exit 1
 fi
 
+QC_CONFIG=
+QC_CONFIG_OVERRIDE=
 if [[ -z $QC_JSON_FROM_OUTSIDE && ! -z $GEN_TOPO_QC_JSON_FILE && -f $GEN_TOPO_QC_JSON_FILE ]]; then
   QC_JSON_FROM_OUTSIDE=$GEN_TOPO_QC_JSON_FILE
 elif [[ -z $QC_JSON_FROM_OUTSIDE ]]; then
@@ -37,9 +39,6 @@ elif [[ -z $QC_JSON_FROM_OUTSIDE ]]; then
     [[ -z "$QC_JSON_ZDC" ]] && has_processing_step ZDC_RECO && QC_JSON_ZDC=consul://o2/components/qc/ANY/any/zdc-rec-epn
     if [[ -z "$QC_JSON_MCH" ]]; then
       if has_detector MCH && has_processing_step MCH_RECO; then
-        if has_track_source "MCH-MID"; then
-          QC_JSON_MCH=consul://o2/components/qc/ANY/any/mch-qcmn-epn-full-track-matching
-        else
           QC_JSON_MCH=consul://o2/components/qc/ANY/any/mch-qcmn-epn-full
         fi
       else
@@ -65,7 +64,19 @@ elif [[ -z $QC_JSON_FROM_OUTSIDE ]]; then
         QC_JSON_TOF_MATCH=consul://o2/components/qc/ANY/any/tof-qcmn-match-itstpctof
       fi
     fi
+
+    if [[ -z "$QC_JSON_MUON_MATCH" ]]; then
+      if has_track_source "MFT-MCH" && has_track_source "MCH-MID"; then
+          QC_JSON_MUON_MATCH=consul://o2/components/qc/ANY/any/muon-qcmn-epn-mft-mch-mid
+      elif has_track_source "MFT-MCH"; then
+        QC_JSON_MUON_MATCH=consul://o2/components/qc/ANY/any/muon-qcmn-epn-mft-mch
+      elif has_track_source "MCH-MID"; then
+        QC_JSON_MUON_MATCH=consul://o2/components/qc/ANY/any/muon-qcmn-epn-mch-mid
+      fi
+    fi
+
     [[ -z "$QC_JSON_GLOBAL" ]] && QC_JSON_GLOBAL=$O2DPG_ROOT/DATA/production/qc-sync/qc-global-epn.json # this must be last
+
   elif [[ $SYNCMODE == 1 ]]; then
     [[ -z "$QC_JSON_TPC" ]] && QC_JSON_TPC=$O2DPG_ROOT/DATA/production/qc-sync/tpc.json
     [[ -z "$QC_JSON_ITS" ]] && QC_JSON_ITS=$O2DPG_ROOT/DATA/production/qc-sync/its.json
@@ -96,7 +107,19 @@ elif [[ -z $QC_JSON_FROM_OUTSIDE ]]; then
         QC_JSON_TOF_MATCH=$O2DPG_ROOT/DATA/production/qc-sync/itstpctof.json
       fi
     fi
+
+    if [[ -z "$QC_JSON_MUON_MATCH" ]]; then
+      if has_track_source "MFT-MCH" && has_track_source "MCH-MID"; then
+        QC_JSON_MUON_MATCH=$O2DPG_ROOT/DATA/production/qc-sync/mft-mch-mid.json
+      elif has_track_source "MFT-MCH"; then
+        QC_JSON_MUON_MATCH=$O2DPG_ROOT/DATA/production/qc-sync/mft-mch.json
+      elif has_track_source "MCH-MID"; then
+        QC_JSON_MUON_MATCH=$O2DPG_ROOT/DATA/production/qc-sync/mch-mid.json
+      fi
+    fi
+
     [[ -z "$QC_JSON_GLOBAL" ]] && QC_JSON_GLOBAL=$O2DPG_ROOT/DATA/production/qc-sync/qc-global.json # this must be last
+
   else
     [[ -z "$QC_JSON_TPC" ]] && QC_JSON_TPC=$O2DPG_ROOT/DATA/production/qc-async/tpc.json
     [[ -z "$QC_JSON_ITS" ]] && QC_JSON_ITS=$O2DPG_ROOT/DATA/production/qc-async/its.json
@@ -133,16 +156,20 @@ elif [[ -z $QC_JSON_FROM_OUTSIDE ]]; then
         echo "Error fetching QC JSON $2"
         exit 1
       fi
-      JSON_FILES+=" $TMP_FILENAME"
     else
-      JSON_FILES+=" ${2}"
+      TMP_FILENAME=$2
+    fi
+    JSON_FILES+=" $TMP_FILENAME"
+    jq -rM '""' > /dev/null < $TMP_FILENAME
+    if [[ $? != 0 ]]; then
+      echo "Invalid QC JSON $2" 1>&2
+      exit 1
     fi
     OUTPUT_SUFFIX+="-$1"
   }
 
   JSON_FILES=
   OUTPUT_SUFFIX=
-  QC_CONFIG=
 
   # TOF matching
   if has_detector_qc TOF && [ ! -z "$QC_JSON_TOF_MATCH" ]; then
@@ -189,7 +216,7 @@ elif [[ -z $QC_JSON_FROM_OUTSIDE ]]; then
     else
       MERGED_JSON_FILENAME=$GEN_TOPO_QC_JSON_FILE
     fi
-    jq -n 'reduce inputs as $s (input; .qc.tasks += ($s.qc.tasks) | .qc.checks += ($s.qc.checks)  | .qc.externalTasks += ($s.qc.externalTasks) | .qc.postprocessing += ($s.qc.postprocessing)| .dataSamplingPolicies += ($s.dataSamplingPolicies))' $QC_JSON_GLOBAL $JSON_FILES >$MERGED_JSON_FILENAME
+    jq -n 'reduce inputs as $s (input; .qc.tasks += ($s.qc.tasks) | .qc.checks += ($s.qc.checks)  | .qc.externalTasks += ($s.qc.externalTasks) | .qc.postprocessing += ($s.qc.postprocessing)| .dataSamplingPolicies += ($s.dataSamplingPolicies))' $QC_JSON_GLOBAL $JSON_FILES > $MERGED_JSON_FILENAME
     if [[ $? != 0 ]]; then
       echo Merging QC workflow with JSON files $JSON_FILES failed 1>&2
       exit 1
@@ -199,7 +226,7 @@ elif [[ -z $QC_JSON_FROM_OUTSIDE ]]; then
     if [[ "0$QC_REDIRECT_MERGER_TO_LOCALHOST" == "01" ]]; then
       sed -i.bak -E 's/( *)"remoteMachine" *: *".*"(,?) *$/\1"remoteMachine": "127.0.0.1"\2/' $MERGED_JSON_FILENAME
       unlink $MERGED_JSON_FILENAME.bak
-      QC_CONFIG+="--override-values \"qc.config.database.host=ccdb-test.cern.ch:8080\""
+      QC_CONFIG_OVERRIDE+="qc.config.database.host=ccdb-test.cern.ch:8080;"
     fi
 
     if [[ "0$GEN_TOPO_QC_OVERRIDE_CCDB_SERVER" != "0" ]]; then
@@ -211,7 +238,21 @@ elif [[ -z $QC_JSON_FROM_OUTSIDE ]]; then
   rm -Rf $FETCHTMPDIR
 fi
 
+[[ $EPNSYNCMODE == 1 && $NUMAGPUIDS == 1 ]] && QC_CONFIG_OVERRIDE+="qc.config.infologger.filterDiscardFile=../../qc-_ID_${NUMAID}.log;"
+
+[[ ! -z $QC_CONFIG_OVERRIDE ]] && QC_CONFIG+=" --override-values \"$QC_CONFIG_OVERRIDE\""
+
 if [[ ! -z "$QC_JSON_FROM_OUTSIDE" ]]; then
+  if [[ ! -f $QC_JSON_FROM_OUTSIDE ]]; then
+    echo QC JSON FILE $QC_JSON_FROM_OUTSIDE missing 1>&2
+    exit 1
+  fi
+  jq -rM '""' > /dev/null < $QC_JSON_FROM_OUTSIDE
+  if [[ $? != 0 ]]; then
+    echo "Final QC JSON FILE $QC_JSON_FROM_OUTSIDE has invalid syntax" 1>&2
+    #cat $QC_JSON_FROM_OUTSIDE 1>&2
+    exit 1
+  fi
   add_W o2-qc "--config json://$QC_JSON_FROM_OUTSIDE ${QC_CONFIG_PARAM:---local --host ${QC_HOST:-localhost}} ${QC_CONFIG}"
 fi
 
