fix(benchmark): capture CPU and fgprof profiles sequentially to avoid corruption

Go's CPU profiler and fgprof conflict when running concurrently on the
same process, producing empty or corrupted profiles. Switch from parallel
background captures to sequential execution (CPU first, then fgprof),
measure actual capture duration for accurate remaining-time calculation,
and make BASE_DIR overridable so benchmark-compare.sh can route profiles
to per-label directories.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: pdrobnjak

benchmark/benchmark-compare.sh

@@ -171,6 +171,7 @@ for i in $(seq 0 $((NUM-1))); do
 
   BENCHMARK_PHASE=start \
   SEI_HOME="$home_dir" \
+  BASE_DIR="$BASE_DIR/$label" \
   SEID_BIN="$seid" \
   PORT_OFFSET="$port_offset" \
   LOG_FILE="$log_file" \
@@ -192,7 +193,11 @@ sleep "$PPROF_WAIT" || true
 
 echo ""
 echo "=== Capturing profiles (30s CPU + 30s fgprof + heap + goroutine + block + mutex) ==="
-PPROF_PIDS=()
+set +e
+PROFILE_CAPTURE_START=$(date +%s)
+
+# Capture CPU across all nodes first, then fgprof, then remaining profiles.
+# CPU and fgprof cannot run together on the same process.
 for i in $(seq 0 $((NUM-1))); do
   label="${LABELS[$i]}"
   port_offset=$((i * 100))
@@ -201,44 +206,57 @@ for i in $(seq 0 $((NUM-1))); do
   mkdir -p "$profile_dir"
 
   # CPU profile (30s) — on-CPU time only
+  echo "  [$label] capturing cpu"
   curl -s "http://localhost:${PPROF_PORT}/debug/pprof/profile?seconds=30" \
-    -o "$profile_dir/cpu.pb.gz" &
-  PPROF_PIDS=("${PPROF_PIDS[@]}" $!)
+    -o "$profile_dir/cpu.pb.gz"
+done
+
+for i in $(seq 0 $((NUM-1))); do
+  label="${LABELS[$i]}"
+  port_offset=$((i * 100))
+  PPROF_PORT=$((6060 + port_offset))
+  profile_dir="$BASE_DIR/$label/pprof"
 
   # fgprof (30s) — wall-clock time (on-CPU + off-CPU/I/O/blocking)
+  echo "  [$label] capturing fgprof"
   curl -s "http://localhost:${PPROF_PORT}/debug/fgprof?seconds=30" \
-    -o "$profile_dir/fgprof.pb.gz" &
-  PPROF_PIDS=("${PPROF_PIDS[@]}" $!)
+    -o "$profile_dir/fgprof.pb.gz"
+done
+
+for i in $(seq 0 $((NUM-1))); do
+  label="${LABELS[$i]}"
+  port_offset=$((i * 100))
+  PPROF_PORT=$((6060 + port_offset))
+  profile_dir="$BASE_DIR/$label/pprof"
+
+  echo "  [$label] capturing heap/goroutine/block/mutex"
 
   # Heap snapshot
   curl -s "http://localhost:${PPROF_PORT}/debug/pprof/heap?debug=0" \
-    -o "$profile_dir/heap.pb.gz" &
-  PPROF_PIDS=("${PPROF_PIDS[@]}" $!)
+    -o "$profile_dir/heap.pb.gz"
 
   # Goroutine dump
   curl -s "http://localhost:${PPROF_PORT}/debug/pprof/goroutine?debug=0" \
-    -o "$profile_dir/goroutine.pb.gz" &
-  PPROF_PIDS=("${PPROF_PIDS[@]}" $!)
+    -o "$profile_dir/goroutine.pb.gz"
 
   # Block profile (time waiting on channels/mutexes)
   curl -s "http://localhost:${PPROF_PORT}/debug/pprof/block?debug=0" \
-    -o "$profile_dir/block.pb.gz" &
-  PPROF_PIDS=("${PPROF_PIDS[@]}" $!)
+    -o "$profile_dir/block.pb.gz"
 
   # Mutex contention profile
   curl -s "http://localhost:${PPROF_PORT}/debug/pprof/mutex?debug=0" \
-    -o "$profile_dir/mutex.pb.gz" &
-  PPROF_PIDS=("${PPROF_PIDS[@]}" $!)
+    -o "$profile_dir/mutex.pb.gz"
 
   echo "  [$label] capturing from localhost:${PPROF_PORT} -> $profile_dir/"
 done
 
-# Wait for all profile captures to finish
-for pid in "${PPROF_PIDS[@]}"; do
-  wait "$pid" 2>/dev/null || true
-done
 echo "  profile capture complete."
 
+set -e
+
+PROFILE_CAPTURE_END=$(date +%s)
+PROFILE_CAPTURE_SECS=$(( PROFILE_CAPTURE_END - PROFILE_CAPTURE_START ))
+
 # Verify profile sizes
 for i in $(seq 0 $((NUM-1))); do
   label="${LABELS[$i]}"
@@ -253,7 +271,7 @@ for i in $(seq 0 $((NUM-1))); do
 done
 
 # Wait remaining time
-REMAINING=$(( DURATION - PPROF_WAIT - 30 ))
+REMAINING=$(( DURATION - PPROF_WAIT - PROFILE_CAPTURE_SECS ))
 if [ "$REMAINING" -gt 0 ]; then
   echo ""
   echo "  Continuing benchmark for ${REMAINING}s..."

benchmark/benchmark.sh

@@ -325,7 +325,7 @@ fi
 
 if [ "$DURATION" -gt 0 ] 2>/dev/null; then
   # ---- Timed run: background seid, capture profiles, extract TPS ----
-  BASE_DIR="/tmp/sei-bench"
+  BASE_DIR="${BASE_DIR:-/tmp/sei-bench}"
   PROFILE_DIR="$BASE_DIR/pprof"
   mkdir -p "$PROFILE_DIR"
 
@@ -352,47 +352,37 @@ if [ "$DURATION" -gt 0 ] 2>/dev/null; then
   sleep "$PPROF_WAIT" || true
 
   echo ""
-  echo "=== Capturing profiles (30s CPU + 30s fgprof + heap + goroutine + block + mutex) ==="
-  PPROF_PIDS=()
+  echo "=== Capturing profiles (30s CPU + 30s fgprof + others) ==="
+  set +e
+  PROFILE_CAPTURE_START=$(date +%s)
 
-  # CPU profile (30s)
+  # Capture sequentially to avoid runtime conflicts:
+  # Go pprof CPU and fgprof profiling cannot run at the same time.
+  echo "  capturing cpu profile ..."
   curl -s "http://localhost:${PPROF_PORT}/debug/pprof/profile?seconds=30" \
-    -o "$PROFILE_DIR/cpu.pb.gz" &
-  PPROF_PIDS=("${PPROF_PIDS[@]}" $!)
+    -o "$PROFILE_DIR/cpu.pb.gz"
 
-  # fgprof (30s) — wall-clock time
+  echo "  capturing fgprof ..."
   curl -s "http://localhost:${PPROF_PORT}/debug/fgprof?seconds=30" \
-    -o "$PROFILE_DIR/fgprof.pb.gz" &
-  PPROF_PIDS=("${PPROF_PIDS[@]}" $!)
+    -o "$PROFILE_DIR/fgprof.pb.gz"
 
-  # Heap snapshot
+  echo "  capturing heap/goroutine/block/mutex ..."
   curl -s "http://localhost:${PPROF_PORT}/debug/pprof/heap?debug=0" \
-    -o "$PROFILE_DIR/heap.pb.gz" &
-  PPROF_PIDS=("${PPROF_PIDS[@]}" $!)
-
-  # Goroutine dump
+    -o "$PROFILE_DIR/heap.pb.gz"
   curl -s "http://localhost:${PPROF_PORT}/debug/pprof/goroutine?debug=0" \
-    -o "$PROFILE_DIR/goroutine.pb.gz" &
-  PPROF_PIDS=("${PPROF_PIDS[@]}" $!)
-
-  # Block profile
+    -o "$PROFILE_DIR/goroutine.pb.gz"
   curl -s "http://localhost:${PPROF_PORT}/debug/pprof/block?debug=0" \
-    -o "$PROFILE_DIR/block.pb.gz" &
-  PPROF_PIDS=("${PPROF_PIDS[@]}" $!)
-
-  # Mutex contention profile
+    -o "$PROFILE_DIR/block.pb.gz"
   curl -s "http://localhost:${PPROF_PORT}/debug/pprof/mutex?debug=0" \
-    -o "$PROFILE_DIR/mutex.pb.gz" &
-  PPROF_PIDS=("${PPROF_PIDS[@]}" $!)
+    -o "$PROFILE_DIR/mutex.pb.gz"
 
-  echo "  Capturing from localhost:${PPROF_PORT} -> $PROFILE_DIR/"
-
-  # Wait for all profile captures to finish
-  for pid in "${PPROF_PIDS[@]}"; do
-    wait "$pid" 2>/dev/null || true
-  done
+  echo "  capturing from localhost:${PPROF_PORT} -> $PROFILE_DIR/"
   echo "  Profile capture complete."
 
+  PROFILE_CAPTURE_END=$(date +%s)
+  PROFILE_CAPTURE_SECS=$(( PROFILE_CAPTURE_END - PROFILE_CAPTURE_START ))
+  set -e
+
   # Verify profile sizes
   cpu_size=$(wc -c < "$PROFILE_DIR/cpu.pb.gz" 2>/dev/null | tr -d ' ' || echo 0)
   fgprof_size=$(wc -c < "$PROFILE_DIR/fgprof.pb.gz" 2>/dev/null | tr -d ' ' || echo 0)
@@ -403,7 +393,7 @@ if [ "$DURATION" -gt 0 ] 2>/dev/null; then
   echo "  cpu=${cpu_size}B fgprof=${fgprof_size}B heap=${heap_size}B goroutine=${goroutine_size}B block=${block_size}B mutex=${mutex_size}B"
 
   # Wait remaining time
-  REMAINING=$(( DURATION - PPROF_WAIT - 30 ))
+  REMAINING=$(( DURATION - PPROF_WAIT - PROFILE_CAPTURE_SECS ))
   if [ "$REMAINING" -gt 0 ]; then
     echo ""
     echo "  Continuing benchmark for ${REMAINING}s..."