From 095756741b482f9a6d81762f175617386e5a53d8 Mon Sep 17 00:00:00 2001 From: ludamad Date: Wed, 25 Feb 2026 13:54:07 -0500 Subject: [PATCH 01/16] dbg --- bootstrap.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/bootstrap.sh b/bootstrap.sh index a241ff273bfd..d285e8982683 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -694,6 +694,7 @@ case "$cmd" in barretenberg/cpp/bootstrap.sh build ;; "ci-barretenberg") + export BUILD_SYSTEM_DEBUG=1 export CI=1 export USE_TEST_CACHE=1 export AVM=0 From e7c6775e78504222f9bea0739a53c4ab63c6161b Mon Sep 17 00:00:00 2001 From: ludamad Date: Wed, 25 Feb 2026 14:08:37 -0500 Subject: [PATCH 02/16] - --- barretenberg/cpp/bootstrap.sh | 3 +++ bootstrap.sh | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/barretenberg/cpp/bootstrap.sh b/barretenberg/cpp/bootstrap.sh index b2c78fd1eba2..32c10853f481 100755 --- a/barretenberg/cpp/bootstrap.sh +++ b/barretenberg/cpp/bootstrap.sh @@ -287,6 +287,7 @@ function build_release_dir { export -f build_preset build_native_objects build_cross_objects build_native build_cross build_ios build_android build_asan_fast build_wasm build_wasm_threads build_gcc_syntax_check_only build_fuzzing_syntax_check_only build_smt_verification inject_version function build { + set -x echo_header "bb cpp build" if [ "$CI_FULL" -eq 1 ]; then @@ -408,6 +409,7 @@ function test_cmds { # This is not called in ci. It is just for a developer to run the tests. function test { + set -x echo_header "bb test" test_cmds | filter_test_cmds | parallelize } @@ -501,6 +503,7 @@ case "$cmd" in build ;; "ci") + set -x build test ;; diff --git a/bootstrap.sh b/bootstrap.sh index d285e8982683..47f27638d9c1 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -694,7 +694,7 @@ case "$cmd" in barretenberg/cpp/bootstrap.sh build ;; "ci-barretenberg") - export BUILD_SYSTEM_DEBUG=1 + set -x export CI=1 export USE_TEST_CACHE=1 export AVM=0 From 6a83bd1538ea10165ed70a0729ad6d108203cff3 Mon Sep 17 00:00:00 2001 From: ludamad Date: Wed, 25 Feb 2026 14:41:11 -0500 Subject: [PATCH 03/16] fix(ci3): emit sentinel to prevent SIGPIPE when all tests are cached When all tests are filtered by filter_cached_test_cmd, parallelize receives empty stdin causing GNU parallel to exit and the upstream pipeline to receive SIGPIPE (exit 141). Emit a trailing empty line as a sentinel; run_test_cmd already handles empty commands with an early exit 0, and parallelize already excludes them from the count. Co-Authored-By: Claude Sonnet 4.6 --- ci3/filter_cached_test_cmd | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ci3/filter_cached_test_cmd b/ci3/filter_cached_test_cmd index 04f288c05c92..73251f8c2feb 100755 --- a/ci3/filter_cached_test_cmd +++ b/ci3/filter_cached_test_cmd @@ -38,3 +38,9 @@ else # Noop. cat fi + +# Always emit an empty sentinel line so that 'parallelize' always receives at least one +# line of input. Without this, when all tests are cache-filtered out, GNU parallel receives +# empty stdin and the upstream pipeline gets SIGPIPE (exit 141). run_test_cmd handles +# empty commands with an early exit 0, and parallelize already excludes them from the count. +echo "" From b63677c6454feee70dcbf63e7836112d56803c0b Mon Sep 17 00:00:00 2001 From: ludamad Date: Wed, 25 Feb 2026 15:00:03 -0500 Subject: [PATCH 04/16] fix(ci3): handle SIGPIPE in bb test when all tests are cached test_cmds_native runs each test binary with --gtest_list_tests to enumerate tests, which takes real time across 50+ binaries. When all tests are cache-hits, the downstream (parallelize) exits early, and test_cmds_native's echo calls receive SIGPIPE (exit 141). Fix by using set +o pipefail + PIPESTATUS to distinguish SIGPIPE from test_cmds (exit 141, expected when all cached) from genuine failures. Revert the earlier sentinel approach which didn't address the root cause. The root-level CI avoids this by writing tests to a file (test_engine), so test_cmds never pipes to a reader that can exit early. Co-Authored-By: Claude Sonnet 4.6 --- barretenberg/cpp/bootstrap.sh | 15 +++++++++++++++ ci3/filter_cached_test_cmd | 6 ------ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/barretenberg/cpp/bootstrap.sh b/barretenberg/cpp/bootstrap.sh index 32c10853f481..8334fe232e9b 100755 --- a/barretenberg/cpp/bootstrap.sh +++ b/barretenberg/cpp/bootstrap.sh @@ -411,7 +411,22 @@ function test_cmds { function test { set -x echo_header "bb test" + # test_cmds_native runs each test binary with --gtest_list_tests to enumerate tests. + # When all tests are cache-hits, the downstream (parallelize) exits early, causing + # test_cmds_native's echo calls to receive SIGPIPE (exit 141). This is expected and OK. + # We use set +o pipefail + PIPESTATUS to distinguish this from genuine failures. + set +o pipefail test_cmds | filter_test_cmds | parallelize + local -a ps=("${PIPESTATUS[@]}") + set -o pipefail + # ps[0]=test_cmds, ps[1]=filter_test_cmds, ps[2]=parallelize + if [ "${ps[0]}" -ne 0 ] && [ "${ps[0]}" -ne 141 ]; then + echo "test_cmds failed (exit ${ps[0]})" + exit "${ps[0]}" + fi + if [ "${ps[2]}" -ne 0 ]; then + exit "${ps[2]}" + fi } function build_bench { diff --git a/ci3/filter_cached_test_cmd b/ci3/filter_cached_test_cmd index 73251f8c2feb..04f288c05c92 100755 --- a/ci3/filter_cached_test_cmd +++ b/ci3/filter_cached_test_cmd @@ -38,9 +38,3 @@ else # Noop. cat fi - -# Always emit an empty sentinel line so that 'parallelize' always receives at least one -# line of input. Without this, when all tests are cache-filtered out, GNU parallel receives -# empty stdin and the upstream pipeline gets SIGPIPE (exit 141). run_test_cmd handles -# empty commands with an early exit 0, and parallelize already excludes them from the count. -echo "" From 872f43c05e08df07ef273b40e13d53b3df2de6c1 Mon Sep 17 00:00:00 2001 From: ludamad Date: Wed, 25 Feb 2026 15:12:02 -0500 Subject: [PATCH 05/16] fix(ci3): buffer test_cmds to file to avoid SIGPIPE when all cached test_cmds_native runs each binary with --gtest_list_tests to enumerate tests. When all tests are cached, the downstream exits early and test_cmds_native receives SIGPIPE (exit 141). Write test commands to a temp file first (analogous to how the root-level build_and_test uses test_engine with a file). Writing to a file is immune to SIGPIPE regardless of when the downstream reader exits. Co-Authored-By: Claude Sonnet 4.6 --- barretenberg/cpp/bootstrap.sh | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/barretenberg/cpp/bootstrap.sh b/barretenberg/cpp/bootstrap.sh index 8334fe232e9b..1acbc4fc6c51 100755 --- a/barretenberg/cpp/bootstrap.sh +++ b/barretenberg/cpp/bootstrap.sh @@ -411,22 +411,13 @@ function test_cmds { function test { set -x echo_header "bb test" - # test_cmds_native runs each test binary with --gtest_list_tests to enumerate tests. - # When all tests are cache-hits, the downstream (parallelize) exits early, causing - # test_cmds_native's echo calls to receive SIGPIPE (exit 141). This is expected and OK. - # We use set +o pipefail + PIPESTATUS to distinguish this from genuine failures. - set +o pipefail - test_cmds | filter_test_cmds | parallelize - local -a ps=("${PIPESTATUS[@]}") - set -o pipefail - # ps[0]=test_cmds, ps[1]=filter_test_cmds, ps[2]=parallelize - if [ "${ps[0]}" -ne 0 ] && [ "${ps[0]}" -ne 141 ]; then - echo "test_cmds failed (exit ${ps[0]})" - exit "${ps[0]}" - fi - if [ "${ps[2]}" -ne 0 ]; then - exit "${ps[2]}" - fi + # Write test commands to a temp file first to avoid SIGPIPE: test_cmds_native runs + # each binary with --gtest_list_tests (slow), and if all tests are cached the downstream + # exits before enumeration completes. Writing to a file is immune to this. + local cmds_file=$(mktemp) + trap "rm -f $cmds_file" RETURN + test_cmds > $cmds_file + cat $cmds_file | filter_test_cmds | parallelize } function build_bench { From 1588f35b4ae0d18f6841729dc189f6afbd27c0d2 Mon Sep 17 00:00:00 2001 From: ludamad Date: Wed, 25 Feb 2026 15:15:29 -0500 Subject: [PATCH 06/16] dbg: add set -x to parallelize to trace why it exits early Co-Authored-By: Claude Sonnet 4.6 --- barretenberg/cpp/bootstrap.sh | 8 +------- ci3/parallelize | 1 + 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/barretenberg/cpp/bootstrap.sh b/barretenberg/cpp/bootstrap.sh index 1acbc4fc6c51..32c10853f481 100755 --- a/barretenberg/cpp/bootstrap.sh +++ b/barretenberg/cpp/bootstrap.sh @@ -411,13 +411,7 @@ function test_cmds { function test { set -x echo_header "bb test" - # Write test commands to a temp file first to avoid SIGPIPE: test_cmds_native runs - # each binary with --gtest_list_tests (slow), and if all tests are cached the downstream - # exits before enumeration completes. Writing to a file is immune to this. - local cmds_file=$(mktemp) - trap "rm -f $cmds_file" RETURN - test_cmds > $cmds_file - cat $cmds_file | filter_test_cmds | parallelize + test_cmds | filter_test_cmds | parallelize } function build_bench { diff --git a/ci3/parallelize b/ci3/parallelize index 8fc7db08092e..2e7929dd36ac 100755 --- a/ci3/parallelize +++ b/ci3/parallelize @@ -3,6 +3,7 @@ # Commands are read from stdin. NO_CD=1 source $(git rev-parse --show-toplevel)/ci3/source +set -x [ "${STRICT_SCHEDULING:-0}" -eq 1 ] && exec parallelize_strict "$@" cd $root From 97f763af465b5b8750895a301bc9e4ee7c39cc4e Mon Sep 17 00:00:00 2001 From: ludamad Date: Wed, 25 Feb 2026 15:33:31 -0500 Subject: [PATCH 07/16] dbg: add set -x to filter_cached_test_cmd to find early exit cause Co-Authored-By: Claude Sonnet 4.6 --- ci3/filter_cached_test_cmd | 3 +++ ci3/parallelize | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/ci3/filter_cached_test_cmd b/ci3/filter_cached_test_cmd index 04f288c05c92..b9bb9e04db46 100755 --- a/ci3/filter_cached_test_cmd +++ b/ci3/filter_cached_test_cmd @@ -29,11 +29,14 @@ function process_batch { export -f process_batch if [ "${USE_TEST_CACHE:-0}" -eq 1 ] && [ "$CI_REDIS_AVAILABLE" -eq 1 ]; then + set -x exec 3> >(cache_log "Skipped tests") # Process stdin in batches of 50. parallel --pipe -N50 --keep-order process_batch + echo "parallel exit: $?" exec 3>&- wait + echo "wait exit: $?" else # Noop. cat diff --git a/ci3/parallelize b/ci3/parallelize index 2e7929dd36ac..8fc7db08092e 100755 --- a/ci3/parallelize +++ b/ci3/parallelize @@ -3,7 +3,6 @@ # Commands are read from stdin. NO_CD=1 source $(git rev-parse --show-toplevel)/ci3/source -set -x [ "${STRICT_SCHEDULING:-0}" -eq 1 ] && exec parallelize_strict "$@" cd $root From 5f8b444a524bef2f71350fdb9c7eeb2ced11f419 Mon Sep 17 00:00:00 2001 From: ludamad Date: Wed, 25 Feb 2026 16:03:18 -0500 Subject: [PATCH 08/16] dbg: capture parallel --pipe exit code in filter_cached_test_cmd Co-Authored-By: Claude Sonnet 4.6 --- ci3/filter_cached_test_cmd | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ci3/filter_cached_test_cmd b/ci3/filter_cached_test_cmd index b9bb9e04db46..ba7892dcafad 100755 --- a/ci3/filter_cached_test_cmd +++ b/ci3/filter_cached_test_cmd @@ -32,8 +32,10 @@ if [ "${USE_TEST_CACHE:-0}" -eq 1 ] && [ "$CI_REDIS_AVAILABLE" -eq 1 ]; then set -x exec 3> >(cache_log "Skipped tests") # Process stdin in batches of 50. - parallel --pipe -N50 --keep-order process_batch + set +e + parallel --pipe -N50 --keep-order process_batch 2>&1 echo "parallel exit: $?" + set -e exec 3>&- wait echo "wait exit: $?" From b889ee9bf75a21802e965eecca70cbba197056bf Mon Sep 17 00:00:00 2001 From: ludamad Date: Wed, 25 Feb 2026 22:07:53 -0500 Subject: [PATCH 09/16] probe --- barretenberg/cpp/bootstrap.sh | 5 ++++- ci3/parallelize | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/barretenberg/cpp/bootstrap.sh b/barretenberg/cpp/bootstrap.sh index 32c10853f481..75301b43767e 100755 --- a/barretenberg/cpp/bootstrap.sh +++ b/barretenberg/cpp/bootstrap.sh @@ -503,9 +503,12 @@ case "$cmd" in build ;; "ci") - set -x build + set -x + set +e test + echo $! + set -e ;; "hash") echo $hash diff --git a/ci3/parallelize b/ci3/parallelize index 8fc7db08092e..381e37cf737a 100755 --- a/ci3/parallelize +++ b/ci3/parallelize @@ -54,7 +54,7 @@ echo "Completed run of $(tail -n+2 joblog.txt | grep -v "''" | wc -l) tests in $ # Process flake log (check thresholds and post PR comment in CI) if [ "$CI" -eq 1 ]; then - $root/scripts/process_flake_log.py "$FLAKES_FILE" --post-comment + $root/scripts/process_flake_log.py "$FLAKES_FILE" --post-comment || true else - $root/scripts/process_flake_log.py "$FLAKES_FILE" + $root/scripts/process_flake_log.py "$FLAKES_FILE" || true fi From 89b38db346f78e6178f5d18ad11e877a7b9a0983 Mon Sep 17 00:00:00 2001 From: ludamad Date: Wed, 25 Feb 2026 23:22:26 -0500 Subject: [PATCH 10/16] probe --- barretenberg/cpp/bootstrap.sh | 9 ++++----- ci3/filter_cached_test_cmd | 2 -- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/barretenberg/cpp/bootstrap.sh b/barretenberg/cpp/bootstrap.sh index 75301b43767e..256d5695743c 100755 --- a/barretenberg/cpp/bootstrap.sh +++ b/barretenberg/cpp/bootstrap.sh @@ -411,7 +411,10 @@ function test_cmds { function test { set -x echo_header "bb test" - test_cmds | filter_test_cmds | parallelize + test_cmds > /tmp/test_cmds + filter_test_cmds < /tmp/test_cmds > /tmp/out + echo "hi" ; cat /tmp/out ; echo "hey" + cat /tmp/out | parallelize } function build_bench { @@ -504,11 +507,7 @@ case "$cmd" in ;; "ci") build - set -x - set +e test - echo $! - set -e ;; "hash") echo $hash diff --git a/ci3/filter_cached_test_cmd b/ci3/filter_cached_test_cmd index ba7892dcafad..c8c9d25423af 100755 --- a/ci3/filter_cached_test_cmd +++ b/ci3/filter_cached_test_cmd @@ -34,11 +34,9 @@ if [ "${USE_TEST_CACHE:-0}" -eq 1 ] && [ "$CI_REDIS_AVAILABLE" -eq 1 ]; then # Process stdin in batches of 50. set +e parallel --pipe -N50 --keep-order process_batch 2>&1 - echo "parallel exit: $?" set -e exec 3>&- wait - echo "wait exit: $?" else # Noop. cat From 526643e39e9e5964c260692d29bc530123b0a39e Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 26 Feb 2026 00:47:36 -0500 Subject: [PATCH 11/16] probe --- barretenberg/cpp/bootstrap.sh | 2 +- bootstrap.sh | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/barretenberg/cpp/bootstrap.sh b/barretenberg/cpp/bootstrap.sh index 256d5695743c..ec3ce95b9fdb 100755 --- a/barretenberg/cpp/bootstrap.sh +++ b/barretenberg/cpp/bootstrap.sh @@ -287,7 +287,6 @@ function build_release_dir { export -f build_preset build_native_objects build_cross_objects build_native build_cross build_ios build_android build_asan_fast build_wasm build_wasm_threads build_gcc_syntax_check_only build_fuzzing_syntax_check_only build_smt_verification inject_version function build { - set -x echo_header "bb cpp build" if [ "$CI_FULL" -eq 1 ]; then @@ -410,6 +409,7 @@ function test_cmds { # This is not called in ci. It is just for a developer to run the tests. function test { set -x + set -e echo_header "bb test" test_cmds > /tmp/test_cmds filter_test_cmds < /tmp/test_cmds > /tmp/out diff --git a/bootstrap.sh b/bootstrap.sh index 47f27638d9c1..4724ab760fe4 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -699,7 +699,9 @@ case "$cmd" in export USE_TEST_CACHE=1 export AVM=0 export AVM_TRANSPILER=0 + set +e barretenberg/cpp/bootstrap.sh ci + echo $? ;; "ci-barretenberg-full") export CI=1 From 8224dffb37d597560e7f8cf1bfb996d8091d5f80 Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 26 Feb 2026 01:10:18 -0500 Subject: [PATCH 12/16] probe --- barretenberg/cpp/bootstrap.sh | 14 ++++++++++++-- ci3/filter_cached_test_cmd | 4 ++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/barretenberg/cpp/bootstrap.sh b/barretenberg/cpp/bootstrap.sh index ec3ce95b9fdb..c34555882ec9 100755 --- a/barretenberg/cpp/bootstrap.sh +++ b/barretenberg/cpp/bootstrap.sh @@ -409,12 +409,22 @@ function test_cmds { # This is not called in ci. It is just for a developer to run the tests. function test { set -x - set -e echo_header "bb test" test_cmds > /tmp/test_cmds + echo "test_cmds line count: $(wc -l < /tmp/test_cmds)" + set +e filter_test_cmds < /tmp/test_cmds > /tmp/out - echo "hi" ; cat /tmp/out ; echo "hey" + filter_exit=$? + set -e + echo "filter_test_cmds exit: $filter_exit" + echo "filter output line count: $(wc -l < /tmp/out)" + cat /tmp/out + set +e cat /tmp/out | parallelize + par_exit=$? + set -e + echo "parallelize exit: $par_exit" + [ $par_exit -ne 0 ] && exit $par_exit || true } function build_bench { diff --git a/ci3/filter_cached_test_cmd b/ci3/filter_cached_test_cmd index c8c9d25423af..a30785e4d947 100755 --- a/ci3/filter_cached_test_cmd +++ b/ci3/filter_cached_test_cmd @@ -34,9 +34,13 @@ if [ "${USE_TEST_CACHE:-0}" -eq 1 ] && [ "$CI_REDIS_AVAILABLE" -eq 1 ]; then # Process stdin in batches of 50. set +e parallel --pipe -N50 --keep-order process_batch 2>&1 + par_exit=$? + echo "filter_cached_test_cmd: parallel --pipe exit=$par_exit" >&2 set -e exec 3>&- wait + wait_exit=$? + echo "filter_cached_test_cmd: wait exit=$wait_exit" >&2 else # Noop. cat From 6a0edcdfe0f48925fd9900bedad23ab5f4d144af Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 26 Feb 2026 01:28:08 -0500 Subject: [PATCH 13/16] probe --- barretenberg/cpp/bootstrap.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/barretenberg/cpp/bootstrap.sh b/barretenberg/cpp/bootstrap.sh index c34555882ec9..ec97b1f504dc 100755 --- a/barretenberg/cpp/bootstrap.sh +++ b/barretenberg/cpp/bootstrap.sh @@ -419,12 +419,16 @@ function test { echo "filter_test_cmds exit: $filter_exit" echo "filter output line count: $(wc -l < /tmp/out)" cat /tmp/out + set +o pipefail set +e cat /tmp/out | parallelize - par_exit=$? + pipe_exit=$? + cat_exit=${PIPESTATUS[0]} + par_exit=${PIPESTATUS[1]} set -e - echo "parallelize exit: $par_exit" - [ $par_exit -ne 0 ] && exit $par_exit || true + set -o pipefail + echo "cat exit: $cat_exit, parallelize exit: $par_exit, pipeline exit: $pipe_exit" + [ $pipe_exit -ne 0 ] && exit $pipe_exit || true } function build_bench { From 6d8f41820aaf870bcb29e16c704649900e0ec5f0 Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 26 Feb 2026 01:41:21 -0500 Subject: [PATCH 14/16] probe --- barretenberg/cpp/bootstrap.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/barretenberg/cpp/bootstrap.sh b/barretenberg/cpp/bootstrap.sh index ec97b1f504dc..af43920c8f97 100755 --- a/barretenberg/cpp/bootstrap.sh +++ b/barretenberg/cpp/bootstrap.sh @@ -422,13 +422,13 @@ function test { set +o pipefail set +e cat /tmp/out | parallelize - pipe_exit=$? - cat_exit=${PIPESTATUS[0]} - par_exit=${PIPESTATUS[1]} + _ps=("${PIPESTATUS[@]}") + cat_exit=${_ps[0]} + par_exit=${_ps[1]} set -e set -o pipefail - echo "cat exit: $cat_exit, parallelize exit: $par_exit, pipeline exit: $pipe_exit" - [ $pipe_exit -ne 0 ] && exit $pipe_exit || true + echo "cat exit: $cat_exit, parallelize exit: $par_exit" + [ "${par_exit:-0}" -ne 0 ] && exit $par_exit || true } function build_bench { From 2dca16ea3bfbcabbd4de306fef88b2b9d94c087c Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 26 Feb 2026 02:06:27 -0500 Subject: [PATCH 15/16] fix: denoise stdin lost for background job - pass stdin explicitly to avoid /dev/null override --- barretenberg/cpp/bootstrap.sh | 21 +-------------------- bootstrap.sh | 3 --- ci3/denoise | 10 ++++++++-- ci3/filter_cached_test_cmd | 9 +-------- ci3/parallelize | 4 ++-- 5 files changed, 12 insertions(+), 35 deletions(-) diff --git a/barretenberg/cpp/bootstrap.sh b/barretenberg/cpp/bootstrap.sh index af43920c8f97..b2c78fd1eba2 100755 --- a/barretenberg/cpp/bootstrap.sh +++ b/barretenberg/cpp/bootstrap.sh @@ -408,27 +408,8 @@ function test_cmds { # This is not called in ci. It is just for a developer to run the tests. function test { - set -x echo_header "bb test" - test_cmds > /tmp/test_cmds - echo "test_cmds line count: $(wc -l < /tmp/test_cmds)" - set +e - filter_test_cmds < /tmp/test_cmds > /tmp/out - filter_exit=$? - set -e - echo "filter_test_cmds exit: $filter_exit" - echo "filter output line count: $(wc -l < /tmp/out)" - cat /tmp/out - set +o pipefail - set +e - cat /tmp/out | parallelize - _ps=("${PIPESTATUS[@]}") - cat_exit=${_ps[0]} - par_exit=${_ps[1]} - set -e - set -o pipefail - echo "cat exit: $cat_exit, parallelize exit: $par_exit" - [ "${par_exit:-0}" -ne 0 ] && exit $par_exit || true + test_cmds | filter_test_cmds | parallelize } function build_bench { diff --git a/bootstrap.sh b/bootstrap.sh index 4724ab760fe4..a241ff273bfd 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -694,14 +694,11 @@ case "$cmd" in barretenberg/cpp/bootstrap.sh build ;; "ci-barretenberg") - set -x export CI=1 export USE_TEST_CACHE=1 export AVM=0 export AVM_TRANSPILER=0 - set +e barretenberg/cpp/bootstrap.sh ci - echo $? ;; "ci-barretenberg-full") export CI=1 diff --git a/ci3/denoise b/ci3/denoise index 27107165a809..71100e439e3e 100755 --- a/ci3/denoise +++ b/ci3/denoise @@ -109,9 +109,15 @@ tail --sleep-interval=0.2 -n +1 -f "$outfile" > >( ) & tail_pid=$! -# Execute the command in background. -PARENT_LOG_ID=$key bash -c "$cmd" > >(redact > $outfile) 2>&1 & +# Save stdin to fd 9 so the background job can read from it. +# In non-interactive bash, background processes get /dev/null as stdin +# unless we explicitly redirect stdin (the "in the absence of explicit +# redirections" clause in the bash manual). +exec 9<&0 +# Execute the command in background, restoring saved stdin. +PARENT_LOG_ID=$key bash -c "$cmd" 0<&9 9>&- > >(redact > $outfile) 2>&1 & job_pid=$! +exec 9>&- # Close saved stdin in this process # Wait for the job to finish and get its exit status. wait $job_pid diff --git a/ci3/filter_cached_test_cmd b/ci3/filter_cached_test_cmd index a30785e4d947..04f288c05c92 100755 --- a/ci3/filter_cached_test_cmd +++ b/ci3/filter_cached_test_cmd @@ -29,18 +29,11 @@ function process_batch { export -f process_batch if [ "${USE_TEST_CACHE:-0}" -eq 1 ] && [ "$CI_REDIS_AVAILABLE" -eq 1 ]; then - set -x exec 3> >(cache_log "Skipped tests") # Process stdin in batches of 50. - set +e - parallel --pipe -N50 --keep-order process_batch 2>&1 - par_exit=$? - echo "filter_cached_test_cmd: parallel --pipe exit=$par_exit" >&2 - set -e + parallel --pipe -N50 --keep-order process_batch exec 3>&- wait - wait_exit=$? - echo "filter_cached_test_cmd: wait exit=$wait_exit" >&2 else # Noop. cat diff --git a/ci3/parallelize b/ci3/parallelize index 381e37cf737a..8fc7db08092e 100755 --- a/ci3/parallelize +++ b/ci3/parallelize @@ -54,7 +54,7 @@ echo "Completed run of $(tail -n+2 joblog.txt | grep -v "''" | wc -l) tests in $ # Process flake log (check thresholds and post PR comment in CI) if [ "$CI" -eq 1 ]; then - $root/scripts/process_flake_log.py "$FLAKES_FILE" --post-comment || true + $root/scripts/process_flake_log.py "$FLAKES_FILE" --post-comment else - $root/scripts/process_flake_log.py "$FLAKES_FILE" || true + $root/scripts/process_flake_log.py "$FLAKES_FILE" fi From eb1cb7af48f8fa5635ca2c250327c08aac92127a Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 26 Feb 2026 12:09:05 -0500 Subject: [PATCH 16/16] Try just a revert --- ci3/denoise | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/ci3/denoise b/ci3/denoise index 71100e439e3e..751484af2d9e 100755 --- a/ci3/denoise +++ b/ci3/denoise @@ -109,15 +109,9 @@ tail --sleep-interval=0.2 -n +1 -f "$outfile" > >( ) & tail_pid=$! -# Save stdin to fd 9 so the background job can read from it. -# In non-interactive bash, background processes get /dev/null as stdin -# unless we explicitly redirect stdin (the "in the absence of explicit -# redirections" clause in the bash manual). -exec 9<&0 -# Execute the command in background, restoring saved stdin. -PARENT_LOG_ID=$key bash -c "$cmd" 0<&9 9>&- > >(redact > $outfile) 2>&1 & +# Execute the command in background. +PARENT_LOG_ID=$key bash -c "$cmd" 2>&1 | redact > $outfile & job_pid=$! -exec 9>&- # Close saved stdin in this process # Wait for the job to finish and get its exit status. wait $job_pid