diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 434b5e16718e..4f54497acb36 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,6 @@ concurrency: env: CI_FAILFAST_TEST_LEAVE_DANGLING: 1 # GHA does not care about dangling processes and setting this variable avoids killing the CI script itself on error - REPO_USE_WARP_RUNNERS: 'bitcoin/bitcoin' # Use warp runners for this repo, instead of falling back to the slow GHA runners defaults: run: @@ -28,12 +27,16 @@ defaults: shell: bash jobs: - runners: - name: '[meta] determine runners' - runs-on: ubuntu-slim - outputs: - provider: ${{ steps.runners.outputs.provider }} + test-each-commit: + name: 'test ancestor commits' + runs-on: ${{ github.repository == 'bitcoin/bitcoin' && 'warp-ubuntu-latest-x64-8x' || 'ubuntu-latest' }} + env: + TEST_RUNNER_PORT_MIN: "14000" # Use a larger port range to avoid colliding with other CI services. + if: github.event_name == 'pull_request' && github.event.pull_request.commits != 1 + timeout-minutes: 360 # Use maximum time, see https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#jobsjob_idtimeout-minutes. steps: + - name: Determine fetch depth + run: echo "FETCH_DEPTH=$((${{ github.event.pull_request.commits }} + 2))" >> "$GITHUB_ENV" - &ANNOTATION_PR_NUMBER name: Annotate with pull request number # This annotation is machine-readable and can be used to assign a check @@ -44,28 +47,6 @@ jobs: if [ "${{ github.event_name }}" = "pull_request" ]; then echo "::notice title=debug_pull_request_number_str::${{ github.event.number }}" fi - - id: runners - run: | - if [[ "${REPO_USE_WARP_RUNNERS}" == "${{ github.repository }}" ]]; then - echo "provider=warp" >> "$GITHUB_OUTPUT" - echo "::notice title=Runner Selection::Using Warp Runners" - else - echo "provider=gha" >> "$GITHUB_OUTPUT" - echo "::notice title=Runner Selection::Using GitHub-hosted runners" - fi - - test-each-commit: - name: 'test ancestor commits' - needs: runners - runs-on: ${{ needs.runners.outputs.provider == 'warp' && 'warp-ubuntu-latest-x64-8x' || 'ubuntu-latest' }} - env: - TEST_RUNNER_PORT_MIN: "14000" # Use a larger port range to avoid colliding with other CI services. - if: github.event_name == 'pull_request' && github.event.pull_request.commits != 1 - timeout-minutes: 360 # Use maximum time, see https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#jobsjob_idtimeout-minutes. - steps: - - name: Determine fetch depth - run: echo "FETCH_DEPTH=$((${{ github.event.pull_request.commits }} + 2))" >> "$GITHUB_ENV" - - *ANNOTATION_PR_NUMBER - uses: actions/checkout@v6 with: ref: ${{ github.event.pull_request.head.sha }} @@ -324,8 +305,8 @@ jobs: windows-cross: name: 'Windows-cross to x86_64, ${{ matrix.crt }}' - needs: [runners, record-frozen-commit] - runs-on: ${{ needs.runners.outputs.provider == 'warp' && 'warp-ubuntu-latest-x64-4x' || 'ubuntu-latest' }} + needs: record-frozen-commit + runs-on: ${{ github.repository == 'bitcoin/bitcoin' && 'warp-ubuntu-latest-x64-4x' || 'ubuntu-latest' }} if: ${{ vars.SKIP_BRANCH_PUSH != 'true' || github.event_name == 'pull_request' }} strategy: @@ -359,7 +340,7 @@ jobs: id: restore-cache uses: ./.github/actions/cache/restore with: - provider: ${{ needs.runners.outputs.provider }} + provider: ${{ github.repository == 'bitcoin/bitcoin' && 'warp' || 'gha' }} - name: Configure Docker uses: ./.github/actions/configure-docker @@ -372,7 +353,7 @@ jobs: - name: Save caches uses: ./.github/actions/cache/save with: - provider: ${{ needs.runners.outputs.provider }} + provider: ${{ github.repository == 'bitcoin/bitcoin' && 'warp' || 'gha' }} - name: Upload built executables uses: actions/upload-artifact@v7 @@ -439,8 +420,7 @@ jobs: ci-matrix: name: ${{ matrix.name }} - needs: runners - runs-on: ${{ needs.runners.outputs.provider == 'warp' && matrix.warp-runner || matrix.fallback-runner }} + runs-on: ${{ github.repository == 'bitcoin/bitcoin' && matrix.warp-runner || matrix.fallback-runner }} if: ${{ vars.SKIP_BRANCH_PUSH != 'true' || github.event_name == 'pull_request' }} timeout-minutes: ${{ matrix.timeout-minutes }} @@ -534,7 +514,7 @@ jobs: - name: 'MSan, fuzz' warp-runner: 'warp-ubuntu-latest-x64-8x' fallback-runner: 'ubuntu-latest' - timeout-minutes: 150 + timeout-minutes: 180 file-env: './ci/test/00_setup_env_native_fuzz_with_msan.sh' - name: 'MSan' @@ -555,7 +535,7 @@ jobs: id: restore-cache uses: ./.github/actions/cache/restore with: - provider: ${{ matrix.provider || needs.runners.outputs.provider }} + provider: ${{ matrix.provider || (github.repository == 'bitcoin/bitcoin' && 'warp' || 'gha') }} - name: Configure Docker uses: ./.github/actions/configure-docker @@ -563,7 +543,7 @@ jobs: provider: ${{ matrix.provider || needs.runners.outputs.provider }} - name: Clear unnecessary files - if: ${{ needs.runners.outputs.provider == 'gha' && true || false }} # Only needed on GHA runners + if: ${{ github.repository != 'bitcoin/bitcoin' || matrix.provider == 'gha' }} # Only needed on GHA runners uses: ./.github/actions/clear-files - name: Enable bpfcc script @@ -583,12 +563,11 @@ jobs: - name: Save caches uses: ./.github/actions/cache/save with: - provider: ${{ matrix.provider || needs.runners.outputs.provider }} + provider: ${{ matrix.provider || (github.repository == 'bitcoin/bitcoin' && 'warp' || 'gha') }} lint: name: 'lint' - needs: runners - runs-on: ${{ needs.runners.outputs.provider == 'warp' && 'warp-ubuntu-latest-x64-2x' || 'ubuntu-latest' }} + runs-on: ${{ github.repository == 'bitcoin/bitcoin' && 'warp-ubuntu-latest-x64-2x' || 'ubuntu-latest' }} if: ${{ vars.SKIP_BRANCH_PUSH != 'true' || github.event_name == 'pull_request' }} timeout-minutes: 20 env: diff --git a/ci/README.md b/ci/README.md index 0b6a34460ca1..33c623dc2afa 100644 --- a/ci/README.md +++ b/ci/README.md @@ -97,5 +97,5 @@ To configure the primary repository, follow these steps: When used in a fork the CI will run on GitHub's free hosted runners by default. In this case, GitHub's cache size limitations may cause caches to be frequently evicted and missed, but the workflows will run (slowly). -It is also possible to use your own WarpBuild Runners in your own fork with an appropriate patch to the `REPO_USE_WARP_RUNNERS` variable in ../.github/workflows/ci.yml +It is also possible to use your own WarpBuild Runners in your own fork by replacing the references to `bitcoin/bitcoin` in `../.github/workflows/ci.yml` with the name of your fork (e.g. `your-org/bitcoin`). NB that WarpBuild Runners only work at an organisation level, therefore in order to use your own WarpBuild Runners, *the fork must be within your own organisation*. diff --git a/ci/test/00_setup_env_arm.sh b/ci/test/00_setup_env_arm.sh index f4ad97d2cc2b..71b519dab189 100755 --- a/ci/test/00_setup_env_arm.sh +++ b/ci/test/00_setup_env_arm.sh @@ -8,7 +8,8 @@ export LC_ALL=C.UTF-8 export HOST=arm-linux-gnueabihf export DPKG_ADD_ARCH="armhf" -export PACKAGES="python3-zmq g++-arm-linux-gnueabihf libc6:armhf libstdc++6:armhf libfontconfig1:armhf libxcb1:armhf" +export PACKAGES="python3-pip python3-zmq g++-arm-linux-gnueabihf libc6:armhf libstdc++6:armhf libfontconfig1:armhf libxcb1:armhf" +export PIP_PACKAGES="--break-system-packages pycapnp" export CONTAINER_NAME=ci_arm_linux export CI_IMAGE_NAME_TAG="mirror.gcr.io/debian:trixie" # Check that https://packages.debian.org/trixie/g++-arm-linux-gnueabihf (version 14.x, similar to guix) can cross-compile export CI_IMAGE_PLATFORM="linux/arm64" diff --git a/ci/test/00_setup_env_freebsd_cross.sh b/ci/test/00_setup_env_freebsd_cross.sh index da7304d1d26d..1be28a22c643 100755 --- a/ci/test/00_setup_env_freebsd_cross.sh +++ b/ci/test/00_setup_env_freebsd_cross.sh @@ -7,7 +7,7 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_freebsd_cross -export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:24.04" +export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:26.04" export APT_LLVM_V="22" export FREEBSD_VERSION=15.0 export PACKAGES="clang-${APT_LLVM_V} llvm-${APT_LLVM_V} lld" diff --git a/ci/test/00_setup_env_i686_no_ipc.sh b/ci/test/00_setup_env_i686_no_ipc.sh index 93d5ab744604..d8df8100fe56 100755 --- a/ci/test/00_setup_env_i686_no_ipc.sh +++ b/ci/test/00_setup_env_i686_no_ipc.sh @@ -8,10 +8,10 @@ export LC_ALL=C.UTF-8 export HOST=i686-pc-linux-gnu export CONTAINER_NAME=ci_i686_no_multiprocess -export CI_IMAGE_NAME_TAG="mirror.gcr.io/debian:trixie" +export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:26.04" export CI_IMAGE_PLATFORM="linux/amd64" export CI_CONTAINER_CAP="--security-opt seccomp=unconfined" -export PACKAGES="llvm clang g++-multilib" +export PACKAGES="g++-multilib" export DEP_OPTS="DEBUG=1 NO_IPC=1" export GOAL="install" export CI_LIMIT_STACK_SIZE=1 @@ -19,7 +19,5 @@ export BITCOIN_CONFIG="\ --preset=dev-mode \ -DENABLE_IPC=OFF \ -DCMAKE_BUILD_TYPE=Debug \ - -DCMAKE_C_COMPILER='clang;-m32' \ - -DCMAKE_CXX_COMPILER='clang++;-m32' \ -DAPPEND_CPPFLAGS='-DBOOST_MULTI_INDEX_ENABLE_SAFE_MODE' \ " diff --git a/ci/test/00_setup_env_native_alpine_musl.sh b/ci/test/00_setup_env_native_alpine_musl.sh index 3bad08d68538..3801dfec403c 100755 --- a/ci/test/00_setup_env_native_alpine_musl.sh +++ b/ci/test/00_setup_env_native_alpine_musl.sh @@ -7,7 +7,7 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_alpine_musl -export CI_IMAGE_NAME_TAG="mirror.gcr.io/alpine:3.23" +export CI_IMAGE_NAME_TAG="mirror.gcr.io/alpine:3.24" export CI_BASE_PACKAGES="build-base musl-dev pkgconf curl ccache make ninja git python3-dev py3-pip which patch xz procps rsync util-linux bison e2fsprogs cmake dash linux-headers" export PIP_PACKAGES="--break-system-packages pyzmq pycapnp" export DEP_OPTS="DEBUG=1" diff --git a/ci/test/00_setup_env_native_asan.sh b/ci/test/00_setup_env_native_asan.sh index 2465c70bb487..58be89fa1041 100755 --- a/ci/test/00_setup_env_native_asan.sh +++ b/ci/test/00_setup_env_native_asan.sh @@ -9,7 +9,7 @@ export LC_ALL=C.UTF-8 export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:24.04" # Only install BCC tracing packages in CI. Container has to match the host for BCC to work. -if [[ "${INSTALL_BCC_TRACING_TOOLS}" == "true" ]]; then +if [[ "${INSTALL_BCC_TRACING_TOOLS:-}" == "true" ]]; then # Required for USDT functional tests to run BPFCC_PACKAGE="bpfcc-tools linux-headers-$(uname --kernel-release)" export CI_CONTAINER_CAP="--privileged -v /sys/kernel:/sys/kernel:rw" diff --git a/ci/test/00_setup_env_native_fuzz.sh b/ci/test/00_setup_env_native_fuzz.sh index cdfe2a0d35b3..276ce08d056e 100755 --- a/ci/test/00_setup_env_native_fuzz.sh +++ b/ci/test/00_setup_env_native_fuzz.sh @@ -6,7 +6,7 @@ export LC_ALL=C.UTF-8 -export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:24.04" +export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:26.04" export CONTAINER_NAME=ci_native_fuzz export APT_LLVM_V="22" export PACKAGES="clang-${APT_LLVM_V} llvm-${APT_LLVM_V} libclang-rt-${APT_LLVM_V}-dev libevent-dev libboost-dev libsqlite3-dev libcapnp-dev capnproto" diff --git a/ci/test/00_setup_env_native_fuzz_with_msan.sh b/ci/test/00_setup_env_native_fuzz_with_msan.sh index 82d9044a8d70..c35a93622df8 100755 --- a/ci/test/00_setup_env_native_fuzz_with_msan.sh +++ b/ci/test/00_setup_env_native_fuzz_with_msan.sh @@ -6,7 +6,7 @@ export LC_ALL=C.UTF-8 -export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:24.04" +export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:26.04" export APT_LLVM_V="22" LIBCXX_DIR="/cxx_build/" export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls" diff --git a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh index ed84ae84cf35..8020ea43da7a 100755 --- a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh +++ b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh @@ -6,7 +6,7 @@ export LC_ALL=C.UTF-8 -export CI_IMAGE_NAME_TAG="mirror.gcr.io/debian:trixie" +export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:26.04" export CONTAINER_NAME=ci_native_fuzz_valgrind export PACKAGES="clang llvm libclang-rt-dev libevent-dev libboost-dev libsqlite3-dev valgrind libcapnp-dev capnproto" export NO_DEPENDS=1 diff --git a/ci/test/00_setup_env_native_iwyu.sh b/ci/test/00_setup_env_native_iwyu.sh index 27b1e02b6db8..3a26be56e7ea 100755 --- a/ci/test/00_setup_env_native_iwyu.sh +++ b/ci/test/00_setup_env_native_iwyu.sh @@ -6,7 +6,7 @@ export LC_ALL=C.UTF-8 -export CI_IMAGE_NAME_TAG="mirror.gcr.io/debian:trixie" # To build codegen, CMake must be 3.31 or newer. +export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:26.04" # To build codegen, CMake must be 3.31 or newer. export CONTAINER_NAME=ci_native_iwyu export IWYU_LLVM_V="22" export APT_LLVM_V="${IWYU_LLVM_V}" diff --git a/ci/test/00_setup_env_native_msan.sh b/ci/test/00_setup_env_native_msan.sh index 4a10bf96eabc..1d005abbd605 100755 --- a/ci/test/00_setup_env_native_msan.sh +++ b/ci/test/00_setup_env_native_msan.sh @@ -6,7 +6,7 @@ export LC_ALL=C.UTF-8 -export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:24.04" +export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:26.04" export APT_LLVM_V="22" LIBCXX_DIR="/cxx_build/" export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls" diff --git a/ci/test/00_setup_env_native_tidy.sh b/ci/test/00_setup_env_native_tidy.sh index b3cda17e8eed..5cebb8cd8ca6 100755 --- a/ci/test/00_setup_env_native_tidy.sh +++ b/ci/test/00_setup_env_native_tidy.sh @@ -6,7 +6,7 @@ export LC_ALL=C.UTF-8 -export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:24.04" +export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:26.04" export CONTAINER_NAME=ci_native_tidy export TIDY_LLVM_V="22" export APT_LLVM_V="${TIDY_LLVM_V}" diff --git a/ci/test/00_setup_env_native_valgrind.sh b/ci/test/00_setup_env_native_valgrind.sh index 0ca02f77824c..ba9d5b404aae 100755 --- a/ci/test/00_setup_env_native_valgrind.sh +++ b/ci/test/00_setup_env_native_valgrind.sh @@ -6,7 +6,7 @@ export LC_ALL=C.UTF-8 -export CI_IMAGE_NAME_TAG="mirror.gcr.io/debian:trixie" +export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:26.04" export CONTAINER_NAME=ci_native_valgrind export PACKAGES="clang llvm libclang-rt-dev valgrind python3-zmq libevent-dev libboost-dev libzmq3-dev libsqlite3-dev libcapnp-dev capnproto python3-pip" export PIP_PACKAGES="--break-system-packages pycapnp" diff --git a/ci/test/00_setup_env_s390x.sh b/ci/test/00_setup_env_s390x.sh index d1e34070ca99..a7c59d769635 100755 --- a/ci/test/00_setup_env_s390x.sh +++ b/ci/test/00_setup_env_s390x.sh @@ -9,7 +9,7 @@ export LC_ALL=C.UTF-8 export HOST=s390x-linux-gnu export PACKAGES="python3-zmq" export CONTAINER_NAME=ci_s390x -export CI_IMAGE_NAME_TAG="mirror.gcr.io/debian:trixie" +export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:26.04" export CI_IMAGE_PLATFORM="linux/s390x" # bind tests excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 export TEST_RUNNER_EXTRA="--exclude rpc_bind --exclude feature_bind_extra" diff --git a/ci/test/01_base_install.sh b/ci/test/01_base_install.sh index 512d1a32e87c..54dafbe8fd40 100755 --- a/ci/test/01_base_install.sh +++ b/ci/test/01_base_install.sh @@ -62,10 +62,13 @@ if [ -n "$PIP_PACKAGES" ]; then fi if [[ -n "${USE_INSTRUMENTED_LIBCPP}" ]]; then - ${CI_RETRY_EXE} git clone --depth=1 https://github.com/llvm/llvm-project -b "llvmorg-22.1.3" /llvm-project + ${CI_RETRY_EXE} git clone --depth=1 https://github.com/llvm/llvm-project -b "llvmorg-22.1.7" /llvm-project +# LLVM is configured with LIBCXXABI_USE_LLVM_UNWINDER=OFF, +# because libunwind doesn't handle exceptions under MSAN. +# https://github.com/llvm/llvm-project/issues/84348 cmake -G Ninja -B /cxx_build/ \ - -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \ + -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi" \ -DCMAKE_BUILD_TYPE=Release \ -DLLVM_USE_SANITIZER="${USE_INSTRUMENTED_LIBCPP}" \ -DCMAKE_C_COMPILER=clang \ @@ -86,7 +89,10 @@ fi if [[ "${RUN_IWYU}" == true ]]; then ${CI_RETRY_EXE} git clone --depth=1 https://github.com/include-what-you-use/include-what-you-use -b clang_"${IWYU_LLVM_V}" /include-what-you-use - (cd /include-what-you-use && patch -p1 < /ci_container_base/ci/test/01_iwyu.patch) + pushd /include-what-you-use + patch -p1 < /ci_container_base/ci/test/01_iwyu.patch + patch -p1 < /ci_container_base/ci/test/02_iwyu_hash.patch + popd cmake -B /iwyu-build/ -G 'Unix Makefiles' -DCMAKE_PREFIX_PATH=/usr/lib/llvm-"${IWYU_LLVM_V}" -S /include-what-you-use make -C /iwyu-build/ install "$MAKEJOBS" fi diff --git a/ci/test/02_iwyu_hash.patch b/ci/test/02_iwyu_hash.patch new file mode 100644 index 000000000000..12df7a04ed5c --- /dev/null +++ b/ci/test/02_iwyu_hash.patch @@ -0,0 +1,44 @@ +Map std::hash to its providing standard headers. +Backport of https://github.com/include-what-you-use/include-what-you-use/pull/2013 +(commit 52f85e1f4d990f55fc6556d543eb051d79364a16) to the clang_22 release +branch. Drop once the upstream fix lands in the IWYU branch tracked by +ci/test/00_setup_env_native_iwyu.sh. + +--- a/std_symbol_map.inc ++++ b/std_symbol_map.inc +@@ -1054,12 +1054,27 @@ + { "std::has_unique_object_representations_v", kPrivate, "", kPublic }, + { "std::has_virtual_destructor", kPrivate, "", kPublic }, + { "std::has_virtual_destructor_v", kPrivate, "", kPublic }, ++{ "std::hash", kPrivate, "", kPublic }, ++{ "std::hash", kPrivate, "", kPublic }, ++{ "std::hash", kPrivate, "", kPublic }, ++{ "std::hash", kPrivate, "", kPublic }, ++{ "std::hash", kPrivate, "", kPublic }, ++{ "std::hash", kPrivate, "", kPublic }, ++{ "std::hash", kPrivate, "", kPublic }, ++{ "std::hash", kPrivate, "", kPublic }, ++{ "std::hash", kPrivate, "", kPublic }, ++{ "std::hash", kPrivate, "", kPublic }, ++{ "std::hash", kPrivate, "", kPublic }, ++{ "std::hash", kPrivate, "", kPublic }, ++{ "std::hash", kPrivate, "", kPublic }, ++{ "std::hash", kPrivate, "", kPublic }, + { "std::hash>", kPrivate, "", kPublic }, + { "std::hash, :0>>", kPrivate, "", kPublic }, + { "std::hash, :0>>", kPrivate, "", kPublic }, + { "std::hash, :0>>", kPrivate, "", kPublic }, + { "std::hash, :0>>", kPrivate, "", kPublic }, + { "std::hash, :0>>", kPrivate, "", kPublic }, ++{ "std::hash>", kPrivate, "", kPublic }, + { "std::hash>", kPrivate, "", kPublic }, + { "std::hash", kPrivate, "", kPublic }, + { "std::hash", kPrivate, "", kPublic }, +@@ -1069,6 +1084,7 @@ + { "std::hash>", kPrivate, "", kPublic }, + { "std::hash", kPrivate, "", kPublic }, + { "std::hash", kPrivate, "", kPublic }, ++{ "std::hash", kPrivate, "", kPublic }, + { "std::hash", kPrivate, "", kPublic }, + { "std::hash", kPrivate, "", kPublic }, + { "std::hash", kPrivate, "", kPublic }, diff --git a/ci/test/03_test_script.sh b/ci/test/03_test_script.sh index 980583d00b56..c55f44301637 100755 --- a/ci/test/03_test_script.sh +++ b/ci/test/03_test_script.sh @@ -168,7 +168,7 @@ if [ -n "${CI_LIMIT_STACK_SIZE}" ]; then fi if [ -n "$USE_VALGRIND" ]; then - "${BASE_ROOT_DIR}/ci/test/wrap-valgrind.sh" + "${BASE_ROOT_DIR}/ci/test/wrap-valgrind.py" fi if [ "$RUN_CHECK_DEPS" = "true" ]; then @@ -229,7 +229,7 @@ fi if [[ "${RUN_IWYU}" == true ]]; then # TODO: Consider enforcing IWYU across the entire codebase. - FILES_WITH_ENFORCED_IWYU="/src/(((crypto|index|kernel|primitives|script|univalue/(lib|test)|util|zmq)/.*|bench/(block_assemble|connectblock)|common/license_info|node/(blockstorage|interfaces|miner|mining_args|utxo_snapshot)|rpc/mining|clientversion|core_io|signet|init)\\.cpp)" + FILES_WITH_ENFORCED_IWYU="/src/(((bench|crypto|index|kernel|primitives|script|univalue/(lib|test)|util|zmq)/.*|common/license_info|node/(blockstorage|interfaces|miner|mining_args|utxo_snapshot)|rpc/mining|clientversion|core_io|signet|init)\\.cpp)" jq --arg patterns "$FILES_WITH_ENFORCED_IWYU" 'map(select(.file | test($patterns)))' "${BASE_BUILD_DIR}/compile_commands.json" > "${BASE_BUILD_DIR}/compile_commands_iwyu_errors.json" jq --arg patterns "$FILES_WITH_ENFORCED_IWYU" 'map(select(.file | test($patterns) | not))' "${BASE_BUILD_DIR}/compile_commands.json" > "${BASE_BUILD_DIR}/compile_commands_iwyu_warnings.json" diff --git a/ci/test/wrap-valgrind.py b/ci/test/wrap-valgrind.py new file mode 100755 index 000000000000..6a79f2aaae1d --- /dev/null +++ b/ci/test/wrap-valgrind.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +# Copyright (c) The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or https://opensource.org/license/mit/. + +import os +import shlex +from pathlib import Path + + +def main(): + base_root = Path(os.environ["BASE_ROOT_DIR"]) + base_out = Path(os.environ["BASE_OUTDIR"]) + suppressions_file = base_root / "test" / "sanitizer_suppressions" / "valgrind.supp" + target_names = {b.name for b in (base_out / "bin").iterdir()} + + for exe in base_root.rglob("*"): + if exe.name in target_names and exe.is_file() and os.access(exe, os.X_OK): + print(f"Wrap {exe} ...") + original_path = exe.with_name(f"{exe.name}_orig") + exe.rename(original_path) + exe.write_text( + "#!/usr/bin/env bash\n" + "exec valgrind --gen-suppressions=all --quiet --error-exitcode=1 " + f"--suppressions={shlex.quote(str(suppressions_file))} " + f'{shlex.quote(str(original_path))} "$@"\n' + ) + exe.chmod(exe.stat().st_mode | 0o111) + + +if __name__ == "__main__": + main() diff --git a/ci/test/wrap-valgrind.sh b/ci/test/wrap-valgrind.sh deleted file mode 100755 index 4ed3f2d66c54..000000000000 --- a/ci/test/wrap-valgrind.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-present The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C.UTF-8 - -for b_name in "${BASE_OUTDIR}/bin"/*; do - # shellcheck disable=SC2044 - for b in $(find "${BASE_ROOT_DIR}" -executable -type f -name "$(basename "$b_name")"); do - echo "Wrap $b ..." - mv "$b" "${b}_orig" - echo '#!/usr/bin/env bash' > "$b" - echo "exec valgrind --gen-suppressions=all --quiet --error-exitcode=1 --suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/valgrind.supp \"${b}_orig\" \"\$@\"" >> "$b" - chmod +x "$b" - done -done diff --git a/ci/test_imagefile b/ci/test_imagefile index 908e9a0f5ab4..dac6b5531564 100644 --- a/ci/test_imagefile +++ b/ci/test_imagefile @@ -16,7 +16,8 @@ ENV BASE_ROOT_DIR=${BASE_ROOT_DIR} # Make retry available in PATH, needed for CI_RETRY_EXE COPY ./ci/retry/retry /usr/bin/retry -COPY ./ci/test/00_setup_env.sh ./${FILE_ENV} ./ci/test/01_base_install.sh ./ci/test/01_iwyu.patch /ci_container_base/ci/test/ +COPY ./ci/test/00_setup_env.sh ./${FILE_ENV} ./ci/test/01_base_install.sh /ci_container_base/ci/test/ +COPY ./ci/test/*.patch /ci_container_base/ci/test/ # Bash is required, so install it when missing RUN sh -c "bash -c 'true' || ( apk update && apk add --no-cache bash )" diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index 51354273efae..0ec058b8de32 100755 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -5,73 +5,13 @@ export LC_ALL=C set -e -o pipefail -# Environment variables for determinism -export TAR_OPTIONS="--no-same-owner --owner=0 --group=0 --numeric-owner --mtime='@${SOURCE_DATE_EPOCH}' --sort=name" -export TZ=UTC - -# Although Guix _does_ set umask when building its own packages (in our case, -# this is all packages in manifest.scm), it does not set it for `guix -# shell`. It does make sense for at least `guix shell --container` -# to set umask, so if that change gets merged upstream and we bump the -# time-machine to a commit which includes the aforementioned change, we can -# remove this line. -# -# This line should be placed before any commands which creates files. -umask 0022 - -if [ -n "$V" ]; then - # Print both unexpanded (-v) and expanded (-x) forms of commands as they are - # read from this file. - set -vx - # Set VERBOSE for CMake-based builds - export VERBOSE="$V" -fi - -# Check that required environment variables are set -cat << EOF -Required environment variables as seen inside the container: - DIST_ARCHIVE_BASE: ${DIST_ARCHIVE_BASE:?not set} - DISTNAME: ${DISTNAME:?not set} - HOST: ${HOST:?not set} - SOURCE_DATE_EPOCH: ${SOURCE_DATE_EPOCH:?not set} - JOBS: ${JOBS:?not set} - DISTSRC: ${DISTSRC:?not set} - OUTDIR: ${OUTDIR:?not set} -EOF - -ACTUAL_OUTDIR="${OUTDIR}" -OUTDIR="${DISTSRC}/output" - -##################### -# Environment Setup # -##################### - -# The depends folder also serves as a base-prefix for depends packages for -# $HOSTs after successfully building. -BASEPREFIX="${PWD}/depends" - -# Given a package name and an output name, return the path of that output in our -# current guix environment -store_path() { - grep --extended-regexp "/[^-]{32}-${1}-[^-]+${2:+-${2}}" "${GUIX_ENVIRONMENT}/manifest" \ - | head --lines=1 \ - | sed --expression='s|\x29*$||' \ - --expression='s|^[[:space:]]*"||' \ - --expression='s|"[[:space:]]*$||' -} - +# shellcheck source=setup.sh +source "$(dirname "${BASH_SOURCE[0]}")/setup.sh" # Set environment variables to point the NATIVE toolchain to the right # includes/libs NATIVE_GCC="$(store_path gcc-toolchain)" -unset LIBRARY_PATH -unset CPATH -unset C_INCLUDE_PATH -unset CPLUS_INCLUDE_PATH -unset OBJC_INCLUDE_PATH -unset OBJCPLUS_INCLUDE_PATH - # Set native toolchain build_CC="${NATIVE_GCC}/bin/gcc -isystem ${NATIVE_GCC}/include" build_CXX="${NATIVE_GCC}/bin/g++ -isystem ${NATIVE_GCC}/include/c++ -isystem ${NATIVE_GCC}/include" @@ -134,15 +74,6 @@ for p in "${PATHS[@]}"; do fi done -# Disable Guix ld auto-rpath behavior -export GUIX_LD_WRAPPER_DISABLE_RPATH=yes - -# Make /usr/bin if it doesn't exist -[ -e /usr/bin ] || mkdir -p /usr/bin - -# Symlink env to a conventional path -[ -e /usr/bin/env ] || ln -s --no-dereference "$(command -v env)" /usr/bin/env - # Determine the correct value for -Wl,--dynamic-linker for the current $HOST case "$HOST" in *linux*) @@ -186,20 +117,6 @@ case "$HOST" in ;; esac -########################### -# Source Tarball Building # -########################### - -GIT_ARCHIVE="${DIST_ARCHIVE_BASE}/${DISTNAME}.tar.gz" - -# Create the source tarball if not already there -if [ ! -e "$GIT_ARCHIVE" ]; then - mkdir -p "$(dirname "$GIT_ARCHIVE")" - git archive --prefix="${DISTNAME}/" --output="$GIT_ARCHIVE" HEAD -fi - -mkdir -p "$OUTDIR" - ########################### # Binary Tarball Building # ########################### @@ -249,17 +166,17 @@ mkdir -p "$DISTSRC" -DWITH_CCACHE=OFF \ -Werror=dev \ ${CONFIGFLAGS} \ - "${CMAKE_EXE_LINKER_FLAGS}" + ${CMAKE_EXE_LINKER_FLAGS+"$CMAKE_EXE_LINKER_FLAGS"} # Build Bitcoin Core - cmake --build build -j "$JOBS" ${V:+--verbose} + cmake --build build -j "$JOBS" mkdir -p "$OUTDIR" # Make the os-specific installers case "$HOST" in *mingw*) - cmake --build build -j "$JOBS" -t deploy ${V:+--verbose} + cmake --build build -j "$JOBS" -t deploy mv build/bitcoin-win64-setup.exe "${OUTDIR}/${DISTNAME}-win64-setup-unsigned.exe" ;; esac @@ -267,15 +184,14 @@ mkdir -p "$DISTSRC" # Setup the directory where our Bitcoin Core build for HOST will be # installed. This directory will also later serve as the input for our # binary tarballs. - INSTALLPATH="${PWD}/installed/${DISTNAME}" mkdir -p "${INSTALLPATH}" # Install built Bitcoin Core to $INSTALLPATH case "$HOST" in *darwin*) - cmake --install build --strip --prefix "${INSTALLPATH}" ${V:+--verbose} + cmake --install build --strip --prefix "${INSTALLPATH}" ;; *) - cmake --install build --prefix "${INSTALLPATH}" ${V:+--verbose} + cmake --install build --prefix "${INSTALLPATH}" ;; esac @@ -285,126 +201,7 @@ mkdir -p "$DISTSRC" # Check that executables only contain allowed version symbols. echo "Running symbol and dynamic library checks on installed executables..." python3 "${DISTSRC}/contrib/guix/symbol-check.py" "${INSTALLPATH}/bin/"* "${INSTALLPATH}/libexec/"* - - ( - cd installed - - case "$HOST" in - *darwin*) ;; - *) - # Split binaries from their debug symbols - { - find "${DISTNAME}/bin" "${DISTNAME}/libexec" -type f -executable -print0 - } | xargs -0 -P"$JOBS" -I{} "${DISTSRC}/build/split-debug.sh" {} {} {}.dbg - ;; - esac - - case "$HOST" in - *mingw*) - cp "${DISTSRC}/doc/README_windows.txt" "${DISTNAME}/readme.txt" - ;; - *linux*) - cp "${DISTSRC}/README.md" "${DISTNAME}/" - cp "${DISTSRC}/doc/INSTALL_linux.md" "${DISTNAME}/INSTALL.md" - ;; - esac - - # copy over the example bitcoin.conf file. if contrib/devtools/gen-bitcoin-conf.sh - # has not been run before buildling, this file will be a stub - cp "${DISTSRC}/share/examples/bitcoin.conf" "${DISTNAME}/" - - cp -r "${DISTSRC}/share/rpcauth" "${DISTNAME}/share/" - - # Deterministically produce {non-,}debug binary tarballs ready - # for release - case "$HOST" in - *mingw*) - find "${DISTNAME}" -not -name "*.dbg" -print0 \ - | xargs -0r touch --no-dereference --date="@${SOURCE_DATE_EPOCH}" - find "${DISTNAME}" -not -name "*.dbg" \ - | sort \ - | zip -X@ "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}-unsigned.zip" \ - || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}-unsigned.zip" && exit 1 ) - find "${DISTNAME}" -name "*.dbg" -print0 \ - | xargs -0r touch --no-dereference --date="@${SOURCE_DATE_EPOCH}" - find "${DISTNAME}" -name "*.dbg" \ - | sort \ - | zip -X@ "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}-debug.zip" \ - || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}-debug.zip" && exit 1 ) - ;; - *linux*) - find "${DISTNAME}" -not -name "*.dbg" -print0 \ - | sort --zero-terminated \ - | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ - | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" \ - || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" && exit 1 ) - find "${DISTNAME}" -name "*.dbg" -print0 \ - | sort --zero-terminated \ - | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ - | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}-debug.tar.gz" \ - || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}-debug.tar.gz" && exit 1 ) - ;; - *darwin*) - find "${DISTNAME}" -print0 \ - | sort --zero-terminated \ - | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ - | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}-unsigned.tar.gz" \ - || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}-unsigned.tar.gz" && exit 1 ) - ;; - esac - ) # $DISTSRC/installed - - # Finally make tarballs for codesigning - case "$HOST" in - *mingw*) - cp -rf --target-directory=. contrib/windeploy - ( - cd ./windeploy - mkdir -p unsigned - cp --target-directory=unsigned/ "${OUTDIR}/${DISTNAME}-win64-setup-unsigned.exe" - cp -r --target-directory=unsigned/ "${INSTALLPATH}" - find unsigned/ -name "*.dbg" -print0 \ - | xargs -0r rm - find . -print0 \ - | sort --zero-terminated \ - | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ - | gzip -9n > "${OUTDIR}/${DISTNAME}-win64-codesigning.tar.gz" \ - || ( rm -f "${OUTDIR}/${DISTNAME}-win64-codesigning.tar.gz" && exit 1 ) - ) - ;; - *darwin*) - cmake --build build --target deploy ${V:+--verbose} - mv build/dist/bitcoin-macos-app.zip "${OUTDIR}/${DISTNAME}-${HOST}-unsigned.zip" - mkdir -p "unsigned-app-${HOST}" - cp --target-directory="unsigned-app-${HOST}" \ - contrib/macdeploy/detached-sig-create.sh - mv --target-directory="unsigned-app-${HOST}" build/dist - cp -r --target-directory="unsigned-app-${HOST}" "${INSTALLPATH}" - ( - cd "unsigned-app-${HOST}" - find . -print0 \ - | sort --zero-terminated \ - | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ - | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}-codesigning.tar.gz" \ - || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}-codesigning.tar.gz" && exit 1 ) - ) - ;; - esac ) # $DISTSRC -rm -rf "$ACTUAL_OUTDIR" -mv --no-target-directory "$OUTDIR" "$ACTUAL_OUTDIR" \ - || ( rm -rf "$ACTUAL_OUTDIR" && exit 1 ) - -( - tmp="$(mktemp)" - cd /outdir-base - { - echo "$GIT_ARCHIVE" - find "$ACTUAL_OUTDIR" -type f - } | xargs realpath --relative-base="$PWD" \ - | xargs sha256sum \ - | sort -k2 \ - > "$tmp"; - mv "$tmp" "$ACTUAL_OUTDIR"/SHA256SUMS.part -) +# shellcheck source=package.sh +source "$(dirname "${BASH_SOURCE[0]}")/package.sh" diff --git a/contrib/guix/libexec/package.sh b/contrib/guix/libexec/package.sh new file mode 100755 index 000000000000..7228346abe7a --- /dev/null +++ b/contrib/guix/libexec/package.sh @@ -0,0 +1,133 @@ +#!/usr/bin/env bash +# Copyright (c) The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or https://opensource.org/license/mit. +export LC_ALL=C +set -e -o pipefail + +( + cd "$DISTSRC" + + ( + cd installed + + case "$HOST" in + *darwin*) ;; + *) + # Split binaries from their debug symbols + { + find "${DISTNAME}/bin" "${DISTNAME}/libexec" -type f -executable -print0 + } | xargs -0 -P"$JOBS" -I{} "${DISTSRC}/build/split-debug.sh" {} {} {}.dbg + ;; + esac + + case "$HOST" in + *mingw*) + cp "${DISTSRC}/doc/README_windows.txt" "${DISTNAME}/readme.txt" + ;; + *linux*) + cp "${DISTSRC}/README.md" "${DISTNAME}/" + cp "${DISTSRC}/doc/INSTALL_linux.md" "${DISTNAME}/INSTALL.md" + ;; + esac + + # copy over the example bitcoin.conf file. if contrib/devtools/gen-bitcoin-conf.sh + # has not been run before buildling, this file will be a stub + cp "${DISTSRC}/share/examples/bitcoin.conf" "${DISTNAME}/" + + cp -r "${DISTSRC}/share/rpcauth" "${DISTNAME}/share/" + + # Deterministically produce {non-,}debug binary tarballs ready + # for release + case "$HOST" in + *mingw*) + find "${DISTNAME}" -not -name "*.dbg" -print0 \ + | xargs -0r touch --no-dereference --date="@${SOURCE_DATE_EPOCH}" + find "${DISTNAME}" -not -name "*.dbg" \ + | sort \ + | zip -X@ "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}-unsigned.zip" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}-unsigned.zip" && exit 1 ) + find "${DISTNAME}" -name "*.dbg" -print0 \ + | xargs -0r touch --no-dereference --date="@${SOURCE_DATE_EPOCH}" + find "${DISTNAME}" -name "*.dbg" \ + | sort \ + | zip -X@ "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}-debug.zip" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}-debug.zip" && exit 1 ) + ;; + *linux*) + find "${DISTNAME}" -not -name "*.dbg" -print0 \ + | sort --zero-terminated \ + | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ + | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" && exit 1 ) + find "${DISTNAME}" -name "*.dbg" -print0 \ + | sort --zero-terminated \ + | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ + | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}-debug.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}-debug.tar.gz" && exit 1 ) + ;; + *darwin*) + find "${DISTNAME}" -print0 \ + | sort --zero-terminated \ + | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ + | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}-unsigned.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}-unsigned.tar.gz" && exit 1 ) + ;; + esac + ) # $DISTSRC/installed + + # Finally make tarballs for codesigning + case "$HOST" in + *mingw*) + cp -rf --target-directory=. contrib/windeploy + ( + cd ./windeploy + mkdir -p unsigned + cp --target-directory=unsigned/ "${OUTDIR}/${DISTNAME}-win64-setup-unsigned.exe" + cp -r --target-directory=unsigned/ "${INSTALLPATH}" + find unsigned/ -name "*.dbg" -print0 \ + | xargs -0r rm + find . -print0 \ + | sort --zero-terminated \ + | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ + | gzip -9n > "${OUTDIR}/${DISTNAME}-win64-codesigning.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-win64-codesigning.tar.gz" && exit 1 ) + ) + ;; + *darwin*) + cmake --build build --target deploy + mv build/dist/bitcoin-macos-app.zip "${OUTDIR}/${DISTNAME}-${HOST}-unsigned.zip" + mkdir -p "unsigned-app-${HOST}" + cp --target-directory="unsigned-app-${HOST}" \ + contrib/macdeploy/detached-sig-create.sh + mv --target-directory="unsigned-app-${HOST}" build/dist + cp -r --target-directory="unsigned-app-${HOST}" "${INSTALLPATH}" + ( + cd "unsigned-app-${HOST}" + find . -print0 \ + | sort --zero-terminated \ + | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ + | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}-codesigning.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}-codesigning.tar.gz" && exit 1 ) + ) + ;; + esac + +) # $DISTSRC + +rm -rf "$ACTUAL_OUTDIR" +mv --no-target-directory "$OUTDIR" "$ACTUAL_OUTDIR" \ + || ( rm -rf "$ACTUAL_OUTDIR" && exit 1 ) + +( + tmp="$(mktemp)" + cd /outdir-base + { + echo "$GIT_ARCHIVE" + find "$ACTUAL_OUTDIR" -type f + } | xargs realpath --relative-base="$PWD" \ + | xargs sha256sum \ + | sort -k2 \ + > "$tmp"; + mv "$tmp" "$ACTUAL_OUTDIR"/SHA256SUMS.part +) diff --git a/contrib/guix/libexec/setup.sh b/contrib/guix/libexec/setup.sh new file mode 100755 index 000000000000..37388f985538 --- /dev/null +++ b/contrib/guix/libexec/setup.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# Copyright (c) The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or https://opensource.org/license/mit. +export LC_ALL=C +set -e -o pipefail + +# Environment variables for determinism +export TAR_OPTIONS="--no-same-owner --owner=0 --group=0 --numeric-owner --mtime='@${SOURCE_DATE_EPOCH}' --sort=name" +export TZ=UTC + +# Although Guix _does_ set umask when building its own packages (in our case, +# this is all packages in manifest.scm), it does not set it for `guix +# shell`. It does make sense for at least `guix shell --container` +# to set umask, so if that change gets merged upstream and we bump the +# time-machine to a commit which includes the aforementioned change, we can +# remove this line. +# +# This line should be placed before any commands which creates files. +umask 0022 + +if [ -n "$V" ]; then + # Print both unexpanded (-v) and expanded (-x) forms of commands as they are + # read from this file. + set -vx + # Set VERBOSE for CMake-based builds + export VERBOSE="$V" +fi + +# Check that required environment variables are set +cat << EOF +Required environment variables as seen inside the container: + DIST_ARCHIVE_BASE: ${DIST_ARCHIVE_BASE:?not set} + DISTNAME: ${DISTNAME:?not set} + HOST: ${HOST:?not set} + SOURCE_DATE_EPOCH: ${SOURCE_DATE_EPOCH:?not set} + JOBS: ${JOBS:?not set} + DISTSRC: ${DISTSRC:?not set} + OUTDIR: ${OUTDIR:?not set} +EOF + +export ACTUAL_OUTDIR="${OUTDIR}" +export OUTDIR="${DISTSRC}/output" +export INSTALLPATH="${DISTSRC}/installed/${DISTNAME}" + +##################### +# Environment Setup # +##################### + +# The depends folder also serves as a base-prefix for depends packages for +# $HOSTs after successfully building. +export BASEPREFIX="${PWD}/depends" + +# Given a package name and an output name, return the path of that output in our +# current guix environment +store_path() { + grep --extended-regexp "/[^-]{32}-${1}-[^-]+${2:+-${2}}" "${GUIX_ENVIRONMENT}/manifest" \ + | head --lines=1 \ + | sed --expression='s|\x29*$||' \ + --expression='s|^[[:space:]]*"||' \ + --expression='s|"[[:space:]]*$||' +} + +# Disable Guix ld auto-rpath behavior +export GUIX_LD_WRAPPER_DISABLE_RPATH=yes + +# Make /usr/bin if it doesn't exist +[ -e /usr/bin ] || mkdir -p /usr/bin + +# Symlink env to a conventional path +[ -e /usr/bin/env ] || ln -s --no-dereference "$(command -v env)" /usr/bin/env + +########################### +# Source Tarball Building # +########################### + +GIT_ARCHIVE="${DIST_ARCHIVE_BASE}/${DISTNAME}.tar.gz" + +# Create the source tarball if not already there +if [ ! -e "$GIT_ARCHIVE" ]; then + mkdir -p "$(dirname "$GIT_ARCHIVE")" + git archive --prefix="${DISTNAME}/" --output="$GIT_ARCHIVE" HEAD +fi + +mkdir -p "$OUTDIR" + +unset LIBRARY_PATH +unset CPATH +unset C_INCLUDE_PATH +unset CPLUS_INCLUDE_PATH +unset OBJC_INCLUDE_PATH +unset OBJCPLUS_INCLUDE_PATH diff --git a/depends/funcs.mk b/depends/funcs.mk index 60e7889a4ec5..f6221ef18f8c 100644 --- a/depends/funcs.mk +++ b/depends/funcs.mk @@ -222,7 +222,7 @@ $(1)_cmake=env CC="$$($(1)_cc)" \ -DCMAKE_AR=`which $$($(1)_ar)` \ -DCMAKE_NM=`which $$($(1)_nm)` \ -DCMAKE_RANLIB=`which $$($(1)_ranlib)` \ - -DCMAKE_INSTALL_LIBDIR=lib/ \ + -DCMAKE_INSTALL_LIBDIR=lib \ -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ -DCMAKE_VERBOSE_MAKEFILE:BOOL=$(V) \ -DCMAKE_EXPORT_NO_PACKAGE_REGISTRY:BOOL=TRUE \ diff --git a/doc/bips.md b/doc/bips.md index ebf6b8fcd7d7..6f9b322e33da 100644 --- a/doc/bips.md +++ b/doc/bips.md @@ -79,3 +79,4 @@ BIPs that are implemented by Bitcoin Core: * [`BIP 390`](https://github.com/bitcoin/bips/blob/master/bip-0390.mediawiki): MuSig2 Descriptor parsing is implemented in **v30.0** ([PR 31244](https://github.com/bitcoin/bitcoin/pull/31244)) and signing in **v31.0** ([PR 29675](https://github.com/bitcoin/bitcoin/pull/29675)) * [`BIP 431`](https://github.com/bitcoin/bips/blob/master/bip-0431.mediawiki): transactions with nVersion=3 are standard and treated as Topologically Restricted Until Confirmation as of **v28.0** ([PR 29496](https://github.com/bitcoin/bitcoin/pull/29496)). * [`BIP 433`](https://github.com/bitcoin/bips/blob/master/bip-0433.mediawiki): Spending of Pay to Anchor (P2A) outputs is standard as of **v28.0** ([PR 30352](https://github.com/bitcoin/bitcoin/pull/30352)). +* [`BIP 434`](https://github.com/bitcoin/bips/blob/master/bip-0434.md): Peer Feature Negotiation as of **v32.0** ([PR 35221](https://github.com/bitcoin/bitcoin/pull/35221)). diff --git a/doc/developer-notes.md b/doc/developer-notes.md index e44a4b91093a..962e8851187c 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -562,9 +562,8 @@ For IWYU pragmas, prefer adding a nearby source comment that explains why the an Profiling is a good way to get a precise idea of where time is being spent in code. One tool for doing profiling on Linux platforms is called -[`perf`](https://www.brendangregg.com/perf.html), and has been integrated into -the functional test framework. Perf can observe a running process and sample -(at some frequency) where its execution is. +[`perf`](https://www.brendangregg.com/perf.html). It can observe a running +process and sample (at some frequency) where its execution is. Perf installation is contingent on which kernel version you're running; see [this thread](https://askubuntu.com/questions/50145/how-to-install-perf-monitoring-tool) @@ -599,8 +598,6 @@ perf report --stdio | c++filt | less or using a graphical tool like [Hotspot](https://github.com/KDAB/hotspot). -See the functional test documentation for how to invoke perf within tests. - ### Valgrind Valgrind is a programming tool for memory debugging, memory leak detection, and @@ -702,16 +699,16 @@ and its `cs_KeyStore` lock for example). : Performs various loading tasks that are part of init but shouldn't block the node from being started: external block import, reindex, reindex-chainstate, main chain activation, spawn indexes background sync threads and mempool load. -- [CCheckQueue::Loop (`b-scriptch.x`)](https://doxygen.bitcoincore.org/class_c_check_queue.html#checkqueue) +- [CCheckQueue::Loop (`b-scriptch.xx`)](https://doxygen.bitcoincore.org/class_c_check_queue.html#checkqueue) : Parallel script validation threads for transactions in blocks. - [ThreadHTTP (`b-http`)](https://doxygen.bitcoincore.org/httpserver_8cpp.html#http) : Libevent thread to listen for RPC and REST connections. -- [HTTP worker threads (`b-http_pool_x`)](https://doxygen.bitcoincore.org/httpserver_8cpp.html#http_pool) +- [HTTP worker threads (`b-http.xx`)](https://doxygen.bitcoincore.org/httpserver_8cpp.html#http_pool) : Threads to service RPC and REST requests. -- [Indexer threads (`b-txindex`, etc)](https://doxygen.bitcoincore.org/class_base_index.html#index_sync) +- [Indexer threads (`b-txidx`, `b-blkfltbscidx`, `b-coinstatsidx`, `b-txospenderidx`)](https://doxygen.bitcoincore.org/class_base_index.html#index_sync) : One thread per indexer. - [SchedulerThread (`b-scheduler`)](https://doxygen.bitcoincore.org/class_c_scheduler.html#scheduler) diff --git a/doc/release-notes-35267.md b/doc/release-notes-35267.md new file mode 100644 index 000000000000..ff9ad4550ef9 --- /dev/null +++ b/doc/release-notes-35267.md @@ -0,0 +1,4 @@ +RPC +--- + +The `getprivatebroadcastinfo` and `abortprivatebroadcast` RPCs now return an error `-32601`, "Method not found", when `-privatebroadcast` is not enabled at startup. diff --git a/src/addrman.cpp b/src/addrman.cpp index 92178f30089b..3050beb7968c 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include diff --git a/src/addrman.h b/src/addrman.h index 94e7d3e65379..f714338610c1 100644 --- a/src/addrman.h +++ b/src/addrman.h @@ -7,18 +7,22 @@ #define BITCOIN_ADDRMAN_H #include -#include #include -#include #include +#include #include +#include #include #include +#include +#include #include #include #include +class NetGroupManager; + /** Over how many buckets entries with tried addresses from a single group (/16 for IPv4) are spread */ static constexpr uint32_t ADDRMAN_TRIED_BUCKETS_PER_GROUP{8}; /** Over how many buckets entries with new addresses originating from a single group are spread */ diff --git a/src/base58.h b/src/base58.h index f258163bc3da..1656cbe1d394 100644 --- a/src/base58.h +++ b/src/base58.h @@ -14,8 +14,7 @@ #ifndef BITCOIN_BASE58_H #define BITCOIN_BASE58_H -#include - +#include #include #include diff --git a/src/bech32.h b/src/bech32.h index ba4c79bb8695..9a43a58f396c 100644 --- a/src/bech32.h +++ b/src/bech32.h @@ -14,8 +14,10 @@ #ifndef BITCOIN_BECH32_H #define BITCOIN_BECH32_H +#include #include #include +#include #include namespace bech32 diff --git a/src/bench/addrman.cpp b/src/bench/addrman.cpp index 703b4d245db4..5ce7893d5365 100644 --- a/src/bench/addrman.cpp +++ b/src/bench/addrman.cpp @@ -10,13 +10,13 @@ #include #include #include -#include #include #include #include #include #include +#include #include /* A "source" is a source address from which we have received a bunch of other addresses. */ diff --git a/src/bench/asmap.cpp b/src/bench/asmap.cpp index e834ea04a278..92b048f746e3 100644 --- a/src/bench/asmap.cpp +++ b/src/bench/asmap.cpp @@ -8,10 +8,10 @@ #include #include #include +#include #include #include -#include #include #include diff --git a/src/bench/base58.cpp b/src/bench/base58.cpp index 0be16439c3c1..ed36dc32cfb9 100644 --- a/src/bench/base58.cpp +++ b/src/bench/base58.cpp @@ -4,7 +4,6 @@ #include #include -#include #include #include diff --git a/src/bench/bech32.cpp b/src/bench/bech32.cpp index c5e864aed03e..1147bd6d2f6c 100644 --- a/src/bench/bech32.cpp +++ b/src/bench/bech32.cpp @@ -6,6 +6,7 @@ #include #include +#include #include using namespace util::hex_literals; diff --git a/src/bench/bench.cpp b/src/bench/bench.cpp index 45d1b360f659..4d1f4da03290 100644 --- a/src/bench/bench.cpp +++ b/src/bench/bench.cpp @@ -7,19 +7,18 @@ #include // IWYU pragma: keep #include #include +#include -#include #include #include #include #include #include #include +#include #include #include -using namespace std::chrono_literals; - /** * Retrieves the available test setup command line arguments that may be used * in the benchmark. They will be used only if the benchmark utilizes a @@ -68,9 +67,9 @@ BenchRunner::BenchmarkMap& BenchRunner::benchmarks() return benchmarks_map; } -BenchRunner::BenchRunner(std::string name, BenchFunction func) +BenchRunner::BenchRunner(std::string_view name, BenchFunction func) { - Assert(benchmarks().try_emplace(std::move(name), std::move(func)).second); + Assert(benchmarks().try_emplace(std::string{name}, std::move(func)).second); } void BenchRunner::RunAll(const Args& args) diff --git a/src/bench/bench.h b/src/bench/bench.h index f7df42a30384..54c75bc21cab 100644 --- a/src/bench/bench.h +++ b/src/bench/bench.h @@ -8,11 +8,12 @@ #include // IWYU pragma: export #include #include +#include -#include #include #include #include +#include #include /* @@ -58,7 +59,7 @@ class BenchRunner static BenchmarkMap& benchmarks(); public: - BenchRunner(std::string name, BenchFunction func); + BenchRunner(std::string_view name, BenchFunction func); static void RunAll(const Args& args); }; diff --git a/src/bench/bench_bitcoin.cpp b/src/bench/bench_bitcoin.cpp index 987523a128b5..fa0c170f93f4 100644 --- a/src/bench/bench_bitcoin.cpp +++ b/src/bench/bench_bitcoin.cpp @@ -3,19 +3,19 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include -#include #include +#include #include #include -#include -#include +#include -#include #include #include #include #include +#include #include +#include #include static const char* DEFAULT_BENCH_FILTER = ".*"; diff --git a/src/bench/bip324_ecdh.cpp b/src/bench/bip324_ecdh.cpp index 65deb8b74e9e..c8d3a35efc53 100644 --- a/src/bench/bip324_ecdh.cpp +++ b/src/bench/bip324_ecdh.cpp @@ -11,6 +11,7 @@ #include #include #include +#include static void BIP324_ECDH(benchmark::Bench& bench) { diff --git a/src/bench/block_assemble.cpp b/src/bench/block_assemble.cpp index be03917417e6..45052ecf71cd 100644 --- a/src/bench/block_assemble.cpp +++ b/src/bench/block_assemble.cpp @@ -12,13 +12,12 @@ #include #include #include +#include #include #include -#include #include #include -#include #include using node::BlockCreateOptions; diff --git a/src/bench/blockencodings.cpp b/src/bench/blockencodings.cpp index ce968dbc6516..274474d498cd 100644 --- a/src/bench/blockencodings.cpp +++ b/src/bench/blockencodings.cpp @@ -7,15 +7,23 @@ #include #include #include +#include #include +#include #include