From 6c59ec641b78a303ddf791a1c4740323b8e2cf90 Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Thu, 14 Sep 2023 13:15:24 -0400 Subject: [PATCH 01/78] Merge bitcoin/bitcoin#27850: test: Add unit & functional test coverage for blockstore MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit de8f9123afbecc3b4f59fa80af8148bc865d0588 test: cover read-only blockstore (Matthew Zipkin) 5c2185b3b624ce87320ec16412f98ab591a5860c ci: enable chattr +i capability inside containers (Matthew Zipkin) e573f2420244c583e218f51cd0d3a3cac6731003 unit test: add coverage for BlockManager (Matthew Zipkin) Pull request description: This PR adds unit and functional tests to cover the behavior described in #2039. In particular, that bitcoind will crash on startup if a reindex is requested but the `blk` files are read-only. Eventually this behavior can be updated with https://github.com/bitcoin/bitcoin/pull/27039. This PR just commits the test coverage from #27039 as suggested in https://github.com/bitcoin/bitcoin/pull/27039#issuecomment-1584915782 ACKs for top commit: jonatack: ACK de8f9123afbecc3b4f59fa80af8148bc865d0588 modulo suggestions in https://github.com/bitcoin/bitcoin/pull/27850#discussion_r1319010039, tested on macOS, but not on Linux for the Linux-related change in the last push achow101: ACK de8f9123afbecc3b4f59fa80af8148bc865d0588 MarcoFalke: lgtm ACK de8f9123afbecc3b4f59fa80af8148bc865d0588 📶 Tree-SHA512: b9bd684035dcea11c901b649fc39f397a2155a9a8459f3348e67947e387e45312fddeccb52981aef486f8a31deebb5356a7901c1bb94b78f82c24192a369af73 --- ci/test/00_setup_env.sh | 3 +- ci/test/04_install.sh | 10 +- src/test/blockmanager_tests.cpp | 154 ++++ test/functional/feature_reindex_readonly.py | 64 ++ test/functional/test_runner.py | 4 +- test/functional/test_runner.py.bak3 | 943 ++++++++++++++++++++ 6 files changed, 1169 insertions(+), 9 deletions(-) create mode 100755 test/functional/feature_reindex_readonly.py create mode 100755 test/functional/test_runner.py.bak3 diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh index d4840e9ebfd8..117b7e7ba6e1 100755 --- a/ci/test/00_setup_env.sh +++ b/ci/test/00_setup_env.sh @@ -70,8 +70,7 @@ export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_SCRATCH_DIR/out/$HOST} export BASE_BUILD_DIR=${BASE_BUILD_DIR:-$BASE_SCRATCH_DIR/build-ci} export PREVIOUS_RELEASES_DIR=${PREVIOUS_RELEASES_DIR:-$BASE_ROOT_DIR/releases/$HOST} export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks} -export DOCKER_PACKAGES=${DOCKER_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 rsync git procps} -export GOAL=${GOAL:-install} +export DOCKER_PACKAGES=${DOCKER_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 rsync git procps bison e2fsprogs}export GOAL=${GOAL:-install} export DIR_QA_ASSETS=${DIR_QA_ASSETS:-${BASE_SCRATCH_DIR}/qa-assets} export PATH=${BASE_ROOT_DIR}/ci/retry:$PATH export CI_RETRY_EXE=${CI_RETRY_EXE:-"retry --"} diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index 3ea6d24e6b86..8f20f1b7b85c 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#\!/usr/bin/env bash # # Copyright (c) 2018-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying @@ -42,7 +42,7 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then ${CI_RETRY_EXE} docker pull "$DOCKER_NAME_TAG" # shellcheck disable=SC2086 - DOCKER_ID=$(docker run $DOCKER_ADMIN -idt \ + DOCKER_ID=$(docker run $DOCKER_ADMIN --cap-add LINUX_IMMUTABLE -idt \ --mount type=bind,src=$BASE_ROOT_DIR,dst=/ro_base,readonly \ --mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR \ --mount type=bind,src=$DEPENDS_DIR,dst=$DEPENDS_DIR \ @@ -83,7 +83,7 @@ fi if [[ $DOCKER_NAME_TAG == *centos* ]]; then CI_EXEC_ROOT yum -y install epel-release CI_EXEC_ROOT yum -y install "$DOCKER_PACKAGES" "$PACKAGES" -elif [ "$CI_USE_APT_INSTALL" != "no" ]; then +elif [ "$CI_USE_APT_INSTALL" \!= "no" ]; then ${CI_RETRY_EXE} CI_EXEC_ROOT apt-get update ${CI_RETRY_EXE} CI_EXEC_ROOT apt-get install --no-install-recommends --no-upgrade -y "$PACKAGES" "$DOCKER_PACKAGES" if [ -n "$PIP_PACKAGES" ]; then @@ -105,7 +105,7 @@ CI_EXEC df -h if [ "$RUN_FUZZ_TESTS" = "true" ]; then export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/ - if [ ! -d "$DIR_FUZZ_IN" ]; then + if [ \! -d "$DIR_FUZZ_IN" ]; then CI_EXEC git clone --depth=1 https://github.com/bitcoin-core/qa-assets "${DIR_QA_ASSETS}" fi ( @@ -115,7 +115,7 @@ if [ "$RUN_FUZZ_TESTS" = "true" ]; then ) elif [ "$RUN_UNIT_TESTS" = "true" ] || [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ]; then export DIR_UNIT_TEST_DATA=${DIR_QA_ASSETS}/unit_test_data/ - if [ ! -d "$DIR_UNIT_TEST_DATA" ]; then + if [ \! -d "$DIR_UNIT_TEST_DATA" ]; then CI_EXEC mkdir -p "$DIR_UNIT_TEST_DATA" CI_EXEC curl --location --fail https://github.com/bitcoin-core/qa-assets/raw/main/unit_test_data/script_assets_test.json -o "${DIR_UNIT_TEST_DATA}/script_assets_test.json" fi diff --git a/src/test/blockmanager_tests.cpp b/src/test/blockmanager_tests.cpp index dd7c32cc46c0..432b6cbf21d6 100644 --- a/src/test/blockmanager_tests.cpp +++ b/src/test/blockmanager_tests.cpp @@ -5,9 +5,14 @@ #include #include #include +#include +#include