From 6b2ba040a858db659b3b87ef07f2d087fe50d891 Mon Sep 17 00:00:00 2001 From: fanquake Date: Mon, 21 Aug 2023 10:41:36 +0100 Subject: [PATCH] Merge bitcoin/bitcoin#28218: refactor: Make IsInitialBlockDownload & NotifyHeaderTip not require a Chainstate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 94a98fbd1d14a4ea10629b917771d50f3191e944 assumeutxo cleanup: Move IsInitialBlockDownload & NotifyHeaderTip to ChainstateManager (Ryan Ofsky) Pull request description: This change makes `IsInitialBlockDownload` and `NotifyHeaderTip` functions no longer tied to individual `Chainstate` objects. It makes them work with the `ChainstateManager` object instead so code is simpler and it is no longer possible to call them incorrectly with an inactive `Chainstate`. This change also makes `m_cached_finished_ibd` caching easier to reason about, because now there is only one cached value instead of two (for background and snapshot chainstates) so the cached IBD state now no longer gets reset when a snapshot is loaded. There should be no change in behavior because these functions were always called on the active `ChainState` objects. These changes were discussed previously https://github.com/bitcoin/bitcoin/pull/27746#discussion_r1246868905 and https://github.com/bitcoin/bitcoin/pull/27746#discussion_r1237552792 as possible followups for that PR. ACKs for top commit: MarcoFalke: re-ACK 94a98fbd1d14a4ea10629b917771d50f3191e944 🐺 naumenkogs: ACK 94a98fbd1d14a4ea10629b917771d50f3191e944 dergoegge: reACK 94a98fbd1d14a4ea10629b917771d50f3191e944 Tree-SHA512: 374d6e5c9bbc7564c143f634bd709a4e8f4a42c8d77e7a8554c832acdcf60fa2a134f3ea10827db1a1e0191006496329c0ebf5c64f3ab868398c3722bb7ff56f --- src/net_processing.cpp | 15 ++-- src/node/interfaces.cpp | 10 ++- src/rpc/blockchain.cpp | 2 +- src/rpc/mining.cpp | 2 +- src/test/fuzz/process_message.cpp | 4 +- src/test/fuzz/process_messages.cpp | 4 +- src/test/net_tests.cpp | 9 +-- src/test/util/validation.cpp | 4 +- src/test/util/validation.h | 2 +- .../validation_chainstatemanager_tests.cpp | 13 +++- src/validation.cpp | 75 +++++++++++++------ src/validation.h | 54 ++++++++++--- 12 files changed, 134 insertions(+), 60 deletions(-) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 1d634907c5bc..ff0e474b04cf 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -2162,7 +2162,7 @@ void PeerManagerImpl::BlockChecked(const CBlock& block, const BlockValidationSta // the tip yet so we have no way to check this directly here. Instead we // just check that there are currently no other blocks in flight. else if (state.IsValid() && - !m_chainman.ActiveChainstate().IsInitialBlockDownload() && + !m_chainman.IsInitialBlockDownload() && mapBlocksInFlight.count(hash) == mapBlocksInFlight.size()) { if (it != mapBlockSource.end()) { MaybeSetPeerAsAnnouncingHeaderAndIDs(it->second.first); @@ -3098,7 +3098,7 @@ void PeerManagerImpl::UpdatePeerStateForReceivedHeaders(CNode& pfrom, // If we're in IBD, we want outbound peers that will serve us a useful // chain. Disconnect peers that are on chains with insufficient work. - if (m_chainman.ActiveChainstate().IsInitialBlockDownload() && !may_have_more_headers) { + if (m_chainman.IsInitialBlockDownload() && !may_have_more_headers) { // If the peer has no more headers to give us, then we know we have // their tip. if (nodestate->pindexBestKnownBlock && nodestate->pindexBestKnownBlock->nChainWork < nMinimumChainWork) { @@ -4149,7 +4149,7 @@ void PeerManagerImpl::ProcessMessage( return; } bool allowWhileInIBD = allowWhileInIBDObjs.count(inv.type); - if (allowWhileInIBD || !m_chainman.ActiveChainstate().IsInitialBlockDownload()) { + if (allowWhileInIBD || !m_chainman.IsInitialBlockDownload()) { RequestObject(pfrom.GetId(), inv, current_time, is_masternode); } } @@ -4434,7 +4434,7 @@ void PeerManagerImpl::ProcessMessage( // Stop processing the transaction early if we are still in IBD since we don't // have enough information to validate it yet. Sending unsolicited transactions // is not considered a protocol violation, so don't punish the peer. - if (m_chainman.ActiveChainstate().IsInitialBlockDownload()) return; + if (m_chainman.IsInitialBlockDownload()) return; CTransactionRef ptx; CCoinJoinBroadcastTx dstx; @@ -4613,7 +4613,7 @@ void PeerManagerImpl::ProcessMessage( if (!m_chainman.m_blockman.LookupBlockIndex(cmpctblock.header.hashPrevBlock)) { // Doesn't connect (or is genesis), instead of DoSing in AcceptBlockHeader, request deeper headers - if (!m_chainman.ActiveChainstate().IsInitialBlockDownload()) { + if (!m_chainman.IsInitialBlockDownload()) { std::string ret_val = UsesCompressedHeaders(*peer) ? NetMsgType::GETHEADERS2 : NetMsgType::GETHEADERS; MaybeSendGetHeaders(pfrom, ret_val, m_chainman.ActiveChain().GetLocator(m_chainman.m_best_header), *peer); } @@ -5663,7 +5663,7 @@ void PeerManagerImpl::MaybeSendAddr(CNode& node, Peer& peer, std::chrono::micros LOCK(peer.m_addr_send_times_mutex); // Periodically advertise our local address to the peer. - if (fListen && !m_chainman.ActiveChainstate().IsInitialBlockDownload() && + if (fListen && !m_chainman.IsInitialBlockDownload() && peer.m_next_local_addr_send < current_time) { // If we've sent before, clear the bloom filter for the peer, so that our // self-announcement will actually go out. @@ -5723,7 +5723,6 @@ void PeerManagerImpl::MaybeSendAddr(CNode& node, Peer& peer, std::chrono::micros peer.m_addrs_to_send.shrink_to_fit(); } } - namespace { class CompareInvMempoolOrder { @@ -6279,7 +6278,7 @@ bool PeerManagerImpl::SendMessages(CNode* pto) // Message: getdata (blocks) // std::vector vGetData; - if (CanServeBlocks(*peer) && pto->CanRelay() && ((sync_blocks_and_headers_from_peer && !IsLimitedPeer(*peer)) || !m_chainman.ActiveChainstate().IsInitialBlockDownload()) && state.nBlocksInFlight < MAX_BLOCKS_IN_TRANSIT_PER_PEER) { + if (CanServeBlocks(*peer) && pto->CanRelay() && ((sync_blocks_and_headers_from_peer && !IsLimitedPeer(*peer)) || !m_chainman.IsInitialBlockDownload()) && state.nBlocksInFlight < MAX_BLOCKS_IN_TRANSIT_PER_PEER) { std::vector vToDownload; NodeId staller = -1; FindNextBlocksToDownload(*peer, MAX_BLOCKS_IN_TRANSIT_PER_PEER - state.nBlocksInFlight, vToDownload, staller); diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp index 493c8606c86c..a0e1c1323b8d 100644 --- a/src/node/interfaces.cpp +++ b/src/node/interfaces.cpp @@ -476,8 +476,9 @@ class NodeImpl : public Node } return GuessVerificationProgress(Params().TxData(), tip); } - bool isInitialBlockDownload() override { - return chainman().ActiveChainstate().IsInitialBlockDownload(); + bool isInitialBlockDownload() override + { + return chainman().IsInitialBlockDownload(); } bool isMasternode() override { @@ -961,8 +962,9 @@ class ChainImpl : public Chain return m_node.chainman->m_blockman.m_have_pruned; } bool isReadyToBroadcast() override { return !node::fImporting && !node::fReindex && !isInitialBlockDownload(); } - bool isInitialBlockDownload() override { - return chainman().ActiveChainstate().IsInitialBlockDownload(); + bool isInitialBlockDownload() override + { + return chainman().IsInitialBlockDownload(); } bool shutdownRequested() override { return ShutdownRequested(); } void initMessage(const std::string& message) override { ::uiInterface.InitMessage(message); } diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index ae6ca2b23ea8..21dbd72b54a6 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1520,7 +1520,7 @@ RPCHelpMan getblockchaininfo() obj.pushKV("time", tip.GetBlockTime()); obj.pushKV("mediantime", tip.GetMedianTimePast()); obj.pushKV("verificationprogress", GuessVerificationProgress(Params().TxData(), &tip)); - obj.pushKV("initialblockdownload", active_chainstate.IsInitialBlockDownload()); + obj.pushKV("initialblockdownload", chainman.IsInitialBlockDownload()); obj.pushKV("chainwork", tip.nChainWork.GetHex()); obj.pushKV("size_on_disk", chainman.m_blockman.CalculateCurrentUsage()); obj.pushKV("pruned", node::fPruneMode); diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index ec3723e1d9bd..5e605df0a1a9 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -730,7 +730,7 @@ static RPCHelpMan getblocktemplate() throw JSONRPCError(RPC_CLIENT_NOT_CONNECTED, PACKAGE_NAME " is not connected!"); } - if (active_chainstate.IsInitialBlockDownload()) { + if (chainman.IsInitialBlockDownload()) { throw JSONRPCError(RPC_CLIENT_IN_INITIAL_DOWNLOAD, PACKAGE_NAME " is in initial sync and waiting for blocks..."); } } diff --git a/src/test/fuzz/process_message.cpp b/src/test/fuzz/process_message.cpp index 5b458ba0a541..d780cfaa6453 100644 --- a/src/test/fuzz/process_message.cpp +++ b/src/test/fuzz/process_message.cpp @@ -65,9 +65,9 @@ FUZZ_TARGET(process_message, .init = initialize_process_message) FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); ConnmanTestMsg& connman = *static_cast(g_setup->m_node.connman.get()); - TestChainState& chainstate = *static_cast(&g_setup->m_node.chainman->ActiveChainstate()); + auto& chainman = static_cast(*g_setup->m_node.chainman); SetMockTime(1610000000); // any time to successfully reset ibd - chainstate.ResetIbd(); + chainman.ResetIbd(); LOCK(NetEventsInterface::g_msgproc_mutex); diff --git a/src/test/fuzz/process_messages.cpp b/src/test/fuzz/process_messages.cpp index 8832ce2ccb46..31431e0afe36 100644 --- a/src/test/fuzz/process_messages.cpp +++ b/src/test/fuzz/process_messages.cpp @@ -38,9 +38,9 @@ FUZZ_TARGET(process_messages, .init = initialize_process_messages) FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); ConnmanTestMsg& connman = *static_cast(g_setup->m_node.connman.get()); - TestChainState& chainstate = *static_cast(&g_setup->m_node.chainman->ActiveChainstate()); + auto& chainman = static_cast(*g_setup->m_node.chainman); SetMockTime(1610000000); // any time to successfully reset ibd - chainstate.ResetIbd(); + chainman.ResetIbd(); LOCK(NetEventsInterface::g_msgproc_mutex); diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index 1c218ecb5f18..8f4b78b69537 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -850,11 +850,10 @@ BOOST_AUTO_TEST_CASE(initial_advertise_from_version_message) const int64_t time{0}; const CNetMsgMaker msg_maker{PROTOCOL_VERSION}; - // Force CChainState::IsInitialBlockDownload() to return false. + // Force ChainstateManager::IsInitialBlockDownload() to return false. // Otherwise PushAddress() isn't called by PeerManager::ProcessMessage(). - TestChainState& chainstate = - *static_cast(&m_node.chainman->ActiveChainstate()); - chainstate.JumpOutOfIbd(); + auto& chainman = static_cast(*m_node.chainman); + chainman.JumpOutOfIbd(); m_node.peerman->InitializeNode(peer, NODE_NETWORK); @@ -904,7 +903,7 @@ BOOST_AUTO_TEST_CASE(initial_advertise_from_version_message) BOOST_CHECK(sent); CaptureMessage = CaptureMessageOrig; - chainstate.ResetIbd(); + chainman.ResetIbd(); m_node.args->ForceSetArg("-capturemessages", "0"); m_node.args->ForceSetArg("-bind", ""); // PeerManager::ProcessMessage() calls AddTimeData() which changes the internal state diff --git a/src/test/util/validation.cpp b/src/test/util/validation.cpp index 49535855f988..e10a44d1d0e1 100644 --- a/src/test/util/validation.cpp +++ b/src/test/util/validation.cpp @@ -9,13 +9,13 @@ #include #include -void TestChainState::ResetIbd() +void TestChainstateManager::ResetIbd() { m_cached_finished_ibd = false; assert(IsInitialBlockDownload()); } -void TestChainState::JumpOutOfIbd() +void TestChainstateManager::JumpOutOfIbd() { Assert(IsInitialBlockDownload()); m_cached_finished_ibd = true; diff --git a/src/test/util/validation.h b/src/test/util/validation.h index b0bc717b6c22..8357b1ff0fb6 100644 --- a/src/test/util/validation.h +++ b/src/test/util/validation.h @@ -9,7 +9,7 @@ class CValidationInterface; -struct TestChainState : public CChainState { +struct TestChainstateManager : public ChainstateManager { /** Reset the ibd cache to its initial state */ void ResetIbd(); /** Toggle IsInitialBlockDownload from true to false */ diff --git a/src/test/validation_chainstatemanager_tests.cpp b/src/test/validation_chainstatemanager_tests.cpp index 2de98a6fc65b..1b2907258e67 100644 --- a/src/test/validation_chainstatemanager_tests.cpp +++ b/src/test/validation_chainstatemanager_tests.cpp @@ -16,6 +16,8 @@ #include #include #include +#include +#include #include #include #include @@ -163,6 +165,15 @@ BOOST_AUTO_TEST_CASE(chainstatemanager_rebalance_caches) c2.InitCoinsDB( /*cache_size_bytes=*/1 << 23, /*in_memory=*/true, /*should_wipe=*/false); + // Reset IBD state so IsInitialBlockDownload() returns true and causes + // MaybeRebalancesCaches() to prioritize the snapshot chainstate, giving it + // more cache space than the snapshot chainstate. Calling ResetIbd() is + // necessary because m_cached_finished_ibd is already latched to true before + // the test starts due to the test setup. After ResetIbd() is called. + // IsInitialBlockDownload will return true because at this point the active + // chainstate has a null chain tip. + static_cast(manager).ResetIbd(); + { LOCK(::cs_main); c2.InitCoinsCache(1 << 23); @@ -171,8 +182,6 @@ BOOST_AUTO_TEST_CASE(chainstatemanager_rebalance_caches) manager.MaybeRebalanceCaches(); } - // Since both chainstates are considered to be in initial block download, - // the snapshot chainstate should take priority. BOOST_CHECK_CLOSE(c1.m_coinstip_cache_size_bytes, max_cache * 0.05, 1); BOOST_CHECK_CLOSE(c1.m_coinsdb_cache_size_bytes, max_cache * 0.05, 1); BOOST_CHECK_CLOSE(c2.m_coinstip_cache_size_bytes, max_cache * 0.95, 1); diff --git a/src/validation.cpp b/src/validation.cpp index 29ce4c2184eb..b9bd7d2000f1 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -366,8 +366,9 @@ static void LimitMempoolSize(CTxMemPool& pool, CCoinsViewCache& coins_cache, siz static bool IsCurrentForFeeEstimation(CChainState& active_chainstate) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { AssertLockHeld(cs_main); - if (active_chainstate.IsInitialBlockDownload()) + if (active_chainstate.m_chainman.IsInitialBlockDownload()) { return false; + } if (active_chainstate.m_chain.Tip()->GetBlockTime() < count_seconds(GetTime() - MAX_FEE_ESTIMATION_TIP_AGE)) return false; if (active_chainstate.m_chain.Height() < active_chainstate.m_chainman.m_best_header->nHeight - 1) { @@ -1613,10 +1614,10 @@ void CChainState::InitCoinsCache(size_t cache_size_bytes) // Note that though this is marked const, we may end up modifying `m_cached_finished_ibd`, which // is a performance-related implementation detail. This function must be marked -// `const` so that `CValidationInterface` clients (which are given a `const CChainState*`) +// `const` so that `CValidationInterface` clients (which are given a `const ChainstateManager*`) // can call it. // -bool CChainState::IsInitialBlockDownload() const +bool ChainstateManager::IsInitialBlockDownload() const { // Optimization: pre-test latch before taking the lock. if (m_cached_finished_ibd.load(std::memory_order_relaxed)) @@ -1627,11 +1628,12 @@ bool CChainState::IsInitialBlockDownload() const return false; if (fImporting || fReindex) return true; - if (m_chain.Tip() == nullptr) + CChain& chain{ActiveChain()}; + if (chain.Tip() == nullptr) return true; - if (m_chain.Tip()->nChainWork < nMinimumChainWork) + if (chain.Tip()->nChainWork < nMinimumChainWork) return true; - if (m_chain.Tip()->GetBlockTime() < (GetTime() - nMaxTipAge)) + if (chain.Tip()->GetBlockTime() < (GetTime() - nMaxTipAge)) return true; LogPrintf("Leaving InitialBlockDownload (latching to false)\n"); m_cached_finished_ibd.store(true, std::memory_order_relaxed); @@ -1664,7 +1666,7 @@ void CChainState::CheckForkWarningConditions() // Before we get past initial download, we cannot reliably alert about forks // (we assume we don't get stuck on a fork before finishing our initial sync) - if (IsInitialBlockDownload()) { + if (m_chainman.IsInitialBlockDownload()) { return; } @@ -2733,7 +2735,7 @@ bool CChainState::FlushStateToDisk( } else { LOG_TIME_MILLIS_WITH_CATEGORY("find files to prune", BCLog::BENCHMARK); - m_blockman.FindFilesToPrune(setFilesToPrune, m_params.PruneAfterHeight(), m_chain.Height(), last_prune, IsInitialBlockDownload()); + m_blockman.FindFilesToPrune(setFilesToPrune, m_params.PruneAfterHeight(), m_chain.Height(), last_prune, m_chainman.IsInitialBlockDownload()); m_blockman.m_check_for_pruning = false; } if (!setFilesToPrune.empty()) { @@ -2923,7 +2925,7 @@ void CChainState::UpdateTip(const CBlockIndex* pindexNew) } bilingual_str warning_messages; - if (!this->IsInitialBlockDownload()) + if (!m_chainman.IsInitialBlockDownload()) { const CBlockIndex* pindex = pindexNew; for (int bit = 0; bit < VERSIONBITS_NUM_BITS; bit++) { @@ -3340,18 +3342,18 @@ static SynchronizationState GetSynchronizationState(bool init) return SynchronizationState::INIT_DOWNLOAD; } -static bool NotifyHeaderTip(CChainState& chainstate) LOCKS_EXCLUDED(cs_main) { +static bool NotifyHeaderTip(ChainstateManager& chainman) LOCKS_EXCLUDED(cs_main) { bool fNotify = false; bool fInitialBlockDownload = false; static CBlockIndex* pindexHeaderOld = nullptr; CBlockIndex* pindexHeader = nullptr; { LOCK(cs_main); - pindexHeader = chainstate.m_chainman.m_best_header; + pindexHeader = chainman.m_best_header; if (pindexHeader != pindexHeaderOld) { fNotify = true; - fInitialBlockDownload = chainstate.IsInitialBlockDownload(); + fInitialBlockDownload = chainman.IsInitialBlockDownload(); pindexHeaderOld = pindexHeader; } } @@ -3443,7 +3445,7 @@ bool CChainState::ActivateBestChain(BlockValidationState& state, std::shared_ptr if (!blocks_connected) return true; const CBlockIndex* pindexFork = m_chain.FindFork(starting_tip); - bool fInitialDownload = IsInitialBlockDownload(); + bool fInitialDownload = m_chainman.IsInitialBlockDownload(); // Notify external listeners about the new tip. // Enqueue while holding cs_main to ensure that UpdatedBlockTip is called in the order in which blocks are connected @@ -3656,13 +3658,13 @@ bool CChainState::InvalidateBlock(BlockValidationState& state, CBlockIndex* pind } InvalidChainFound(to_mark_failed); - GetMainSignals().SynchronousUpdatedBlockTip(m_chain.Tip(), nullptr, IsInitialBlockDownload()); - GetMainSignals().UpdatedBlockTip(m_chain.Tip(), nullptr, IsInitialBlockDownload()); + GetMainSignals().SynchronousUpdatedBlockTip(m_chain.Tip(), nullptr, m_chainman.IsInitialBlockDownload()); + GetMainSignals().UpdatedBlockTip(m_chain.Tip(), nullptr, m_chainman.IsInitialBlockDownload()); } // Only notify about a new block tip if the active chain was modified. if (pindex_was_in_chain) { - uiInterface.NotifyBlockTip(GetSynchronizationState(IsInitialBlockDownload()), to_mark_failed->pprev); + uiInterface.NotifyBlockTip(GetSynchronizationState(m_chainman.IsInitialBlockDownload()), to_mark_failed->pprev); } return true; } @@ -4221,6 +4223,23 @@ bool ChainstateManager::AcceptBlockHeader(const CBlockHeader& block, BlockValida // Notify external listeners about accepted block header GetMainSignals().AcceptedBlockHeader(pindex); + // Since this is the earliest point at which we have determined that a + // header is both new and valid, log here. + // + // These messages are valuable for detecting potential selfish mining behavior; + // if multiple displacing headers are seen near simultaneously across many + // nodes in the network, this might be an indication of selfish mining. Having + // this log by default when not in IBD ensures broad availability of this data + // in case investigation is merited. + const auto msg = strprintf( + "Saw new header hash=%s height=%d", hash.ToString(), pindex->nHeight); + + if (IsInitialBlockDownload()) { + LogPrintLevel(BCLog::VALIDATION, BCLog::Level::Debug, "%s\n", msg); + } else { + LogPrintf("%s\n", msg); + } + return true; } @@ -4243,8 +4262,8 @@ bool ChainstateManager::ProcessNewBlockHeaders(const std::vector& } } } - if (NotifyHeaderTip(ActiveChainstate())) { - if (ActiveChainstate().IsInitialBlockDownload() && ppindex && *ppindex) { + if (NotifyHeaderTip(*this)) { + if (IsInitialBlockDownload() && ppindex && *ppindex) { const CBlockIndex& last_accepted{**ppindex}; const int64_t blocks_left{(GetTime() - last_accepted.GetBlockTime()) / chainparams.GetConsensus().nPowTargetSpacing}; const double progress{100.0 * last_accepted.nHeight / (last_accepted.nHeight + blocks_left)}; @@ -4317,7 +4336,7 @@ bool CChainState::AcceptBlock(const std::shared_ptr& pblock, Block // Header is valid/has work, merkle tree is good...RELAY NOW // (but if it does not build on our best tip, let the SendMessages loop relay it) - if (!IsInitialBlockDownload() && m_chain.Tip() == pindex->pprev) + if (!m_chainman.IsInitialBlockDownload() && m_chain.Tip() == pindex->pprev) GetMainSignals().NewPoWValidBlock(pindex, pblock); // Write block to history file @@ -4375,7 +4394,7 @@ bool ChainstateManager::ProcessNewBlock(const CChainParams& chainparams, const s } } - NotifyHeaderTip(ActiveChainstate()); + NotifyHeaderTip(*this); BlockValidationState state; // Only used to report errors, not invalidity - ignore it if (!ActiveChainstate().ActivateBestChain(state, block)) @@ -5077,6 +5096,20 @@ void CChainState::LoadExternalBlockFile( } } + if (m_blockman.IsPruneMode() && !fReindex && pblock) { + // must update the tip for pruning to work while importing with -loadblock. + // this is a tradeoff to conserve disk space at the expense of time + // spent updating the tip to be able to prune. + // otherwise, ActivateBestChain won't be called by the import process + // until after all of the block files are loaded. ActivateBestChain can be + // called by concurrent network message processing. but, that is not + // reliable for the purpose of pruning while importing. + BlockValidationState state; + if (!ActivateBestChain(state, pblock)) { + LogPrint(BCLog::REINDEX, "failed to activate chain (%s)\n", state.ToString()); + break; + } + } NotifyHeaderTip(*this); if (!blocks_with_unknown_parent) continue; @@ -5945,7 +5978,7 @@ void ChainstateManager::MaybeRebalanceCaches() // If both chainstates exist, determine who needs more cache based on IBD status. // // Note: shrink caches first so that we don't inadvertently overwhelm available memory. - if (m_snapshot_chainstate->IsInitialBlockDownload()) { + if (IsInitialBlockDownload()) { m_ibd_chainstate->ResizeCoinsCaches( m_total_coinstip_cache * 0.05, m_total_coinsdb_cache * 0.05); m_snapshot_chainstate->ResizeCoinsCaches( diff --git a/src/validation.h b/src/validation.h index 6eb7608491f3..c16e900c2fb0 100644 --- a/src/validation.h +++ b/src/validation.h @@ -484,14 +484,6 @@ class CChainState */ Mutex m_chainstate_mutex; - /** - * Whether this chainstate is undergoing initial block download. - * - * Mutable because we need to be able to mark IsInitialBlockDownload() - * const, which latches this for caching purposes. - */ - mutable std::atomic m_cached_finished_ibd{false}; - //! Optional mempool that is kept in sync with the chain. //! Only the active chainstate has a mempool. CTxMemPool* m_mempool; @@ -736,9 +728,6 @@ class CChainState void UnloadBlockIndex() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); - /** Check whether we are doing an initial block download (synchronizing from disk or network) */ - bool IsInitialBlockDownload() const; - /** Find the last common block of this chain and a locator. */ const CBlockIndex* FindForkInGlobalIndex(const CBlockLocator& locator) const EXCLUSIVE_LOCKS_REQUIRED(cs_main); @@ -910,6 +899,15 @@ class ChainstateManager //! chainstate to avoid duplicating block metadata. node::BlockManager m_blockman; + /** + * Whether initial block download has ended and IsInitialBlockDownload + * should return false from now on. + * + * Mutable because we need to be able to mark IsInitialBlockDownload() + * const, which latches this for caching purposes. + */ + mutable std::atomic m_cached_finished_ibd{false}; + /** * In order to efficiently track invalidity of headers, we keep the set of * blocks which we tried to connect and found to be invalid here (ie which @@ -1000,6 +998,40 @@ class ChainstateManager //! Is there a snapshot in use and has it been fully validated? bool IsSnapshotValidated() const EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { return m_snapshot_validated; } + /** Check whether we are doing an initial block download (synchronizing from disk or network) */ + bool IsInitialBlockDownload() const; + + /** + * Import blocks from an external file + * + * During reindexing, this function is called for each block file (datadir/blocks/blk?????.dat). + * It reads all blocks contained in the given file and attempts to process them (add them to the + * block index). The blocks may be out of order within each file and across files. Often this + * function reads a block but finds that its parent hasn't been read yet, so the block can't be + * processed yet. The function will add an entry to the blocks_with_unknown_parent map (which is + * passed as an argument), so that when the block's parent is later read and processed, this + * function can re-read the child block from disk and process it. + * + * Because a block's parent may be in a later file, not just later in the same file, the + * blocks_with_unknown_parent map must be passed in and out with each call. It's a multimap, + * rather than just a map, because multiple blocks may have the same parent (when chain splits + * or stale blocks exist). It maps from parent-hash to child-disk-position. + * + * This function can also be used to read blocks from user-specified block files using the + * -loadblock= option. There's no unknown-parent tracking, so the last two arguments are omitted. + * + * + * @param[in] fileIn FILE handle to file containing blocks to read + * @param[in] dbp (optional) Disk block position (only for reindex) + * @param[in,out] blocks_with_unknown_parent (optional) Map of disk positions for blocks with + * unknown parent, key is parent block hash + * (only used for reindex) + * */ + void LoadExternalBlockFile( + FILE* fileIn, + FlatFilePos* dbp = nullptr, + std::multimap* blocks_with_unknown_parent = nullptr); + /** * Process an incoming block. This only returns after the best known valid * block is made active. Note that it does not, however, guarantee that the