From f4d49bc74b2c7ad45215b5a0411f31a712195c02 Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Wed, 22 Jun 2016 14:20:43 +0200 Subject: [PATCH 01/51] Merge #8068: Compact Blocks 48efec8 Fix some minor compact block issues that came up in review (Matt Corallo) ccd06b9 Elaborate bucket size math (Pieter Wuille) 0d4cb48 Use vTxHashes to optimize InitData significantly (Matt Corallo) 8119026 Provide a flat list of txid/terators to txn in CTxMemPool (Matt Corallo) 678ee97 Add BIP 152 to implemented BIPs list (Matt Corallo) 56ba516 Add reconstruction debug logging (Matt Corallo) 2f34a2e Get our "best three" peers to announce blocks using cmpctblocks (Matt Corallo) 927f8ee Add ability to fetch CNode by NodeId (Matt Corallo) d25cd3e Add receiver-side protocol implementation for CMPCTBLOCK stuff (Matt Corallo) 9c837d5 Add sender-side protocol implementation for CMPCTBLOCK stuff (Matt Corallo) 00c4078 Add protocol messages for short-ids blocks (Matt Corallo) e3b2222 Add some blockencodings tests (Matt Corallo) f4f8f14 Add TestMemPoolEntryHelper::FromTx version for CTransaction (Matt Corallo) 85ad31e Add partial-block block encodings API (Matt Corallo) 5249dac Add COMPACTSIZE wrapper similar to VARINT for serialization (Matt Corallo) cbda71c Move context-required checks from CheckBlockHeader to Contextual... (Matt Corallo) 7c29ec9 If AcceptBlockHeader returns true, pindex will be set. (Matt Corallo) 96806c3 Stop trimming when mapTx is empty (Pieter Wuille) --- doc/bips.md | 1 + src/Makefile.am | 2 + src/Makefile.test.include | 1 + src/blockencodings.cpp | 180 +++++++++++++++ src/blockencodings.h | 206 +++++++++++++++++ src/net.cpp | 10 + src/net.h | 1 + src/net_processing.cpp | 354 ++++++++++++++++++++++++++++-- src/protocol.cpp | 11 +- src/protocol.h | 31 +++ src/test/blockencodings_tests.cpp | 315 ++++++++++++++++++++++++++ src/test/test_dash.cpp | 7 +- src/test/test_dash.h | 3 +- src/txmempool.cpp | 14 +- src/txmempool.h | 5 + src/validation.cpp | 1 + src/version.h | 6 +- 17 files changed, 1117 insertions(+), 31 deletions(-) create mode 100644 src/blockencodings.cpp create mode 100644 src/blockencodings.h create mode 100644 src/test/blockencodings_tests.cpp diff --git a/doc/bips.md b/doc/bips.md index 3e9153a25c1d..b01f0f3215b1 100644 --- a/doc/bips.md +++ b/doc/bips.md @@ -28,3 +28,4 @@ BIPs that are implemented by Bitcoin Core (up-to-date up to **v0.13.0**): * [`BIP 130`](https://github.com/bitcoin/bips/blob/master/bip-0130.mediawiki): direct headers announcement is negotiated with peer versions `>=70012` as of **v0.12.0** ([PR 6494](https://github.com/bitcoin/bitcoin/pull/6494)). * [`BIP 133`](https://github.com/bitcoin/bips/blob/master/bip-0133.mediawiki): feefilter messages are respected and sent for peer versions `>=70013` as of **v0.13.0** ([PR 7542](https://github.com/bitcoin/bitcoin/pull/7542)). * [`BIP 147`](https://github.com/bitcoin/bips/blob/master/bip-0147.mediawiki): NULLDUMMY softfork as of **v0.13.1** ([PR 8636](https://github.com/bitcoin/bitcoin/pull/8636) and [PR 8937](https://github.com/bitcoin/bitcoin/pull/8937)). +* [`BIP 152`](https://github.com/bitcoin/bips/blob/master/bip-0152.mediawiki): Compact block transfer and related optimizations are used as of **v0.13.0** ([PR 8068](https://github.com/bitcoin/bitcoin/pull/8068)). diff --git a/src/Makefile.am b/src/Makefile.am index f7c085a69b74..53b9c8a6c416 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -85,6 +85,7 @@ BITCOIN_CORE_H = \ base58.h \ bip39.h \ bip39_english.h \ + blockencodings.h \ bloom.h \ cachemap.h \ cachemultimap.h \ @@ -211,6 +212,7 @@ libbitcoin_server_a_SOURCES = \ addrdb.cpp \ alert.cpp \ bloom.cpp \ + blockencodings.cpp \ chain.cpp \ checkpoints.cpp \ dsnotificationinterface.cpp \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index e043b2a05ea9..404dc70b0fa9 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -78,6 +78,7 @@ BITCOIN_TESTS =\ test/base64_tests.cpp \ test/bip32_tests.cpp \ test/bip39_tests.cpp \ + test/blockencodings_tests.cpp \ test/bloom_tests.cpp \ test/bswap_tests.cpp \ test/checkqueue_tests.cpp \ diff --git a/src/blockencodings.cpp b/src/blockencodings.cpp new file mode 100644 index 000000000000..aa92aacf6d4e --- /dev/null +++ b/src/blockencodings.cpp @@ -0,0 +1,180 @@ +// Copyright (c) 2016 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "blockencodings.h" +#include "consensus/consensus.h" +#include "consensus/validation.h" +#include "chainparams.h" +#include "hash.h" +#include "random.h" +#include "streams.h" +#include "txmempool.h" +#include "main.h" +#include "util.h" + +#include + +#define MIN_TRANSACTION_SIZE (::GetSerializeSize(CTransaction(), SER_NETWORK, PROTOCOL_VERSION)) + +CBlockHeaderAndShortTxIDs::CBlockHeaderAndShortTxIDs(const CBlock& block) : + nonce(GetRand(std::numeric_limits::max())), + shorttxids(block.vtx.size() - 1), prefilledtxn(1), header(block) { + FillShortTxIDSelector(); + //TODO: Use our mempool prior to block acceptance to predictively fill more than just the coinbase + prefilledtxn[0] = {0, block.vtx[0]}; + for (size_t i = 1; i < block.vtx.size(); i++) { + const CTransaction& tx = block.vtx[i]; + shorttxids[i - 1] = GetShortID(tx.GetHash()); + } +} + +void CBlockHeaderAndShortTxIDs::FillShortTxIDSelector() const { + CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); + stream << header << nonce; + CSHA256 hasher; + hasher.Write((unsigned char*)&(*stream.begin()), stream.end() - stream.begin()); + uint256 shorttxidhash; + hasher.Finalize(shorttxidhash.begin()); + shorttxidk0 = shorttxidhash.GetUint64(0); + shorttxidk1 = shorttxidhash.GetUint64(1); +} + +uint64_t CBlockHeaderAndShortTxIDs::GetShortID(const uint256& txhash) const { + static_assert(SHORTTXIDS_LENGTH == 6, "shorttxids calculation assumes 6-byte shorttxids"); + return SipHashUint256(shorttxidk0, shorttxidk1, txhash) & 0xffffffffffffL; +} + + + +ReadStatus PartiallyDownloadedBlock::InitData(const CBlockHeaderAndShortTxIDs& cmpctblock) { + if (cmpctblock.header.IsNull() || (cmpctblock.shorttxids.empty() && cmpctblock.prefilledtxn.empty())) + return READ_STATUS_INVALID; + if (cmpctblock.shorttxids.size() + cmpctblock.prefilledtxn.size() > MaxBlockSize(true) / MIN_TRANSACTION_SIZE) + return READ_STATUS_INVALID; + + assert(header.IsNull() && txn_available.empty()); + header = cmpctblock.header; + txn_available.resize(cmpctblock.BlockTxCount()); + + int32_t lastprefilledindex = -1; + for (size_t i = 0; i < cmpctblock.prefilledtxn.size(); i++) { + if (cmpctblock.prefilledtxn[i].tx.IsNull()) + return READ_STATUS_INVALID; + + lastprefilledindex += cmpctblock.prefilledtxn[i].index + 1; //index is a uint16_t, so cant overflow here + if (lastprefilledindex > std::numeric_limits::max()) + return READ_STATUS_INVALID; + if ((uint32_t)lastprefilledindex > cmpctblock.shorttxids.size() + i) { + // If we are inserting a tx at an index greater than our full list of shorttxids + // plus the number of prefilled txn we've inserted, then we have txn for which we + // have neither a prefilled txn or a shorttxid! + return READ_STATUS_INVALID; + } + txn_available[lastprefilledindex] = std::make_shared(cmpctblock.prefilledtxn[i].tx); + } + prefilled_count = cmpctblock.prefilledtxn.size(); + + // Calculate map of txids -> positions and check mempool to see what we have (or dont) + // Because well-formed cmpctblock messages will have a (relatively) uniform distribution + // of short IDs, any highly-uneven distribution of elements can be safely treated as a + // READ_STATUS_FAILED. + std::unordered_map shorttxids(cmpctblock.shorttxids.size()); + uint16_t index_offset = 0; + for (size_t i = 0; i < cmpctblock.shorttxids.size(); i++) { + while (txn_available[i + index_offset]) + index_offset++; + shorttxids[cmpctblock.shorttxids[i]] = i + index_offset; + // To determine the chance that the number of entries in a bucket exceeds N, + // we use the fact that the number of elements in a single bucket is + // binomially distributed (with n = the number of shorttxids S, and p = + // 1 / the number of buckets), that in the worst case the number of buckets is + // equal to S (due to std::unordered_map having a default load factor of 1.0), + // and that the chance for any bucket to exceed N elements is at most + // buckets * (the chance that any given bucket is above N elements). + // Thus: P(max_elements_per_bucket > N) <= S * (1 - cdf(binomial(n=S,p=1/S), N)). + // If we assume blocks of up to 16000, allowing 12 elements per bucket should + // only fail once per ~1 million block transfers (per peer and connection). + if (shorttxids.bucket_size(shorttxids.bucket(cmpctblock.shorttxids[i])) > 12) + return READ_STATUS_FAILED; + } + // TODO: in the shortid-collision case, we should instead request both transactions + // which collided. Falling back to full-block-request here is overkill. + if (shorttxids.size() != cmpctblock.shorttxids.size()) + return READ_STATUS_FAILED; // Short ID collision + + std::vector have_txn(txn_available.size()); + LOCK(pool->cs); + const std::vector >& vTxHashes = pool->vTxHashes; + for (size_t i = 0; i < vTxHashes.size(); i++) { + uint64_t shortid = cmpctblock.GetShortID(vTxHashes[i].first); + std::unordered_map::iterator idit = shorttxids.find(shortid); + if (idit != shorttxids.end()) { + if (!have_txn[idit->second]) { + txn_available[idit->second] = vTxHashes[i].second->GetSharedTx(); + have_txn[idit->second] = true; + mempool_count++; + } else { + // If we find two mempool txn that match the short id, just request it. + // This should be rare enough that the extra bandwidth doesn't matter, + // but eating a round-trip due to FillBlock failure would be annoying + if (txn_available[idit->second]) { + txn_available[idit->second].reset(); + mempool_count--; + } + } + } + // Though ideally we'd continue scanning for the two-txn-match-shortid case, + // the performance win of an early exit here is too good to pass up and worth + // the extra risk. + if (mempool_count == shorttxids.size()) + break; + } + + LogPrint("cmpctblock", "Initialized PartiallyDownloadedBlock for block %s using a cmpctblock of size %lu\n", cmpctblock.header.GetHash().ToString(), cmpctblock.GetSerializeSize(SER_NETWORK, PROTOCOL_VERSION)); + + return READ_STATUS_OK; +} + +bool PartiallyDownloadedBlock::IsTxAvailable(size_t index) const { + assert(!header.IsNull()); + assert(index < txn_available.size()); + return txn_available[index] ? true : false; +} + +ReadStatus PartiallyDownloadedBlock::FillBlock(CBlock& block, const std::vector& vtx_missing) const { + assert(!header.IsNull()); + block = header; + block.vtx.resize(txn_available.size()); + + size_t tx_missing_offset = 0; + for (size_t i = 0; i < txn_available.size(); i++) { + if (!txn_available[i]) { + if (vtx_missing.size() <= tx_missing_offset) + return READ_STATUS_INVALID; + block.vtx[i] = vtx_missing[tx_missing_offset++]; + } else + block.vtx[i] = *txn_available[i]; + } + if (vtx_missing.size() != tx_missing_offset) + return READ_STATUS_INVALID; + + CValidationState state; + if (!CheckBlock(block, state, Params().GetConsensus())) { + // TODO: We really want to just check merkle tree manually here, + // but that is expensive, and CheckBlock caches a block's + // "checked-status" (in the CBlock?). CBlock should be able to + // check its own merkle root and cache that check. + if (state.CorruptionPossible()) + return READ_STATUS_FAILED; // Possible Short ID collision + return READ_STATUS_INVALID; + } + + LogPrint("cmpctblock", "Successfully reconstructed block %s with %lu txn prefilled, %lu txn from mempool and %lu txn requested\n", header.GetHash().ToString(), prefilled_count, mempool_count, vtx_missing.size()); + if (vtx_missing.size() < 5) { + for(const CTransaction& tx : vtx_missing) + LogPrint("cmpctblock", "Reconstructed block %s required tx %s\n", header.GetHash().ToString(), tx.GetHash().ToString()); + } + + return READ_STATUS_OK; +} diff --git a/src/blockencodings.h b/src/blockencodings.h new file mode 100644 index 000000000000..b980e9e28660 --- /dev/null +++ b/src/blockencodings.h @@ -0,0 +1,206 @@ +// Copyright (c) 2016 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_BLOCK_ENCODINGS_H +#define BITCOIN_BLOCK_ENCODINGS_H + +#include "primitives/block.h" + +#include + +class CTxMemPool; + +// Dumb helper to handle CTransaction compression at serialize-time +struct TransactionCompressor { +private: + CTransaction& tx; +public: + TransactionCompressor(CTransaction& txIn) : tx(txIn) {} + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + READWRITE(tx); //TODO: Compress tx encoding + } +}; + +class BlockTransactionsRequest { +public: + // A BlockTransactionsRequest message + uint256 blockhash; + std::vector indexes; + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + READWRITE(blockhash); + uint64_t indexes_size = (uint64_t)indexes.size(); + READWRITE(COMPACTSIZE(indexes_size)); + if (ser_action.ForRead()) { + size_t i = 0; + while (indexes.size() < indexes_size) { + indexes.resize(std::min((uint64_t)(1000 + indexes.size()), indexes_size)); + for (; i < indexes.size(); i++) { + uint64_t index = 0; + READWRITE(COMPACTSIZE(index)); + if (index > std::numeric_limits::max()) + throw std::ios_base::failure("index overflowed 16 bits"); + indexes[i] = index; + } + } + + uint16_t offset = 0; + for (size_t i = 0; i < indexes.size(); i++) { + if (uint64_t(indexes[i]) + uint64_t(offset) > std::numeric_limits::max()) + throw std::ios_base::failure("indexes overflowed 16 bits"); + indexes[i] = indexes[i] + offset; + offset = indexes[i] + 1; + } + } else { + for (size_t i = 0; i < indexes.size(); i++) { + uint64_t index = indexes[i] - (i == 0 ? 0 : (indexes[i - 1] + 1)); + READWRITE(COMPACTSIZE(index)); + } + } + } +}; + +class BlockTransactions { +public: + // A BlockTransactions message + uint256 blockhash; + std::vector txn; + + BlockTransactions() {} + BlockTransactions(const BlockTransactionsRequest& req) : + blockhash(req.blockhash), txn(req.indexes.size()) {} + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + READWRITE(blockhash); + uint64_t txn_size = (uint64_t)txn.size(); + READWRITE(COMPACTSIZE(txn_size)); + if (ser_action.ForRead()) { + size_t i = 0; + while (txn.size() < txn_size) { + txn.resize(std::min((uint64_t)(1000 + txn.size()), txn_size)); + for (; i < txn.size(); i++) + READWRITE(REF(TransactionCompressor(txn[i]))); + } + } else { + for (size_t i = 0; i < txn.size(); i++) + READWRITE(REF(TransactionCompressor(txn[i]))); + } + } +}; + +// Dumb serialization/storage-helper for CBlockHeaderAndShortTxIDs and PartiallyDownlaodedBlock +struct PrefilledTransaction { + // Used as an offset since last prefilled tx in CBlockHeaderAndShortTxIDs, + // as a proper transaction-in-block-index in PartiallyDownloadedBlock + uint16_t index; + CTransaction tx; + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + uint64_t idx = index; + READWRITE(COMPACTSIZE(idx)); + if (idx > std::numeric_limits::max()) + throw std::ios_base::failure("index overflowed 16-bits"); + index = idx; + READWRITE(REF(TransactionCompressor(tx))); + } +}; + +typedef enum ReadStatus_t +{ + READ_STATUS_OK, + READ_STATUS_INVALID, // Invalid object, peer is sending bogus crap + READ_STATUS_FAILED, // Failed to process object +} ReadStatus; + +class CBlockHeaderAndShortTxIDs { +private: + mutable uint64_t shorttxidk0, shorttxidk1; + uint64_t nonce; + + void FillShortTxIDSelector() const; + + friend class PartiallyDownloadedBlock; + + static const int SHORTTXIDS_LENGTH = 6; +protected: + std::vector shorttxids; + std::vector prefilledtxn; + +public: + CBlockHeader header; + + // Dummy for deserialization + CBlockHeaderAndShortTxIDs() {} + + CBlockHeaderAndShortTxIDs(const CBlock& block); + + uint64_t GetShortID(const uint256& txhash) const; + + size_t BlockTxCount() const { return shorttxids.size() + prefilledtxn.size(); } + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + READWRITE(header); + READWRITE(nonce); + + uint64_t shorttxids_size = (uint64_t)shorttxids.size(); + READWRITE(COMPACTSIZE(shorttxids_size)); + if (ser_action.ForRead()) { + size_t i = 0; + while (shorttxids.size() < shorttxids_size) { + shorttxids.resize(std::min((uint64_t)(1000 + shorttxids.size()), shorttxids_size)); + for (; i < shorttxids.size(); i++) { + uint32_t lsb = 0; uint16_t msb = 0; + READWRITE(lsb); + READWRITE(msb); + shorttxids[i] = (uint64_t(msb) << 32) | uint64_t(lsb); + static_assert(SHORTTXIDS_LENGTH == 6, "shorttxids serialization assumes 6-byte shorttxids"); + } + } + } else { + for (size_t i = 0; i < shorttxids.size(); i++) { + uint32_t lsb = shorttxids[i] & 0xffffffff; + uint16_t msb = (shorttxids[i] >> 32) & 0xffff; + READWRITE(lsb); + READWRITE(msb); + } + } + + READWRITE(prefilledtxn); + + if (ser_action.ForRead()) + FillShortTxIDSelector(); + } +}; + +class PartiallyDownloadedBlock { +protected: + std::vector > txn_available; + size_t prefilled_count = 0, mempool_count = 0; + CTxMemPool* pool; +public: + CBlockHeader header; + PartiallyDownloadedBlock(CTxMemPool* poolIn) : pool(poolIn) {} + + ReadStatus InitData(const CBlockHeaderAndShortTxIDs& cmpctblock); + bool IsTxAvailable(size_t index) const; + ReadStatus FillBlock(CBlock& block, const std::vector& vtx_missing) const; +}; + +#endif diff --git a/src/net.cpp b/src/net.cpp index 73dc6e82272e..88e8b8d89a6e 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -335,6 +335,16 @@ CNode* CConnman::FindNode(const CService& addr) return NULL; } +//TODO: This is used in only one place in main, and should be removed +CNode* CConnman::FindNode(const NodeId& nodeid) +{ + LOCK(cs_vNodes); + BOOST_FOREACH(CNode* pnode, vNodes) + if (pnode->GetId() == nodeid) + return (pnode); + return NULL; +} + bool CConnman::CheckIncomingNonce(uint64_t nonce) { LOCK(cs_vNodes); diff --git a/src/net.h b/src/net.h index 574192698933..ed92583a0835 100644 --- a/src/net.h +++ b/src/net.h @@ -418,6 +418,7 @@ class CConnman CNode* FindNode(const CSubNet& subNet); CNode* FindNode(const std::string& addrName); CNode* FindNode(const CService& addr); + CNode* FindNode(const NodeId& id); //TODO: Remove this bool AttemptToEvictConnection(); CNode* ConnectNode(CAddress addrConnect, const char *pszDest = NULL, bool fCountFailure = false); diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 770e27b733ff..2fa2a6c23193 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -111,10 +111,14 @@ namespace { struct QueuedBlock { uint256 hash; const CBlockIndex* pindex; //!< Optional. - bool fValidatedHeaders; //!< Whether this block has validated headers at the time of request. + bool fValidatedHeaders; //!< Whether this block has validated headers at the time of request. + std::unique_ptr partialBlock; //!< Optional, used for CMPCTBLOCK downloads }; std::map::iterator> > mapBlocksInFlight; + /** Stack of nodes which we have set to announce using compact blocks */ + std::list lNodesAnnouncingHeaderAndIDs; + /** Number of preferable block download peers. */ int nPreferredDownload = 0; @@ -185,6 +189,10 @@ struct CNodeState { bool fPreferredDownload; //! Whether this peer wants invs or headers (when possible) for block announcements. bool fPreferHeaders; + //! Whether this peer wants invs or cmpctblocks (when possible) for block announcements. + bool fPreferHeaderAndIDs; + //! Whether this peer will send us cmpctblocks if we request them + bool fProvidesHeaderAndIDs; CNodeState(CAddress addrIn, std::string addrNameIn) : address(addrIn), name(addrNameIn) { fCurrentlyConnected = false; @@ -203,6 +211,8 @@ struct CNodeState { nBlocksInFlightValidHeaders = 0; fPreferredDownload = false; fPreferHeaders = false; + fPreferHeaderAndIDs = false; + fProvidesHeaderAndIDs = false; } }; @@ -291,6 +301,7 @@ void FinalizeNode(NodeId nodeid, bool& fUpdateConnectionTime) { // Requires cs_main. // Returns a bool indicating whether we requested this block. +// Also used if a block was /not/ received and timed out or started with another peer bool MarkBlockAsReceived(const uint256& hash) { std::map::iterator> >::iterator itInFlight = mapBlocksInFlight.find(hash); if (itInFlight != mapBlocksInFlight.end()) { @@ -314,17 +325,26 @@ bool MarkBlockAsReceived(const uint256& hash) { } // Requires cs_main. -void MarkBlockAsInFlight(NodeId nodeid, const uint256& hash, const Consensus::Params& consensusParams, const CBlockIndex *pindex = NULL) { +// returns false, still setting pit, if the block was already in flight from the same peer +// pit will only be valid as long as the same cs_main lock is being held +void MarkBlockAsInFlight(NodeId nodeid, const uint256& hash, const Consensus::Params& consensusParams, const CBlockIndex *pindex = NULL, list::iterator **pit = NULL) { CNodeState *state = State(nodeid); assert(state != NULL); + // Short-circuit most stuff in case its from the same node + map::iterator> >::iterator itInFlight = mapBlocksInFlight.find(hash); + if (itInFlight != mapBlocksInFlight.end() && itInFlight->second.first == nodeid) { + *pit = &itInFlight->second.second; + return false; + } + // Make sure it's not listed somewhere already. MarkBlockAsReceived(hash); - QueuedBlock newentry = {hash, pindex, pindex != NULL}; - std::list::iterator it = state->vBlocksInFlight.insert(state->vBlocksInFlight.end(), newentry); + list::iterator it = state->vBlocksInFlight.insert(state->vBlocksInFlight.end(), + {hash, pindex, pindex != NULL, std::unique_ptr(pit ? new PartiallyDownloadedBlock(&mempool) : NULL)}); state->nBlocksInFlight++; - state->nBlocksInFlightValidHeaders += newentry.fValidatedHeaders; + state->nBlocksInFlightValidHeaders += it->fValidatedHeaders; if (state->nBlocksInFlight == 1) { // We're starting a block download (batch) from this peer. state->nDownloadingSince = GetTimeMicros(); @@ -332,7 +352,10 @@ void MarkBlockAsInFlight(NodeId nodeid, const uint256& hash, const Consensus::Pa if (state->nBlocksInFlightValidHeaders == 1 && pindex != NULL) { nPeersWithValidatedDownloads++; } - mapBlocksInFlight[hash] = std::make_pair(nodeid, it); + itInFlight = mapBlocksInFlight.insert(std::make_pair(hash, std::make_pair(nodeid, it))).first; + if (pit) + *pit = &itInFlight->second.second; + return true; } /** Check whether the last unknown block a peer advertised is not yet known. */ @@ -368,6 +391,28 @@ void UpdateBlockAvailability(NodeId nodeid, const uint256 &hash) { } } +void MaybeSetPeerAsAnnouncingHeaderAndIDs(const CNodeState* nodestate, CNode* pfrom, CConnman& connman, CNetMsgMaker& msgMaker) { + if (nodestate->fProvidesHeaderAndIDs) { + BOOST_FOREACH(const NodeId nodeid, lNodesAnnouncingHeaderAndIDs) + if (nodeid == pfrom->GetId()) + return; + bool fAnnounceUsingCMPCTBLOCK = false; + uint64_t nCMPCTBLOCKVersion = 1; + if (lNodesAnnouncingHeaderAndIDs.size() >= 3) { + // As per BIP152, we only get 3 of our peers to announce + // blocks using compact encodings. + CNode* pnodeStop = FindNode(lNodesAnnouncingHeaderAndIDs.front()); + if (pnodeStop) { + connman.PushMessage(pnodeStop, msgMaker.Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); + lNodesAnnouncingHeaderAndIDs.pop_front(); + } + } + fAnnounceUsingCMPCTBLOCK = true; + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); + lNodesAnnouncingHeaderAndIDs.push_back(pfrom->GetId()); + } +} + // Requires cs_main bool CanDirectFetch(const Consensus::Params &consensusParams) { @@ -880,7 +925,7 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam it++; - if (inv.type == MSG_BLOCK || inv.type == MSG_FILTERED_BLOCK) + if (inv.type == MSG_BLOCK || inv.type == MSG_FILTERED_BLOCK || inv.type == MSG_CMPCT_BLOCK) { bool send = false; BlockMap::iterator mi = mapBlockIndex.find(inv.hash); @@ -921,7 +966,7 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam assert(!"cannot load block from disk"); if (inv.type == MSG_BLOCK) connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::BLOCK, block)); - else // MSG_FILTERED_BLOCK) + else if (inv.type == MSG_FILTERED_BLOCK) { bool sendMerkleBlock = false; CMerkleBlock merkleBlock; @@ -947,6 +992,18 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam // else // no response } + else if (inv.type == MSG_CMPCT_BLOCK) + { + // If a peer is asking for old blocks, we're almost guaranteed + // they wont have a useful mempool to match against a compact block, + // and we dont feel like constructing the object for them, so + // instead we respond with the full, non-compact block. + if (mi->second->nHeight >= chainActive.Height() - 10) { + CBlockHeaderAndShortTxIDs cmpctblock(block); + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::CMPCTBLOCK, cmpctblock)); + } else + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::BLOCK, block)); + } // Trigger the peer node to send a getblocks request for the next batch of inventory if (inv.hash == pfrom->hashContinue) @@ -1102,7 +1159,7 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam // Track requests for our stuff. GetMainSignals().Inventory(inv.hash); - if (inv.type == MSG_BLOCK || inv.type == MSG_FILTERED_BLOCK) + if (inv.type == MSG_BLOCK || inv.type == MSG_FILTERED_BLOCK || inv.type == MSG_CMPCT_BLOCK) break; } } @@ -1372,6 +1429,18 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // nodes) connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::SENDHEADERS)); } + + if (pfrom->nVersion >= SHORT_IDS_BLOCKS_VERSION) { + // Tell our peer we are willing to provide version-1 cmpctblocks + // However, we do not request new block announcements using + // cmpctblock messages. + // We send this to non-NODE NETWORK peers as well, because + // they may wish to request compact blocks from us + bool fAnnounceUsingCMPCTBLOCK = false; + uint64_t nCMPCTBLOCKVersion = 1; + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); + } + pfrom->fSuccessfullyConnected = true; } @@ -1436,6 +1505,19 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr } + else if (strCommand == NetMsgType::SENDCMPCT) + { + bool fAnnounceUsingCMPCTBLOCK = false; + uint64_t nCMPCTBLOCKVersion = 1; + vRecv >> fAnnounceUsingCMPCTBLOCK >> nCMPCTBLOCKVersion; + if (nCMPCTBLOCKVersion == 1) { + LOCK(cs_main); + State(pfrom->GetId())->fProvidesHeaderAndIDs = true; + State(pfrom->GetId())->fPreferHeaderAndIDs = fAnnounceUsingCMPCTBLOCK; + } + } + + else if (strCommand == NetMsgType::INV) { std::vector vInv; @@ -1586,6 +1668,40 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr } + + else if (strCommand == NetMsgType::GETBLOCKTXN) + { + BlockTransactionsRequest req; + vRecv >> req; + + BlockMap::iterator it = mapBlockIndex.find(req.blockhash); + if (it == mapBlockIndex.end() || !(it->second->nStatus & BLOCK_HAVE_DATA)) { + Misbehaving(pfrom->GetId(), 100); + LogPrintf("Peer %d sent us a getblocktxn for a block we don't have", pfrom->id); + return true; + } + + if (it->second->nHeight < chainActive.Height() - 15) { + LogPrint("net", "Peer %d sent us a getblocktxn for a block > 15 deep", pfrom->id); + return true; + } + + CBlock block; + assert(ReadBlockFromDisk(block, it->second, chainparams.GetConsensus())); + + BlockTransactions resp(req); + for (size_t i = 0; i < req.indexes.size(); i++) { + if (req.indexes[i] >= block.vtx.size()) { + Misbehaving(pfrom->GetId(), 100); + LogPrintf("Peer %d sent us a getblocktxn with out-of-bounds tx indices", pfrom->id); + return true; + } + resp.txn[i] = block.vtx[req.indexes[i]]; + } + pfrom->PushMessage(NetMsgType::BLOCKTXN, resp); + } + + else if (strCommand == NetMsgType::GETHEADERS) { CBlockLocator locator; @@ -1875,6 +1991,172 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr } } + else if (strCommand == NetMsgType::CMPCTBLOCK && !fImporting && !fReindex) // Ignore blocks received while importing + { + CBlockHeaderAndShortTxIDs cmpctblock; + vRecv >> cmpctblock; + + LOCK(cs_main); + + if (mapBlockIndex.find(cmpctblock.header.hashPrevBlock) == mapBlockIndex.end()) { + // Doesn't connect (or is genesis), instead of DoSing in AcceptBlockHeader, request deeper headers + if (!IsInitialBlockDownload()) + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), uint256())); + return true; + } + + CBlockIndex *pindex = NULL; + CValidationState state; + if (!AcceptBlockHeader(cmpctblock.header, state, chainparams, &pindex)) { + int nDoS; + if (state.IsInvalid(nDoS)) { + if (nDoS > 0) + Misbehaving(pfrom->GetId(), nDoS); + LogPrintf("Peer %d sent us invalid header via cmpctblock\n", pfrom->id); + return true; + } + } + + // If AcceptBlockHeader returned true, it set pindex + assert(pindex); + UpdateBlockAvailability(pfrom->GetId(), pindex->GetBlockHash()); + + std::map::iterator> >::iterator blockInFlightIt = mapBlocksInFlight.find(pindex->GetBlockHash()); + bool fAlreadyInFlight = blockInFlightIt != mapBlocksInFlight.end(); + + if (pindex->nStatus & BLOCK_HAVE_DATA) // Nothing to do here + return true; + + if (pindex->nChainWork <= chainActive.Tip()->nChainWork || // We know something better + pindex->nTx != 0) { // We had this block at some point, but pruned it + if (fAlreadyInFlight) { + // We requested this block for some reason, but our mempool will probably be useless + // so we just grab the block via normal getdata + std::vector vInv(1); + vInv[0] = CInv(MSG_BLOCK, cmpctblock.header.GetHash()); + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); + return true; + } + } + + // If we're not close to tip yet, give up and let parallel block fetch work its magic + if (!fAlreadyInFlight && !CanDirectFetch(chainparams.GetConsensus())) + return true; + + CNodeState *nodestate = State(pfrom->GetId()); + + // We want to be a bit conservative just to be extra careful about DoS + // possibilities in compact block processing... + if (pindex->nHeight <= chainActive.Height() + 2) { + if ((!fAlreadyInFlight && nodestate->nBlocksInFlight < MAX_BLOCKS_IN_TRANSIT_PER_PEER) || + (fAlreadyInFlight && blockInFlightIt->second.first == pfrom->GetId())) { + std::list::iterator *queuedBlockIt = NULL; + if (!MarkBlockAsInFlight(pfrom->GetId(), pindex->GetBlockHash(), chainparams.GetConsensus(), pindex, &queuedBlockIt)) { + if (!(*queuedBlockIt)->partialBlock) + (*queuedBlockIt)->partialBlock.reset(new PartiallyDownloadedBlock(&mempool)); + else { + // The block was already in flight using compact blocks from the same peer + LogPrint("net", "Peer sent us compact block we were already syncing!\n"); + return true; + } + } + + PartiallyDownloadedBlock& partialBlock = *(*queuedBlockIt)->partialBlock; + ReadStatus status = partialBlock.InitData(cmpctblock); + if (status == READ_STATUS_INVALID) { + MarkBlockAsReceived(pindex->GetBlockHash()); // Reset in-flight state in case of whitelist + Misbehaving(pfrom->GetId(), 100); + LogPrintf("Peer %d sent us invalid compact block\n", pfrom->id); + return true; + } else if (status == READ_STATUS_FAILED) { + // Duplicate txindexes, the block is now in-flight, so just request it + std::vector vInv(1); + vInv[0] = CInv(MSG_BLOCK, cmpctblock.header.GetHash()); + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); + return true; + } + + BlockTransactionsRequest req; + for (size_t i = 0; i < cmpctblock.BlockTxCount(); i++) { + if (!partialBlock.IsTxAvailable(i)) + req.indexes.push_back(i); + } + if (req.indexes.empty()) { + // Dirty hack to jump to BLOCKTXN code (TODO: move message handling into their own functions) + BlockTransactions txn; + txn.blockhash = cmpctblock.header.GetHash(); + CDataStream blockTxnMsg(SER_NETWORK, PROTOCOL_VERSION); + blockTxnMsg << txn; + return connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::BLOCKTXN, blockTxnMsg, nTimeReceived, chainparams)); + } else { + req.blockhash = pindex->GetBlockHash(); + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETBLOCKTXN, req)); + } + } + } else { + if (fAlreadyInFlight) { + // We requested this block, but its far into the future, so our + // mempool will probably be useless - request the block normally + std::vector vInv(1); + vInv[0] = CInv(MSG_BLOCK, cmpctblock.header.GetHash()); + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); + return true; + } else { + // If this was an announce-cmpctblock, we want the same treatment as a header message + // Dirty hack to process as if it were just a headers message (TODO: move message handling into their own functions) + std::vector headers; + headers.push_back(cmpctblock.header); + CDataStream vHeadersMsg(SER_NETWORK, PROTOCOL_VERSION); + vHeadersMsg << headers; + return ProcessMessage(pfrom, NetMsgType::HEADERS, vHeadersMsg, nTimeReceived, chainparams, connman, interruptMsgProc); + } + } + + CheckBlockIndex(chainparams.GetConsensus()); + } + + else if (strCommand == NetMsgType::BLOCKTXN && !fImporting && !fReindex) // Ignore blocks received while importing + { + BlockTransactions resp; + vRecv >> resp; + + LOCK(cs_main); + + map::iterator> >::iterator it = mapBlocksInFlight.find(resp.blockhash); + if (it == mapBlocksInFlight.end() || !it->second.second->partialBlock || + it->second.first != pfrom->GetId()) { + LogPrint("net", "Peer %d sent us block transactions for block we weren't expecting\n", pfrom->id); + return true; + } + + PartiallyDownloadedBlock& partialBlock = *it->second.second->partialBlock; + CBlock block; + ReadStatus status = partialBlock.FillBlock(block, resp.txn); + if (status == READ_STATUS_INVALID) { + MarkBlockAsReceived(resp.blockhash); // Reset in-flight state in case of whitelist + Misbehaving(pfrom->GetId(), 100); + LogPrintf("Peer %d sent us invalid compact block/non-matching block transactions\n", pfrom->id); + return true; + } else if (status == READ_STATUS_FAILED) { + // Might have collided, fall back to getdata now :( + std::vector invs; + invs.push_back(CInv(MSG_BLOCK, resp.blockhash)); + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, invs)); + } else { + ProcessNewBlock(chainparams, &block, false, NULL); + int nDoS; + if (state.IsInvalid(nDoS)) { + assert (state.GetRejectCode() < REJECT_INTERNAL); // Blocks are never rejected with internal reject codes + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::REJECT, strCommand, (unsigned char)state.GetRejectCode(), + state.GetRejectReason().substr(0, MAX_REJECT_MESSAGE_LENGTH), block.GetHash())); + if (nDoS > 0) { + LOCK(cs_main); + Misbehaving(pfrom->GetId(), nDoS); + } + } + } + } + else if (strCommand == NetMsgType::HEADERS && !fImporting && !fReindex) // Ignore headers received while importing { @@ -1955,8 +2237,8 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr } nodestate->nUnconnectingHeaders = 0; - if (pindexLast) - UpdateBlockAvailability(pfrom->GetId(), pindexLast->GetBlockHash()); + assert(pindexLast); + UpdateBlockAvailability(pfrom->GetId(), pindexLast->GetBlockHash()); if (nCount == MAX_HEADERS_RESULTS && pindexLast) { // Headers message had its maximum size; the peer may have more headers. @@ -2021,6 +2303,13 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr pindexLast->GetBlockHash().ToString(), pindexLast->nHeight); } if (vGetData.size() > 0) { + if (nodestate->fProvidesHeaderAndIDs && vGetData.size() == 1 && mapBlocksInFlight.size() == 1 && pindexLast->pprev->IsValid(BLOCK_VALID_CHAIN)) { + // We seem to be rather well-synced, so it appears pfrom was the first to provide us + // with this block! Let's get them to announce using compact blocks in the future. + MaybeSetPeerAsAnnouncingHeaderAndIDs(nodestate, pfrom, connman, msgMaker); + // In any case, we want to download using a compact block, not a regular one + vGetData[0] = CInv(MSG_CMPCT_BLOCK, vGetData[0].hash); + } connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vGetData)); } } @@ -2633,7 +2922,9 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic& interr // add all to the inv queue. LOCK(pto->cs_inventory); std::vector vHeaders; - bool fRevertToInv = (!state.fPreferHeaders || pto->vBlockHashesToAnnounce.size() > MAX_BLOCKS_TO_ANNOUNCE); + bool fRevertToInv = ((!state.fPreferHeaders && + (!state.fPreferHeaderAndIDs || pto->vBlockHashesToAnnounce.size() > 1)) || + pto->vBlockHashesToAnnounce.size() > MAX_BLOCKS_TO_ANNOUNCE); const CBlockIndex *pBestIndex = NULL; // last header queued for delivery ProcessBlockAvailability(pto->id); // ensure pindexBestKnownBlock is up-to-date @@ -2685,6 +2976,33 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic& interr } } } + if (!fRevertToInv && !vHeaders.empty()) { + if (vHeaders.size() == 1 && state.fPreferHeaderAndIDs) { + // We only send up to 1 block as header-and-ids, as otherwise + // probably means we're doing an initial-ish-sync or they're slow + LogPrint("net", "%s sending header-and-ids %s to peer %d\n", __func__, + vHeaders.front().GetHash().ToString(), pto->id); + //TODO: Shouldn't need to reload block from disk, but requires refactor + CBlock block; + assert(ReadBlockFromDisk(block, pBestIndex, consensusParams)); + CBlockHeaderAndShortTxIDs cmpctblock(block); + pto->PushMessage(NetMsgType::CMPCTBLOCK, cmpctblock); + state.pindexBestHeaderSent = pBestIndex; + } else if (state.fPreferHeaders) { + if (vHeaders.size() > 1) { + LogPrint("net", "%s: %u headers, range (%s, %s), to peer=%d\n", __func__, + vHeaders.size(), + vHeaders.front().GetHash().ToString(), + vHeaders.back().GetHash().ToString(), pto->id); + } else { + LogPrint("net", "%s: sending header %s to peer=%d\n", __func__, + vHeaders.front().GetHash().ToString(), pto->id); + } + connman.PushMessage(pto, msgMaker.Make(NetMsgType::HEADERS, vHeaders)); + state.pindexBestHeaderSent = pBestIndex; + } else + fRevertToInv = true; + } if (fRevertToInv) { // If falling back to using an inv, just try to inv the tip. // The last entry in vBlockHashesToAnnounce was our tip at some point @@ -2710,18 +3028,6 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic& interr pto->id, hashToAnnounce.ToString()); } } - } else if (!vHeaders.empty()) { - if (vHeaders.size() > 1) { - LogPrint("net", "%s: %u headers, range (%s, %s), to peer=%d\n", __func__, - vHeaders.size(), - vHeaders.front().GetHash().ToString(), - vHeaders.back().GetHash().ToString(), pto->id); - } else { - LogPrint("net", "%s: sending header %s to peer=%d\n", __func__, - vHeaders.front().GetHash().ToString(), pto->id); - } - connman.PushMessage(pto, msgMaker.Make(NetMsgType::HEADERS, vHeaders)); - state.pindexBestHeaderSent = pBestIndex; } pto->vBlockHashesToAnnounce.clear(); } diff --git a/src/protocol.cpp b/src/protocol.cpp index ae1f6ef9e5f0..1ac44645cbc2 100644 --- a/src/protocol.cpp +++ b/src/protocol.cpp @@ -36,6 +36,10 @@ const char *FILTERCLEAR="filterclear"; const char *REJECT="reject"; const char *SENDHEADERS="sendheaders"; const char *FEEFILTER="feefilter"; +const char *SENDCMPCT="sendcmpct"; +const char *CMPCTBLOCK="cmpctblock"; +const char *GETBLOCKTXN="getblocktxn"; +const char *BLOCKTXN="blocktxn"; // Dash message types const char *TXLOCKREQUEST="ix"; const char *TXLOCKVOTE="txlvote"; @@ -92,6 +96,7 @@ static const char* ppszTypeName[] = NetMsgType::MNGOVERNANCEOBJECT, NetMsgType::MNGOVERNANCEOBJECTVOTE, NetMsgType::MNVERIFY, + "compact block", // Should never occur }; /** All known message types. Keep this in the same order as the list of @@ -120,7 +125,11 @@ const static std::string allNetMessageTypes[] = { NetMsgType::FILTERCLEAR, NetMsgType::REJECT, NetMsgType::SENDHEADERS, - NetMsgType::FEEFILTER, + NetMsgType::FEEFILTER, + NetMsgType::SENDCMPCT, + NetMsgType::CMPCTBLOCK, + NetMsgType::GETBLOCKTXN, + NetMsgType::BLOCKTXN, // Dash message types // NOTE: do NOT include non-implmented here, we want them to be "Unknown command" in ProcessMessage() NetMsgType::TXLOCKREQUEST, diff --git a/src/protocol.h b/src/protocol.h index dba95e94fc81..8cdc1a734e18 100644 --- a/src/protocol.h +++ b/src/protocol.h @@ -222,6 +222,33 @@ extern const char *SENDHEADERS; */ extern const char *FEEFILTER; +/** + * Contains a 1-byte bool and 8-byte LE version number. + * Indicates that a node is willing to provide blocks via "cmpctblock" messages. + * May indicate that a node prefers to receive new block announcements via a + * "cmpctblock" message rather than an "inv", depending on message contents. + * @since protocol version 70209 as described by BIP 152 + */ +extern const char *SENDCMPCT; +/** + * Contains a CBlockHeaderAndShortTxIDs object - providing a header and + * list of "short txids". + * @since protocol version 70209 as described by BIP 152 + */ +extern const char *CMPCTBLOCK; +/** + * Contains a BlockTransactionsRequest + * Peer should respond with "blocktxn" message. + * @since protocol version 70209 as described by BIP 152 + */ +extern const char *GETBLOCKTXN; +/** + * Contains a BlockTransactions. + * Sent in response to a "getblocktxn" message. + * @since protocol version 70209 as described by BIP 152 + */ +extern const char *BLOCKTXN; + // Dash message types // NOTE: do NOT declare non-implmented here, we don't want them to be exposed to the outside // TODO: add description @@ -345,6 +372,9 @@ enum GetDataMsg { MSG_GOVERNANCE_OBJECT = 17, MSG_GOVERNANCE_OBJECT_VOTE = 18, MSG_MASTERNODE_VERIFY = 19, + // Nodes may always request a MSG_CMPCT_BLOCK in a getdata, however, + // MSG_CMPCT_BLOCK should not appear in any invs except as a part of getdata. + MSG_CMPCT_BLOCK = 20, //!< Defined in BIP152 }; /** inv message data */ @@ -376,4 +406,5 @@ class CInv uint256 hash; }; + #endif // BITCOIN_PROTOCOL_H diff --git a/src/test/blockencodings_tests.cpp b/src/test/blockencodings_tests.cpp new file mode 100644 index 000000000000..6d5962ed8265 --- /dev/null +++ b/src/test/blockencodings_tests.cpp @@ -0,0 +1,315 @@ +// Copyright (c) 2011-2015 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "blockencodings.h" +#include "consensus/merkle.h" +#include "chainparams.h" +#include "random.h" + +#include "test/test_dash.h" + +#include + +struct RegtestingSetup : public TestingSetup { + RegtestingSetup() : TestingSetup(CBaseChainParams::REGTEST) {} +}; + +BOOST_FIXTURE_TEST_SUITE(blockencodings_tests, RegtestingSetup) + +static CBlock BuildBlockTestCase() { + CBlock block; + CMutableTransaction tx; + tx.vin.resize(1); + tx.vin[0].scriptSig.resize(10); + tx.vout.resize(1); + tx.vout[0].nValue = 42; + + block.vtx.resize(3); + block.vtx[0] = tx; + block.nVersion = 42; + block.hashPrevBlock = GetRandHash(); + block.nBits = 0x207fffff; + + tx.vin[0].prevout.hash = GetRandHash(); + tx.vin[0].prevout.n = 0; + block.vtx[1] = tx; + + tx.vin.resize(10); + for (size_t i = 0; i < tx.vin.size(); i++) { + tx.vin[i].prevout.hash = GetRandHash(); + tx.vin[i].prevout.n = 0; + } + block.vtx[2] = tx; + + bool mutated; + block.hashMerkleRoot = BlockMerkleRoot(block, &mutated); + assert(!mutated); + while (!CheckProofOfWork(block.GetHash(), block.nBits, Params().GetConsensus())) ++block.nNonce; + return block; +} + +// Number of shared use_counts we expect for a tx we havent touched +// == 2 (mempool + our copy from the GetSharedTx call) +#define SHARED_TX_OFFSET 2 + +BOOST_AUTO_TEST_CASE(SimpleRoundTripTest) +{ + CTxMemPool pool(CFeeRate(0)); + TestMemPoolEntryHelper entry; + CBlock block(BuildBlockTestCase()); + + pool.addUnchecked(block.vtx[2].GetHash(), entry.FromTx(block.vtx[2])); + BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2].GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 0); + + // Do a simple ShortTxIDs RT + { + CBlockHeaderAndShortTxIDs shortIDs(block); + + CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); + stream << shortIDs; + + CBlockHeaderAndShortTxIDs shortIDs2; + stream >> shortIDs2; + + PartiallyDownloadedBlock partialBlock(&pool); + BOOST_CHECK(partialBlock.InitData(shortIDs2) == READ_STATUS_OK); + BOOST_CHECK( partialBlock.IsTxAvailable(0)); + BOOST_CHECK(!partialBlock.IsTxAvailable(1)); + BOOST_CHECK( partialBlock.IsTxAvailable(2)); + + BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2].GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); + + std::list removed; + pool.removeRecursive(block.vtx[2], removed); + BOOST_CHECK_EQUAL(removed.size(), 1); + + CBlock block2; + std::vector vtx_missing; + BOOST_CHECK(partialBlock.FillBlock(block2, vtx_missing) == READ_STATUS_INVALID); // No transactions + + vtx_missing.push_back(block.vtx[2]); // Wrong transaction + partialBlock.FillBlock(block2, vtx_missing); // Current implementation doesn't check txn here, but don't require that + bool mutated; + BOOST_CHECK(block.hashMerkleRoot != BlockMerkleRoot(block2, &mutated)); + + vtx_missing[0] = block.vtx[1]; + CBlock block3; + BOOST_CHECK(partialBlock.FillBlock(block3, vtx_missing) == READ_STATUS_OK); + BOOST_CHECK_EQUAL(block.GetHash().ToString(), block3.GetHash().ToString()); + BOOST_CHECK_EQUAL(block.hashMerkleRoot.ToString(), BlockMerkleRoot(block3, &mutated).ToString()); + BOOST_CHECK(!mutated); + } +} + +class TestHeaderAndShortIDs { + // Utility to encode custom CBlockHeaderAndShortTxIDs +public: + CBlockHeader header; + uint64_t nonce; + std::vector shorttxids; + std::vector prefilledtxn; + + TestHeaderAndShortIDs(const CBlockHeaderAndShortTxIDs& orig) { + CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); + stream << orig; + stream >> *this; + } + TestHeaderAndShortIDs(const CBlock& block) : + TestHeaderAndShortIDs(CBlockHeaderAndShortTxIDs(block)) {} + + uint64_t GetShortID(const uint256& txhash) const { + CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); + stream << *this; + CBlockHeaderAndShortTxIDs base; + stream >> base; + return base.GetShortID(txhash); + } + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + READWRITE(header); + READWRITE(nonce); + size_t shorttxids_size = shorttxids.size(); + READWRITE(VARINT(shorttxids_size)); + shorttxids.resize(shorttxids_size); + for (size_t i = 0; i < shorttxids.size(); i++) { + uint32_t lsb = shorttxids[i] & 0xffffffff; + uint16_t msb = (shorttxids[i] >> 32) & 0xffff; + READWRITE(lsb); + READWRITE(msb); + shorttxids[i] = (uint64_t(msb) << 32) | uint64_t(lsb); + } + READWRITE(prefilledtxn); + } +}; + +BOOST_AUTO_TEST_CASE(NonCoinbasePreforwardRTTest) +{ + CTxMemPool pool(CFeeRate(0)); + TestMemPoolEntryHelper entry; + CBlock block(BuildBlockTestCase()); + + pool.addUnchecked(block.vtx[2].GetHash(), entry.FromTx(block.vtx[2])); + BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2].GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 0); + + // Test with pre-forwarding tx 1, but not coinbase + { + TestHeaderAndShortIDs shortIDs(block); + shortIDs.prefilledtxn.resize(1); + shortIDs.prefilledtxn[0] = {1, block.vtx[1]}; + shortIDs.shorttxids.resize(2); + shortIDs.shorttxids[0] = shortIDs.GetShortID(block.vtx[0].GetHash()); + shortIDs.shorttxids[1] = shortIDs.GetShortID(block.vtx[2].GetHash()); + + CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); + stream << shortIDs; + + CBlockHeaderAndShortTxIDs shortIDs2; + stream >> shortIDs2; + + PartiallyDownloadedBlock partialBlock(&pool); + BOOST_CHECK(partialBlock.InitData(shortIDs2) == READ_STATUS_OK); + BOOST_CHECK(!partialBlock.IsTxAvailable(0)); + BOOST_CHECK( partialBlock.IsTxAvailable(1)); + BOOST_CHECK( partialBlock.IsTxAvailable(2)); + + BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2].GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); + + CBlock block2; + std::vector vtx_missing; + BOOST_CHECK(partialBlock.FillBlock(block2, vtx_missing) == READ_STATUS_INVALID); // No transactions + + vtx_missing.push_back(block.vtx[1]); // Wrong transaction + partialBlock.FillBlock(block2, vtx_missing); // Current implementation doesn't check txn here, but don't require that + bool mutated; + BOOST_CHECK(block.hashMerkleRoot != BlockMerkleRoot(block2, &mutated)); + + vtx_missing[0] = block.vtx[0]; + CBlock block3; + BOOST_CHECK(partialBlock.FillBlock(block3, vtx_missing) == READ_STATUS_OK); + BOOST_CHECK_EQUAL(block.GetHash().ToString(), block3.GetHash().ToString()); + BOOST_CHECK_EQUAL(block.hashMerkleRoot.ToString(), BlockMerkleRoot(block3, &mutated).ToString()); + BOOST_CHECK(!mutated); + + BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2].GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); + } + BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2].GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 0); +} + +BOOST_AUTO_TEST_CASE(SufficientPreforwardRTTest) +{ + CTxMemPool pool(CFeeRate(0)); + TestMemPoolEntryHelper entry; + CBlock block(BuildBlockTestCase()); + + pool.addUnchecked(block.vtx[1].GetHash(), entry.FromTx(block.vtx[1])); + BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[1].GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 0); + + // Test with pre-forwarding coinbase + tx 2 with tx 1 in mempool + { + TestHeaderAndShortIDs shortIDs(block); + shortIDs.prefilledtxn.resize(2); + shortIDs.prefilledtxn[0] = {0, block.vtx[0]}; + shortIDs.prefilledtxn[1] = {1, block.vtx[2]}; // id == 1 as it is 1 after index 1 + shortIDs.shorttxids.resize(1); + shortIDs.shorttxids[0] = shortIDs.GetShortID(block.vtx[1].GetHash()); + + CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); + stream << shortIDs; + + CBlockHeaderAndShortTxIDs shortIDs2; + stream >> shortIDs2; + + PartiallyDownloadedBlock partialBlock(&pool); + BOOST_CHECK(partialBlock.InitData(shortIDs2) == READ_STATUS_OK); + BOOST_CHECK( partialBlock.IsTxAvailable(0)); + BOOST_CHECK( partialBlock.IsTxAvailable(1)); + BOOST_CHECK( partialBlock.IsTxAvailable(2)); + + BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[1].GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); + + CBlock block2; + std::vector vtx_missing; + BOOST_CHECK(partialBlock.FillBlock(block2, vtx_missing) == READ_STATUS_OK); + BOOST_CHECK_EQUAL(block.GetHash().ToString(), block2.GetHash().ToString()); + bool mutated; + BOOST_CHECK_EQUAL(block.hashMerkleRoot.ToString(), BlockMerkleRoot(block2, &mutated).ToString()); + BOOST_CHECK(!mutated); + + BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[1].GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); + } + BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[1].GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 0); +} + +BOOST_AUTO_TEST_CASE(EmptyBlockRoundTripTest) +{ + CTxMemPool pool(CFeeRate(0)); + CMutableTransaction coinbase; + coinbase.vin.resize(1); + coinbase.vin[0].scriptSig.resize(10); + coinbase.vout.resize(1); + coinbase.vout[0].nValue = 42; + + CBlock block; + block.vtx.resize(1); + block.vtx[0] = coinbase; + block.nVersion = 42; + block.hashPrevBlock = GetRandHash(); + block.nBits = 0x207fffff; + + bool mutated; + block.hashMerkleRoot = BlockMerkleRoot(block, &mutated); + assert(!mutated); + while (!CheckProofOfWork(block.GetHash(), block.nBits, Params().GetConsensus())) ++block.nNonce; + + // Test simple header round-trip with only coinbase + { + CBlockHeaderAndShortTxIDs shortIDs(block); + + CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); + stream << shortIDs; + + CBlockHeaderAndShortTxIDs shortIDs2; + stream >> shortIDs2; + + PartiallyDownloadedBlock partialBlock(&pool); + BOOST_CHECK(partialBlock.InitData(shortIDs2) == READ_STATUS_OK); + BOOST_CHECK(partialBlock.IsTxAvailable(0)); + + CBlock block2; + std::vector vtx_missing; + BOOST_CHECK(partialBlock.FillBlock(block2, vtx_missing) == READ_STATUS_OK); + BOOST_CHECK_EQUAL(block.GetHash().ToString(), block2.GetHash().ToString()); + bool mutated; + BOOST_CHECK_EQUAL(block.hashMerkleRoot.ToString(), BlockMerkleRoot(block2, &mutated).ToString()); + BOOST_CHECK(!mutated); + } +} + +BOOST_AUTO_TEST_CASE(TransactionsRequestSerializationTest) { + BlockTransactionsRequest req1; + req1.blockhash = GetRandHash(); + req1.indexes.resize(4); + req1.indexes[0] = 0; + req1.indexes[1] = 1; + req1.indexes[2] = 3; + req1.indexes[3] = 4; + + CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); + stream << req1; + + BlockTransactionsRequest req2; + stream >> req2; + + BOOST_CHECK_EQUAL(req1.blockhash.ToString(), req2.blockhash.ToString()); + BOOST_CHECK_EQUAL(req1.indexes.size(), req2.indexes.size()); + BOOST_CHECK_EQUAL(req1.indexes[0], req2.indexes[0]); + BOOST_CHECK_EQUAL(req1.indexes[1], req2.indexes[1]); + BOOST_CHECK_EQUAL(req1.indexes[2], req2.indexes[2]); + BOOST_CHECK_EQUAL(req1.indexes[3], req2.indexes[3]); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/test_dash.cpp b/src/test/test_dash.cpp index 574b240def04..ae459123078a 100644 --- a/src/test/test_dash.cpp +++ b/src/test/test_dash.cpp @@ -142,11 +142,12 @@ TestChain100Setup::~TestChain100Setup() CTxMemPoolEntry TestMemPoolEntryHelper::FromTx(const CMutableTransaction &tx, CTxMemPool *pool) { CTransaction txn(tx); - // Hack to assume either it's completely dependent on other mempool txs or not at all - CAmount inChainValue = pool && pool->HasNoInputsOf(txn) ? txn.GetValueOut() : 0; + return FromTx(txn, pool); +} +CTxMemPoolEntry TestMemPoolEntryHelper::FromTx(const CTransaction &txn, CTxMemPool *pool) { return CTxMemPoolEntry(MakeTransactionRef(txn), nFee, nTime, dPriority, nHeight, - inChainValue, spendsCoinbase, sigOpCount, lp); + txn.GetValueOut(), spendsCoinbase, sigOpCount, lp); } void Shutdown(void* parg) diff --git a/src/test/test_dash.h b/src/test/test_dash.h index c1f6b593aaa2..41440c684f5a 100644 --- a/src/test/test_dash.h +++ b/src/test/test_dash.h @@ -79,7 +79,8 @@ struct TestMemPoolEntryHelper nFee(0), nTime(0), dPriority(0.0), nHeight(1), spendsCoinbase(false), sigOpCount(4) { } - CTxMemPoolEntry FromTx(const CMutableTransaction &tx, CTxMemPool *pool = NULL); + CTxMemPoolEntry FromTx(CMutableTransaction &tx, CTxMemPool *pool = NULL); + CTxMemPoolEntry FromTx(CTransaction &tx, CTxMemPool *pool = NULL); // Change the default value TestMemPoolEntryHelper &Fee(CAmount _fee) { nFee = _fee; return *this; } diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 391afa74958b..8ed45633ac98 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -434,6 +434,9 @@ bool CTxMemPool::addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, totalTxSize += entry.GetTxSize(); minerPolicyEstimator->processTransaction(entry, validFeeEstimate); + vTxHashes.emplace_back(hash, newit); + newit->vTxHashesIdx = vTxHashes.size() - 1; + return true; } @@ -600,6 +603,15 @@ void CTxMemPool::removeUnchecked(txiter it, MemPoolRemovalReason reason) BOOST_FOREACH(const CTxIn& txin, it->GetTx().vin) mapNextTx.erase(txin.prevout); + if (vTxHashes.size() > 1) { + vTxHashes[it->vTxHashesIdx] = std::move(vTxHashes.back()); + vTxHashes[it->vTxHashesIdx].second->vTxHashesIdx = it->vTxHashesIdx; + vTxHashes.pop_back(); + if (vTxHashes.size() * 2 < vTxHashes.capacity()) + vTxHashes.shrink_to_fit(); + } else + vTxHashes.clear(); + totalTxSize -= it->GetTxSize(); cachedInnerUsage -= it->DynamicMemoryUsage(); cachedInnerUsage -= memusage::DynamicUsage(mapLinks[it].parents) + memusage::DynamicUsage(mapLinks[it].children); @@ -1132,7 +1144,7 @@ bool CCoinsViewMemPool::GetCoin(const COutPoint &outpoint, Coin &coin) const { size_t CTxMemPool::DynamicMemoryUsage() const { LOCK(cs); // Estimate the overhead of mapTx to be 15 pointers + an allocation, as no exact formula for boost::multi_index_contained is implemented. - return memusage::MallocUsage(sizeof(CTxMemPoolEntry) + 15 * sizeof(void*)) * mapTx.size() + memusage::DynamicUsage(mapNextTx) + memusage::DynamicUsage(mapDeltas) + memusage::DynamicUsage(mapLinks) + cachedInnerUsage; + return memusage::MallocUsage(sizeof(CTxMemPoolEntry) + 15 * sizeof(void*)) * mapTx.size() + memusage::DynamicUsage(mapNextTx) + memusage::DynamicUsage(mapDeltas) + memusage::DynamicUsage(mapLinks) + memusage::DynamicUsage(vTxHashes) + cachedInnerUsage; } void CTxMemPool::RemoveStaged(setEntries &stage, bool updateDescendants, MemPoolRemovalReason reason) { diff --git a/src/txmempool.h b/src/txmempool.h index 8a5dddf0d992..93a78a8fcbdb 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -158,6 +158,8 @@ class CTxMemPoolEntry uint64_t GetSizeWithAncestors() const { return nSizeWithAncestors; } CAmount GetModFeesWithAncestors() const { return nModFeesWithAncestors; } unsigned int GetSigOpCountWithAncestors() const { return nSigOpCountWithAncestors; } + + mutable size_t vTxHashesIdx; //!< Index in mempool's vTxHashes }; // Helpers for modifying CTxMemPool::mapTx, which is a boost multi_index. @@ -496,7 +498,10 @@ class CTxMemPool mutable CCriticalSection cs; indexed_transaction_set mapTx; + typedef indexed_transaction_set::nth_index<0>::type::iterator txiter; + std::vector > vTxHashes; //!< All tx hashes/entries in mapTx, in random order + struct CompareIteratorByHash { bool operator()(const txiter &a, const txiter &b) const { return a->GetTx().GetHash() < b->GetTx().GetHash(); diff --git a/src/validation.cpp b/src/validation.cpp index cc53355a1061..e6040ef20f87 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -8,6 +8,7 @@ #include "alert.h" #include "arith_uint256.h" +#include "blockencodings.h" #include "chainparams.h" #include "checkpoints.h" #include "checkqueue.h" diff --git a/src/version.h b/src/version.h index c801eb332679..8e7d9f19a208 100644 --- a/src/version.h +++ b/src/version.h @@ -10,7 +10,8 @@ * network protocol versioning */ -static const int PROTOCOL_VERSION = 70208; + +static const int PROTOCOL_VERSION = 70209; //! initial proto version, to be increased after version/verack negotiation static const int INIT_PROTO_VERSION = 209; @@ -43,4 +44,7 @@ static const int FEEFILTER_VERSION = 99999; // disable for now (clarify deployme //! DIP0001 was activated in this version static const int DIP0001_PROTOCOL_VERSION = 70208; +//! shord-id-based block download starts with this version +static const int SHORT_IDS_BLOCKS_VERSION = 70209; + #endif // BITCOIN_VERSION_H From 91d0bd91a933a273480b5b5db4a96d8be35c171f Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Fri, 29 Jul 2016 08:20:57 +0200 Subject: [PATCH 02/51] Merge #8408: Prevent fingerprinting, disk-DoS with compact blocks 1d06e49 Ignore CMPCTBLOCK messages for pruned blocks (Suhas Daftuar) 1de2a46 Ignore GETBLOCKTXN requests for unknown blocks (Suhas Daftuar) --- src/net_processing.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 2fa2a6c23193..57c6a1cba584 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -1676,7 +1676,6 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr BlockMap::iterator it = mapBlockIndex.find(req.blockhash); if (it == mapBlockIndex.end() || !(it->second->nStatus & BLOCK_HAVE_DATA)) { - Misbehaving(pfrom->GetId(), 100); LogPrintf("Peer %d sent us a getblocktxn for a block we don't have", pfrom->id); return true; } @@ -2035,8 +2034,8 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr std::vector vInv(1); vInv[0] = CInv(MSG_BLOCK, cmpctblock.header.GetHash()); connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); - return true; } + return true; } // If we're not close to tip yet, give up and let parallel block fetch work its magic From c0c8d8bd8385100171ca19089609b66435100a19 Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Tue, 2 Aug 2016 08:25:25 +0200 Subject: [PATCH 03/51] Merge #8418: Add tests for compact blocks 45c7ddd Add p2p test for BIP 152 (compact blocks) (Suhas Daftuar) 9a22a6c Add support for compactblocks to mininode (Suhas Daftuar) a8689fd Tests: refactor compact size serialization in mininode (Suhas Daftuar) 9c8593d Implement SipHash in Python (Pieter Wuille) 56c87e9 Allow changing BIP9 parameters on regtest (Suhas Daftuar) --- qa/pull-tester/rpc-tests.py | 1 + qa/rpc-tests/p2p-compactblocks.py | 608 ++++++++++++++++++++++++ qa/rpc-tests/test_framework/mininode.py | 277 ++++++++++- src/test/hash_tests.cpp | 2 + 4 files changed, 884 insertions(+), 4 deletions(-) create mode 100755 qa/rpc-tests/p2p-compactblocks.py diff --git a/qa/pull-tester/rpc-tests.py b/qa/pull-tester/rpc-tests.py index b978f7d5c7f3..e1faaca08977 100755 --- a/qa/pull-tester/rpc-tests.py +++ b/qa/pull-tester/rpc-tests.py @@ -165,6 +165,7 @@ 'rpcnamedargs.py', 'listsinceblock.py', 'p2p-leaktests.py', + 'p2p-compactblocks.py', ] if ENABLE_ZMQ: testScripts.append('zmq_test.py') diff --git a/qa/rpc-tests/p2p-compactblocks.py b/qa/rpc-tests/p2p-compactblocks.py new file mode 100755 index 000000000000..7fe7ecc16c6e --- /dev/null +++ b/qa/rpc-tests/p2p-compactblocks.py @@ -0,0 +1,608 @@ +#!/usr/bin/env python3 +# Copyright (c) 2016 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +from test_framework.mininode import * +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import * +from test_framework.blocktools import create_block, create_coinbase +from test_framework.siphash import siphash256 +from test_framework.script import CScript, OP_TRUE + +''' +CompactBlocksTest -- test compact blocks (BIP 152) +''' + + +# TestNode: A peer we use to send messages to bitcoind, and store responses. +class TestNode(SingleNodeConnCB): + def __init__(self): + SingleNodeConnCB.__init__(self) + self.last_sendcmpct = None + self.last_headers = None + self.last_inv = None + self.last_cmpctblock = None + self.block_announced = False + self.last_getdata = None + self.last_getblocktxn = None + self.last_block = None + self.last_blocktxn = None + + def on_sendcmpct(self, conn, message): + self.last_sendcmpct = message + + def on_block(self, conn, message): + self.last_block = message + + def on_cmpctblock(self, conn, message): + self.last_cmpctblock = message + self.block_announced = True + + def on_headers(self, conn, message): + self.last_headers = message + self.block_announced = True + + def on_inv(self, conn, message): + self.last_inv = message + self.block_announced = True + + def on_getdata(self, conn, message): + self.last_getdata = message + + def on_getblocktxn(self, conn, message): + self.last_getblocktxn = message + + def on_blocktxn(self, conn, message): + self.last_blocktxn = message + + # Requires caller to hold mininode_lock + def received_block_announcement(self): + return self.block_announced + + def clear_block_announcement(self): + with mininode_lock: + self.block_announced = False + self.last_inv = None + self.last_headers = None + self.last_cmpctblock = None + + def get_headers(self, locator, hashstop): + msg = msg_getheaders() + msg.locator.vHave = locator + msg.hashstop = hashstop + self.connection.send_message(msg) + + def send_header_for_blocks(self, new_blocks): + headers_message = msg_headers() + headers_message.headers = [CBlockHeader(b) for b in new_blocks] + self.send_message(headers_message) + + +class CompactBlocksTest(BitcoinTestFramework): + def __init__(self): + super().__init__() + self.setup_clean_chain = True + self.num_nodes = 1 + self.utxos = [] + + def setup_network(self): + self.nodes = [] + + # Turn off segwit in this test, as compact blocks don't currently work + # with segwit. (After BIP 152 is updated to support segwit, we can + # test behavior with and without segwit enabled by adding a second node + # to the test.) + self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, [["-debug", "-logtimemicros=1", "-bip9params=segwit:0:0"]]) + + def build_block_on_tip(self): + height = self.nodes[0].getblockcount() + tip = self.nodes[0].getbestblockhash() + mtp = self.nodes[0].getblockheader(tip)['mediantime'] + block = create_block(int(tip, 16), create_coinbase(height + 1), mtp + 1) + block.solve() + return block + + # Create 10 more anyone-can-spend utxo's for testing. + def make_utxos(self): + block = self.build_block_on_tip() + self.test_node.send_and_ping(msg_block(block)) + assert(int(self.nodes[0].getbestblockhash(), 16) == block.sha256) + self.nodes[0].generate(100) + + total_value = block.vtx[0].vout[0].nValue + out_value = total_value // 10 + tx = CTransaction() + tx.vin.append(CTxIn(COutPoint(block.vtx[0].sha256, 0), b'')) + for i in range(10): + tx.vout.append(CTxOut(out_value, CScript([OP_TRUE]))) + tx.rehash() + + block2 = self.build_block_on_tip() + block2.vtx.append(tx) + block2.hashMerkleRoot = block2.calc_merkle_root() + block2.solve() + self.test_node.send_and_ping(msg_block(block2)) + assert_equal(int(self.nodes[0].getbestblockhash(), 16), block2.sha256) + self.utxos.extend([[tx.sha256, i, out_value] for i in range(10)]) + return + + # Test "sendcmpct": + # - No compact block announcements or getdata(MSG_CMPCT_BLOCK) unless + # sendcmpct is sent. + # - If sendcmpct is sent with version > 0, the message is ignored. + # - If sendcmpct is sent with boolean 0, then block announcements are not + # made with compact blocks. + # - If sendcmpct is then sent with boolean 1, then new block announcements + # are made with compact blocks. + def test_sendcmpct(self): + print("Testing SENDCMPCT p2p message... ") + + # Make sure we get a version 0 SENDCMPCT message from our peer + def received_sendcmpct(): + return (self.test_node.last_sendcmpct is not None) + got_message = wait_until(received_sendcmpct, timeout=30) + assert(got_message) + assert_equal(self.test_node.last_sendcmpct.version, 1) + + tip = int(self.nodes[0].getbestblockhash(), 16) + + def check_announcement_of_new_block(node, peer, predicate): + self.test_node.clear_block_announcement() + node.generate(1) + got_message = wait_until(peer.received_block_announcement, timeout=30) + assert(got_message) + with mininode_lock: + assert(predicate) + + # We shouldn't get any block announcements via cmpctblock yet. + check_announcement_of_new_block(self.nodes[0], self.test_node, lambda: self.test_node.last_cmpctblock is None) + + # Try one more time, this time after requesting headers. + self.test_node.clear_block_announcement() + self.test_node.get_headers(locator=[tip], hashstop=0) + wait_until(self.test_node.received_block_announcement, timeout=30) + self.test_node.clear_block_announcement() + + check_announcement_of_new_block(self.nodes[0], self.test_node, lambda: self.test_node.last_cmpctblock is None and self.test_node.last_inv is not None) + + # Now try a SENDCMPCT message with too-high version + sendcmpct = msg_sendcmpct() + sendcmpct.version = 2 + self.test_node.send_message(sendcmpct) + + check_announcement_of_new_block(self.nodes[0], self.test_node, lambda: self.test_node.last_cmpctblock is None) + + # Now try a SENDCMPCT message with valid version, but announce=False + self.test_node.send_message(msg_sendcmpct()) + check_announcement_of_new_block(self.nodes[0], self.test_node, lambda: self.test_node.last_cmpctblock is None) + + # Finally, try a SENDCMPCT message with announce=True + sendcmpct.version = 1 + sendcmpct.announce = True + self.test_node.send_message(sendcmpct) + check_announcement_of_new_block(self.nodes[0], self.test_node, lambda: self.test_node.last_cmpctblock is not None) + + # Try one more time + check_announcement_of_new_block(self.nodes[0], self.test_node, lambda: self.test_node.last_cmpctblock is not None) + + # Try one more time, after turning on sendheaders + self.test_node.send_message(msg_sendheaders()) + check_announcement_of_new_block(self.nodes[0], self.test_node, lambda: self.test_node.last_cmpctblock is not None) + + # Now turn off announcements + sendcmpct.announce = False + check_announcement_of_new_block(self.nodes[0], self.test_node, lambda: self.test_node.last_cmpctblock is None and self.test_node.last_headers is not None) + + # This test actually causes bitcoind to (reasonably!) disconnect us, so do this last. + def test_invalid_cmpctblock_message(self): + print("Testing invalid index in cmpctblock message...") + self.nodes[0].generate(101) + block = self.build_block_on_tip() + + cmpct_block = P2PHeaderAndShortIDs() + cmpct_block.header = CBlockHeader(block) + cmpct_block.prefilled_txn_length = 1 + # This index will be too high + prefilled_txn = PrefilledTransaction(1, block.vtx[0]) + cmpct_block.prefilled_txn = [prefilled_txn] + self.test_node.send_and_ping(msg_cmpctblock(cmpct_block)) + assert(int(self.nodes[0].getbestblockhash(), 16) == block.hashPrevBlock) + + # Compare the generated shortids to what we expect based on BIP 152, given + # bitcoind's choice of nonce. + def test_compactblock_construction(self): + print("Testing compactblock headers and shortIDs are correct...") + + # Generate a bunch of transactions. + self.nodes[0].generate(101) + num_transactions = 25 + address = self.nodes[0].getnewaddress() + for i in range(num_transactions): + self.nodes[0].sendtoaddress(address, 0.1) + + # Now mine a block, and look at the resulting compact block. + self.test_node.clear_block_announcement() + block_hash = int(self.nodes[0].generate(1)[0], 16) + + # Store the raw block in our internal format. + block = FromHex(CBlock(), self.nodes[0].getblock("%02x" % block_hash, False)) + [tx.calc_sha256() for tx in block.vtx] + block.rehash() + + # Don't care which type of announcement came back for this test; just + # request the compact block if we didn't get one yet. + wait_until(self.test_node.received_block_announcement, timeout=30) + + with mininode_lock: + if self.test_node.last_cmpctblock is None: + self.test_node.clear_block_announcement() + inv = CInv(4, block_hash) # 4 == "CompactBlock" + self.test_node.send_message(msg_getdata([inv])) + + wait_until(self.test_node.received_block_announcement, timeout=30) + + # Now we should have the compactblock + header_and_shortids = None + with mininode_lock: + assert(self.test_node.last_cmpctblock is not None) + # Convert the on-the-wire representation to absolute indexes + header_and_shortids = HeaderAndShortIDs(self.test_node.last_cmpctblock.header_and_shortids) + + # Check that we got the right block! + header_and_shortids.header.calc_sha256() + assert_equal(header_and_shortids.header.sha256, block_hash) + + # Make sure the prefilled_txn appears to have included the coinbase + assert(len(header_and_shortids.prefilled_txn) >= 1) + assert_equal(header_and_shortids.prefilled_txn[0].index, 0) + + # Check that all prefilled_txn entries match what's in the block. + for entry in header_and_shortids.prefilled_txn: + entry.tx.calc_sha256() + assert_equal(entry.tx.sha256, block.vtx[entry.index].sha256) + + # Check that the cmpctblock message announced all the transactions. + assert_equal(len(header_and_shortids.prefilled_txn) + len(header_and_shortids.shortids), len(block.vtx)) + + # And now check that all the shortids are as expected as well. + # Determine the siphash keys to use. + [k0, k1] = header_and_shortids.get_siphash_keys() + + index = 0 + while index < len(block.vtx): + if (len(header_and_shortids.prefilled_txn) > 0 and + header_and_shortids.prefilled_txn[0].index == index): + # Already checked prefilled transactions above + header_and_shortids.prefilled_txn.pop(0) + else: + shortid = calculate_shortid(k0, k1, block.vtx[index].sha256) + assert_equal(shortid, header_and_shortids.shortids[0]) + header_and_shortids.shortids.pop(0) + index += 1 + + # Test that bitcoind requests compact blocks when we announce new blocks + # via header or inv, and that responding to getblocktxn causes the block + # to be successfully reconstructed. + def test_compactblock_requests(self): + print("Testing compactblock requests... ") + + # Try announcing a block with an inv or header, expect a compactblock + # request + for announce in ["inv", "header"]: + block = self.build_block_on_tip() + with mininode_lock: + self.test_node.last_getdata = None + + if announce == "inv": + self.test_node.send_message(msg_inv([CInv(2, block.sha256)])) + else: + self.test_node.send_header_for_blocks([block]) + success = wait_until(lambda: self.test_node.last_getdata is not None, timeout=30) + assert(success) + assert_equal(len(self.test_node.last_getdata.inv), 1) + assert_equal(self.test_node.last_getdata.inv[0].type, 4) + assert_equal(self.test_node.last_getdata.inv[0].hash, block.sha256) + + # Send back a compactblock message that omits the coinbase + comp_block = HeaderAndShortIDs() + comp_block.header = CBlockHeader(block) + comp_block.nonce = 0 + comp_block.shortids = [1] # this is useless, and wrong + self.test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) + assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.hashPrevBlock) + # Expect a getblocktxn message. + with mininode_lock: + assert(self.test_node.last_getblocktxn is not None) + absolute_indexes = self.test_node.last_getblocktxn.block_txn_request.to_absolute() + assert_equal(absolute_indexes, [0]) # should be a coinbase request + + # Send the coinbase, and verify that the tip advances. + msg = msg_blocktxn() + msg.block_transactions.blockhash = block.sha256 + msg.block_transactions.transactions = [block.vtx[0]] + self.test_node.send_and_ping(msg) + assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256) + + # Create a chain of transactions from given utxo, and add to a new block. + def build_block_with_transactions(self, utxo, num_transactions): + block = self.build_block_on_tip() + + for i in range(num_transactions): + tx = CTransaction() + tx.vin.append(CTxIn(COutPoint(utxo[0], utxo[1]), b'')) + tx.vout.append(CTxOut(utxo[2] - 1000, CScript([OP_TRUE]))) + tx.rehash() + utxo = [tx.sha256, 0, tx.vout[0].nValue] + block.vtx.append(tx) + + block.hashMerkleRoot = block.calc_merkle_root() + block.solve() + return block + + # Test that we only receive getblocktxn requests for transactions that the + # node needs, and that responding to them causes the block to be + # reconstructed. + def test_getblocktxn_requests(self): + print("Testing getblocktxn requests...") + + # First try announcing compactblocks that won't reconstruct, and verify + # that we receive getblocktxn messages back. + utxo = self.utxos.pop(0) + + block = self.build_block_with_transactions(utxo, 5) + self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) + + comp_block = HeaderAndShortIDs() + comp_block.initialize_from_block(block) + + self.test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) + with mininode_lock: + assert(self.test_node.last_getblocktxn is not None) + absolute_indexes = self.test_node.last_getblocktxn.block_txn_request.to_absolute() + assert_equal(absolute_indexes, [1, 2, 3, 4, 5]) + msg = msg_blocktxn() + msg.block_transactions = BlockTransactions(block.sha256, block.vtx[1:]) + self.test_node.send_and_ping(msg) + assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256) + + utxo = self.utxos.pop(0) + block = self.build_block_with_transactions(utxo, 5) + self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) + + # Now try interspersing the prefilled transactions + comp_block.initialize_from_block(block, prefill_list=[0, 1, 5]) + self.test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) + with mininode_lock: + assert(self.test_node.last_getblocktxn is not None) + absolute_indexes = self.test_node.last_getblocktxn.block_txn_request.to_absolute() + assert_equal(absolute_indexes, [2, 3, 4]) + msg.block_transactions = BlockTransactions(block.sha256, block.vtx[2:5]) + self.test_node.send_and_ping(msg) + assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256) + + # Now try giving one transaction ahead of time. + utxo = self.utxos.pop(0) + block = self.build_block_with_transactions(utxo, 5) + self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) + self.test_node.send_and_ping(msg_tx(block.vtx[1])) + assert(block.vtx[1].hash in self.nodes[0].getrawmempool()) + + # Prefill 4 out of the 6 transactions, and verify that only the one + # that was not in the mempool is requested. + comp_block.initialize_from_block(block, prefill_list=[0, 2, 3, 4]) + self.test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) + with mininode_lock: + assert(self.test_node.last_getblocktxn is not None) + absolute_indexes = self.test_node.last_getblocktxn.block_txn_request.to_absolute() + assert_equal(absolute_indexes, [5]) + + msg.block_transactions = BlockTransactions(block.sha256, [block.vtx[5]]) + self.test_node.send_and_ping(msg) + assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256) + + # Now provide all transactions to the node before the block is + # announced and verify reconstruction happens immediately. + utxo = self.utxos.pop(0) + block = self.build_block_with_transactions(utxo, 10) + self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) + for tx in block.vtx[1:]: + self.test_node.send_message(msg_tx(tx)) + self.test_node.sync_with_ping() + # Make sure all transactions were accepted. + mempool = self.nodes[0].getrawmempool() + for tx in block.vtx[1:]: + assert(tx.hash in mempool) + + # Clear out last request. + with mininode_lock: + self.test_node.last_getblocktxn = None + + # Send compact block + comp_block.initialize_from_block(block, prefill_list=[0]) + self.test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) + with mininode_lock: + # Shouldn't have gotten a request for any transaction + assert(self.test_node.last_getblocktxn is None) + # Tip should have updated + assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256) + + # Incorrectly responding to a getblocktxn shouldn't cause the block to be + # permanently failed. + def test_incorrect_blocktxn_response(self): + print("Testing handling of incorrect blocktxn responses...") + + if (len(self.utxos) == 0): + self.make_utxos() + utxo = self.utxos.pop(0) + + block = self.build_block_with_transactions(utxo, 10) + self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) + # Relay the first 5 transactions from the block in advance + for tx in block.vtx[1:6]: + self.test_node.send_message(msg_tx(tx)) + self.test_node.sync_with_ping() + # Make sure all transactions were accepted. + mempool = self.nodes[0].getrawmempool() + for tx in block.vtx[1:6]: + assert(tx.hash in mempool) + + # Send compact block + comp_block = HeaderAndShortIDs() + comp_block.initialize_from_block(block, prefill_list=[0]) + self.test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) + absolute_indexes = [] + with mininode_lock: + assert(self.test_node.last_getblocktxn is not None) + absolute_indexes = self.test_node.last_getblocktxn.block_txn_request.to_absolute() + assert_equal(absolute_indexes, [6, 7, 8, 9, 10]) + + # Now give an incorrect response. + # Note that it's possible for bitcoind to be smart enough to know we're + # lying, since it could check to see if the shortid matches what we're + # sending, and eg disconnect us for misbehavior. If that behavior + # change were made, we could just modify this test by having a + # different peer provide the block further down, so that we're still + # verifying that the block isn't marked bad permanently. This is good + # enough for now. + msg = msg_blocktxn() + msg.block_transactions = BlockTransactions(block.sha256, [block.vtx[5]] + block.vtx[7:]) + self.test_node.send_and_ping(msg) + + # Tip should not have updated + assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.hashPrevBlock) + + # We should receive a getdata request + success = wait_until(lambda: self.test_node.last_getdata is not None, timeout=10) + assert(success) + assert_equal(len(self.test_node.last_getdata.inv), 1) + assert_equal(self.test_node.last_getdata.inv[0].type, 2) + assert_equal(self.test_node.last_getdata.inv[0].hash, block.sha256) + + # Deliver the block + self.test_node.send_and_ping(msg_block(block)) + assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256) + + def test_getblocktxn_handler(self): + print("Testing getblocktxn handler...") + + # bitcoind won't respond for blocks whose height is more than 15 blocks + # deep. + MAX_GETBLOCKTXN_DEPTH = 15 + chain_height = self.nodes[0].getblockcount() + current_height = chain_height + while (current_height >= chain_height - MAX_GETBLOCKTXN_DEPTH): + block_hash = self.nodes[0].getblockhash(current_height) + block = FromHex(CBlock(), self.nodes[0].getblock(block_hash, False)) + + msg = msg_getblocktxn() + msg.block_txn_request = BlockTransactionsRequest(int(block_hash, 16), []) + num_to_request = random.randint(1, len(block.vtx)) + msg.block_txn_request.from_absolute(sorted(random.sample(range(len(block.vtx)), num_to_request))) + self.test_node.send_message(msg) + success = wait_until(lambda: self.test_node.last_blocktxn is not None, timeout=10) + assert(success) + + [tx.calc_sha256() for tx in block.vtx] + with mininode_lock: + assert_equal(self.test_node.last_blocktxn.block_transactions.blockhash, int(block_hash, 16)) + all_indices = msg.block_txn_request.to_absolute() + for index in all_indices: + tx = self.test_node.last_blocktxn.block_transactions.transactions.pop(0) + tx.calc_sha256() + assert_equal(tx.sha256, block.vtx[index].sha256) + self.test_node.last_blocktxn = None + current_height -= 1 + + # Next request should be ignored, as we're past the allowed depth. + block_hash = self.nodes[0].getblockhash(current_height) + msg.block_txn_request = BlockTransactionsRequest(int(block_hash, 16), [0]) + self.test_node.send_and_ping(msg) + with mininode_lock: + assert_equal(self.test_node.last_blocktxn, None) + + def test_compactblocks_not_at_tip(self): + print("Testing compactblock requests/announcements not at chain tip...") + + # Test that requesting old compactblocks doesn't work. + MAX_CMPCTBLOCK_DEPTH = 11 + new_blocks = [] + for i in range(MAX_CMPCTBLOCK_DEPTH): + self.test_node.clear_block_announcement() + new_blocks.append(self.nodes[0].generate(1)[0]) + wait_until(self.test_node.received_block_announcement, timeout=30) + + self.test_node.clear_block_announcement() + self.test_node.send_message(msg_getdata([CInv(4, int(new_blocks[0], 16))])) + success = wait_until(lambda: self.test_node.last_cmpctblock is not None, timeout=30) + assert(success) + + self.test_node.clear_block_announcement() + self.nodes[0].generate(1) + wait_until(self.test_node.received_block_announcement, timeout=30) + self.test_node.clear_block_announcement() + self.test_node.send_message(msg_getdata([CInv(4, int(new_blocks[0], 16))])) + success = wait_until(lambda: self.test_node.last_block is not None, timeout=30) + assert(success) + with mininode_lock: + self.test_node.last_block.block.calc_sha256() + assert_equal(self.test_node.last_block.block.sha256, int(new_blocks[0], 16)) + + # Generate an old compactblock, and verify that it's not accepted. + cur_height = self.nodes[0].getblockcount() + hashPrevBlock = int(self.nodes[0].getblockhash(cur_height-5), 16) + block = self.build_block_on_tip() + block.hashPrevBlock = hashPrevBlock + block.solve() + + comp_block = HeaderAndShortIDs() + comp_block.initialize_from_block(block) + self.test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) + + tips = self.nodes[0].getchaintips() + found = False + for x in tips: + if x["hash"] == block.hash: + assert_equal(x["status"], "headers-only") + found = True + break + assert(found) + + # Requesting this block via getblocktxn should silently fail + # (to avoid fingerprinting attacks). + msg = msg_getblocktxn() + msg.block_txn_request = BlockTransactionsRequest(block.sha256, [0]) + with mininode_lock: + self.test_node.last_blocktxn = None + self.test_node.send_and_ping(msg) + with mininode_lock: + assert(self.test_node.last_blocktxn is None) + + def run_test(self): + # Setup the p2p connections and start up the network thread. + self.test_node = TestNode() + + connections = [] + connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], self.test_node)) + self.test_node.add_connection(connections[0]) + + NetworkThread().start() # Start up network handling in another thread + + # Test logic begins here + self.test_node.wait_for_verack() + + # We will need UTXOs to construct transactions in later tests. + self.make_utxos() + + self.test_sendcmpct() + self.test_compactblock_construction() + self.test_compactblock_requests() + self.test_getblocktxn_requests() + self.test_getblocktxn_handler() + self.test_compactblocks_not_at_tip() + self.test_incorrect_blocktxn_response() + self.test_invalid_cmpctblock_message() + + +if __name__ == '__main__': + CompactBlocksTest().main() diff --git a/qa/rpc-tests/test_framework/mininode.py b/qa/rpc-tests/test_framework/mininode.py index b32b8dcfae7e..0affe70b08bc 100755 --- a/qa/rpc-tests/test_framework/mininode.py +++ b/qa/rpc-tests/test_framework/mininode.py @@ -36,11 +36,12 @@ from threading import Thread import logging import copy +from test_framework.siphash import siphash256 import dash_hash BIP0031_VERSION = 60000 -MY_VERSION = 70208 # current MIN_PEER_PROTO_VERSION +MY_VERSION = 70209 # current SHORT_IDS_BLOCKS_VERSION to support cmpct blocks? MY_SUBVERSION = b"/python-mininode-tester:0.0.3/" MY_RELAY = 1 # from version 70001 onwards, fRelay should be appended to version messages (BIP37) @@ -54,7 +55,7 @@ NODE_BLOOM = (1 << 2) # Keep our own socket map for asyncore, so that we can track disconnects -# ourselves (to workaround an issue with closing an asyncore socket when +# ourselves (to workaround an issue with closing an asyncore socket when # using select) mininode_socket_map = dict() @@ -241,7 +242,9 @@ class CInv(object): typemap = { 0: "Error", 1: "TX", - 2: "Block"} + 2: "Block", + 20: "CompactBlock" + } def __init__(self, t=0, h=0): self.type = t @@ -604,6 +607,187 @@ def __repr__(self): % (len(self.vchMsg), len(self.vchSig)) +class PrefilledTransaction(object): + def __init__(self, index=0, tx = None): + self.index = index + self.tx = tx + + def deserialize(self, f): + self.index = deser_compact_size(f) + self.tx = CTransaction() + self.tx.deserialize(f) + + def serialize(self, with_witness=False): + r = b"" + r += ser_compact_size(self.index) + if with_witness: + r += self.tx.serialize_with_witness() + else: + r += self.tx.serialize_without_witness() + return r + + def __repr__(self): + return "PrefilledTransaction(index=%d, tx=%s)" % (self.index, repr(self.tx)) + +# This is what we send on the wire, in a cmpctblock message. +class P2PHeaderAndShortIDs(object): + def __init__(self): + self.header = CBlockHeader() + self.nonce = 0 + self.shortids_length = 0 + self.shortids = [] + self.prefilled_txn_length = 0 + self.prefilled_txn = [] + + def deserialize(self, f): + self.header.deserialize(f) + self.nonce = struct.unpack(" Date: Wed, 17 Aug 2016 12:50:28 +0200 Subject: [PATCH 04/51] Merge #8505: Trivial: Fix typos in various files 1aacfc2 various typos (leijurv) --- src/blockencodings.cpp | 2 +- src/net.cpp | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/blockencodings.cpp b/src/blockencodings.cpp index aa92aacf6d4e..6136240bbe67 100644 --- a/src/blockencodings.cpp +++ b/src/blockencodings.cpp @@ -75,7 +75,7 @@ ReadStatus PartiallyDownloadedBlock::InitData(const CBlockHeaderAndShortTxIDs& c } prefilled_count = cmpctblock.prefilledtxn.size(); - // Calculate map of txids -> positions and check mempool to see what we have (or dont) + // Calculate map of txids -> positions and check mempool to see what we have (or don't) // Because well-formed cmpctblock messages will have a (relatively) uniform distribution // of short IDs, any highly-uneven distribution of elements can be safely treated as a // READ_STATUS_FAILED. diff --git a/src/net.cpp b/src/net.cpp index 88e8b8d89a6e..ea2a69b38a40 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -1817,6 +1817,7 @@ void CConnman::ThreadOpenConnections() if (nANow - addr.nLastTry < 600 && nTries < 30) continue; + // only consider nodes missing relevant services after 40 failed attempts and only if less than half the outbound are up. ServiceFlags nRequiredServices = nRelevantServices; if (nTries >= 40 && nOutbound < (nMaxOutbound >> 1)) { From 63c33e210d96e9601fe33bc21e4b1b3670d8700b Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Fri, 2 Sep 2016 12:25:49 +0200 Subject: [PATCH 05/51] Merge #8449: [Trivial] Do not shadow local variable, cleanup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit a159f25 Remove redundand (and shadowing) declaration (Pavel Janík) cce3024 Do not shadow local variable, cleanup (Pavel Janík) --- src/blockencodings.h | 8 ++++---- src/test/blockencodings_tests.cpp | 1 - 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/blockencodings.h b/src/blockencodings.h index b980e9e28660..349fcbd50f8c 100644 --- a/src/blockencodings.h +++ b/src/blockencodings.h @@ -53,11 +53,11 @@ class BlockTransactionsRequest { } uint16_t offset = 0; - for (size_t i = 0; i < indexes.size(); i++) { - if (uint64_t(indexes[i]) + uint64_t(offset) > std::numeric_limits::max()) + for (size_t j = 0; j < indexes.size(); j++) { + if (uint64_t(indexes[j]) + uint64_t(offset) > std::numeric_limits::max()) throw std::ios_base::failure("indexes overflowed 16 bits"); - indexes[i] = indexes[i] + offset; - offset = indexes[i] + 1; + indexes[j] = indexes[j] + offset; + offset = indexes[j] + 1; } } else { for (size_t i = 0; i < indexes.size(); i++) { diff --git a/src/test/blockencodings_tests.cpp b/src/test/blockencodings_tests.cpp index 6d5962ed8265..179e906159f9 100644 --- a/src/test/blockencodings_tests.cpp +++ b/src/test/blockencodings_tests.cpp @@ -283,7 +283,6 @@ BOOST_AUTO_TEST_CASE(EmptyBlockRoundTripTest) std::vector vtx_missing; BOOST_CHECK(partialBlock.FillBlock(block2, vtx_missing) == READ_STATUS_OK); BOOST_CHECK_EQUAL(block.GetHash().ToString(), block2.GetHash().ToString()); - bool mutated; BOOST_CHECK_EQUAL(block.hashMerkleRoot.ToString(), BlockMerkleRoot(block2, &mutated).ToString()); BOOST_CHECK(!mutated); } From caae165aa53d22df136ae6d2cc609218358a7658 Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Tue, 20 Sep 2016 16:21:45 +0200 Subject: [PATCH 06/51] Merge #8739: [qa] Fix broken sendcmpct test in p2p-compactblocks.py 157254a Fix broken sendcmpct test in p2p-compactblocks.py (Suhas Daftuar) --- qa/rpc-tests/p2p-compactblocks.py | 58 ++++++++++++++++++++----------- 1 file changed, 37 insertions(+), 21 deletions(-) diff --git a/qa/rpc-tests/p2p-compactblocks.py b/qa/rpc-tests/p2p-compactblocks.py index 7fe7ecc16c6e..bf4fb43add3d 100755 --- a/qa/rpc-tests/p2p-compactblocks.py +++ b/qa/rpc-tests/p2p-compactblocks.py @@ -78,6 +78,13 @@ def send_header_for_blocks(self, new_blocks): headers_message.headers = [CBlockHeader(b) for b in new_blocks] self.send_message(headers_message) + def request_headers_and_sync(self, locator, hashstop=0): + self.clear_block_announcement() + self.get_headers(locator, hashstop) + assert(wait_until(self.received_block_announcement, timeout=30)) + assert(self.received_block_announcement()) + self.clear_block_announcement() + class CompactBlocksTest(BitcoinTestFramework): def __init__(self): @@ -130,7 +137,7 @@ def make_utxos(self): # Test "sendcmpct": # - No compact block announcements or getdata(MSG_CMPCT_BLOCK) unless # sendcmpct is sent. - # - If sendcmpct is sent with version > 0, the message is ignored. + # - If sendcmpct is sent with version > 1, the message is ignored. # - If sendcmpct is sent with boolean 0, then block announcements are not # made with compact blocks. # - If sendcmpct is then sent with boolean 1, then new block announcements @@ -142,57 +149,66 @@ def test_sendcmpct(self): def received_sendcmpct(): return (self.test_node.last_sendcmpct is not None) got_message = wait_until(received_sendcmpct, timeout=30) + assert(received_sendcmpct()) assert(got_message) assert_equal(self.test_node.last_sendcmpct.version, 1) tip = int(self.nodes[0].getbestblockhash(), 16) def check_announcement_of_new_block(node, peer, predicate): - self.test_node.clear_block_announcement() + peer.clear_block_announcement() node.generate(1) - got_message = wait_until(peer.received_block_announcement, timeout=30) + got_message = wait_until(lambda: peer.block_announced, timeout=30) + assert(peer.block_announced) assert(got_message) with mininode_lock: - assert(predicate) + assert(predicate(peer)) # We shouldn't get any block announcements via cmpctblock yet. - check_announcement_of_new_block(self.nodes[0], self.test_node, lambda: self.test_node.last_cmpctblock is None) + check_announcement_of_new_block(self.nodes[0], self.test_node, lambda p: p.last_cmpctblock is None) # Try one more time, this time after requesting headers. - self.test_node.clear_block_announcement() - self.test_node.get_headers(locator=[tip], hashstop=0) - wait_until(self.test_node.received_block_announcement, timeout=30) - self.test_node.clear_block_announcement() + self.test_node.request_headers_and_sync(locator=[tip]) + check_announcement_of_new_block(self.nodes[0], self.test_node, lambda p: p.last_cmpctblock is None and p.last_inv is not None) - check_announcement_of_new_block(self.nodes[0], self.test_node, lambda: self.test_node.last_cmpctblock is None and self.test_node.last_inv is not None) + # Test a few ways of using sendcmpct that should NOT + # result in compact block announcements. + # Before each test, sync the headers chain. + self.test_node.request_headers_and_sync(locator=[tip]) # Now try a SENDCMPCT message with too-high version sendcmpct = msg_sendcmpct() sendcmpct.version = 2 - self.test_node.send_message(sendcmpct) + self.test_node.send_and_ping(sendcmpct) + check_announcement_of_new_block(self.nodes[0], self.test_node, lambda p: p.last_cmpctblock is None) - check_announcement_of_new_block(self.nodes[0], self.test_node, lambda: self.test_node.last_cmpctblock is None) + # Headers sync before next test. + self.test_node.request_headers_and_sync(locator=[tip]) # Now try a SENDCMPCT message with valid version, but announce=False - self.test_node.send_message(msg_sendcmpct()) - check_announcement_of_new_block(self.nodes[0], self.test_node, lambda: self.test_node.last_cmpctblock is None) + self.test_node.send_and_ping(msg_sendcmpct()) + check_announcement_of_new_block(self.nodes[0], self.test_node, lambda p: p.last_cmpctblock is None) + + # Headers sync before next test. + self.test_node.request_headers_and_sync(locator=[tip]) # Finally, try a SENDCMPCT message with announce=True sendcmpct.version = 1 sendcmpct.announce = True - self.test_node.send_message(sendcmpct) - check_announcement_of_new_block(self.nodes[0], self.test_node, lambda: self.test_node.last_cmpctblock is not None) + self.test_node.send_and_ping(sendcmpct) + check_announcement_of_new_block(self.nodes[0], self.test_node, lambda p: p.last_cmpctblock is not None) - # Try one more time - check_announcement_of_new_block(self.nodes[0], self.test_node, lambda: self.test_node.last_cmpctblock is not None) + # Try one more time (no headers sync should be needed!) + check_announcement_of_new_block(self.nodes[0], self.test_node, lambda p: p.last_cmpctblock is not None) # Try one more time, after turning on sendheaders - self.test_node.send_message(msg_sendheaders()) - check_announcement_of_new_block(self.nodes[0], self.test_node, lambda: self.test_node.last_cmpctblock is not None) + self.test_node.send_and_ping(msg_sendheaders()) + check_announcement_of_new_block(self.nodes[0], self.test_node, lambda p: p.last_cmpctblock is not None) # Now turn off announcements sendcmpct.announce = False - check_announcement_of_new_block(self.nodes[0], self.test_node, lambda: self.test_node.last_cmpctblock is None and self.test_node.last_headers is not None) + self.test_node.send_and_ping(sendcmpct) + check_announcement_of_new_block(self.nodes[0], self.test_node, lambda p: p.last_cmpctblock is None and p.last_headers is not None) # This test actually causes bitcoind to (reasonably!) disconnect us, so do this last. def test_invalid_cmpctblock_message(self): From 3d1804fcfef0c44fe6d86370411264636059c4eb Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Sat, 1 Oct 2016 11:19:16 +0200 Subject: [PATCH 07/51] Merge #8854: [qa] Fix race condition in p2p-compactblocks test b5fd666 [qa] Fix race condition in p2p-compactblocks test (Suhas Daftuar) --- qa/rpc-tests/p2p-compactblocks.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qa/rpc-tests/p2p-compactblocks.py b/qa/rpc-tests/p2p-compactblocks.py index bf4fb43add3d..ac4655a841ec 100755 --- a/qa/rpc-tests/p2p-compactblocks.py +++ b/qa/rpc-tests/p2p-compactblocks.py @@ -237,6 +237,8 @@ def test_compactblock_construction(self): for i in range(num_transactions): self.nodes[0].sendtoaddress(address, 0.1) + self.test_node.sync_with_ping() + # Now mine a block, and look at the resulting compact block. self.test_node.clear_block_announcement() block_hash = int(self.nodes[0].generate(1)[0], 16) From f4b6ddbdc852a7a3f2cdb8874e5e4e09a3da3498 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Mon, 10 Oct 2016 12:32:17 +0200 Subject: [PATCH 08/51] Merge #8393: Support for compact blocks together with segwit 27acfc1 [qa] Update p2p-compactblocks.py for compactblocks v2 (Suhas Daftuar) 422fac6 [qa] Add support for compactblocks v2 to mininode (Suhas Daftuar) f5b9b8f [qa] Fix bug in mininode witness deserialization (Suhas Daftuar) 6aa28ab Use cmpctblock type 2 for segwit-enabled transfer (Pieter Wuille) be7555f Fix overly-prescriptive p2p-segwit test for new fetch logic (Matt Corallo) 06128da Make GetFetchFlags always request witness objects from witness peers (Matt Corallo) --- qa/rpc-tests/p2p-compactblocks.py | 473 ++++++++++++++---------- qa/rpc-tests/test_framework/mininode.py | 16 +- src/net_processing.cpp | 6 + 3 files changed, 293 insertions(+), 202 deletions(-) diff --git a/qa/rpc-tests/p2p-compactblocks.py b/qa/rpc-tests/p2p-compactblocks.py index ac4655a841ec..691779634fe1 100755 --- a/qa/rpc-tests/p2p-compactblocks.py +++ b/qa/rpc-tests/p2p-compactblocks.py @@ -11,15 +11,14 @@ from test_framework.script import CScript, OP_TRUE ''' -CompactBlocksTest -- test compact blocks (BIP 152) +CompactBlocksTest -- test compact blocks (BIP 152, without segwit support, version 1) ''' - # TestNode: A peer we use to send messages to bitcoind, and store responses. class TestNode(SingleNodeConnCB): def __init__(self): SingleNodeConnCB.__init__(self) - self.last_sendcmpct = None + self.last_sendcmpct = [] self.last_headers = None self.last_inv = None self.last_cmpctblock = None @@ -30,7 +29,7 @@ def __init__(self): self.last_blocktxn = None def on_sendcmpct(self, conn, message): - self.last_sendcmpct = message + self.last_sendcmpct.append(message) def on_block(self, conn, message): self.last_block = message @@ -90,29 +89,31 @@ class CompactBlocksTest(BitcoinTestFramework): def __init__(self): super().__init__() self.setup_clean_chain = True - self.num_nodes = 1 + # both nodes has the same version + self.num_nodes = 2 self.utxos = [] def setup_network(self): self.nodes = [] - # Turn off segwit in this test, as compact blocks don't currently work - # with segwit. (After BIP 152 is updated to support segwit, we can - # test behavior with and without segwit enabled by adding a second node - # to the test.) - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, [["-debug", "-logtimemicros=1", "-bip9params=segwit:0:0"]]) + # Start up node0 to be a version 1, pre-segwit node. + self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, + [["-debug", "-logtimemicros=1", "-txindex"], + ["-debug", "-logtimemicros", "-txindex"]]) + connect_nodes(self.nodes[0], 1) - def build_block_on_tip(self): - height = self.nodes[0].getblockcount() - tip = self.nodes[0].getbestblockhash() - mtp = self.nodes[0].getblockheader(tip)['mediantime'] + def build_block_on_tip(self, node): + height = node.getblockcount() + tip = node.getbestblockhash() + mtp = node.getblockheader(tip)['mediantime'] block = create_block(int(tip, 16), create_coinbase(height + 1), mtp + 1) block.solve() return block # Create 10 more anyone-can-spend utxo's for testing. def make_utxos(self): - block = self.build_block_on_tip() + # Doesn't matter which node we use, just use node0. + block = self.build_block_on_tip(self.nodes[0]) self.test_node.send_and_ping(msg_block(block)) assert(int(self.nodes[0].getbestblockhash(), 16) == block.sha256) self.nodes[0].generate(100) @@ -125,7 +126,7 @@ def make_utxos(self): tx.vout.append(CTxOut(out_value, CScript([OP_TRUE]))) tx.rehash() - block2 = self.build_block_on_tip() + block2 = self.build_block_on_tip(self.nodes[0]) block2.vtx.append(tx) block2.hashMerkleRoot = block2.calc_merkle_root() block2.solve() @@ -134,26 +135,27 @@ def make_utxos(self): self.utxos.extend([[tx.sha256, i, out_value] for i in range(10)]) return - # Test "sendcmpct": - # - No compact block announcements or getdata(MSG_CMPCT_BLOCK) unless - # sendcmpct is sent. - # - If sendcmpct is sent with version > 1, the message is ignored. + # Test "sendcmpct" (between peers with the same version): + # - No compact block announcements unless sendcmpct is sent. # - If sendcmpct is sent with boolean 0, then block announcements are not # made with compact blocks. # - If sendcmpct is then sent with boolean 1, then new block announcements # are made with compact blocks. - def test_sendcmpct(self): - print("Testing SENDCMPCT p2p message... ") - - # Make sure we get a version 0 SENDCMPCT message from our peer + # If old_node is passed in, request compact blocks with version=preferred-1 + # and verify that it receives block announcements via compact block. + def test_sendcmpct(self, node, test_node, preferred_version, old_node=None): + # Make sure we get a SENDCMPCT message from our peer def received_sendcmpct(): - return (self.test_node.last_sendcmpct is not None) + return (len(test_node.last_sendcmpct) > 0) got_message = wait_until(received_sendcmpct, timeout=30) assert(received_sendcmpct()) assert(got_message) - assert_equal(self.test_node.last_sendcmpct.version, 1) + with mininode_lock: + # Check that the first version received is the preferred one + assert_equal(test_node.last_sendcmpct[0].version, preferred_version) + test_node.last_sendcmpct = [] - tip = int(self.nodes[0].getbestblockhash(), 16) + tip = int(node.getbestblockhash(), 16) def check_announcement_of_new_block(node, peer, predicate): peer.clear_block_announcement() @@ -165,56 +167,70 @@ def check_announcement_of_new_block(node, peer, predicate): assert(predicate(peer)) # We shouldn't get any block announcements via cmpctblock yet. - check_announcement_of_new_block(self.nodes[0], self.test_node, lambda p: p.last_cmpctblock is None) + check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is None) # Try one more time, this time after requesting headers. - self.test_node.request_headers_and_sync(locator=[tip]) - check_announcement_of_new_block(self.nodes[0], self.test_node, lambda p: p.last_cmpctblock is None and p.last_inv is not None) + test_node.request_headers_and_sync(locator=[tip]) + check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is None and p.last_inv is not None) # Test a few ways of using sendcmpct that should NOT # result in compact block announcements. # Before each test, sync the headers chain. - self.test_node.request_headers_and_sync(locator=[tip]) + test_node.request_headers_and_sync(locator=[tip]) # Now try a SENDCMPCT message with too-high version sendcmpct = msg_sendcmpct() - sendcmpct.version = 2 - self.test_node.send_and_ping(sendcmpct) - check_announcement_of_new_block(self.nodes[0], self.test_node, lambda p: p.last_cmpctblock is None) + sendcmpct.version = preferred_version+1 + sendcmpct.announce = True + test_node.send_and_ping(sendcmpct) + check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is None) # Headers sync before next test. - self.test_node.request_headers_and_sync(locator=[tip]) + test_node.request_headers_and_sync(locator=[tip]) # Now try a SENDCMPCT message with valid version, but announce=False - self.test_node.send_and_ping(msg_sendcmpct()) - check_announcement_of_new_block(self.nodes[0], self.test_node, lambda p: p.last_cmpctblock is None) + sendcmpct.version = preferred_version + sendcmpct.announce = False + test_node.send_and_ping(sendcmpct) + check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is None) # Headers sync before next test. - self.test_node.request_headers_and_sync(locator=[tip]) + test_node.request_headers_and_sync(locator=[tip]) # Finally, try a SENDCMPCT message with announce=True - sendcmpct.version = 1 + sendcmpct.version = preferred_version sendcmpct.announce = True - self.test_node.send_and_ping(sendcmpct) - check_announcement_of_new_block(self.nodes[0], self.test_node, lambda p: p.last_cmpctblock is not None) + test_node.send_and_ping(sendcmpct) + check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is not None) # Try one more time (no headers sync should be needed!) - check_announcement_of_new_block(self.nodes[0], self.test_node, lambda p: p.last_cmpctblock is not None) + check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is not None) # Try one more time, after turning on sendheaders - self.test_node.send_and_ping(msg_sendheaders()) - check_announcement_of_new_block(self.nodes[0], self.test_node, lambda p: p.last_cmpctblock is not None) + test_node.send_and_ping(msg_sendheaders()) + check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is not None) # Now turn off announcements + sendcmpct.version = preferred_version sendcmpct.announce = False - self.test_node.send_and_ping(sendcmpct) - check_announcement_of_new_block(self.nodes[0], self.test_node, lambda p: p.last_cmpctblock is None and p.last_headers is not None) + test_node.send_and_ping(sendcmpct) + check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is None and p.last_headers is not None) + + # This code should be enabled after increasing cmctblk version + #if old_node is not None: + # Verify that a peer using an older protocol version can receive + # announcements from this node. + #sendcmpct.version = preferred_version-1 + #sendcmpct.announce = True + #old_node.send_and_ping(sendcmpct) + # Header sync + #old_node.request_headers_and_sync(locator=[tip]) + #check_announcement_of_new_block(node, old_node, lambda p: p.last_cmpctblock is not None) # This test actually causes bitcoind to (reasonably!) disconnect us, so do this last. def test_invalid_cmpctblock_message(self): - print("Testing invalid index in cmpctblock message...") self.nodes[0].generate(101) - block = self.build_block_on_tip() + block = self.build_block_on_tip(self.nodes[0]) cmpct_block = P2PHeaderAndShortIDs() cmpct_block.header = CBlockHeader(block) @@ -227,45 +243,48 @@ def test_invalid_cmpctblock_message(self): # Compare the generated shortids to what we expect based on BIP 152, given # bitcoind's choice of nonce. - def test_compactblock_construction(self): - print("Testing compactblock headers and shortIDs are correct...") - + def test_compactblock_construction(self, node, test_node, version): # Generate a bunch of transactions. - self.nodes[0].generate(101) + node.generate(101) num_transactions = 25 - address = self.nodes[0].getnewaddress() + address = node.getnewaddress() + for i in range(num_transactions): - self.nodes[0].sendtoaddress(address, 0.1) + txid = node.sendtoaddress(address, 0.1) + hex_tx = node.gettransaction(txid)["hex"] + tx = FromHex(CTransaction(), hex_tx) self.test_node.sync_with_ping() # Now mine a block, and look at the resulting compact block. - self.test_node.clear_block_announcement() - block_hash = int(self.nodes[0].generate(1)[0], 16) + test_node.clear_block_announcement() + block_hash = int(node.generate(1)[0], 16) # Store the raw block in our internal format. - block = FromHex(CBlock(), self.nodes[0].getblock("%02x" % block_hash, False)) + block = FromHex(CBlock(), node.getblock("%02x" % block_hash, False)) [tx.calc_sha256() for tx in block.vtx] block.rehash() # Don't care which type of announcement came back for this test; just # request the compact block if we didn't get one yet. - wait_until(self.test_node.received_block_announcement, timeout=30) + wait_until(test_node.received_block_announcement, timeout=30) + assert(test_node.received_block_announcement()) with mininode_lock: - if self.test_node.last_cmpctblock is None: - self.test_node.clear_block_announcement() - inv = CInv(4, block_hash) # 4 == "CompactBlock" - self.test_node.send_message(msg_getdata([inv])) + if test_node.last_cmpctblock is None: + test_node.clear_block_announcement() + inv = CInv(20, block_hash) # 20 == "CompactBlock" + test_node.send_message(msg_getdata([inv])) - wait_until(self.test_node.received_block_announcement, timeout=30) + wait_until(test_node.received_block_announcement, timeout=30) + assert(test_node.received_block_announcement()) # Now we should have the compactblock header_and_shortids = None with mininode_lock: - assert(self.test_node.last_cmpctblock is not None) + assert(test_node.last_cmpctblock is not None) # Convert the on-the-wire representation to absolute indexes - header_and_shortids = HeaderAndShortIDs(self.test_node.last_cmpctblock.header_and_shortids) + header_and_shortids = HeaderAndShortIDs(test_node.last_cmpctblock.header_and_shortids) # Check that we got the right block! header_and_shortids.header.calc_sha256() @@ -278,6 +297,7 @@ def test_compactblock_construction(self): # Check that all prefilled_txn entries match what's in the block. for entry in header_and_shortids.prefilled_txn: entry.tx.calc_sha256() + # This checks the tx agree assert_equal(entry.tx.sha256, block.vtx[entry.index].sha256) # Check that the cmpctblock message announced all the transactions. @@ -294,7 +314,8 @@ def test_compactblock_construction(self): # Already checked prefilled transactions above header_and_shortids.prefilled_txn.pop(0) else: - shortid = calculate_shortid(k0, k1, block.vtx[index].sha256) + tx_hash = block.vtx[index].sha256 + shortid = calculate_shortid(k0, k1, tx_hash) assert_equal(shortid, header_and_shortids.shortids[0]) header_and_shortids.shortids.pop(0) index += 1 @@ -302,49 +323,47 @@ def test_compactblock_construction(self): # Test that bitcoind requests compact blocks when we announce new blocks # via header or inv, and that responding to getblocktxn causes the block # to be successfully reconstructed. - def test_compactblock_requests(self): - print("Testing compactblock requests... ") - + def test_compactblock_requests(self, node, test_node): # Try announcing a block with an inv or header, expect a compactblock # request for announce in ["inv", "header"]: - block = self.build_block_on_tip() + block = self.build_block_on_tip(node) with mininode_lock: - self.test_node.last_getdata = None + test_node.last_getdata = None if announce == "inv": - self.test_node.send_message(msg_inv([CInv(2, block.sha256)])) + test_node.send_message(msg_inv([CInv(2, block.sha256)])) else: - self.test_node.send_header_for_blocks([block]) - success = wait_until(lambda: self.test_node.last_getdata is not None, timeout=30) + test_node.send_header_for_blocks([block]) + success = wait_until(lambda: test_node.last_getdata is not None, timeout=30) assert(success) - assert_equal(len(self.test_node.last_getdata.inv), 1) - assert_equal(self.test_node.last_getdata.inv[0].type, 4) - assert_equal(self.test_node.last_getdata.inv[0].hash, block.sha256) + assert_equal(len(test_node.last_getdata.inv), 1) + assert_equal(test_node.last_getdata.inv[0].type, 20) + assert_equal(test_node.last_getdata.inv[0].hash, block.sha256) # Send back a compactblock message that omits the coinbase comp_block = HeaderAndShortIDs() comp_block.header = CBlockHeader(block) comp_block.nonce = 0 comp_block.shortids = [1] # this is useless, and wrong - self.test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) - assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.hashPrevBlock) + test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) + assert_equal(int(node.getbestblockhash(), 16), block.hashPrevBlock) # Expect a getblocktxn message. with mininode_lock: - assert(self.test_node.last_getblocktxn is not None) - absolute_indexes = self.test_node.last_getblocktxn.block_txn_request.to_absolute() + assert(test_node.last_getblocktxn is not None) + absolute_indexes = test_node.last_getblocktxn.block_txn_request.to_absolute() assert_equal(absolute_indexes, [0]) # should be a coinbase request # Send the coinbase, and verify that the tip advances. msg = msg_blocktxn() msg.block_transactions.blockhash = block.sha256 msg.block_transactions.transactions = [block.vtx[0]] - self.test_node.send_and_ping(msg) - assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256) + test_node.send_and_ping(msg) + assert_equal(int(node.getbestblockhash(), 16), block.sha256) # Create a chain of transactions from given utxo, and add to a new block. - def build_block_with_transactions(self, utxo, num_transactions): - block = self.build_block_on_tip() + def build_block_with_transactions(self, node, utxo, num_transactions): + block = self.build_block_on_tip(node) for i in range(num_transactions): tx = CTransaction() @@ -361,118 +380,110 @@ def build_block_with_transactions(self, utxo, num_transactions): # Test that we only receive getblocktxn requests for transactions that the # node needs, and that responding to them causes the block to be # reconstructed. - def test_getblocktxn_requests(self): - print("Testing getblocktxn requests...") + def test_getblocktxn_requests(self, node, test_node, version): + + def test_getblocktxn_response(compact_block, peer, expected_result): + msg = msg_cmpctblock(compact_block.to_p2p()) + peer.send_and_ping(msg) + with mininode_lock: + assert(peer.last_getblocktxn is not None) + absolute_indexes = peer.last_getblocktxn.block_txn_request.to_absolute() + assert_equal(absolute_indexes, expected_result) + + def test_tip_after_message(node, peer, msg, tip): + peer.send_and_ping(msg) + assert_equal(int(node.getbestblockhash(), 16), tip) # First try announcing compactblocks that won't reconstruct, and verify # that we receive getblocktxn messages back. utxo = self.utxos.pop(0) - block = self.build_block_with_transactions(utxo, 5) + block = self.build_block_with_transactions(node, utxo, 5) self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) - comp_block = HeaderAndShortIDs() comp_block.initialize_from_block(block) - self.test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) - with mininode_lock: - assert(self.test_node.last_getblocktxn is not None) - absolute_indexes = self.test_node.last_getblocktxn.block_txn_request.to_absolute() - assert_equal(absolute_indexes, [1, 2, 3, 4, 5]) - msg = msg_blocktxn() - msg.block_transactions = BlockTransactions(block.sha256, block.vtx[1:]) - self.test_node.send_and_ping(msg) - assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256) + test_getblocktxn_response(comp_block, test_node, [1, 2, 3, 4, 5]) + + msg_bt = msg_blocktxn() + msg_bt.block_transactions = BlockTransactions(block.sha256, block.vtx[1:]) + test_tip_after_message(node, test_node, msg_bt, block.sha256) utxo = self.utxos.pop(0) - block = self.build_block_with_transactions(utxo, 5) + block = self.build_block_with_transactions(node, utxo, 5) self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) # Now try interspersing the prefilled transactions comp_block.initialize_from_block(block, prefill_list=[0, 1, 5]) - self.test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) - with mininode_lock: - assert(self.test_node.last_getblocktxn is not None) - absolute_indexes = self.test_node.last_getblocktxn.block_txn_request.to_absolute() - assert_equal(absolute_indexes, [2, 3, 4]) - msg.block_transactions = BlockTransactions(block.sha256, block.vtx[2:5]) - self.test_node.send_and_ping(msg) - assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256) + test_getblocktxn_response(comp_block, test_node, [2, 3, 4]) + msg_bt.block_transactions = BlockTransactions(block.sha256, block.vtx[2:5]) + test_tip_after_message(node, test_node, msg_bt, block.sha256) # Now try giving one transaction ahead of time. utxo = self.utxos.pop(0) - block = self.build_block_with_transactions(utxo, 5) + block = self.build_block_with_transactions(node, utxo, 5) self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) - self.test_node.send_and_ping(msg_tx(block.vtx[1])) - assert(block.vtx[1].hash in self.nodes[0].getrawmempool()) + test_node.send_and_ping(msg_tx(block.vtx[1])) + assert(block.vtx[1].hash in node.getrawmempool()) # Prefill 4 out of the 6 transactions, and verify that only the one # that was not in the mempool is requested. comp_block.initialize_from_block(block, prefill_list=[0, 2, 3, 4]) - self.test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) - with mininode_lock: - assert(self.test_node.last_getblocktxn is not None) - absolute_indexes = self.test_node.last_getblocktxn.block_txn_request.to_absolute() - assert_equal(absolute_indexes, [5]) + test_getblocktxn_response(comp_block, test_node, [5]) - msg.block_transactions = BlockTransactions(block.sha256, [block.vtx[5]]) - self.test_node.send_and_ping(msg) - assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256) + msg_bt.block_transactions = BlockTransactions(block.sha256, [block.vtx[5]]) + test_tip_after_message(node, test_node, msg_bt, block.sha256) # Now provide all transactions to the node before the block is # announced and verify reconstruction happens immediately. utxo = self.utxos.pop(0) - block = self.build_block_with_transactions(utxo, 10) + block = self.build_block_with_transactions(node, utxo, 10) self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) for tx in block.vtx[1:]: - self.test_node.send_message(msg_tx(tx)) - self.test_node.sync_with_ping() + test_node.send_message(msg_tx(tx)) + test_node.sync_with_ping() # Make sure all transactions were accepted. - mempool = self.nodes[0].getrawmempool() + mempool = node.getrawmempool() for tx in block.vtx[1:]: assert(tx.hash in mempool) # Clear out last request. with mininode_lock: - self.test_node.last_getblocktxn = None + test_node.last_getblocktxn = None # Send compact block comp_block.initialize_from_block(block, prefill_list=[0]) - self.test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) + test_tip_after_message(node, test_node, msg_cmpctblock(comp_block.to_p2p()), block.sha256) with mininode_lock: # Shouldn't have gotten a request for any transaction - assert(self.test_node.last_getblocktxn is None) - # Tip should have updated - assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256) + assert(test_node.last_getblocktxn is None) # Incorrectly responding to a getblocktxn shouldn't cause the block to be # permanently failed. - def test_incorrect_blocktxn_response(self): - print("Testing handling of incorrect blocktxn responses...") - + def test_incorrect_blocktxn_response(self, node, test_node, version): if (len(self.utxos) == 0): self.make_utxos() utxo = self.utxos.pop(0) - block = self.build_block_with_transactions(utxo, 10) + block = self.build_block_with_transactions(node, utxo, 10) self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) # Relay the first 5 transactions from the block in advance for tx in block.vtx[1:6]: - self.test_node.send_message(msg_tx(tx)) - self.test_node.sync_with_ping() + test_node.send_message(msg_tx(tx)) + test_node.sync_with_ping() # Make sure all transactions were accepted. - mempool = self.nodes[0].getrawmempool() + mempool = node.getrawmempool() for tx in block.vtx[1:6]: assert(tx.hash in mempool) # Send compact block comp_block = HeaderAndShortIDs() comp_block.initialize_from_block(block, prefill_list=[0]) - self.test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) + test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) absolute_indexes = [] with mininode_lock: - assert(self.test_node.last_getblocktxn is not None) - absolute_indexes = self.test_node.last_getblocktxn.block_txn_request.to_absolute() + assert(test_node.last_getblocktxn is not None) + absolute_indexes = test_node.last_getblocktxn.block_txn_request.to_absolute() assert_equal(absolute_indexes, [6, 7, 8, 9, 10]) # Now give an incorrect response. @@ -485,99 +496,95 @@ def test_incorrect_blocktxn_response(self): # enough for now. msg = msg_blocktxn() msg.block_transactions = BlockTransactions(block.sha256, [block.vtx[5]] + block.vtx[7:]) - self.test_node.send_and_ping(msg) + test_node.send_and_ping(msg) # Tip should not have updated - assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.hashPrevBlock) + assert_equal(int(node.getbestblockhash(), 16), block.hashPrevBlock) # We should receive a getdata request - success = wait_until(lambda: self.test_node.last_getdata is not None, timeout=10) + success = wait_until(lambda: test_node.last_getdata is not None, timeout=10) assert(success) - assert_equal(len(self.test_node.last_getdata.inv), 1) - assert_equal(self.test_node.last_getdata.inv[0].type, 2) - assert_equal(self.test_node.last_getdata.inv[0].hash, block.sha256) + assert_equal(len(test_node.last_getdata.inv), 1) + assert(test_node.last_getdata.inv[0].type == 2) + assert_equal(test_node.last_getdata.inv[0].hash, block.sha256) # Deliver the block - self.test_node.send_and_ping(msg_block(block)) - assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256) - - def test_getblocktxn_handler(self): - print("Testing getblocktxn handler...") + test_node.send_and_ping(msg_block(block)) + assert_equal(int(node.getbestblockhash(), 16), block.sha256) + def test_getblocktxn_handler(self, node, test_node, version): # bitcoind won't respond for blocks whose height is more than 15 blocks # deep. MAX_GETBLOCKTXN_DEPTH = 15 - chain_height = self.nodes[0].getblockcount() + chain_height = node.getblockcount() current_height = chain_height while (current_height >= chain_height - MAX_GETBLOCKTXN_DEPTH): - block_hash = self.nodes[0].getblockhash(current_height) - block = FromHex(CBlock(), self.nodes[0].getblock(block_hash, False)) + block_hash = node.getblockhash(current_height) + block = FromHex(CBlock(), node.getblock(block_hash, False)) msg = msg_getblocktxn() msg.block_txn_request = BlockTransactionsRequest(int(block_hash, 16), []) num_to_request = random.randint(1, len(block.vtx)) msg.block_txn_request.from_absolute(sorted(random.sample(range(len(block.vtx)), num_to_request))) - self.test_node.send_message(msg) - success = wait_until(lambda: self.test_node.last_blocktxn is not None, timeout=10) + test_node.send_message(msg) + success = wait_until(lambda: test_node.last_blocktxn is not None, timeout=10) assert(success) [tx.calc_sha256() for tx in block.vtx] with mininode_lock: - assert_equal(self.test_node.last_blocktxn.block_transactions.blockhash, int(block_hash, 16)) + assert_equal(test_node.last_blocktxn.block_transactions.blockhash, int(block_hash, 16)) all_indices = msg.block_txn_request.to_absolute() for index in all_indices: - tx = self.test_node.last_blocktxn.block_transactions.transactions.pop(0) + tx = test_node.last_blocktxn.block_transactions.transactions.pop(0) tx.calc_sha256() assert_equal(tx.sha256, block.vtx[index].sha256) - self.test_node.last_blocktxn = None + test_node.last_blocktxn = None current_height -= 1 # Next request should be ignored, as we're past the allowed depth. - block_hash = self.nodes[0].getblockhash(current_height) + block_hash = node.getblockhash(current_height) msg.block_txn_request = BlockTransactionsRequest(int(block_hash, 16), [0]) - self.test_node.send_and_ping(msg) + test_node.send_and_ping(msg) with mininode_lock: - assert_equal(self.test_node.last_blocktxn, None) - - def test_compactblocks_not_at_tip(self): - print("Testing compactblock requests/announcements not at chain tip...") + assert_equal(test_node.last_blocktxn, None) + def test_compactblocks_not_at_tip(self, node, test_node): # Test that requesting old compactblocks doesn't work. MAX_CMPCTBLOCK_DEPTH = 11 new_blocks = [] for i in range(MAX_CMPCTBLOCK_DEPTH): - self.test_node.clear_block_announcement() - new_blocks.append(self.nodes[0].generate(1)[0]) - wait_until(self.test_node.received_block_announcement, timeout=30) + test_node.clear_block_announcement() + new_blocks.append(node.generate(1)[0]) + wait_until(test_node.received_block_announcement, timeout=30) - self.test_node.clear_block_announcement() - self.test_node.send_message(msg_getdata([CInv(4, int(new_blocks[0], 16))])) - success = wait_until(lambda: self.test_node.last_cmpctblock is not None, timeout=30) + test_node.clear_block_announcement() + test_node.send_message(msg_getdata([CInv(20, int(new_blocks[0], 16))])) + success = wait_until(lambda: test_node.last_cmpctblock is not None, timeout=30) assert(success) - self.test_node.clear_block_announcement() - self.nodes[0].generate(1) - wait_until(self.test_node.received_block_announcement, timeout=30) - self.test_node.clear_block_announcement() - self.test_node.send_message(msg_getdata([CInv(4, int(new_blocks[0], 16))])) - success = wait_until(lambda: self.test_node.last_block is not None, timeout=30) + test_node.clear_block_announcement() + node.generate(1) + wait_until(test_node.received_block_announcement, timeout=30) + test_node.clear_block_announcement() + test_node.send_message(msg_getdata([CInv(20, int(new_blocks[0], 16))])) + success = wait_until(lambda: test_node.last_block is not None, timeout=30) assert(success) with mininode_lock: - self.test_node.last_block.block.calc_sha256() - assert_equal(self.test_node.last_block.block.sha256, int(new_blocks[0], 16)) + test_node.last_block.block.calc_sha256() + assert_equal(test_node.last_block.block.sha256, int(new_blocks[0], 16)) # Generate an old compactblock, and verify that it's not accepted. - cur_height = self.nodes[0].getblockcount() - hashPrevBlock = int(self.nodes[0].getblockhash(cur_height-5), 16) - block = self.build_block_on_tip() + cur_height = node.getblockcount() + hashPrevBlock = int(node.getblockhash(cur_height-5), 16) + block = self.build_block_on_tip(node) block.hashPrevBlock = hashPrevBlock block.solve() comp_block = HeaderAndShortIDs() comp_block.initialize_from_block(block) - self.test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) + test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) - tips = self.nodes[0].getchaintips() + tips = node.getchaintips() found = False for x in tips: if x["hash"] == block.hash: @@ -591,18 +598,54 @@ def test_compactblocks_not_at_tip(self): msg = msg_getblocktxn() msg.block_txn_request = BlockTransactionsRequest(block.sha256, [0]) with mininode_lock: - self.test_node.last_blocktxn = None - self.test_node.send_and_ping(msg) + test_node.last_blocktxn = None + test_node.send_and_ping(msg) + with mininode_lock: + assert(test_node.last_blocktxn is None) + + def test_end_to_end_block_relay(self, node, listeners): + utxo = self.utxos.pop(0) + + block = self.build_block_with_transactions(node, utxo, 10) + + [l.clear_block_announcement() for l in listeners] + + node.submitblock(ToHex(block)) + + for l in listeners: + wait_until(lambda: l.received_block_announcement(), timeout=30) with mininode_lock: - assert(self.test_node.last_blocktxn is None) + for l in listeners: + assert(l.last_cmpctblock is not None) + l.last_cmpctblock.header_and_shortids.header.calc_sha256() + assert_equal(l.last_cmpctblock.header_and_shortids.header.sha256, block.sha256) + + # Helper for enabling cb announcements + # Send the sendcmpct request and sync headers + def request_cb_announcements(self, peer, node, version): + tip = node.getbestblockhash() + peer.get_headers(locator=[int(tip, 16)], hashstop=0) + + msg = msg_sendcmpct() + msg.version = version + msg.announce = True + peer.send_and_ping(msg) def run_test(self): # Setup the p2p connections and start up the network thread. self.test_node = TestNode() + self.second_node = TestNode() + self.old_node = TestNode() connections = [] connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], self.test_node)) + connections.append(NodeConn('127.0.0.1', p2p_port(1), self.nodes[1], + self.second_node, services=NODE_NETWORK)) + connections.append(NodeConn('127.0.0.1', p2p_port(1), self.nodes[1], + self.old_node, services=NODE_NETWORK)) self.test_node.add_connection(connections[0]) + self.second_node.add_connection(connections[1]) + self.old_node.add_connection(connections[2]) NetworkThread().start() # Start up network handling in another thread @@ -612,13 +655,61 @@ def run_test(self): # We will need UTXOs to construct transactions in later tests. self.make_utxos() - self.test_sendcmpct() - self.test_compactblock_construction() - self.test_compactblock_requests() - self.test_getblocktxn_requests() - self.test_getblocktxn_handler() - self.test_compactblocks_not_at_tip() - self.test_incorrect_blocktxn_response() + print("Running tests, activation:") + + print("\tTesting SENDCMPCT p2p message... ") + self.test_sendcmpct(self.nodes[0], self.test_node, 1) + sync_blocks(self.nodes) + self.test_sendcmpct(self.nodes[1], self.second_node, 1) + sync_blocks(self.nodes) + + print("\tTesting compactblock construction...") + self.test_compactblock_construction(self.nodes[0], self.test_node, 1) + sync_blocks(self.nodes) + self.test_compactblock_construction(self.nodes[1], self.second_node, 1) + sync_blocks(self.nodes) + + print("\tTesting compactblock requests... ") + self.test_compactblock_requests(self.nodes[0], self.test_node) + sync_blocks(self.nodes) + self.test_compactblock_requests(self.nodes[1], self.second_node) + sync_blocks(self.nodes) + + print("\tTesting getblocktxn requests...") + self.test_getblocktxn_requests(self.nodes[0], self.test_node, 1) + sync_blocks(self.nodes) + self.test_getblocktxn_requests(self.nodes[1], self.second_node, 1) + sync_blocks(self.nodes) + + print("\tTesting getblocktxn handler...") + self.test_getblocktxn_handler(self.nodes[0], self.test_node, 1) + sync_blocks(self.nodes) + self.test_getblocktxn_handler(self.nodes[1], self.second_node, 1) + self.test_getblocktxn_handler(self.nodes[1], self.old_node, 1) + sync_blocks(self.nodes) + + print("\tTesting compactblock requests/announcements not at chain tip...") + self.test_compactblocks_not_at_tip(self.nodes[0], self.test_node) + sync_blocks(self.nodes) + self.test_compactblocks_not_at_tip(self.nodes[1], self.second_node) + self.test_compactblocks_not_at_tip(self.nodes[1], self.old_node) + sync_blocks(self.nodes) + + print("\tTesting handling of incorrect blocktxn responses...") + self.test_incorrect_blocktxn_response(self.nodes[0], self.test_node, 1) + sync_blocks(self.nodes) + self.test_incorrect_blocktxn_response(self.nodes[1], self.second_node, 1) + sync_blocks(self.nodes) + + # End-to-end block relay tests + print("\tTesting end-to-end block relay...") + self.request_cb_announcements(self.test_node, self.nodes[0], 1) + self.request_cb_announcements(self.old_node, self.nodes[1], 1) + self.request_cb_announcements(self.second_node, self.nodes[1], 1) + self.test_end_to_end_block_relay(self.nodes[0], [self.second_node, self.test_node, self.old_node]) + self.test_end_to_end_block_relay(self.nodes[1], [self.second_node, self.test_node, self.old_node]) + + print("\tTesting invalid index in cmpctblock message...") self.test_invalid_cmpctblock_message() diff --git a/qa/rpc-tests/test_framework/mininode.py b/qa/rpc-tests/test_framework/mininode.py index 0affe70b08bc..d0e59cf1f6eb 100755 --- a/qa/rpc-tests/test_framework/mininode.py +++ b/qa/rpc-tests/test_framework/mininode.py @@ -617,13 +617,10 @@ def deserialize(self, f): self.tx = CTransaction() self.tx.deserialize(f) - def serialize(self, with_witness=False): + def serialize(self): r = b"" r += ser_compact_size(self.index) - if with_witness: - r += self.tx.serialize_with_witness() - else: - r += self.tx.serialize_without_witness() + r += self.tx.serialize() return r def __repr__(self): @@ -650,7 +647,7 @@ def deserialize(self, f): self.prefilled_txn = deser_vector(f, PrefilledTransaction) self.prefilled_txn_length = len(self.prefilled_txn) - def serialize(self, with_witness=False): + def serialize(self): r = b"" r += self.header.serialize() r += struct.pack(" Date: Tue, 11 Oct 2016 10:49:52 +0200 Subject: [PATCH 09/51] Merge #8882: [qa] Fix race conditions in p2p-compactblocks.py and sendheaders.py b55d941 [qa] Fix race condition in sendheaders.py (Suhas Daftuar) 6976db2 [qa] Another attempt to fix race condition in p2p-compactblocks.py (Suhas Daftuar) --- qa/rpc-tests/p2p-compactblocks.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/qa/rpc-tests/p2p-compactblocks.py b/qa/rpc-tests/p2p-compactblocks.py index 691779634fe1..2fa82fbbcb6e 100755 --- a/qa/rpc-tests/p2p-compactblocks.py +++ b/qa/rpc-tests/p2p-compactblocks.py @@ -27,6 +27,10 @@ def __init__(self): self.last_getblocktxn = None self.last_block = None self.last_blocktxn = None + # Store the hashes of blocks we've seen announced. + # This is for synchronizing the p2p message traffic, + # so we can eg wait until a particular block is announced. + self.set_announced_blockhashes = set() def on_sendcmpct(self, conn, message): self.last_sendcmpct.append(message) @@ -37,14 +41,22 @@ def on_block(self, conn, message): def on_cmpctblock(self, conn, message): self.last_cmpctblock = message self.block_announced = True + self.last_cmpctblock.header_and_shortids.header.calc_sha256() + self.set_announced_blockhashes.add(self.last_cmpctblock.header_and_shortids.header.sha256) def on_headers(self, conn, message): self.last_headers = message self.block_announced = True + for x in self.last_headers.headers: + x.calc_sha256() + self.set_announced_blockhashes.add(x.sha256) def on_inv(self, conn, message): self.last_inv = message - self.block_announced = True + for x in self.last_inv.inv: + if x.type == 2: + self.block_announced = True + self.set_announced_blockhashes.add(x.hash) def on_getdata(self, conn, message): self.last_getdata = message @@ -84,6 +96,12 @@ def request_headers_and_sync(self, locator, hashstop=0): assert(self.received_block_announcement()) self.clear_block_announcement() + # Block until a block announcement for a particular block hash is + # received. + def wait_for_block_announcement(self, block_hash, timeout=30): + def received_hash(): + return (block_hash in self.set_announced_blockhashes) + return wait_until(received_hash, timeout=timeout) class CompactBlocksTest(BitcoinTestFramework): def __init__(self): @@ -254,7 +272,9 @@ def test_compactblock_construction(self, node, test_node, version): hex_tx = node.gettransaction(txid)["hex"] tx = FromHex(CTransaction(), hex_tx) - self.test_node.sync_with_ping() + # Wait until we've seen the block announcement for the resulting tip + tip = int(self.nodes[0].getbestblockhash(), 16) + assert(self.test_node.wait_for_block_announcement(tip)) # Now mine a block, and look at the resulting compact block. test_node.clear_block_announcement() From c31fde05cc8859d550f468c1cec028cd2c1b8fbe Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Thu, 13 Oct 2016 18:47:45 +0200 Subject: [PATCH 10/51] Merge #8904: [qa] Fix compact block shortids for a test case 4cdece4 [qa] Fix compact block shortids for a test case (Dagur Valberg Johannsson) --- qa/rpc-tests/p2p-compactblocks.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/qa/rpc-tests/p2p-compactblocks.py b/qa/rpc-tests/p2p-compactblocks.py index 2fa82fbbcb6e..edd9825d6c2e 100755 --- a/qa/rpc-tests/p2p-compactblocks.py +++ b/qa/rpc-tests/p2p-compactblocks.py @@ -365,7 +365,9 @@ def test_compactblock_requests(self, node, test_node): comp_block = HeaderAndShortIDs() comp_block.header = CBlockHeader(block) comp_block.nonce = 0 - comp_block.shortids = [1] # this is useless, and wrong + [k0, k1] = comp_block.get_siphash_keys() + comp_block.shortids = [ + calculate_shortid(k0, k1, block.vtx[0].sha256) ] test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) assert_equal(int(node.getbestblockhash(), 16), block.hashPrevBlock) # Expect a getblocktxn message. From cd2c4c6a7ed8d2a99970f464645aeca16ef12130 Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Tue, 18 Oct 2016 21:18:10 +0200 Subject: [PATCH 11/51] Merge #8637: Compact Block Tweaks (rebase of #8235) 3ac6de0 Align constant names for maximum compact block / blocktxn depth (Pieter Wuille) b2e93a3 Add cmpctblock to debug help list (instagibbs) fe998e9 More agressively filter compact block requests (Matt Corallo) 02a337d Dont remove a "preferred" cmpctblock peer if they provide a block (Matt Corallo) --- qa/rpc-tests/p2p-compactblocks.py | 6 +++--- src/init.cpp | 2 +- src/net_processing.cpp | 20 +++++++++++++++----- src/validation.h | 5 +++++ 4 files changed, 24 insertions(+), 9 deletions(-) diff --git a/qa/rpc-tests/p2p-compactblocks.py b/qa/rpc-tests/p2p-compactblocks.py index edd9825d6c2e..9e839e239b43 100755 --- a/qa/rpc-tests/p2p-compactblocks.py +++ b/qa/rpc-tests/p2p-compactblocks.py @@ -537,7 +537,7 @@ def test_incorrect_blocktxn_response(self, node, test_node, version): def test_getblocktxn_handler(self, node, test_node, version): # bitcoind won't respond for blocks whose height is more than 15 blocks # deep. - MAX_GETBLOCKTXN_DEPTH = 15 + MAX_GETBLOCKTXN_DEPTH = 10 chain_height = node.getblockcount() current_height = chain_height while (current_height >= chain_height - MAX_GETBLOCKTXN_DEPTH): @@ -572,9 +572,9 @@ def test_getblocktxn_handler(self, node, test_node, version): def test_compactblocks_not_at_tip(self, node, test_node): # Test that requesting old compactblocks doesn't work. - MAX_CMPCTBLOCK_DEPTH = 11 + MAX_CMPCTBLOCK_DEPTH = 5 new_blocks = [] - for i in range(MAX_CMPCTBLOCK_DEPTH): + for i in range(MAX_CMPCTBLOCK_DEPTH + 1): test_node.clear_block_announcement() new_blocks.append(node.generate(1)[0]) wait_until(test_node.received_block_announcement, timeout=30) diff --git a/src/init.cpp b/src/init.cpp index 6d99d0429835..900dbe866644 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -509,7 +509,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-limitdescendantsize=", strprintf("Do not accept transactions if any ancestor would have more than kilobytes of in-mempool descendants (default: %u).", DEFAULT_DESCENDANT_SIZE_LIMIT)); strUsage += HelpMessageOpt("-bip9params=deployment:start:end", "Use given start/end times for specified BIP9 deployment (regtest-only)"); } - std::string debugCategories = "addrman, alert, bench, coindb, db, http, leveldb, libevent, lock, mempool, mempoolrej, net, proxy, prune, rand, reindex, rpc, selectcoins, tor, zmq, " + std::string debugCategories = "addrman, alert, bench, cmpctblock, coindb, db, http, leveldb, libevent, lock, mempool, mempoolrej, net, proxy, prune, rand, reindex, rpc, selectcoins, tor, zmq, " "dash (or specifically: gobject, instantsend, keepass, masternode, mnpayments, mnsync, privatesend, spork)"; // Don't translate these and qt below if (mode == HMM_BITCOIN_QT) debugCategories += ", qt"; diff --git a/src/net_processing.cpp b/src/net_processing.cpp index afd3b3fd0eda..e37c1af01646 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -399,9 +399,13 @@ void UpdateBlockAvailability(NodeId nodeid, const uint256 &hash) { void MaybeSetPeerAsAnnouncingHeaderAndIDs(const CNodeState* nodestate, CNode* pfrom, CConnman& connman, CNetMsgMaker& msgMaker) { if (nodestate->fProvidesHeaderAndIDs) { - BOOST_FOREACH(const NodeId nodeid, lNodesAnnouncingHeaderAndIDs) - if (nodeid == pfrom->GetId()) + for (std::list::iterator it = lNodesAnnouncingHeaderAndIDs.begin(); it != lNodesAnnouncingHeaderAndIDs.end(); it++) { + if (*it == pfrom->GetId()) { + lNodesAnnouncingHeaderAndIDs.erase(it); + lNodesAnnouncingHeaderAndIDs.push_back(pfrom->GetId()); return; + } + } bool fAnnounceUsingCMPCTBLOCK = false; uint64_t nCMPCTBLOCKVersion = 1; if (lNodesAnnouncingHeaderAndIDs.size() >= 3) { @@ -1004,7 +1008,7 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam // they wont have a useful mempool to match against a compact block, // and we dont feel like constructing the object for them, so // instead we respond with the full, non-compact block. - if (mi->second->nHeight >= chainActive.Height() - 10) { + if (CanDirectFetch(consensusParams) && mi->second->nHeight >= chainActive.Height() - MAX_CMPCTBLOCK_DEPTH) { CBlockHeaderAndShortTxIDs cmpctblock(block); connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::CMPCTBLOCK, cmpctblock)); } else @@ -1686,8 +1690,8 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr return true; } - if (it->second->nHeight < chainActive.Height() - 15) { - LogPrint("net", "Peer %d sent us a getblocktxn for a block > 15 deep", pfrom->id); + if (it->second->nHeight < chainActive.Height() - MAX_BLOCKTXN_DEPTH) { + LogPrint("net", "Peer %d sent us a getblocktxn for a block > %i deep", pfrom->id, MAX_BLOCKTXN_DEPTH); return true; } @@ -2081,6 +2085,12 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr return true; } + if (!fAlreadyInFlight && mapBlocksInFlight.size() == 1 && pindex->pprev->IsValid(BLOCK_VALID_CHAIN)) { + // We seem to be rather well-synced, so it appears pfrom was the first to provide us + // with this block! Let's get them to announce using compact blocks in the future. + MaybeSetPeerAsAnnouncingHeaderAndIDs(nodestate, pfrom, connman, msgMaker); + } + BlockTransactionsRequest req; for (size_t i = 0; i < cmpctblock.BlockTxCount(); i++) { if (!partialBlock.IsTxAvailable(i)) diff --git a/src/validation.h b/src/validation.h index 58ced71b806f..12f8c7aa86af 100644 --- a/src/validation.h +++ b/src/validation.h @@ -91,6 +91,11 @@ static const unsigned int BLOCK_STALLING_TIMEOUT = 2; /** Number of headers sent in one getheaders result. We rely on the assumption that if a peer sends * less than this number, we reached its tip. Changing this value is a protocol upgrade. */ static const unsigned int MAX_HEADERS_RESULTS = 2000; +/** Maximum depth of blocks we're willing to serve as compact blocks to peers + * when requested. For older blocks, a regular BLOCK response will be sent. */ +static const int MAX_CMPCTBLOCK_DEPTH = 5; +/** Maximum depth of blocks we're willing to respond to GETBLOCKTXN requests for. */ +static const int MAX_BLOCKTXN_DEPTH = 10; /** Size of the "block download window": how far ahead of our current height do we fetch? * Larger windows tolerate larger download speed differences between peer, but increase the potential * degree of disordering of blocks on disk (which make reindexing and in the future perhaps pruning From 08ec624b11d2092ed347169dc89b5f4cbdec3315 Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Fri, 21 Oct 2016 16:57:32 +0200 Subject: [PATCH 12/51] Merge #8975: Chainparams: Trivial: In AppInit2(), s/Params()/chainparams/ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 6f2f639 Chainparams: Trivial: In AppInit2(), s/Params()/chainparams/ (Jorge Timón) --- src/init.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/init.cpp b/src/init.cpp index 900dbe866644..96094094ccaf 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -1197,8 +1197,8 @@ bool AppInitParameterInteraction() dustRelayFee = CFeeRate(n); } - fRequireStandard = !GetBoolArg("-acceptnonstdtxn", !Params().RequireStandard()); - if (Params().RequireStandard() && !fRequireStandard) + fRequireStandard = !GetBoolArg("-acceptnonstdtxn", !chainparams.RequireStandard()); + if (chainparams.RequireStandard() && !fRequireStandard) return InitError(strprintf("acceptnonstdtxn is not currently supported for %s chain", chainparams.NetworkIDString())); nBytesPerSigOp = GetArg("-bytespersigop", nBytesPerSigOp); From ca643650c3b02b78fbce7a95dcf67c8f9ef0f20b Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Fri, 21 Oct 2016 16:58:20 +0200 Subject: [PATCH 13/51] Merge #8968: Don't hold cs_main when calling ProcessNewBlock from a cmpctblock 72ca7d9 Don't hold cs_main when calling ProcessNewBlock from a cmpctblock (Matt Corallo) --- src/net_processing.cpp | 62 ++++++++++++++++++++---------------------- 1 file changed, 30 insertions(+), 32 deletions(-) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index e37c1af01646..8f80d2419c54 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -2135,40 +2135,38 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr BlockTransactions resp; vRecv >> resp; - LOCK(cs_main); - - map::iterator> >::iterator it = mapBlocksInFlight.find(resp.blockhash); - if (it == mapBlocksInFlight.end() || !it->second.second->partialBlock || - it->second.first != pfrom->GetId()) { - LogPrint("net", "Peer %d sent us block transactions for block we weren't expecting\n", pfrom->id); - return true; - } - - PartiallyDownloadedBlock& partialBlock = *it->second.second->partialBlock; CBlock block; - ReadStatus status = partialBlock.FillBlock(block, resp.txn); - if (status == READ_STATUS_INVALID) { - MarkBlockAsReceived(resp.blockhash); // Reset in-flight state in case of whitelist - Misbehaving(pfrom->GetId(), 100); - LogPrintf("Peer %d sent us invalid compact block/non-matching block transactions\n", pfrom->id); - return true; - } else if (status == READ_STATUS_FAILED) { - // Might have collided, fall back to getdata now :( - std::vector invs; - invs.push_back(CInv(MSG_BLOCK, resp.blockhash)); - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, invs)); - } else { - ProcessNewBlock(chainparams, &block, false, NULL); - int nDoS; - if (state.IsInvalid(nDoS)) { - assert (state.GetRejectCode() < REJECT_INTERNAL); // Blocks are never rejected with internal reject codes - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::REJECT, strCommand, (unsigned char)state.GetRejectCode(), - state.GetRejectReason().substr(0, MAX_REJECT_MESSAGE_LENGTH), block.GetHash())); - if (nDoS > 0) { - LOCK(cs_main); - Misbehaving(pfrom->GetId(), nDoS); - } + bool fBlockRead = false; + { + LOCK(cs_main); + + std::map::iterator> >::iterator it = mapBlocksInFlight.find(resp.blockhash); + if (it == mapBlocksInFlight.end() || !it->second.second->partialBlock || + it->second.first != pfrom->GetId()) { + LogPrint("net", "Peer %d sent us block transactions for block we weren't expecting\n", pfrom->id); + return true; } + + PartiallyDownloadedBlock& partialBlock = *it->second.second->partialBlock; + ReadStatus status = partialBlock.FillBlock(block, resp.txn); + if (status == READ_STATUS_INVALID) { + MarkBlockAsReceived(resp.blockhash); // Reset in-flight state in case of whitelist + Misbehaving(pfrom->GetId(), 100); + LogPrintf("Peer %d sent us invalid compact block/non-matching block transactions\n", pfrom->id); + return true; + } else if (status == READ_STATUS_FAILED) { + // Might have collided, fall back to getdata now :( + std::vector invs; + invs.push_back(CInv(MSG_BLOCK, resp.blockhash)); + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, invs)); + } else + fBlockRead = true; + } // Don't hold cs_main when we call into ProcessNewBlock + if(fBlockRead) { + bool fNewBlock = false; + ProcessNewBlock(chainparams, &block, false, &fNewBlock); + if (fNewBlock) + pfrom->nLastBlockTime = GetTime(); } } From 8f3f6385f8724893a36003aa63a375789541ffcb Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Mon, 24 Oct 2016 11:17:04 +0200 Subject: [PATCH 14/51] Merge #8995: Add missing cs_main lock to ::GETBLOCKTXN processing dfe7906 Add missing cs_main lock to ::GETBLOCKTXN processing (Matt Corallo) --- src/net_processing.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 8f80d2419c54..a8e1c41e224b 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -1684,6 +1684,8 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr BlockTransactionsRequest req; vRecv >> req; + LOCK(cs_main); + BlockMap::iterator it = mapBlockIndex.find(req.blockhash); if (it == mapBlockIndex.end() || !(it->second->nStatus & BLOCK_HAVE_DATA)) { LogPrintf("Peer %d sent us a getblocktxn for a block we don't have", pfrom->id); From bc47ef3db2d523c46a73ae127793fff7f5fee332 Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Tue, 25 Oct 2016 14:00:27 +0200 Subject: [PATCH 15/51] Merge #8515: A few mempool removal optimizations 0334430 Add some missing includes (Pieter Wuille) 4100499 Return shared_ptr from mempool removes (Pieter Wuille) 51f2783 Make removed and conflicted arguments optional to remove (Pieter Wuille) f48211b Bypass removeRecursive in removeForReorg (Pieter Wuille) --- src/test/blockencodings_tests.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/test/blockencodings_tests.cpp b/src/test/blockencodings_tests.cpp index 179e906159f9..1f76c68523e3 100644 --- a/src/test/blockencodings_tests.cpp +++ b/src/test/blockencodings_tests.cpp @@ -80,8 +80,9 @@ BOOST_AUTO_TEST_CASE(SimpleRoundTripTest) BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2].GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); - std::list removed; - pool.removeRecursive(block.vtx[2], removed); + + std::vector> removed; + pool.removeRecursive(block.vtx[2], &removed); BOOST_CHECK_EQUAL(removed.size(), 1); CBlock block2; From 0f3ea35bfd40bcaa6611555f39a0025da5cdb295 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Mon, 7 Nov 2016 17:59:46 -0800 Subject: [PATCH 16/51] Merge #9026: Fix handling of invalid compact blocks d4833ff Bump the protocol version to distinguish new banning behavior. (Suhas Daftuar) 88c3549 Fix compact block handling to not ban if block is invalid (Suhas Daftuar) c93beac [qa] Test that invalid compactblocks don't result in ban (Suhas Daftuar) --- qa/rpc-tests/p2p-compactblocks.py | 27 +++++++++++++++++++++++++++ src/blockencodings.cpp | 2 +- src/blockencodings.h | 2 ++ src/net_processing.cpp | 20 ++++++++++++-------- 4 files changed, 42 insertions(+), 9 deletions(-) diff --git a/qa/rpc-tests/p2p-compactblocks.py b/qa/rpc-tests/p2p-compactblocks.py index 9e839e239b43..b05191e5b7a4 100755 --- a/qa/rpc-tests/p2p-compactblocks.py +++ b/qa/rpc-tests/p2p-compactblocks.py @@ -642,6 +642,28 @@ def test_end_to_end_block_relay(self, node, listeners): l.last_cmpctblock.header_and_shortids.header.calc_sha256() assert_equal(l.last_cmpctblock.header_and_shortids.header.sha256, block.sha256) + # Test that we don't get disconnected if we relay a compact block with valid header, + # but invalid transactions. + def test_invalid_tx_in_compactblock(self, node, test_node): + assert(len(self.utxos)) + utxo = self.utxos[0] + + block = self.build_block_with_transactions(node, utxo, 5) + del block.vtx[3] + block.hashMerkleRoot = block.calc_merkle_root() + block.solve() + + # Now send the compact block with all transactions prefilled, and + # verify that we don't get disconnected. + comp_block = HeaderAndShortIDs() + comp_block.initialize_from_block(block, prefill_list=[0, 1, 2, 3, 4]) + msg = msg_cmpctblock(comp_block.to_p2p()) + test_node.send_and_ping(msg) + + # Check that the tip didn't advance + assert(int(node.getbestblockhash(), 16) is not block.sha256) + test_node.sync_with_ping() + # Helper for enabling cb announcements # Send the sendcmpct request and sync headers def request_cb_announcements(self, peer, node, version): @@ -731,6 +753,11 @@ def run_test(self): self.test_end_to_end_block_relay(self.nodes[0], [self.second_node, self.test_node, self.old_node]) self.test_end_to_end_block_relay(self.nodes[1], [self.second_node, self.test_node, self.old_node]) + print("\tTesting handling of invalid compact blocks...") + self.test_invalid_tx_in_compactblock(self.nodes[0], self.test_node) + self.test_invalid_tx_in_compactblock(self.nodes[1], self.second_node) + self.test_invalid_tx_in_compactblock(self.nodes[1], self.old_node) + print("\tTesting invalid index in cmpctblock message...") self.test_invalid_cmpctblock_message() diff --git a/src/blockencodings.cpp b/src/blockencodings.cpp index 6136240bbe67..d47bdb9434ad 100644 --- a/src/blockencodings.cpp +++ b/src/blockencodings.cpp @@ -167,7 +167,7 @@ ReadStatus PartiallyDownloadedBlock::FillBlock(CBlock& block, const std::vector< // check its own merkle root and cache that check. if (state.CorruptionPossible()) return READ_STATUS_FAILED; // Possible Short ID collision - return READ_STATUS_INVALID; + return READ_STATUS_CHECKBLOCK_FAILED; } LogPrint("cmpctblock", "Successfully reconstructed block %s with %lu txn prefilled, %lu txn from mempool and %lu txn requested\n", header.GetHash().ToString(), prefilled_count, mempool_count, vtx_missing.size()); diff --git a/src/blockencodings.h b/src/blockencodings.h index 349fcbd50f8c..ccc33fc16732 100644 --- a/src/blockencodings.h +++ b/src/blockencodings.h @@ -124,6 +124,8 @@ typedef enum ReadStatus_t READ_STATUS_OK, READ_STATUS_INVALID, // Invalid object, peer is sending bogus crap READ_STATUS_FAILED, // Failed to process object + READ_STATUS_CHECKBLOCK_FAILED, // Used only by FillBlock to indicate a + // failure in CheckBlock. } ReadStatus; class CBlockHeaderAndShortTxIDs { diff --git a/src/net_processing.cpp b/src/net_processing.cpp index a8e1c41e224b..aa553a7ce03b 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -82,7 +82,7 @@ namespace { * messages or ban them when processing happens afterwards. Protected by * cs_main. */ - std::map mapBlockSource; + std::map > mapBlockSource; /** * Filter for transactions that were recently rejected by @@ -781,16 +781,16 @@ void PeerLogicValidation::BlockChecked(const CBlock& block, const CValidationSta LOCK(cs_main); const uint256 hash(block.GetHash()); - std::map::iterator it = mapBlockSource.find(hash); + std::map >::iterator it = mapBlockSource.find(hash); int nDoS = 0; if (state.IsInvalid(nDoS)) { - if (it != mapBlockSource.end() && State(it->second)) { + if (it != mapBlockSource.end() && State(it->second.first)) { assert (state.GetRejectCode() < REJECT_INTERNAL); // Blocks are never rejected with internal reject codes CBlockReject reject = {(unsigned char)state.GetRejectCode(), state.GetRejectReason().substr(0, MAX_REJECT_MESSAGE_LENGTH), hash}; - State(it->second)->rejects.push_back(reject); - if (nDoS > 0) - Misbehaving(it->second, nDoS); + State(it->second.first)->rejects.push_back(reject); + if ((nDoS > 0) && (it->second.second)) + Misbehaving(it->second.first, nDoS); } } if (it != mapBlockSource.end()) @@ -2161,11 +2161,15 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr std::vector invs; invs.push_back(CInv(MSG_BLOCK, resp.blockhash)); connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, invs)); - } else + } else { + MarkBlockAsReceived(resp.blockhash); // it is now an empty pointer fBlockRead = true; + } } // Don't hold cs_main when we call into ProcessNewBlock if(fBlockRead) { bool fNewBlock = false; + // Since we requested this block (it was in mapBlocksInFlight), force it to be processed, + // even if it would not be a candidate for new tip (missing previous block, chain not long enough, etc) ProcessNewBlock(chainparams, &block, false, &fNewBlock); if (fNewBlock) pfrom->nLastBlockTime = GetTime(); @@ -2352,7 +2356,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr forceProcessing |= MarkBlockAsReceived(hash); // mapBlockSource is only used for sending reject messages and DoS scores, // so the race between here and cs_main in ProcessNewBlock is fine. - mapBlockSource.emplace(hash, pfrom->GetId()); + mapBlockSource.emplace(hash, std::make_pair(pfrom->GetId(), true)); } bool fNewBlock = false; ProcessNewBlock(chainparams, pblock, forceProcessing, &fNewBlock); From a526e9bf89825e88b9d649097b69bf7375cc2e08 Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Wed, 9 Nov 2016 12:32:57 +0100 Subject: [PATCH 17/51] Merge #9039: Various serialization simplifcations and optimizations d59a518 Use fixed preallocation instead of costly GetSerializeSize (Pieter Wuille) 25a211a Add optimized CSizeComputer serializers (Pieter Wuille) a2929a2 Make CSerAction's ForRead() constexpr (Pieter Wuille) a603925 Avoid -Wshadow errors (Pieter Wuille) 5284721 Get rid of nType and nVersion (Pieter Wuille) 657e05a Make GetSerializeSize a wrapper on top of CSizeComputer (Pieter Wuille) fad9b66 Make nType and nVersion private and sometimes const (Pieter Wuille) c2c5d42 Make streams' read and write return void (Pieter Wuille) 50e8a9c Remove unused ReadVersion and WriteVersion (Pieter Wuille) --- src/blockencodings.cpp | 2 +- src/blockencodings.h | 10 +++++----- src/test/blockencodings_tests.cpp | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/blockencodings.cpp b/src/blockencodings.cpp index d47bdb9434ad..23a1c6988631 100644 --- a/src/blockencodings.cpp +++ b/src/blockencodings.cpp @@ -131,7 +131,7 @@ ReadStatus PartiallyDownloadedBlock::InitData(const CBlockHeaderAndShortTxIDs& c break; } - LogPrint("cmpctblock", "Initialized PartiallyDownloadedBlock for block %s using a cmpctblock of size %lu\n", cmpctblock.header.GetHash().ToString(), cmpctblock.GetSerializeSize(SER_NETWORK, PROTOCOL_VERSION)); + LogPrint("cmpctblock", "Initialized PartiallyDownloadedBlock for block %s using a cmpctblock of size %lu\n", cmpctblock.header.GetHash().ToString(), GetSerializeSize(cmpctblock, SER_NETWORK, PROTOCOL_VERSION)); return READ_STATUS_OK; } diff --git a/src/blockencodings.h b/src/blockencodings.h index ccc33fc16732..c9e80b4a483b 100644 --- a/src/blockencodings.h +++ b/src/blockencodings.h @@ -21,7 +21,7 @@ struct TransactionCompressor { ADD_SERIALIZE_METHODS; template - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + inline void SerializationOp(Stream& s, Operation ser_action) { READWRITE(tx); //TODO: Compress tx encoding } }; @@ -35,7 +35,7 @@ class BlockTransactionsRequest { ADD_SERIALIZE_METHODS; template - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + inline void SerializationOp(Stream& s, Operation ser_action) { READWRITE(blockhash); uint64_t indexes_size = (uint64_t)indexes.size(); READWRITE(COMPACTSIZE(indexes_size)); @@ -81,7 +81,7 @@ class BlockTransactions { ADD_SERIALIZE_METHODS; template - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + inline void SerializationOp(Stream& s, Operation ser_action) { READWRITE(blockhash); uint64_t txn_size = (uint64_t)txn.size(); READWRITE(COMPACTSIZE(txn_size)); @@ -109,7 +109,7 @@ struct PrefilledTransaction { ADD_SERIALIZE_METHODS; template - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + inline void SerializationOp(Stream& s, Operation ser_action) { uint64_t idx = index; READWRITE(COMPACTSIZE(idx)); if (idx > std::numeric_limits::max()) @@ -157,7 +157,7 @@ class CBlockHeaderAndShortTxIDs { ADD_SERIALIZE_METHODS; template - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + inline void SerializationOp(Stream& s, Operation ser_action) { READWRITE(header); READWRITE(nonce); diff --git a/src/test/blockencodings_tests.cpp b/src/test/blockencodings_tests.cpp index 1f76c68523e3..91f9f1a043c2 100644 --- a/src/test/blockencodings_tests.cpp +++ b/src/test/blockencodings_tests.cpp @@ -130,7 +130,7 @@ class TestHeaderAndShortIDs { ADD_SERIALIZE_METHODS; template - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + inline void SerializationOp(Stream& s, Operation ser_action) { READWRITE(header); READWRITE(nonce); size_t shorttxids_size = shorttxids.size(); From ac2cd34f517599ea8048f9a3b04cbc4880db4d12 Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Fri, 11 Nov 2016 11:02:34 +0100 Subject: [PATCH 18/51] Merge #9058: Fixes for p2p-compactblocks.py test timeouts on travis (#8842) dac53b5 Modify getblocktxn handler not to drop requests for old blocks (Russell Yanofsky) 55bfddc [qa] Fix stale data bug in test_compactblocks_not_at_tip (Russell Yanofsky) 47e9659 [qa] Fix bug in compactblocks v2 merge (Russell Yanofsky) --- qa/rpc-tests/p2p-compactblocks.py | 18 +++++++++++++----- src/net_processing.cpp | 12 ++++++++++++ 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/qa/rpc-tests/p2p-compactblocks.py b/qa/rpc-tests/p2p-compactblocks.py index b05191e5b7a4..b4985ba40c81 100755 --- a/qa/rpc-tests/p2p-compactblocks.py +++ b/qa/rpc-tests/p2p-compactblocks.py @@ -273,8 +273,8 @@ def test_compactblock_construction(self, node, test_node, version): tx = FromHex(CTransaction(), hex_tx) # Wait until we've seen the block announcement for the resulting tip - tip = int(self.nodes[0].getbestblockhash(), 16) - assert(self.test_node.wait_for_block_announcement(tip)) + tip = int(node.getbestblockhash(), 16) + assert(test_node.wait_for_block_announcement(tip)) # Now mine a block, and look at the resulting compact block. test_node.clear_block_announcement() @@ -535,8 +535,8 @@ def test_incorrect_blocktxn_response(self, node, test_node, version): assert_equal(int(node.getbestblockhash(), 16), block.sha256) def test_getblocktxn_handler(self, node, test_node, version): - # bitcoind won't respond for blocks whose height is more than 15 blocks - # deep. + # bitcoind will not send blocktxn responses for blocks whose height is + # more than 10 blocks deep. MAX_GETBLOCKTXN_DEPTH = 10 chain_height = node.getblockcount() current_height = chain_height @@ -563,11 +563,17 @@ def test_getblocktxn_handler(self, node, test_node, version): test_node.last_blocktxn = None current_height -= 1 - # Next request should be ignored, as we're past the allowed depth. + # Next request should send a full block response, as we're past the + # allowed depth for a blocktxn response. block_hash = node.getblockhash(current_height) msg.block_txn_request = BlockTransactionsRequest(int(block_hash, 16), [0]) + with mininode_lock: + test_node.last_block = None + test_node.last_blocktxn = None test_node.send_and_ping(msg) with mininode_lock: + test_node.last_block.block.calc_sha256() + assert_equal(test_node.last_block.block.sha256, int(block_hash, 16)) assert_equal(test_node.last_blocktxn, None) def test_compactblocks_not_at_tip(self, node, test_node): @@ -588,6 +594,8 @@ def test_compactblocks_not_at_tip(self, node, test_node): node.generate(1) wait_until(test_node.received_block_announcement, timeout=30) test_node.clear_block_announcement() + with mininode_lock: + test_node.last_block = None test_node.send_message(msg_getdata([CInv(20, int(new_blocks[0], 16))])) success = wait_until(lambda: test_node.last_block is not None, timeout=30) assert(success) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index aa553a7ce03b..83519128ea10 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -1693,7 +1693,19 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr } if (it->second->nHeight < chainActive.Height() - MAX_BLOCKTXN_DEPTH) { + // If an older block is requested (should never happen in practice, + // but can happen in tests) send a block response instead of a + // blocktxn response. Sending a full block response instead of a + // small blocktxn response is preferable in the case where a peer + // might maliciously send lots of getblocktxn requests to trigger + // expensive disk reads, because it will require the peer to + // actually receive all the data read from disk over the network. LogPrint("net", "Peer %d sent us a getblocktxn for a block > %i deep", pfrom->id, MAX_BLOCKTXN_DEPTH); + CInv vInv; + vInv.type = MSG_BLOCK; + vInv.hash = req.blockhash; + pfrom->vRecvGetData.push_back(vInv); + ProcessGetData(pfrom, chainparams.GetConsensus(), connman, interruptMsgProc); return true; } From 64393ed996aef5eae2e1826766e6a4a842080220 Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Tue, 15 Nov 2016 09:51:46 +0100 Subject: [PATCH 19/51] Merge #9160: [trivial] Fix hungarian variable name ec34648 [trivial] Fix hungarian variable name (Russell Yanofsky) --- src/net_processing.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 83519128ea10..3427ecb90f9b 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -1701,10 +1701,10 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // expensive disk reads, because it will require the peer to // actually receive all the data read from disk over the network. LogPrint("net", "Peer %d sent us a getblocktxn for a block > %i deep", pfrom->id, MAX_BLOCKTXN_DEPTH); - CInv vInv; - vInv.type = MSG_BLOCK; - vInv.hash = req.blockhash; - pfrom->vRecvGetData.push_back(vInv); + CInv inv; + inv.type = MSG_BLOCK; + inv.hash = req.blockhash; + pfrom->vRecvGetData.push_back(inv); ProcessGetData(pfrom, chainparams.GetConsensus(), connman, interruptMsgProc); return true; } From 629ddc3d24cffe4e177ed33862c51459a319f3f6 Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Sat, 19 Nov 2016 23:32:47 +0100 Subject: [PATCH 20/51] Merge #9159: [qa] Wait for specific block announcement in p2p-compactblocks dfa44d1 [qa] Wait for specific block announcement in p2p-compactblocks (Russell Yanofsky) --- qa/rpc-tests/p2p-compactblocks.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/qa/rpc-tests/p2p-compactblocks.py b/qa/rpc-tests/p2p-compactblocks.py index b4985ba40c81..161ec12831c1 100755 --- a/qa/rpc-tests/p2p-compactblocks.py +++ b/qa/rpc-tests/p2p-compactblocks.py @@ -177,12 +177,15 @@ def received_sendcmpct(): def check_announcement_of_new_block(node, peer, predicate): peer.clear_block_announcement() - node.generate(1) - got_message = wait_until(lambda: peer.block_announced, timeout=30) + block_hash = int(node.generate(1)[0], 16) + peer.wait_for_block_announcement(block_hash, timeout=30) assert(peer.block_announced) assert(got_message) + with mininode_lock: - assert(predicate(peer)) + assert predicate(peer), ( + "block_hash={!r}, cmpctblock={!r}, inv={!r}".format( + block_hash, peer.last_cmpctblock, peer.last_inv)) # We shouldn't get any block announcements via cmpctblock yet. check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is None) From a96ed5874d2b94dc336cc8df78e84320226776a8 Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Mon, 21 Nov 2016 10:51:32 +0100 Subject: [PATCH 21/51] Merge #9125: Make CBlock a vector of shared_ptr of CTransactions b4e4ba4 Introduce convenience type CTransactionRef (Pieter Wuille) 1662b43 Make CBlock::vtx a vector of shared_ptr (Pieter Wuille) da60506 Add deserializing constructors to CTransaction and CMutableTransaction (Pieter Wuille) 0e85204 Add serialization for unique_ptr and shared_ptr (Pieter Wuille) --- src/blockencodings.cpp | 14 +++---- src/blockencodings.h | 12 +++--- src/test/blockencodings_tests.cpp | 64 ++++++++++++++++++------------- src/test/test_dash.h | 4 +- 4 files changed, 52 insertions(+), 42 deletions(-) diff --git a/src/blockencodings.cpp b/src/blockencodings.cpp index 23a1c6988631..3c80d2556dba 100644 --- a/src/blockencodings.cpp +++ b/src/blockencodings.cpp @@ -24,7 +24,7 @@ CBlockHeaderAndShortTxIDs::CBlockHeaderAndShortTxIDs(const CBlock& block) : //TODO: Use our mempool prior to block acceptance to predictively fill more than just the coinbase prefilledtxn[0] = {0, block.vtx[0]}; for (size_t i = 1; i < block.vtx.size(); i++) { - const CTransaction& tx = block.vtx[i]; + const CTransaction& tx = *block.vtx[i]; shorttxids[i - 1] = GetShortID(tx.GetHash()); } } @@ -59,7 +59,7 @@ ReadStatus PartiallyDownloadedBlock::InitData(const CBlockHeaderAndShortTxIDs& c int32_t lastprefilledindex = -1; for (size_t i = 0; i < cmpctblock.prefilledtxn.size(); i++) { - if (cmpctblock.prefilledtxn[i].tx.IsNull()) + if (cmpctblock.prefilledtxn[i].tx->IsNull()) return READ_STATUS_INVALID; lastprefilledindex += cmpctblock.prefilledtxn[i].index + 1; //index is a uint16_t, so cant overflow here @@ -71,7 +71,7 @@ ReadStatus PartiallyDownloadedBlock::InitData(const CBlockHeaderAndShortTxIDs& c // have neither a prefilled txn or a shorttxid! return READ_STATUS_INVALID; } - txn_available[lastprefilledindex] = std::make_shared(cmpctblock.prefilledtxn[i].tx); + txn_available[lastprefilledindex] = cmpctblock.prefilledtxn[i].tx; } prefilled_count = cmpctblock.prefilledtxn.size(); @@ -142,7 +142,7 @@ bool PartiallyDownloadedBlock::IsTxAvailable(size_t index) const { return txn_available[index] ? true : false; } -ReadStatus PartiallyDownloadedBlock::FillBlock(CBlock& block, const std::vector& vtx_missing) const { +ReadStatus PartiallyDownloadedBlock::FillBlock(CBlock& block, const std::vector& vtx_missing) const { assert(!header.IsNull()); block = header; block.vtx.resize(txn_available.size()); @@ -154,7 +154,7 @@ ReadStatus PartiallyDownloadedBlock::FillBlock(CBlock& block, const std::vector< return READ_STATUS_INVALID; block.vtx[i] = vtx_missing[tx_missing_offset++]; } else - block.vtx[i] = *txn_available[i]; + block.vtx[i] = txn_available[i]; } if (vtx_missing.size() != tx_missing_offset) return READ_STATUS_INVALID; @@ -172,8 +172,8 @@ ReadStatus PartiallyDownloadedBlock::FillBlock(CBlock& block, const std::vector< LogPrint("cmpctblock", "Successfully reconstructed block %s with %lu txn prefilled, %lu txn from mempool and %lu txn requested\n", header.GetHash().ToString(), prefilled_count, mempool_count, vtx_missing.size()); if (vtx_missing.size() < 5) { - for(const CTransaction& tx : vtx_missing) - LogPrint("cmpctblock", "Reconstructed block %s required tx %s\n", header.GetHash().ToString(), tx.GetHash().ToString()); + for (const auto& tx : vtx_missing) + LogPrint("cmpctblock", "Reconstructed block %s required tx %s\n", header.GetHash().ToString(), tx->GetHash().ToString()); } return READ_STATUS_OK; diff --git a/src/blockencodings.h b/src/blockencodings.h index c9e80b4a483b..95ce9e51803d 100644 --- a/src/blockencodings.h +++ b/src/blockencodings.h @@ -14,9 +14,9 @@ class CTxMemPool; // Dumb helper to handle CTransaction compression at serialize-time struct TransactionCompressor { private: - CTransaction& tx; + CTransactionRef& tx; public: - TransactionCompressor(CTransaction& txIn) : tx(txIn) {} + TransactionCompressor(CTransactionRef& txIn) : tx(txIn) {} ADD_SERIALIZE_METHODS; @@ -72,7 +72,7 @@ class BlockTransactions { public: // A BlockTransactions message uint256 blockhash; - std::vector txn; + std::vector txn; BlockTransactions() {} BlockTransactions(const BlockTransactionsRequest& req) : @@ -104,7 +104,7 @@ struct PrefilledTransaction { // Used as an offset since last prefilled tx in CBlockHeaderAndShortTxIDs, // as a proper transaction-in-block-index in PartiallyDownloadedBlock uint16_t index; - CTransaction tx; + CTransactionRef tx; ADD_SERIALIZE_METHODS; @@ -193,7 +193,7 @@ class CBlockHeaderAndShortTxIDs { class PartiallyDownloadedBlock { protected: - std::vector > txn_available; + std::vector txn_available; size_t prefilled_count = 0, mempool_count = 0; CTxMemPool* pool; public: @@ -202,7 +202,7 @@ class PartiallyDownloadedBlock { ReadStatus InitData(const CBlockHeaderAndShortTxIDs& cmpctblock); bool IsTxAvailable(size_t index) const; - ReadStatus FillBlock(CBlock& block, const std::vector& vtx_missing) const; + ReadStatus FillBlock(CBlock& block, const std::vector& vtx_missing) const; }; #endif diff --git a/src/test/blockencodings_tests.cpp b/src/test/blockencodings_tests.cpp index 91f9f1a043c2..603cc6827ce4 100644 --- a/src/test/blockencodings_tests.cpp +++ b/src/test/blockencodings_tests.cpp @@ -26,21 +26,21 @@ static CBlock BuildBlockTestCase() { tx.vout[0].nValue = 42; block.vtx.resize(3); - block.vtx[0] = tx; + block.vtx[0] = MakeTransactionRef(tx); block.nVersion = 42; block.hashPrevBlock = GetRandHash(); block.nBits = 0x207fffff; tx.vin[0].prevout.hash = GetRandHash(); tx.vin[0].prevout.n = 0; - block.vtx[1] = tx; + block.vtx[1] = MakeTransactionRef(tx); tx.vin.resize(10); for (size_t i = 0; i < tx.vin.size(); i++) { tx.vin[i].prevout.hash = GetRandHash(); tx.vin[i].prevout.n = 0; } - block.vtx[2] = tx; + block.vtx[2] = MakeTransactionRef(tx); bool mutated; block.hashMerkleRoot = BlockMerkleRoot(block, &mutated); @@ -59,8 +59,8 @@ BOOST_AUTO_TEST_CASE(SimpleRoundTripTest) TestMemPoolEntryHelper entry; CBlock block(BuildBlockTestCase()); - pool.addUnchecked(block.vtx[2].GetHash(), entry.FromTx(block.vtx[2])); - BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2].GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 0); + pool.addUnchecked(block.vtx[2]->GetHash(), entry.FromTx(*block.vtx[2])); + BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 0); // Do a simple ShortTxIDs RT { @@ -78,15 +78,14 @@ BOOST_AUTO_TEST_CASE(SimpleRoundTripTest) BOOST_CHECK(!partialBlock.IsTxAvailable(1)); BOOST_CHECK( partialBlock.IsTxAvailable(2)); - BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2].GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); + BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); - - std::vector> removed; - pool.removeRecursive(block.vtx[2], &removed); + std::vector removed; + pool.removeRecursive(*block.vtx[2], &removed); BOOST_CHECK_EQUAL(removed.size(), 1); CBlock block2; - std::vector vtx_missing; + std::vector vtx_missing; BOOST_CHECK(partialBlock.FillBlock(block2, vtx_missing) == READ_STATUS_INVALID); // No transactions vtx_missing.push_back(block.vtx[2]); // Wrong transaction @@ -153,8 +152,10 @@ BOOST_AUTO_TEST_CASE(NonCoinbasePreforwardRTTest) TestMemPoolEntryHelper entry; CBlock block(BuildBlockTestCase()); - pool.addUnchecked(block.vtx[2].GetHash(), entry.FromTx(block.vtx[2])); - BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2].GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 0); + pool.addUnchecked(block.vtx[2]->GetHash(), entry.FromTx(*block.vtx[2])); + BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 0); + + uint256 txhash; // Test with pre-forwarding tx 1, but not coinbase { @@ -162,8 +163,8 @@ BOOST_AUTO_TEST_CASE(NonCoinbasePreforwardRTTest) shortIDs.prefilledtxn.resize(1); shortIDs.prefilledtxn[0] = {1, block.vtx[1]}; shortIDs.shorttxids.resize(2); - shortIDs.shorttxids[0] = shortIDs.GetShortID(block.vtx[0].GetHash()); - shortIDs.shorttxids[1] = shortIDs.GetShortID(block.vtx[2].GetHash()); + shortIDs.shorttxids[0] = shortIDs.GetShortID(block.vtx[0]->GetHash()); + shortIDs.shorttxids[1] = shortIDs.GetShortID(block.vtx[2]->GetHash()); CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); stream << shortIDs; @@ -177,10 +178,10 @@ BOOST_AUTO_TEST_CASE(NonCoinbasePreforwardRTTest) BOOST_CHECK( partialBlock.IsTxAvailable(1)); BOOST_CHECK( partialBlock.IsTxAvailable(2)); - BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2].GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); + BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); CBlock block2; - std::vector vtx_missing; + std::vector vtx_missing; BOOST_CHECK(partialBlock.FillBlock(block2, vtx_missing) == READ_STATUS_INVALID); // No transactions vtx_missing.push_back(block.vtx[1]); // Wrong transaction @@ -195,9 +196,13 @@ BOOST_AUTO_TEST_CASE(NonCoinbasePreforwardRTTest) BOOST_CHECK_EQUAL(block.hashMerkleRoot.ToString(), BlockMerkleRoot(block3, &mutated).ToString()); BOOST_CHECK(!mutated); - BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2].GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); + txhash = block.vtx[2]->GetHash(); + block.vtx.clear(); + block2.vtx.clear(); + block3.vtx.clear(); + BOOST_CHECK_EQUAL(pool.mapTx.find(txhash)->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); } - BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2].GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 0); + BOOST_CHECK_EQUAL(pool.mapTx.find(txhash)->GetSharedTx().use_count(), SHARED_TX_OFFSET + 0); } BOOST_AUTO_TEST_CASE(SufficientPreforwardRTTest) @@ -206,8 +211,10 @@ BOOST_AUTO_TEST_CASE(SufficientPreforwardRTTest) TestMemPoolEntryHelper entry; CBlock block(BuildBlockTestCase()); - pool.addUnchecked(block.vtx[1].GetHash(), entry.FromTx(block.vtx[1])); - BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[1].GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 0); + pool.addUnchecked(block.vtx[1]->GetHash(), entry.FromTx(*block.vtx[1])); + BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[1]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 0); + + uint256 txhash; // Test with pre-forwarding coinbase + tx 2 with tx 1 in mempool { @@ -216,7 +223,7 @@ BOOST_AUTO_TEST_CASE(SufficientPreforwardRTTest) shortIDs.prefilledtxn[0] = {0, block.vtx[0]}; shortIDs.prefilledtxn[1] = {1, block.vtx[2]}; // id == 1 as it is 1 after index 1 shortIDs.shorttxids.resize(1); - shortIDs.shorttxids[0] = shortIDs.GetShortID(block.vtx[1].GetHash()); + shortIDs.shorttxids[0] = shortIDs.GetShortID(block.vtx[1]->GetHash()); CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); stream << shortIDs; @@ -230,19 +237,22 @@ BOOST_AUTO_TEST_CASE(SufficientPreforwardRTTest) BOOST_CHECK( partialBlock.IsTxAvailable(1)); BOOST_CHECK( partialBlock.IsTxAvailable(2)); - BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[1].GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); + BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[1]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); CBlock block2; - std::vector vtx_missing; + std::vector vtx_missing; BOOST_CHECK(partialBlock.FillBlock(block2, vtx_missing) == READ_STATUS_OK); BOOST_CHECK_EQUAL(block.GetHash().ToString(), block2.GetHash().ToString()); bool mutated; BOOST_CHECK_EQUAL(block.hashMerkleRoot.ToString(), BlockMerkleRoot(block2, &mutated).ToString()); BOOST_CHECK(!mutated); - BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[1].GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); + txhash = block.vtx[1]->GetHash(); + block.vtx.clear(); + block2.vtx.clear(); + BOOST_CHECK_EQUAL(pool.mapTx.find(txhash)->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); } - BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[1].GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 0); + BOOST_CHECK_EQUAL(pool.mapTx.find(txhash)->GetSharedTx().use_count(), SHARED_TX_OFFSET + 0); } BOOST_AUTO_TEST_CASE(EmptyBlockRoundTripTest) @@ -256,7 +266,7 @@ BOOST_AUTO_TEST_CASE(EmptyBlockRoundTripTest) CBlock block; block.vtx.resize(1); - block.vtx[0] = coinbase; + block.vtx[0] = MakeTransactionRef(std::move(coinbase)); block.nVersion = 42; block.hashPrevBlock = GetRandHash(); block.nBits = 0x207fffff; @@ -281,7 +291,7 @@ BOOST_AUTO_TEST_CASE(EmptyBlockRoundTripTest) BOOST_CHECK(partialBlock.IsTxAvailable(0)); CBlock block2; - std::vector vtx_missing; + std::vector vtx_missing; BOOST_CHECK(partialBlock.FillBlock(block2, vtx_missing) == READ_STATUS_OK); BOOST_CHECK_EQUAL(block.GetHash().ToString(), block2.GetHash().ToString()); BOOST_CHECK_EQUAL(block.hashMerkleRoot.ToString(), BlockMerkleRoot(block2, &mutated).ToString()); diff --git a/src/test/test_dash.h b/src/test/test_dash.h index 41440c684f5a..0bb1717a839b 100644 --- a/src/test/test_dash.h +++ b/src/test/test_dash.h @@ -79,8 +79,8 @@ struct TestMemPoolEntryHelper nFee(0), nTime(0), dPriority(0.0), nHeight(1), spendsCoinbase(false), sigOpCount(4) { } - CTxMemPoolEntry FromTx(CMutableTransaction &tx, CTxMemPool *pool = NULL); - CTxMemPoolEntry FromTx(CTransaction &tx, CTxMemPool *pool = NULL); + CTxMemPoolEntry FromTx(const CMutableTransaction &tx, CTxMemPool *pool = NULL); + CTxMemPoolEntry FromTx(const CTransaction &tx, CTxMemPool *pool = NULL); // Change the default value TestMemPoolEntryHelper &Fee(CAmount _fee) { nFee = _fee; return *this; } From 8fdfa6e4145579f811c3f6565d6eb034ebc32e24 Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Mon, 21 Nov 2016 15:35:53 +0100 Subject: [PATCH 22/51] Merge #8872: Remove block-request logic from INV message processing 037159c Remove block-request logic from INV message processing (Matt Corallo) 3451203 [qa] Respond to getheaders and do not assume a getdata on inv (Matt Corallo) d768f15 [qa] Make comptool push blocks instead of relying on inv-fetch (mrbandrews) --- qa/rpc-tests/p2p-compactblocks.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/qa/rpc-tests/p2p-compactblocks.py b/qa/rpc-tests/p2p-compactblocks.py index 161ec12831c1..ea0a4f1d44ec 100755 --- a/qa/rpc-tests/p2p-compactblocks.py +++ b/qa/rpc-tests/p2p-compactblocks.py @@ -24,6 +24,7 @@ def __init__(self): self.last_cmpctblock = None self.block_announced = False self.last_getdata = None + self.last_getheaders = None self.last_getblocktxn = None self.last_block = None self.last_blocktxn = None @@ -61,6 +62,9 @@ def on_inv(self, conn, message): def on_getdata(self, conn, message): self.last_getdata = message + def on_getheaders(self, conn, message): + self.last_getheaders = message + def on_getblocktxn(self, conn, message): self.last_getblocktxn = message @@ -356,6 +360,9 @@ def test_compactblock_requests(self, node, test_node): if announce == "inv": test_node.send_message(msg_inv([CInv(2, block.sha256)])) + success = wait_until(lambda: test_node.last_getheaders is not None, timeout=30) + assert(success) + test_node.send_header_for_blocks([block]) else: test_node.send_header_for_blocks([block]) success = wait_until(lambda: test_node.last_getdata is not None, timeout=30) From 77731737420f0feb7b4f9a6fd9fe7a60121ffe99 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Wed, 23 Nov 2016 16:12:55 -0800 Subject: [PATCH 23/51] Merge #9199: Always drop the least preferred HB peer when adding a new one. ca8549d Always drop the least preferred HB peer when adding a new one. (Gregory Maxwell) --- src/net_processing.cpp | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 3427ecb90f9b..42c5e1870398 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -397,7 +397,10 @@ void UpdateBlockAvailability(NodeId nodeid, const uint256 &hash) { } } -void MaybeSetPeerAsAnnouncingHeaderAndIDs(const CNodeState* nodestate, CNode* pfrom, CConnman& connman, CNetMsgMaker& msgMaker) { +void MaybeSetPeerAsAnnouncingHeaderAndIDs(const CNodeState* nodestate, CNode* pfrom, CConnman& connman) { + if (!nodestate->fSupportsDesiredCmpctVersion) { + return; + } if (nodestate->fProvidesHeaderAndIDs) { for (std::list::iterator it = lNodesAnnouncingHeaderAndIDs.begin(); it != lNodesAnnouncingHeaderAndIDs.end(); it++) { if (*it == pfrom->GetId()) { @@ -411,14 +414,14 @@ void MaybeSetPeerAsAnnouncingHeaderAndIDs(const CNodeState* nodestate, CNode* pf if (lNodesAnnouncingHeaderAndIDs.size() >= 3) { // As per BIP152, we only get 3 of our peers to announce // blocks using compact encodings. - CNode* pnodeStop = FindNode(lNodesAnnouncingHeaderAndIDs.front()); - if (pnodeStop) { - connman.PushMessage(pnodeStop, msgMaker.Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); - lNodesAnnouncingHeaderAndIDs.pop_front(); - } + connman.ForNode(lNodesAnnouncingHeaderAndIDs.front(), [&connman, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion](CNode* pnodeStop){ + connman.PushMessage(pnodeStop, NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion); + return true; + }); + lNodesAnnouncingHeaderAndIDs.pop_front(); } fAnnounceUsingCMPCTBLOCK = true; - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); + connman.PushMessage(pfrom, NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion); lNodesAnnouncingHeaderAndIDs.push_back(pfrom->GetId()); } } From b77ff6c1e355f4133738dfc00fd35e469ee9ee7c Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Tue, 29 Nov 2016 11:12:39 +0100 Subject: [PATCH 24/51] Merge #9233: Fix some typos 15fa95d Fix some typos (fsb4000) --- src/blockencodings.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/blockencodings.cpp b/src/blockencodings.cpp index 3c80d2556dba..493131a489e5 100644 --- a/src/blockencodings.cpp +++ b/src/blockencodings.cpp @@ -62,7 +62,7 @@ ReadStatus PartiallyDownloadedBlock::InitData(const CBlockHeaderAndShortTxIDs& c if (cmpctblock.prefilledtxn[i].tx->IsNull()) return READ_STATUS_INVALID; - lastprefilledindex += cmpctblock.prefilledtxn[i].index + 1; //index is a uint16_t, so cant overflow here + lastprefilledindex += cmpctblock.prefilledtxn[i].index + 1; //index is a uint16_t, so can't overflow here if (lastprefilledindex > std::numeric_limits::max()) return READ_STATUS_INVALID; if ((uint32_t)lastprefilledindex > cmpctblock.shorttxids.size() + i) { From 1fc226dab57c7ff198c602d27b6d735d745c51de Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Fri, 2 Dec 2016 18:17:25 -0800 Subject: [PATCH 25/51] Merge #9260: Mrs Peacock in The Library with The Candlestick (killed main.{h,cpp}) 76faa3c Rename the remaining main.{h,cpp} to validation.{h,cpp} (Matt Corallo) e736772 Move network-msg-processing code out of main to its own file (Matt Corallo) 87c35f5 Remove orphan state wipe from UnloadBlockIndex. (Matt Corallo) --- src/blockencodings.cpp | 2 +- src/net_processing.cpp | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/blockencodings.cpp b/src/blockencodings.cpp index 493131a489e5..5dfeeafc84ed 100644 --- a/src/blockencodings.cpp +++ b/src/blockencodings.cpp @@ -10,7 +10,7 @@ #include "random.h" #include "streams.h" #include "txmempool.h" -#include "main.h" +#include "validation.h" #include "util.h" #include diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 42c5e1870398..f69bbdfb929a 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -8,6 +8,7 @@ #include "alert.h" #include "addrman.h" #include "arith_uint256.h" +#include "blockencodings.h" #include "chainparams.h" #include "consensus/validation.h" #include "hash.h" @@ -2033,7 +2034,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr CBlockIndex *pindex = NULL; CValidationState state; - if (!AcceptBlockHeader(cmpctblock.header, state, chainparams, &pindex)) { + if (!ProcessNewBlockHeaders({cmpctblock.header}, state, chainparams, &pindex)) { int nDoS; if (state.IsInvalid(nDoS)) { if (nDoS > 0) From ca694e861abffe3f0aad1b62b14b1f431840e820 Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Mon, 5 Dec 2016 08:07:22 +0100 Subject: [PATCH 26/51] Merge #9014: Fix block-connection performance regression dd0df81 Document ConnectBlock connectTrace postconditions (Matt Corallo) 2d6e561 Switch pblock in ProcessNewBlock to a shared_ptr (Matt Corallo) 2736c44 Make the optional pblock in ActivateBestChain a shared_ptr (Matt Corallo) ae4db44 Create a shared_ptr for the block we're connecting in ActivateBCS (Matt Corallo) fd9d890 Keep blocks as shared_ptrs, instead of copying txn in ConnectTip (Matt Corallo) 6fdd43b Add struct to track block-connect-time-generated info for callbacks (Matt Corallo) --- src/net_processing.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index f69bbdfb929a..316da843c972 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -2153,7 +2153,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr BlockTransactions resp; vRecv >> resp; - CBlock block; + std::shared_ptr pblock = std::make_shared(); bool fBlockRead = false; { LOCK(cs_main); @@ -2166,7 +2166,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr } PartiallyDownloadedBlock& partialBlock = *it->second.second->partialBlock; - ReadStatus status = partialBlock.FillBlock(block, resp.txn); + ReadStatus status = partialBlock.FillBlock(*pblock, resp.txn); if (status == READ_STATUS_INVALID) { MarkBlockAsReceived(resp.blockhash); // Reset in-flight state in case of whitelist Misbehaving(pfrom->GetId(), 100); @@ -2186,7 +2186,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr bool fNewBlock = false; // Since we requested this block (it was in mapBlocksInFlight), force it to be processed, // even if it would not be a candidate for new tip (missing previous block, chain not long enough, etc) - ProcessNewBlock(chainparams, &block, false, &fNewBlock); + ProcessNewBlock(chainparams, pblock, true, NULL, &fNewBlock); if (fNewBlock) pfrom->nLastBlockTime = GetTime(); } @@ -2375,7 +2375,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr mapBlockSource.emplace(hash, std::make_pair(pfrom->GetId(), true)); } bool fNewBlock = false; - ProcessNewBlock(chainparams, pblock, forceProcessing, &fNewBlock); + ProcessNewBlock(chainparams, pblock, forceProcessing, NULL, &fNewBlock); if (fNewBlock) pfrom->nLastBlockTime = GetTime(); } From 5af46eec4737f3dd9abe2354bf6a4992de1610af Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Fri, 9 Dec 2016 16:22:33 -0800 Subject: [PATCH 27/51] Merge #9240: Remove txConflicted a874ab5 remove internal tracking of mempool conflicts for reporting to wallet (Alex Morcos) bf663f8 remove external usage of mempool conflict tracking (Alex Morcos) --- src/test/blockencodings_tests.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/blockencodings_tests.cpp b/src/test/blockencodings_tests.cpp index 603cc6827ce4..8b546ee2f348 100644 --- a/src/test/blockencodings_tests.cpp +++ b/src/test/blockencodings_tests.cpp @@ -80,9 +80,9 @@ BOOST_AUTO_TEST_CASE(SimpleRoundTripTest) BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); - std::vector removed; - pool.removeRecursive(*block.vtx[2], &removed); - BOOST_CHECK_EQUAL(removed.size(), 1); + size_t poolSize = pool.size(); + pool.removeRecursive(*block.vtx[2]); + BOOST_CHECK_EQUAL(pool.size(), poolSize - 1); CBlock block2; std::vector vtx_missing; From c7a4fc96713c93194cfbe42030b280891c114fda Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Wed, 14 Dec 2016 09:33:14 +0100 Subject: [PATCH 28/51] Merge #9344: Do not run functions with necessary side-effects in assert() da9cdd2 Do not run functions with necessary side-effects in assert() (Gregory Maxwell) --- src/net_processing.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 316da843c972..bdfe0644b938 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -1714,7 +1714,8 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr } CBlock block; - assert(ReadBlockFromDisk(block, it->second, chainparams.GetConsensus())); + bool ret = ReadBlockFromDisk(block, it->second, chainparams.GetConsensus()); + assert(ret); BlockTransactions resp(req); for (size_t i = 0; i < req.indexes.size(); i++) { @@ -3019,9 +3020,10 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic& interr vHeaders.front().GetHash().ToString(), pto->id); //TODO: Shouldn't need to reload block from disk, but requires refactor CBlock block; - assert(ReadBlockFromDisk(block, pBestIndex, consensusParams)); + bool ret = ReadBlockFromDisk(block, pBestIndex, consensusParams); + assert(ret); CBlockHeaderAndShortTxIDs cmpctblock(block); - pto->PushMessage(NetMsgType::CMPCTBLOCK, cmpctblock); + connman.PushMessage(pto, msgMaker.Make(NetMsgType::CMPCTBLOCK, cmpctblock)); state.pindexBestHeaderSent = pBestIndex; } else if (state.fPreferHeaders) { if (vHeaders.size() > 1) { From 4be3fd7da597173e8ed7c1945ee1d6a4db51e896 Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Wed, 14 Dec 2016 10:55:20 +0100 Subject: [PATCH 29/51] Merge #9273: Remove unused CDiskBlockPos* argument from ProcessNewBlock a13fa4c Remove unused CDiskBlockPos* argument from ProcessNewBlock (Matt Corallo) --- src/net_processing.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index bdfe0644b938..4b81007a6f9e 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -2187,7 +2187,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr bool fNewBlock = false; // Since we requested this block (it was in mapBlocksInFlight), force it to be processed, // even if it would not be a candidate for new tip (missing previous block, chain not long enough, etc) - ProcessNewBlock(chainparams, pblock, true, NULL, &fNewBlock); + ProcessNewBlock(chainparams, pblock, true, &fNewBlock); if (fNewBlock) pfrom->nLastBlockTime = GetTime(); } @@ -2376,7 +2376,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr mapBlockSource.emplace(hash, std::make_pair(pfrom->GetId(), true)); } bool fNewBlock = false; - ProcessNewBlock(chainparams, pblock, forceProcessing, NULL, &fNewBlock); + ProcessNewBlock(chainparams, pblock, forceProcessing, &fNewBlock); if (fNewBlock) pfrom->nLastBlockTime = GetTime(); } From 36415f86815c8e91e8dcadb488df342466e8f8fe Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Mon, 19 Dec 2016 08:52:23 +0100 Subject: [PATCH 30/51] Merge #9352: Attempt reconstruction from all compact block announcements 813ede9 [qa] Update compactblocks test for multi-peer reconstruction (Suhas Daftuar) 7017298 Allow compactblock reconstruction when block is in flight (Suhas Daftuar) --- qa/rpc-tests/p2p-compactblocks.py | 53 ++++++++++++++++++++++++++++++- src/net_processing.cpp | 48 ++++++++++++++++++++++++++-- 2 files changed, 98 insertions(+), 3 deletions(-) diff --git a/qa/rpc-tests/p2p-compactblocks.py b/qa/rpc-tests/p2p-compactblocks.py index ea0a4f1d44ec..8cac28e14d1e 100755 --- a/qa/rpc-tests/p2p-compactblocks.py +++ b/qa/rpc-tests/p2p-compactblocks.py @@ -693,6 +693,53 @@ def request_cb_announcements(self, peer, node, version): msg.announce = True peer.send_and_ping(msg) + def test_compactblock_reconstruction_multiple_peers(self, node, stalling_peer, delivery_peer): + assert(len(self.utxos)) + + def announce_cmpct_block(node, peer): + utxo = self.utxos.pop(0) + block = self.build_block_with_transactions(node, utxo, 5) + + cmpct_block = HeaderAndShortIDs() + cmpct_block.initialize_from_block(block) + msg = msg_cmpctblock(cmpct_block.to_p2p()) + peer.send_and_ping(msg) + with mininode_lock: + assert(peer.last_getblocktxn is not None) + return block, cmpct_block + + block, cmpct_block = announce_cmpct_block(node, stalling_peer) + + for tx in block.vtx[1:]: + delivery_peer.send_message(msg_tx(tx)) + delivery_peer.sync_with_ping() + mempool = node.getrawmempool() + for tx in block.vtx[1:]: + assert(tx.hash in mempool) + + delivery_peer.send_and_ping(msg_cmpctblock(cmpct_block.to_p2p())) + assert_equal(int(node.getbestblockhash(), 16), block.sha256) + + self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) + + # Now test that delivering an invalid compact block won't break relay + + block, cmpct_block = announce_cmpct_block(node, stalling_peer) + for tx in block.vtx[1:]: + delivery_peer.send_message(msg_tx(tx)) + delivery_peer.sync_with_ping() + + cmpct_block.prefilled_txn[0].tx = CTxIn() + + delivery_peer.send_and_ping(msg_cmpctblock(cmpct_block.to_p2p())) + assert(int(node.getbestblockhash(), 16) != block.sha256) + + msg = msg_blocktxn() + msg.block_transactions.blockhash = block.sha256 + msg.block_transactions.transactions = block.vtx[1:] + stalling_peer.send_and_ping(msg) + assert_equal(int(node.getbestblockhash(), 16), block.sha256) + def run_test(self): # Setup the p2p connections and start up the network thread. self.test_node = TestNode() @@ -717,7 +764,7 @@ def run_test(self): # We will need UTXOs to construct transactions in later tests. self.make_utxos() - print("Running tests, activation:") + print("Running tests:") print("\tTesting SENDCMPCT p2p message... ") self.test_sendcmpct(self.nodes[0], self.test_node, 1) @@ -776,6 +823,10 @@ def run_test(self): self.test_invalid_tx_in_compactblock(self.nodes[1], self.second_node) self.test_invalid_tx_in_compactblock(self.nodes[1], self.old_node) + print("\tTesting reconstructing compact blocks from all peers...") + self.test_compactblock_reconstruction_multiple_peers(self.nodes[1], self.second_node, self.old_node) + sync_blocks(self.nodes) + print("\tTesting invalid index in cmpctblock message...") self.test_invalid_cmpctblock_message() diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 4b81007a6f9e..b1ae7198f811 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -2045,6 +2045,12 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr } } + // Keep a CBlock for "optimistic" compactblock reconstructions (see + // below) + std::shared_ptr pblock = std::make_shared(); + bool fBlockReconstructed = false; + + LOCK(cs_main); // If AcceptBlockHeader returned true, it set pindex assert(pindex); UpdateBlockAvailability(pfrom->GetId(), pindex->GetBlockHash()); @@ -2107,7 +2113,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (!fAlreadyInFlight && mapBlocksInFlight.size() == 1 && pindex->pprev->IsValid(BLOCK_VALID_CHAIN)) { // We seem to be rather well-synced, so it appears pfrom was the first to provide us // with this block! Let's get them to announce using compact blocks in the future. - MaybeSetPeerAsAnnouncingHeaderAndIDs(nodestate, pfrom, connman, msgMaker); + MaybeSetPeerAsAnnouncingHeaderAndIDs(nodestate, pfrom, connman); } BlockTransactionsRequest req; @@ -2126,6 +2132,23 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr req.blockhash = pindex->GetBlockHash(); connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETBLOCKTXN, req)); } + } else { + // This block is either already in flight from a different + // peer, or this peer has too many blocks outstanding to + // download from. + // Optimistically try to reconstruct anyway since we might be + // able to without any round trips. + PartiallyDownloadedBlock tempBlock(&mempool); + ReadStatus status = tempBlock.InitData(cmpctblock); + if (status != READ_STATUS_OK) { + // TODO: don't ignore failures + return true; + } + std::vector dummy; + status = tempBlock.FillBlock(*pblock, dummy); + if (status == READ_STATUS_OK) { + fBlockReconstructed = true; + } } } else { if (fAlreadyInFlight) { @@ -2146,7 +2169,28 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr } } - CheckBlockIndex(chainparams.GetConsensus()); + if (fBlockReconstructed) { + // If we got here, we were able to optimistically reconstruct a + // block that is in flight from some other peer. + { + LOCK(cs_main); + mapBlockSource.emplace(pblock->GetHash(), std::make_pair(pfrom->GetId(), false)); + } + bool fNewBlock = false; + ProcessNewBlock(chainparams, pblock, true, &fNewBlock); + if (fNewBlock) + pfrom->nLastBlockTime = GetTime(); + + LOCK(cs_main); // hold cs_main for CBlockIndex::IsValid() + if (pindex->IsValid(BLOCK_VALID_TRANSACTIONS)) { + // Clear download state for this block, which is in + // process from some other peer. We do this after calling + // ProcessNewBlock so that a malleated cmpctblock announcement + // can't be used to interfere with block relay. + MarkBlockAsReceived(pblock->GetHash()); + } + } + } else if (strCommand == NetMsgType::BLOCKTXN && !fImporting && !fReindex) // Ignore blocks received while importing From 085e9ce46b505bccd3b43e7c23589f5e2f72e5c0 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Tue, 3 Jan 2017 13:41:42 -0800 Subject: [PATCH 31/51] Merge #9252: Release cs_main before calling ProcessNewBlock, or processing headers (cmpctblock handling) bd02bdd Release cs_main before processing cmpctblock as header (Suhas Daftuar) 680b0c0 Release cs_main before calling ProcessNewBlock (cmpctblock handling) (Suhas Daftuar) --- src/net_processing.cpp | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index b1ae7198f811..cac4351f6047 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -2045,11 +2045,24 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr } } + // When we succeed in decoding a block's txids from a cmpctblock + // message we typically jump to the BLOCKTXN handling code, with a + // dummy (empty) BLOCKTXN message, to re-use the logic there in + // completing processing of the putative block (without cs_main). + bool fProcessBLOCKTXN = false; + CDataStream blockTxnMsg(SER_NETWORK, PROTOCOL_VERSION); + + // If we end up treating this as a plain headers message, call that as well + // without cs_main. + bool fRevertToHeaderProcessing = false; + CDataStream vHeadersMsg(SER_NETWORK, PROTOCOL_VERSION); + // Keep a CBlock for "optimistic" compactblock reconstructions (see // below) std::shared_ptr pblock = std::make_shared(); bool fBlockReconstructed = false; + { LOCK(cs_main); // If AcceptBlockHeader returned true, it set pindex assert(pindex); @@ -2125,9 +2138,8 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // Dirty hack to jump to BLOCKTXN code (TODO: move message handling into their own functions) BlockTransactions txn; txn.blockhash = cmpctblock.header.GetHash(); - CDataStream blockTxnMsg(SER_NETWORK, PROTOCOL_VERSION); blockTxnMsg << txn; - return connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::BLOCKTXN, blockTxnMsg, nTimeReceived, chainparams)); + fProcessBLOCKTXN = true; } else { req.blockhash = pindex->GetBlockHash(); connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETBLOCKTXN, req)); @@ -2163,11 +2175,17 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // Dirty hack to process as if it were just a headers message (TODO: move message handling into their own functions) std::vector headers; headers.push_back(cmpctblock.header); - CDataStream vHeadersMsg(SER_NETWORK, PROTOCOL_VERSION); vHeadersMsg << headers; - return ProcessMessage(pfrom, NetMsgType::HEADERS, vHeadersMsg, nTimeReceived, chainparams, connman, interruptMsgProc); + fRevertToHeaderProcessing = true; } } + } // cs_main + + if (fProcessBLOCKTXN) + return ProcessMessage(pfrom, NetMsgType::BLOCKTXN, blockTxnMsg, nTimeReceived, chainparams, connman, interruptMsgProc); + + if (fRevertToHeaderProcessing) + return ProcessMessage(pfrom, NetMsgType::HEADERS, vHeadersMsg, nTimeReceived, chainparams, connman, interruptMsgProc); if (fBlockReconstructed) { // If we got here, we were able to optimistically reconstruct a From 385fb8d7f90021a6b71821dcae32ee9a0d12be6e Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Wed, 4 Jan 2017 12:22:49 +0100 Subject: [PATCH 32/51] Merge #9283: A few more CTransactionRef optimizations 91335ba Remove unused MakeTransactionRef overloads (Pieter Wuille) 6713f0f Make FillBlock consume txn_available to avoid shared_ptr copies (Pieter Wuille) 62607d7 Convert COrphanTx to keep a CTransactionRef (Pieter Wuille) c44e4c4 Make AcceptToMemoryPool take CTransactionRef (Pieter Wuille) --- src/blockencodings.cpp | 14 +++++++--- src/blockencodings.h | 2 +- src/net_processing.cpp | 3 +-- src/test/blockencodings_tests.cpp | 45 ++++++++++++++++++++----------- 4 files changed, 41 insertions(+), 23 deletions(-) diff --git a/src/blockencodings.cpp b/src/blockencodings.cpp index 5dfeeafc84ed..bf3612c6f42b 100644 --- a/src/blockencodings.cpp +++ b/src/blockencodings.cpp @@ -142,8 +142,9 @@ bool PartiallyDownloadedBlock::IsTxAvailable(size_t index) const { return txn_available[index] ? true : false; } -ReadStatus PartiallyDownloadedBlock::FillBlock(CBlock& block, const std::vector& vtx_missing) const { +ReadStatus PartiallyDownloadedBlock::FillBlock(CBlock& block, const std::vector& vtx_missing) { assert(!header.IsNull()); + uint256 hash = header.GetHash(); block = header; block.vtx.resize(txn_available.size()); @@ -154,8 +155,13 @@ ReadStatus PartiallyDownloadedBlock::FillBlock(CBlock& block, const std::vector< return READ_STATUS_INVALID; block.vtx[i] = vtx_missing[tx_missing_offset++]; } else - block.vtx[i] = txn_available[i]; + block.vtx[i] = std::move(txn_available[i]); } + + // Make sure we can't call FillBlock again. + header.SetNull(); + txn_available.clear(); + if (vtx_missing.size() != tx_missing_offset) return READ_STATUS_INVALID; @@ -170,10 +176,10 @@ ReadStatus PartiallyDownloadedBlock::FillBlock(CBlock& block, const std::vector< return READ_STATUS_CHECKBLOCK_FAILED; } - LogPrint("cmpctblock", "Successfully reconstructed block %s with %lu txn prefilled, %lu txn from mempool and %lu txn requested\n", header.GetHash().ToString(), prefilled_count, mempool_count, vtx_missing.size()); + LogPrint("cmpctblock", "Successfully reconstructed block %s with %lu txn prefilled, %lu txn from mempool and %lu txn requested\n", hash.ToString(), prefilled_count, mempool_count, vtx_missing.size()); if (vtx_missing.size() < 5) { for (const auto& tx : vtx_missing) - LogPrint("cmpctblock", "Reconstructed block %s required tx %s\n", header.GetHash().ToString(), tx->GetHash().ToString()); + LogPrint("cmpctblock", "Reconstructed block %s required tx %s\n", hash.ToString(), tx->GetHash().ToString()); } return READ_STATUS_OK; diff --git a/src/blockencodings.h b/src/blockencodings.h index 95ce9e51803d..63a174c783fb 100644 --- a/src/blockencodings.h +++ b/src/blockencodings.h @@ -202,7 +202,7 @@ class PartiallyDownloadedBlock { ReadStatus InitData(const CBlockHeaderAndShortTxIDs& cmpctblock); bool IsTxAvailable(size_t index) const; - ReadStatus FillBlock(CBlock& block, const std::vector& vtx_missing) const; + ReadStatus FillBlock(CBlock& block, const std::vector& vtx_missing); }; #endif diff --git a/src/net_processing.cpp b/src/net_processing.cpp index cac4351f6047..ae129f2b4571 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -1908,8 +1908,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (setMisbehaving.count(fromPeer)) continue; - if (AcceptToMemoryPool(mempool, stateDummy, porphanTx, true, &fMissingInputs2)) - { + if (AcceptToMemoryPool(mempool, stateDummy, porphanTx, true, &fMissingInputs2)) { LogPrint("mempool", " accepted orphan tx %s\n", orphanHash.ToString()); connman.RelayTransaction(orphanTx); for (unsigned int i = 0; i < orphanTx.vout.size(); i++) { diff --git a/src/test/blockencodings_tests.cpp b/src/test/blockencodings_tests.cpp index 8b546ee2f348..8db51a786183 100644 --- a/src/test/blockencodings_tests.cpp +++ b/src/test/blockencodings_tests.cpp @@ -85,17 +85,23 @@ BOOST_AUTO_TEST_CASE(SimpleRoundTripTest) BOOST_CHECK_EQUAL(pool.size(), poolSize - 1); CBlock block2; - std::vector vtx_missing; - BOOST_CHECK(partialBlock.FillBlock(block2, vtx_missing) == READ_STATUS_INVALID); // No transactions + { + PartiallyDownloadedBlock tmp = partialBlock; + BOOST_CHECK(partialBlock.FillBlock(block2, {}) == READ_STATUS_INVALID); // No transactions + partialBlock = tmp; + } - vtx_missing.push_back(block.vtx[2]); // Wrong transaction - partialBlock.FillBlock(block2, vtx_missing); // Current implementation doesn't check txn here, but don't require that + // Wrong transaction + { + PartiallyDownloadedBlock tmp = partialBlock; + partialBlock.FillBlock(block2, {block.vtx[2]}); // Current implementation doesn't check txn here, but don't require that + partialBlock = tmp; + } bool mutated; BOOST_CHECK(block.hashMerkleRoot != BlockMerkleRoot(block2, &mutated)); - vtx_missing[0] = block.vtx[1]; CBlock block3; - BOOST_CHECK(partialBlock.FillBlock(block3, vtx_missing) == READ_STATUS_OK); + BOOST_CHECK(partialBlock.FillBlock(block3, {block.vtx[1]}) == READ_STATUS_OK); BOOST_CHECK_EQUAL(block.GetHash().ToString(), block3.GetHash().ToString()); BOOST_CHECK_EQUAL(block.hashMerkleRoot.ToString(), BlockMerkleRoot(block3, &mutated).ToString()); BOOST_CHECK(!mutated); @@ -181,17 +187,24 @@ BOOST_AUTO_TEST_CASE(NonCoinbasePreforwardRTTest) BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); CBlock block2; - std::vector vtx_missing; - BOOST_CHECK(partialBlock.FillBlock(block2, vtx_missing) == READ_STATUS_INVALID); // No transactions + { + PartiallyDownloadedBlock tmp = partialBlock; + BOOST_CHECK(partialBlock.FillBlock(block2, {}) == READ_STATUS_INVALID); // No transactions + partialBlock = tmp; + } - vtx_missing.push_back(block.vtx[1]); // Wrong transaction - partialBlock.FillBlock(block2, vtx_missing); // Current implementation doesn't check txn here, but don't require that + // Wrong transaction + { + PartiallyDownloadedBlock tmp = partialBlock; + partialBlock.FillBlock(block2, {block.vtx[1]}); // Current implementation doesn't check txn here, but don't require that + partialBlock = tmp; + } bool mutated; BOOST_CHECK(block.hashMerkleRoot != BlockMerkleRoot(block2, &mutated)); - vtx_missing[0] = block.vtx[0]; CBlock block3; - BOOST_CHECK(partialBlock.FillBlock(block3, vtx_missing) == READ_STATUS_OK); + PartiallyDownloadedBlock partialBlockCopy = partialBlock; + BOOST_CHECK(partialBlock.FillBlock(block3, {block.vtx[0]}) == READ_STATUS_OK); BOOST_CHECK_EQUAL(block.GetHash().ToString(), block3.GetHash().ToString()); BOOST_CHECK_EQUAL(block.hashMerkleRoot.ToString(), BlockMerkleRoot(block3, &mutated).ToString()); BOOST_CHECK(!mutated); @@ -200,7 +213,7 @@ BOOST_AUTO_TEST_CASE(NonCoinbasePreforwardRTTest) block.vtx.clear(); block2.vtx.clear(); block3.vtx.clear(); - BOOST_CHECK_EQUAL(pool.mapTx.find(txhash)->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); + BOOST_CHECK_EQUAL(pool.mapTx.find(txhash)->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); // + 1 because of partialBlockCopy. } BOOST_CHECK_EQUAL(pool.mapTx.find(txhash)->GetSharedTx().use_count(), SHARED_TX_OFFSET + 0); } @@ -240,8 +253,8 @@ BOOST_AUTO_TEST_CASE(SufficientPreforwardRTTest) BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[1]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); CBlock block2; - std::vector vtx_missing; - BOOST_CHECK(partialBlock.FillBlock(block2, vtx_missing) == READ_STATUS_OK); + PartiallyDownloadedBlock partialBlockCopy = partialBlock; + BOOST_CHECK(partialBlock.FillBlock(block2, {}) == READ_STATUS_OK); BOOST_CHECK_EQUAL(block.GetHash().ToString(), block2.GetHash().ToString()); bool mutated; BOOST_CHECK_EQUAL(block.hashMerkleRoot.ToString(), BlockMerkleRoot(block2, &mutated).ToString()); @@ -250,7 +263,7 @@ BOOST_AUTO_TEST_CASE(SufficientPreforwardRTTest) txhash = block.vtx[1]->GetHash(); block.vtx.clear(); block2.vtx.clear(); - BOOST_CHECK_EQUAL(pool.mapTx.find(txhash)->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); + BOOST_CHECK_EQUAL(pool.mapTx.find(txhash)->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); // + 1 because of partialBlockCopy. } BOOST_CHECK_EQUAL(pool.mapTx.find(txhash)->GetSharedTx().use_count(), SHARED_TX_OFFSET + 0); } From e7ae1ecfb2d05bc0d4fe8066766e05b9c35cb703 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Fri, 13 Jan 2017 14:42:08 -0800 Subject: [PATCH 33/51] Merge #9375: Relay compact block messages prior to full block connection 02ee4eb Make most_recent_compact_block a pointer to a const (Matt Corallo) 73666ad Add comment to describe callers to ActivateBestChain (Matt Corallo) 962f7f0 Call ActivateBestChain without cs_main/with most_recent_block (Matt Corallo) 0df777d Use a temp pindex to avoid a const_cast in ProcessNewBlockHeaders (Matt Corallo) c1ae4fc Avoid holding cs_most_recent_block while calling ReadBlockFromDisk (Matt Corallo) 9eb67f5 Ensure we meet the BIP 152 old-relay-types response requirements (Matt Corallo) 5749a85 Cache most-recently-connected compact block (Matt Corallo) 9eaec08 Cache most-recently-announced block's shared_ptr (Matt Corallo) c802092 Relay compact block messages prior to full block connection (Matt Corallo) 6987219 Add a CValidationInterface::NewPoWValidBlock callback (Matt Corallo) 180586f Call AcceptBlock with the block's shared_ptr instead of CBlock& (Matt Corallo) 8baaba6 [qa] Avoid race in preciousblock test. (Matt Corallo) 9a0b2f4 [qa] Make compact blocks test construction using fetch methods (Matt Corallo) 8017547 Make CBlockIndex*es in net_processing const (Matt Corallo) --- qa/rpc-tests/p2p-compactblocks.py | 26 +++-- src/net_processing.cpp | 163 ++++++++++++++++++++++++++---- src/net_processing.h | 1 + src/validation.cpp | 5 + 4 files changed, 168 insertions(+), 27 deletions(-) diff --git a/qa/rpc-tests/p2p-compactblocks.py b/qa/rpc-tests/p2p-compactblocks.py index 8cac28e14d1e..da257b15808f 100755 --- a/qa/rpc-tests/p2p-compactblocks.py +++ b/qa/rpc-tests/p2p-compactblocks.py @@ -283,6 +283,9 @@ def test_compactblock_construction(self, node, test_node, version): tip = int(node.getbestblockhash(), 16) assert(test_node.wait_for_block_announcement(tip)) + # Make sure we will receive a fast-announce compact block + self.request_cb_announcements(test_node, node, version) + # Now mine a block, and look at the resulting compact block. test_node.clear_block_announcement() block_hash = int(node.generate(1)[0], 16) @@ -292,27 +295,36 @@ def test_compactblock_construction(self, node, test_node, version): [tx.calc_sha256() for tx in block.vtx] block.rehash() - # Don't care which type of announcement came back for this test; just - # request the compact block if we didn't get one yet. + # Wait until the block was announced (via compact blocks) wait_until(test_node.received_block_announcement, timeout=30) assert(test_node.received_block_announcement()) + # Now fetch and check the compact block + header_and_shortids = None + with mininode_lock: + assert(test_node.last_cmpctblock is not None) + # Convert the on-the-wire representation to absolute indexes + header_and_shortids = HeaderAndShortIDs(test_node.last_cmpctblock.header_and_shortids) + self.check_compactblock_construction_from_block(version, header_and_shortids, block_hash, block) + + # Now fetch the compact block using a normal non-announce getdata with mininode_lock: - if test_node.last_cmpctblock is None: - test_node.clear_block_announcement() - inv = CInv(20, block_hash) # 20 == "CompactBlock" - test_node.send_message(msg_getdata([inv])) + test_node.clear_block_announcement() + inv = CInv(20, block_hash) # 20 == "CompactBlock" + test_node.send_message(msg_getdata([inv])) wait_until(test_node.received_block_announcement, timeout=30) assert(test_node.received_block_announcement()) - # Now we should have the compactblock + # Now fetch and check the compact block header_and_shortids = None with mininode_lock: assert(test_node.last_cmpctblock is not None) # Convert the on-the-wire representation to absolute indexes header_and_shortids = HeaderAndShortIDs(test_node.last_cmpctblock.header_and_shortids) + self.check_compactblock_construction_from_block(version, header_and_shortids, block_hash, block) + def check_compactblock_construction_from_block(self, version, header_and_shortids, block_hash, block): # Check that we got the right block! header_and_shortids.header.calc_sha256() assert_equal(header_and_shortids.header.sha256, block_hash) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index ae129f2b4571..2a73ba4bb972 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -334,12 +334,12 @@ bool MarkBlockAsReceived(const uint256& hash) { // Requires cs_main. // returns false, still setting pit, if the block was already in flight from the same peer // pit will only be valid as long as the same cs_main lock is being held -void MarkBlockAsInFlight(NodeId nodeid, const uint256& hash, const Consensus::Params& consensusParams, const CBlockIndex *pindex = NULL, list::iterator **pit = NULL) { +bool MarkBlockAsInFlight(NodeId nodeid, const uint256& hash, const Consensus::Params& consensusParams, const CBlockIndex *pindex = NULL, std::list::iterator **pit = NULL) { CNodeState *state = State(nodeid); assert(state != NULL); // Short-circuit most stuff in case its from the same node - map::iterator> >::iterator itInFlight = mapBlocksInFlight.find(hash); + std::map::iterator> >::iterator itInFlight = mapBlocksInFlight.find(hash); if (itInFlight != mapBlocksInFlight.end() && itInFlight->second.first == nodeid) { *pit = &itInFlight->second.second; return false; @@ -348,7 +348,7 @@ void MarkBlockAsInFlight(NodeId nodeid, const uint256& hash, const Consensus::Pa // Make sure it's not listed somewhere already. MarkBlockAsReceived(hash); - list::iterator it = state->vBlocksInFlight.insert(state->vBlocksInFlight.end(), + std::list::iterator it = state->vBlocksInFlight.insert(state->vBlocksInFlight.end(), {hash, pindex, pindex != NULL, std::unique_ptr(pit ? new PartiallyDownloadedBlock(&mempool) : NULL)}); state->nBlocksInFlight++; state->nBlocksInFlightValidHeaders += it->fValidatedHeaders; @@ -750,6 +750,50 @@ void PeerLogicValidation::SyncTransaction(const CTransaction& tx, const CBlockIn } } +static CCriticalSection cs_most_recent_block; +static std::shared_ptr most_recent_block; +static std::shared_ptr most_recent_compact_block; +static uint256 most_recent_block_hash; + +void PeerLogicValidation::NewPoWValidBlock(const CBlockIndex *pindex, const std::shared_ptr& pblock) { + std::shared_ptr pcmpctblock = std::make_shared (*pblock); + const CNetMsgMaker msgMaker(PROTOCOL_VERSION); + + LOCK(cs_main); + + static int nHighestFastAnnounce = 0; + if (pindex->nHeight <= nHighestFastAnnounce) + return; + nHighestFastAnnounce = pindex->nHeight; + + uint256 hashBlock(pblock->GetHash()); + + { + LOCK(cs_most_recent_block); + most_recent_block_hash = hashBlock; + most_recent_block = pblock; + most_recent_compact_block = pcmpctblock; + } + + connman->ForEachNode([this, &pcmpctblock, pindex, &msgMaker, &hashBlock](CNode* pnode) { + // TODO: Avoid the repeated-serialization here + if (pnode->fDisconnect) + return; + ProcessBlockAvailability(pnode->GetId()); + CNodeState &state = *State(pnode->GetId()); + // If the peer has, or we announced to them the previous block already, + // but we don't think they have this one, go ahead and announce it + if (state.fPreferHeaderAndIDs && + !PeerHasHeader(&state, pindex) && PeerHasHeader(&state, pindex->pprev)) { + + LogPrint("net", "%s sending header-and-ids %s to peer %d\n", "PeerLogicValidation::NewPoWValidBlock", + hashBlock.ToString(), pnode->id); + connman->PushMessage(pnode, msgMaker.Make(NetMsgType::CMPCTBLOCK, *pcmpctblock)); + state.pindexBestHeaderSent = pindex; + } + }); +} + void PeerLogicValidation::UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) { const int nNewHeight = pindexNew->nHeight; connman->SetBestHeight(nNewHeight); @@ -945,6 +989,21 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam BlockMap::iterator mi = mapBlockIndex.find(inv.hash); if (mi != mapBlockIndex.end()) { + if (mi->second->nChainTx && !mi->second->IsValid(BLOCK_VALID_SCRIPTS) && + mi->second->IsValid(BLOCK_VALID_TREE)) { + // If we have the block and all of its parents, but have not yet validated it, + // we might be in the middle of connecting it (ie in the unlock of cs_main + // before ActivateBestChain but after AcceptBlock). + // In this case, we need to run ActivateBestChain prior to checking the relay + // conditions below. + std::shared_ptr a_recent_block; + { + LOCK(cs_most_recent_block); + a_recent_block = most_recent_block; + } + CValidationState dummy; + ActivateBestChain(dummy, Params(), a_recent_block); + } if (chainActive.Contains(mi->second)) { send = true; } else { @@ -1192,6 +1251,22 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam } } +inline void static SendBlockTransactions(const CBlock& block, const BlockTransactionsRequest& req, CNode* pfrom, CConnman& connman) { + BlockTransactions resp(req); + for (size_t i = 0; i < req.indexes.size(); i++) { + if (req.indexes[i] >= block.vtx.size()) { + LOCK(cs_main); + Misbehaving(pfrom->GetId(), 100); + LogPrintf("Peer %d sent us a getblocktxn with out-of-bounds tx indices", pfrom->id); + return; + } + resp.txn[i] = block.vtx[req.indexes[i]]; + } + LOCK(cs_main); + CNetMsgMaker msgMaker(pfrom->GetSendVersion()); + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::BLOCKTXN, resp)); +} + bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vRecv, int64_t nTimeReceived, const CChainParams& chainparams, CConnman& connman, const std::atomic& interruptMsgProc) { LogPrint("net", "received: %s (%u bytes) peer=%d\n", SanitizeString(strCommand), vRecv.size(), pfrom->id); @@ -1644,6 +1719,23 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr uint256 hashStop; vRecv >> locator >> hashStop; + // We might have announced the currently-being-connected tip using a + // compact block, which resulted in the peer sending a getblocks + // request, which we would otherwise respond to without the new block. + // To avoid this situation we simply verify that we are on our best + // known chain now. This is super overkill, but we handle it better + // for getheaders requests, and there are no known nodes which support + // compact blocks but still use getblocks to request blocks. + { + std::shared_ptr a_recent_block; + { + LOCK(cs_most_recent_block); + a_recent_block = most_recent_block; + } + CValidationState dummy; + ActivateBestChain(dummy, Params(), a_recent_block); + } + LOCK(cs_main); // Find the last block the caller has in the main chain @@ -1688,6 +1780,18 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr BlockTransactionsRequest req; vRecv >> req; + std::shared_ptr recent_block; + { + LOCK(cs_most_recent_block); + if (most_recent_block_hash == req.blockhash) + recent_block = most_recent_block; + // Unlock cs_most_recent_block to avoid cs_main lock inversion + } + if (recent_block) { + SendBlockTransactions(*recent_block, req, pfrom, connman); + return true; + } + LOCK(cs_main); BlockMap::iterator it = mapBlockIndex.find(req.blockhash); @@ -1717,16 +1821,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr bool ret = ReadBlockFromDisk(block, it->second, chainparams.GetConsensus()); assert(ret); - BlockTransactions resp(req); - for (size_t i = 0; i < req.indexes.size(); i++) { - if (req.indexes[i] >= block.vtx.size()) { - Misbehaving(pfrom->GetId(), 100); - LogPrintf("Peer %d sent us a getblocktxn with out-of-bounds tx indices", pfrom->id); - return true; - } - resp.txn[i] = block.vtx[req.indexes[i]]; - } - pfrom->PushMessage(NetMsgType::BLOCKTXN, resp); + SendBlockTransactions(block, req, pfrom, connman); } @@ -1774,6 +1869,14 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // if our peer has chainActive.Tip() (and thus we are sending an empty // headers message). In both cases it's safe to update // pindexBestHeaderSent to be our tip. + // + // It is important that we simply reset the BestHeaderSent value here, + // and not max(BestHeaderSent, newHeaderSent). We might have announced + // the currently-being-connected tip using a compact block, which + // resulted in the peer sending a headers request, which we respond to + // without the new block. By resetting the BestHeaderSent, we ensure we + // will re-announce the new block via headers (or compact blocks again) + // in the SendMessages logic. nodestate->pindexBestHeaderSent = pindex ? pindex : chainActive.Tip(); connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::HEADERS, vHeaders)); } @@ -2032,7 +2135,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr return true; } - CBlockIndex *pindex = NULL; + const CBlockIndex *pindex = NULL; CValidationState state; if (!ProcessNewBlockHeaders({cmpctblock.header}, state, chainparams, &pindex)) { int nDoS; @@ -2272,6 +2375,11 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr ReadCompactSize(vRecv); // ignore tx count; assume it is 0. } + if (nCount == 0) { + // Nothing interesting. Stop asking this peers for more headers. + return true; + } + const CBlockIndex *pindexLast = NULL; { LOCK(cs_main); @@ -3079,12 +3187,27 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic& interr // probably means we're doing an initial-ish-sync or they're slow LogPrint("net", "%s sending header-and-ids %s to peer %d\n", __func__, vHeaders.front().GetHash().ToString(), pto->id); - //TODO: Shouldn't need to reload block from disk, but requires refactor - CBlock block; - bool ret = ReadBlockFromDisk(block, pBestIndex, consensusParams); - assert(ret); - CBlockHeaderAndShortTxIDs cmpctblock(block); - connman.PushMessage(pto, msgMaker.Make(NetMsgType::CMPCTBLOCK, cmpctblock)); + + bool fGotBlockFromCache = false; + { + LOCK(cs_most_recent_block); + if (most_recent_block_hash == pBestIndex->GetBlockHash()) { + if (state.fWantsCmpctWitness) + connman.PushMessage(pto, msgMaker.Make(NetMsgType::CMPCTBLOCK, *most_recent_compact_block)); + else { + CBlockHeaderAndShortTxIDs cmpctblock(*most_recent_block, state.fWantsCmpctWitness); + connman.PushMessage(pto, msgMaker.Make(NetMsgType::CMPCTBLOCK, cmpctblock)); + } + fGotBlockFromCache = true; + } + } + if (!fGotBlockFromCache) { + CBlock block; + bool ret = ReadBlockFromDisk(block, pBestIndex, consensusParams); + assert(ret); + CBlockHeaderAndShortTxIDs cmpctblock(block, state.fWantsCmpctWitness); + connman.PushMessage(pto, msgMaker.Make(NetMsgType::CMPCTBLOCK, cmpctblock)); + } state.pindexBestHeaderSent = pBestIndex; } else if (state.fPreferHeaders) { if (vHeaders.size() > 1) { diff --git a/src/net_processing.h b/src/net_processing.h index 317f8eafa6f7..6e1345e0162a 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -36,6 +36,7 @@ class PeerLogicValidation : public CValidationInterface { virtual void SyncTransaction(const CTransaction& tx, const CBlockIndex* pindex, int nPosInBlock) override; virtual void UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) override; virtual void BlockChecked(const CBlock& block, const CValidationState& state) override; + virtual void NewPoWValidBlock(const CBlockIndex *pindex, const std::shared_ptr& pblock); }; struct CNodeStateStats { diff --git a/src/validation.cpp b/src/validation.cpp index e6040ef20f87..91997179885f 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -2913,6 +2913,11 @@ static void NotifyHeaderTip() { * that is already loaded (to avoid loading it again from disk). */ bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, std::shared_ptr pblock) { + // Note that while we're often called here from ProcessNewBlock, this is + // far from a guarantee. Things in the P2P/RPC will often end up calling + // us in the middle of ProcessNewBlock - do not assume pblock is set + // sanely for performance or correctness! + CBlockIndex *pindexMostWork = NULL; CBlockIndex *pindexNewTip = NULL; do { From 22553a022c3bad3c7e799c15c9736b75372ee49f Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Sun, 15 Jan 2017 06:44:20 +0100 Subject: [PATCH 34/51] Merge #9486: Make peer=%d log prints consistent e6111b2 Make peer id logging consistent ("peer=%d" instead of "peer %d") (Matt Corallo) --- src/net_processing.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 2a73ba4bb972..92f63589ac27 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -786,7 +786,7 @@ void PeerLogicValidation::NewPoWValidBlock(const CBlockIndex *pindex, const std: if (state.fPreferHeaderAndIDs && !PeerHasHeader(&state, pindex) && PeerHasHeader(&state, pindex->pprev)) { - LogPrint("net", "%s sending header-and-ids %s to peer %d\n", "PeerLogicValidation::NewPoWValidBlock", + LogPrint("net", "%s sending header-and-ids %s to peer=%d\n", "PeerLogicValidation::NewPoWValidBlock", hashBlock.ToString(), pnode->id); connman->PushMessage(pnode, msgMaker.Make(NetMsgType::CMPCTBLOCK, *pcmpctblock)); state.pindexBestHeaderSent = pindex; @@ -3185,7 +3185,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic& interr if (vHeaders.size() == 1 && state.fPreferHeaderAndIDs) { // We only send up to 1 block as header-and-ids, as otherwise // probably means we're doing an initial-ish-sync or they're slow - LogPrint("net", "%s sending header-and-ids %s to peer %d\n", __func__, + LogPrint("net", "%s sending header-and-ids %s to peer=%d\n", __func__, vHeaders.front().GetHash().ToString(), pto->id); bool fGotBlockFromCache = false; From 2f8f47557ed3d16534d571038a5ac38faec69952 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Sun, 15 Jan 2017 09:12:22 -0800 Subject: [PATCH 35/51] Merge #9400: Set peers as HB peers upon full block validation d4781ac Set peers as HB peers upon full block validation (Gregory Sanders) --- src/net_processing.cpp | 68 ++++++++++++++++++++---------------------- 1 file changed, 33 insertions(+), 35 deletions(-) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 92f63589ac27..6c06301c724c 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -398,32 +398,38 @@ void UpdateBlockAvailability(NodeId nodeid, const uint256 &hash) { } } -void MaybeSetPeerAsAnnouncingHeaderAndIDs(const CNodeState* nodestate, CNode* pfrom, CConnman& connman) { - if (!nodestate->fSupportsDesiredCmpctVersion) { +void MaybeSetPeerAsAnnouncingHeaderAndIDs(NodeId nodeid, CConnman& connman) { + AssertLockHeld(cs_main); + CNodeState* nodestate = State(nodeid); + if (!nodestate || !nodestate->fSupportsDesiredCmpctVersion) { + // Never ask from peers who can't provide desired version. return; } if (nodestate->fProvidesHeaderAndIDs) { for (std::list::iterator it = lNodesAnnouncingHeaderAndIDs.begin(); it != lNodesAnnouncingHeaderAndIDs.end(); it++) { - if (*it == pfrom->GetId()) { + if (*it == nodeid) { lNodesAnnouncingHeaderAndIDs.erase(it); - lNodesAnnouncingHeaderAndIDs.push_back(pfrom->GetId()); + lNodesAnnouncingHeaderAndIDs.push_back(nodeid); return; } } - bool fAnnounceUsingCMPCTBLOCK = false; - uint64_t nCMPCTBLOCKVersion = 1; - if (lNodesAnnouncingHeaderAndIDs.size() >= 3) { - // As per BIP152, we only get 3 of our peers to announce - // blocks using compact encodings. - connman.ForNode(lNodesAnnouncingHeaderAndIDs.front(), [&connman, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion](CNode* pnodeStop){ - connman.PushMessage(pnodeStop, NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion); - return true; - }); - lNodesAnnouncingHeaderAndIDs.pop_front(); - } - fAnnounceUsingCMPCTBLOCK = true; - connman.PushMessage(pfrom, NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion); - lNodesAnnouncingHeaderAndIDs.push_back(pfrom->GetId()); + connman.ForNode(nodeid, [&connman](CNode* pfrom){ + bool fAnnounceUsingCMPCTBLOCK = false; + uint64_t nCMPCTBLOCKVersion = 1; + if (lNodesAnnouncingHeaderAndIDs.size() >= 3) { + // As per BIP152, we only get 3 of our peers to announce + // blocks using compact encodings. + connman.ForNode(lNodesAnnouncingHeaderAndIDs.front(), [&connman, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion](CNode* pnodeStop){ + connman.PushMessage(pnodeStop, CNetMsgMaker(pnodeStop->GetSendVersion()).Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); + return true; + }); + lNodesAnnouncingHeaderAndIDs.pop_front(); + } + fAnnounceUsingCMPCTBLOCK = true; + connman.PushMessage(pfrom, CNetMsgMaker(pfrom->GetSendVersion()).Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); + lNodesAnnouncingHeaderAndIDs.push_back(pfrom->GetId()); + return true; + }); } } @@ -841,6 +847,11 @@ void PeerLogicValidation::BlockChecked(const CBlock& block, const CValidationSta Misbehaving(it->second.first, nDoS); } } + else if (state.IsValid() && !IsInitialBlockDownload() && mapBlocksInFlight.count(hash) == mapBlocksInFlight.size()) { + if (it != mapBlockSource.end()) { + MaybeSetPeerAsAnnouncingHeaderAndIDs(it->second.first, *connman); + } + } if (it != mapBlockSource.end()) mapBlockSource.erase(it); } @@ -2225,12 +2236,6 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr return true; } - if (!fAlreadyInFlight && mapBlocksInFlight.size() == 1 && pindex->pprev->IsValid(BLOCK_VALID_CHAIN)) { - // We seem to be rather well-synced, so it appears pfrom was the first to provide us - // with this block! Let's get them to announce using compact blocks in the future. - MaybeSetPeerAsAnnouncingHeaderAndIDs(nodestate, pfrom, connman); - } - BlockTransactionsRequest req; for (size_t i = 0; i < cmpctblock.BlockTxCount(); i++) { if (!partialBlock.IsTxAvailable(i)) @@ -2508,10 +2513,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr pindexLast->GetBlockHash().ToString(), pindexLast->nHeight); } if (vGetData.size() > 0) { - if (nodestate->fProvidesHeaderAndIDs && vGetData.size() == 1 && mapBlocksInFlight.size() == 1 && pindexLast->pprev->IsValid(BLOCK_VALID_CHAIN)) { - // We seem to be rather well-synced, so it appears pfrom was the first to provide us - // with this block! Let's get them to announce using compact blocks in the future. - MaybeSetPeerAsAnnouncingHeaderAndIDs(nodestate, pfrom, connman, msgMaker); + if (nodestate->fSupportsDesiredCmpctVersion && vGetData.size() == 1 && mapBlocksInFlight.size() == 1 && pindexLast->pprev->IsValid(BLOCK_VALID_CHAIN)) { // In any case, we want to download using a compact block, not a regular one vGetData[0] = CInv(MSG_CMPCT_BLOCK, vGetData[0].hash); } @@ -3192,12 +3194,8 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic& interr { LOCK(cs_most_recent_block); if (most_recent_block_hash == pBestIndex->GetBlockHash()) { - if (state.fWantsCmpctWitness) - connman.PushMessage(pto, msgMaker.Make(NetMsgType::CMPCTBLOCK, *most_recent_compact_block)); - else { - CBlockHeaderAndShortTxIDs cmpctblock(*most_recent_block, state.fWantsCmpctWitness); - connman.PushMessage(pto, msgMaker.Make(NetMsgType::CMPCTBLOCK, cmpctblock)); - } + CBlockHeaderAndShortTxIDs cmpctblock(*most_recent_block); + connman.PushMessage(pto, msgMaker.Make(NetMsgType::CMPCTBLOCK, cmpctblock)); fGotBlockFromCache = true; } } @@ -3205,7 +3203,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic& interr CBlock block; bool ret = ReadBlockFromDisk(block, pBestIndex, consensusParams); assert(ret); - CBlockHeaderAndShortTxIDs cmpctblock(block, state.fWantsCmpctWitness); + CBlockHeaderAndShortTxIDs cmpctblock(block); connman.PushMessage(pto, msgMaker.Make(NetMsgType::CMPCTBLOCK, cmpctblock)); } state.pindexBestHeaderSent = pBestIndex; From 4ab5e6ec8843a9f190ce8ad88f2f1ef0f5b8df97 Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Thu, 19 Jan 2017 09:02:26 +0100 Subject: [PATCH 36/51] Merge #9499: Use recent-rejects, orphans, and recently-replaced txn for compact-block-reconstruction c594580 Add braces around AddToCompactExtraTransactions (Matt Corallo) 1ccfe9b Clarify comment about mempool/extra conflicts (Matt Corallo) fac4c78 Make PartiallyDownloadedBlock::InitData's second param const (Matt Corallo) b55b416 Add extra_count lower bound to compact reconstruction debug print (Matt Corallo) 863edb4 Consider all (<100k memusage) txn for compact-block-extra-txn cache (Matt Corallo) 7f8c8ca Consider all orphan txn for compact-block-extra-txn cache (Matt Corallo) 93380c5 Use replaced transactions in compact block reconstruction (Matt Corallo) 1531652 Keep shared_ptrs to recently-replaced txn for compact blocks (Matt Corallo) edded80 Make ATMP optionally return the CTransactionRefs it replaced (Matt Corallo) c735540 Move ORPHAN constants from validation.h to net_processing.h (Matt Corallo) --- src/blockencodings.cpp | 37 +++++++++++++++++++++++++-- src/blockencodings.h | 5 ++-- src/init.cpp | 1 + src/net_processing.cpp | 38 +++++++++++++++++++++++----- src/net_processing.h | 3 +++ src/rpc/rawtransaction.cpp | 2 +- src/test/blockencodings_tests.cpp | 10 +++++--- src/test/txvalidationcache_tests.cpp | 2 +- src/validation.cpp | 34 ++++++++----------------- src/validation.h | 9 ++++--- src/wallet/wallet.cpp | 2 +- 11 files changed, 100 insertions(+), 43 deletions(-) diff --git a/src/blockencodings.cpp b/src/blockencodings.cpp index bf3612c6f42b..2c9d4f23df39 100644 --- a/src/blockencodings.cpp +++ b/src/blockencodings.cpp @@ -47,7 +47,7 @@ uint64_t CBlockHeaderAndShortTxIDs::GetShortID(const uint256& txhash) const { -ReadStatus PartiallyDownloadedBlock::InitData(const CBlockHeaderAndShortTxIDs& cmpctblock) { +ReadStatus PartiallyDownloadedBlock::InitData(const CBlockHeaderAndShortTxIDs& cmpctblock, const std::vector>& extra_txn) { if (cmpctblock.header.IsNull() || (cmpctblock.shorttxids.empty() && cmpctblock.prefilledtxn.empty())) return READ_STATUS_INVALID; if (cmpctblock.shorttxids.size() + cmpctblock.prefilledtxn.size() > MaxBlockSize(true) / MIN_TRANSACTION_SIZE) @@ -104,6 +104,7 @@ ReadStatus PartiallyDownloadedBlock::InitData(const CBlockHeaderAndShortTxIDs& c return READ_STATUS_FAILED; // Short ID collision std::vector have_txn(txn_available.size()); + { LOCK(pool->cs); const std::vector >& vTxHashes = pool->vTxHashes; for (size_t i = 0; i < vTxHashes.size(); i++) { @@ -130,6 +131,38 @@ ReadStatus PartiallyDownloadedBlock::InitData(const CBlockHeaderAndShortTxIDs& c if (mempool_count == shorttxids.size()) break; } + } + + for (size_t i = 0; i < extra_txn.size(); i++) { + uint64_t shortid = cmpctblock.GetShortID(extra_txn[i].first); + std::unordered_map::iterator idit = shorttxids.find(shortid); + if (idit != shorttxids.end()) { + if (!have_txn[idit->second]) { + txn_available[idit->second] = extra_txn[i].second; + have_txn[idit->second] = true; + mempool_count++; + extra_count++; + } else { + // If we find two mempool/extra txn that match the short id, just + // request it. + // This should be rare enough that the extra bandwidth doesn't matter, + // but eating a round-trip due to FillBlock failure would be annoying + // Note that we dont want duplication between extra_txn and mempool to + // trigger this case, so we compare hashes first + if (txn_available[idit->second] && + txn_available[idit->second]->GetHash() != extra_txn[i].second->GetHash()) { + txn_available[idit->second].reset(); + mempool_count--; + extra_count--; + } + } + } + // Though ideally we'd continue scanning for the two-txn-match-shortid case, + // the performance win of an early exit here is too good to pass up and worth + // the extra risk. + if (mempool_count == shorttxids.size()) + break; + } LogPrint("cmpctblock", "Initialized PartiallyDownloadedBlock for block %s using a cmpctblock of size %lu\n", cmpctblock.header.GetHash().ToString(), GetSerializeSize(cmpctblock, SER_NETWORK, PROTOCOL_VERSION)); @@ -176,7 +209,7 @@ ReadStatus PartiallyDownloadedBlock::FillBlock(CBlock& block, const std::vector< return READ_STATUS_CHECKBLOCK_FAILED; } - LogPrint("cmpctblock", "Successfully reconstructed block %s with %lu txn prefilled, %lu txn from mempool and %lu txn requested\n", hash.ToString(), prefilled_count, mempool_count, vtx_missing.size()); + LogPrint("cmpctblock", "Successfully reconstructed block %s with %lu txn prefilled, %lu txn from mempool (incl at least %lu from extra pool) and %lu txn requested\n", hash.ToString(), prefilled_count, mempool_count, extra_count, vtx_missing.size()); if (vtx_missing.size() < 5) { for (const auto& tx : vtx_missing) LogPrint("cmpctblock", "Reconstructed block %s required tx %s\n", hash.ToString(), tx->GetHash().ToString()); diff --git a/src/blockencodings.h b/src/blockencodings.h index 63a174c783fb..5dfc39e4d15e 100644 --- a/src/blockencodings.h +++ b/src/blockencodings.h @@ -194,13 +194,14 @@ class CBlockHeaderAndShortTxIDs { class PartiallyDownloadedBlock { protected: std::vector txn_available; - size_t prefilled_count = 0, mempool_count = 0; + size_t prefilled_count = 0, mempool_count = 0, extra_count = 0; CTxMemPool* pool; public: CBlockHeader header; PartiallyDownloadedBlock(CTxMemPool* poolIn) : pool(poolIn) {} - ReadStatus InitData(const CBlockHeaderAndShortTxIDs& cmpctblock); + // extra_txn is a list of extra transactions to look at, in form + ReadStatus InitData(const CBlockHeaderAndShortTxIDs& cmpctblock, const std::vector>& extra_txn); bool IsTxAvailable(size_t index) const; ReadStatus FillBlock(CBlock& block, const std::vector& vtx_missing); }; diff --git a/src/init.cpp b/src/init.cpp index 96094094ccaf..b1bd93f38156 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -411,6 +411,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-maxorphantx=", strprintf(_("Keep at most unconnectable transactions in memory (default: %u)"), DEFAULT_MAX_ORPHAN_TRANSACTIONS)); strUsage += HelpMessageOpt("-maxmempool=", strprintf(_("Keep the transaction memory pool below megabytes (default: %u)"), DEFAULT_MAX_MEMPOOL_SIZE)); strUsage += HelpMessageOpt("-mempoolexpiry=", strprintf(_("Do not keep transactions in the mempool longer than hours (default: %u)"), DEFAULT_MEMPOOL_EXPIRY)); + strUsage += HelpMessageOpt("-blockreconstructionextratxn=", strprintf(_("Extra transactions to keep in memory for compact block reconstructions (default: %u)"), DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN)); strUsage += HelpMessageOpt("-par=", strprintf(_("Set the number of script verification threads (%u to %d, 0 = auto, <0 = leave that many cores free, default: %d)"), -GetNumCores(), MAX_SCRIPTCHECK_THREADS, DEFAULT_SCRIPTCHECK_THREADS)); #ifndef WIN32 diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 6c06301c724c..f437a4218e8f 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -71,6 +71,9 @@ std::map mapOrphanTransactions GUARDED_BY(cs_main); std::map::iterator, IteratorComparator>> mapOrphanTransactionsByPrev GUARDED_BY(cs_main); void EraseOrphansFor(NodeId peer) EXCLUSIVE_LOCKS_REQUIRED(cs_main); +static size_t vExtraTxnForCompactIt = 0; +static std::vector> vExtraTxnForCompact GUARDED_BY(cs_main); + static const uint64_t RANDOMIZER_ID_ADDRESS_RELAY = 0x3cac0035b5866b90ULL; // SHA256("main address relay")[0:8] // Internal stuff @@ -590,6 +593,17 @@ void UnregisterNodeSignals(CNodeSignals& nodeSignals) // mapOrphanTransactions // +void AddToCompactExtraTransactions(const CTransactionRef& tx) +{ + size_t max_extra_txn = GetArg("-blockreconstructionextratxn", DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN); + if (max_extra_txn <= 0) + return; + if (!vExtraTxnForCompact.size()) + vExtraTxnForCompact.resize(max_extra_txn); + vExtraTxnForCompact[vExtraTxnForCompactIt] = std::make_pair(tx->GetHash(), tx); + vExtraTxnForCompactIt = (vExtraTxnForCompactIt + 1) % max_extra_txn; +} + bool AddOrphanTx(const CTransactionRef& tx, NodeId peer) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { const uint256& hash = tx->GetHash(); @@ -616,6 +630,8 @@ bool AddOrphanTx(const CTransactionRef& tx, NodeId peer) EXCLUSIVE_LOCKS_REQUIRE mapOrphanTransactionsByPrev[txin.prevout].insert(ret.first); } + AddToCompactExtraTransactions(tx); + LogPrint("mempool", "stored orphan tx %s (mapsz %u outsz %u)\n", hash.ToString(), mapOrphanTransactions.size(), mapOrphanTransactionsByPrev.size()); return true; @@ -1972,7 +1988,9 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr mapAlreadyAskedFor.erase(inv.hash); - if (!AlreadyHave(inv) && AcceptToMemoryPool(mempool, state, ptx, true, &fMissingInputs)) { + std::list lRemovedTxn; + + if (!AlreadyHave(inv) && AcceptToMemoryPool(mempool, state, ptx, true, &fMissingInputs, &lRemovedTxn)) { // Process custom txes, this changes AlreadyHave to "true" if (strCommand == NetMsgType::DSTX) { LogPrintf("DSTX -- Masternode transaction accepted, txid=%s, peer=%d\n", @@ -2022,7 +2040,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (setMisbehaving.count(fromPeer)) continue; - if (AcceptToMemoryPool(mempool, stateDummy, porphanTx, true, &fMissingInputs2)) { + if (AcceptToMemoryPool(mempool, stateDummy, porphanTx, true, &fMissingInputs2, &lRemovedTxn)) { LogPrint("mempool", " accepted orphan tx %s\n", orphanHash.ToString()); connman.RelayTransaction(orphanTx); for (unsigned int i = 0; i < orphanTx.vout.size(); i++) { @@ -2083,8 +2101,13 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr recentRejects->insert(tx.GetHash()); } } else { - assert(recentRejects); - recentRejects->insert(tx.GetHash()); + if (!state.CorruptionPossible()) { + assert(recentRejects); + recentRejects->insert(tx.GetHash()); + if (RecursiveDynamicUsage(*ptx) < 100000) { + AddToCompactExtraTransactions(ptx); + } + } if (strCommand == NetMsgType::TXLOCKREQUEST && !AlreadyHave(inv)) { // i.e. AcceptToMemoryPool failed, probably because it's conflicting @@ -2118,6 +2141,9 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr } } + for (const CTransactionRef& tx : lRemovedTxn) + AddToCompactExtraTransactions(tx); + int nDoS = 0; if (state.IsInvalid(nDoS)) { @@ -2222,7 +2248,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr } PartiallyDownloadedBlock& partialBlock = *(*queuedBlockIt)->partialBlock; - ReadStatus status = partialBlock.InitData(cmpctblock); + ReadStatus status = partialBlock.InitData(cmpctblock, vExtraTxnForCompact); if (status == READ_STATUS_INVALID) { MarkBlockAsReceived(pindex->GetBlockHash()); // Reset in-flight state in case of whitelist Misbehaving(pfrom->GetId(), 100); @@ -2258,7 +2284,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // Optimistically try to reconstruct anyway since we might be // able to without any round trips. PartiallyDownloadedBlock tempBlock(&mempool); - ReadStatus status = tempBlock.InitData(cmpctblock); + ReadStatus status = tempBlock.InitData(cmpctblock, vExtraTxnForCompact); if (status != READ_STATUS_OK) { // TODO: don't ignore failures return true; diff --git a/src/net_processing.h b/src/net_processing.h index 6e1345e0162a..e130839b0a6b 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -21,6 +21,9 @@ static const int64_t ORPHAN_TX_EXPIRE_INTERVAL = 5 * 60; static constexpr int64_t HEADERS_DOWNLOAD_TIMEOUT_BASE = 15 * 60 * 1000000; // 15 minutes static constexpr int64_t HEADERS_DOWNLOAD_TIMEOUT_PER_HEADER = 1000; // 1ms/header +/** Default number of orphan+recently-replaced txn to keep around for block reconstruction */ +static const unsigned int DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN = 100; + /** Register with a network node to receive its signals */ void RegisterNodeSignals(CNodeSignals& nodeSignals); /** Unregister a network node */ diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index a9fd236551cb..2f7cb412751d 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -928,7 +928,7 @@ UniValue sendrawtransaction(const JSONRPCRequest& request) } CValidationState state; bool fMissingInputs; - if (!AcceptToMemoryPool(mempool, state, std::move(tx), fLimitFree, &fMissingInputs, false, nMaxRawTxFee)) { + if (!AcceptToMemoryPool(mempool, state, std::move(tx), fLimitFree, &fMissingInputs, NULL, false, nMaxRawTxFee)) { if (state.IsInvalid()) { throw JSONRPCError(RPC_TRANSACTION_REJECTED, strprintf("%i: %s", state.GetRejectCode(), state.GetRejectReason())); } else { diff --git a/src/test/blockencodings_tests.cpp b/src/test/blockencodings_tests.cpp index 8db51a786183..69c489de0012 100644 --- a/src/test/blockencodings_tests.cpp +++ b/src/test/blockencodings_tests.cpp @@ -11,6 +11,8 @@ #include +std::vector> extra_txn; + struct RegtestingSetup : public TestingSetup { RegtestingSetup() : TestingSetup(CBaseChainParams::REGTEST) {} }; @@ -73,7 +75,7 @@ BOOST_AUTO_TEST_CASE(SimpleRoundTripTest) stream >> shortIDs2; PartiallyDownloadedBlock partialBlock(&pool); - BOOST_CHECK(partialBlock.InitData(shortIDs2) == READ_STATUS_OK); + BOOST_CHECK(partialBlock.InitData(shortIDs2, extra_txn) == READ_STATUS_OK); BOOST_CHECK( partialBlock.IsTxAvailable(0)); BOOST_CHECK(!partialBlock.IsTxAvailable(1)); BOOST_CHECK( partialBlock.IsTxAvailable(2)); @@ -179,7 +181,7 @@ BOOST_AUTO_TEST_CASE(NonCoinbasePreforwardRTTest) stream >> shortIDs2; PartiallyDownloadedBlock partialBlock(&pool); - BOOST_CHECK(partialBlock.InitData(shortIDs2) == READ_STATUS_OK); + BOOST_CHECK(partialBlock.InitData(shortIDs2, extra_txn) == READ_STATUS_OK); BOOST_CHECK(!partialBlock.IsTxAvailable(0)); BOOST_CHECK( partialBlock.IsTxAvailable(1)); BOOST_CHECK( partialBlock.IsTxAvailable(2)); @@ -245,7 +247,7 @@ BOOST_AUTO_TEST_CASE(SufficientPreforwardRTTest) stream >> shortIDs2; PartiallyDownloadedBlock partialBlock(&pool); - BOOST_CHECK(partialBlock.InitData(shortIDs2) == READ_STATUS_OK); + BOOST_CHECK(partialBlock.InitData(shortIDs2, extra_txn) == READ_STATUS_OK); BOOST_CHECK( partialBlock.IsTxAvailable(0)); BOOST_CHECK( partialBlock.IsTxAvailable(1)); BOOST_CHECK( partialBlock.IsTxAvailable(2)); @@ -300,7 +302,7 @@ BOOST_AUTO_TEST_CASE(EmptyBlockRoundTripTest) stream >> shortIDs2; PartiallyDownloadedBlock partialBlock(&pool); - BOOST_CHECK(partialBlock.InitData(shortIDs2) == READ_STATUS_OK); + BOOST_CHECK(partialBlock.InitData(shortIDs2, extra_txn) == READ_STATUS_OK); BOOST_CHECK(partialBlock.IsTxAvailable(0)); CBlock block2; diff --git a/src/test/txvalidationcache_tests.cpp b/src/test/txvalidationcache_tests.cpp index 8e6637e4f1be..af2495abf3ef 100644 --- a/src/test/txvalidationcache_tests.cpp +++ b/src/test/txvalidationcache_tests.cpp @@ -23,7 +23,7 @@ ToMemPool(CMutableTransaction& tx) LOCK(cs_main); CValidationState state; - return AcceptToMemoryPool(mempool, state, MakeTransactionRef(tx), false, NULL, true, 0); + return AcceptToMemoryPool(mempool, state, MakeTransactionRef(tx), false, NULL, NULL, true, 0); } BOOST_FIXTURE_TEST_CASE(tx_mempool_block_doublespend, TestChain100Setup) diff --git a/src/validation.cpp b/src/validation.cpp index 91997179885f..d0bc8cfc54b1 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -604,7 +604,8 @@ static bool IsCurrentForFeeEstimation() } bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const CTransactionRef& ptx, bool fLimitFree, - bool* pfMissingInputs, int64_t nAcceptTime, bool fOverrideMempoolLimit, const CAmount& nAbsurdFee, + bool* pfMissingInputs, int64_t nAcceptTime, std::list* plTxnReplaced, + bool fOverrideMempoolLimit, const CAmount& nAbsurdFee, std::vector& coins_to_uncache, bool fDryRun) { const CTransaction& tx = *ptx; @@ -1027,6 +1028,8 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C FormatMoney(nModifiedFees - nConflictingFees), CURRENCY_UNIT, (int)nSize - (int)nConflictingSize); + if (plTxnReplaced) + plTxnReplaced->push_back(it->GetSharedTx()); } pool.RemoveStaged(allConflicting, false, MemPoolRemovalReason::REPLACED); @@ -1064,10 +1067,11 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C } bool AcceptToMemoryPoolWithTime(CTxMemPool& pool, CValidationState &state, const CTransactionRef &tx, bool fLimitFree, - bool* pfMissingInputs, int64_t nAcceptTime, bool fOverrideMempoolLimit, const CAmount nAbsurdFee, bool fDryRun) + bool* pfMissingInputs, int64_t nAcceptTime, std::list* plTxnReplaced, + bool fOverrideMempoolLimit, const CAmount nAbsurdFee, bool fDryRun) { std::vector coins_to_uncache; - bool res = AcceptToMemoryPoolWorker(pool, state, tx, fLimitFree, pfMissingInputs, nAcceptTime, fOverrideMempoolLimit, nAbsurdFee, coins_to_uncache, fDryRun); + bool res = AcceptToMemoryPoolWorker(pool, state, tx, fLimitFree, pfMissingInputs, nAcceptTime, plTxnReplaced, fOverrideMempoolLimit, nAbsurdFee, coins_to_uncache, fDryRun); if (!res || fDryRun) { if(!res) LogPrint("mempool", "%s: %s %s (%s)\n", __func__, tx->GetHash().ToString(), state.GetRejectReason(), state.GetDebugMessage()); BOOST_FOREACH(const COutPoint& hashTx, coins_to_uncache) @@ -1080,9 +1084,10 @@ bool AcceptToMemoryPoolWithTime(CTxMemPool& pool, CValidationState &state, const } bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransactionRef &tx, bool fLimitFree, - bool* pfMissingInputs, bool fOverrideMempoolLimit, const CAmount nAbsurdFee, bool fDryRun) + bool* pfMissingInputs, std::list* plTxnReplaced, + bool fOverrideMempoolLimit, const CAmount nAbsurdFee, bool fDryRun) { - return AcceptToMemoryPoolWithTime(pool, state, tx, fLimitFree, pfMissingInputs, GetTime(), fOverrideMempoolLimit, nAbsurdFee, fDryRun); + return AcceptToMemoryPoolWithTime(pool, state, tx, fLimitFree, pfMissingInputs, GetTime(), plTxnReplaced, fOverrideMempoolLimit, nAbsurdFee, fDryRun); } bool GetTimestampIndex(const unsigned int &high, const unsigned int &low, std::vector &hashes) @@ -2594,24 +2599,7 @@ bool static DisconnectTip(CValidationState& state, const CChainParams& chainpara // Write the chain state to disk, if necessary. if (!FlushStateToDisk(state, FLUSH_STATE_IF_NEEDED)) return false; - // Resurrect mempool transactions from the disconnected block. - std::vector vHashUpdate; - for (const auto& it : block.vtx) { - const CTransaction& tx = *it; - // ignore validation errors in resurrected transactions - CValidationState stateDummy; - if (tx.IsCoinBase() || !AcceptToMemoryPool(mempool, stateDummy, it, false, NULL, true)) { - mempool.removeRecursive(tx, MemPoolRemovalReason::REORG); - } else if (mempool.exists(tx.GetHash())) { - vHashUpdate.push_back(tx.GetHash()); - } - } - // AcceptToMemoryPool/addUnchecked all assume that new mempool entries have - // no in-mempool children, which is generally not true when adding - // previously-confirmed transactions back to the mempool. - // UpdateTransactionsFromBlock finds descendants of any transactions in this - // block that were added back and cleans up the mempool state. - mempool.UpdateTransactionsFromBlock(vHashUpdate); + // Update chainActive and related variables. UpdateTip(pindexDelete->pprev, chainparams); // Let wallets know transactions went from 1-confirmed to diff --git a/src/validation.h b/src/validation.h index 12f8c7aa86af..f5c57a4388da 100644 --- a/src/validation.h +++ b/src/validation.h @@ -337,13 +337,16 @@ void PruneAndFlush(); /** Prune block files up to a given height */ void PruneBlockFilesManual(int nPruneUpToHeight); -/** (try to) add transaction to memory pool **/ +/** (try to) add transaction to memory pool + * plTxnReplaced will be appended to with all transactions replaced from mempool **/ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransactionRef &tx, bool fLimitFree, - bool* pfMissingInputs, bool fOverrideMempoolLimit=false, const CAmount nAbsurdFee=0, bool fDryRun=false); + bool* pfMissingInputs, std::list* plTxnReplaced = NULL, bool fOverrideMempoolLimit=false, + const CAmount nAbsurdFee=0, bool fDryRun=false); /** (try to) add transaction to memory pool with a specified acceptance time **/ bool AcceptToMemoryPoolWithTime(CTxMemPool& pool, CValidationState &state, const CTransactionRef &tx, bool fLimitFree, - bool* pfMissingInputs, int64_t nAcceptTime, bool fOverrideMempoolLimit=false, const CAmount nAbsurdFee=0, bool fDryRun=false); + bool* pfMissingInputs, int64_t nAcceptTime, std::list* plTxnReplaced = NULL, + bool fOverrideMempoolLimit=false, const CAmount nAbsurdFee=0, bool fDryRun=false); bool GetUTXOCoin(const COutPoint& outpoint, Coin& coin); int GetUTXOHeight(const COutPoint& outpoint); diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index f65c27bee350..6cc6671225b8 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -5393,5 +5393,5 @@ int CMerkleTx::GetBlocksToMaturity() const bool CMerkleTx::AcceptToMemoryPool(const CAmount& nAbsurdFee, CValidationState& state) { - return ::AcceptToMemoryPool(mempool, state, tx, true, NULL, false, nAbsurdFee); + return ::AcceptToMemoryPool(mempool, state, tx, true, NULL, NULL, false, nAbsurdFee); } From c728f7d3b0f2764c109859c0a17e0c40b282973d Mon Sep 17 00:00:00 2001 From: gladcow Date: Wed, 14 Mar 2018 15:52:38 +0300 Subject: [PATCH 37/51] Fix using of AcceptToMemoryPool in PrivateSend code --- src/privatesend-server.cpp | 2 +- src/privatesend.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/privatesend-server.cpp b/src/privatesend-server.cpp index 6db6990846eb..b727f772f1a6 100644 --- a/src/privatesend-server.cpp +++ b/src/privatesend-server.cpp @@ -334,7 +334,7 @@ void CPrivateSendServer::CommitFinalTransaction(CConnman& connman) TRY_LOCK(cs_main, lockMain); CValidationState validationState; mempool.PrioritiseTransaction(hashTx, hashTx.ToString(), 1000, 0.1*COIN); - if(!lockMain || !AcceptToMemoryPool(mempool, validationState, finalTransaction, false, NULL, false, maxTxFee, true)) + if(!lockMain || !AcceptToMemoryPool(mempool, validationState, finalTransaction, false, NULL, NULL, false, maxTxFee, true)) { LogPrintf("CPrivateSendServer::CommitFinalTransaction -- AcceptToMemoryPool() error: Transaction not valid\n"); SetNull(); diff --git a/src/privatesend.cpp b/src/privatesend.cpp index cc44d0b3085e..c61685ba490e 100644 --- a/src/privatesend.cpp +++ b/src/privatesend.cpp @@ -294,7 +294,7 @@ bool CPrivateSend::IsCollateralValid(const CTransaction& txCollateral) { LOCK(cs_main); CValidationState validationState; - if(!AcceptToMemoryPool(mempool, validationState, MakeTransactionRef(txCollateral), false, NULL, false, maxTxFee, true)) { + if(!AcceptToMemoryPool(mempool, validationState, MakeTransactionRef(txCollateral), false, NULL, NULL, false, maxTxFee, true)) { LogPrint("privatesend", "CPrivateSend::IsCollateralValid -- didn't pass AcceptToMemoryPool()\n"); return false; } From 39256a73730c4582fd20c8efac51bb048977a59a Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Thu, 26 Jan 2017 10:14:34 +0100 Subject: [PATCH 38/51] Merge #9587: Do not shadow local variable named `tx`. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 44f2baa Do not shadow local variable named `tx`. (Pavel Janík) --- src/net_processing.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index f437a4218e8f..bba89d08dcd6 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -2141,8 +2141,8 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr } } - for (const CTransactionRef& tx : lRemovedTxn) - AddToCompactExtraTransactions(tx); + for (const CTransactionRef& removedTx : lRemovedTxn) + AddToCompactExtraTransactions(removedTx); int nDoS = 0; if (state.IsInvalid(nDoS)) From 8e04e55bab751279f778f42c14b31de996002522 Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Sun, 29 Jan 2017 16:05:06 +0100 Subject: [PATCH 39/51] Merge #9510: [trivial] Fix typos in comments cc16d99 [trivial] Fix typos in comments (practicalswift) --- src/blockencodings.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/blockencodings.h b/src/blockencodings.h index 5dfc39e4d15e..a768a753f594 100644 --- a/src/blockencodings.h +++ b/src/blockencodings.h @@ -99,7 +99,7 @@ class BlockTransactions { } }; -// Dumb serialization/storage-helper for CBlockHeaderAndShortTxIDs and PartiallyDownlaodedBlock +// Dumb serialization/storage-helper for CBlockHeaderAndShortTxIDs and PartiallyDownloadedBlock struct PrefilledTransaction { // Used as an offset since last prefilled tx in CBlockHeaderAndShortTxIDs, // as a proper transaction-in-block-index in PartiallyDownloadedBlock From 519187cb157d6a9ffa530851d91c3e70ddfa2d76 Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Tue, 7 Feb 2017 13:03:21 +0100 Subject: [PATCH 40/51] Merge #9604: [Trivial] add comment about setting peer as HB peer. dd5b011 [Trivial] add comment about setting peer as HB peer. (John Newbery) --- src/net_processing.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index bba89d08dcd6..33cf59c867f4 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -863,7 +863,15 @@ void PeerLogicValidation::BlockChecked(const CBlock& block, const CValidationSta Misbehaving(it->second.first, nDoS); } } - else if (state.IsValid() && !IsInitialBlockDownload() && mapBlocksInFlight.count(hash) == mapBlocksInFlight.size()) { + // Check that: + // 1. The block is valid + // 2. We're not in initial block download + // 3. This is currently the best block we're aware of. We haven't updated + // the tip yet so we have no way to check this directly here. Instead we + // just check that there are currently no other blocks in flight. + else if (state.IsValid() && + !IsInitialBlockDownload() && + mapBlocksInFlight.count(hash) == mapBlocksInFlight.size()) { if (it != mapBlockSource.end()) { MaybeSetPeerAsAnnouncingHeaderAndIDs(it->second.first, *connman); } From 0400258be76128753a293f098d33bafebc43d436 Mon Sep 17 00:00:00 2001 From: UdjinM6 Date: Sun, 25 Mar 2018 22:08:48 +0300 Subject: [PATCH 41/51] bring back tx ressurection in DisconnectTip --- src/validation.cpp | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/validation.cpp b/src/validation.cpp index d0bc8cfc54b1..805f3ae28430 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -2599,7 +2599,24 @@ bool static DisconnectTip(CValidationState& state, const CChainParams& chainpara // Write the chain state to disk, if necessary. if (!FlushStateToDisk(state, FLUSH_STATE_IF_NEEDED)) return false; - + // Resurrect mempool transactions from the disconnected block. + std::vector vHashUpdate; + for (const auto& it : block.vtx) { + const CTransaction& tx = *it; + // ignore validation errors in resurrected transactions + CValidationState stateDummy; + if (tx.IsCoinBase() || !AcceptToMemoryPool(mempool, stateDummy, it, false, NULL, NULL, true)) { + mempool.removeRecursive(tx, MemPoolRemovalReason::REORG); + } else if (mempool.exists(tx.GetHash())) { + vHashUpdate.push_back(tx.GetHash()); + } + } + // AcceptToMemoryPool/addUnchecked all assume that new mempool entries have + // no in-mempool children, which is generally not true when adding + // previously-confirmed transactions back to the mempool. + // UpdateTransactionsFromBlock finds descendants of any transactions in this + // block that were added back and cleans up the mempool state. + mempool.UpdateTransactionsFromBlock(vHashUpdate); // Update chainActive and related variables. UpdateTip(pindexDelete->pprev, chainparams); // Let wallets know transactions went from 1-confirmed to From da49fce1bc78b208492799df2db7a4f19c430143 Mon Sep 17 00:00:00 2001 From: UdjinM6 Date: Sun, 25 Mar 2018 22:08:23 +0300 Subject: [PATCH 42/51] fSupportsDesiredCmpctVersion --- src/net_processing.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 33cf59c867f4..0725210f8e3b 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -222,7 +222,7 @@ struct CNodeState { fPreferHeaders = false; fPreferHeaderAndIDs = false; fProvidesHeaderAndIDs = false; - fSupportsDesiredCmpctVersion = true; // we have only one version for now + fSupportsDesiredCmpctVersion = false; } }; @@ -1638,6 +1638,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr LOCK(cs_main); State(pfrom->GetId())->fProvidesHeaderAndIDs = true; State(pfrom->GetId())->fPreferHeaderAndIDs = fAnnounceUsingCMPCTBLOCK; + State(pfrom->GetId())->fSupportsDesiredCmpctVersion = true; } } From ba27cdeb9c3bd0e18aa6b283854b05e048b171cf Mon Sep 17 00:00:00 2001 From: UdjinM6 Date: Sat, 24 Mar 2018 19:17:31 +0300 Subject: [PATCH 43/51] add `override` --- src/net_processing.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/net_processing.h b/src/net_processing.h index e130839b0a6b..3e7d596a597b 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -39,7 +39,7 @@ class PeerLogicValidation : public CValidationInterface { virtual void SyncTransaction(const CTransaction& tx, const CBlockIndex* pindex, int nPosInBlock) override; virtual void UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) override; virtual void BlockChecked(const CBlock& block, const CValidationState& state) override; - virtual void NewPoWValidBlock(const CBlockIndex *pindex, const std::shared_ptr& pblock); + virtual void NewPoWValidBlock(const CBlockIndex *pindex, const std::shared_ptr& pblock) override; }; struct CNodeStateStats { From 1a070be1aec2bda0dbdccd820bcf9cff93f16d25 Mon Sep 17 00:00:00 2001 From: UdjinM6 Date: Wed, 28 Mar 2018 03:49:36 +0300 Subject: [PATCH 44/51] Fix delayed headers --- src/net_processing.cpp | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 0725210f8e3b..544651aa45cc 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -2491,20 +2491,17 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // from there instead. LogPrint("net", "more getheaders (%d) to end to peer=%d (startheight:%d)\n", pindexLast->nHeight, pfrom->id, pfrom->nStartingHeight); connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexLast), uint256())); - } else { - if (GetAdjustedTime() - pindexBestHeader->GetBlockTime() > chainparams.DelayGetHeadersTime()) { - // peer has sent us a HEADERS message below maximum size and we are still quite far from being fully - // synced, this means we probably got a bad peer for initial sync and need to continue with another one. - // By disconnecting we force to start a new iteration of initial headers sync in SendMessages - // TODO should we handle whitelisted peers here as we do in headers sync timeout handling? - pfrom->fDisconnect = true; - return error("detected bad peer for initial headers sync, disconnecting %d", pfrom->id); - } + } - if (nCount == 0) { - // Nothing interesting. Stop asking this peers for more headers. - return true; - } + bool fGenesis = pindexBestHeader->GetBlockHash() == chainparams.GetConsensus().hashGenesisBlock; + bool fDevNetGenesis = !chainparams.GetConsensus().hashDevnetGenesisBlock.IsNull() && pindexBestHeader->GetBlockHash() == chainparams.GetConsensus().hashDevnetGenesisBlock; + if (!fGenesis && !fDevNetGenesis && nCount < MAX_HEADERS_RESULTS && GetAdjustedTime() - pindexBestHeader->GetBlockTime() > chainparams.DelayGetHeadersTime()) { + // peer has sent us a HEADERS message below maximum size and we are still quite far from being fully + // synced, this means we probably got a bad peer for initial sync and need to continue with another one. + // By disconnecting we force to start a new iteration of initial headers sync in SendMessages + // TODO should we handle whitelisted peers here as we do in headers sync timeout handling? + pfrom->fDisconnect = true; + return error("detected bad peer for initial headers sync, disconnecting peer=%d", pfrom->id); } bool fCanDirectFetch = CanDirectFetch(chainparams.GetConsensus()); From c0939129c7f8c9967d00b2768a84d139b9ad94b2 Mon Sep 17 00:00:00 2001 From: gladcow Date: Fri, 30 Mar 2018 13:21:43 +0300 Subject: [PATCH 45/51] Remove unused CConnman::FindNode overload --- src/net.cpp | 10 ---------- src/net.h | 1 - 2 files changed, 11 deletions(-) diff --git a/src/net.cpp b/src/net.cpp index ea2a69b38a40..e957b5e3b9a6 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -335,16 +335,6 @@ CNode* CConnman::FindNode(const CService& addr) return NULL; } -//TODO: This is used in only one place in main, and should be removed -CNode* CConnman::FindNode(const NodeId& nodeid) -{ - LOCK(cs_vNodes); - BOOST_FOREACH(CNode* pnode, vNodes) - if (pnode->GetId() == nodeid) - return (pnode); - return NULL; -} - bool CConnman::CheckIncomingNonce(uint64_t nonce) { LOCK(cs_vNodes); diff --git a/src/net.h b/src/net.h index ed92583a0835..574192698933 100644 --- a/src/net.h +++ b/src/net.h @@ -418,7 +418,6 @@ class CConnman CNode* FindNode(const CSubNet& subNet); CNode* FindNode(const std::string& addrName); CNode* FindNode(const CService& addr); - CNode* FindNode(const NodeId& id); //TODO: Remove this bool AttemptToEvictConnection(); CNode* ConnectNode(CAddress addrConnect, const char *pszDest = NULL, bool fCountFailure = false); From 89b65bdbb930851c7ffa09cce6d1c62bfeb60ab2 Mon Sep 17 00:00:00 2001 From: gladcow Date: Fri, 30 Mar 2018 15:58:52 +0300 Subject: [PATCH 46/51] Fix typos and comments --- qa/rpc-tests/test_framework/mininode.py | 2 +- src/net.cpp | 1 - src/net_processing.cpp | 2 +- src/version.h | 2 +- 4 files changed, 3 insertions(+), 4 deletions(-) diff --git a/qa/rpc-tests/test_framework/mininode.py b/qa/rpc-tests/test_framework/mininode.py index d0e59cf1f6eb..9e86f9aa4aae 100755 --- a/qa/rpc-tests/test_framework/mininode.py +++ b/qa/rpc-tests/test_framework/mininode.py @@ -41,7 +41,7 @@ import dash_hash BIP0031_VERSION = 60000 -MY_VERSION = 70209 # current SHORT_IDS_BLOCKS_VERSION to support cmpct blocks? +MY_VERSION = 70209 # SHORT_IDS_BLOCKS_VERSION to support cmpct blocks MY_SUBVERSION = b"/python-mininode-tester:0.0.3/" MY_RELAY = 1 # from version 70001 onwards, fRelay should be appended to version messages (BIP37) diff --git a/src/net.cpp b/src/net.cpp index e957b5e3b9a6..73dc6e82272e 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -1807,7 +1807,6 @@ void CConnman::ThreadOpenConnections() if (nANow - addr.nLastTry < 600 && nTries < 30) continue; - // only consider nodes missing relevant services after 40 failed attempts and only if less than half the outbound are up. ServiceFlags nRequiredServices = nRelevantServices; if (nTries >= 40 && nOutbound < (nMaxOutbound >> 1)) { diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 544651aa45cc..1d5ed9258297 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -2485,7 +2485,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr assert(pindexLast); UpdateBlockAvailability(pfrom->GetId(), pindexLast->GetBlockHash()); - if (nCount == MAX_HEADERS_RESULTS && pindexLast) { + if (nCount == MAX_HEADERS_RESULTS) { // Headers message had its maximum size; the peer may have more headers. // TODO: optimize: if pindexLast is an ancestor of chainActive.Tip or pindexBestHeader, continue // from there instead. diff --git a/src/version.h b/src/version.h index 8e7d9f19a208..9686b886391f 100644 --- a/src/version.h +++ b/src/version.h @@ -44,7 +44,7 @@ static const int FEEFILTER_VERSION = 99999; // disable for now (clarify deployme //! DIP0001 was activated in this version static const int DIP0001_PROTOCOL_VERSION = 70208; -//! shord-id-based block download starts with this version +//! short-id-based block download starts with this version static const int SHORT_IDS_BLOCKS_VERSION = 70209; #endif // BITCOIN_VERSION_H From 4e24f73af1954a93f226b248aac05b4835580003 Mon Sep 17 00:00:00 2001 From: Alexander Block Date: Wed, 4 Apr 2018 11:26:08 +0200 Subject: [PATCH 47/51] Fix minor code differences --- src/net_processing.cpp | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 1d5ed9258297..7e3c13b3ae28 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -85,8 +85,10 @@ namespace { * Sources of received blocks, saved to be able to send them reject * messages or ban them when processing happens afterwards. Protected by * cs_main. + * Set mapBlockSource[hash].second to false if the node should not be + * punished if the block is invalid. */ - std::map > mapBlockSource; + std::map> mapBlockSource; /** * Filter for transactions that were recently rejected by @@ -352,7 +354,7 @@ bool MarkBlockAsInFlight(NodeId nodeid, const uint256& hash, const Consensus::Pa MarkBlockAsReceived(hash); std::list::iterator it = state->vBlocksInFlight.insert(state->vBlocksInFlight.end(), - {hash, pindex, pindex != NULL, std::unique_ptr(pit ? new PartiallyDownloadedBlock(&mempool) : NULL)}); + {hash, pindex, pindex != NULL, std::unique_ptr(pit ? new PartiallyDownloadedBlock(&mempool) : NULL)}); state->nBlocksInFlight++; state->nBlocksInFlightValidHeaders += it->fValidatedHeaders; if (state->nBlocksInFlight == 1) { @@ -859,7 +861,7 @@ void PeerLogicValidation::BlockChecked(const CBlock& block, const CValidationSta assert (state.GetRejectCode() < REJECT_INTERNAL); // Blocks are never rejected with internal reject codes CBlockReject reject = {(unsigned char)state.GetRejectCode(), state.GetRejectReason().substr(0, MAX_REJECT_MESSAGE_LENGTH), hash}; State(it->second.first)->rejects.push_back(reject); - if ((nDoS > 0) && (it->second.second)) + if (nDoS > 0 && it->second.second) Misbehaving(it->second.first, nDoS); } } @@ -1103,8 +1105,8 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam else if (inv.type == MSG_CMPCT_BLOCK) { // If a peer is asking for old blocks, we're almost guaranteed - // they wont have a useful mempool to match against a compact block, - // and we dont feel like constructing the object for them, so + // they won't have a useful mempool to match against a compact block, + // and we don't feel like constructing the object for them, so // instead we respond with the full, non-compact block. if (CanDirectFetch(consensusParams) && mi->second->nHeight >= chainActive.Height() - MAX_CMPCTBLOCK_DEPTH) { CBlockHeaderAndShortTxIDs cmpctblock(block); @@ -1810,7 +1812,6 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr } - else if (strCommand == NetMsgType::GETBLOCKTXN) { BlockTransactionsRequest req; @@ -2162,8 +2163,9 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (state.GetRejectCode() < REJECT_INTERNAL) // Never send AcceptToMemoryPool's internal codes over P2P connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::REJECT, strCommand, (unsigned char)state.GetRejectCode(), state.GetRejectReason().substr(0, MAX_REJECT_MESSAGE_LENGTH), inv.hash)); - if (nDoS > 0) + if (nDoS > 0) { Misbehaving(pfrom->GetId(), nDoS); + } } } @@ -2387,7 +2389,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr fBlockRead = true; } } // Don't hold cs_main when we call into ProcessNewBlock - if(fBlockRead) { + if (fBlockRead) { bool fNewBlock = false; // Since we requested this block (it was in mapBlocksInFlight), force it to be processed, // even if it would not be a candidate for new tip (missing previous block, chain not long enough, etc) From 67910590a1c3b39a0b713c960bfdca47e26ab6aa Mon Sep 17 00:00:00 2001 From: Alexander Block Date: Wed, 4 Apr 2018 11:27:45 +0200 Subject: [PATCH 48/51] Don't use rejection cache for corrupted transactions Partly based on https://github.com/bitcoin/bitcoin/pull/8525 --- src/net_processing.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 7e3c13b3ae28..c7b4088274d1 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -2072,8 +2072,10 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // Probably non-standard or insufficient fee/priority LogPrint("mempool", " removed orphan tx %s\n", orphanHash.ToString()); vEraseQueue.push_back(orphanHash); - assert(recentRejects); - recentRejects->insert(orphanHash); + if (!stateDummy.CorruptionPossible()) { + assert(recentRejects); + recentRejects->insert(orphanHash); + } } mempool.check(pcoinsTip); } From fcbfaa6fa5863c443f933e8ed57e3416702ad22c Mon Sep 17 00:00:00 2001 From: Alexander Block Date: Wed, 4 Apr 2018 11:31:09 +0200 Subject: [PATCH 49/51] Backport missed cs_main locking changes Missed from https://github.com/bitcoin/bitcoin/commit/58a215ce8c13b900cf982c39f8ee4879290d1a95 --- src/net_processing.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index c7b4088274d1..592fbcc7e124 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -2176,6 +2176,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr CBlockHeaderAndShortTxIDs cmpctblock; vRecv >> cmpctblock; + { LOCK(cs_main); if (mapBlockIndex.find(cmpctblock.header.hashPrevBlock) == mapBlockIndex.end()) { @@ -2184,14 +2185,17 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), uint256())); return true; } + } const CBlockIndex *pindex = NULL; CValidationState state; if (!ProcessNewBlockHeaders({cmpctblock.header}, state, chainparams, &pindex)) { int nDoS; if (state.IsInvalid(nDoS)) { - if (nDoS > 0) + if (nDoS > 0) { + LOCK(cs_main); Misbehaving(pfrom->GetId(), nDoS); + } LogPrintf("Peer %d sent us invalid header via cmpctblock\n", pfrom->id); return true; } From 3a51596c2e91b5ecf70bcc979d520382d5747848 Mon Sep 17 00:00:00 2001 From: Alexander Block Date: Wed, 4 Apr 2018 11:34:48 +0200 Subject: [PATCH 50/51] Backport missed comments and mapBlockSource.emplace call Missed from two commits: https://github.com/bitcoin/bitcoin/commit/88c35491ab19f9afdf9b3fa9356a072f70ef2f55 https://github.com/bitcoin/bitcoin/commit/7c98ce584ec23bcddcba8cdb33efa6547212f6ef --- src/net_processing.cpp | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 592fbcc7e124..69eed98361ca 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -2391,8 +2391,31 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr invs.push_back(CInv(MSG_BLOCK, resp.blockhash)); connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, invs)); } else { + // Block is either okay, or possibly we received + // READ_STATUS_CHECKBLOCK_FAILED. + // Note that CheckBlock can only fail for one of a few reasons: + // 1. bad-proof-of-work (impossible here, because we've already + // accepted the header) + // 2. merkleroot doesn't match the transactions given (already + // caught in FillBlock with READ_STATUS_FAILED, so + // impossible here) + // 3. the block is otherwise invalid (eg invalid coinbase, + // block is too big, too many legacy sigops, etc). + // So if CheckBlock failed, #3 is the only possibility. + // Under BIP 152, we don't DoS-ban unless proof of work is + // invalid (we don't require all the stateless checks to have + // been run). This is handled below, so just treat this as + // though the block was successfully read, and rely on the + // handling in ProcessNewBlock to ensure the block index is + // updated, reject messages go out, etc. MarkBlockAsReceived(resp.blockhash); // it is now an empty pointer fBlockRead = true; + // mapBlockSource is only used for sending reject messages and DoS scores, + // so the race between here and cs_main in ProcessNewBlock is fine. + // BIP 152 permits peers to relay compact blocks after validating + // the header only; we should not punish peers if the block turns + // out to be invalid. + mapBlockSource.emplace(resp.blockhash, std::make_pair(pfrom->GetId(), false)); } } // Don't hold cs_main when we call into ProcessNewBlock if (fBlockRead) { From 0bf93ce5746a4c82cdee05ebc4e6e92806f5af3a Mon Sep 17 00:00:00 2001 From: UdjinM6 Date: Wed, 4 Apr 2018 15:31:05 +0300 Subject: [PATCH 51/51] Add CheckPeerHeaders() helper and check in (nCount == 0) too --- src/net_processing.cpp | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 69eed98361ca..151e7fd58135 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -1304,6 +1304,24 @@ inline void static SendBlockTransactions(const CBlock& block, const BlockTransac connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::BLOCKTXN, resp)); } +bool static CheckGoodHeadersSyncPeer(CNode* pfrom, const CChainParams& chainparams, int nCount) +{ + AssertLockHeld(cs_main); + + bool fGenesis = pindexBestHeader->GetBlockHash() == chainparams.GetConsensus().hashGenesisBlock; + bool fDevNetGenesis = !chainparams.GetConsensus().hashDevnetGenesisBlock.IsNull() && pindexBestHeader->GetBlockHash() == chainparams.GetConsensus().hashDevnetGenesisBlock; + if (!fGenesis && !fDevNetGenesis && nCount < MAX_HEADERS_RESULTS && GetAdjustedTime() - pindexBestHeader->GetBlockTime() > chainparams.DelayGetHeadersTime()) { + // peer has sent us a HEADERS message below maximum size and we are still quite far from being fully + // synced, this means we probably got a bad peer for initial sync and need to continue with another one. + // By disconnecting we force to start a new iteration of initial headers sync in SendMessages + // TODO should we handle whitelisted peers here as we do in headers sync timeout handling? + pfrom->fDisconnect = true; + return error("detected bad peer for initial headers sync, disconnecting peer=%d", pfrom->id); + } + + return true; +} + bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vRecv, int64_t nTimeReceived, const CChainParams& chainparams, CConnman& connman, const std::atomic& interruptMsgProc) { LogPrint("net", "received: %s (%u bytes) peer=%d\n", SanitizeString(strCommand), vRecv.size(), pfrom->id); @@ -2448,7 +2466,9 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (nCount == 0) { // Nothing interesting. Stop asking this peers for more headers. - return true; + // See if it's a peer with "good" headers though. + LOCK(cs_main); + return CheckGoodHeadersSyncPeer(pfrom, chainparams, nCount); } const CBlockIndex *pindexLast = NULL; @@ -2522,17 +2542,8 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // from there instead. LogPrint("net", "more getheaders (%d) to end to peer=%d (startheight:%d)\n", pindexLast->nHeight, pfrom->id, pfrom->nStartingHeight); connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexLast), uint256())); - } - - bool fGenesis = pindexBestHeader->GetBlockHash() == chainparams.GetConsensus().hashGenesisBlock; - bool fDevNetGenesis = !chainparams.GetConsensus().hashDevnetGenesisBlock.IsNull() && pindexBestHeader->GetBlockHash() == chainparams.GetConsensus().hashDevnetGenesisBlock; - if (!fGenesis && !fDevNetGenesis && nCount < MAX_HEADERS_RESULTS && GetAdjustedTime() - pindexBestHeader->GetBlockTime() > chainparams.DelayGetHeadersTime()) { - // peer has sent us a HEADERS message below maximum size and we are still quite far from being fully - // synced, this means we probably got a bad peer for initial sync and need to continue with another one. - // By disconnecting we force to start a new iteration of initial headers sync in SendMessages - // TODO should we handle whitelisted peers here as we do in headers sync timeout handling? - pfrom->fDisconnect = true; - return error("detected bad peer for initial headers sync, disconnecting peer=%d", pfrom->id); + } else if (!CheckGoodHeadersSyncPeer(pfrom, chainparams, nCount)) { + return false; } bool fCanDirectFetch = CanDirectFetch(chainparams.GetConsensus());