diff --git a/doc/bips.md b/doc/bips.md index 3e9153a25c1d..b01f0f3215b1 100644 --- a/doc/bips.md +++ b/doc/bips.md @@ -28,3 +28,4 @@ BIPs that are implemented by Bitcoin Core (up-to-date up to **v0.13.0**): * [`BIP 130`](https://github.com/bitcoin/bips/blob/master/bip-0130.mediawiki): direct headers announcement is negotiated with peer versions `>=70012` as of **v0.12.0** ([PR 6494](https://github.com/bitcoin/bitcoin/pull/6494)). * [`BIP 133`](https://github.com/bitcoin/bips/blob/master/bip-0133.mediawiki): feefilter messages are respected and sent for peer versions `>=70013` as of **v0.13.0** ([PR 7542](https://github.com/bitcoin/bitcoin/pull/7542)). * [`BIP 147`](https://github.com/bitcoin/bips/blob/master/bip-0147.mediawiki): NULLDUMMY softfork as of **v0.13.1** ([PR 8636](https://github.com/bitcoin/bitcoin/pull/8636) and [PR 8937](https://github.com/bitcoin/bitcoin/pull/8937)). +* [`BIP 152`](https://github.com/bitcoin/bips/blob/master/bip-0152.mediawiki): Compact block transfer and related optimizations are used as of **v0.13.0** ([PR 8068](https://github.com/bitcoin/bitcoin/pull/8068)). diff --git a/qa/pull-tester/rpc-tests.py b/qa/pull-tester/rpc-tests.py index b978f7d5c7f3..e1faaca08977 100755 --- a/qa/pull-tester/rpc-tests.py +++ b/qa/pull-tester/rpc-tests.py @@ -165,6 +165,7 @@ 'rpcnamedargs.py', 'listsinceblock.py', 'p2p-leaktests.py', + 'p2p-compactblocks.py', ] if ENABLE_ZMQ: testScripts.append('zmq_test.py') diff --git a/qa/rpc-tests/p2p-compactblocks.py b/qa/rpc-tests/p2p-compactblocks.py new file mode 100755 index 000000000000..da257b15808f --- /dev/null +++ b/qa/rpc-tests/p2p-compactblocks.py @@ -0,0 +1,847 @@ +#!/usr/bin/env python3 +# Copyright (c) 2016 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +from test_framework.mininode import * +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import * +from test_framework.blocktools import create_block, create_coinbase +from test_framework.siphash import siphash256 +from test_framework.script import CScript, OP_TRUE + +''' +CompactBlocksTest -- test compact blocks (BIP 152, without segwit support, version 1) +''' + +# TestNode: A peer we use to send messages to bitcoind, and store responses. +class TestNode(SingleNodeConnCB): + def __init__(self): + SingleNodeConnCB.__init__(self) + self.last_sendcmpct = [] + self.last_headers = None + self.last_inv = None + self.last_cmpctblock = None + self.block_announced = False + self.last_getdata = None + self.last_getheaders = None + self.last_getblocktxn = None + self.last_block = None + self.last_blocktxn = None + # Store the hashes of blocks we've seen announced. + # This is for synchronizing the p2p message traffic, + # so we can eg wait until a particular block is announced. + self.set_announced_blockhashes = set() + + def on_sendcmpct(self, conn, message): + self.last_sendcmpct.append(message) + + def on_block(self, conn, message): + self.last_block = message + + def on_cmpctblock(self, conn, message): + self.last_cmpctblock = message + self.block_announced = True + self.last_cmpctblock.header_and_shortids.header.calc_sha256() + self.set_announced_blockhashes.add(self.last_cmpctblock.header_and_shortids.header.sha256) + + def on_headers(self, conn, message): + self.last_headers = message + self.block_announced = True + for x in self.last_headers.headers: + x.calc_sha256() + self.set_announced_blockhashes.add(x.sha256) + + def on_inv(self, conn, message): + self.last_inv = message + for x in self.last_inv.inv: + if x.type == 2: + self.block_announced = True + self.set_announced_blockhashes.add(x.hash) + + def on_getdata(self, conn, message): + self.last_getdata = message + + def on_getheaders(self, conn, message): + self.last_getheaders = message + + def on_getblocktxn(self, conn, message): + self.last_getblocktxn = message + + def on_blocktxn(self, conn, message): + self.last_blocktxn = message + + # Requires caller to hold mininode_lock + def received_block_announcement(self): + return self.block_announced + + def clear_block_announcement(self): + with mininode_lock: + self.block_announced = False + self.last_inv = None + self.last_headers = None + self.last_cmpctblock = None + + def get_headers(self, locator, hashstop): + msg = msg_getheaders() + msg.locator.vHave = locator + msg.hashstop = hashstop + self.connection.send_message(msg) + + def send_header_for_blocks(self, new_blocks): + headers_message = msg_headers() + headers_message.headers = [CBlockHeader(b) for b in new_blocks] + self.send_message(headers_message) + + def request_headers_and_sync(self, locator, hashstop=0): + self.clear_block_announcement() + self.get_headers(locator, hashstop) + assert(wait_until(self.received_block_announcement, timeout=30)) + assert(self.received_block_announcement()) + self.clear_block_announcement() + + # Block until a block announcement for a particular block hash is + # received. + def wait_for_block_announcement(self, block_hash, timeout=30): + def received_hash(): + return (block_hash in self.set_announced_blockhashes) + return wait_until(received_hash, timeout=timeout) + +class CompactBlocksTest(BitcoinTestFramework): + def __init__(self): + super().__init__() + self.setup_clean_chain = True + # both nodes has the same version + self.num_nodes = 2 + self.utxos = [] + + def setup_network(self): + self.nodes = [] + + # Start up node0 to be a version 1, pre-segwit node. + self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, + [["-debug", "-logtimemicros=1", "-txindex"], + ["-debug", "-logtimemicros", "-txindex"]]) + connect_nodes(self.nodes[0], 1) + + def build_block_on_tip(self, node): + height = node.getblockcount() + tip = node.getbestblockhash() + mtp = node.getblockheader(tip)['mediantime'] + block = create_block(int(tip, 16), create_coinbase(height + 1), mtp + 1) + block.solve() + return block + + # Create 10 more anyone-can-spend utxo's for testing. + def make_utxos(self): + # Doesn't matter which node we use, just use node0. + block = self.build_block_on_tip(self.nodes[0]) + self.test_node.send_and_ping(msg_block(block)) + assert(int(self.nodes[0].getbestblockhash(), 16) == block.sha256) + self.nodes[0].generate(100) + + total_value = block.vtx[0].vout[0].nValue + out_value = total_value // 10 + tx = CTransaction() + tx.vin.append(CTxIn(COutPoint(block.vtx[0].sha256, 0), b'')) + for i in range(10): + tx.vout.append(CTxOut(out_value, CScript([OP_TRUE]))) + tx.rehash() + + block2 = self.build_block_on_tip(self.nodes[0]) + block2.vtx.append(tx) + block2.hashMerkleRoot = block2.calc_merkle_root() + block2.solve() + self.test_node.send_and_ping(msg_block(block2)) + assert_equal(int(self.nodes[0].getbestblockhash(), 16), block2.sha256) + self.utxos.extend([[tx.sha256, i, out_value] for i in range(10)]) + return + + # Test "sendcmpct" (between peers with the same version): + # - No compact block announcements unless sendcmpct is sent. + # - If sendcmpct is sent with boolean 0, then block announcements are not + # made with compact blocks. + # - If sendcmpct is then sent with boolean 1, then new block announcements + # are made with compact blocks. + # If old_node is passed in, request compact blocks with version=preferred-1 + # and verify that it receives block announcements via compact block. + def test_sendcmpct(self, node, test_node, preferred_version, old_node=None): + # Make sure we get a SENDCMPCT message from our peer + def received_sendcmpct(): + return (len(test_node.last_sendcmpct) > 0) + got_message = wait_until(received_sendcmpct, timeout=30) + assert(received_sendcmpct()) + assert(got_message) + with mininode_lock: + # Check that the first version received is the preferred one + assert_equal(test_node.last_sendcmpct[0].version, preferred_version) + test_node.last_sendcmpct = [] + + tip = int(node.getbestblockhash(), 16) + + def check_announcement_of_new_block(node, peer, predicate): + peer.clear_block_announcement() + block_hash = int(node.generate(1)[0], 16) + peer.wait_for_block_announcement(block_hash, timeout=30) + assert(peer.block_announced) + assert(got_message) + + with mininode_lock: + assert predicate(peer), ( + "block_hash={!r}, cmpctblock={!r}, inv={!r}".format( + block_hash, peer.last_cmpctblock, peer.last_inv)) + + # We shouldn't get any block announcements via cmpctblock yet. + check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is None) + + # Try one more time, this time after requesting headers. + test_node.request_headers_and_sync(locator=[tip]) + check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is None and p.last_inv is not None) + + # Test a few ways of using sendcmpct that should NOT + # result in compact block announcements. + # Before each test, sync the headers chain. + test_node.request_headers_and_sync(locator=[tip]) + + # Now try a SENDCMPCT message with too-high version + sendcmpct = msg_sendcmpct() + sendcmpct.version = preferred_version+1 + sendcmpct.announce = True + test_node.send_and_ping(sendcmpct) + check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is None) + + # Headers sync before next test. + test_node.request_headers_and_sync(locator=[tip]) + + # Now try a SENDCMPCT message with valid version, but announce=False + sendcmpct.version = preferred_version + sendcmpct.announce = False + test_node.send_and_ping(sendcmpct) + check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is None) + + # Headers sync before next test. + test_node.request_headers_and_sync(locator=[tip]) + + # Finally, try a SENDCMPCT message with announce=True + sendcmpct.version = preferred_version + sendcmpct.announce = True + test_node.send_and_ping(sendcmpct) + check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is not None) + + # Try one more time (no headers sync should be needed!) + check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is not None) + + # Try one more time, after turning on sendheaders + test_node.send_and_ping(msg_sendheaders()) + check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is not None) + + # Now turn off announcements + sendcmpct.version = preferred_version + sendcmpct.announce = False + test_node.send_and_ping(sendcmpct) + check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is None and p.last_headers is not None) + + # This code should be enabled after increasing cmctblk version + #if old_node is not None: + # Verify that a peer using an older protocol version can receive + # announcements from this node. + #sendcmpct.version = preferred_version-1 + #sendcmpct.announce = True + #old_node.send_and_ping(sendcmpct) + # Header sync + #old_node.request_headers_and_sync(locator=[tip]) + #check_announcement_of_new_block(node, old_node, lambda p: p.last_cmpctblock is not None) + + # This test actually causes bitcoind to (reasonably!) disconnect us, so do this last. + def test_invalid_cmpctblock_message(self): + self.nodes[0].generate(101) + block = self.build_block_on_tip(self.nodes[0]) + + cmpct_block = P2PHeaderAndShortIDs() + cmpct_block.header = CBlockHeader(block) + cmpct_block.prefilled_txn_length = 1 + # This index will be too high + prefilled_txn = PrefilledTransaction(1, block.vtx[0]) + cmpct_block.prefilled_txn = [prefilled_txn] + self.test_node.send_and_ping(msg_cmpctblock(cmpct_block)) + assert(int(self.nodes[0].getbestblockhash(), 16) == block.hashPrevBlock) + + # Compare the generated shortids to what we expect based on BIP 152, given + # bitcoind's choice of nonce. + def test_compactblock_construction(self, node, test_node, version): + # Generate a bunch of transactions. + node.generate(101) + num_transactions = 25 + address = node.getnewaddress() + + for i in range(num_transactions): + txid = node.sendtoaddress(address, 0.1) + hex_tx = node.gettransaction(txid)["hex"] + tx = FromHex(CTransaction(), hex_tx) + + # Wait until we've seen the block announcement for the resulting tip + tip = int(node.getbestblockhash(), 16) + assert(test_node.wait_for_block_announcement(tip)) + + # Make sure we will receive a fast-announce compact block + self.request_cb_announcements(test_node, node, version) + + # Now mine a block, and look at the resulting compact block. + test_node.clear_block_announcement() + block_hash = int(node.generate(1)[0], 16) + + # Store the raw block in our internal format. + block = FromHex(CBlock(), node.getblock("%02x" % block_hash, False)) + [tx.calc_sha256() for tx in block.vtx] + block.rehash() + + # Wait until the block was announced (via compact blocks) + wait_until(test_node.received_block_announcement, timeout=30) + assert(test_node.received_block_announcement()) + + # Now fetch and check the compact block + header_and_shortids = None + with mininode_lock: + assert(test_node.last_cmpctblock is not None) + # Convert the on-the-wire representation to absolute indexes + header_and_shortids = HeaderAndShortIDs(test_node.last_cmpctblock.header_and_shortids) + self.check_compactblock_construction_from_block(version, header_and_shortids, block_hash, block) + + # Now fetch the compact block using a normal non-announce getdata + with mininode_lock: + test_node.clear_block_announcement() + inv = CInv(20, block_hash) # 20 == "CompactBlock" + test_node.send_message(msg_getdata([inv])) + + wait_until(test_node.received_block_announcement, timeout=30) + assert(test_node.received_block_announcement()) + + # Now fetch and check the compact block + header_and_shortids = None + with mininode_lock: + assert(test_node.last_cmpctblock is not None) + # Convert the on-the-wire representation to absolute indexes + header_and_shortids = HeaderAndShortIDs(test_node.last_cmpctblock.header_and_shortids) + self.check_compactblock_construction_from_block(version, header_and_shortids, block_hash, block) + + def check_compactblock_construction_from_block(self, version, header_and_shortids, block_hash, block): + # Check that we got the right block! + header_and_shortids.header.calc_sha256() + assert_equal(header_and_shortids.header.sha256, block_hash) + + # Make sure the prefilled_txn appears to have included the coinbase + assert(len(header_and_shortids.prefilled_txn) >= 1) + assert_equal(header_and_shortids.prefilled_txn[0].index, 0) + + # Check that all prefilled_txn entries match what's in the block. + for entry in header_and_shortids.prefilled_txn: + entry.tx.calc_sha256() + # This checks the tx agree + assert_equal(entry.tx.sha256, block.vtx[entry.index].sha256) + + # Check that the cmpctblock message announced all the transactions. + assert_equal(len(header_and_shortids.prefilled_txn) + len(header_and_shortids.shortids), len(block.vtx)) + + # And now check that all the shortids are as expected as well. + # Determine the siphash keys to use. + [k0, k1] = header_and_shortids.get_siphash_keys() + + index = 0 + while index < len(block.vtx): + if (len(header_and_shortids.prefilled_txn) > 0 and + header_and_shortids.prefilled_txn[0].index == index): + # Already checked prefilled transactions above + header_and_shortids.prefilled_txn.pop(0) + else: + tx_hash = block.vtx[index].sha256 + shortid = calculate_shortid(k0, k1, tx_hash) + assert_equal(shortid, header_and_shortids.shortids[0]) + header_and_shortids.shortids.pop(0) + index += 1 + + # Test that bitcoind requests compact blocks when we announce new blocks + # via header or inv, and that responding to getblocktxn causes the block + # to be successfully reconstructed. + def test_compactblock_requests(self, node, test_node): + # Try announcing a block with an inv or header, expect a compactblock + # request + for announce in ["inv", "header"]: + block = self.build_block_on_tip(node) + with mininode_lock: + test_node.last_getdata = None + + if announce == "inv": + test_node.send_message(msg_inv([CInv(2, block.sha256)])) + success = wait_until(lambda: test_node.last_getheaders is not None, timeout=30) + assert(success) + test_node.send_header_for_blocks([block]) + else: + test_node.send_header_for_blocks([block]) + success = wait_until(lambda: test_node.last_getdata is not None, timeout=30) + assert(success) + assert_equal(len(test_node.last_getdata.inv), 1) + assert_equal(test_node.last_getdata.inv[0].type, 20) + assert_equal(test_node.last_getdata.inv[0].hash, block.sha256) + + # Send back a compactblock message that omits the coinbase + comp_block = HeaderAndShortIDs() + comp_block.header = CBlockHeader(block) + comp_block.nonce = 0 + [k0, k1] = comp_block.get_siphash_keys() + comp_block.shortids = [ + calculate_shortid(k0, k1, block.vtx[0].sha256) ] + test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) + assert_equal(int(node.getbestblockhash(), 16), block.hashPrevBlock) + # Expect a getblocktxn message. + with mininode_lock: + assert(test_node.last_getblocktxn is not None) + absolute_indexes = test_node.last_getblocktxn.block_txn_request.to_absolute() + assert_equal(absolute_indexes, [0]) # should be a coinbase request + + # Send the coinbase, and verify that the tip advances. + msg = msg_blocktxn() + msg.block_transactions.blockhash = block.sha256 + msg.block_transactions.transactions = [block.vtx[0]] + test_node.send_and_ping(msg) + assert_equal(int(node.getbestblockhash(), 16), block.sha256) + + # Create a chain of transactions from given utxo, and add to a new block. + def build_block_with_transactions(self, node, utxo, num_transactions): + block = self.build_block_on_tip(node) + + for i in range(num_transactions): + tx = CTransaction() + tx.vin.append(CTxIn(COutPoint(utxo[0], utxo[1]), b'')) + tx.vout.append(CTxOut(utxo[2] - 1000, CScript([OP_TRUE]))) + tx.rehash() + utxo = [tx.sha256, 0, tx.vout[0].nValue] + block.vtx.append(tx) + + block.hashMerkleRoot = block.calc_merkle_root() + block.solve() + return block + + # Test that we only receive getblocktxn requests for transactions that the + # node needs, and that responding to them causes the block to be + # reconstructed. + def test_getblocktxn_requests(self, node, test_node, version): + + def test_getblocktxn_response(compact_block, peer, expected_result): + msg = msg_cmpctblock(compact_block.to_p2p()) + peer.send_and_ping(msg) + with mininode_lock: + assert(peer.last_getblocktxn is not None) + absolute_indexes = peer.last_getblocktxn.block_txn_request.to_absolute() + assert_equal(absolute_indexes, expected_result) + + def test_tip_after_message(node, peer, msg, tip): + peer.send_and_ping(msg) + assert_equal(int(node.getbestblockhash(), 16), tip) + + # First try announcing compactblocks that won't reconstruct, and verify + # that we receive getblocktxn messages back. + utxo = self.utxos.pop(0) + + block = self.build_block_with_transactions(node, utxo, 5) + self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) + comp_block = HeaderAndShortIDs() + comp_block.initialize_from_block(block) + + test_getblocktxn_response(comp_block, test_node, [1, 2, 3, 4, 5]) + + msg_bt = msg_blocktxn() + msg_bt.block_transactions = BlockTransactions(block.sha256, block.vtx[1:]) + test_tip_after_message(node, test_node, msg_bt, block.sha256) + + utxo = self.utxos.pop(0) + block = self.build_block_with_transactions(node, utxo, 5) + self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) + + # Now try interspersing the prefilled transactions + comp_block.initialize_from_block(block, prefill_list=[0, 1, 5]) + test_getblocktxn_response(comp_block, test_node, [2, 3, 4]) + msg_bt.block_transactions = BlockTransactions(block.sha256, block.vtx[2:5]) + test_tip_after_message(node, test_node, msg_bt, block.sha256) + + # Now try giving one transaction ahead of time. + utxo = self.utxos.pop(0) + block = self.build_block_with_transactions(node, utxo, 5) + self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) + test_node.send_and_ping(msg_tx(block.vtx[1])) + assert(block.vtx[1].hash in node.getrawmempool()) + + # Prefill 4 out of the 6 transactions, and verify that only the one + # that was not in the mempool is requested. + comp_block.initialize_from_block(block, prefill_list=[0, 2, 3, 4]) + test_getblocktxn_response(comp_block, test_node, [5]) + + msg_bt.block_transactions = BlockTransactions(block.sha256, [block.vtx[5]]) + test_tip_after_message(node, test_node, msg_bt, block.sha256) + + # Now provide all transactions to the node before the block is + # announced and verify reconstruction happens immediately. + utxo = self.utxos.pop(0) + block = self.build_block_with_transactions(node, utxo, 10) + self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) + for tx in block.vtx[1:]: + test_node.send_message(msg_tx(tx)) + test_node.sync_with_ping() + # Make sure all transactions were accepted. + mempool = node.getrawmempool() + for tx in block.vtx[1:]: + assert(tx.hash in mempool) + + # Clear out last request. + with mininode_lock: + test_node.last_getblocktxn = None + + # Send compact block + comp_block.initialize_from_block(block, prefill_list=[0]) + test_tip_after_message(node, test_node, msg_cmpctblock(comp_block.to_p2p()), block.sha256) + with mininode_lock: + # Shouldn't have gotten a request for any transaction + assert(test_node.last_getblocktxn is None) + + # Incorrectly responding to a getblocktxn shouldn't cause the block to be + # permanently failed. + def test_incorrect_blocktxn_response(self, node, test_node, version): + if (len(self.utxos) == 0): + self.make_utxos() + utxo = self.utxos.pop(0) + + block = self.build_block_with_transactions(node, utxo, 10) + self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) + # Relay the first 5 transactions from the block in advance + for tx in block.vtx[1:6]: + test_node.send_message(msg_tx(tx)) + test_node.sync_with_ping() + # Make sure all transactions were accepted. + mempool = node.getrawmempool() + for tx in block.vtx[1:6]: + assert(tx.hash in mempool) + + # Send compact block + comp_block = HeaderAndShortIDs() + comp_block.initialize_from_block(block, prefill_list=[0]) + test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) + absolute_indexes = [] + with mininode_lock: + assert(test_node.last_getblocktxn is not None) + absolute_indexes = test_node.last_getblocktxn.block_txn_request.to_absolute() + assert_equal(absolute_indexes, [6, 7, 8, 9, 10]) + + # Now give an incorrect response. + # Note that it's possible for bitcoind to be smart enough to know we're + # lying, since it could check to see if the shortid matches what we're + # sending, and eg disconnect us for misbehavior. If that behavior + # change were made, we could just modify this test by having a + # different peer provide the block further down, so that we're still + # verifying that the block isn't marked bad permanently. This is good + # enough for now. + msg = msg_blocktxn() + msg.block_transactions = BlockTransactions(block.sha256, [block.vtx[5]] + block.vtx[7:]) + test_node.send_and_ping(msg) + + # Tip should not have updated + assert_equal(int(node.getbestblockhash(), 16), block.hashPrevBlock) + + # We should receive a getdata request + success = wait_until(lambda: test_node.last_getdata is not None, timeout=10) + assert(success) + assert_equal(len(test_node.last_getdata.inv), 1) + assert(test_node.last_getdata.inv[0].type == 2) + assert_equal(test_node.last_getdata.inv[0].hash, block.sha256) + + # Deliver the block + test_node.send_and_ping(msg_block(block)) + assert_equal(int(node.getbestblockhash(), 16), block.sha256) + + def test_getblocktxn_handler(self, node, test_node, version): + # bitcoind will not send blocktxn responses for blocks whose height is + # more than 10 blocks deep. + MAX_GETBLOCKTXN_DEPTH = 10 + chain_height = node.getblockcount() + current_height = chain_height + while (current_height >= chain_height - MAX_GETBLOCKTXN_DEPTH): + block_hash = node.getblockhash(current_height) + block = FromHex(CBlock(), node.getblock(block_hash, False)) + + msg = msg_getblocktxn() + msg.block_txn_request = BlockTransactionsRequest(int(block_hash, 16), []) + num_to_request = random.randint(1, len(block.vtx)) + msg.block_txn_request.from_absolute(sorted(random.sample(range(len(block.vtx)), num_to_request))) + test_node.send_message(msg) + success = wait_until(lambda: test_node.last_blocktxn is not None, timeout=10) + assert(success) + + [tx.calc_sha256() for tx in block.vtx] + with mininode_lock: + assert_equal(test_node.last_blocktxn.block_transactions.blockhash, int(block_hash, 16)) + all_indices = msg.block_txn_request.to_absolute() + for index in all_indices: + tx = test_node.last_blocktxn.block_transactions.transactions.pop(0) + tx.calc_sha256() + assert_equal(tx.sha256, block.vtx[index].sha256) + test_node.last_blocktxn = None + current_height -= 1 + + # Next request should send a full block response, as we're past the + # allowed depth for a blocktxn response. + block_hash = node.getblockhash(current_height) + msg.block_txn_request = BlockTransactionsRequest(int(block_hash, 16), [0]) + with mininode_lock: + test_node.last_block = None + test_node.last_blocktxn = None + test_node.send_and_ping(msg) + with mininode_lock: + test_node.last_block.block.calc_sha256() + assert_equal(test_node.last_block.block.sha256, int(block_hash, 16)) + assert_equal(test_node.last_blocktxn, None) + + def test_compactblocks_not_at_tip(self, node, test_node): + # Test that requesting old compactblocks doesn't work. + MAX_CMPCTBLOCK_DEPTH = 5 + new_blocks = [] + for i in range(MAX_CMPCTBLOCK_DEPTH + 1): + test_node.clear_block_announcement() + new_blocks.append(node.generate(1)[0]) + wait_until(test_node.received_block_announcement, timeout=30) + + test_node.clear_block_announcement() + test_node.send_message(msg_getdata([CInv(20, int(new_blocks[0], 16))])) + success = wait_until(lambda: test_node.last_cmpctblock is not None, timeout=30) + assert(success) + + test_node.clear_block_announcement() + node.generate(1) + wait_until(test_node.received_block_announcement, timeout=30) + test_node.clear_block_announcement() + with mininode_lock: + test_node.last_block = None + test_node.send_message(msg_getdata([CInv(20, int(new_blocks[0], 16))])) + success = wait_until(lambda: test_node.last_block is not None, timeout=30) + assert(success) + with mininode_lock: + test_node.last_block.block.calc_sha256() + assert_equal(test_node.last_block.block.sha256, int(new_blocks[0], 16)) + + # Generate an old compactblock, and verify that it's not accepted. + cur_height = node.getblockcount() + hashPrevBlock = int(node.getblockhash(cur_height-5), 16) + block = self.build_block_on_tip(node) + block.hashPrevBlock = hashPrevBlock + block.solve() + + comp_block = HeaderAndShortIDs() + comp_block.initialize_from_block(block) + test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) + + tips = node.getchaintips() + found = False + for x in tips: + if x["hash"] == block.hash: + assert_equal(x["status"], "headers-only") + found = True + break + assert(found) + + # Requesting this block via getblocktxn should silently fail + # (to avoid fingerprinting attacks). + msg = msg_getblocktxn() + msg.block_txn_request = BlockTransactionsRequest(block.sha256, [0]) + with mininode_lock: + test_node.last_blocktxn = None + test_node.send_and_ping(msg) + with mininode_lock: + assert(test_node.last_blocktxn is None) + + def test_end_to_end_block_relay(self, node, listeners): + utxo = self.utxos.pop(0) + + block = self.build_block_with_transactions(node, utxo, 10) + + [l.clear_block_announcement() for l in listeners] + + node.submitblock(ToHex(block)) + + for l in listeners: + wait_until(lambda: l.received_block_announcement(), timeout=30) + with mininode_lock: + for l in listeners: + assert(l.last_cmpctblock is not None) + l.last_cmpctblock.header_and_shortids.header.calc_sha256() + assert_equal(l.last_cmpctblock.header_and_shortids.header.sha256, block.sha256) + + # Test that we don't get disconnected if we relay a compact block with valid header, + # but invalid transactions. + def test_invalid_tx_in_compactblock(self, node, test_node): + assert(len(self.utxos)) + utxo = self.utxos[0] + + block = self.build_block_with_transactions(node, utxo, 5) + del block.vtx[3] + block.hashMerkleRoot = block.calc_merkle_root() + block.solve() + + # Now send the compact block with all transactions prefilled, and + # verify that we don't get disconnected. + comp_block = HeaderAndShortIDs() + comp_block.initialize_from_block(block, prefill_list=[0, 1, 2, 3, 4]) + msg = msg_cmpctblock(comp_block.to_p2p()) + test_node.send_and_ping(msg) + + # Check that the tip didn't advance + assert(int(node.getbestblockhash(), 16) is not block.sha256) + test_node.sync_with_ping() + + # Helper for enabling cb announcements + # Send the sendcmpct request and sync headers + def request_cb_announcements(self, peer, node, version): + tip = node.getbestblockhash() + peer.get_headers(locator=[int(tip, 16)], hashstop=0) + + msg = msg_sendcmpct() + msg.version = version + msg.announce = True + peer.send_and_ping(msg) + + def test_compactblock_reconstruction_multiple_peers(self, node, stalling_peer, delivery_peer): + assert(len(self.utxos)) + + def announce_cmpct_block(node, peer): + utxo = self.utxos.pop(0) + block = self.build_block_with_transactions(node, utxo, 5) + + cmpct_block = HeaderAndShortIDs() + cmpct_block.initialize_from_block(block) + msg = msg_cmpctblock(cmpct_block.to_p2p()) + peer.send_and_ping(msg) + with mininode_lock: + assert(peer.last_getblocktxn is not None) + return block, cmpct_block + + block, cmpct_block = announce_cmpct_block(node, stalling_peer) + + for tx in block.vtx[1:]: + delivery_peer.send_message(msg_tx(tx)) + delivery_peer.sync_with_ping() + mempool = node.getrawmempool() + for tx in block.vtx[1:]: + assert(tx.hash in mempool) + + delivery_peer.send_and_ping(msg_cmpctblock(cmpct_block.to_p2p())) + assert_equal(int(node.getbestblockhash(), 16), block.sha256) + + self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) + + # Now test that delivering an invalid compact block won't break relay + + block, cmpct_block = announce_cmpct_block(node, stalling_peer) + for tx in block.vtx[1:]: + delivery_peer.send_message(msg_tx(tx)) + delivery_peer.sync_with_ping() + + cmpct_block.prefilled_txn[0].tx = CTxIn() + + delivery_peer.send_and_ping(msg_cmpctblock(cmpct_block.to_p2p())) + assert(int(node.getbestblockhash(), 16) != block.sha256) + + msg = msg_blocktxn() + msg.block_transactions.blockhash = block.sha256 + msg.block_transactions.transactions = block.vtx[1:] + stalling_peer.send_and_ping(msg) + assert_equal(int(node.getbestblockhash(), 16), block.sha256) + + def run_test(self): + # Setup the p2p connections and start up the network thread. + self.test_node = TestNode() + self.second_node = TestNode() + self.old_node = TestNode() + + connections = [] + connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], self.test_node)) + connections.append(NodeConn('127.0.0.1', p2p_port(1), self.nodes[1], + self.second_node, services=NODE_NETWORK)) + connections.append(NodeConn('127.0.0.1', p2p_port(1), self.nodes[1], + self.old_node, services=NODE_NETWORK)) + self.test_node.add_connection(connections[0]) + self.second_node.add_connection(connections[1]) + self.old_node.add_connection(connections[2]) + + NetworkThread().start() # Start up network handling in another thread + + # Test logic begins here + self.test_node.wait_for_verack() + + # We will need UTXOs to construct transactions in later tests. + self.make_utxos() + + print("Running tests:") + + print("\tTesting SENDCMPCT p2p message... ") + self.test_sendcmpct(self.nodes[0], self.test_node, 1) + sync_blocks(self.nodes) + self.test_sendcmpct(self.nodes[1], self.second_node, 1) + sync_blocks(self.nodes) + + print("\tTesting compactblock construction...") + self.test_compactblock_construction(self.nodes[0], self.test_node, 1) + sync_blocks(self.nodes) + self.test_compactblock_construction(self.nodes[1], self.second_node, 1) + sync_blocks(self.nodes) + + print("\tTesting compactblock requests... ") + self.test_compactblock_requests(self.nodes[0], self.test_node) + sync_blocks(self.nodes) + self.test_compactblock_requests(self.nodes[1], self.second_node) + sync_blocks(self.nodes) + + print("\tTesting getblocktxn requests...") + self.test_getblocktxn_requests(self.nodes[0], self.test_node, 1) + sync_blocks(self.nodes) + self.test_getblocktxn_requests(self.nodes[1], self.second_node, 1) + sync_blocks(self.nodes) + + print("\tTesting getblocktxn handler...") + self.test_getblocktxn_handler(self.nodes[0], self.test_node, 1) + sync_blocks(self.nodes) + self.test_getblocktxn_handler(self.nodes[1], self.second_node, 1) + self.test_getblocktxn_handler(self.nodes[1], self.old_node, 1) + sync_blocks(self.nodes) + + print("\tTesting compactblock requests/announcements not at chain tip...") + self.test_compactblocks_not_at_tip(self.nodes[0], self.test_node) + sync_blocks(self.nodes) + self.test_compactblocks_not_at_tip(self.nodes[1], self.second_node) + self.test_compactblocks_not_at_tip(self.nodes[1], self.old_node) + sync_blocks(self.nodes) + + print("\tTesting handling of incorrect blocktxn responses...") + self.test_incorrect_blocktxn_response(self.nodes[0], self.test_node, 1) + sync_blocks(self.nodes) + self.test_incorrect_blocktxn_response(self.nodes[1], self.second_node, 1) + sync_blocks(self.nodes) + + # End-to-end block relay tests + print("\tTesting end-to-end block relay...") + self.request_cb_announcements(self.test_node, self.nodes[0], 1) + self.request_cb_announcements(self.old_node, self.nodes[1], 1) + self.request_cb_announcements(self.second_node, self.nodes[1], 1) + self.test_end_to_end_block_relay(self.nodes[0], [self.second_node, self.test_node, self.old_node]) + self.test_end_to_end_block_relay(self.nodes[1], [self.second_node, self.test_node, self.old_node]) + + print("\tTesting handling of invalid compact blocks...") + self.test_invalid_tx_in_compactblock(self.nodes[0], self.test_node) + self.test_invalid_tx_in_compactblock(self.nodes[1], self.second_node) + self.test_invalid_tx_in_compactblock(self.nodes[1], self.old_node) + + print("\tTesting reconstructing compact blocks from all peers...") + self.test_compactblock_reconstruction_multiple_peers(self.nodes[1], self.second_node, self.old_node) + sync_blocks(self.nodes) + + print("\tTesting invalid index in cmpctblock message...") + self.test_invalid_cmpctblock_message() + + +if __name__ == '__main__': + CompactBlocksTest().main() diff --git a/qa/rpc-tests/test_framework/mininode.py b/qa/rpc-tests/test_framework/mininode.py index b32b8dcfae7e..9e86f9aa4aae 100755 --- a/qa/rpc-tests/test_framework/mininode.py +++ b/qa/rpc-tests/test_framework/mininode.py @@ -36,11 +36,12 @@ from threading import Thread import logging import copy +from test_framework.siphash import siphash256 import dash_hash BIP0031_VERSION = 60000 -MY_VERSION = 70208 # current MIN_PEER_PROTO_VERSION +MY_VERSION = 70209 # SHORT_IDS_BLOCKS_VERSION to support cmpct blocks MY_SUBVERSION = b"/python-mininode-tester:0.0.3/" MY_RELAY = 1 # from version 70001 onwards, fRelay should be appended to version messages (BIP37) @@ -54,7 +55,7 @@ NODE_BLOOM = (1 << 2) # Keep our own socket map for asyncore, so that we can track disconnects -# ourselves (to workaround an issue with closing an asyncore socket when +# ourselves (to workaround an issue with closing an asyncore socket when # using select) mininode_socket_map = dict() @@ -241,7 +242,9 @@ class CInv(object): typemap = { 0: "Error", 1: "TX", - 2: "Block"} + 2: "Block", + 20: "CompactBlock" + } def __init__(self, t=0, h=0): self.type = t @@ -604,6 +607,181 @@ def __repr__(self): % (len(self.vchMsg), len(self.vchSig)) +class PrefilledTransaction(object): + def __init__(self, index=0, tx = None): + self.index = index + self.tx = tx + + def deserialize(self, f): + self.index = deser_compact_size(f) + self.tx = CTransaction() + self.tx.deserialize(f) + + def serialize(self): + r = b"" + r += ser_compact_size(self.index) + r += self.tx.serialize() + return r + + def __repr__(self): + return "PrefilledTransaction(index=%d, tx=%s)" % (self.index, repr(self.tx)) + +# This is what we send on the wire, in a cmpctblock message. +class P2PHeaderAndShortIDs(object): + def __init__(self): + self.header = CBlockHeader() + self.nonce = 0 + self.shortids_length = 0 + self.shortids = [] + self.prefilled_txn_length = 0 + self.prefilled_txn = [] + + def deserialize(self, f): + self.header.deserialize(f) + self.nonce = struct.unpack(" + +#define MIN_TRANSACTION_SIZE (::GetSerializeSize(CTransaction(), SER_NETWORK, PROTOCOL_VERSION)) + +CBlockHeaderAndShortTxIDs::CBlockHeaderAndShortTxIDs(const CBlock& block) : + nonce(GetRand(std::numeric_limits::max())), + shorttxids(block.vtx.size() - 1), prefilledtxn(1), header(block) { + FillShortTxIDSelector(); + //TODO: Use our mempool prior to block acceptance to predictively fill more than just the coinbase + prefilledtxn[0] = {0, block.vtx[0]}; + for (size_t i = 1; i < block.vtx.size(); i++) { + const CTransaction& tx = *block.vtx[i]; + shorttxids[i - 1] = GetShortID(tx.GetHash()); + } +} + +void CBlockHeaderAndShortTxIDs::FillShortTxIDSelector() const { + CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); + stream << header << nonce; + CSHA256 hasher; + hasher.Write((unsigned char*)&(*stream.begin()), stream.end() - stream.begin()); + uint256 shorttxidhash; + hasher.Finalize(shorttxidhash.begin()); + shorttxidk0 = shorttxidhash.GetUint64(0); + shorttxidk1 = shorttxidhash.GetUint64(1); +} + +uint64_t CBlockHeaderAndShortTxIDs::GetShortID(const uint256& txhash) const { + static_assert(SHORTTXIDS_LENGTH == 6, "shorttxids calculation assumes 6-byte shorttxids"); + return SipHashUint256(shorttxidk0, shorttxidk1, txhash) & 0xffffffffffffL; +} + + + +ReadStatus PartiallyDownloadedBlock::InitData(const CBlockHeaderAndShortTxIDs& cmpctblock, const std::vector>& extra_txn) { + if (cmpctblock.header.IsNull() || (cmpctblock.shorttxids.empty() && cmpctblock.prefilledtxn.empty())) + return READ_STATUS_INVALID; + if (cmpctblock.shorttxids.size() + cmpctblock.prefilledtxn.size() > MaxBlockSize(true) / MIN_TRANSACTION_SIZE) + return READ_STATUS_INVALID; + + assert(header.IsNull() && txn_available.empty()); + header = cmpctblock.header; + txn_available.resize(cmpctblock.BlockTxCount()); + + int32_t lastprefilledindex = -1; + for (size_t i = 0; i < cmpctblock.prefilledtxn.size(); i++) { + if (cmpctblock.prefilledtxn[i].tx->IsNull()) + return READ_STATUS_INVALID; + + lastprefilledindex += cmpctblock.prefilledtxn[i].index + 1; //index is a uint16_t, so can't overflow here + if (lastprefilledindex > std::numeric_limits::max()) + return READ_STATUS_INVALID; + if ((uint32_t)lastprefilledindex > cmpctblock.shorttxids.size() + i) { + // If we are inserting a tx at an index greater than our full list of shorttxids + // plus the number of prefilled txn we've inserted, then we have txn for which we + // have neither a prefilled txn or a shorttxid! + return READ_STATUS_INVALID; + } + txn_available[lastprefilledindex] = cmpctblock.prefilledtxn[i].tx; + } + prefilled_count = cmpctblock.prefilledtxn.size(); + + // Calculate map of txids -> positions and check mempool to see what we have (or don't) + // Because well-formed cmpctblock messages will have a (relatively) uniform distribution + // of short IDs, any highly-uneven distribution of elements can be safely treated as a + // READ_STATUS_FAILED. + std::unordered_map shorttxids(cmpctblock.shorttxids.size()); + uint16_t index_offset = 0; + for (size_t i = 0; i < cmpctblock.shorttxids.size(); i++) { + while (txn_available[i + index_offset]) + index_offset++; + shorttxids[cmpctblock.shorttxids[i]] = i + index_offset; + // To determine the chance that the number of entries in a bucket exceeds N, + // we use the fact that the number of elements in a single bucket is + // binomially distributed (with n = the number of shorttxids S, and p = + // 1 / the number of buckets), that in the worst case the number of buckets is + // equal to S (due to std::unordered_map having a default load factor of 1.0), + // and that the chance for any bucket to exceed N elements is at most + // buckets * (the chance that any given bucket is above N elements). + // Thus: P(max_elements_per_bucket > N) <= S * (1 - cdf(binomial(n=S,p=1/S), N)). + // If we assume blocks of up to 16000, allowing 12 elements per bucket should + // only fail once per ~1 million block transfers (per peer and connection). + if (shorttxids.bucket_size(shorttxids.bucket(cmpctblock.shorttxids[i])) > 12) + return READ_STATUS_FAILED; + } + // TODO: in the shortid-collision case, we should instead request both transactions + // which collided. Falling back to full-block-request here is overkill. + if (shorttxids.size() != cmpctblock.shorttxids.size()) + return READ_STATUS_FAILED; // Short ID collision + + std::vector have_txn(txn_available.size()); + { + LOCK(pool->cs); + const std::vector >& vTxHashes = pool->vTxHashes; + for (size_t i = 0; i < vTxHashes.size(); i++) { + uint64_t shortid = cmpctblock.GetShortID(vTxHashes[i].first); + std::unordered_map::iterator idit = shorttxids.find(shortid); + if (idit != shorttxids.end()) { + if (!have_txn[idit->second]) { + txn_available[idit->second] = vTxHashes[i].second->GetSharedTx(); + have_txn[idit->second] = true; + mempool_count++; + } else { + // If we find two mempool txn that match the short id, just request it. + // This should be rare enough that the extra bandwidth doesn't matter, + // but eating a round-trip due to FillBlock failure would be annoying + if (txn_available[idit->second]) { + txn_available[idit->second].reset(); + mempool_count--; + } + } + } + // Though ideally we'd continue scanning for the two-txn-match-shortid case, + // the performance win of an early exit here is too good to pass up and worth + // the extra risk. + if (mempool_count == shorttxids.size()) + break; + } + } + + for (size_t i = 0; i < extra_txn.size(); i++) { + uint64_t shortid = cmpctblock.GetShortID(extra_txn[i].first); + std::unordered_map::iterator idit = shorttxids.find(shortid); + if (idit != shorttxids.end()) { + if (!have_txn[idit->second]) { + txn_available[idit->second] = extra_txn[i].second; + have_txn[idit->second] = true; + mempool_count++; + extra_count++; + } else { + // If we find two mempool/extra txn that match the short id, just + // request it. + // This should be rare enough that the extra bandwidth doesn't matter, + // but eating a round-trip due to FillBlock failure would be annoying + // Note that we dont want duplication between extra_txn and mempool to + // trigger this case, so we compare hashes first + if (txn_available[idit->second] && + txn_available[idit->second]->GetHash() != extra_txn[i].second->GetHash()) { + txn_available[idit->second].reset(); + mempool_count--; + extra_count--; + } + } + } + // Though ideally we'd continue scanning for the two-txn-match-shortid case, + // the performance win of an early exit here is too good to pass up and worth + // the extra risk. + if (mempool_count == shorttxids.size()) + break; + } + + LogPrint("cmpctblock", "Initialized PartiallyDownloadedBlock for block %s using a cmpctblock of size %lu\n", cmpctblock.header.GetHash().ToString(), GetSerializeSize(cmpctblock, SER_NETWORK, PROTOCOL_VERSION)); + + return READ_STATUS_OK; +} + +bool PartiallyDownloadedBlock::IsTxAvailable(size_t index) const { + assert(!header.IsNull()); + assert(index < txn_available.size()); + return txn_available[index] ? true : false; +} + +ReadStatus PartiallyDownloadedBlock::FillBlock(CBlock& block, const std::vector& vtx_missing) { + assert(!header.IsNull()); + uint256 hash = header.GetHash(); + block = header; + block.vtx.resize(txn_available.size()); + + size_t tx_missing_offset = 0; + for (size_t i = 0; i < txn_available.size(); i++) { + if (!txn_available[i]) { + if (vtx_missing.size() <= tx_missing_offset) + return READ_STATUS_INVALID; + block.vtx[i] = vtx_missing[tx_missing_offset++]; + } else + block.vtx[i] = std::move(txn_available[i]); + } + + // Make sure we can't call FillBlock again. + header.SetNull(); + txn_available.clear(); + + if (vtx_missing.size() != tx_missing_offset) + return READ_STATUS_INVALID; + + CValidationState state; + if (!CheckBlock(block, state, Params().GetConsensus())) { + // TODO: We really want to just check merkle tree manually here, + // but that is expensive, and CheckBlock caches a block's + // "checked-status" (in the CBlock?). CBlock should be able to + // check its own merkle root and cache that check. + if (state.CorruptionPossible()) + return READ_STATUS_FAILED; // Possible Short ID collision + return READ_STATUS_CHECKBLOCK_FAILED; + } + + LogPrint("cmpctblock", "Successfully reconstructed block %s with %lu txn prefilled, %lu txn from mempool (incl at least %lu from extra pool) and %lu txn requested\n", hash.ToString(), prefilled_count, mempool_count, extra_count, vtx_missing.size()); + if (vtx_missing.size() < 5) { + for (const auto& tx : vtx_missing) + LogPrint("cmpctblock", "Reconstructed block %s required tx %s\n", hash.ToString(), tx->GetHash().ToString()); + } + + return READ_STATUS_OK; +} diff --git a/src/blockencodings.h b/src/blockencodings.h new file mode 100644 index 000000000000..a768a753f594 --- /dev/null +++ b/src/blockencodings.h @@ -0,0 +1,209 @@ +// Copyright (c) 2016 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_BLOCK_ENCODINGS_H +#define BITCOIN_BLOCK_ENCODINGS_H + +#include "primitives/block.h" + +#include + +class CTxMemPool; + +// Dumb helper to handle CTransaction compression at serialize-time +struct TransactionCompressor { +private: + CTransactionRef& tx; +public: + TransactionCompressor(CTransactionRef& txIn) : tx(txIn) {} + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) { + READWRITE(tx); //TODO: Compress tx encoding + } +}; + +class BlockTransactionsRequest { +public: + // A BlockTransactionsRequest message + uint256 blockhash; + std::vector indexes; + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) { + READWRITE(blockhash); + uint64_t indexes_size = (uint64_t)indexes.size(); + READWRITE(COMPACTSIZE(indexes_size)); + if (ser_action.ForRead()) { + size_t i = 0; + while (indexes.size() < indexes_size) { + indexes.resize(std::min((uint64_t)(1000 + indexes.size()), indexes_size)); + for (; i < indexes.size(); i++) { + uint64_t index = 0; + READWRITE(COMPACTSIZE(index)); + if (index > std::numeric_limits::max()) + throw std::ios_base::failure("index overflowed 16 bits"); + indexes[i] = index; + } + } + + uint16_t offset = 0; + for (size_t j = 0; j < indexes.size(); j++) { + if (uint64_t(indexes[j]) + uint64_t(offset) > std::numeric_limits::max()) + throw std::ios_base::failure("indexes overflowed 16 bits"); + indexes[j] = indexes[j] + offset; + offset = indexes[j] + 1; + } + } else { + for (size_t i = 0; i < indexes.size(); i++) { + uint64_t index = indexes[i] - (i == 0 ? 0 : (indexes[i - 1] + 1)); + READWRITE(COMPACTSIZE(index)); + } + } + } +}; + +class BlockTransactions { +public: + // A BlockTransactions message + uint256 blockhash; + std::vector txn; + + BlockTransactions() {} + BlockTransactions(const BlockTransactionsRequest& req) : + blockhash(req.blockhash), txn(req.indexes.size()) {} + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) { + READWRITE(blockhash); + uint64_t txn_size = (uint64_t)txn.size(); + READWRITE(COMPACTSIZE(txn_size)); + if (ser_action.ForRead()) { + size_t i = 0; + while (txn.size() < txn_size) { + txn.resize(std::min((uint64_t)(1000 + txn.size()), txn_size)); + for (; i < txn.size(); i++) + READWRITE(REF(TransactionCompressor(txn[i]))); + } + } else { + for (size_t i = 0; i < txn.size(); i++) + READWRITE(REF(TransactionCompressor(txn[i]))); + } + } +}; + +// Dumb serialization/storage-helper for CBlockHeaderAndShortTxIDs and PartiallyDownloadedBlock +struct PrefilledTransaction { + // Used as an offset since last prefilled tx in CBlockHeaderAndShortTxIDs, + // as a proper transaction-in-block-index in PartiallyDownloadedBlock + uint16_t index; + CTransactionRef tx; + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) { + uint64_t idx = index; + READWRITE(COMPACTSIZE(idx)); + if (idx > std::numeric_limits::max()) + throw std::ios_base::failure("index overflowed 16-bits"); + index = idx; + READWRITE(REF(TransactionCompressor(tx))); + } +}; + +typedef enum ReadStatus_t +{ + READ_STATUS_OK, + READ_STATUS_INVALID, // Invalid object, peer is sending bogus crap + READ_STATUS_FAILED, // Failed to process object + READ_STATUS_CHECKBLOCK_FAILED, // Used only by FillBlock to indicate a + // failure in CheckBlock. +} ReadStatus; + +class CBlockHeaderAndShortTxIDs { +private: + mutable uint64_t shorttxidk0, shorttxidk1; + uint64_t nonce; + + void FillShortTxIDSelector() const; + + friend class PartiallyDownloadedBlock; + + static const int SHORTTXIDS_LENGTH = 6; +protected: + std::vector shorttxids; + std::vector prefilledtxn; + +public: + CBlockHeader header; + + // Dummy for deserialization + CBlockHeaderAndShortTxIDs() {} + + CBlockHeaderAndShortTxIDs(const CBlock& block); + + uint64_t GetShortID(const uint256& txhash) const; + + size_t BlockTxCount() const { return shorttxids.size() + prefilledtxn.size(); } + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) { + READWRITE(header); + READWRITE(nonce); + + uint64_t shorttxids_size = (uint64_t)shorttxids.size(); + READWRITE(COMPACTSIZE(shorttxids_size)); + if (ser_action.ForRead()) { + size_t i = 0; + while (shorttxids.size() < shorttxids_size) { + shorttxids.resize(std::min((uint64_t)(1000 + shorttxids.size()), shorttxids_size)); + for (; i < shorttxids.size(); i++) { + uint32_t lsb = 0; uint16_t msb = 0; + READWRITE(lsb); + READWRITE(msb); + shorttxids[i] = (uint64_t(msb) << 32) | uint64_t(lsb); + static_assert(SHORTTXIDS_LENGTH == 6, "shorttxids serialization assumes 6-byte shorttxids"); + } + } + } else { + for (size_t i = 0; i < shorttxids.size(); i++) { + uint32_t lsb = shorttxids[i] & 0xffffffff; + uint16_t msb = (shorttxids[i] >> 32) & 0xffff; + READWRITE(lsb); + READWRITE(msb); + } + } + + READWRITE(prefilledtxn); + + if (ser_action.ForRead()) + FillShortTxIDSelector(); + } +}; + +class PartiallyDownloadedBlock { +protected: + std::vector txn_available; + size_t prefilled_count = 0, mempool_count = 0, extra_count = 0; + CTxMemPool* pool; +public: + CBlockHeader header; + PartiallyDownloadedBlock(CTxMemPool* poolIn) : pool(poolIn) {} + + // extra_txn is a list of extra transactions to look at, in form + ReadStatus InitData(const CBlockHeaderAndShortTxIDs& cmpctblock, const std::vector>& extra_txn); + bool IsTxAvailable(size_t index) const; + ReadStatus FillBlock(CBlock& block, const std::vector& vtx_missing); +}; + +#endif diff --git a/src/init.cpp b/src/init.cpp index 6d99d0429835..b1bd93f38156 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -411,6 +411,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-maxorphantx=", strprintf(_("Keep at most unconnectable transactions in memory (default: %u)"), DEFAULT_MAX_ORPHAN_TRANSACTIONS)); strUsage += HelpMessageOpt("-maxmempool=", strprintf(_("Keep the transaction memory pool below megabytes (default: %u)"), DEFAULT_MAX_MEMPOOL_SIZE)); strUsage += HelpMessageOpt("-mempoolexpiry=", strprintf(_("Do not keep transactions in the mempool longer than hours (default: %u)"), DEFAULT_MEMPOOL_EXPIRY)); + strUsage += HelpMessageOpt("-blockreconstructionextratxn=", strprintf(_("Extra transactions to keep in memory for compact block reconstructions (default: %u)"), DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN)); strUsage += HelpMessageOpt("-par=", strprintf(_("Set the number of script verification threads (%u to %d, 0 = auto, <0 = leave that many cores free, default: %d)"), -GetNumCores(), MAX_SCRIPTCHECK_THREADS, DEFAULT_SCRIPTCHECK_THREADS)); #ifndef WIN32 @@ -509,7 +510,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-limitdescendantsize=", strprintf("Do not accept transactions if any ancestor would have more than kilobytes of in-mempool descendants (default: %u).", DEFAULT_DESCENDANT_SIZE_LIMIT)); strUsage += HelpMessageOpt("-bip9params=deployment:start:end", "Use given start/end times for specified BIP9 deployment (regtest-only)"); } - std::string debugCategories = "addrman, alert, bench, coindb, db, http, leveldb, libevent, lock, mempool, mempoolrej, net, proxy, prune, rand, reindex, rpc, selectcoins, tor, zmq, " + std::string debugCategories = "addrman, alert, bench, cmpctblock, coindb, db, http, leveldb, libevent, lock, mempool, mempoolrej, net, proxy, prune, rand, reindex, rpc, selectcoins, tor, zmq, " "dash (or specifically: gobject, instantsend, keepass, masternode, mnpayments, mnsync, privatesend, spork)"; // Don't translate these and qt below if (mode == HMM_BITCOIN_QT) debugCategories += ", qt"; @@ -1197,8 +1198,8 @@ bool AppInitParameterInteraction() dustRelayFee = CFeeRate(n); } - fRequireStandard = !GetBoolArg("-acceptnonstdtxn", !Params().RequireStandard()); - if (Params().RequireStandard() && !fRequireStandard) + fRequireStandard = !GetBoolArg("-acceptnonstdtxn", !chainparams.RequireStandard()); + if (chainparams.RequireStandard() && !fRequireStandard) return InitError(strprintf("acceptnonstdtxn is not currently supported for %s chain", chainparams.NetworkIDString())); nBytesPerSigOp = GetArg("-bytespersigop", nBytesPerSigOp); diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 770e27b733ff..151e7fd58135 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -8,6 +8,7 @@ #include "alert.h" #include "addrman.h" #include "arith_uint256.h" +#include "blockencodings.h" #include "chainparams.h" #include "consensus/validation.h" #include "hash.h" @@ -70,6 +71,9 @@ std::map mapOrphanTransactions GUARDED_BY(cs_main); std::map::iterator, IteratorComparator>> mapOrphanTransactionsByPrev GUARDED_BY(cs_main); void EraseOrphansFor(NodeId peer) EXCLUSIVE_LOCKS_REQUIRED(cs_main); +static size_t vExtraTxnForCompactIt = 0; +static std::vector> vExtraTxnForCompact GUARDED_BY(cs_main); + static const uint64_t RANDOMIZER_ID_ADDRESS_RELAY = 0x3cac0035b5866b90ULL; // SHA256("main address relay")[0:8] // Internal stuff @@ -81,8 +85,10 @@ namespace { * Sources of received blocks, saved to be able to send them reject * messages or ban them when processing happens afterwards. Protected by * cs_main. + * Set mapBlockSource[hash].second to false if the node should not be + * punished if the block is invalid. */ - std::map mapBlockSource; + std::map> mapBlockSource; /** * Filter for transactions that were recently rejected by @@ -111,10 +117,14 @@ namespace { struct QueuedBlock { uint256 hash; const CBlockIndex* pindex; //!< Optional. - bool fValidatedHeaders; //!< Whether this block has validated headers at the time of request. + bool fValidatedHeaders; //!< Whether this block has validated headers at the time of request. + std::unique_ptr partialBlock; //!< Optional, used for CMPCTBLOCK downloads }; std::map::iterator> > mapBlocksInFlight; + /** Stack of nodes which we have set to announce using compact blocks */ + std::list lNodesAnnouncingHeaderAndIDs; + /** Number of preferable block download peers. */ int nPreferredDownload = 0; @@ -185,6 +195,15 @@ struct CNodeState { bool fPreferredDownload; //! Whether this peer wants invs or headers (when possible) for block announcements. bool fPreferHeaders; + //! Whether this peer wants invs or cmpctblocks (when possible) for block announcements. + bool fPreferHeaderAndIDs; + //! Whether this peer will send us cmpctblocks if we request them + bool fProvidesHeaderAndIDs; + /** + * If we've announced last version to this peer: whether the peer sends last version in cmpctblocks/blocktxns, + * otherwise: whether this peer sends non-last version in cmpctblocks/blocktxns. + */ + bool fSupportsDesiredCmpctVersion; CNodeState(CAddress addrIn, std::string addrNameIn) : address(addrIn), name(addrNameIn) { fCurrentlyConnected = false; @@ -203,6 +222,9 @@ struct CNodeState { nBlocksInFlightValidHeaders = 0; fPreferredDownload = false; fPreferHeaders = false; + fPreferHeaderAndIDs = false; + fProvidesHeaderAndIDs = false; + fSupportsDesiredCmpctVersion = false; } }; @@ -291,6 +313,7 @@ void FinalizeNode(NodeId nodeid, bool& fUpdateConnectionTime) { // Requires cs_main. // Returns a bool indicating whether we requested this block. +// Also used if a block was /not/ received and timed out or started with another peer bool MarkBlockAsReceived(const uint256& hash) { std::map::iterator> >::iterator itInFlight = mapBlocksInFlight.find(hash); if (itInFlight != mapBlocksInFlight.end()) { @@ -314,17 +337,26 @@ bool MarkBlockAsReceived(const uint256& hash) { } // Requires cs_main. -void MarkBlockAsInFlight(NodeId nodeid, const uint256& hash, const Consensus::Params& consensusParams, const CBlockIndex *pindex = NULL) { +// returns false, still setting pit, if the block was already in flight from the same peer +// pit will only be valid as long as the same cs_main lock is being held +bool MarkBlockAsInFlight(NodeId nodeid, const uint256& hash, const Consensus::Params& consensusParams, const CBlockIndex *pindex = NULL, std::list::iterator **pit = NULL) { CNodeState *state = State(nodeid); assert(state != NULL); + // Short-circuit most stuff in case its from the same node + std::map::iterator> >::iterator itInFlight = mapBlocksInFlight.find(hash); + if (itInFlight != mapBlocksInFlight.end() && itInFlight->second.first == nodeid) { + *pit = &itInFlight->second.second; + return false; + } + // Make sure it's not listed somewhere already. MarkBlockAsReceived(hash); - QueuedBlock newentry = {hash, pindex, pindex != NULL}; - std::list::iterator it = state->vBlocksInFlight.insert(state->vBlocksInFlight.end(), newentry); + std::list::iterator it = state->vBlocksInFlight.insert(state->vBlocksInFlight.end(), + {hash, pindex, pindex != NULL, std::unique_ptr(pit ? new PartiallyDownloadedBlock(&mempool) : NULL)}); state->nBlocksInFlight++; - state->nBlocksInFlightValidHeaders += newentry.fValidatedHeaders; + state->nBlocksInFlightValidHeaders += it->fValidatedHeaders; if (state->nBlocksInFlight == 1) { // We're starting a block download (batch) from this peer. state->nDownloadingSince = GetTimeMicros(); @@ -332,7 +364,10 @@ void MarkBlockAsInFlight(NodeId nodeid, const uint256& hash, const Consensus::Pa if (state->nBlocksInFlightValidHeaders == 1 && pindex != NULL) { nPeersWithValidatedDownloads++; } - mapBlocksInFlight[hash] = std::make_pair(nodeid, it); + itInFlight = mapBlocksInFlight.insert(std::make_pair(hash, std::make_pair(nodeid, it))).first; + if (pit) + *pit = &itInFlight->second.second; + return true; } /** Check whether the last unknown block a peer advertised is not yet known. */ @@ -368,6 +403,41 @@ void UpdateBlockAvailability(NodeId nodeid, const uint256 &hash) { } } +void MaybeSetPeerAsAnnouncingHeaderAndIDs(NodeId nodeid, CConnman& connman) { + AssertLockHeld(cs_main); + CNodeState* nodestate = State(nodeid); + if (!nodestate || !nodestate->fSupportsDesiredCmpctVersion) { + // Never ask from peers who can't provide desired version. + return; + } + if (nodestate->fProvidesHeaderAndIDs) { + for (std::list::iterator it = lNodesAnnouncingHeaderAndIDs.begin(); it != lNodesAnnouncingHeaderAndIDs.end(); it++) { + if (*it == nodeid) { + lNodesAnnouncingHeaderAndIDs.erase(it); + lNodesAnnouncingHeaderAndIDs.push_back(nodeid); + return; + } + } + connman.ForNode(nodeid, [&connman](CNode* pfrom){ + bool fAnnounceUsingCMPCTBLOCK = false; + uint64_t nCMPCTBLOCKVersion = 1; + if (lNodesAnnouncingHeaderAndIDs.size() >= 3) { + // As per BIP152, we only get 3 of our peers to announce + // blocks using compact encodings. + connman.ForNode(lNodesAnnouncingHeaderAndIDs.front(), [&connman, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion](CNode* pnodeStop){ + connman.PushMessage(pnodeStop, CNetMsgMaker(pnodeStop->GetSendVersion()).Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); + return true; + }); + lNodesAnnouncingHeaderAndIDs.pop_front(); + } + fAnnounceUsingCMPCTBLOCK = true; + connman.PushMessage(pfrom, CNetMsgMaker(pfrom->GetSendVersion()).Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); + lNodesAnnouncingHeaderAndIDs.push_back(pfrom->GetId()); + return true; + }); + } +} + // Requires cs_main bool CanDirectFetch(const Consensus::Params &consensusParams) { @@ -525,6 +595,17 @@ void UnregisterNodeSignals(CNodeSignals& nodeSignals) // mapOrphanTransactions // +void AddToCompactExtraTransactions(const CTransactionRef& tx) +{ + size_t max_extra_txn = GetArg("-blockreconstructionextratxn", DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN); + if (max_extra_txn <= 0) + return; + if (!vExtraTxnForCompact.size()) + vExtraTxnForCompact.resize(max_extra_txn); + vExtraTxnForCompact[vExtraTxnForCompactIt] = std::make_pair(tx->GetHash(), tx); + vExtraTxnForCompactIt = (vExtraTxnForCompactIt + 1) % max_extra_txn; +} + bool AddOrphanTx(const CTransactionRef& tx, NodeId peer) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { const uint256& hash = tx->GetHash(); @@ -551,6 +632,8 @@ bool AddOrphanTx(const CTransactionRef& tx, NodeId peer) EXCLUSIVE_LOCKS_REQUIRE mapOrphanTransactionsByPrev[txin.prevout].insert(ret.first); } + AddToCompactExtraTransactions(tx); + LogPrint("mempool", "stored orphan tx %s (mapsz %u outsz %u)\n", hash.ToString(), mapOrphanTransactions.size(), mapOrphanTransactionsByPrev.size()); return true; @@ -691,6 +774,50 @@ void PeerLogicValidation::SyncTransaction(const CTransaction& tx, const CBlockIn } } +static CCriticalSection cs_most_recent_block; +static std::shared_ptr most_recent_block; +static std::shared_ptr most_recent_compact_block; +static uint256 most_recent_block_hash; + +void PeerLogicValidation::NewPoWValidBlock(const CBlockIndex *pindex, const std::shared_ptr& pblock) { + std::shared_ptr pcmpctblock = std::make_shared (*pblock); + const CNetMsgMaker msgMaker(PROTOCOL_VERSION); + + LOCK(cs_main); + + static int nHighestFastAnnounce = 0; + if (pindex->nHeight <= nHighestFastAnnounce) + return; + nHighestFastAnnounce = pindex->nHeight; + + uint256 hashBlock(pblock->GetHash()); + + { + LOCK(cs_most_recent_block); + most_recent_block_hash = hashBlock; + most_recent_block = pblock; + most_recent_compact_block = pcmpctblock; + } + + connman->ForEachNode([this, &pcmpctblock, pindex, &msgMaker, &hashBlock](CNode* pnode) { + // TODO: Avoid the repeated-serialization here + if (pnode->fDisconnect) + return; + ProcessBlockAvailability(pnode->GetId()); + CNodeState &state = *State(pnode->GetId()); + // If the peer has, or we announced to them the previous block already, + // but we don't think they have this one, go ahead and announce it + if (state.fPreferHeaderAndIDs && + !PeerHasHeader(&state, pindex) && PeerHasHeader(&state, pindex->pprev)) { + + LogPrint("net", "%s sending header-and-ids %s to peer=%d\n", "PeerLogicValidation::NewPoWValidBlock", + hashBlock.ToString(), pnode->id); + connman->PushMessage(pnode, msgMaker.Make(NetMsgType::CMPCTBLOCK, *pcmpctblock)); + state.pindexBestHeaderSent = pindex; + } + }); +} + void PeerLogicValidation::UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) { const int nNewHeight = pindexNew->nHeight; connman->SetBestHeight(nNewHeight); @@ -726,16 +853,29 @@ void PeerLogicValidation::BlockChecked(const CBlock& block, const CValidationSta LOCK(cs_main); const uint256 hash(block.GetHash()); - std::map::iterator it = mapBlockSource.find(hash); + std::map >::iterator it = mapBlockSource.find(hash); int nDoS = 0; if (state.IsInvalid(nDoS)) { - if (it != mapBlockSource.end() && State(it->second)) { + if (it != mapBlockSource.end() && State(it->second.first)) { assert (state.GetRejectCode() < REJECT_INTERNAL); // Blocks are never rejected with internal reject codes CBlockReject reject = {(unsigned char)state.GetRejectCode(), state.GetRejectReason().substr(0, MAX_REJECT_MESSAGE_LENGTH), hash}; - State(it->second)->rejects.push_back(reject); - if (nDoS > 0) - Misbehaving(it->second, nDoS); + State(it->second.first)->rejects.push_back(reject); + if (nDoS > 0 && it->second.second) + Misbehaving(it->second.first, nDoS); + } + } + // Check that: + // 1. The block is valid + // 2. We're not in initial block download + // 3. This is currently the best block we're aware of. We haven't updated + // the tip yet so we have no way to check this directly here. Instead we + // just check that there are currently no other blocks in flight. + else if (state.IsValid() && + !IsInitialBlockDownload() && + mapBlocksInFlight.count(hash) == mapBlocksInFlight.size()) { + if (it != mapBlockSource.end()) { + MaybeSetPeerAsAnnouncingHeaderAndIDs(it->second.first, *connman); } } if (it != mapBlockSource.end()) @@ -880,12 +1020,27 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam it++; - if (inv.type == MSG_BLOCK || inv.type == MSG_FILTERED_BLOCK) + if (inv.type == MSG_BLOCK || inv.type == MSG_FILTERED_BLOCK || inv.type == MSG_CMPCT_BLOCK) { bool send = false; BlockMap::iterator mi = mapBlockIndex.find(inv.hash); if (mi != mapBlockIndex.end()) { + if (mi->second->nChainTx && !mi->second->IsValid(BLOCK_VALID_SCRIPTS) && + mi->second->IsValid(BLOCK_VALID_TREE)) { + // If we have the block and all of its parents, but have not yet validated it, + // we might be in the middle of connecting it (ie in the unlock of cs_main + // before ActivateBestChain but after AcceptBlock). + // In this case, we need to run ActivateBestChain prior to checking the relay + // conditions below. + std::shared_ptr a_recent_block; + { + LOCK(cs_most_recent_block); + a_recent_block = most_recent_block; + } + CValidationState dummy; + ActivateBestChain(dummy, Params(), a_recent_block); + } if (chainActive.Contains(mi->second)) { send = true; } else { @@ -921,7 +1076,7 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam assert(!"cannot load block from disk"); if (inv.type == MSG_BLOCK) connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::BLOCK, block)); - else // MSG_FILTERED_BLOCK) + else if (inv.type == MSG_FILTERED_BLOCK) { bool sendMerkleBlock = false; CMerkleBlock merkleBlock; @@ -947,6 +1102,18 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam // else // no response } + else if (inv.type == MSG_CMPCT_BLOCK) + { + // If a peer is asking for old blocks, we're almost guaranteed + // they won't have a useful mempool to match against a compact block, + // and we don't feel like constructing the object for them, so + // instead we respond with the full, non-compact block. + if (CanDirectFetch(consensusParams) && mi->second->nHeight >= chainActive.Height() - MAX_CMPCTBLOCK_DEPTH) { + CBlockHeaderAndShortTxIDs cmpctblock(block); + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::CMPCTBLOCK, cmpctblock)); + } else + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::BLOCK, block)); + } // Trigger the peer node to send a getblocks request for the next batch of inventory if (inv.hash == pfrom->hashContinue) @@ -1102,7 +1269,7 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam // Track requests for our stuff. GetMainSignals().Inventory(inv.hash); - if (inv.type == MSG_BLOCK || inv.type == MSG_FILTERED_BLOCK) + if (inv.type == MSG_BLOCK || inv.type == MSG_FILTERED_BLOCK || inv.type == MSG_CMPCT_BLOCK) break; } } @@ -1121,6 +1288,40 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam } } +inline void static SendBlockTransactions(const CBlock& block, const BlockTransactionsRequest& req, CNode* pfrom, CConnman& connman) { + BlockTransactions resp(req); + for (size_t i = 0; i < req.indexes.size(); i++) { + if (req.indexes[i] >= block.vtx.size()) { + LOCK(cs_main); + Misbehaving(pfrom->GetId(), 100); + LogPrintf("Peer %d sent us a getblocktxn with out-of-bounds tx indices", pfrom->id); + return; + } + resp.txn[i] = block.vtx[req.indexes[i]]; + } + LOCK(cs_main); + CNetMsgMaker msgMaker(pfrom->GetSendVersion()); + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::BLOCKTXN, resp)); +} + +bool static CheckGoodHeadersSyncPeer(CNode* pfrom, const CChainParams& chainparams, int nCount) +{ + AssertLockHeld(cs_main); + + bool fGenesis = pindexBestHeader->GetBlockHash() == chainparams.GetConsensus().hashGenesisBlock; + bool fDevNetGenesis = !chainparams.GetConsensus().hashDevnetGenesisBlock.IsNull() && pindexBestHeader->GetBlockHash() == chainparams.GetConsensus().hashDevnetGenesisBlock; + if (!fGenesis && !fDevNetGenesis && nCount < MAX_HEADERS_RESULTS && GetAdjustedTime() - pindexBestHeader->GetBlockTime() > chainparams.DelayGetHeadersTime()) { + // peer has sent us a HEADERS message below maximum size and we are still quite far from being fully + // synced, this means we probably got a bad peer for initial sync and need to continue with another one. + // By disconnecting we force to start a new iteration of initial headers sync in SendMessages + // TODO should we handle whitelisted peers here as we do in headers sync timeout handling? + pfrom->fDisconnect = true; + return error("detected bad peer for initial headers sync, disconnecting peer=%d", pfrom->id); + } + + return true; +} + bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vRecv, int64_t nTimeReceived, const CChainParams& chainparams, CConnman& connman, const std::atomic& interruptMsgProc) { LogPrint("net", "received: %s (%u bytes) peer=%d\n", SanitizeString(strCommand), vRecv.size(), pfrom->id); @@ -1372,6 +1573,18 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // nodes) connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::SENDHEADERS)); } + + if (pfrom->nVersion >= SHORT_IDS_BLOCKS_VERSION) { + // Tell our peer we are willing to provide version-1 cmpctblocks + // However, we do not request new block announcements using + // cmpctblock messages. + // We send this to non-NODE NETWORK peers as well, because + // they may wish to request compact blocks from us + bool fAnnounceUsingCMPCTBLOCK = false; + uint64_t nCMPCTBLOCKVersion = 1; + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); + } + pfrom->fSuccessfullyConnected = true; } @@ -1436,6 +1649,20 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr } + else if (strCommand == NetMsgType::SENDCMPCT) + { + bool fAnnounceUsingCMPCTBLOCK = false; + uint64_t nCMPCTBLOCKVersion = 1; + vRecv >> fAnnounceUsingCMPCTBLOCK >> nCMPCTBLOCKVersion; + if (nCMPCTBLOCKVersion == 1) { + LOCK(cs_main); + State(pfrom->GetId())->fProvidesHeaderAndIDs = true; + State(pfrom->GetId())->fPreferHeaderAndIDs = fAnnounceUsingCMPCTBLOCK; + State(pfrom->GetId())->fSupportsDesiredCmpctVersion = true; + } + } + + else if (strCommand == NetMsgType::INV) { std::vector vInv; @@ -1548,6 +1775,23 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr uint256 hashStop; vRecv >> locator >> hashStop; + // We might have announced the currently-being-connected tip using a + // compact block, which resulted in the peer sending a getblocks + // request, which we would otherwise respond to without the new block. + // To avoid this situation we simply verify that we are on our best + // known chain now. This is super overkill, but we handle it better + // for getheaders requests, and there are no known nodes which support + // compact blocks but still use getblocks to request blocks. + { + std::shared_ptr a_recent_block; + { + LOCK(cs_most_recent_block); + a_recent_block = most_recent_block; + } + CValidationState dummy; + ActivateBestChain(dummy, Params(), a_recent_block); + } + LOCK(cs_main); // Find the last block the caller has in the main chain @@ -1586,6 +1830,56 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr } + else if (strCommand == NetMsgType::GETBLOCKTXN) + { + BlockTransactionsRequest req; + vRecv >> req; + + std::shared_ptr recent_block; + { + LOCK(cs_most_recent_block); + if (most_recent_block_hash == req.blockhash) + recent_block = most_recent_block; + // Unlock cs_most_recent_block to avoid cs_main lock inversion + } + if (recent_block) { + SendBlockTransactions(*recent_block, req, pfrom, connman); + return true; + } + + LOCK(cs_main); + + BlockMap::iterator it = mapBlockIndex.find(req.blockhash); + if (it == mapBlockIndex.end() || !(it->second->nStatus & BLOCK_HAVE_DATA)) { + LogPrintf("Peer %d sent us a getblocktxn for a block we don't have", pfrom->id); + return true; + } + + if (it->second->nHeight < chainActive.Height() - MAX_BLOCKTXN_DEPTH) { + // If an older block is requested (should never happen in practice, + // but can happen in tests) send a block response instead of a + // blocktxn response. Sending a full block response instead of a + // small blocktxn response is preferable in the case where a peer + // might maliciously send lots of getblocktxn requests to trigger + // expensive disk reads, because it will require the peer to + // actually receive all the data read from disk over the network. + LogPrint("net", "Peer %d sent us a getblocktxn for a block > %i deep", pfrom->id, MAX_BLOCKTXN_DEPTH); + CInv inv; + inv.type = MSG_BLOCK; + inv.hash = req.blockhash; + pfrom->vRecvGetData.push_back(inv); + ProcessGetData(pfrom, chainparams.GetConsensus(), connman, interruptMsgProc); + return true; + } + + CBlock block; + bool ret = ReadBlockFromDisk(block, it->second, chainparams.GetConsensus()); + assert(ret); + + SendBlockTransactions(block, req, pfrom, connman); + } + + else if (strCommand == NetMsgType::GETHEADERS) { CBlockLocator locator; @@ -1630,6 +1924,14 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // if our peer has chainActive.Tip() (and thus we are sending an empty // headers message). In both cases it's safe to update // pindexBestHeaderSent to be our tip. + // + // It is important that we simply reset the BestHeaderSent value here, + // and not max(BestHeaderSent, newHeaderSent). We might have announced + // the currently-being-connected tip using a compact block, which + // resulted in the peer sending a headers request, which we respond to + // without the new block. By resetting the BestHeaderSent, we ensure we + // will re-announce the new block via headers (or compact blocks again) + // in the SendMessages logic. nodestate->pindexBestHeaderSent = pindex ? pindex : chainActive.Tip(); connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::HEADERS, vHeaders)); } @@ -1714,7 +2016,9 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr mapAlreadyAskedFor.erase(inv.hash); - if (!AlreadyHave(inv) && AcceptToMemoryPool(mempool, state, ptx, true, &fMissingInputs)) { + std::list lRemovedTxn; + + if (!AlreadyHave(inv) && AcceptToMemoryPool(mempool, state, ptx, true, &fMissingInputs, &lRemovedTxn)) { // Process custom txes, this changes AlreadyHave to "true" if (strCommand == NetMsgType::DSTX) { LogPrintf("DSTX -- Masternode transaction accepted, txid=%s, peer=%d\n", @@ -1764,8 +2068,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (setMisbehaving.count(fromPeer)) continue; - if (AcceptToMemoryPool(mempool, stateDummy, porphanTx, true, &fMissingInputs2)) - { + if (AcceptToMemoryPool(mempool, stateDummy, porphanTx, true, &fMissingInputs2, &lRemovedTxn)) { LogPrint("mempool", " accepted orphan tx %s\n", orphanHash.ToString()); connman.RelayTransaction(orphanTx); for (unsigned int i = 0; i < orphanTx.vout.size(); i++) { @@ -1787,8 +2090,10 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // Probably non-standard or insufficient fee/priority LogPrint("mempool", " removed orphan tx %s\n", orphanHash.ToString()); vEraseQueue.push_back(orphanHash); - assert(recentRejects); - recentRejects->insert(orphanHash); + if (!stateDummy.CorruptionPossible()) { + assert(recentRejects); + recentRejects->insert(orphanHash); + } } mempool.check(pcoinsTip); } @@ -1826,8 +2131,13 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr recentRejects->insert(tx.GetHash()); } } else { - assert(recentRejects); - recentRejects->insert(tx.GetHash()); + if (!state.CorruptionPossible()) { + assert(recentRejects); + recentRejects->insert(tx.GetHash()); + if (RecursiveDynamicUsage(*ptx) < 100000) { + AddToCompactExtraTransactions(ptx); + } + } if (strCommand == NetMsgType::TXLOCKREQUEST && !AlreadyHave(inv)) { // i.e. AcceptToMemoryPool failed, probably because it's conflicting @@ -1861,6 +2171,9 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr } } + for (const CTransactionRef& removedTx : lRemovedTxn) + AddToCompactExtraTransactions(removedTx); + int nDoS = 0; if (state.IsInvalid(nDoS)) { @@ -1870,8 +2183,266 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (state.GetRejectCode() < REJECT_INTERNAL) // Never send AcceptToMemoryPool's internal codes over P2P connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::REJECT, strCommand, (unsigned char)state.GetRejectCode(), state.GetRejectReason().substr(0, MAX_REJECT_MESSAGE_LENGTH), inv.hash)); - if (nDoS > 0) + if (nDoS > 0) { Misbehaving(pfrom->GetId(), nDoS); + } + } + } + + else if (strCommand == NetMsgType::CMPCTBLOCK && !fImporting && !fReindex) // Ignore blocks received while importing + { + CBlockHeaderAndShortTxIDs cmpctblock; + vRecv >> cmpctblock; + + { + LOCK(cs_main); + + if (mapBlockIndex.find(cmpctblock.header.hashPrevBlock) == mapBlockIndex.end()) { + // Doesn't connect (or is genesis), instead of DoSing in AcceptBlockHeader, request deeper headers + if (!IsInitialBlockDownload()) + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), uint256())); + return true; + } + } + + const CBlockIndex *pindex = NULL; + CValidationState state; + if (!ProcessNewBlockHeaders({cmpctblock.header}, state, chainparams, &pindex)) { + int nDoS; + if (state.IsInvalid(nDoS)) { + if (nDoS > 0) { + LOCK(cs_main); + Misbehaving(pfrom->GetId(), nDoS); + } + LogPrintf("Peer %d sent us invalid header via cmpctblock\n", pfrom->id); + return true; + } + } + + // When we succeed in decoding a block's txids from a cmpctblock + // message we typically jump to the BLOCKTXN handling code, with a + // dummy (empty) BLOCKTXN message, to re-use the logic there in + // completing processing of the putative block (without cs_main). + bool fProcessBLOCKTXN = false; + CDataStream blockTxnMsg(SER_NETWORK, PROTOCOL_VERSION); + + // If we end up treating this as a plain headers message, call that as well + // without cs_main. + bool fRevertToHeaderProcessing = false; + CDataStream vHeadersMsg(SER_NETWORK, PROTOCOL_VERSION); + + // Keep a CBlock for "optimistic" compactblock reconstructions (see + // below) + std::shared_ptr pblock = std::make_shared(); + bool fBlockReconstructed = false; + + { + LOCK(cs_main); + // If AcceptBlockHeader returned true, it set pindex + assert(pindex); + UpdateBlockAvailability(pfrom->GetId(), pindex->GetBlockHash()); + + std::map::iterator> >::iterator blockInFlightIt = mapBlocksInFlight.find(pindex->GetBlockHash()); + bool fAlreadyInFlight = blockInFlightIt != mapBlocksInFlight.end(); + + if (pindex->nStatus & BLOCK_HAVE_DATA) // Nothing to do here + return true; + + if (pindex->nChainWork <= chainActive.Tip()->nChainWork || // We know something better + pindex->nTx != 0) { // We had this block at some point, but pruned it + if (fAlreadyInFlight) { + // We requested this block for some reason, but our mempool will probably be useless + // so we just grab the block via normal getdata + std::vector vInv(1); + vInv[0] = CInv(MSG_BLOCK, cmpctblock.header.GetHash()); + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); + } + return true; + } + + // If we're not close to tip yet, give up and let parallel block fetch work its magic + if (!fAlreadyInFlight && !CanDirectFetch(chainparams.GetConsensus())) + return true; + + CNodeState *nodestate = State(pfrom->GetId()); + + // We want to be a bit conservative just to be extra careful about DoS + // possibilities in compact block processing... + if (pindex->nHeight <= chainActive.Height() + 2) { + if ((!fAlreadyInFlight && nodestate->nBlocksInFlight < MAX_BLOCKS_IN_TRANSIT_PER_PEER) || + (fAlreadyInFlight && blockInFlightIt->second.first == pfrom->GetId())) { + std::list::iterator *queuedBlockIt = NULL; + if (!MarkBlockAsInFlight(pfrom->GetId(), pindex->GetBlockHash(), chainparams.GetConsensus(), pindex, &queuedBlockIt)) { + if (!(*queuedBlockIt)->partialBlock) + (*queuedBlockIt)->partialBlock.reset(new PartiallyDownloadedBlock(&mempool)); + else { + // The block was already in flight using compact blocks from the same peer + LogPrint("net", "Peer sent us compact block we were already syncing!\n"); + return true; + } + } + + PartiallyDownloadedBlock& partialBlock = *(*queuedBlockIt)->partialBlock; + ReadStatus status = partialBlock.InitData(cmpctblock, vExtraTxnForCompact); + if (status == READ_STATUS_INVALID) { + MarkBlockAsReceived(pindex->GetBlockHash()); // Reset in-flight state in case of whitelist + Misbehaving(pfrom->GetId(), 100); + LogPrintf("Peer %d sent us invalid compact block\n", pfrom->id); + return true; + } else if (status == READ_STATUS_FAILED) { + // Duplicate txindexes, the block is now in-flight, so just request it + std::vector vInv(1); + vInv[0] = CInv(MSG_BLOCK, cmpctblock.header.GetHash()); + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); + return true; + } + + BlockTransactionsRequest req; + for (size_t i = 0; i < cmpctblock.BlockTxCount(); i++) { + if (!partialBlock.IsTxAvailable(i)) + req.indexes.push_back(i); + } + if (req.indexes.empty()) { + // Dirty hack to jump to BLOCKTXN code (TODO: move message handling into their own functions) + BlockTransactions txn; + txn.blockhash = cmpctblock.header.GetHash(); + blockTxnMsg << txn; + fProcessBLOCKTXN = true; + } else { + req.blockhash = pindex->GetBlockHash(); + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETBLOCKTXN, req)); + } + } else { + // This block is either already in flight from a different + // peer, or this peer has too many blocks outstanding to + // download from. + // Optimistically try to reconstruct anyway since we might be + // able to without any round trips. + PartiallyDownloadedBlock tempBlock(&mempool); + ReadStatus status = tempBlock.InitData(cmpctblock, vExtraTxnForCompact); + if (status != READ_STATUS_OK) { + // TODO: don't ignore failures + return true; + } + std::vector dummy; + status = tempBlock.FillBlock(*pblock, dummy); + if (status == READ_STATUS_OK) { + fBlockReconstructed = true; + } + } + } else { + if (fAlreadyInFlight) { + // We requested this block, but its far into the future, so our + // mempool will probably be useless - request the block normally + std::vector vInv(1); + vInv[0] = CInv(MSG_BLOCK, cmpctblock.header.GetHash()); + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); + return true; + } else { + // If this was an announce-cmpctblock, we want the same treatment as a header message + // Dirty hack to process as if it were just a headers message (TODO: move message handling into their own functions) + std::vector headers; + headers.push_back(cmpctblock.header); + vHeadersMsg << headers; + fRevertToHeaderProcessing = true; + } + } + } // cs_main + + if (fProcessBLOCKTXN) + return ProcessMessage(pfrom, NetMsgType::BLOCKTXN, blockTxnMsg, nTimeReceived, chainparams, connman, interruptMsgProc); + + if (fRevertToHeaderProcessing) + return ProcessMessage(pfrom, NetMsgType::HEADERS, vHeadersMsg, nTimeReceived, chainparams, connman, interruptMsgProc); + + if (fBlockReconstructed) { + // If we got here, we were able to optimistically reconstruct a + // block that is in flight from some other peer. + { + LOCK(cs_main); + mapBlockSource.emplace(pblock->GetHash(), std::make_pair(pfrom->GetId(), false)); + } + bool fNewBlock = false; + ProcessNewBlock(chainparams, pblock, true, &fNewBlock); + if (fNewBlock) + pfrom->nLastBlockTime = GetTime(); + + LOCK(cs_main); // hold cs_main for CBlockIndex::IsValid() + if (pindex->IsValid(BLOCK_VALID_TRANSACTIONS)) { + // Clear download state for this block, which is in + // process from some other peer. We do this after calling + // ProcessNewBlock so that a malleated cmpctblock announcement + // can't be used to interfere with block relay. + MarkBlockAsReceived(pblock->GetHash()); + } + } + + } + + else if (strCommand == NetMsgType::BLOCKTXN && !fImporting && !fReindex) // Ignore blocks received while importing + { + BlockTransactions resp; + vRecv >> resp; + + std::shared_ptr pblock = std::make_shared(); + bool fBlockRead = false; + { + LOCK(cs_main); + + std::map::iterator> >::iterator it = mapBlocksInFlight.find(resp.blockhash); + if (it == mapBlocksInFlight.end() || !it->second.second->partialBlock || + it->second.first != pfrom->GetId()) { + LogPrint("net", "Peer %d sent us block transactions for block we weren't expecting\n", pfrom->id); + return true; + } + + PartiallyDownloadedBlock& partialBlock = *it->second.second->partialBlock; + ReadStatus status = partialBlock.FillBlock(*pblock, resp.txn); + if (status == READ_STATUS_INVALID) { + MarkBlockAsReceived(resp.blockhash); // Reset in-flight state in case of whitelist + Misbehaving(pfrom->GetId(), 100); + LogPrintf("Peer %d sent us invalid compact block/non-matching block transactions\n", pfrom->id); + return true; + } else if (status == READ_STATUS_FAILED) { + // Might have collided, fall back to getdata now :( + std::vector invs; + invs.push_back(CInv(MSG_BLOCK, resp.blockhash)); + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, invs)); + } else { + // Block is either okay, or possibly we received + // READ_STATUS_CHECKBLOCK_FAILED. + // Note that CheckBlock can only fail for one of a few reasons: + // 1. bad-proof-of-work (impossible here, because we've already + // accepted the header) + // 2. merkleroot doesn't match the transactions given (already + // caught in FillBlock with READ_STATUS_FAILED, so + // impossible here) + // 3. the block is otherwise invalid (eg invalid coinbase, + // block is too big, too many legacy sigops, etc). + // So if CheckBlock failed, #3 is the only possibility. + // Under BIP 152, we don't DoS-ban unless proof of work is + // invalid (we don't require all the stateless checks to have + // been run). This is handled below, so just treat this as + // though the block was successfully read, and rely on the + // handling in ProcessNewBlock to ensure the block index is + // updated, reject messages go out, etc. + MarkBlockAsReceived(resp.blockhash); // it is now an empty pointer + fBlockRead = true; + // mapBlockSource is only used for sending reject messages and DoS scores, + // so the race between here and cs_main in ProcessNewBlock is fine. + // BIP 152 permits peers to relay compact blocks after validating + // the header only; we should not punish peers if the block turns + // out to be invalid. + mapBlockSource.emplace(resp.blockhash, std::make_pair(pfrom->GetId(), false)); + } + } // Don't hold cs_main when we call into ProcessNewBlock + if (fBlockRead) { + bool fNewBlock = false; + // Since we requested this block (it was in mapBlocksInFlight), force it to be processed, + // even if it would not be a candidate for new tip (missing previous block, chain not long enough, etc) + ProcessNewBlock(chainparams, pblock, true, &fNewBlock); + if (fNewBlock) + pfrom->nLastBlockTime = GetTime(); } } @@ -1893,6 +2464,13 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr ReadCompactSize(vRecv); // ignore tx count; assume it is 0. } + if (nCount == 0) { + // Nothing interesting. Stop asking this peers for more headers. + // See if it's a peer with "good" headers though. + LOCK(cs_main); + return CheckGoodHeadersSyncPeer(pfrom, chainparams, nCount); + } + const CBlockIndex *pindexLast = NULL; { LOCK(cs_main); @@ -1955,29 +2533,17 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr } nodestate->nUnconnectingHeaders = 0; - if (pindexLast) - UpdateBlockAvailability(pfrom->GetId(), pindexLast->GetBlockHash()); + assert(pindexLast); + UpdateBlockAvailability(pfrom->GetId(), pindexLast->GetBlockHash()); - if (nCount == MAX_HEADERS_RESULTS && pindexLast) { + if (nCount == MAX_HEADERS_RESULTS) { // Headers message had its maximum size; the peer may have more headers. // TODO: optimize: if pindexLast is an ancestor of chainActive.Tip or pindexBestHeader, continue // from there instead. LogPrint("net", "more getheaders (%d) to end to peer=%d (startheight:%d)\n", pindexLast->nHeight, pfrom->id, pfrom->nStartingHeight); connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexLast), uint256())); - } else { - if (GetAdjustedTime() - pindexBestHeader->GetBlockTime() > chainparams.DelayGetHeadersTime()) { - // peer has sent us a HEADERS message below maximum size and we are still quite far from being fully - // synced, this means we probably got a bad peer for initial sync and need to continue with another one. - // By disconnecting we force to start a new iteration of initial headers sync in SendMessages - // TODO should we handle whitelisted peers here as we do in headers sync timeout handling? - pfrom->fDisconnect = true; - return error("detected bad peer for initial headers sync, disconnecting %d", pfrom->id); - } - - if (nCount == 0) { - // Nothing interesting. Stop asking this peers for more headers. - return true; - } + } else if (!CheckGoodHeadersSyncPeer(pfrom, chainparams, nCount)) { + return false; } bool fCanDirectFetch = CanDirectFetch(chainparams.GetConsensus()); @@ -2021,6 +2587,10 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr pindexLast->GetBlockHash().ToString(), pindexLast->nHeight); } if (vGetData.size() > 0) { + if (nodestate->fSupportsDesiredCmpctVersion && vGetData.size() == 1 && mapBlocksInFlight.size() == 1 && pindexLast->pprev->IsValid(BLOCK_VALID_CHAIN)) { + // In any case, we want to download using a compact block, not a regular one + vGetData[0] = CInv(MSG_CMPCT_BLOCK, vGetData[0].hash); + } connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vGetData)); } } @@ -2048,7 +2618,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr forceProcessing |= MarkBlockAsReceived(hash); // mapBlockSource is only used for sending reject messages and DoS scores, // so the race between here and cs_main in ProcessNewBlock is fine. - mapBlockSource.emplace(hash, pfrom->GetId()); + mapBlockSource.emplace(hash, std::make_pair(pfrom->GetId(), true)); } bool fNewBlock = false; ProcessNewBlock(chainparams, pblock, forceProcessing, &fNewBlock); @@ -2633,7 +3203,9 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic& interr // add all to the inv queue. LOCK(pto->cs_inventory); std::vector vHeaders; - bool fRevertToInv = (!state.fPreferHeaders || pto->vBlockHashesToAnnounce.size() > MAX_BLOCKS_TO_ANNOUNCE); + bool fRevertToInv = ((!state.fPreferHeaders && + (!state.fPreferHeaderAndIDs || pto->vBlockHashesToAnnounce.size() > 1)) || + pto->vBlockHashesToAnnounce.size() > MAX_BLOCKS_TO_ANNOUNCE); const CBlockIndex *pBestIndex = NULL; // last header queued for delivery ProcessBlockAvailability(pto->id); // ensure pindexBestKnownBlock is up-to-date @@ -2685,6 +3257,45 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic& interr } } } + if (!fRevertToInv && !vHeaders.empty()) { + if (vHeaders.size() == 1 && state.fPreferHeaderAndIDs) { + // We only send up to 1 block as header-and-ids, as otherwise + // probably means we're doing an initial-ish-sync or they're slow + LogPrint("net", "%s sending header-and-ids %s to peer=%d\n", __func__, + vHeaders.front().GetHash().ToString(), pto->id); + + bool fGotBlockFromCache = false; + { + LOCK(cs_most_recent_block); + if (most_recent_block_hash == pBestIndex->GetBlockHash()) { + CBlockHeaderAndShortTxIDs cmpctblock(*most_recent_block); + connman.PushMessage(pto, msgMaker.Make(NetMsgType::CMPCTBLOCK, cmpctblock)); + fGotBlockFromCache = true; + } + } + if (!fGotBlockFromCache) { + CBlock block; + bool ret = ReadBlockFromDisk(block, pBestIndex, consensusParams); + assert(ret); + CBlockHeaderAndShortTxIDs cmpctblock(block); + connman.PushMessage(pto, msgMaker.Make(NetMsgType::CMPCTBLOCK, cmpctblock)); + } + state.pindexBestHeaderSent = pBestIndex; + } else if (state.fPreferHeaders) { + if (vHeaders.size() > 1) { + LogPrint("net", "%s: %u headers, range (%s, %s), to peer=%d\n", __func__, + vHeaders.size(), + vHeaders.front().GetHash().ToString(), + vHeaders.back().GetHash().ToString(), pto->id); + } else { + LogPrint("net", "%s: sending header %s to peer=%d\n", __func__, + vHeaders.front().GetHash().ToString(), pto->id); + } + connman.PushMessage(pto, msgMaker.Make(NetMsgType::HEADERS, vHeaders)); + state.pindexBestHeaderSent = pBestIndex; + } else + fRevertToInv = true; + } if (fRevertToInv) { // If falling back to using an inv, just try to inv the tip. // The last entry in vBlockHashesToAnnounce was our tip at some point @@ -2710,18 +3321,6 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic& interr pto->id, hashToAnnounce.ToString()); } } - } else if (!vHeaders.empty()) { - if (vHeaders.size() > 1) { - LogPrint("net", "%s: %u headers, range (%s, %s), to peer=%d\n", __func__, - vHeaders.size(), - vHeaders.front().GetHash().ToString(), - vHeaders.back().GetHash().ToString(), pto->id); - } else { - LogPrint("net", "%s: sending header %s to peer=%d\n", __func__, - vHeaders.front().GetHash().ToString(), pto->id); - } - connman.PushMessage(pto, msgMaker.Make(NetMsgType::HEADERS, vHeaders)); - state.pindexBestHeaderSent = pBestIndex; } pto->vBlockHashesToAnnounce.clear(); } diff --git a/src/net_processing.h b/src/net_processing.h index 317f8eafa6f7..3e7d596a597b 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -21,6 +21,9 @@ static const int64_t ORPHAN_TX_EXPIRE_INTERVAL = 5 * 60; static constexpr int64_t HEADERS_DOWNLOAD_TIMEOUT_BASE = 15 * 60 * 1000000; // 15 minutes static constexpr int64_t HEADERS_DOWNLOAD_TIMEOUT_PER_HEADER = 1000; // 1ms/header +/** Default number of orphan+recently-replaced txn to keep around for block reconstruction */ +static const unsigned int DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN = 100; + /** Register with a network node to receive its signals */ void RegisterNodeSignals(CNodeSignals& nodeSignals); /** Unregister a network node */ @@ -36,6 +39,7 @@ class PeerLogicValidation : public CValidationInterface { virtual void SyncTransaction(const CTransaction& tx, const CBlockIndex* pindex, int nPosInBlock) override; virtual void UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) override; virtual void BlockChecked(const CBlock& block, const CValidationState& state) override; + virtual void NewPoWValidBlock(const CBlockIndex *pindex, const std::shared_ptr& pblock) override; }; struct CNodeStateStats { diff --git a/src/privatesend-server.cpp b/src/privatesend-server.cpp index 6db6990846eb..b727f772f1a6 100644 --- a/src/privatesend-server.cpp +++ b/src/privatesend-server.cpp @@ -334,7 +334,7 @@ void CPrivateSendServer::CommitFinalTransaction(CConnman& connman) TRY_LOCK(cs_main, lockMain); CValidationState validationState; mempool.PrioritiseTransaction(hashTx, hashTx.ToString(), 1000, 0.1*COIN); - if(!lockMain || !AcceptToMemoryPool(mempool, validationState, finalTransaction, false, NULL, false, maxTxFee, true)) + if(!lockMain || !AcceptToMemoryPool(mempool, validationState, finalTransaction, false, NULL, NULL, false, maxTxFee, true)) { LogPrintf("CPrivateSendServer::CommitFinalTransaction -- AcceptToMemoryPool() error: Transaction not valid\n"); SetNull(); diff --git a/src/privatesend.cpp b/src/privatesend.cpp index cc44d0b3085e..c61685ba490e 100644 --- a/src/privatesend.cpp +++ b/src/privatesend.cpp @@ -294,7 +294,7 @@ bool CPrivateSend::IsCollateralValid(const CTransaction& txCollateral) { LOCK(cs_main); CValidationState validationState; - if(!AcceptToMemoryPool(mempool, validationState, MakeTransactionRef(txCollateral), false, NULL, false, maxTxFee, true)) { + if(!AcceptToMemoryPool(mempool, validationState, MakeTransactionRef(txCollateral), false, NULL, NULL, false, maxTxFee, true)) { LogPrint("privatesend", "CPrivateSend::IsCollateralValid -- didn't pass AcceptToMemoryPool()\n"); return false; } diff --git a/src/protocol.cpp b/src/protocol.cpp index ae1f6ef9e5f0..1ac44645cbc2 100644 --- a/src/protocol.cpp +++ b/src/protocol.cpp @@ -36,6 +36,10 @@ const char *FILTERCLEAR="filterclear"; const char *REJECT="reject"; const char *SENDHEADERS="sendheaders"; const char *FEEFILTER="feefilter"; +const char *SENDCMPCT="sendcmpct"; +const char *CMPCTBLOCK="cmpctblock"; +const char *GETBLOCKTXN="getblocktxn"; +const char *BLOCKTXN="blocktxn"; // Dash message types const char *TXLOCKREQUEST="ix"; const char *TXLOCKVOTE="txlvote"; @@ -92,6 +96,7 @@ static const char* ppszTypeName[] = NetMsgType::MNGOVERNANCEOBJECT, NetMsgType::MNGOVERNANCEOBJECTVOTE, NetMsgType::MNVERIFY, + "compact block", // Should never occur }; /** All known message types. Keep this in the same order as the list of @@ -120,7 +125,11 @@ const static std::string allNetMessageTypes[] = { NetMsgType::FILTERCLEAR, NetMsgType::REJECT, NetMsgType::SENDHEADERS, - NetMsgType::FEEFILTER, + NetMsgType::FEEFILTER, + NetMsgType::SENDCMPCT, + NetMsgType::CMPCTBLOCK, + NetMsgType::GETBLOCKTXN, + NetMsgType::BLOCKTXN, // Dash message types // NOTE: do NOT include non-implmented here, we want them to be "Unknown command" in ProcessMessage() NetMsgType::TXLOCKREQUEST, diff --git a/src/protocol.h b/src/protocol.h index dba95e94fc81..8cdc1a734e18 100644 --- a/src/protocol.h +++ b/src/protocol.h @@ -222,6 +222,33 @@ extern const char *SENDHEADERS; */ extern const char *FEEFILTER; +/** + * Contains a 1-byte bool and 8-byte LE version number. + * Indicates that a node is willing to provide blocks via "cmpctblock" messages. + * May indicate that a node prefers to receive new block announcements via a + * "cmpctblock" message rather than an "inv", depending on message contents. + * @since protocol version 70209 as described by BIP 152 + */ +extern const char *SENDCMPCT; +/** + * Contains a CBlockHeaderAndShortTxIDs object - providing a header and + * list of "short txids". + * @since protocol version 70209 as described by BIP 152 + */ +extern const char *CMPCTBLOCK; +/** + * Contains a BlockTransactionsRequest + * Peer should respond with "blocktxn" message. + * @since protocol version 70209 as described by BIP 152 + */ +extern const char *GETBLOCKTXN; +/** + * Contains a BlockTransactions. + * Sent in response to a "getblocktxn" message. + * @since protocol version 70209 as described by BIP 152 + */ +extern const char *BLOCKTXN; + // Dash message types // NOTE: do NOT declare non-implmented here, we don't want them to be exposed to the outside // TODO: add description @@ -345,6 +372,9 @@ enum GetDataMsg { MSG_GOVERNANCE_OBJECT = 17, MSG_GOVERNANCE_OBJECT_VOTE = 18, MSG_MASTERNODE_VERIFY = 19, + // Nodes may always request a MSG_CMPCT_BLOCK in a getdata, however, + // MSG_CMPCT_BLOCK should not appear in any invs except as a part of getdata. + MSG_CMPCT_BLOCK = 20, //!< Defined in BIP152 }; /** inv message data */ @@ -376,4 +406,5 @@ class CInv uint256 hash; }; + #endif // BITCOIN_PROTOCOL_H diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index a9fd236551cb..2f7cb412751d 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -928,7 +928,7 @@ UniValue sendrawtransaction(const JSONRPCRequest& request) } CValidationState state; bool fMissingInputs; - if (!AcceptToMemoryPool(mempool, state, std::move(tx), fLimitFree, &fMissingInputs, false, nMaxRawTxFee)) { + if (!AcceptToMemoryPool(mempool, state, std::move(tx), fLimitFree, &fMissingInputs, NULL, false, nMaxRawTxFee)) { if (state.IsInvalid()) { throw JSONRPCError(RPC_TRANSACTION_REJECTED, strprintf("%i: %s", state.GetRejectCode(), state.GetRejectReason())); } else { diff --git a/src/test/blockencodings_tests.cpp b/src/test/blockencodings_tests.cpp new file mode 100644 index 000000000000..69c489de0012 --- /dev/null +++ b/src/test/blockencodings_tests.cpp @@ -0,0 +1,340 @@ +// Copyright (c) 2011-2015 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "blockencodings.h" +#include "consensus/merkle.h" +#include "chainparams.h" +#include "random.h" + +#include "test/test_dash.h" + +#include + +std::vector> extra_txn; + +struct RegtestingSetup : public TestingSetup { + RegtestingSetup() : TestingSetup(CBaseChainParams::REGTEST) {} +}; + +BOOST_FIXTURE_TEST_SUITE(blockencodings_tests, RegtestingSetup) + +static CBlock BuildBlockTestCase() { + CBlock block; + CMutableTransaction tx; + tx.vin.resize(1); + tx.vin[0].scriptSig.resize(10); + tx.vout.resize(1); + tx.vout[0].nValue = 42; + + block.vtx.resize(3); + block.vtx[0] = MakeTransactionRef(tx); + block.nVersion = 42; + block.hashPrevBlock = GetRandHash(); + block.nBits = 0x207fffff; + + tx.vin[0].prevout.hash = GetRandHash(); + tx.vin[0].prevout.n = 0; + block.vtx[1] = MakeTransactionRef(tx); + + tx.vin.resize(10); + for (size_t i = 0; i < tx.vin.size(); i++) { + tx.vin[i].prevout.hash = GetRandHash(); + tx.vin[i].prevout.n = 0; + } + block.vtx[2] = MakeTransactionRef(tx); + + bool mutated; + block.hashMerkleRoot = BlockMerkleRoot(block, &mutated); + assert(!mutated); + while (!CheckProofOfWork(block.GetHash(), block.nBits, Params().GetConsensus())) ++block.nNonce; + return block; +} + +// Number of shared use_counts we expect for a tx we havent touched +// == 2 (mempool + our copy from the GetSharedTx call) +#define SHARED_TX_OFFSET 2 + +BOOST_AUTO_TEST_CASE(SimpleRoundTripTest) +{ + CTxMemPool pool(CFeeRate(0)); + TestMemPoolEntryHelper entry; + CBlock block(BuildBlockTestCase()); + + pool.addUnchecked(block.vtx[2]->GetHash(), entry.FromTx(*block.vtx[2])); + BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 0); + + // Do a simple ShortTxIDs RT + { + CBlockHeaderAndShortTxIDs shortIDs(block); + + CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); + stream << shortIDs; + + CBlockHeaderAndShortTxIDs shortIDs2; + stream >> shortIDs2; + + PartiallyDownloadedBlock partialBlock(&pool); + BOOST_CHECK(partialBlock.InitData(shortIDs2, extra_txn) == READ_STATUS_OK); + BOOST_CHECK( partialBlock.IsTxAvailable(0)); + BOOST_CHECK(!partialBlock.IsTxAvailable(1)); + BOOST_CHECK( partialBlock.IsTxAvailable(2)); + + BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); + + size_t poolSize = pool.size(); + pool.removeRecursive(*block.vtx[2]); + BOOST_CHECK_EQUAL(pool.size(), poolSize - 1); + + CBlock block2; + { + PartiallyDownloadedBlock tmp = partialBlock; + BOOST_CHECK(partialBlock.FillBlock(block2, {}) == READ_STATUS_INVALID); // No transactions + partialBlock = tmp; + } + + // Wrong transaction + { + PartiallyDownloadedBlock tmp = partialBlock; + partialBlock.FillBlock(block2, {block.vtx[2]}); // Current implementation doesn't check txn here, but don't require that + partialBlock = tmp; + } + bool mutated; + BOOST_CHECK(block.hashMerkleRoot != BlockMerkleRoot(block2, &mutated)); + + CBlock block3; + BOOST_CHECK(partialBlock.FillBlock(block3, {block.vtx[1]}) == READ_STATUS_OK); + BOOST_CHECK_EQUAL(block.GetHash().ToString(), block3.GetHash().ToString()); + BOOST_CHECK_EQUAL(block.hashMerkleRoot.ToString(), BlockMerkleRoot(block3, &mutated).ToString()); + BOOST_CHECK(!mutated); + } +} + +class TestHeaderAndShortIDs { + // Utility to encode custom CBlockHeaderAndShortTxIDs +public: + CBlockHeader header; + uint64_t nonce; + std::vector shorttxids; + std::vector prefilledtxn; + + TestHeaderAndShortIDs(const CBlockHeaderAndShortTxIDs& orig) { + CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); + stream << orig; + stream >> *this; + } + TestHeaderAndShortIDs(const CBlock& block) : + TestHeaderAndShortIDs(CBlockHeaderAndShortTxIDs(block)) {} + + uint64_t GetShortID(const uint256& txhash) const { + CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); + stream << *this; + CBlockHeaderAndShortTxIDs base; + stream >> base; + return base.GetShortID(txhash); + } + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) { + READWRITE(header); + READWRITE(nonce); + size_t shorttxids_size = shorttxids.size(); + READWRITE(VARINT(shorttxids_size)); + shorttxids.resize(shorttxids_size); + for (size_t i = 0; i < shorttxids.size(); i++) { + uint32_t lsb = shorttxids[i] & 0xffffffff; + uint16_t msb = (shorttxids[i] >> 32) & 0xffff; + READWRITE(lsb); + READWRITE(msb); + shorttxids[i] = (uint64_t(msb) << 32) | uint64_t(lsb); + } + READWRITE(prefilledtxn); + } +}; + +BOOST_AUTO_TEST_CASE(NonCoinbasePreforwardRTTest) +{ + CTxMemPool pool(CFeeRate(0)); + TestMemPoolEntryHelper entry; + CBlock block(BuildBlockTestCase()); + + pool.addUnchecked(block.vtx[2]->GetHash(), entry.FromTx(*block.vtx[2])); + BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 0); + + uint256 txhash; + + // Test with pre-forwarding tx 1, but not coinbase + { + TestHeaderAndShortIDs shortIDs(block); + shortIDs.prefilledtxn.resize(1); + shortIDs.prefilledtxn[0] = {1, block.vtx[1]}; + shortIDs.shorttxids.resize(2); + shortIDs.shorttxids[0] = shortIDs.GetShortID(block.vtx[0]->GetHash()); + shortIDs.shorttxids[1] = shortIDs.GetShortID(block.vtx[2]->GetHash()); + + CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); + stream << shortIDs; + + CBlockHeaderAndShortTxIDs shortIDs2; + stream >> shortIDs2; + + PartiallyDownloadedBlock partialBlock(&pool); + BOOST_CHECK(partialBlock.InitData(shortIDs2, extra_txn) == READ_STATUS_OK); + BOOST_CHECK(!partialBlock.IsTxAvailable(0)); + BOOST_CHECK( partialBlock.IsTxAvailable(1)); + BOOST_CHECK( partialBlock.IsTxAvailable(2)); + + BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); + + CBlock block2; + { + PartiallyDownloadedBlock tmp = partialBlock; + BOOST_CHECK(partialBlock.FillBlock(block2, {}) == READ_STATUS_INVALID); // No transactions + partialBlock = tmp; + } + + // Wrong transaction + { + PartiallyDownloadedBlock tmp = partialBlock; + partialBlock.FillBlock(block2, {block.vtx[1]}); // Current implementation doesn't check txn here, but don't require that + partialBlock = tmp; + } + bool mutated; + BOOST_CHECK(block.hashMerkleRoot != BlockMerkleRoot(block2, &mutated)); + + CBlock block3; + PartiallyDownloadedBlock partialBlockCopy = partialBlock; + BOOST_CHECK(partialBlock.FillBlock(block3, {block.vtx[0]}) == READ_STATUS_OK); + BOOST_CHECK_EQUAL(block.GetHash().ToString(), block3.GetHash().ToString()); + BOOST_CHECK_EQUAL(block.hashMerkleRoot.ToString(), BlockMerkleRoot(block3, &mutated).ToString()); + BOOST_CHECK(!mutated); + + txhash = block.vtx[2]->GetHash(); + block.vtx.clear(); + block2.vtx.clear(); + block3.vtx.clear(); + BOOST_CHECK_EQUAL(pool.mapTx.find(txhash)->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); // + 1 because of partialBlockCopy. + } + BOOST_CHECK_EQUAL(pool.mapTx.find(txhash)->GetSharedTx().use_count(), SHARED_TX_OFFSET + 0); +} + +BOOST_AUTO_TEST_CASE(SufficientPreforwardRTTest) +{ + CTxMemPool pool(CFeeRate(0)); + TestMemPoolEntryHelper entry; + CBlock block(BuildBlockTestCase()); + + pool.addUnchecked(block.vtx[1]->GetHash(), entry.FromTx(*block.vtx[1])); + BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[1]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 0); + + uint256 txhash; + + // Test with pre-forwarding coinbase + tx 2 with tx 1 in mempool + { + TestHeaderAndShortIDs shortIDs(block); + shortIDs.prefilledtxn.resize(2); + shortIDs.prefilledtxn[0] = {0, block.vtx[0]}; + shortIDs.prefilledtxn[1] = {1, block.vtx[2]}; // id == 1 as it is 1 after index 1 + shortIDs.shorttxids.resize(1); + shortIDs.shorttxids[0] = shortIDs.GetShortID(block.vtx[1]->GetHash()); + + CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); + stream << shortIDs; + + CBlockHeaderAndShortTxIDs shortIDs2; + stream >> shortIDs2; + + PartiallyDownloadedBlock partialBlock(&pool); + BOOST_CHECK(partialBlock.InitData(shortIDs2, extra_txn) == READ_STATUS_OK); + BOOST_CHECK( partialBlock.IsTxAvailable(0)); + BOOST_CHECK( partialBlock.IsTxAvailable(1)); + BOOST_CHECK( partialBlock.IsTxAvailable(2)); + + BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[1]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); + + CBlock block2; + PartiallyDownloadedBlock partialBlockCopy = partialBlock; + BOOST_CHECK(partialBlock.FillBlock(block2, {}) == READ_STATUS_OK); + BOOST_CHECK_EQUAL(block.GetHash().ToString(), block2.GetHash().ToString()); + bool mutated; + BOOST_CHECK_EQUAL(block.hashMerkleRoot.ToString(), BlockMerkleRoot(block2, &mutated).ToString()); + BOOST_CHECK(!mutated); + + txhash = block.vtx[1]->GetHash(); + block.vtx.clear(); + block2.vtx.clear(); + BOOST_CHECK_EQUAL(pool.mapTx.find(txhash)->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); // + 1 because of partialBlockCopy. + } + BOOST_CHECK_EQUAL(pool.mapTx.find(txhash)->GetSharedTx().use_count(), SHARED_TX_OFFSET + 0); +} + +BOOST_AUTO_TEST_CASE(EmptyBlockRoundTripTest) +{ + CTxMemPool pool(CFeeRate(0)); + CMutableTransaction coinbase; + coinbase.vin.resize(1); + coinbase.vin[0].scriptSig.resize(10); + coinbase.vout.resize(1); + coinbase.vout[0].nValue = 42; + + CBlock block; + block.vtx.resize(1); + block.vtx[0] = MakeTransactionRef(std::move(coinbase)); + block.nVersion = 42; + block.hashPrevBlock = GetRandHash(); + block.nBits = 0x207fffff; + + bool mutated; + block.hashMerkleRoot = BlockMerkleRoot(block, &mutated); + assert(!mutated); + while (!CheckProofOfWork(block.GetHash(), block.nBits, Params().GetConsensus())) ++block.nNonce; + + // Test simple header round-trip with only coinbase + { + CBlockHeaderAndShortTxIDs shortIDs(block); + + CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); + stream << shortIDs; + + CBlockHeaderAndShortTxIDs shortIDs2; + stream >> shortIDs2; + + PartiallyDownloadedBlock partialBlock(&pool); + BOOST_CHECK(partialBlock.InitData(shortIDs2, extra_txn) == READ_STATUS_OK); + BOOST_CHECK(partialBlock.IsTxAvailable(0)); + + CBlock block2; + std::vector vtx_missing; + BOOST_CHECK(partialBlock.FillBlock(block2, vtx_missing) == READ_STATUS_OK); + BOOST_CHECK_EQUAL(block.GetHash().ToString(), block2.GetHash().ToString()); + BOOST_CHECK_EQUAL(block.hashMerkleRoot.ToString(), BlockMerkleRoot(block2, &mutated).ToString()); + BOOST_CHECK(!mutated); + } +} + +BOOST_AUTO_TEST_CASE(TransactionsRequestSerializationTest) { + BlockTransactionsRequest req1; + req1.blockhash = GetRandHash(); + req1.indexes.resize(4); + req1.indexes[0] = 0; + req1.indexes[1] = 1; + req1.indexes[2] = 3; + req1.indexes[3] = 4; + + CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); + stream << req1; + + BlockTransactionsRequest req2; + stream >> req2; + + BOOST_CHECK_EQUAL(req1.blockhash.ToString(), req2.blockhash.ToString()); + BOOST_CHECK_EQUAL(req1.indexes.size(), req2.indexes.size()); + BOOST_CHECK_EQUAL(req1.indexes[0], req2.indexes[0]); + BOOST_CHECK_EQUAL(req1.indexes[1], req2.indexes[1]); + BOOST_CHECK_EQUAL(req1.indexes[2], req2.indexes[2]); + BOOST_CHECK_EQUAL(req1.indexes[3], req2.indexes[3]); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/hash_tests.cpp b/src/test/hash_tests.cpp index 46dc10ef203f..604a1a2a2d80 100644 --- a/src/test/hash_tests.cpp +++ b/src/test/hash_tests.cpp @@ -145,6 +145,8 @@ BOOST_AUTO_TEST_CASE(siphash) BOOST_CHECK_EQUAL(SipHashUint256(k1, k2, x), sip256.Finalize()); BOOST_CHECK_EQUAL(SipHashUint256Extra(k1, k2, x, n), sip288.Finalize()); }*/ + + BOOST_CHECK_EQUAL(SipHashUint256(1, 2, ss.GetHash()), 0x79751e980c2a0a35ULL); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/test_dash.cpp b/src/test/test_dash.cpp index 574b240def04..ae459123078a 100644 --- a/src/test/test_dash.cpp +++ b/src/test/test_dash.cpp @@ -142,11 +142,12 @@ TestChain100Setup::~TestChain100Setup() CTxMemPoolEntry TestMemPoolEntryHelper::FromTx(const CMutableTransaction &tx, CTxMemPool *pool) { CTransaction txn(tx); - // Hack to assume either it's completely dependent on other mempool txs or not at all - CAmount inChainValue = pool && pool->HasNoInputsOf(txn) ? txn.GetValueOut() : 0; + return FromTx(txn, pool); +} +CTxMemPoolEntry TestMemPoolEntryHelper::FromTx(const CTransaction &txn, CTxMemPool *pool) { return CTxMemPoolEntry(MakeTransactionRef(txn), nFee, nTime, dPriority, nHeight, - inChainValue, spendsCoinbase, sigOpCount, lp); + txn.GetValueOut(), spendsCoinbase, sigOpCount, lp); } void Shutdown(void* parg) diff --git a/src/test/test_dash.h b/src/test/test_dash.h index c1f6b593aaa2..0bb1717a839b 100644 --- a/src/test/test_dash.h +++ b/src/test/test_dash.h @@ -80,6 +80,7 @@ struct TestMemPoolEntryHelper spendsCoinbase(false), sigOpCount(4) { } CTxMemPoolEntry FromTx(const CMutableTransaction &tx, CTxMemPool *pool = NULL); + CTxMemPoolEntry FromTx(const CTransaction &tx, CTxMemPool *pool = NULL); // Change the default value TestMemPoolEntryHelper &Fee(CAmount _fee) { nFee = _fee; return *this; } diff --git a/src/test/txvalidationcache_tests.cpp b/src/test/txvalidationcache_tests.cpp index 8e6637e4f1be..af2495abf3ef 100644 --- a/src/test/txvalidationcache_tests.cpp +++ b/src/test/txvalidationcache_tests.cpp @@ -23,7 +23,7 @@ ToMemPool(CMutableTransaction& tx) LOCK(cs_main); CValidationState state; - return AcceptToMemoryPool(mempool, state, MakeTransactionRef(tx), false, NULL, true, 0); + return AcceptToMemoryPool(mempool, state, MakeTransactionRef(tx), false, NULL, NULL, true, 0); } BOOST_FIXTURE_TEST_CASE(tx_mempool_block_doublespend, TestChain100Setup) diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 391afa74958b..8ed45633ac98 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -434,6 +434,9 @@ bool CTxMemPool::addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, totalTxSize += entry.GetTxSize(); minerPolicyEstimator->processTransaction(entry, validFeeEstimate); + vTxHashes.emplace_back(hash, newit); + newit->vTxHashesIdx = vTxHashes.size() - 1; + return true; } @@ -600,6 +603,15 @@ void CTxMemPool::removeUnchecked(txiter it, MemPoolRemovalReason reason) BOOST_FOREACH(const CTxIn& txin, it->GetTx().vin) mapNextTx.erase(txin.prevout); + if (vTxHashes.size() > 1) { + vTxHashes[it->vTxHashesIdx] = std::move(vTxHashes.back()); + vTxHashes[it->vTxHashesIdx].second->vTxHashesIdx = it->vTxHashesIdx; + vTxHashes.pop_back(); + if (vTxHashes.size() * 2 < vTxHashes.capacity()) + vTxHashes.shrink_to_fit(); + } else + vTxHashes.clear(); + totalTxSize -= it->GetTxSize(); cachedInnerUsage -= it->DynamicMemoryUsage(); cachedInnerUsage -= memusage::DynamicUsage(mapLinks[it].parents) + memusage::DynamicUsage(mapLinks[it].children); @@ -1132,7 +1144,7 @@ bool CCoinsViewMemPool::GetCoin(const COutPoint &outpoint, Coin &coin) const { size_t CTxMemPool::DynamicMemoryUsage() const { LOCK(cs); // Estimate the overhead of mapTx to be 15 pointers + an allocation, as no exact formula for boost::multi_index_contained is implemented. - return memusage::MallocUsage(sizeof(CTxMemPoolEntry) + 15 * sizeof(void*)) * mapTx.size() + memusage::DynamicUsage(mapNextTx) + memusage::DynamicUsage(mapDeltas) + memusage::DynamicUsage(mapLinks) + cachedInnerUsage; + return memusage::MallocUsage(sizeof(CTxMemPoolEntry) + 15 * sizeof(void*)) * mapTx.size() + memusage::DynamicUsage(mapNextTx) + memusage::DynamicUsage(mapDeltas) + memusage::DynamicUsage(mapLinks) + memusage::DynamicUsage(vTxHashes) + cachedInnerUsage; } void CTxMemPool::RemoveStaged(setEntries &stage, bool updateDescendants, MemPoolRemovalReason reason) { diff --git a/src/txmempool.h b/src/txmempool.h index 8a5dddf0d992..93a78a8fcbdb 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -158,6 +158,8 @@ class CTxMemPoolEntry uint64_t GetSizeWithAncestors() const { return nSizeWithAncestors; } CAmount GetModFeesWithAncestors() const { return nModFeesWithAncestors; } unsigned int GetSigOpCountWithAncestors() const { return nSigOpCountWithAncestors; } + + mutable size_t vTxHashesIdx; //!< Index in mempool's vTxHashes }; // Helpers for modifying CTxMemPool::mapTx, which is a boost multi_index. @@ -496,7 +498,10 @@ class CTxMemPool mutable CCriticalSection cs; indexed_transaction_set mapTx; + typedef indexed_transaction_set::nth_index<0>::type::iterator txiter; + std::vector > vTxHashes; //!< All tx hashes/entries in mapTx, in random order + struct CompareIteratorByHash { bool operator()(const txiter &a, const txiter &b) const { return a->GetTx().GetHash() < b->GetTx().GetHash(); diff --git a/src/validation.cpp b/src/validation.cpp index cc53355a1061..805f3ae28430 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -8,6 +8,7 @@ #include "alert.h" #include "arith_uint256.h" +#include "blockencodings.h" #include "chainparams.h" #include "checkpoints.h" #include "checkqueue.h" @@ -603,7 +604,8 @@ static bool IsCurrentForFeeEstimation() } bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const CTransactionRef& ptx, bool fLimitFree, - bool* pfMissingInputs, int64_t nAcceptTime, bool fOverrideMempoolLimit, const CAmount& nAbsurdFee, + bool* pfMissingInputs, int64_t nAcceptTime, std::list* plTxnReplaced, + bool fOverrideMempoolLimit, const CAmount& nAbsurdFee, std::vector& coins_to_uncache, bool fDryRun) { const CTransaction& tx = *ptx; @@ -1026,6 +1028,8 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C FormatMoney(nModifiedFees - nConflictingFees), CURRENCY_UNIT, (int)nSize - (int)nConflictingSize); + if (plTxnReplaced) + plTxnReplaced->push_back(it->GetSharedTx()); } pool.RemoveStaged(allConflicting, false, MemPoolRemovalReason::REPLACED); @@ -1063,10 +1067,11 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C } bool AcceptToMemoryPoolWithTime(CTxMemPool& pool, CValidationState &state, const CTransactionRef &tx, bool fLimitFree, - bool* pfMissingInputs, int64_t nAcceptTime, bool fOverrideMempoolLimit, const CAmount nAbsurdFee, bool fDryRun) + bool* pfMissingInputs, int64_t nAcceptTime, std::list* plTxnReplaced, + bool fOverrideMempoolLimit, const CAmount nAbsurdFee, bool fDryRun) { std::vector coins_to_uncache; - bool res = AcceptToMemoryPoolWorker(pool, state, tx, fLimitFree, pfMissingInputs, nAcceptTime, fOverrideMempoolLimit, nAbsurdFee, coins_to_uncache, fDryRun); + bool res = AcceptToMemoryPoolWorker(pool, state, tx, fLimitFree, pfMissingInputs, nAcceptTime, plTxnReplaced, fOverrideMempoolLimit, nAbsurdFee, coins_to_uncache, fDryRun); if (!res || fDryRun) { if(!res) LogPrint("mempool", "%s: %s %s (%s)\n", __func__, tx->GetHash().ToString(), state.GetRejectReason(), state.GetDebugMessage()); BOOST_FOREACH(const COutPoint& hashTx, coins_to_uncache) @@ -1079,9 +1084,10 @@ bool AcceptToMemoryPoolWithTime(CTxMemPool& pool, CValidationState &state, const } bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransactionRef &tx, bool fLimitFree, - bool* pfMissingInputs, bool fOverrideMempoolLimit, const CAmount nAbsurdFee, bool fDryRun) + bool* pfMissingInputs, std::list* plTxnReplaced, + bool fOverrideMempoolLimit, const CAmount nAbsurdFee, bool fDryRun) { - return AcceptToMemoryPoolWithTime(pool, state, tx, fLimitFree, pfMissingInputs, GetTime(), fOverrideMempoolLimit, nAbsurdFee, fDryRun); + return AcceptToMemoryPoolWithTime(pool, state, tx, fLimitFree, pfMissingInputs, GetTime(), plTxnReplaced, fOverrideMempoolLimit, nAbsurdFee, fDryRun); } bool GetTimestampIndex(const unsigned int &high, const unsigned int &low, std::vector &hashes) @@ -2599,7 +2605,7 @@ bool static DisconnectTip(CValidationState& state, const CChainParams& chainpara const CTransaction& tx = *it; // ignore validation errors in resurrected transactions CValidationState stateDummy; - if (tx.IsCoinBase() || !AcceptToMemoryPool(mempool, stateDummy, it, false, NULL, true)) { + if (tx.IsCoinBase() || !AcceptToMemoryPool(mempool, stateDummy, it, false, NULL, NULL, true)) { mempool.removeRecursive(tx, MemPoolRemovalReason::REORG); } else if (mempool.exists(tx.GetHash())) { vHashUpdate.push_back(tx.GetHash()); @@ -2912,6 +2918,11 @@ static void NotifyHeaderTip() { * that is already loaded (to avoid loading it again from disk). */ bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, std::shared_ptr pblock) { + // Note that while we're often called here from ProcessNewBlock, this is + // far from a guarantee. Things in the P2P/RPC will often end up calling + // us in the middle of ProcessNewBlock - do not assume pblock is set + // sanely for performance or correctness! + CBlockIndex *pindexMostWork = NULL; CBlockIndex *pindexNewTip = NULL; do { diff --git a/src/validation.h b/src/validation.h index 58ced71b806f..f5c57a4388da 100644 --- a/src/validation.h +++ b/src/validation.h @@ -91,6 +91,11 @@ static const unsigned int BLOCK_STALLING_TIMEOUT = 2; /** Number of headers sent in one getheaders result. We rely on the assumption that if a peer sends * less than this number, we reached its tip. Changing this value is a protocol upgrade. */ static const unsigned int MAX_HEADERS_RESULTS = 2000; +/** Maximum depth of blocks we're willing to serve as compact blocks to peers + * when requested. For older blocks, a regular BLOCK response will be sent. */ +static const int MAX_CMPCTBLOCK_DEPTH = 5; +/** Maximum depth of blocks we're willing to respond to GETBLOCKTXN requests for. */ +static const int MAX_BLOCKTXN_DEPTH = 10; /** Size of the "block download window": how far ahead of our current height do we fetch? * Larger windows tolerate larger download speed differences between peer, but increase the potential * degree of disordering of blocks on disk (which make reindexing and in the future perhaps pruning @@ -332,13 +337,16 @@ void PruneAndFlush(); /** Prune block files up to a given height */ void PruneBlockFilesManual(int nPruneUpToHeight); -/** (try to) add transaction to memory pool **/ +/** (try to) add transaction to memory pool + * plTxnReplaced will be appended to with all transactions replaced from mempool **/ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransactionRef &tx, bool fLimitFree, - bool* pfMissingInputs, bool fOverrideMempoolLimit=false, const CAmount nAbsurdFee=0, bool fDryRun=false); + bool* pfMissingInputs, std::list* plTxnReplaced = NULL, bool fOverrideMempoolLimit=false, + const CAmount nAbsurdFee=0, bool fDryRun=false); /** (try to) add transaction to memory pool with a specified acceptance time **/ bool AcceptToMemoryPoolWithTime(CTxMemPool& pool, CValidationState &state, const CTransactionRef &tx, bool fLimitFree, - bool* pfMissingInputs, int64_t nAcceptTime, bool fOverrideMempoolLimit=false, const CAmount nAbsurdFee=0, bool fDryRun=false); + bool* pfMissingInputs, int64_t nAcceptTime, std::list* plTxnReplaced = NULL, + bool fOverrideMempoolLimit=false, const CAmount nAbsurdFee=0, bool fDryRun=false); bool GetUTXOCoin(const COutPoint& outpoint, Coin& coin); int GetUTXOHeight(const COutPoint& outpoint); diff --git a/src/version.h b/src/version.h index c801eb332679..9686b886391f 100644 --- a/src/version.h +++ b/src/version.h @@ -10,7 +10,8 @@ * network protocol versioning */ -static const int PROTOCOL_VERSION = 70208; + +static const int PROTOCOL_VERSION = 70209; //! initial proto version, to be increased after version/verack negotiation static const int INIT_PROTO_VERSION = 209; @@ -43,4 +44,7 @@ static const int FEEFILTER_VERSION = 99999; // disable for now (clarify deployme //! DIP0001 was activated in this version static const int DIP0001_PROTOCOL_VERSION = 70208; +//! short-id-based block download starts with this version +static const int SHORT_IDS_BLOCKS_VERSION = 70209; + #endif // BITCOIN_VERSION_H diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index f65c27bee350..6cc6671225b8 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -5393,5 +5393,5 @@ int CMerkleTx::GetBlocksToMaturity() const bool CMerkleTx::AcceptToMemoryPool(const CAmount& nAbsurdFee, CValidationState& state) { - return ::AcceptToMemoryPool(mempool, state, tx, true, NULL, false, nAbsurdFee); + return ::AcceptToMemoryPool(mempool, state, tx, true, NULL, NULL, false, nAbsurdFee); }