diff --git a/src/httpserver.cpp b/src/httpserver.cpp index 633c486463c8..15f29beeac46 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -24,6 +24,7 @@ #include #include +#include #include #include @@ -240,6 +241,16 @@ static std::string RequestMethodString(HTTPRequest::RequestMethod m) /** HTTP request callback */ static void http_request_cb(struct evhttp_request* req, void* arg) { + // Disable reading to work around a libevent bug, fixed in 2.2.0. + if (event_get_version_number() >= 0x02010600 && event_get_version_number() < 0x02020001) { + evhttp_connection* conn = evhttp_request_get_connection(req); + if (conn) { + bufferevent* bev = evhttp_connection_get_bufferevent(conn); + if (bev) { + bufferevent_disable(bev, EV_READ); + } + } + } std::unique_ptr hreq(new HTTPRequest(req)); LogPrint(BCLog::HTTP, "Received a %s request for %s from %s\n", @@ -606,8 +617,21 @@ void HTTPRequest::WriteReply(int nStatus, const std::string& strReply) struct evbuffer* evb = evhttp_request_get_output_buffer(req); assert(evb); evbuffer_add(evb, strReply.data(), strReply.size()); - HTTPEvent* ev = new HTTPEvent(eventBase, true, - std::bind(evhttp_send_reply, req, nStatus, (const char*)nullptr, (struct evbuffer *)nullptr)); + auto req_copy = req; + HTTPEvent* ev = new HTTPEvent(eventBase, true, [req_copy, nStatus]{ + evhttp_send_reply(req_copy, nStatus, nullptr, nullptr); + // Re-enable reading from the socket. This is the second part of the libevent + // workaround above. + if (event_get_version_number() >= 0x02010600 && event_get_version_number() < 0x02020001) { + evhttp_connection* conn = evhttp_request_get_connection(req_copy); + if (conn) { + bufferevent* bev = evhttp_connection_get_bufferevent(conn); + if (bev) { + bufferevent_enable(bev, EV_READ | EV_WRITE); + } + } + } + }); ev->trigger(0); replySent = true; req = 0; // transferred back to main thread diff --git a/src/init.cpp b/src/init.cpp index 57048ba491ba..aaeb5e4b0b45 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -243,12 +243,12 @@ void PrepareShutdown() } #endif MapPort(false); + + // Because these depend on each-other, we make sure that neither can be + // using the other before destroying them. UnregisterValidationInterface(peerLogic.get()); + if(g_connman) g_connman->Stop(); peerLogic.reset(); - if (g_connman) { - // make sure to stop all threads before g_connman is reset to nullptr as these threads might still be accessing it - g_connman->Stop(); - } g_connman.reset(); if (!fLiteMode && !fRPCInWarmup) { @@ -263,7 +263,6 @@ void PrepareShutdown() flatdb6.Dump(sporkManager); } - UnregisterNodeSignals(GetNodeSignals()); if (fDumpMempoolLater && gArgs.GetArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) { DumpMempool(); } @@ -462,6 +461,9 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-maxorphantx=", strprintf(_("Keep at most unconnectable transactions in memory (default: %u)"), DEFAULT_MAX_ORPHAN_TRANSACTIONS)); strUsage += HelpMessageOpt("-maxmempool=", strprintf(_("Keep the transaction memory pool below megabytes (default: %u)"), DEFAULT_MAX_MEMPOOL_SIZE)); strUsage += HelpMessageOpt("-mempoolexpiry=", strprintf(_("Do not keep transactions in the mempool longer than hours (default: %u)"), DEFAULT_MEMPOOL_EXPIRY)); + if (showDebug) { + strUsage += HelpMessageOpt("-minimumchainwork=", strprintf("Minimum work assumed to exist on a valid chain in hex (default: %s, testnet: %s)", defaultChainParams->GetConsensus().nMinimumChainWork.GetHex(), testnetChainParams->GetConsensus().nMinimumChainWork.GetHex())); + } strUsage += HelpMessageOpt("-persistmempool", strprintf(_("Whether to save the mempool on shutdown and load on restart (default: %u)"), DEFAULT_PERSIST_MEMPOOL)); strUsage += HelpMessageOpt("-blockreconstructionextratxn=", strprintf(_("Extra transactions to keep in memory for compact block reconstructions (default: %u)"), DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN)); strUsage += HelpMessageOpt("-par=", strprintf(_("Set the number of script verification threads (%u to %d, 0 = auto, <0 = leave that many cores free, default: %d)"), @@ -484,12 +486,12 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-spentindex", strprintf(_("Maintain a full spent index, used to query the spending txid and input index for an outpoint (default: %u)"), DEFAULT_SPENTINDEX)); strUsage += HelpMessageGroup(_("Connection options:")); - strUsage += HelpMessageOpt("-addnode=", _("Add a node to connect to and attempt to keep the connection open")); + strUsage += HelpMessageOpt("-addnode=", _("Add a node to connect to and attempt to keep the connection open (see the `addnode` RPC command help for more info)")); strUsage += HelpMessageOpt("-allowprivatenet", strprintf(_("Allow RFC1918 addresses to be relayed and connected to (default: %u)"), DEFAULT_ALLOWPRIVATENET)); strUsage += HelpMessageOpt("-banscore=", strprintf(_("Threshold for disconnecting misbehaving peers (default: %u)"), DEFAULT_BANSCORE_THRESHOLD)); strUsage += HelpMessageOpt("-bantime=", strprintf(_("Number of seconds to keep misbehaving peers from reconnecting (default: %u)"), DEFAULT_MISBEHAVING_BANTIME)); strUsage += HelpMessageOpt("-bind=", _("Bind to given address and always listen on it. Use [host]:port notation for IPv6")); - strUsage += HelpMessageOpt("-connect=", _("Connect only to the specified node(s); -connect=0 disables automatic connections")); + strUsage += HelpMessageOpt("-connect=", _("Connect only to the specified node(s); -connect=0 disables automatic connections (the rules for this peer are the same as for -addnode)")); strUsage += HelpMessageOpt("-discover", _("Discover own IP addresses (default: 1 when listening and no -externalip or -proxy)")); strUsage += HelpMessageOpt("-dns", _("Allow DNS lookups for -addnode, -seednode and -connect") + " " + strprintf(_("(default: %u)"), DEFAULT_NAME_LOOKUP)); strUsage += HelpMessageOpt("-dnsseed", _("Query for peer addresses via DNS lookup, if low on addresses (default: 1 unless -connect used)")); @@ -1013,7 +1015,6 @@ void InitLogging() namespace { // Variables internal to initialization process only -ServiceFlags nRelevantServices = NODE_NETWORK; int nMaxConnections; int nUserMaxConnections; int nFD; @@ -1190,6 +1191,20 @@ bool AppInitParameterInteraction() else LogPrintf("Validating signatures for all blocks.\n"); + if (gArgs.IsArgSet("-minimumchainwork")) { + const std::string minChainWorkStr = gArgs.GetArg("-minimumchainwork", ""); + if (!IsHexNumber(minChainWorkStr)) { + return InitError(strprintf("Invalid non-hex (%s) minimum chain work value specified", minChainWorkStr)); + } + nMinimumChainWork = UintToArith256(uint256S(minChainWorkStr)); + } else { + nMinimumChainWork = UintToArith256(chainparams.GetConsensus().nMinimumChainWork); + } + LogPrintf("Setting nMinimumChainWork=%s\n", nMinimumChainWork.GetHex()); + if (nMinimumChainWork < UintToArith256(chainparams.GetConsensus().nMinimumChainWork)) { + LogPrintf("Warning: nMinimumChainWork set below default value of %s\n", chainparams.GetConsensus().nMinimumChainWork.GetHex()); + } + // mempool limits int64_t nMempoolSizeMax = gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; int64_t nMempoolSizeMin = gArgs.GetArg("-limitdescendantsize", DEFAULT_DESCENDANT_SIZE_LIMIT) * 1000 * 40; @@ -1570,9 +1585,8 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) g_connman = std::unique_ptr(new CConnman(GetRand(std::numeric_limits::max()), GetRand(std::numeric_limits::max()))); CConnman& connman = *g_connman; - peerLogic.reset(new PeerLogicValidation(&connman)); + peerLogic.reset(new PeerLogicValidation(&connman, scheduler)); RegisterValidationInterface(peerLogic.get()); - RegisterNodeSignals(GetNodeSignals()); // sanitize comments per BIP-0014, format user agent and check total size std::vector uacomments; @@ -2105,13 +2119,13 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) CConnman::Options connOptions; connOptions.nLocalServices = nLocalServices; - connOptions.nRelevantServices = nRelevantServices; connOptions.nMaxConnections = nMaxConnections; connOptions.nMaxOutbound = std::min(MAX_OUTBOUND_CONNECTIONS, connOptions.nMaxConnections); connOptions.nMaxAddnode = MAX_ADDNODE_CONNECTIONS; connOptions.nMaxFeeler = 1; connOptions.nBestHeight = chainActive.Height(); connOptions.uiInterface = &uiInterface; + connOptions.m_msgproc = peerLogic.get(); connOptions.nSendBufferMaxSize = 1000*gArgs.GetArg("-maxsendbuffer", DEFAULT_MAXSENDBUFFER); connOptions.nReceiveFloodSize = 1000*gArgs.GetArg("-maxreceivebuffer", DEFAULT_MAXRECEIVEBUFFER); diff --git a/src/net.cpp b/src/net.cpp index 1160f278d769..5b37104b3477 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -99,10 +99,6 @@ std::string strSubVersion; unordered_limitedmap mapAlreadyAskedFor(MAX_INV_SZ, MAX_INV_SZ * 2); -// Signals for message handling -static CNodeSignals g_signals; -CNodeSignals& GetNodeSignals() { return g_signals; } - void CConnman::AddOneShot(const std::string& strDest) { LOCK(cs_vOneShots); @@ -436,7 +432,6 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo uint64_t nonce = GetDeterministicRandomizer(RANDOMIZER_ID_LOCALHOSTNONCE).Write(id).Finalize(); CAddress addr_bind = GetBindAddress(hSocket); CNode* pnode = new CNode(id, nLocalServices, GetBestHeight(), hSocket, addrConnect, CalculateKeyedNetGroup(addrConnect), nonce, addr_bind, pszDest ? pszDest : "", false); - pnode->nServicesExpected = ServiceFlags(addrConnect.nServices & nRelevantServices); pnode->AddRef(); @@ -688,7 +683,7 @@ void CNode::copyStats(CNodeStats &stats) X(cleanSubVer); } X(fInbound); - X(fAddnode); + X(m_manual_connection); X(nStartingHeight); { LOCK(cs_vSend); @@ -1026,7 +1021,7 @@ bool CConnman::AttemptToEvictConnection() NodeEvictionCandidate candidate = {node->GetId(), node->nTimeConnected, node->nMinPingUsecTime, node->nLastBlockTime, node->nLastTXTime, - (node->nServices & nRelevantServices) == nRelevantServices, + HasAllDesirableServiceFlags(node->nServices), node->fRelayTxes, node->pfilter != nullptr, node->nKeyedNetGroup}; vEvictionCandidates.push_back(candidate); } @@ -1202,7 +1197,7 @@ void CConnman::AcceptConnection(const ListenSocket& hListenSocket) { CNode* pnode = new CNode(id, nLocalServices, GetBestHeight(), hSocket, addr, CalculateKeyedNetGroup(addr), nonce, addr_bind, "", true); pnode->AddRef(); pnode->fWhitelisted = whitelisted; - GetNodeSignals().InitializeNode(pnode, *this); + m_msgproc->InitializeNode(pnode); if (fLogIPs) { LogPrint(BCLog::NET, "connection from %s accepted\n", addr.ToString()); @@ -1717,7 +1712,7 @@ void CConnman::ThreadDNSAddressSeed() LOCK(cs_vNodes); int nRelevant = 0; for (auto pnode : vNodes) { - nRelevant += pnode->fSuccessfullyConnected && ((pnode->nServices & nRelevantServices) == nRelevantServices); + nRelevant += pnode->fSuccessfullyConnected && !pnode->fFeeler && !pnode->fOneShot && !pnode->m_manual_connection && !pnode->fInbound; } if (nRelevant >= 2) { LogPrintf("P2P peers available. Skipped DNS seeding.\n"); @@ -1739,7 +1734,7 @@ void CConnman::ThreadDNSAddressSeed() } else { std::vector vIPs; std::vector vAdd; - ServiceFlags requiredServiceBits = nRelevantServices; + ServiceFlags requiredServiceBits = GetDesirableServiceFlags(NODE_NONE); std::string host = GetDNSHost(seed, &requiredServiceBits); CNetAddr resolveSource; if (!resolveSource.SetInternal(host)) { @@ -1809,6 +1804,41 @@ void CConnman::ProcessOneShot() } } +bool CConnman::GetTryNewOutboundPeer() +{ + return m_try_another_outbound_peer; +} + +void CConnman::SetTryNewOutboundPeer(bool flag) +{ + m_try_another_outbound_peer = flag; + LogPrint(BCLog::NET, "net: setting try another outbound peer=%s\n", flag ? "true" : "false"); +} + +// Return the number of peers we have over our outbound connection limit +// Exclude peers that are marked for disconnect, or are going to be +// disconnected soon (eg one-shots and feelers) +// Also exclude peers that haven't finished initial connection handshake yet +// (so that we don't decide we're over our desired connection limit, and then +// evict some peer that has finished the handshake) +int CConnman::GetExtraOutboundCount() +{ + int nOutbound = 0; + { + LOCK(cs_vNodes); + for (CNode* pnode : vNodes) { + // don't count outbound masternodes + if (pnode->fMasternode) { + continue; + } + if (!pnode->fInbound && !pnode->m_manual_connection && !pnode->fFeeler && !pnode->fDisconnect && !pnode->fOneShot && pnode->fSuccessfullyConnected) { + ++nOutbound; + } + } + } + return std::max(nOutbound - nMaxOutbound, 0); +} + void CConnman::ThreadOpenConnections() { // Connect to specific addresses @@ -1820,7 +1850,7 @@ void CConnman::ThreadOpenConnections() for (const std::string& strAddr : gArgs.GetArgs("-connect")) { CAddress addr(CService(), NODE_NONE); - OpenNetworkConnection(addr, false, nullptr, strAddr.c_str()); + OpenNetworkConnection(addr, false, nullptr, strAddr.c_str(), false, false, true); for (int i = 0; i < 10 && i < nLoop; i++) { if (!interruptNet.sleep_for(std::chrono::milliseconds(500))) @@ -1869,17 +1899,11 @@ void CConnman::ThreadOpenConnections() // Do this here so we don't have to critsect vNodes inside mapAddresses critsect. // This is only done for mainnet and testnet int nOutbound = 0; - int nOutboundRelevant = 0; std::set > setConnected; if (!Params().AllowMultipleAddressesFromGroup()) { LOCK(cs_vNodes); for (CNode* pnode : vNodes) { - if (!pnode->fInbound && !pnode->fAddnode && !pnode->fMasternode) { - - // Count the peers that have all relevant services - if (pnode->fSuccessfullyConnected && !pnode->fFeeler && ((pnode->nServices & nRelevantServices) == nRelevantServices)) { - nOutboundRelevant++; - } + if (!pnode->fInbound && !pnode->fMasternode && !pnode->m_manual_connection) { // Netgroups for inbound and addnode peers are not excluded because our goal here // is to not use multiple of our limited outbound slots on a single netgroup // but inbound and addnode peers do not use our outbound slots. Inbound peers @@ -1904,7 +1928,8 @@ void CConnman::ThreadOpenConnections() // * Only make a feeler connection once every few minutes. // bool fFeeler = false; - if (nOutbound >= nMaxOutbound) { + + if (nOutbound >= nMaxOutbound && !GetTryNewOutboundPeer()) { int64_t nTime = GetTimeMicros(); // The current time right now (in microseconds). if (nTime > nNextFeeler) { nNextFeeler = PoissonNextSend(nTime, FEELER_INTERVAL); @@ -1943,21 +1968,16 @@ void CConnman::ThreadOpenConnections() if (IsLimited(addr)) continue; - // only connect to full nodes - if (!isMasternode && (addr.nServices & REQUIRED_SERVICES) != REQUIRED_SERVICES) - continue; - // only consider very recently tried nodes after 30 failed attempts if (nANow - addr.nLastTry < 600 && nTries < 30) continue; - // only consider nodes missing relevant services after 40 failed attempts and only if less than half the outbound are up. - ServiceFlags nRequiredServices = nRelevantServices; - if (nTries >= 40 && nOutbound < (nMaxOutbound >> 1)) { - nRequiredServices = REQUIRED_SERVICES; - } - - if (!isMasternode && (addr.nServices & nRequiredServices) != nRequiredServices) { + // for non-feelers, require all the services we'll want, + // for feelers, only require they be a full node (only because most + // SPV clients don't have a good address DB available) + if (!isMasternode && !fFeeler && !HasAllDesirableServiceFlags(addr.nServices)) { + continue; + } else if (!isMasternode && fFeeler && !MayHaveUsefulAddressDB(addr.nServices)) { continue; } @@ -1966,13 +1986,6 @@ void CConnman::ThreadOpenConnections() continue; addrConnect = addr; - - // regardless of the services assumed to be available, only require the minimum if half or more outbound have relevant services - if (nOutboundRelevant >= (nMaxOutbound >> 1)) { - addrConnect.nServices = REQUIRED_SERVICES; - } else { - addrConnect.nServices = nRequiredServices; - } break; } @@ -2165,7 +2178,7 @@ void CConnman::ThreadOpenMasternodeConnections() } // if successful, this moves the passed grant to the constructed node -bool CConnman::OpenNetworkConnection(const CAddress& addrConnect, bool fCountFailure, CSemaphoreGrant *grantOutbound, const char *pszDest, bool fOneShot, bool fFeeler, bool fAddnode, bool fConnectToMasternode) +bool CConnman::OpenNetworkConnection(const CAddress& addrConnect, bool fCountFailure, CSemaphoreGrant *grantOutbound, const char *pszDest, bool fOneShot, bool fFeeler, bool manual_connection, bool fConnectToMasternode) { // // Initiate outbound network connection @@ -2201,12 +2214,12 @@ bool CConnman::OpenNetworkConnection(const CAddress& addrConnect, bool fCountFai pnode->fOneShot = true; if (fFeeler) pnode->fFeeler = true; - if (fAddnode) - pnode->fAddnode = true; + if (manual_connection) + pnode->m_manual_connection = true; if (fConnectToMasternode) pnode->fMasternode = true; - GetNodeSignals().InitializeNode(pnode, *this); + m_msgproc->InitializeNode(pnode); { LOCK(cs_vNodes); vNodes.push_back(pnode); @@ -2233,16 +2246,16 @@ void CConnman::ThreadMessageHandler() continue; // Receive messages - bool fMoreNodeWork = GetNodeSignals().ProcessMessages(pnode, *this, flagInterruptMsgProc); + bool fMoreNodeWork = m_msgproc->ProcessMessages(pnode, flagInterruptMsgProc); fMoreWork |= (fMoreNodeWork && !pnode->fPauseSend); if (flagInterruptMsgProc) return; - // Send messages { LOCK(pnode->cs_sendProcessing); - GetNodeSignals().SendMessages(pnode, *this, flagInterruptMsgProc); + m_msgproc->SendMessages(pnode, flagInterruptMsgProc); } + if (flagInterruptMsgProc) return; } @@ -2447,6 +2460,7 @@ CConnman::CConnman(uint64_t nSeed0In, uint64_t nSeed1In) : semAddnode = nullptr; semMasternodeOutbound = nullptr; flagInterruptMsgProc = false; + SetTryNewOutboundPeer(false); Options connOptions; Init(connOptions); @@ -2565,6 +2579,7 @@ bool CConnman::Start(CScheduler& scheduler, const Options& connOptions) // // Start threads // + assert(m_msgproc); InterruptSocks5(false); interruptNet.reset(); flagInterruptMsgProc = false; @@ -2728,9 +2743,10 @@ void CConnman::DeleteNode(CNode* pnode) { assert(pnode); bool fUpdateConnectionTime = false; - GetNodeSignals().FinalizeNode(pnode->GetId(), fUpdateConnectionTime); - if(fUpdateConnectionTime) + m_msgproc->FinalizeNode(pnode->GetId(), fUpdateConnectionTime); + if(fUpdateConnectionTime) { addrman.Connected(pnode->addr); + } delete pnode; } @@ -3138,7 +3154,6 @@ CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn nSendVersion(0) { nServices = NODE_NONE; - nServicesExpected = NODE_NONE; hSocket = hSocketIn; nRecvVersion = INIT_PROTO_VERSION; nLastSend = 0; @@ -3153,7 +3168,7 @@ CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn strSubVer = ""; fWhitelisted = false; fOneShot = false; - fAddnode = false; + m_manual_connection = false; fClient = false; // set by version message fFeeler = false; fSuccessfullyConnected = false; diff --git a/src/net.h b/src/net.h index b3fd255a7e4c..2032a3aefef4 100644 --- a/src/net.h +++ b/src/net.h @@ -37,7 +37,6 @@ #include #endif -#include // "Optimistic send" was introduced in the beginning of the Bitcoin project. I assume this was done because it was // thought that "send" would be very cheap when the send buffer is empty. This is not true, as shown by profiling. @@ -111,8 +110,6 @@ static const bool DEFAULT_FORCEDNSSEED = false; static const size_t DEFAULT_MAXRECEIVEBUFFER = 5 * 1000; static const size_t DEFAULT_MAXSENDBUFFER = 1 * 1000; -static const ServiceFlags REQUIRED_SERVICES = NODE_NETWORK; - // NOTE: When adjusting this, update rpcnet:setban's help ("24h") static const unsigned int DEFAULT_MISBEHAVING_BANTIME = 60 * 60 * 24; // Default 24-hour ban @@ -142,7 +139,7 @@ struct CSerializedNetMsg std::string command; }; - +class NetEventsInterface; class CConnman { public: @@ -157,13 +154,13 @@ class CConnman struct Options { ServiceFlags nLocalServices = NODE_NONE; - ServiceFlags nRelevantServices = NODE_NONE; int nMaxConnections = 0; int nMaxOutbound = 0; int nMaxAddnode = 0; int nMaxFeeler = 0; int nBestHeight = 0; CClientUIInterface* uiInterface = nullptr; + NetEventsInterface* m_msgproc = nullptr; unsigned int nSendBufferMaxSize = 0; unsigned int nReceiveFloodSize = 0; uint64_t nMaxOutboundTimeframe = 0; @@ -175,13 +172,13 @@ class CConnman void Init(const Options& connOptions) { nLocalServices = connOptions.nLocalServices; - nRelevantServices = connOptions.nRelevantServices; nMaxConnections = connOptions.nMaxConnections; nMaxOutbound = std::min(connOptions.nMaxOutbound, connOptions.nMaxConnections); nMaxAddnode = connOptions.nMaxAddnode; nMaxFeeler = connOptions.nMaxFeeler; nBestHeight = connOptions.nBestHeight; clientInterface = connOptions.uiInterface; + m_msgproc = connOptions.m_msgproc; nSendBufferMaxSize = connOptions.nSendBufferMaxSize; nReceiveFloodSize = connOptions.nReceiveFloodSize; nMaxOutboundTimeframe = connOptions.nMaxOutboundTimeframe; @@ -196,7 +193,7 @@ class CConnman void Interrupt(); bool GetNetworkActive() const { return fNetworkActive; }; void SetNetworkActive(bool active); - bool OpenNetworkConnection(const CAddress& addrConnect, bool fCountFailure, CSemaphoreGrant *grantOutbound = nullptr, const char *strDest = nullptr, bool fOneShot = false, bool fFeeler = false, bool fAddnode = false, bool fConnectToMasternode = false); + bool OpenNetworkConnection(const CAddress& addrConnect, bool fCountFailure, CSemaphoreGrant *grantOutbound = nullptr, const char *strDest = nullptr, bool fOneShot = false, bool fFeeler = false, bool manual_connection = false, bool fConnectToMasternode = false); bool OpenMasternodeConnection(const CAddress& addrConnect); bool CheckIncomingNonce(uint64_t nonce); @@ -382,6 +379,19 @@ class CConnman void GetBanned(banmap_t &banmap); void SetBanned(const banmap_t &banmap); + // This allows temporarily exceeding nMaxOutbound, with the goal of finding + // a peer that is better than all our current peers. + void SetTryNewOutboundPeer(bool flag); + bool GetTryNewOutboundPeer(); + + // Return the number of outbound peers we have in excess of our target (eg, + // if we previously called SetTryNewOutboundPeer(true), and have since set + // to false, we may have extra peers that we wish to disconnect). This may + // return a value less than (num_outbound_connections - num_outbound_slots) + // in cases where some outbound connections are not yet fully connected, or + // not yet fully disconnected. + int GetExtraOutboundCount(); + bool AddNode(const std::string& node); bool RemoveAddedNode(const std::string& node); std::vector GetAddedNodeInfo(); @@ -532,9 +542,6 @@ class CConnman /** Services this instance offers */ ServiceFlags nLocalServices; - /** Services this instance cares about */ - ServiceFlags nRelevantServices; - CSemaphore *semOutbound; CSemaphore *semAddnode; CSemaphore *semMasternodeOutbound; @@ -544,6 +551,7 @@ class CConnman int nMaxFeeler; std::atomic nBestHeight; CClientUIInterface* clientInterface; + NetEventsInterface* m_msgproc; /** SipHasher seeds for deterministic randomness */ const uint64_t nSeed0, nSeed1; @@ -569,6 +577,13 @@ class CConnman std::thread threadOpenConnections; std::thread threadOpenMasternodeConnections; std::thread threadMessageHandler; + + /** flag for deciding to connect to an extra outbound peer, + * in excess of nMaxOutbound + * This takes the place of a feeler connection */ + std::atomic_bool m_try_another_outbound_peer; + + friend struct CConnmanTest; }; extern std::unique_ptr g_connman; void Discover(boost::thread_group& threadGroup); @@ -591,19 +606,18 @@ struct CombinerAll } }; -// Signals for message handling -struct CNodeSignals +/** + * Interface for message handling + */ +class NetEventsInterface { - boost::signals2::signal&), CombinerAll> ProcessMessages; - boost::signals2::signal&), CombinerAll> SendMessages; - boost::signals2::signal InitializeNode; - boost::signals2::signal FinalizeNode; +public: + virtual bool ProcessMessages(CNode* pnode, std::atomic& interrupt) = 0; + virtual bool SendMessages(CNode* pnode, std::atomic& interrupt) = 0; + virtual void InitializeNode(CNode* pnode) = 0; + virtual void FinalizeNode(NodeId id, bool& update_connection_time) = 0; }; - -CNodeSignals& GetNodeSignals(); - - enum { LOCAL_NONE, // unknown @@ -663,7 +677,7 @@ class CNodeStats int nVersion; std::string cleanSubVer; bool fInbound; - bool fAddnode; + bool m_manual_connection; int nStartingHeight; uint64_t nSendBytes; mapMsgCmdSize mapSendBytesPerMsgCmd; @@ -735,7 +749,6 @@ class CNode public: // socket std::atomic nServices; - ServiceFlags nServicesExpected; SOCKET hSocket; size_t nSendSize; // total size of all vSendMsg entries size_t nSendOffset; // offset inside the first vSendMsg already sent @@ -777,7 +790,7 @@ class CNode bool fWhitelisted; // This peer can bypass DoS banning. bool fFeeler; // If true this node is being used as a short lived feeler. bool fOneShot; - bool fAddnode; + bool m_manual_connection; bool fClient; const bool fInbound; std::atomic_bool fSuccessfullyConnected; diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 2d3dc4e8d06f..39d59f358b42 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -23,6 +23,7 @@ #include "primitives/transaction.h" #include "random.h" #include "reverse_iterator.h" +#include "scheduler.h" #include "tinyformat.h" #include "txmempool.h" #include "ui_interface.h" @@ -147,12 +148,11 @@ namespace { /** Number of peers from which we're downloading blocks. */ int nPeersWithValidatedDownloads = 0; - /* This comment is here to force a merge conflict when bitcoin#11560 is backported - * It introduces this member as int64_t while bitcoin#11824 changes it to atomic. - * bitcoin#11824 is partially backported already, which means you'll have to take the atomic - * version when you encounter this merge conflict! + /** Number of outbound peers with m_chain_sync.m_protect. */ + int g_outbound_peers_with_protect_from_disconnect = 0; + + /** When our tip was last updated. */ std::atomic g_last_tip_update(0); - */ /** Relay map, protected by cs_main. */ typedef std::map MapRelay; @@ -161,11 +161,6 @@ namespace { std::deque> vRelayExpiration; } // namespace -////////////////////////////////////////////////////////////////////////////// -// -// Registration of network node signals. -// - namespace { struct CBlockReject { @@ -228,6 +223,36 @@ struct CNodeState { */ bool fSupportsDesiredCmpctVersion; + /** State used to enforce CHAIN_SYNC_TIMEOUT + * Only in effect for outbound, non-manual connections, with + * m_protect == false + * Algorithm: if a peer's best known block has less work than our tip, + * set a timeout CHAIN_SYNC_TIMEOUT seconds in the future: + * - If at timeout their best known block now has more work than our tip + * when the timeout was set, then either reset the timeout or clear it + * (after comparing against our current tip's work) + * - If at timeout their best known block still has less work than our + * tip did when the timeout was set, then send a getheaders message, + * and set a shorter timeout, HEADERS_RESPONSE_TIME seconds in future. + * If their best known block is still behind when that new timeout is + * reached, disconnect. + */ + struct ChainSyncTimeoutState { + //! A timeout used for checking whether our peer has sufficiently synced + int64_t m_timeout; + //! A header with the work we require on our peer's chain + const CBlockIndex * m_work_header; + //! After timeout is reached, set to true after sending getheaders + bool m_sent_getheaders; + //! Whether this peer is protected from disconnection due to a bad/slow chain + bool m_protect; + }; + + ChainSyncTimeoutState m_chain_sync; + + //! Time of last new block announcement + int64_t m_last_block_announcement; + CNodeState(CAddress addrIn, std::string addrNameIn) : address(addrIn), name(addrNameIn) { fCurrentlyConnected = false; nMisbehavior = 0; @@ -248,6 +273,8 @@ struct CNodeState { fPreferHeaderAndIDs = false; fProvidesHeaderAndIDs = false; fSupportsDesiredCmpctVersion = false; + m_chain_sync = { 0, nullptr, false, false }; + m_last_block_announcement = 0; } }; @@ -272,7 +299,7 @@ void UpdatePreferredDownload(CNode* node, CNodeState* state) nPreferredDownload += state->fPreferredDownload; } -void PushNodeVersion(CNode *pnode, CConnman& connman, int64_t nTime) +void PushNodeVersion(CNode *pnode, CConnman* connman, int64_t nTime) { ServiceFlags nLocalNodeServices = pnode->GetLocalServices(); uint64_t nonce = pnode->GetLocalNonce(); @@ -290,7 +317,7 @@ void PushNodeVersion(CNode *pnode, CConnman& connman, int64_t nTime) pnode->sentMNAuthChallenge = mnauthChallenge; } - connman.PushMessage(pnode, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::VERSION, PROTOCOL_VERSION, (uint64_t)nLocalNodeServices, nTime, addrYou, addrMe, + connman->PushMessage(pnode, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::VERSION, PROTOCOL_VERSION, (uint64_t)nLocalNodeServices, nTime, addrYou, addrMe, nonce, strSubVersion, nNodeStartingHeight, ::fRelayTxes, mnauthChallenge)); if (fLogIPs) { @@ -300,49 +327,6 @@ void PushNodeVersion(CNode *pnode, CConnman& connman, int64_t nTime) } } -void InitializeNode(CNode *pnode, CConnman& connman) { - CAddress addr = pnode->addr; - std::string addrName = pnode->GetAddrName(); - NodeId nodeid = pnode->GetId(); - { - LOCK(cs_main); - mapNodeState.emplace_hint(mapNodeState.end(), std::piecewise_construct, std::forward_as_tuple(nodeid), std::forward_as_tuple(addr, std::move(addrName))); - } - if(!pnode->fInbound) - PushNodeVersion(pnode, connman, GetTime()); -} - -void FinalizeNode(NodeId nodeid, bool& fUpdateConnectionTime) { - fUpdateConnectionTime = false; - LOCK(cs_main); - CNodeState *state = State(nodeid); - - if (state->fSyncStarted) - nSyncStarted--; - - if (state->nMisbehavior == 0 && state->fCurrentlyConnected) { - fUpdateConnectionTime = true; - } - - for (const QueuedBlock& entry : state->vBlocksInFlight) { - mapBlocksInFlight.erase(entry.hash); - } - EraseOrphansFor(nodeid); - nPreferredDownload -= state->fPreferredDownload; - nPeersWithValidatedDownloads -= (state->nBlocksInFlightValidHeaders != 0); - assert(nPeersWithValidatedDownloads >= 0); - - mapNodeState.erase(nodeid); - - if (mapNodeState.empty()) { - // Do a consistency check after the last peer is removed. - assert(mapBlocksInFlight.empty()); - assert(nPreferredDownload == 0); - assert(nPeersWithValidatedDownloads == 0); - } - LogPrint(BCLog::NET, "Cleared nodestate for peer=%d\n", nodeid); -} - // Requires cs_main. // Returns a bool indicating whether we requested this block. // Also used if a block was /not/ received and timed out or started with another peer @@ -437,7 +421,7 @@ void UpdateBlockAvailability(NodeId nodeid, const uint256 &hash) { } } -void MaybeSetPeerAsAnnouncingHeaderAndIDs(NodeId nodeid, CConnman& connman) { +void MaybeSetPeerAsAnnouncingHeaderAndIDs(NodeId nodeid, CConnman* connman) { AssertLockHeld(cs_main); CNodeState* nodestate = State(nodeid); if (!nodestate || !nodestate->fSupportsDesiredCmpctVersion) { @@ -452,26 +436,35 @@ void MaybeSetPeerAsAnnouncingHeaderAndIDs(NodeId nodeid, CConnman& connman) { return; } } - connman.ForNode(nodeid, [&connman](CNode* pfrom){ + connman->ForNode(nodeid, [connman](CNode* pfrom){ bool fAnnounceUsingCMPCTBLOCK = false; uint64_t nCMPCTBLOCKVersion = 1; if (lNodesAnnouncingHeaderAndIDs.size() >= 3) { // As per BIP152, we only get 3 of our peers to announce // blocks using compact encodings. - connman.ForNode(lNodesAnnouncingHeaderAndIDs.front(), [&connman, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion](CNode* pnodeStop){ - connman.PushMessage(pnodeStop, CNetMsgMaker(pnodeStop->GetSendVersion()).Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); + connman->ForNode(lNodesAnnouncingHeaderAndIDs.front(), [connman, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion](CNode* pnodeStop){ + connman->PushMessage(pnodeStop, CNetMsgMaker(pnodeStop->GetSendVersion()).Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); return true; }); lNodesAnnouncingHeaderAndIDs.pop_front(); } fAnnounceUsingCMPCTBLOCK = true; - connman.PushMessage(pfrom, CNetMsgMaker(pfrom->GetSendVersion()).Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); + connman->PushMessage(pfrom, CNetMsgMaker(pfrom->GetSendVersion()).Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); lNodesAnnouncingHeaderAndIDs.push_back(pfrom->GetId()); return true; }); } } +bool TipMayBeStale(const Consensus::Params &consensusParams) +{ + AssertLockHeld(cs_main); + if (g_last_tip_update == 0) { + g_last_tip_update = GetTime(); + } + return g_last_tip_update < GetTime() - consensusParams.nPowTargetSpacing * 3 && mapBlocksInFlight.empty(); +} + // Requires cs_main bool CanDirectFetch(const Consensus::Params &consensusParams) { @@ -501,7 +494,7 @@ void FindNextBlocksToDownload(NodeId nodeid, unsigned int count, std::vectorpindexBestKnownBlock == nullptr || state->pindexBestKnownBlock->nChainWork < chainActive.Tip()->nChainWork || state->pindexBestKnownBlock->nChainWork < UintToArith256(consensusParams.nMinimumChainWork)) { + if (state->pindexBestKnownBlock == nullptr || state->pindexBestKnownBlock->nChainWork < chainActive.Tip()->nChainWork || state->pindexBestKnownBlock->nChainWork < nMinimumChainWork) { // This peer has nothing interesting. return; } @@ -574,6 +567,69 @@ void FindNextBlocksToDownload(NodeId nodeid, unsigned int count, std::vectorm_last_block_announcement = time_in_seconds; +} + +// Returns true for outbound peers, excluding manual connections, feelers, and +// one-shots +bool IsOutboundDisconnectionCandidate(const CNode *node) +{ + return !(node->fInbound || node->m_manual_connection || node->fFeeler || node->fOneShot); +} + +void PeerLogicValidation::InitializeNode(CNode *pnode) { + CAddress addr = pnode->addr; + std::string addrName = pnode->GetAddrName(); + NodeId nodeid = pnode->GetId(); + { + LOCK(cs_main); + mapNodeState.emplace_hint(mapNodeState.end(), std::piecewise_construct, std::forward_as_tuple(nodeid), std::forward_as_tuple(addr, std::move(addrName))); + } + if(!pnode->fInbound) + PushNodeVersion(pnode, connman, GetTime()); +} + +void PeerLogicValidation::FinalizeNode(NodeId nodeid, bool& fUpdateConnectionTime) { + fUpdateConnectionTime = false; + LOCK(cs_main); + CNodeState *state = State(nodeid); + assert(state != nullptr); + + if (state->fSyncStarted) + nSyncStarted--; + + if (state->nMisbehavior == 0 && state->fCurrentlyConnected) { + fUpdateConnectionTime = true; + } + + for (const QueuedBlock& entry : state->vBlocksInFlight) { + mapBlocksInFlight.erase(entry.hash); + } + EraseOrphansFor(nodeid); + nPreferredDownload -= state->fPreferredDownload; + nPeersWithValidatedDownloads -= (state->nBlocksInFlightValidHeaders != 0); + assert(nPeersWithValidatedDownloads >= 0); + g_outbound_peers_with_protect_from_disconnect -= state->m_chain_sync.m_protect; + assert(g_outbound_peers_with_protect_from_disconnect >= 0); + + mapNodeState.erase(nodeid); + + if (mapNodeState.empty()) { + // Do a consistency check after the last peer is removed. + assert(mapBlocksInFlight.empty()); + assert(nPreferredDownload == 0); + assert(nPeersWithValidatedDownloads == 0); + assert(g_outbound_peers_with_protect_from_disconnect == 0); + } + LogPrint(BCLog::NET, "Cleared nodestate for peer=%d\n", nodeid); +} + bool GetNodeStateStats(NodeId nodeid, CNodeStateStats &stats) { LOCK(cs_main); CNodeState *state = State(nodeid); @@ -589,22 +645,6 @@ bool GetNodeStateStats(NodeId nodeid, CNodeStateStats &stats) { return true; } -void RegisterNodeSignals(CNodeSignals& nodeSignals) -{ - nodeSignals.ProcessMessages.connect(&ProcessMessages); - nodeSignals.SendMessages.connect(&SendMessages); - nodeSignals.InitializeNode.connect(&InitializeNode); - nodeSignals.FinalizeNode.connect(&FinalizeNode); -} - -void UnregisterNodeSignals(CNodeSignals& nodeSignals) -{ - nodeSignals.ProcessMessages.disconnect(&ProcessMessages); - nodeSignals.SendMessages.disconnect(&SendMessages); - nodeSignals.InitializeNode.disconnect(&InitializeNode); - nodeSignals.FinalizeNode.disconnect(&FinalizeNode); -} - ////////////////////////////////////////////////////////////////////////////// // // mapOrphanTransactions @@ -782,9 +822,17 @@ static bool BlockRequestAllowed(const CBlockIndex* pindex, const Consensus::Para (GetBlockProofEquivalentTime(*pindexBestHeader, *pindex, *pindexBestHeader, consensusParams) < STALE_RELAY_AGE_LIMIT); } -PeerLogicValidation::PeerLogicValidation(CConnman* connmanIn) : connman(connmanIn) { +PeerLogicValidation::PeerLogicValidation(CConnman* connmanIn, CScheduler &scheduler) : connman(connmanIn), m_stale_tip_check_time(0) { // Initialize global variables that cannot be constructed at startup. recentRejects.reset(new CRollingBloomFilter(120000, 0.000001)); + + const Consensus::Params& consensusParams = Params().GetConsensus(); + // Stale tip checking and peer eviction are on two different timers, but we + // don't want them to get out of sync due to drift in the scheduler, so we + // combine them in one function and schedule at the quicker (peer-eviction) + // timer. + static_assert(EXTRA_PEER_CHECK_INTERVAL < STALE_CHECK_INTERVAL, "peer eviction timer should be less than stale tip check timer"); + scheduler.scheduleEvery(std::bind(&PeerLogicValidation::CheckForStaleTipAndEvictPeers, this, consensusParams), EXTRA_PEER_CHECK_INTERVAL * 1000); } void PeerLogicValidation::BlockConnected(const std::shared_ptr& pblock, const CBlockIndex* pindex, const std::vector& vtxConflicted) { @@ -815,6 +863,8 @@ void PeerLogicValidation::BlockConnected(const std::shared_ptr& pb } LogPrint(BCLog::MEMPOOL, "Erased %d orphan tx included or conflicted by block\n", nErased); } + + g_last_tip_update = GetTime(); } // All of the following cache a recent block, and are protected by cs_most_recent_block @@ -919,7 +969,7 @@ void PeerLogicValidation::BlockChecked(const CBlock& block, const CValidationSta !IsInitialBlockDownload() && mapBlocksInFlight.count(hash) == mapBlocksInFlight.size()) { if (it != mapBlockSource.end()) { - MaybeSetPeerAsAnnouncingHeaderAndIDs(it->second.first, *connman); + MaybeSetPeerAsAnnouncingHeaderAndIDs(it->second.first, connman); } } if (it != mapBlockSource.end()) @@ -1007,7 +1057,7 @@ bool static AlreadyHave(const CInv& inv) EXCLUSIVE_LOCKS_REQUIRED(cs_main) return true; } -static void RelayAddress(const CAddress& addr, bool fReachable, CConnman& connman) +static void RelayAddress(const CAddress& addr, bool fReachable, CConnman* connman) { unsigned int nRelayNodes = fReachable ? 2 : 1; // limited relaying of addresses outside our network(s) @@ -1015,7 +1065,7 @@ static void RelayAddress(const CAddress& addr, bool fReachable, CConnman& connma // Use deterministic randomness to send to the same nodes for 24 hours // at a time so the addrKnowns of the chosen nodes prevent repeats uint64_t hashAddr = addr.GetHash(); - const CSipHasher hasher = connman.GetDeterministicRandomizer(RANDOMIZER_ID_ADDRESS_RELAY).Write(hashAddr << 32).Write((GetTime() + hashAddr) / (24*60*60)); + const CSipHasher hasher = connman->GetDeterministicRandomizer(RANDOMIZER_ID_ADDRESS_RELAY).Write(hashAddr << 32).Write((GetTime() + hashAddr) / (24*60*60)); FastRandomContext insecure_rand; std::array,2> best{{{0, nullptr}, {0, nullptr}}}; @@ -1040,10 +1090,10 @@ static void RelayAddress(const CAddress& addr, bool fReachable, CConnman& connma } }; - connman.ForEachNodeThen(std::move(sortfunc), std::move(pushfunc)); + connman->ForEachNodeThen(std::move(sortfunc), std::move(pushfunc)); } -void static ProcessGetBlockData(CNode* pfrom, const Consensus::Params& consensusParams, const CInv& inv, CConnman& connman, const std::atomic& interruptMsgProc) +void static ProcessGetBlockData(CNode* pfrom, const Consensus::Params& consensusParams, const CInv& inv, CConnman* connman, const std::atomic& interruptMsgProc) { bool send = false; std::shared_ptr a_recent_block; @@ -1087,7 +1137,7 @@ void static ProcessGetBlockData(CNode* pfrom, const Consensus::Params& consensus const CNetMsgMaker msgMaker(pfrom->GetSendVersion()); // disconnect node in case we have reached the outbound limit for serving historical blocks // never disconnect whitelisted nodes - if (send && connman.OutboundTargetReached(true) && ( ((pindexBestHeader != nullptr) && (pindexBestHeader->GetBlockTime() - mi->second->GetBlockTime() > HISTORICAL_BLOCK_AGE)) || inv.type == MSG_FILTERED_BLOCK) && !pfrom->fWhitelisted) + if (send && connman->OutboundTargetReached(true) && ( ((pindexBestHeader != nullptr) && (pindexBestHeader->GetBlockTime() - mi->second->GetBlockTime() > HISTORICAL_BLOCK_AGE)) || inv.type == MSG_FILTERED_BLOCK) && !pfrom->fWhitelisted) { LogPrint(BCLog::NET, "historical block serving limit reached, disconnect peer=%d\n", pfrom->GetId()); @@ -1110,7 +1160,7 @@ void static ProcessGetBlockData(CNode* pfrom, const Consensus::Params& consensus pblock = pblockRead; } if (inv.type == MSG_BLOCK) - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::BLOCK, *pblock)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::BLOCK, *pblock)); else if (inv.type == MSG_FILTERED_BLOCK) { bool sendMerkleBlock = false; @@ -1123,7 +1173,7 @@ void static ProcessGetBlockData(CNode* pfrom, const Consensus::Params& consensus } } if (sendMerkleBlock) { - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::MERKLEBLOCK, merkleBlock)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::MERKLEBLOCK, merkleBlock)); // CMerkleBlock just contains hashes, so also push any transactions in the block the client did not see // This avoids hurting performance by pointlessly requiring a round-trip // Note that there is currently no way for a node to request any single transactions we didn't send here - @@ -1132,7 +1182,7 @@ void static ProcessGetBlockData(CNode* pfrom, const Consensus::Params& consensus // however we MUST always provide at least what the remote peer needs typedef std::pair PairType; for (PairType& pair : merkleBlock.vMatchedTxn) - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::TX, *pblock->vtx[pair.first])); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::TX, *pblock->vtx[pair.first])); } // else // no response @@ -1145,13 +1195,13 @@ void static ProcessGetBlockData(CNode* pfrom, const Consensus::Params& consensus // instead we respond with the full, non-compact block. if (CanDirectFetch(consensusParams) && mi->second->nHeight >= chainActive.Height() - MAX_CMPCTBLOCK_DEPTH) { if (a_recent_compact_block && a_recent_compact_block->header.GetHash() == mi->second->GetBlockHash()) { - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::CMPCTBLOCK, *a_recent_compact_block)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::CMPCTBLOCK, *a_recent_compact_block)); } else { CBlockHeaderAndShortTxIDs cmpctblock(*pblock); - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::CMPCTBLOCK, cmpctblock)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::CMPCTBLOCK, cmpctblock)); } } else { - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::BLOCK, *pblock)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::BLOCK, *pblock)); } } @@ -1163,13 +1213,13 @@ void static ProcessGetBlockData(CNode* pfrom, const Consensus::Params& consensus // wait for other stuff first. std::vector vInv; vInv.push_back(CInv(MSG_BLOCK, chainActive.Tip()->GetBlockHash())); - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::INV, vInv)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::INV, vInv)); pfrom->hashContinue.SetNull(); } } } -void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParams, CConnman& connman, const std::atomic& interruptMsgProc) +void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParams, CConnman* connman, const std::atomic& interruptMsgProc) { AssertLockNotHeld(cs_main); @@ -1197,14 +1247,14 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam if (inv.type == MSG_TX) { auto mi = mapRelay.find(inv.hash); if (mi != mapRelay.end()) { - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::TX, *mi->second)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::TX, *mi->second)); push = true; } else if (pfrom->timeLastMempoolReq) { auto txinfo = mempool.info(inv.hash); // To protect privacy, do not answer getdata using the mempool when // that TX couldn't have been INVed in reply to a MEMPOOL request. if (txinfo.tx && txinfo.nTime <= pfrom->timeLastMempoolReq) { - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::TX, *txinfo.tx)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::TX, *txinfo.tx)); push = true; } } @@ -1213,7 +1263,7 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam if (!push && inv.type == MSG_SPORK) { CSporkMessage spork; if(sporkManager.GetSporkByHash(inv.hash, spork)) { - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::SPORK, spork)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::SPORK, spork)); push = true; } } @@ -1221,7 +1271,7 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam if (!push && inv.type == MSG_DSTX) { CPrivateSendBroadcastTx dstx = CPrivateSend::GetDSTX(inv.hash); if(dstx) { - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::DSTX, dstx)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::DSTX, dstx)); push = true; } } @@ -1240,7 +1290,7 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam } LogPrint(BCLog::NET, "ProcessGetData -- MSG_GOVERNANCE_OBJECT: topush = %d, inv = %s\n", topush, inv.ToString()); if(topush) { - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::MNGOVERNANCEOBJECT, ss)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::MNGOVERNANCEOBJECT, ss)); push = true; } } @@ -1258,7 +1308,7 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam } if(topush) { LogPrint(BCLog::NET, "ProcessGetData -- pushing: inv = %s\n", inv.ToString()); - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::MNGOVERNANCEOBJECTVOTE, ss)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::MNGOVERNANCEOBJECTVOTE, ss)); push = true; } } @@ -1266,7 +1316,7 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam if (!push && (inv.type == MSG_QUORUM_FINAL_COMMITMENT)) { llmq::CFinalCommitment o; if (llmq::quorumBlockProcessor->GetMinableCommitmentByHash(inv.hash, o)) { - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::QFCOMMITMENT, o)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::QFCOMMITMENT, o)); push = true; } } @@ -1274,35 +1324,35 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam if (!push && (inv.type == MSG_QUORUM_CONTRIB)) { llmq::CDKGContribution o; if (llmq::quorumDKGSessionManager->GetContribution(inv.hash, o)) { - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::QCONTRIB, o)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::QCONTRIB, o)); push = true; } } if (!push && (inv.type == MSG_QUORUM_COMPLAINT)) { llmq::CDKGComplaint o; if (llmq::quorumDKGSessionManager->GetComplaint(inv.hash, o)) { - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::QCOMPLAINT, o)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::QCOMPLAINT, o)); push = true; } } if (!push && (inv.type == MSG_QUORUM_JUSTIFICATION)) { llmq::CDKGJustification o; if (llmq::quorumDKGSessionManager->GetJustification(inv.hash, o)) { - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::QJUSTIFICATION, o)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::QJUSTIFICATION, o)); push = true; } } if (!push && (inv.type == MSG_QUORUM_PREMATURE_COMMITMENT)) { llmq::CDKGPrematureCommitment o; if (llmq::quorumDKGSessionManager->GetPrematureCommitment(inv.hash, o)) { - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::QPCOMMITMENT, o)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::QPCOMMITMENT, o)); push = true; } } if (!push && (inv.type == MSG_QUORUM_RECOVERED_SIG)) { llmq::CRecoveredSig o; if (llmq::quorumSigningManager->GetRecoveredSigForGetData(inv.hash, o)) { - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::QSIGREC, o)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::QSIGREC, o)); push = true; } } @@ -1310,7 +1360,7 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam if (!push && (inv.type == MSG_CLSIG)) { llmq::CChainLockSig o; if (llmq::chainLocksHandler->GetChainLockByHash(inv.hash, o)) { - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::CLSIG, o)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::CLSIG, o)); push = true; } } @@ -1318,7 +1368,7 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam if (!push && (inv.type == MSG_ISLOCK)) { llmq::CInstantSendLock o; if (llmq::quorumInstantSendManager->GetInstantSendLockByHash(inv.hash, o)) { - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::ISLOCK, o)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::ISLOCK, o)); push = true; } } @@ -1349,11 +1399,11 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam // do that because they want to know about (and store and rebroadcast and // risk analyze) the dependencies of transactions relevant to them, without // having to download the entire memory pool. - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::NOTFOUND, vNotFound)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::NOTFOUND, vNotFound)); } } -inline void static SendBlockTransactions(const CBlock& block, const BlockTransactionsRequest& req, CNode* pfrom, CConnman& connman) { +inline void static SendBlockTransactions(const CBlock& block, const BlockTransactionsRequest& req, CNode* pfrom, CConnman* connman) { BlockTransactions resp(req); for (size_t i = 0; i < req.indexes.size(); i++) { if (req.indexes[i] >= block.vtx.size()) { @@ -1366,10 +1416,227 @@ inline void static SendBlockTransactions(const CBlock& block, const BlockTransac } LOCK(cs_main); CNetMsgMaker msgMaker(pfrom->GetSendVersion()); - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::BLOCKTXN, resp)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::BLOCKTXN, resp)); } -bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vRecv, int64_t nTimeReceived, const CChainParams& chainparams, CConnman& connman, const std::atomic& interruptMsgProc) +bool static ProcessHeadersMessage(CNode *pfrom, CConnman *connman, const std::vector& headers, const CChainParams& chainparams, bool punish_duplicate_invalid) +{ + const CNetMsgMaker msgMaker(pfrom->GetSendVersion()); + size_t nCount = headers.size(); + + if (nCount == 0) { + // Nothing interesting. Stop asking this peers for more headers. + return true; + } + + bool received_new_header = false; + const CBlockIndex *pindexLast = nullptr; + { + LOCK(cs_main); + CNodeState *nodestate = State(pfrom->GetId()); + + // If this looks like it could be a block announcement (nCount < + // MAX_BLOCKS_TO_ANNOUNCE), use special logic for handling headers that + // don't connect: + // - Send a getheaders message in response to try to connect the chain. + // - The peer can send up to MAX_UNCONNECTING_HEADERS in a row that + // don't connect before giving DoS points + // - Once a headers message is received that is valid and does connect, + // nUnconnectingHeaders gets reset back to 0. + if (mapBlockIndex.find(headers[0].hashPrevBlock) == mapBlockIndex.end() && nCount < MAX_BLOCKS_TO_ANNOUNCE) { + nodestate->nUnconnectingHeaders++; + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), uint256())); + LogPrint(BCLog::NET, "received header %s: missing prev block %s, sending getheaders (%d) to end (peer=%d, nUnconnectingHeaders=%d)\n", + headers[0].GetHash().ToString(), + headers[0].hashPrevBlock.ToString(), + pindexBestHeader->nHeight, + pfrom->GetId(), nodestate->nUnconnectingHeaders); + // Set hashLastUnknownBlock for this peer, so that if we + // eventually get the headers - even from a different peer - + // we can use this peer to download. + UpdateBlockAvailability(pfrom->GetId(), headers.back().GetHash()); + + if (nodestate->nUnconnectingHeaders % MAX_UNCONNECTING_HEADERS == 0) { + Misbehaving(pfrom->GetId(), 20); + } + return true; + } + + uint256 hashLastBlock; + for (const CBlockHeader& header : headers) { + if (!hashLastBlock.IsNull() && header.hashPrevBlock != hashLastBlock) { + Misbehaving(pfrom->GetId(), 20); + return error("non-continuous headers sequence"); + } + hashLastBlock = header.GetHash(); + } + + // If we don't have the last header, then they'll have given us + // something new (if these headers are valid). + if (mapBlockIndex.find(hashLastBlock) == mapBlockIndex.end()) { + received_new_header = true; + } + } + + CValidationState state; + CBlockHeader first_invalid_header; + if (!ProcessNewBlockHeaders(headers, state, chainparams, &pindexLast, &first_invalid_header)) { + int nDoS; + if (state.IsInvalid(nDoS)) { + LOCK(cs_main); + if (nDoS > 0) { + Misbehaving(pfrom->GetId(), nDoS); + } + if (punish_duplicate_invalid && mapBlockIndex.find(first_invalid_header.GetHash()) != mapBlockIndex.end()) { + // Goal: don't allow outbound peers to use up our outbound + // connection slots if they are on incompatible chains. + // + // We ask the caller to set punish_invalid appropriately based + // on the peer and the method of header delivery (compact + // blocks are allowed to be invalid in some circumstances, + // under BIP 152). + // Here, we try to detect the narrow situation that we have a + // valid block header (ie it was valid at the time the header + // was received, and hence stored in mapBlockIndex) but know the + // block is invalid, and that a peer has announced that same + // block as being on its active chain. + // Disconnect the peer in such a situation. + // + // Note: if the header that is invalid was not accepted to our + // mapBlockIndex at all, that may also be grounds for + // disconnecting the peer, as the chain they are on is likely + // to be incompatible. However, there is a circumstance where + // that does not hold: if the header's timestamp is more than + // 2 hours ahead of our current time. In that case, the header + // may become valid in the future, and we don't want to + // disconnect a peer merely for serving us one too-far-ahead + // block header, to prevent an attacker from splitting the + // network by mining a block right at the 2 hour boundary. + // + // TODO: update the DoS logic (or, rather, rewrite the + // DoS-interface between validation and net_processing) so that + // the interface is cleaner, and so that we disconnect on all the + // reasons that a peer's headers chain is incompatible + // with ours (eg block->nVersion softforks, MTP violations, + // etc), and not just the duplicate-invalid case. + pfrom->fDisconnect = true; + } + return error("invalid header received"); + } + } + + { + LOCK(cs_main); + CNodeState *nodestate = State(pfrom->GetId()); + if (nodestate->nUnconnectingHeaders > 0) { + LogPrint(BCLog::NET, "peer=%d: resetting nUnconnectingHeaders (%d -> 0)\n", pfrom->GetId(), nodestate->nUnconnectingHeaders); + } + nodestate->nUnconnectingHeaders = 0; + + assert(pindexLast); + UpdateBlockAvailability(pfrom->GetId(), pindexLast->GetBlockHash()); + + // From here, pindexBestKnownBlock should be guaranteed to be non-null, + // because it is set in UpdateBlockAvailability. Some nullptr checks + // are still present, however, as belt-and-suspenders. + + if (received_new_header && pindexLast->nChainWork > chainActive.Tip()->nChainWork) { + nodestate->m_last_block_announcement = GetTime(); + } + + if (nCount == MAX_HEADERS_RESULTS) { + // Headers message had its maximum size; the peer may have more headers. + // TODO: optimize: if pindexLast is an ancestor of chainActive.Tip or pindexBestHeader, continue + // from there instead. + LogPrint(BCLog::NET, "more getheaders (%d) to end to peer=%d (startheight:%d)\n", pindexLast->nHeight, pfrom->GetId(), pfrom->nStartingHeight); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexLast), uint256())); + } + + bool fCanDirectFetch = CanDirectFetch(chainparams.GetConsensus()); + // If this set of headers is valid and ends in a block with at least as + // much work as our tip, download as much as possible. + if (fCanDirectFetch && pindexLast->IsValid(BLOCK_VALID_TREE) && chainActive.Tip()->nChainWork <= pindexLast->nChainWork) { + std::vector vToFetch; + const CBlockIndex *pindexWalk = pindexLast; + // Calculate all the blocks we'd need to switch to pindexLast, up to a limit. + while (pindexWalk && !chainActive.Contains(pindexWalk) && vToFetch.size() <= MAX_BLOCKS_IN_TRANSIT_PER_PEER) { + if (!(pindexWalk->nStatus & BLOCK_HAVE_DATA) && + !mapBlocksInFlight.count(pindexWalk->GetBlockHash())) { + // We don't have this block, and it's not yet in flight. + vToFetch.push_back(pindexWalk); + } + pindexWalk = pindexWalk->pprev; + } + // If pindexWalk still isn't on our main chain, we're looking at a + // very large reorg at a time we think we're close to caught up to + // the main chain -- this shouldn't really happen. Bail out on the + // direct fetch and rely on parallel download instead. + if (!chainActive.Contains(pindexWalk)) { + LogPrint(BCLog::NET, "Large reorg, won't direct fetch to %s (%d)\n", + pindexLast->GetBlockHash().ToString(), + pindexLast->nHeight); + } else { + std::vector vGetData; + // Download as much as possible, from earliest to latest. + for (const CBlockIndex *pindex : reverse_iterate(vToFetch)) { + if (nodestate->nBlocksInFlight >= MAX_BLOCKS_IN_TRANSIT_PER_PEER) { + // Can't download any more from this peer + break; + } + vGetData.push_back(CInv(MSG_BLOCK, pindex->GetBlockHash())); + MarkBlockAsInFlight(pfrom->GetId(), pindex->GetBlockHash(), pindex); + LogPrint(BCLog::NET, "Requesting block %s from peer=%d\n", + pindex->GetBlockHash().ToString(), pfrom->GetId()); + } + if (vGetData.size() > 1) { + LogPrint(BCLog::NET, "Downloading blocks toward %s (%d) via headers direct fetch\n", + pindexLast->GetBlockHash().ToString(), pindexLast->nHeight); + } + if (vGetData.size() > 0) { + if (nodestate->fSupportsDesiredCmpctVersion && vGetData.size() == 1 && mapBlocksInFlight.size() == 1 && pindexLast->pprev->IsValid(BLOCK_VALID_CHAIN)) { + // In any case, we want to download using a compact block, not a regular one + vGetData[0] = CInv(MSG_CMPCT_BLOCK, vGetData[0].hash); + } + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vGetData)); + } + } + } + // If we're in IBD, we want outbound peers that will serve us a useful + // chain. Disconnect peers that are on chains with insufficient work. + if (IsInitialBlockDownload() && nCount != MAX_HEADERS_RESULTS) { + // When nCount < MAX_HEADERS_RESULTS, we know we have no more + // headers to fetch from this peer. + if (nodestate->pindexBestKnownBlock && nodestate->pindexBestKnownBlock->nChainWork < nMinimumChainWork) { + // This peer has too little work on their headers chain to help + // us sync -- disconnect if using an outbound slot (unless + // whitelisted or addnode). + // Note: We compare their tip to nMinimumChainWork (rather than + // chainActive.Tip()) because we won't start block download + // until we have a headers chain that has at least + // nMinimumChainWork, even if a peer has a chain past our tip, + // as an anti-DoS measure. + if (IsOutboundDisconnectionCandidate(pfrom)) { + LogPrintf("Disconnecting outbound peer %d -- headers chain has insufficient work\n", pfrom->GetId()); + pfrom->fDisconnect = true; + } + } + } + + if (!pfrom->fDisconnect && IsOutboundDisconnectionCandidate(pfrom) && nodestate->pindexBestKnownBlock != nullptr) { + // If this is an outbound peer, check to see if we should protect + // it from the bad/lagging chain logic. + if (g_outbound_peers_with_protect_from_disconnect < MAX_OUTBOUND_PEERS_TO_PROTECT_FROM_DISCONNECT && nodestate->pindexBestKnownBlock->nChainWork >= chainActive.Tip()->nChainWork && !nodestate->m_chain_sync.m_protect) { + LogPrint(BCLog::NET, "Protecting outbound peer=%d from eviction\n", pfrom->GetId()); + nodestate->m_chain_sync.m_protect = true; + ++g_outbound_peers_with_protect_from_disconnect; + } + } + } + + return true; +} + +bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vRecv, int64_t nTimeReceived, const CChainParams& chainparams, CConnman* connman, const std::atomic& interruptMsgProc) { LogPrint(BCLog::NET, "received: %s (%u bytes) peer=%d\n", SanitizeString(strCommand), vRecv.size(), pfrom->GetId()); if (gArgs.IsArgSet("-dropmessagestest") && GetRand(gArgs.GetArg("-dropmessagestest", 0)) == 0) @@ -1430,7 +1697,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // Each connection can only send one version message if (pfrom->nVersion != 0) { - connman.PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::REJECT, strCommand, REJECT_DUPLICATE, std::string("Duplicate version message"))); + connman->PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::REJECT, strCommand, REJECT_DUPLICATE, std::string("Duplicate version message"))); LOCK(cs_main); Misbehaving(pfrom->GetId(), 1); return false; @@ -1454,13 +1721,13 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr nServices = ServiceFlags(nServiceInt); if (!pfrom->fInbound) { - connman.SetServices(pfrom->addr, nServices); + connman->SetServices(pfrom->addr, nServices); } - if (pfrom->nServicesExpected & ~nServices) + if (!pfrom->fInbound && !pfrom->fFeeler && !pfrom->m_manual_connection && !HasAllDesirableServiceFlags(nServices)) { - LogPrint(BCLog::NET, "peer=%d does not offer the expected services (%08x offered, %08x expected); disconnecting\n", pfrom->GetId(), nServices, pfrom->nServicesExpected); - connman.PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::REJECT, strCommand, REJECT_NONSTANDARD, - strprintf("Expected to offer services %08x", pfrom->nServicesExpected))); + LogPrint(BCLog::NET, "peer=%d does not offer the expected services (%08x offered, %08x expected); disconnecting\n", pfrom->GetId(), nServices, GetDesirableServiceFlags(nServices)); + connman->PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::REJECT, strCommand, REJECT_NONSTANDARD, + strprintf("Expected to offer services %08x", GetDesirableServiceFlags(nServices)))); pfrom->fDisconnect = true; return false; } @@ -1469,7 +1736,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr { // disconnect from peers older than this proto version LogPrintf("peer=%d using obsolete version %i; disconnecting\n", pfrom->GetId(), nVersion); - connman.PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::REJECT, strCommand, REJECT_OBSOLETE, + connman->PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::REJECT, strCommand, REJECT_OBSOLETE, strprintf("Version must be %d or greater", MIN_PEER_PROTO_VERSION))); pfrom->fDisconnect = true; return false; @@ -1493,7 +1760,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr vRecv >> pfrom->receivedMNAuthChallenge; } // Disconnect if we connected to ourself - if (pfrom->fInbound && !connman.CheckIncomingNonce(nNonce)) + if (pfrom->fInbound && !connman->CheckIncomingNonce(nNonce)) { LogPrintf("connected to self at %s, disconnecting\n", pfrom->addr.ToString()); pfrom->fDisconnect = true; @@ -1522,7 +1789,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr } } - connman.PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::VERACK)); + connman->PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::VERACK)); pfrom->nServices = nServices; pfrom->SetAddrLocal(addrMe); @@ -1567,12 +1834,12 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr } // Get recent addresses - if (pfrom->fOneShot || pfrom->nVersion >= CADDR_TIME_VERSION || connman.GetAddressCount() < 1000) + if (pfrom->fOneShot || pfrom->nVersion >= CADDR_TIME_VERSION || connman->GetAddressCount() < 1000) { - connman.PushMessage(pfrom, CNetMsgMaker(nSendVersion).Make(NetMsgType::GETADDR)); + connman->PushMessage(pfrom, CNetMsgMaker(nSendVersion).Make(NetMsgType::GETADDR)); pfrom->fGetAddr = true; } - connman.MarkAddressGood(pfrom->addr); + connman->MarkAddressGood(pfrom->addr); } std::string remoteAddr; @@ -1619,7 +1886,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr } if (pfrom->nVersion >= LLMQS_PROTO_VERSION) { - CMNAuth::PushMNAUTH(pfrom, connman); + CMNAuth::PushMNAUTH(pfrom, *connman); } if (pfrom->nVersion >= SENDHEADERS_VERSION) { @@ -1627,7 +1894,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // We send this to non-NODE NETWORK peers as well, because even // non-NODE NETWORK peers can announce blocks (such as pruning // nodes) - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::SENDHEADERS)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::SENDHEADERS)); } if (pfrom->nVersion >= SHORT_IDS_BLOCKS_VERSION) { @@ -1638,12 +1905,12 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // they may wish to request compact blocks from us bool fAnnounceUsingCMPCTBLOCK = false; uint64_t nCMPCTBLOCKVersion = 1; - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); } if (pfrom->nVersion >= SENDDSQUEUE_PROTO_VERSION) { // Tell our peer that he should send us PrivateSend queue messages - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::SENDDSQUEUE, true)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::SENDDSQUEUE, true)); } else { // older nodes do not support SENDDSQUEUE and expect us to always send PrivateSend queue messages // TODO we can remove this compatibility code in 0.15.0 @@ -1655,11 +1922,11 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // Otherwise the peer would only announce/send messages resulting from QRECSIG, // e.g. InstantSend locks or ChainLocks. SPV nodes should not send this message // as they are usually only interested in the higher level messages - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::QSENDRECSIGS, true)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::QSENDRECSIGS, true)); } if (gArgs.GetBoolArg("-watchquorums", llmq::DEFAULT_WATCH_QUORUMS)) { - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::QWATCH)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::QWATCH)); } pfrom->fSuccessfullyConnected = true; @@ -1679,7 +1946,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr vRecv >> vAddr; // Don't want addr from older versions unless seeding - if (pfrom->nVersion < CADDR_TIME_VERSION && connman.GetAddressCount() > 1000) + if (pfrom->nVersion < CADDR_TIME_VERSION && connman->GetAddressCount() > 1000) return true; if (vAddr.size() > 1000) { @@ -1697,7 +1964,10 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (interruptMsgProc) return true; - if ((addr.nServices & REQUIRED_SERVICES) != REQUIRED_SERVICES) + // We only bother storing full nodes, though this may include + // things which we would not make an outbound connection to, in + // part because we may make feeler connections to them. + if (!MayHaveUsefulAddressDB(addr.nServices)) continue; if (addr.nTime <= 100000000 || addr.nTime > nNow + 10 * 60) @@ -1712,7 +1982,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (fReachable) vAddrOk.push_back(addr); } - connman.AddNewAddresses(vAddrOk, pfrom->addr, 2 * 60 * 60); + connman->AddNewAddresses(vAddrOk, pfrom->addr, 2 * 60 * 60); if (vAddr.size() < 1000) pfrom->fGetAddr = false; if (pfrom->fOneShot) @@ -1812,7 +2082,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // fell back to inv we probably have a reorg which we should get the headers for first, // we now only provide a getheaders response here. When we receive the headers, we will // then ask for the blocks we need. - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), inv.hash)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), inv.hash)); LogPrint(BCLog::NET, "getheaders (%d) %s to peer=%d\n", pindexBestHeader->nHeight, inv.hash.ToString(), pfrom->GetId()); } } @@ -2044,7 +2314,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // will re-announce the new block via headers (or compact blocks again) // in the SendMessages logic. nodestate->pindexBestHeaderSent = pindex ? pindex : chainActive.Tip(); - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::HEADERS, vHeaders)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::HEADERS, vHeaders)); } @@ -2081,7 +2351,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr pfrom->AddInventoryKnown(inv); { LOCK(cs_main); - connman.RemoveAskFor(inv.hash); + connman->RemoveAskFor(inv.hash); } // Process custom logic, no matter if tx will be accepted to mempool later or not @@ -2133,7 +2403,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr } mempool.check(pcoinsTip); - connman.RelayTransaction(tx); + connman->RelayTransaction(tx); for (unsigned int i = 0; i < tx.vout.size(); i++) { vWorkQueue.emplace_back(inv.hash, i); } @@ -2171,7 +2441,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr continue; if (AcceptToMemoryPool(mempool, stateDummy, porphanTx, true, &fMissingInputs2)) { LogPrint(BCLog::MEMPOOL, " accepted orphan tx %s\n", orphanHash.ToString()); - connman.RelayTransaction(orphanTx); + connman->RelayTransaction(orphanTx); for (unsigned int i = 0; i < orphanTx.vout.size(); i++) { vWorkQueue.emplace_back(orphanHash, i); } @@ -2253,7 +2523,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr int nDoS = 0; if (!state.IsInvalid(nDoS) || nDoS == 0) { LogPrintf("Force relaying tx %s from whitelisted peer=%d\n", tx.GetHash().ToString(), pfrom->GetId()); - connman.RelayTransaction(tx); + connman->RelayTransaction(tx); } else { LogPrintf("Not relaying invalid transaction %s from whitelisted peer=%d (%s)\n", tx.GetHash().ToString(), pfrom->GetId(), FormatStateMessage(state)); } @@ -2267,7 +2537,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr pfrom->GetId(), FormatStateMessage(state)); if (state.GetRejectCode() > 0 && state.GetRejectCode() < REJECT_INTERNAL) // Never send AcceptToMemoryPool's internal codes over P2P - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::REJECT, strCommand, (unsigned char)state.GetRejectCode(), + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::REJECT, strCommand, (unsigned char)state.GetRejectCode(), state.GetRejectReason().substr(0, MAX_REJECT_MESSAGE_LENGTH), inv.hash)); if (nDoS > 0) { Misbehaving(pfrom->GetId(), nDoS); @@ -2280,15 +2550,21 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr CBlockHeaderAndShortTxIDs cmpctblock; vRecv >> cmpctblock; + bool received_new_header = false; + { LOCK(cs_main); if (mapBlockIndex.find(cmpctblock.header.hashPrevBlock) == mapBlockIndex.end()) { // Doesn't connect (or is genesis), instead of DoSing in AcceptBlockHeader, request deeper headers if (!IsInitialBlockDownload()) - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), uint256())); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), uint256())); return true; } + + if (mapBlockIndex.find(cmpctblock.header.GetHash()) == mapBlockIndex.end()) { + received_new_header = true; + } } const CBlockIndex *pindex = nullptr; @@ -2315,7 +2591,6 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // If we end up treating this as a plain headers message, call that as well // without cs_main. bool fRevertToHeaderProcessing = false; - CDataStream vHeadersMsg(SER_NETWORK, PROTOCOL_VERSION); // Keep a CBlock for "optimistic" compactblock reconstructions (see // below) @@ -2328,6 +2603,14 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr assert(pindex); UpdateBlockAvailability(pfrom->GetId(), pindex->GetBlockHash()); + CNodeState *nodestate = State(pfrom->GetId()); + + // If this was a new header with more work than our tip, update the + // peer's last block announcement time + if (received_new_header && pindex->nChainWork > chainActive.Tip()->nChainWork) { + nodestate->m_last_block_announcement = GetTime(); + } + std::map::iterator> >::iterator blockInFlightIt = mapBlocksInFlight.find(pindex->GetBlockHash()); bool fAlreadyInFlight = blockInFlightIt != mapBlocksInFlight.end(); @@ -2341,7 +2624,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // so we just grab the block via normal getdata std::vector vInv(1); vInv[0] = CInv(MSG_BLOCK, cmpctblock.header.GetHash()); - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); } return true; } @@ -2350,8 +2633,6 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (!fAlreadyInFlight && !CanDirectFetch(chainparams.GetConsensus())) return true; - CNodeState *nodestate = State(pfrom->GetId()); - // We want to be a bit conservative just to be extra careful about DoS // possibilities in compact block processing... if (pindex->nHeight <= chainActive.Height() + 2) { @@ -2379,7 +2660,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // Duplicate txindexes, the block is now in-flight, so just request it std::vector vInv(1); vInv[0] = CInv(MSG_BLOCK, cmpctblock.header.GetHash()); - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); return true; } @@ -2396,7 +2677,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr fProcessBLOCKTXN = true; } else { req.blockhash = pindex->GetBlockHash(); - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETBLOCKTXN, req)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETBLOCKTXN, req)); } } else { // This block is either already in flight from a different @@ -2422,14 +2703,10 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // mempool will probably be useless - request the block normally std::vector vInv(1); vInv[0] = CInv(MSG_BLOCK, cmpctblock.header.GetHash()); - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); return true; } else { // If this was an announce-cmpctblock, we want the same treatment as a header message - // Dirty hack to process as if it were just a headers message (TODO: move message handling into their own functions) - std::vector headers; - headers.push_back(cmpctblock.header); - vHeadersMsg << headers; fRevertToHeaderProcessing = true; } } @@ -2438,8 +2715,14 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (fProcessBLOCKTXN) return ProcessMessage(pfrom, NetMsgType::BLOCKTXN, blockTxnMsg, nTimeReceived, chainparams, connman, interruptMsgProc); - if (fRevertToHeaderProcessing) - return ProcessMessage(pfrom, NetMsgType::HEADERS, vHeadersMsg, nTimeReceived, chainparams, connman, interruptMsgProc); + if (fRevertToHeaderProcessing) { + // Headers received from HB compact block peers are permitted to be + // relayed before full validation (see BIP 152), so we don't want to disconnect + // the peer if the header turns out to be for an invalid block. + // Note that if a peer tries to build on an invalid chain, that + // will be detected and the peer will be banned. + return ProcessHeadersMessage(pfrom, connman, {cmpctblock.header}, chainparams, /*punish_duplicate_invalid=*/false); + } if (fBlockReconstructed) { // If we got here, we were able to optimistically reconstruct a @@ -2449,7 +2732,16 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr mapBlockSource.emplace(pblock->GetHash(), std::make_pair(pfrom->GetId(), false)); } bool fNewBlock = false; - ProcessNewBlock(chainparams, pblock, true, &fNewBlock); + // Setting fForceProcessing to true means that we bypass some of + // our anti-DoS protections in AcceptBlock, which filters + // unrequested blocks that might be trying to waste our resources + // (eg disk space). Because we only try to reconstruct blocks when + // we're close to caught up (via the CanDirectFetch() requirement + // above, combined with the behavior of not requesting blocks until + // we have a chain with at least nMinimumChainWork), and we ignore + // compact blocks with less work than our tip, it is safe to treat + // reconstructed compact blocks as having been requested. + ProcessNewBlock(chainparams, pblock, /*fForceProcessing=*/true, &fNewBlock); if (fNewBlock) { pfrom->nLastBlockTime = GetTime(); } else { @@ -2496,7 +2788,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // Might have collided, fall back to getdata now :( std::vector invs; invs.push_back(CInv(MSG_BLOCK, resp.blockhash)); - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, invs)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, invs)); } else { // Block is either okay, or possibly we received // READ_STATUS_CHECKBLOCK_FAILED. @@ -2529,7 +2821,11 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr bool fNewBlock = false; // Since we requested this block (it was in mapBlocksInFlight), force it to be processed, // even if it would not be a candidate for new tip (missing previous block, chain not long enough, etc) - ProcessNewBlock(chainparams, pblock, true, &fNewBlock); + // This bypasses some anti-DoS logic in AcceptBlock (eg to prevent + // disk-space attacks), but this should be safe due to the + // protections in the compact block handler -- see related comment + // in compact block optimistic reconstruction handling. + ProcessNewBlock(chainparams, pblock, /*fForceProcessing=*/true, &fNewBlock); if (fNewBlock) { pfrom->nLastBlockTime = GetTime(); } else { @@ -2557,134 +2853,12 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr ReadCompactSize(vRecv); // ignore tx count; assume it is 0. } - if (nCount == 0) { - // Nothing interesting. Stop asking this peers for more headers. - return true; - } - - const CBlockIndex *pindexLast = nullptr; - { - LOCK(cs_main); - CNodeState *nodestate = State(pfrom->GetId()); - - // If this looks like it could be a block announcement (nCount < - // MAX_BLOCKS_TO_ANNOUNCE), use special logic for handling headers that - // don't connect: - // - Send a getheaders message in response to try to connect the chain. - // - The peer can send up to MAX_UNCONNECTING_HEADERS in a row that - // don't connect before giving DoS points - // - Once a headers message is received that is valid and does connect, - // nUnconnectingHeaders gets reset back to 0. - if (mapBlockIndex.find(headers[0].hashPrevBlock) == mapBlockIndex.end() && nCount < MAX_BLOCKS_TO_ANNOUNCE) { - nodestate->nUnconnectingHeaders++; - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), uint256())); - LogPrint(BCLog::NET, "received header %s: missing prev block %s, sending getheaders (%d) to end (peer=%d, nUnconnectingHeaders=%d)\n", - headers[0].GetHash().ToString(), - headers[0].hashPrevBlock.ToString(), - pindexBestHeader->nHeight, - pfrom->GetId(), nodestate->nUnconnectingHeaders); - // Set hashLastUnknownBlock for this peer, so that if we - // eventually get the headers - even from a different peer - - // we can use this peer to download. - UpdateBlockAvailability(pfrom->GetId(), headers.back().GetHash()); - - if (nodestate->nUnconnectingHeaders % MAX_UNCONNECTING_HEADERS == 0) { - Misbehaving(pfrom->GetId(), 20); - } - return true; - } - - uint256 hashLastBlock; - for (const CBlockHeader& header : headers) { - if (!hashLastBlock.IsNull() && header.hashPrevBlock != hashLastBlock) { - Misbehaving(pfrom->GetId(), 20); - return error("non-continuous headers sequence"); - } - hashLastBlock = header.GetHash(); - } - } - - CValidationState state; - if (!ProcessNewBlockHeaders(headers, state, chainparams, &pindexLast)) { - int nDoS; - if (state.IsInvalid(nDoS)) { - if (nDoS > 0) { - LOCK(cs_main); - Misbehaving(pfrom->GetId(), nDoS); - } - return error("invalid header received"); - } - } - - { - LOCK(cs_main); - CNodeState *nodestate = State(pfrom->GetId()); - if (nodestate->nUnconnectingHeaders > 0) { - LogPrint(BCLog::NET, "peer=%d: resetting nUnconnectingHeaders (%d -> 0)\n", pfrom->GetId(), nodestate->nUnconnectingHeaders); - } - nodestate->nUnconnectingHeaders = 0; - - assert(pindexLast); - UpdateBlockAvailability(pfrom->GetId(), pindexLast->GetBlockHash()); - - if (nCount == MAX_HEADERS_RESULTS) { - // Headers message had its maximum size; the peer may have more headers. - // TODO: optimize: if pindexLast is an ancestor of chainActive.Tip or pindexBestHeader, continue - // from there instead. - LogPrint(BCLog::NET, "more getheaders (%d) to end to peer=%d (startheight:%d)\n", pindexLast->nHeight, pfrom->GetId(), pfrom->nStartingHeight); - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexLast), uint256())); - } - - bool fCanDirectFetch = CanDirectFetch(chainparams.GetConsensus()); - // If this set of headers is valid and ends in a block with at least as - // much work as our tip, download as much as possible. - if (fCanDirectFetch && pindexLast->IsValid(BLOCK_VALID_TREE) && chainActive.Tip()->nChainWork <= pindexLast->nChainWork) { - std::vector vToFetch; - const CBlockIndex *pindexWalk = pindexLast; - // Calculate all the blocks we'd need to switch to pindexLast, up to a limit. - while (pindexWalk && !chainActive.Contains(pindexWalk) && vToFetch.size() <= MAX_BLOCKS_IN_TRANSIT_PER_PEER) { - if (!(pindexWalk->nStatus & BLOCK_HAVE_DATA) && - !mapBlocksInFlight.count(pindexWalk->GetBlockHash())) { - // We don't have this block, and it's not yet in flight. - vToFetch.push_back(pindexWalk); - } - pindexWalk = pindexWalk->pprev; - } - // If pindexWalk still isn't on our main chain, we're looking at a - // very large reorg at a time we think we're close to caught up to - // the main chain -- this shouldn't really happen. Bail out on the - // direct fetch and rely on parallel download instead. - if (!chainActive.Contains(pindexWalk)) { - LogPrint(BCLog::NET, "Large reorg, won't direct fetch to %s (%d)\n", - pindexLast->GetBlockHash().ToString(), - pindexLast->nHeight); - } else { - std::vector vGetData; - // Download as much as possible, from earliest to latest. - for (const CBlockIndex *pindex : reverse_iterate(vToFetch)) { - if (nodestate->nBlocksInFlight >= MAX_BLOCKS_IN_TRANSIT_PER_PEER) { - // Can't download any more from this peer - break; - } - vGetData.push_back(CInv(MSG_BLOCK, pindex->GetBlockHash())); - MarkBlockAsInFlight(pfrom->GetId(), pindex->GetBlockHash(), pindex); - LogPrint(BCLog::NET, "Requesting block %s from peer=%d\n", - pindex->GetBlockHash().ToString(), pfrom->GetId()); - } - if (vGetData.size() > 1) { - LogPrint(BCLog::NET, "Downloading blocks toward %s (%d) via headers direct fetch\n", - pindexLast->GetBlockHash().ToString(), pindexLast->nHeight); - } - if (vGetData.size() > 0) { - if (nodestate->fSupportsDesiredCmpctVersion && vGetData.size() == 1 && mapBlocksInFlight.size() == 1 && pindexLast->pprev->IsValid(BLOCK_VALID_CHAIN)) { - // In any case, we want to download using a compact block, not a regular one - vGetData[0] = CInv(MSG_CMPCT_BLOCK, vGetData[0].hash); - } - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vGetData)); - } - } - } - } + // Headers received via a HEADERS message should be valid, and reflect + // the chain the peer is on. If we receive a known-invalid header, + // disconnect the peer if it is using one of our outbound connection + // slots. + bool should_punish = !pfrom->fInbound && !pfrom->m_manual_connection; + return ProcessHeadersMessage(pfrom, connman, headers, chainparams, should_punish); } else if (strCommand == NetMsgType::BLOCK && !fImporting && !fReindex) // Ignore blocks received while importing @@ -2694,11 +2868,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr LogPrint(BCLog::NET, "received block %s peer=%d\n", pblock->GetHash().ToString(), pfrom->GetId()); - // Process all blocks from whitelisted peers, even if not requested, - // unless we're still syncing with the network. - // Such an unrequested block may still be processed, subject to the - // conditions in AcceptBlock(). - bool forceProcessing = pfrom->fWhitelisted && !IsInitialBlockDownload(); + bool forceProcessing = false; const uint256 hash(pblock->GetHash()); { LOCK(cs_main); @@ -2741,7 +2911,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr pfrom->fSentAddr = true; pfrom->vAddrToSend.clear(); - std::vector vAddr = connman.GetAddresses(); + std::vector vAddr = connman->GetAddresses(); FastRandomContext insecure_rand; for (const CAddress &addr : vAddr) pfrom->PushAddress(addr, insecure_rand); @@ -2757,7 +2927,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr return true; } - if (connman.OutboundTargetReached(false) && !pfrom->fWhitelisted) + if (connman->OutboundTargetReached(false) && !pfrom->fWhitelisted) { LogPrint(BCLog::NET, "mempool request with bandwidth limit reached, disconnect peer=%d\n", pfrom->GetId()); pfrom->fDisconnect = true; @@ -2786,7 +2956,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // it, if the remote node sends a ping once per second and this node takes 5 // seconds to respond to each, the 5th ping the remote sends would appear to // return very quickly. - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::PONG, nonce)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::PONG, nonce)); } } @@ -2915,7 +3085,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr CSimplifiedMNListDiff mnListDiff; std::string strError; if (BuildSimplifiedMNListDiff(cmd.baseBlockHash, cmd.blockHash, mnListDiff, strError)) { - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::MNLISTDIFF, mnListDiff)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::MNLISTDIFF, mnListDiff)); } else { LogPrint(BCLog::NET, "getmnlistdiff failed for baseBlockHash=%s, blockHash=%s. error=%s\n", cmd.baseBlockHash.ToString(), cmd.blockHash.ToString(), strError); Misbehaving(pfrom->GetId(), 1); @@ -2950,19 +3120,19 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr { //probably one the extensions #ifdef ENABLE_WALLET - privateSendClient.ProcessMessage(pfrom, strCommand, vRecv, connman); + privateSendClient.ProcessMessage(pfrom, strCommand, vRecv, *connman); #endif // ENABLE_WALLET - privateSendServer.ProcessMessage(pfrom, strCommand, vRecv, connman); - sporkManager.ProcessSpork(pfrom, strCommand, vRecv, connman); + privateSendServer.ProcessMessage(pfrom, strCommand, vRecv, *connman); + sporkManager.ProcessSpork(pfrom, strCommand, vRecv, *connman); masternodeSync.ProcessMessage(pfrom, strCommand, vRecv); - governance.ProcessMessage(pfrom, strCommand, vRecv, connman); - CMNAuth::ProcessMessage(pfrom, strCommand, vRecv, connman); - llmq::quorumBlockProcessor->ProcessMessage(pfrom, strCommand, vRecv, connman); - llmq::quorumDKGSessionManager->ProcessMessage(pfrom, strCommand, vRecv, connman); - llmq::quorumSigSharesManager->ProcessMessage(pfrom, strCommand, vRecv, connman); - llmq::quorumSigningManager->ProcessMessage(pfrom, strCommand, vRecv, connman); - llmq::chainLocksHandler->ProcessMessage(pfrom, strCommand, vRecv, connman); - llmq::quorumInstantSendManager->ProcessMessage(pfrom, strCommand, vRecv, connman); + governance.ProcessMessage(pfrom, strCommand, vRecv, *connman); + CMNAuth::ProcessMessage(pfrom, strCommand, vRecv, *connman); + llmq::quorumBlockProcessor->ProcessMessage(pfrom, strCommand, vRecv, *connman); + llmq::quorumDKGSessionManager->ProcessMessage(pfrom, strCommand, vRecv, *connman); + llmq::quorumSigSharesManager->ProcessMessage(pfrom, strCommand, vRecv, *connman); + llmq::quorumSigningManager->ProcessMessage(pfrom, strCommand, vRecv, *connman); + llmq::chainLocksHandler->ProcessMessage(pfrom, strCommand, vRecv, *connman); + llmq::quorumInstantSendManager->ProcessMessage(pfrom, strCommand, vRecv, *connman); } else { @@ -2974,13 +3144,13 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr return true; } -static bool SendRejectsAndCheckIfBanned(CNode* pnode, CConnman& connman) +static bool SendRejectsAndCheckIfBanned(CNode* pnode, CConnman* connman) { AssertLockHeld(cs_main); CNodeState &state = *State(pnode->GetId()); for (const CBlockReject& reject : state.rejects) { - connman.PushMessage(pnode, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::REJECT, (std::string)NetMsgType::BLOCK, reject.chRejectCode, reject.strRejectReason, reject.hashBlock)); + connman->PushMessage(pnode, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::REJECT, (std::string)NetMsgType::BLOCK, reject.chRejectCode, reject.strRejectReason, reject.hashBlock)); } state.rejects.clear(); @@ -2988,15 +3158,15 @@ static bool SendRejectsAndCheckIfBanned(CNode* pnode, CConnman& connman) state.fShouldBan = false; if (pnode->fWhitelisted) LogPrintf("Warning: not punishing whitelisted peer %s!\n", pnode->GetLogString()); - else if (pnode->fAddnode) - LogPrintf("Warning: not punishing addnoded peer %s!\n", pnode->GetLogString()); + else if (pnode->m_manual_connection) + LogPrintf("Warning: not punishing manually-connected peer %s!\n", pnode->GetLogString()); else { pnode->fDisconnect = true; if (pnode->addr.IsLocal()) LogPrintf("Warning: not banning local peer %s!\n", pnode->GetLogString()); else { - connman.Ban(pnode->addr, BanReasonNodeMisbehaving); + connman->Ban(pnode->addr, BanReasonNodeMisbehaving); } } return true; @@ -3004,7 +3174,7 @@ static bool SendRejectsAndCheckIfBanned(CNode* pnode, CConnman& connman) return false; } -bool ProcessMessages(CNode* pfrom, CConnman& connman, const std::atomic& interruptMsgProc) +bool PeerLogicValidation::ProcessMessages(CNode* pfrom, std::atomic& interruptMsgProc) { const CChainParams& chainparams = Params(); // @@ -3038,7 +3208,7 @@ bool ProcessMessages(CNode* pfrom, CConnman& connman, const std::atomic& i // Just take one message msgs.splice(msgs.begin(), pfrom->vProcessMsg, pfrom->vProcessMsg.begin()); pfrom->nProcessQueueSize -= msgs.front().vRecv.size() + CMessageHeader::HEADER_SIZE; - pfrom->fPauseRecv = pfrom->nProcessQueueSize > connman.GetReceiveFloodSize(); + pfrom->fPauseRecv = pfrom->nProcessQueueSize > connman->GetReceiveFloodSize(); fMoreWork = !pfrom->vProcessMsg.empty(); } CNetMessage& msg(msgs.front()); @@ -3087,7 +3257,7 @@ bool ProcessMessages(CNode* pfrom, CConnman& connman, const std::atomic& i } catch (const std::ios_base::failure& e) { - connman.PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::REJECT, strCommand, REJECT_MALFORMED, std::string("error parsing message"))); + connman->PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::REJECT, strCommand, REJECT_MALFORMED, std::string("error parsing message"))); if (strstr(e.what(), "end of data")) { // Allow exceptions from under-length message on vRecv @@ -3121,6 +3291,137 @@ bool ProcessMessages(CNode* pfrom, CConnman& connman, const std::atomic& i return fMoreWork; } +void PeerLogicValidation::ConsiderEviction(CNode *pto, int64_t time_in_seconds) +{ + AssertLockHeld(cs_main); + + CNodeState &state = *State(pto->GetId()); + const CNetMsgMaker msgMaker(pto->GetSendVersion()); + + if (!state.m_chain_sync.m_protect && IsOutboundDisconnectionCandidate(pto) && state.fSyncStarted) { + // This is an outbound peer subject to disconnection if they don't + // announce a block with as much work as the current tip within + // CHAIN_SYNC_TIMEOUT + HEADERS_RESPONSE_TIME seconds (note: if + // their chain has more work than ours, we should sync to it, + // unless it's invalid, in which case we should find that out and + // disconnect from them elsewhere). + if (state.pindexBestKnownBlock != nullptr && state.pindexBestKnownBlock->nChainWork >= chainActive.Tip()->nChainWork) { + if (state.m_chain_sync.m_timeout != 0) { + state.m_chain_sync.m_timeout = 0; + state.m_chain_sync.m_work_header = nullptr; + state.m_chain_sync.m_sent_getheaders = false; + } + } else if (state.m_chain_sync.m_timeout == 0 || (state.m_chain_sync.m_work_header != nullptr && state.pindexBestKnownBlock != nullptr && state.pindexBestKnownBlock->nChainWork >= state.m_chain_sync.m_work_header->nChainWork)) { + // Our best block known by this peer is behind our tip, and we're either noticing + // that for the first time, OR this peer was able to catch up to some earlier point + // where we checked against our tip. + // Either way, set a new timeout based on current tip. + state.m_chain_sync.m_timeout = time_in_seconds + CHAIN_SYNC_TIMEOUT; + state.m_chain_sync.m_work_header = chainActive.Tip(); + state.m_chain_sync.m_sent_getheaders = false; + } else if (state.m_chain_sync.m_timeout > 0 && time_in_seconds > state.m_chain_sync.m_timeout) { + // No evidence yet that our peer has synced to a chain with work equal to that + // of our tip, when we first detected it was behind. Send a single getheaders + // message to give the peer a chance to update us. + if (state.m_chain_sync.m_sent_getheaders) { + // They've run out of time to catch up! + LogPrintf("Disconnecting outbound peer %d for old chain, best known block = %s\n", pto->GetId(), state.pindexBestKnownBlock != nullptr ? state.pindexBestKnownBlock->GetBlockHash().ToString() : ""); + pto->fDisconnect = true; + } else { + LogPrint(BCLog::NET, "sending getheaders to outbound peer=%d to verify chain work (current best known block:%s, benchmark blockhash: %s)\n", pto->GetId(), state.pindexBestKnownBlock != nullptr ? state.pindexBestKnownBlock->GetBlockHash().ToString() : "", state.m_chain_sync.m_work_header->GetBlockHash().ToString()); + connman->PushMessage(pto, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(state.m_chain_sync.m_work_header->pprev), uint256())); + state.m_chain_sync.m_sent_getheaders = true; + constexpr int64_t HEADERS_RESPONSE_TIME = 120; // 2 minutes + // Bump the timeout to allow a response, which could clear the timeout + // (if the response shows the peer has synced), reset the timeout (if + // the peer syncs to the required work but not to our tip), or result + // in disconnect (if we advance to the timeout and pindexBestKnownBlock + // has not sufficiently progressed) + state.m_chain_sync.m_timeout = time_in_seconds + HEADERS_RESPONSE_TIME; + } + } + } +} + +void PeerLogicValidation::EvictExtraOutboundPeers(int64_t time_in_seconds) +{ + // Check whether we have too many outbound peers + int extra_peers = connman->GetExtraOutboundCount(); + if (extra_peers > 0) { + // If we have more outbound peers than we target, disconnect one. + // Pick the outbound peer that least recently announced + // us a new block, with ties broken by choosing the more recent + // connection (higher node id) + NodeId worst_peer = -1; + int64_t oldest_block_announcement = std::numeric_limits::max(); + + LOCK(cs_main); + + connman->ForEachNode([&](CNode* pnode) { + // Don't disconnect masternodes just because they were slow in block announcement + if (pnode->fMasternode) return; + // Ignore non-outbound peers, or nodes marked for disconnect already + if (!IsOutboundDisconnectionCandidate(pnode) || pnode->fDisconnect) return; + CNodeState *state = State(pnode->GetId()); + if (state == nullptr) return; // shouldn't be possible, but just in case + // Don't evict our protected peers + if (state->m_chain_sync.m_protect) return; + if (state->m_last_block_announcement < oldest_block_announcement || (state->m_last_block_announcement == oldest_block_announcement && pnode->GetId() > worst_peer)) { + worst_peer = pnode->GetId(); + oldest_block_announcement = state->m_last_block_announcement; + } + }); + if (worst_peer != -1) { + bool disconnected = connman->ForNode(worst_peer, [&](CNode *pnode) { + // Only disconnect a peer that has been connected to us for + // some reasonable fraction of our check-frequency, to give + // it time for new information to have arrived. + // Also don't disconnect any peer we're trying to download a + // block from. + CNodeState &state = *State(pnode->GetId()); + if (time_in_seconds - pnode->nTimeConnected > MINIMUM_CONNECT_TIME && state.nBlocksInFlight == 0) { + LogPrint(BCLog::NET, "disconnecting extra outbound peer=%d (last block announcement received at time %d)\n", pnode->GetId(), oldest_block_announcement); + pnode->fDisconnect = true; + return true; + } else { + LogPrint(BCLog::NET, "keeping outbound peer=%d chosen for eviction (connect time: %d, blocks_in_flight: %d)\n", pnode->GetId(), pnode->nTimeConnected, state.nBlocksInFlight); + return false; + } + }); + if (disconnected) { + // If we disconnected an extra peer, that means we successfully + // connected to at least one peer after the last time we + // detected a stale tip. Don't try any more extra peers until + // we next detect a stale tip, to limit the load we put on the + // network from these extra connections. + connman->SetTryNewOutboundPeer(false); + } + } + } +} + +void PeerLogicValidation::CheckForStaleTipAndEvictPeers(const Consensus::Params &consensusParams) +{ + if (connman == nullptr) return; + + int64_t time_in_seconds = GetTime(); + + EvictExtraOutboundPeers(time_in_seconds); + + if (time_in_seconds > m_stale_tip_check_time) { + LOCK(cs_main); + // Check whether our tip is stale, and if so, allow using an extra + // outbound peer + if (TipMayBeStale(consensusParams)) { + LogPrintf("Potential stale tip detected, will try using extra outbound peer (last tip update: %d seconds ago)\n", time_in_seconds - g_last_tip_update); + connman->SetTryNewOutboundPeer(true); + } else if (connman->GetTryNewOutboundPeer()) { + connman->SetTryNewOutboundPeer(false); + } + m_stale_tip_check_time = time_in_seconds + STALE_CHECK_INTERVAL; + } +} + class CompareInvMempoolOrder { CTxMemPool *mp; @@ -3138,7 +3439,7 @@ class CompareInvMempoolOrder } }; -bool SendMessages(CNode* pto, CConnman& connman, const std::atomic& interruptMsgProc) +bool PeerLogicValidation::SendMessages(CNode* pto, std::atomic& interruptMsgProc) { const Consensus::Params& consensusParams = Params().GetConsensus(); { @@ -3170,11 +3471,11 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic& interr pto->nPingUsecStart = GetTimeMicros(); if (pto->nVersion > BIP0031_VERSION) { pto->nPingNonceSent = nonce; - connman.PushMessage(pto, msgMaker.Make(NetMsgType::PING, nonce)); + connman->PushMessage(pto, msgMaker.Make(NetMsgType::PING, nonce)); } else { // Peer is too old to support ping command with nonce, pong will never arrive. pto->nPingNonceSent = 0; - connman.PushMessage(pto, msgMaker.Make(NetMsgType::PING)); + connman->PushMessage(pto, msgMaker.Make(NetMsgType::PING)); } } @@ -3209,14 +3510,14 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic& interr // receiver rejects addr messages larger than 1000 if (vAddr.size() >= 1000) { - connman.PushMessage(pto, msgMaker.Make(NetMsgType::ADDR, vAddr)); + connman->PushMessage(pto, msgMaker.Make(NetMsgType::ADDR, vAddr)); vAddr.clear(); } } } pto->vAddrToSend.clear(); if (!vAddr.empty()) - connman.PushMessage(pto, msgMaker.Make(NetMsgType::ADDR, vAddr)); + connman->PushMessage(pto, msgMaker.Make(NetMsgType::ADDR, vAddr)); // we only send the big addr message once if (pto->vAddrToSend.capacity() > 40) pto->vAddrToSend.shrink_to_fit(); @@ -3243,7 +3544,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic& interr if (pindexStart->pprev) pindexStart = pindexStart->pprev; LogPrint(BCLog::NET, "initial getheaders (%d) to peer=%d (startheight:%d)\n", pindexStart->nHeight, pto->GetId(), pto->nStartingHeight); - connman.PushMessage(pto, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexStart), uint256())); + connman->PushMessage(pto, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexStart), uint256())); } } @@ -3252,7 +3553,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic& interr // transactions become unconfirmed and spams other nodes. if (!fReindex && !fImporting && !IsInitialBlockDownload()) { - GetMainSignals().Broadcast(nTimeBestReceived, &connman); + GetMainSignals().Broadcast(nTimeBestReceived, connman); } // @@ -3342,7 +3643,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic& interr { LOCK(cs_most_recent_block); if (most_recent_block_hash == pBestIndex->GetBlockHash()) { - connman.PushMessage(pto, msgMaker.Make(NetMsgType::CMPCTBLOCK, *most_recent_compact_block)); + connman->PushMessage(pto, msgMaker.Make(NetMsgType::CMPCTBLOCK, *most_recent_compact_block)); fGotBlockFromCache = true; } } @@ -3351,7 +3652,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic& interr bool ret = ReadBlockFromDisk(block, pBestIndex, consensusParams); assert(ret); CBlockHeaderAndShortTxIDs cmpctblock(block); - connman.PushMessage(pto, msgMaker.Make(NetMsgType::CMPCTBLOCK, cmpctblock)); + connman->PushMessage(pto, msgMaker.Make(NetMsgType::CMPCTBLOCK, cmpctblock)); } state.pindexBestHeaderSent = pBestIndex; } else if (state.fPreferHeaders) { @@ -3364,7 +3665,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic& interr LogPrint(BCLog::NET, "%s: sending header %s to peer=%d\n", __func__, vHeaders.front().GetHash().ToString(), pto->GetId()); } - connman.PushMessage(pto, msgMaker.Make(NetMsgType::HEADERS, vHeaders)); + connman->PushMessage(pto, msgMaker.Make(NetMsgType::HEADERS, vHeaders)); state.pindexBestHeaderSent = pBestIndex; } else fRevertToInv = true; @@ -3410,7 +3711,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic& interr for (const uint256& hash : pto->vInventoryBlockToSend) { vInv.push_back(CInv(MSG_BLOCK, hash)); if (vInv.size() == MAX_INV_SZ) { - connman.PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); + connman->PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); vInv.clear(); } } @@ -3453,7 +3754,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic& interr vInv.push_back(inv); if (vInv.size() == MAX_INV_SZ) { LogPrint(BCLog::NET, "SendMessages -- pushing inv's: count=%d peer=%d\n", vInv.size(), pto->GetId()); - connman.PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); + connman->PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); vInv.clear(); } } @@ -3511,7 +3812,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic& interr } } if (vInv.size() == MAX_INV_SZ) { - connman.PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); + connman->PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); vInv.clear(); } pto->filterInventoryKnown.insert(hash); @@ -3526,14 +3827,14 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic& interr vInv.push_back(inv); pto->filterInventoryKnown.insert(inv.hash); if (vInv.size() == MAX_INV_SZ) { - connman.PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); + connman->PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); vInv.clear(); } } pto->vInventoryOtherToSend.clear(); } if (!vInv.empty()) - connman.PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); + connman->PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); // Detect whether we're stalling nNow = GetTimeMicros(); @@ -3592,6 +3893,9 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic& interr } } + // Check that outbound peers have reasonable chains + // GetTime() is used by this anti-DoS logic so we can test this using mocktime + ConsiderEviction(pto, GetTime()); // // Message: getdata (blocks) @@ -3629,7 +3933,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic& interr vGetData.push_back(inv); if (vGetData.size() >= 1000) { - connman.PushMessage(pto, msgMaker.Make(NetMsgType::GETDATA, vGetData)); + connman->PushMessage(pto, msgMaker.Make(NetMsgType::GETDATA, vGetData)); LogPrint(BCLog::NET, "SendMessages -- GETDATA -- pushed size = %lu peer=%d\n", vGetData.size(), pto->GetId()); vGetData.clear(); } @@ -3642,7 +3946,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic& interr } pto->vecAskFor.erase(pto->vecAskFor.begin(), it); if (!vGetData.empty()) { - connman.PushMessage(pto, msgMaker.Make(NetMsgType::GETDATA, vGetData)); + connman->PushMessage(pto, msgMaker.Make(NetMsgType::GETDATA, vGetData)); LogPrint(BCLog::NET, "SendMessages -- GETDATA -- pushed size = %lu peer=%d\n", vGetData.size(), pto->GetId()); } diff --git a/src/net_processing.h b/src/net_processing.h index b6c0e8ef6436..bb9a76045eb0 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -8,6 +8,7 @@ #include "net.h" #include "validationinterface.h" +#include "consensus/params.h" /** Default for -maxorphantx, maximum number of orphan transactions kept in memory */ static const unsigned int DEFAULT_MAX_ORPHAN_TRANSACTIONS = 100; @@ -15,31 +16,59 @@ static const unsigned int DEFAULT_MAX_ORPHAN_TRANSACTIONS = 100; static const int64_t ORPHAN_TX_EXPIRE_TIME = 20 * 60; /** Minimum time between orphan transactions expire time checks in seconds */ static const int64_t ORPHAN_TX_EXPIRE_INTERVAL = 5 * 60; +/** Default number of orphan+recently-replaced txn to keep around for block reconstruction */ +static const unsigned int DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN = 100; /** Headers download timeout expressed in microseconds * Timeout = base + per_header * (expected number of headers) */ static constexpr int64_t HEADERS_DOWNLOAD_TIMEOUT_BASE = 15 * 60 * 1000000; // 15 minutes static constexpr int64_t HEADERS_DOWNLOAD_TIMEOUT_PER_HEADER = 1000; // 1ms/header +/** Protect at least this many outbound peers from disconnection due to slow/ + * behind headers chain. + */ +static constexpr int32_t MAX_OUTBOUND_PEERS_TO_PROTECT_FROM_DISCONNECT = 4; +/** Timeout for (unprotected) outbound peers to sync to our chainwork, in seconds */ +static constexpr int64_t CHAIN_SYNC_TIMEOUT = 20 * 60; // 20 minutes -/** Default number of orphan+recently-replaced txn to keep around for block reconstruction */ -static const unsigned int DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN = 100; - -/** Register with a network node to receive its signals */ -void RegisterNodeSignals(CNodeSignals& nodeSignals); -/** Unregister a network node */ -void UnregisterNodeSignals(CNodeSignals& nodeSignals); +/** How frequently to check for stale tips, in seconds */ +static constexpr int64_t STALE_CHECK_INTERVAL = 150; // 2.5 minutes (~block interval) +/** How frequently to check for extra outbound peers and disconnect, in seconds */ +static constexpr int64_t EXTRA_PEER_CHECK_INTERVAL = 45; +/** Minimum time an outbound-peer-eviction candidate must be connected for, in order to evict, in seconds */ +static constexpr int64_t MINIMUM_CONNECT_TIME = 30; -class PeerLogicValidation : public CValidationInterface { +class PeerLogicValidation : public CValidationInterface, public NetEventsInterface { private: - CConnman* connman; + CConnman* const connman; public: - PeerLogicValidation(CConnman* connmanIn); + explicit PeerLogicValidation(CConnman* connmanIn, CScheduler &scheduler); void BlockConnected(const std::shared_ptr& pblock, const CBlockIndex* pindexConnected, const std::vector& vtxConflicted) override; void UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) override; void BlockChecked(const CBlock& block, const CValidationState& state) override; void NewPoWValidBlock(const CBlockIndex *pindex, const std::shared_ptr& pblock) override; + + + void InitializeNode(CNode* pnode) override; + void FinalizeNode(NodeId nodeid, bool& fUpdateConnectionTime) override; + /** Process protocol messages received from a given node */ + bool ProcessMessages(CNode* pfrom, std::atomic& interrupt) override; + /** + * Send queued protocol messages to be sent to a give node. + * + * @param[in] pto The node which we are sending messages to. + * @param[in] interrupt Interrupt condition for processing threads + * @return True if there is more work to be done + */ + bool SendMessages(CNode* pto, std::atomic& interrupt) override; + + void ConsiderEviction(CNode *pto, int64_t time_in_seconds); + void CheckForStaleTipAndEvictPeers(const Consensus::Params &consensusParams); + void EvictExtraOutboundPeers(int64_t time_in_seconds); + +private: + int64_t m_stale_tip_check_time; //! Next time to check for stale tip }; struct CNodeStateStats { @@ -55,16 +84,4 @@ bool GetNodeStateStats(NodeId nodeid, CNodeStateStats &stats); void Misbehaving(NodeId nodeid, int howmuch); bool IsBanned(NodeId nodeid); -/** Process protocol messages received from a given node */ -bool ProcessMessages(CNode* pfrom, CConnman& connman, const std::atomic& interrupt); -/** - * Send queued protocol messages to be sent to a give node. - * - * @param[in] pto The node which we are sending messages to. - * @param[in] connman The connection manager for that node. - * @param[in] interrupt Interrupt condition for processing threads - * @return True if there is more work to be done - */ -bool SendMessages(CNode* pto, CConnman& connman, const std::atomic& interrupt); - #endif // BITCOIN_NET_PROCESSING_H diff --git a/src/protocol.h b/src/protocol.h index e7ee254f0a18..128f0adb19b4 100644 --- a/src/protocol.h +++ b/src/protocol.h @@ -306,6 +306,43 @@ enum ServiceFlags : uint64_t { // BIP process. }; +/** + * Gets the set of service flags which are "desirable" for a given peer. + * + * These are the flags which are required for a peer to support for them + * to be "interesting" to us, ie for us to wish to use one of our few + * outbound connection slots for or for us to wish to prioritize keeping + * their connection around. + * + * Relevant service flags may be peer- and state-specific in that the + * version of the peer may determine which flags are required (eg in the + * case of NODE_NETWORK_LIMITED where we seek out NODE_NETWORK peers + * unless they set NODE_NETWORK_LIMITED and we are out of IBD, in which + * case NODE_NETWORK_LIMITED suffices). + * + * Thus, generally, avoid calling with peerServices == NODE_NONE. + */ +static ServiceFlags GetDesirableServiceFlags(ServiceFlags services) { + return ServiceFlags(NODE_NETWORK); +} + +/** + * A shortcut for (services & GetDesirableServiceFlags(services)) + * == GetDesirableServiceFlags(services), ie determines whether the given + * set of service flags are sufficient for a peer to be "relevant". + */ +static inline bool HasAllDesirableServiceFlags(ServiceFlags services) { + return !(GetDesirableServiceFlags(services) & (~services)); +} + +/** + * Checks if a peer with the given service flags may be capable of having a + * robust address-storage DB. Currently an alias for checking NODE_NETWORK. + */ +static inline bool MayHaveUsefulAddressDB(ServiceFlags services) { + return services & NODE_NETWORK; +} + /** A CService with information about it as peer */ class CAddress : public CService { diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index f16c2d116353..d90ddf282922 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -93,7 +93,7 @@ UniValue getpeerinfo(const JSONRPCRequest& request) " \"version\": v, (numeric) The peer version, such as 7001\n" " \"subver\": \"/Dash Core:x.x.x/\", (string) The string version\n" " \"inbound\": true|false, (boolean) Inbound (true) or Outbound (false)\n" - " \"addnode\": true|false, (boolean) Whether connection was due to addnode and is using an addnode slot\n" + " \"addnode\": true|false, (boolean) Whether connection was due to addnode/-connect or if it was an automatic/inbound connection\n" " \"startingheight\": n, (numeric) The starting height (block) of the peer\n" " \"banscore\": n, (numeric) The ban score\n" " \"synced_headers\": n, (numeric) The last header we have in common with this peer\n" @@ -157,7 +157,7 @@ UniValue getpeerinfo(const JSONRPCRequest& request) // their ver message. obj.push_back(Pair("subver", stats.cleanSubVer)); obj.push_back(Pair("inbound", stats.fInbound)); - obj.push_back(Pair("addnode", stats.fAddnode)); + obj.push_back(Pair("addnode", stats.m_manual_connection)); obj.push_back(Pair("startingheight", stats.nStartingHeight)); if (fStateStats) { obj.push_back(Pair("banscore", statestats.nMisbehavior)); @@ -202,6 +202,8 @@ UniValue addnode(const JSONRPCRequest& request) "addnode \"node\" \"add|remove|onetry\"\n" "\nAttempts to add or remove a node from the addnode list.\n" "Or try a connection to a node once.\n" + "Nodes added using addnode (or -connect) are protected from DoS disconnection and are not required to be\n" + "full nodes/support SegWit as other outbound peers are (though such peers will not be synced from).\n" "\nArguments:\n" "1. \"node\" (string, required) The node (see getpeerinfo for nodes)\n" "2. \"command\" (string, required) 'add' to add a node to the list, 'remove' to remove a node from the list, 'onetry' to try a connection to the node once\n" @@ -218,7 +220,7 @@ UniValue addnode(const JSONRPCRequest& request) if (strCommand == "onetry") { CAddress addr; - g_connman->OpenNetworkConnection(addr, false, nullptr, strNode.c_str()); + g_connman->OpenNetworkConnection(addr, false, nullptr, strNode.c_str(), false, false, true); return NullUniValue; } diff --git a/src/test/DoS_tests.cpp b/src/test/DoS_tests.cpp index 961b18c45425..df9370b06dc5 100644 --- a/src/test/DoS_tests.cpp +++ b/src/test/DoS_tests.cpp @@ -40,8 +40,138 @@ CService ip(uint32_t i) static NodeId id = 0; +void UpdateLastBlockAnnounceTime(NodeId node, int64_t time_in_seconds); + BOOST_FIXTURE_TEST_SUITE(DoS_tests, TestingSetup) +// Test eviction of an outbound peer whose chain never advances +// Mock a node connection, and use mocktime to simulate a peer +// which never sends any headers messages. PeerLogic should +// decide to evict that outbound peer, after the appropriate timeouts. +// Note that we protect 4 outbound nodes from being subject to +// this logic; this test takes advantage of that protection only +// being applied to nodes which send headers with sufficient +// work. +BOOST_AUTO_TEST_CASE(outbound_slow_chain_eviction) +{ + std::atomic interruptDummy(false); + + // Mock an outbound peer + CAddress addr1(ip(0xa0b0c001), NODE_NONE); + CNode dummyNode1(id++, ServiceFlags(NODE_NETWORK), 0, INVALID_SOCKET, addr1, 0, 0, CAddress(), "", /*fInboundIn=*/ false); + dummyNode1.SetSendVersion(PROTOCOL_VERSION); + + peerLogic->InitializeNode(&dummyNode1); + dummyNode1.nVersion = 1; + dummyNode1.fSuccessfullyConnected = true; + + // This test requires that we have a chain with non-zero work. + BOOST_CHECK(chainActive.Tip() != nullptr); + BOOST_CHECK(chainActive.Tip()->nChainWork > 0); + + // Test starts here + peerLogic->SendMessages(&dummyNode1, interruptDummy); // should result in getheaders + BOOST_CHECK(dummyNode1.vSendMsg.size() > 0); + dummyNode1.vSendMsg.clear(); + + int64_t nStartTime = GetTime(); + // Wait 21 minutes + SetMockTime(nStartTime+21*60); + peerLogic->SendMessages(&dummyNode1, interruptDummy); // should result in getheaders + BOOST_CHECK(dummyNode1.vSendMsg.size() > 0); + // Wait 3 more minutes + SetMockTime(nStartTime+24*60); + peerLogic->SendMessages(&dummyNode1, interruptDummy); // should result in disconnect + BOOST_CHECK(dummyNode1.fDisconnect == true); + SetMockTime(0); + + bool dummy; + peerLogic->FinalizeNode(dummyNode1.GetId(), dummy); +} + +void AddRandomOutboundPeer(std::vector &vNodes, PeerLogicValidation &peerLogic) +{ + CAddress addr(ip(GetRandInt(0xffffffff)), NODE_NONE); + vNodes.emplace_back(new CNode(id++, ServiceFlags(NODE_NETWORK), 0, INVALID_SOCKET, addr, 0, 0, CAddress(), "", /*fInboundIn=*/ false)); + CNode &node = *vNodes.back(); + node.SetSendVersion(PROTOCOL_VERSION); + + peerLogic.InitializeNode(&node); + node.nVersion = 1; + node.fSuccessfullyConnected = true; + + CConnmanTest::AddNode(node); +} + +BOOST_AUTO_TEST_CASE(stale_tip_peer_management) +{ + const Consensus::Params& consensusParams = Params().GetConsensus(); + constexpr int nMaxOutbound = 8; + CConnman::Options options; + options.nMaxConnections = 125; + options.nMaxOutbound = nMaxOutbound; + options.nMaxFeeler = 1; + + connman->Init(options); + std::vector vNodes; + + // Mock some outbound peers + for (int i=0; iCheckForStaleTipAndEvictPeers(consensusParams); + + // No nodes should be marked for disconnection while we have no extra peers + for (const CNode *node : vNodes) { + BOOST_CHECK(node->fDisconnect == false); + } + + SetMockTime(GetTime() + 3*consensusParams.nPowTargetSpacing + 1); + + // Now tip should definitely be stale, and we should look for an extra + // outbound peer + peerLogic->CheckForStaleTipAndEvictPeers(consensusParams); + BOOST_CHECK(connman->GetTryNewOutboundPeer()); + + // Still no peers should be marked for disconnection + for (const CNode *node : vNodes) { + BOOST_CHECK(node->fDisconnect == false); + } + + // If we add one more peer, something should get marked for eviction + // on the next check (since we're mocking the time to be in the future, the + // required time connected check should be satisfied). + AddRandomOutboundPeer(vNodes, *peerLogic); + + peerLogic->CheckForStaleTipAndEvictPeers(consensusParams); + for (int i=0; ifDisconnect == false); + } + // Last added node should get marked for eviction + BOOST_CHECK(vNodes.back()->fDisconnect == true); + + vNodes.back()->fDisconnect = false; + + // Update the last announced block time for the last + // peer, and check that the next newest node gets evicted. + UpdateLastBlockAnnounceTime(vNodes.back()->GetId(), GetTime()); + + peerLogic->CheckForStaleTipAndEvictPeers(consensusParams); + for (int i=0; ifDisconnect == false); + } + BOOST_CHECK(vNodes[nMaxOutbound-1]->fDisconnect == true); + BOOST_CHECK(vNodes.back()->fDisconnect == false); + + bool dummy; + for (const CNode *node : vNodes) { + peerLogic->FinalizeNode(node->GetId(), dummy); + } + + CConnmanTest::ClearNodes(); +} + BOOST_AUTO_TEST_CASE(DoS_banning) { std::atomic interruptDummy(false); @@ -50,27 +180,31 @@ BOOST_AUTO_TEST_CASE(DoS_banning) CAddress addr1(ip(0xa0b0c001), NODE_NONE); CNode dummyNode1(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr1, 0, 0, CAddress(), "", true); dummyNode1.SetSendVersion(PROTOCOL_VERSION); - GetNodeSignals().InitializeNode(&dummyNode1, *connman); + peerLogic->InitializeNode(&dummyNode1); dummyNode1.nVersion = 1; dummyNode1.fSuccessfullyConnected = true; Misbehaving(dummyNode1.GetId(), 100); // Should get banned - SendMessages(&dummyNode1, *connman, interruptDummy); + peerLogic->SendMessages(&dummyNode1, interruptDummy); BOOST_CHECK(connman->IsBanned(addr1)); BOOST_CHECK(!connman->IsBanned(ip(0xa0b0c001|0x0000ff00))); // Different IP, not banned CAddress addr2(ip(0xa0b0c002), NODE_NONE); CNode dummyNode2(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr2, 1, 1, CAddress(), "", true); dummyNode2.SetSendVersion(PROTOCOL_VERSION); - GetNodeSignals().InitializeNode(&dummyNode2, *connman); + peerLogic->InitializeNode(&dummyNode2); dummyNode2.nVersion = 1; dummyNode2.fSuccessfullyConnected = true; Misbehaving(dummyNode2.GetId(), 50); - SendMessages(&dummyNode2, *connman, interruptDummy); + peerLogic->SendMessages(&dummyNode2, interruptDummy); BOOST_CHECK(!connman->IsBanned(addr2)); // 2 not banned yet... BOOST_CHECK(connman->IsBanned(addr1)); // ... but 1 still should be Misbehaving(dummyNode2.GetId(), 50); - SendMessages(&dummyNode2, *connman, interruptDummy); + peerLogic->SendMessages(&dummyNode2, interruptDummy); BOOST_CHECK(connman->IsBanned(addr2)); + + bool dummy; + peerLogic->FinalizeNode(dummyNode1.GetId(), dummy); + peerLogic->FinalizeNode(dummyNode2.GetId(), dummy); } BOOST_AUTO_TEST_CASE(DoS_banscore) @@ -82,19 +216,22 @@ BOOST_AUTO_TEST_CASE(DoS_banscore) CAddress addr1(ip(0xa0b0c001), NODE_NONE); CNode dummyNode1(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr1, 3, 1, CAddress(), "", true); dummyNode1.SetSendVersion(PROTOCOL_VERSION); - GetNodeSignals().InitializeNode(&dummyNode1, *connman); + peerLogic->InitializeNode(&dummyNode1); dummyNode1.nVersion = 1; dummyNode1.fSuccessfullyConnected = true; Misbehaving(dummyNode1.GetId(), 100); - SendMessages(&dummyNode1, *connman, interruptDummy); + peerLogic->SendMessages(&dummyNode1, interruptDummy); BOOST_CHECK(!connman->IsBanned(addr1)); Misbehaving(dummyNode1.GetId(), 10); - SendMessages(&dummyNode1, *connman, interruptDummy); + peerLogic->SendMessages(&dummyNode1, interruptDummy); BOOST_CHECK(!connman->IsBanned(addr1)); Misbehaving(dummyNode1.GetId(), 1); - SendMessages(&dummyNode1, *connman, interruptDummy); + peerLogic->SendMessages(&dummyNode1, interruptDummy); BOOST_CHECK(connman->IsBanned(addr1)); gArgs.ForceSetArg("-banscore", std::to_string(DEFAULT_BANSCORE_THRESHOLD)); + + bool dummy; + peerLogic->FinalizeNode(dummyNode1.GetId(), dummy); } BOOST_AUTO_TEST_CASE(DoS_bantime) @@ -108,12 +245,12 @@ BOOST_AUTO_TEST_CASE(DoS_bantime) CAddress addr(ip(0xa0b0c001), NODE_NONE); CNode dummyNode(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr, 4, 4, CAddress(), "", true); dummyNode.SetSendVersion(PROTOCOL_VERSION); - GetNodeSignals().InitializeNode(&dummyNode, *connman); + peerLogic->InitializeNode(&dummyNode); dummyNode.nVersion = 1; dummyNode.fSuccessfullyConnected = true; Misbehaving(dummyNode.GetId(), 100); - SendMessages(&dummyNode, *connman, interruptDummy); + peerLogic->SendMessages(&dummyNode, interruptDummy); BOOST_CHECK(connman->IsBanned(addr)); SetMockTime(nStartTime+60*60); @@ -121,6 +258,9 @@ BOOST_AUTO_TEST_CASE(DoS_bantime) SetMockTime(nStartTime+60*60*24+1); BOOST_CHECK(!connman->IsBanned(addr)); + + bool dummy; + peerLogic->FinalizeNode(dummyNode.GetId(), dummy); } CTransactionRef RandomOrphan() diff --git a/src/test/test_dash.cpp b/src/test/test_dash.cpp index e7054324a7b9..c494bcb3e3c6 100644 --- a/src/test/test_dash.cpp +++ b/src/test/test_dash.cpp @@ -29,6 +29,18 @@ #include +void CConnmanTest::AddNode(CNode& node) +{ + LOCK(g_connman->cs_vNodes); + g_connman->vNodes.push_back(&node); +} + +void CConnmanTest::ClearNodes() +{ + LOCK(g_connman->cs_vNodes); + g_connman->vNodes.clear(); +} + uint256 insecure_rand_seed = GetRandHash(); FastRandomContext insecure_rand_ctx(insecure_rand_seed); @@ -59,7 +71,6 @@ BasicTestingSetup::~BasicTestingSetup() delete evoDb; ECC_Stop(); - g_connman.reset(); } TestingSetup::TestingSetup(const std::string& chainName) : BasicTestingSetup(chainName) @@ -96,17 +107,18 @@ TestingSetup::TestingSetup(const std::string& chainName) : BasicTestingSetup(cha nScriptCheckThreads = 3; for (int i=0; i < nScriptCheckThreads-1; i++) threadGroup.create_thread(&ThreadScriptCheck); - RegisterNodeSignals(GetNodeSignals()); + peerLogic.reset(new PeerLogicValidation(connman, scheduler)); } TestingSetup::~TestingSetup() { - UnregisterNodeSignals(GetNodeSignals()); llmq::InterruptLLMQSystem(); threadGroup.interrupt_all(); threadGroup.join_all(); GetMainSignals().FlushBackgroundCallbacks(); GetMainSignals().UnregisterBackgroundSignalScheduler(); + g_connman.reset(); + peerLogic.reset(); UnloadBlockIndex(); delete pcoinsTip; llmq::DestroyLLMQSystem(); diff --git a/src/test/test_dash.h b/src/test/test_dash.h index 98a631d790a1..856523c19536 100644 --- a/src/test/test_dash.h +++ b/src/test/test_dash.h @@ -50,12 +50,20 @@ struct BasicTestingSetup { * Included are data directory, coins database, script check threads setup. */ class CConnman; +class CNode; +struct CConnmanTest { + static void AddNode(CNode& node); + static void ClearNodes(); +}; + +class PeerLogicValidation; struct TestingSetup: public BasicTestingSetup { CCoinsViewDB *pcoinsdbview; fs::path pathTemp; boost::thread_group threadGroup; CConnman* connman; CScheduler scheduler; + std::unique_ptr peerLogic; TestingSetup(const std::string& chainName = CBaseChainParams::MAIN); ~TestingSetup(); diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index e1de704f4eeb..9cf9899c212f 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -253,6 +253,31 @@ BOOST_AUTO_TEST_CASE(util_IsHex) BOOST_CHECK(!IsHex("0x0000")); } +BOOST_AUTO_TEST_CASE(util_IsHexNumber) +{ + BOOST_CHECK(IsHexNumber("0x0")); + BOOST_CHECK(IsHexNumber("0")); + BOOST_CHECK(IsHexNumber("0x10")); + BOOST_CHECK(IsHexNumber("10")); + BOOST_CHECK(IsHexNumber("0xff")); + BOOST_CHECK(IsHexNumber("ff")); + BOOST_CHECK(IsHexNumber("0xFfa")); + BOOST_CHECK(IsHexNumber("Ffa")); + BOOST_CHECK(IsHexNumber("0x00112233445566778899aabbccddeeffAABBCCDDEEFF")); + BOOST_CHECK(IsHexNumber("00112233445566778899aabbccddeeffAABBCCDDEEFF")); + + BOOST_CHECK(!IsHexNumber("")); // empty string not allowed + BOOST_CHECK(!IsHexNumber("0x")); // empty string after prefix not allowed + BOOST_CHECK(!IsHexNumber("0x0 ")); // no spaces at end, + BOOST_CHECK(!IsHexNumber(" 0x0")); // or beginning, + BOOST_CHECK(!IsHexNumber("0x 0")); // or middle, + BOOST_CHECK(!IsHexNumber(" ")); // etc. + BOOST_CHECK(!IsHexNumber("0x0ga")); // invalid character + BOOST_CHECK(!IsHexNumber("x0")); // broken prefix + BOOST_CHECK(!IsHexNumber("0x0x00")); // two prefixes not allowed + +} + BOOST_AUTO_TEST_CASE(util_seed_insecure_rand) { SeedInsecureRand(true); diff --git a/src/utilstrencodings.cpp b/src/utilstrencodings.cpp index 5dbf4fd7e034..4ac0e025e55c 100644 --- a/src/utilstrencodings.cpp +++ b/src/utilstrencodings.cpp @@ -65,6 +65,19 @@ bool IsHex(const std::string& str) return (str.size() > 0) && (str.size()%2 == 0); } +bool IsHexNumber(const std::string& str) +{ + size_t starting_location = 0; + if (str.size() > 2 && *str.begin() == '0' && *(str.begin()+1) == 'x') { + starting_location = 2; + } + for (auto c : str.substr(starting_location)) { + if (HexDigit(c) < 0) return false; + } + // Return false for empty string or "0x". + return (str.size() > starting_location); +} + std::vector ParseHex(const char* psz) { // convert hex dump to vector diff --git a/src/utilstrencodings.h b/src/utilstrencodings.h index a1475f3be931..c5b9f985fa2b 100644 --- a/src/utilstrencodings.h +++ b/src/utilstrencodings.h @@ -38,7 +38,13 @@ std::string SanitizeString(const std::string& str, int rule = SAFE_CHARS_DEFAULT std::vector ParseHex(const char* psz); std::vector ParseHex(const std::string& str); signed char HexDigit(char c); +/* Returns true if each character in str is a hex character, and has an even + * number of hex digits.*/ bool IsHex(const std::string& str); +/** +* Return true if the string is a hex number, optionally prefixed with "0x" +*/ +bool IsHexNumber(const std::string& str); std::vector DecodeBase64(const char* p, bool* pfInvalid = nullptr); std::string DecodeBase64(const std::string& str); std::string EncodeBase64(const unsigned char* pch, size_t len); diff --git a/src/validation.cpp b/src/validation.cpp index e55c4df742ef..4c468ddfe241 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -98,6 +98,7 @@ std::atomic fDIP0001ActiveAtTip{false}; std::atomic fDIP0003ActiveAtTip{false}; uint256 hashAssumeValid; +arith_uint256 nMinimumChainWork; CFeeRate minRelayTxFee = CFeeRate(DEFAULT_MIN_RELAY_TX_FEE); CAmount maxTxFee = DEFAULT_TRANSACTION_MAXFEE; @@ -170,6 +171,26 @@ namespace { /** chainwork for the last block that preciousblock has been applied to. */ arith_uint256 nLastPreciousChainwork = 0; + /** In order to efficiently track invalidity of headers, we keep the set of + * blocks which we tried to connect and found to be invalid here (ie which + * were set to BLOCK_FAILED_VALID since the last restart). We can then + * walk this set and check if a new header is a descendant of something in + * this set, preventing us from having to walk mapBlockIndex when we try + * to connect a bad block and fail. + * + * While this is more complicated than marking everything which descends + * from an invalid block as invalid at the time we discover it to be + * invalid, doing so would require walking all of mapBlockIndex to find all + * descendants. Since this case should be very rare, keeping track of all + * BLOCK_FAILED_VALID blocks in a set should be just fine and work just as + * well. + * + * Because we alreardy walk mapBlockIndex in height-order at startup, we go + * ahead and mark descendants of invalid blocks as FAILED_CHILD at that time, + * instead of putting things in this set. + */ + std::set g_failed_blocks; + /** Dirty block index entries. */ std::set setDirtyBlockIndex; @@ -1075,7 +1096,7 @@ bool IsInitialBlockDownload() const CChainParams& chainParams = Params(); if (chainActive.Tip() == nullptr) return true; - if (chainActive.Tip()->nChainWork < UintToArith256(chainParams.GetConsensus().nMinimumChainWork)) + if (chainActive.Tip()->nChainWork < nMinimumChainWork) return true; if (chainActive.Tip()->GetBlockTime() < (GetTime() - nMaxTipAge)) return true; @@ -1204,6 +1225,7 @@ void static InvalidChainFound(CBlockIndex* pindexNew) void static InvalidBlockFound(CBlockIndex *pindex, const CValidationState &state) { if (!state.CorruptionPossible()) { pindex->nStatus |= BLOCK_FAILED_VALID; + g_failed_blocks.insert(pindex); setDirtyBlockIndex.insert(pindex); setBlockIndexCandidates.erase(pindex); InvalidChainFound(pindex); @@ -1838,7 +1860,7 @@ static bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockInd if (it != mapBlockIndex.end()) { if (it->second->GetAncestor(pindex->nHeight) == pindex && pindexBestHeader->GetAncestor(pindex->nHeight) == pindex && - pindexBestHeader->nChainWork >= UintToArith256(chainparams.GetConsensus().nMinimumChainWork)) { + pindexBestHeader->nChainWork >= nMinimumChainWork) { // This block is a member of the assumed verified chain and an ancestor of the best header. // The equivalent time check discourages hash power from extorting the network via DOS attack // into accepting an invalid block through telling users they must manually set assumevalid. @@ -2890,10 +2912,14 @@ bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, C { AssertLockHeld(cs_main); - // Mark the block itself as invalid. - pindex->nStatus |= BLOCK_FAILED_VALID; - setDirtyBlockIndex.insert(pindex); - setBlockIndexCandidates.erase(pindex); + // We first disconnect backwards and then mark the blocks as invalid. + // This prevents a case where pruned nodes may fail to invalidateblock + // and be left unable to start as they have no tip candidates (as there + // are no blocks that meet the "have data and are not invalid per + // nStatus" criteria for inclusion in setBlockIndexCandidates). + + bool pindex_was_in_chain = false; + CBlockIndex *invalid_walk_tip = chainActive.Tip(); if (pindex == pindexBestHeader) { pindexBestInvalid = pindexBestHeader; @@ -2902,10 +2928,8 @@ bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, C DisconnectedBlockTransactions disconnectpool; while (chainActive.Contains(pindex)) { - CBlockIndex *pindexWalk = chainActive.Tip(); - pindexWalk->nStatus |= BLOCK_FAILED_CHILD; - setDirtyBlockIndex.insert(pindexWalk); - setBlockIndexCandidates.erase(pindexWalk); + const CBlockIndex* pindexOldTip = chainActive.Tip(); + pindex_was_in_chain = true; // ActivateBestChain considers blocks already in chainActive // unconditionally valid already, so force disconnect away from it. if (!DisconnectTip(state, chainparams, &disconnectpool)) { @@ -2914,12 +2938,27 @@ bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, C UpdateMempoolForReorg(disconnectpool, false); return false; } - if (pindexWalk == pindexBestHeader) { + if (pindexOldTip == pindexBestHeader) { pindexBestInvalid = pindexBestHeader; pindexBestHeader = pindexBestHeader->pprev; } } + // Now mark the blocks we just disconnected as descendants invalid + // (note this may not be all descendants). + while (pindex_was_in_chain && invalid_walk_tip != pindex) { + invalid_walk_tip->nStatus |= BLOCK_FAILED_CHILD; + setDirtyBlockIndex.insert(invalid_walk_tip); + setBlockIndexCandidates.erase(invalid_walk_tip); + invalid_walk_tip = invalid_walk_tip->pprev; + } + + // Mark the block itself as invalid. + pindex->nStatus |= BLOCK_FAILED_VALID; + setDirtyBlockIndex.insert(pindex); + setBlockIndexCandidates.erase(pindex); + g_failed_blocks.insert(pindex); + // DisconnectTip will add transactions to disconnectpool; try to add these // back to the mempool. UpdateMempoolForReorg(disconnectpool, true); @@ -2958,6 +2997,7 @@ bool ResetBlockFailureFlags(CBlockIndex *pindex) { // Reset invalid block marker if it was pointing to one of those. pindexBestInvalid = nullptr; } + g_failed_blocks.erase(it->second); } it++; } @@ -3385,6 +3425,21 @@ static bool AcceptBlockHeader(const CBlockHeader& block, CValidationState& state if (!ContextualCheckBlockHeader(block, state, chainparams, pindexPrev, GetAdjustedTime())) return error("%s: Consensus::ContextualCheckBlockHeader: %s, %s", __func__, hash.ToString(), FormatStateMessage(state)); + if (!pindexPrev->IsValid(BLOCK_VALID_SCRIPTS)) { + for (const CBlockIndex* failedit : g_failed_blocks) { + if (pindexPrev->GetAncestor(failedit->nHeight) == failedit) { + assert(failedit->nStatus & BLOCK_FAILED_VALID); + CBlockIndex* invalid_walk = pindexPrev; + while (invalid_walk != failedit) { + invalid_walk->nStatus |= BLOCK_FAILED_CHILD; + setDirtyBlockIndex.insert(invalid_walk); + invalid_walk = invalid_walk->pprev; + } + return state.DoS(100, error("%s: prev block invalid", __func__), REJECT_INVALID, "bad-prevblk"); + } + } + } + if (llmq::chainLocksHandler->HasConflictingChainLock(pindexPrev->nHeight + 1, hash)) { if (pindex == nullptr) { AddToBlockIndex(block, BLOCK_CONFLICT_CHAINLOCK); @@ -3407,13 +3462,15 @@ static bool AcceptBlockHeader(const CBlockHeader& block, CValidationState& state } // Exposed wrapper for AcceptBlockHeader -bool ProcessNewBlockHeaders(const std::vector& headers, CValidationState& state, const CChainParams& chainparams, const CBlockIndex** ppindex) +bool ProcessNewBlockHeaders(const std::vector& headers, CValidationState& state, const CChainParams& chainparams, const CBlockIndex** ppindex, CBlockHeader *first_invalid) { + if (first_invalid != nullptr) first_invalid->SetNull(); { LOCK(cs_main); for (const CBlockHeader& header : headers) { CBlockIndex *pindex = nullptr; // Use a temp pindex instead of ppindex to avoid a const_cast if (!AcceptBlockHeader(header, state, chainparams, &pindex)) { + if (first_invalid) *first_invalid = header; return false; } if (ppindex) { @@ -3443,7 +3500,7 @@ static bool AcceptBlock(const std::shared_ptr& pblock, CValidation // process an unrequested block if it's new and has enough work to // advance our tip, and isn't too many blocks ahead. bool fAlreadyHave = pindex->nStatus & BLOCK_HAVE_DATA; - bool fHasMoreWork = (chainActive.Tip() ? pindex->nChainWork > chainActive.Tip()->nChainWork : true); + bool fHasMoreOrSameWork = (chainActive.Tip() ? pindex->nChainWork >= chainActive.Tip()->nChainWork : true); // Blocks that are too out-of-order needlessly limit the effectiveness of // pruning, because pruning will not delete block files that contain any // blocks which are too close in height to the tip. Apply this test @@ -3460,9 +3517,15 @@ static bool AcceptBlock(const std::shared_ptr& pblock, CValidation // and unrequested blocks. if (fAlreadyHave) return true; if (!fRequested) { // If we didn't ask for it: - if (pindex->nTx != 0) return true; // This is a previously-processed block that was pruned - if (!fHasMoreWork) return true; // Don't process less-work chains - if (fTooFarAhead) return true; // Block height is too high + if (pindex->nTx != 0) return true; // This is a previously-processed block that was pruned + if (!fHasMoreOrSameWork) return true; // Don't process less-work chains + if (fTooFarAhead) return true; // Block height is too high + + // Protect against DoS attacks from low-work chains. + // If our tip is behind, a peer could try to send us + // low-work blocks on a fake chain that we would never + // request; don't process these. + if (pindex->nChainWork < nMinimumChainWork) return true; } if (fNewBlock) *fNewBlock = true; @@ -3829,6 +3892,10 @@ bool static LoadBlockIndexDB(const CChainParams& chainparams) pindex->nChainTx = pindex->nTx; } } + if (!(pindex->nStatus & BLOCK_FAILED_MASK) && pindex->pprev && (pindex->pprev->nStatus & BLOCK_FAILED_MASK)) { + pindex->nStatus |= BLOCK_FAILED_CHILD; + setDirtyBlockIndex.insert(pindex); + } if (pindex->IsValid(BLOCK_VALID_TRANSACTIONS) && (pindex->nChainTx || pindex->pprev == nullptr)) setBlockIndexCandidates.insert(pindex); if (pindex->nStatus & BLOCK_FAILED_MASK && (!pindexBestInvalid || pindex->nChainWork > pindexBestInvalid->nChainWork)) @@ -4142,6 +4209,7 @@ void UnloadBlockIndex() nLastBlockFile = 0; nBlockSequenceId = 1; setDirtyBlockIndex.clear(); + g_failed_blocks.clear(); setDirtyFileInfo.clear(); versionbitscache.Clear(); for (int b = 0; b < VERSIONBITS_NUM_BITS; b++) { diff --git a/src/validation.h b/src/validation.h index 580b709b26a5..aecbd25f2fb4 100644 --- a/src/validation.h +++ b/src/validation.h @@ -192,6 +192,9 @@ extern std::atomic fDIP0001ActiveAtTip; /** Block hash whose ancestors we will assume to have valid scripts without checking them. */ extern uint256 hashAssumeValid; +/** Minimum work we will assume exists on some valid chain. */ +extern arith_uint256 nMinimumChainWork; + /** Best header we've seen so far (used for getheaders queries' starting points). */ extern CBlockIndex *pindexBestHeader; @@ -251,8 +254,9 @@ bool ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr& block, CValidationState& state, const CChainParams& chainparams, const CBlockIndex** ppindex=nullptr); +bool ProcessNewBlockHeaders(const std::vector& block, CValidationState& state, const CChainParams& chainparams, const CBlockIndex** ppindex=nullptr, CBlockHeader *first_invalid=nullptr); /** Check whether enough disk space is available for an incoming block */ bool CheckDiskSpace(uint64_t nAdditionalBytes = 0); diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp index 8459d6ad47be..192ae5daebdc 100644 --- a/src/wallet/db.cpp +++ b/src/wallet/db.cpp @@ -20,6 +20,40 @@ #include +namespace { +//! Make sure database has a unique fileid within the environment. If it +//! doesn't, throw an error. BDB caches do not work properly when more than one +//! open database has the same fileid (values written to one database may show +//! up in reads to other databases). +//! +//! BerkeleyDB generates unique fileids by default +//! (https://docs.oracle.com/cd/E17275_01/html/programmer_reference/program_copy.html), +//! so bitcoin should never create different databases with the same fileid, but +//! this error can be triggered if users manually copy database files. +void CheckUniqueFileid(const CDBEnv& env, const std::string& filename, Db& db) +{ + if (env.IsMock()) return; + + u_int8_t fileid[DB_FILE_ID_LEN]; + int ret = db.get_mpf()->get_fileid(fileid); + if (ret != 0) { + throw std::runtime_error(strprintf("CDB: Can't open database %s (get_fileid failed with %d)", filename, ret)); + } + + for (const auto& item : env.mapDb) { + u_int8_t item_fileid[DB_FILE_ID_LEN]; + if (item.second && item.second->get_mpf()->get_fileid(item_fileid) == 0 && + memcmp(fileid, item_fileid, sizeof(fileid)) == 0) { + const char* item_filename = nullptr; + item.second->get_dbname(&item_filename, nullptr); + throw std::runtime_error(strprintf("CDB: Can't open database %s (duplicates fileid %s from %s)", filename, + HexStr(std::begin(item_fileid), std::end(item_fileid)), + item_filename ? item_filename : "(unknown database)")); + } + } +} +} // namespace + // // CDB // @@ -379,35 +413,34 @@ CDB::CDB(CWalletDBWrapper& dbw, const char* pszMode, bool fFlushOnCloseIn) : pdb if (!env->Open(GetDataDir())) throw std::runtime_error("CDB: Failed to open database environment."); - strFile = strFilename; - ++env->mapFileUseCount[strFile]; - pdb = env->mapDb[strFile]; + pdb = env->mapDb[strFilename]; if (pdb == nullptr) { int ret; - pdb = new Db(env->dbenv, 0); + std::unique_ptr pdb_temp(new Db(env->dbenv, 0)); bool fMockDb = env->IsMock(); if (fMockDb) { - DbMpoolFile* mpf = pdb->get_mpf(); + DbMpoolFile* mpf = pdb_temp->get_mpf(); ret = mpf->set_flags(DB_MPOOL_NOFILE, 1); - if (ret != 0) - throw std::runtime_error(strprintf("CDB: Failed to configure for no temp file backing for database %s", strFile)); + if (ret != 0) { + throw std::runtime_error(strprintf("CDB: Failed to configure for no temp file backing for database %s", strFilename)); + } } - ret = pdb->open(nullptr, // Txn pointer - fMockDb ? nullptr : strFile.c_str(), // Filename - fMockDb ? strFile.c_str() : "main", // Logical db name - DB_BTREE, // Database type - nFlags, // Flags + ret = pdb_temp->open(nullptr, // Txn pointer + fMockDb ? nullptr : strFilename.c_str(), // Filename + fMockDb ? strFilename.c_str() : "main", // Logical db name + DB_BTREE, // Database type + nFlags, // Flags 0); if (ret != 0) { - delete pdb; - pdb = nullptr; - --env->mapFileUseCount[strFile]; - strFile = ""; throw std::runtime_error(strprintf("CDB: Error %d, can't open database %s", ret, strFilename)); } + CheckUniqueFileid(*env, strFilename, *pdb_temp); + + pdb = pdb_temp.release(); + env->mapDb[strFilename] = pdb; if (fCreate && !Exists(std::string("version"))) { bool fTmp = fReadOnly; @@ -415,9 +448,9 @@ CDB::CDB(CWalletDBWrapper& dbw, const char* pszMode, bool fFlushOnCloseIn) : pdb WriteVersion(CLIENT_VERSION); fReadOnly = fTmp; } - - env->mapDb[strFile] = pdb; } + ++env->mapFileUseCount[strFilename]; + strFile = strFilename; } } @@ -673,6 +706,11 @@ bool CWalletDBWrapper::Backup(const std::string& strDest) pathDest /= strFile; try { + if (fs::equivalent(pathSrc, pathDest)) { + LogPrintf("cannot backup to wallet source file %s\n", pathDest.string()); + return false; + } + fs::copy_file(pathSrc, pathDest, fs::copy_option::overwrite_if_exists); LogPrintf("copied %s to %s\n", strFile, pathDest.string()); return true; diff --git a/src/wallet/db.h b/src/wallet/db.h index 8d28d2dcec73..8b1105cccba7 100644 --- a/src/wallet/db.h +++ b/src/wallet/db.h @@ -45,7 +45,7 @@ class CDBEnv void Reset(); void MakeMock(); - bool IsMock() { return fMockDb; } + bool IsMock() const { return fMockDb; } /** * Verify that database file strFile is OK. If it is not, diff --git a/test/functional/bip65-cltv-p2p.py b/test/functional/bip65-cltv-p2p.py index 95a88fcffb3c..37bab3729066 100755 --- a/test/functional/bip65-cltv-p2p.py +++ b/test/functional/bip65-cltv-p2p.py @@ -124,7 +124,7 @@ def run_test(self): # First we show that this tx is valid except for CLTV by getting it # rejected from the mempool for exactly that reason. - assert_raises_jsonrpc(-26, '64: non-mandatory-script-verify-flag (Negative locktime)', self.nodes[0].sendrawtransaction, bytes_to_hex_str(spendtx.serialize()), True) + assert_raises_rpc_error(-26, '64: non-mandatory-script-verify-flag (Negative locktime)', self.nodes[0].sendrawtransaction, bytes_to_hex_str(spendtx.serialize()), True) # Now we verify that a block with this transaction is also invalid. block.vtx.append(spendtx) diff --git a/test/functional/bip68-sequence.py b/test/functional/bip68-sequence.py index 9da5ad998b3f..a193b057ff12 100755 --- a/test/functional/bip68-sequence.py +++ b/test/functional/bip68-sequence.py @@ -83,7 +83,7 @@ def test_disable_flag(self): tx2.vout = [CTxOut(int(value-self.relayfee*COIN), CScript([b'a']))] tx2.rehash() - assert_raises_jsonrpc(-26, NOT_FINAL_ERROR, self.nodes[0].sendrawtransaction, ToHex(tx2)) + assert_raises_rpc_error(-26, NOT_FINAL_ERROR, self.nodes[0].sendrawtransaction, ToHex(tx2)) # Setting the version back down to 1 should disable the sequence lock, # so this should be accepted. @@ -180,7 +180,7 @@ def test_sequence_lock_confirmed_inputs(self): if (using_sequence_locks and not should_pass): # This transaction should be rejected - assert_raises_jsonrpc(-26, NOT_FINAL_ERROR, self.nodes[0].sendrawtransaction, rawtx) + assert_raises_rpc_error(-26, NOT_FINAL_ERROR, self.nodes[0].sendrawtransaction, rawtx) else: # This raw transaction should be accepted self.nodes[0].sendrawtransaction(rawtx) @@ -227,7 +227,7 @@ def test_nonzero_locks(orig_tx, node, relayfee, use_height_lock): if (orig_tx.hash in node.getrawmempool()): # sendrawtransaction should fail if the tx is in the mempool - assert_raises_jsonrpc(-26, NOT_FINAL_ERROR, node.sendrawtransaction, ToHex(tx)) + assert_raises_rpc_error(-26, NOT_FINAL_ERROR, node.sendrawtransaction, ToHex(tx)) else: # sendrawtransaction should succeed if the tx is not in the mempool node.sendrawtransaction(ToHex(tx)) @@ -280,7 +280,7 @@ def test_nonzero_locks(orig_tx, node, relayfee, use_height_lock): tx5.vout[0].nValue += int(utxos[0]["amount"]*COIN) raw_tx5 = self.nodes[0].signrawtransaction(ToHex(tx5))["hex"] - assert_raises_jsonrpc(-26, NOT_FINAL_ERROR, self.nodes[0].sendrawtransaction, raw_tx5) + assert_raises_rpc_error(-26, NOT_FINAL_ERROR, self.nodes[0].sendrawtransaction, raw_tx5) # Test mempool-BIP68 consistency after reorg # @@ -353,7 +353,7 @@ def test_bip68_not_consensus(self): tx3.vout = [CTxOut(int(tx2.vout[0].nValue - self.relayfee*COIN), CScript([b'a']))] tx3.rehash() - assert_raises_jsonrpc(-26, NOT_FINAL_ERROR, self.nodes[0].sendrawtransaction, ToHex(tx3)) + assert_raises_rpc_error(-26, NOT_FINAL_ERROR, self.nodes[0].sendrawtransaction, ToHex(tx3)) # make a block that violates bip68; ensure that the tip updates tip = int(self.nodes[0].getbestblockhash(), 16) diff --git a/test/functional/bipdersig-p2p.py b/test/functional/bipdersig-p2p.py index e8067b49b07c..52ab9714eb8e 100755 --- a/test/functional/bipdersig-p2p.py +++ b/test/functional/bipdersig-p2p.py @@ -114,7 +114,7 @@ def run_test(self): # First we show that this tx is valid except for DERSIG by getting it # rejected from the mempool for exactly that reason. - assert_raises_jsonrpc(-26, '64: non-mandatory-script-verify-flag (Non-canonical DER signature)', self.nodes[0].sendrawtransaction, bytes_to_hex_str(spendtx.serialize()), True) + assert_raises_rpc_error(-26, '64: non-mandatory-script-verify-flag (Non-canonical DER signature)', self.nodes[0].sendrawtransaction, bytes_to_hex_str(spendtx.serialize()), True) # Now we verify that a block with this transaction is also invalid. block.vtx.append(spendtx) diff --git a/test/functional/blockchain.py b/test/functional/blockchain.py index 0d44f5cb928e..6d10fb171106 100755 --- a/test/functional/blockchain.py +++ b/test/functional/blockchain.py @@ -25,7 +25,7 @@ from test_framework.util import ( assert_equal, assert_raises, - assert_raises_jsonrpc, + assert_raises_rpc_error, assert_is_hex_string, assert_is_hash_string, ) @@ -95,7 +95,7 @@ def _test_gettxoutsetinfo(self): def _test_getblockheader(self): node = self.nodes[0] - assert_raises_jsonrpc(-5, "Block not found", + assert_raises_rpc_error(-5, "Block not found", node.getblockheader, "nonsense") besthash = node.getbestblockhash() diff --git a/test/functional/disablewallet.py b/test/functional/disablewallet.py index 58a751b49d69..8591652fc08e 100755 --- a/test/functional/disablewallet.py +++ b/test/functional/disablewallet.py @@ -19,7 +19,7 @@ def set_test_params(self): def run_test (self): # Make sure wallet is really disabled - assert_raises_jsonrpc(-32601, 'Method not found', self.nodes[0].getwalletinfo) + assert_raises_rpc_error(-32601, 'Method not found', self.nodes[0].getwalletinfo) x = self.nodes[0].validateaddress('7TSBtVu959hGEGPKyHjJz9k55RpWrPffXz') assert(x['isvalid'] == False) x = self.nodes[0].validateaddress('ycwedq2f3sz2Yf9JqZsBCQPxp18WU3Hp4J') @@ -28,7 +28,7 @@ def run_test (self): # Checking mining to an address without a wallet. Generating to a valid address should succeed # but generating to an invalid address will fail. self.nodes[0].generatetoaddress(1, 'ycwedq2f3sz2Yf9JqZsBCQPxp18WU3Hp4J') - assert_raises_jsonrpc(-5, "Invalid address", self.nodes[0].generatetoaddress, 1, '7TSBtVu959hGEGPKyHjJz9k55RpWrPffXz') + assert_raises_rpc_error(-5, "Invalid address", self.nodes[0].generatetoaddress, 1, '7TSBtVu959hGEGPKyHjJz9k55RpWrPffXz') if __name__ == '__main__': DisableWalletTest ().main () diff --git a/test/functional/disconnect_ban.py b/test/functional/disconnect_ban.py index e957cf3b3399..7fe540679a92 100755 --- a/test/functional/disconnect_ban.py +++ b/test/functional/disconnect_ban.py @@ -7,7 +7,7 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - assert_raises_jsonrpc, + assert_raises_rpc_error, connect_nodes_bi, wait_until, set_node_times, @@ -34,14 +34,14 @@ def run_test(self): self.log.info("setban: fail to ban an already banned subnet") assert_equal(len(self.nodes[1].listbanned()), 1) - assert_raises_jsonrpc(-23, "IP/Subnet already banned", self.nodes[1].setban, "127.0.0.1", "add") + assert_raises_rpc_error(-23, "IP/Subnet already banned", self.nodes[1].setban, "127.0.0.1", "add") self.log.info("setban: fail to ban an invalid subnet") - assert_raises_jsonrpc(-30, "Error: Invalid IP/Subnet", self.nodes[1].setban, "127.0.0.1/42", "add") + assert_raises_rpc_error(-30, "Error: Invalid IP/Subnet", self.nodes[1].setban, "127.0.0.1/42", "add") assert_equal(len(self.nodes[1].listbanned()), 1) # still only one banned ip because 127.0.0.1 is within the range of 127.0.0.0/24 self.log.info("setban remove: fail to unban a non-banned subnet") - assert_raises_jsonrpc(-30, "Error: Unban failed", self.nodes[1].setban, "127.0.0.1", "remove") + assert_raises_rpc_error(-30, "Error: Unban failed", self.nodes[1].setban, "127.0.0.1", "remove") assert_equal(len(self.nodes[1].listbanned()), 1) self.log.info("setban remove: successfully unban subnet") @@ -78,10 +78,10 @@ def run_test(self): self.log.info("disconnectnode: fail to disconnect when calling with address and nodeid") address1 = self.nodes[0].getpeerinfo()[0]['addr'] node1 = self.nodes[0].getpeerinfo()[0]['addr'] - assert_raises_jsonrpc(-32602, "Only one of address and nodeid should be provided.", self.nodes[0].disconnectnode, address=address1, nodeid=node1) + assert_raises_rpc_error(-32602, "Only one of address and nodeid should be provided.", self.nodes[0].disconnectnode, address=address1, nodeid=node1) self.log.info("disconnectnode: fail to disconnect when calling with junk address") - assert_raises_jsonrpc(-29, "Node not found in connected nodes", self.nodes[0].disconnectnode, address="221B Baker Street") + assert_raises_rpc_error(-29, "Node not found in connected nodes", self.nodes[0].disconnectnode, address="221B Baker Street") self.log.info("disconnectnode: successfully disconnect node by address") address1 = self.nodes[0].getpeerinfo()[0]['addr'] diff --git a/test/functional/fundrawtransaction.py b/test/functional/fundrawtransaction.py index d2156a113765..e5e86327b95e 100755 --- a/test/functional/fundrawtransaction.py +++ b/test/functional/fundrawtransaction.py @@ -180,7 +180,7 @@ def run_test(self): dec_tx = self.nodes[2].decoderawtransaction(rawtx) assert_equal(utx['txid'], dec_tx['vin'][0]['txid']) - assert_raises_jsonrpc(-3, "Unexpected key foo", self.nodes[2].fundrawtransaction, rawtx, {'foo':'bar'}) + assert_raises_rpc_error(-3, "Unexpected key foo", self.nodes[2].fundrawtransaction, rawtx, {'foo':'bar'}) ############################################################ # test a fundrawtransaction with an invalid change address # @@ -193,7 +193,7 @@ def run_test(self): dec_tx = self.nodes[2].decoderawtransaction(rawtx) assert_equal(utx['txid'], dec_tx['vin'][0]['txid']) - assert_raises_jsonrpc(-5, "changeAddress must be a valid dash address", self.nodes[2].fundrawtransaction, rawtx, {'changeAddress':'foobar'}) + assert_raises_rpc_error(-5, "changeAddress must be a valid dash address", self.nodes[2].fundrawtransaction, rawtx, {'changeAddress':'foobar'}) ############################################################ # test a fundrawtransaction with a provided change address # @@ -207,7 +207,7 @@ def run_test(self): assert_equal(utx['txid'], dec_tx['vin'][0]['txid']) change = self.nodes[2].getnewaddress() - assert_raises_jsonrpc(-8, "changePosition out of bounds", self.nodes[2].fundrawtransaction, rawtx, {'changeAddress':change, 'changePosition':2}) + assert_raises_rpc_error(-8, "changePosition out of bounds", self.nodes[2].fundrawtransaction, rawtx, {'changeAddress':change, 'changePosition':2}) rawtxfund = self.nodes[2].fundrawtransaction(rawtx, {'changeAddress': change, 'changePosition': 0}) dec_tx = self.nodes[2].decoderawtransaction(rawtxfund['hex']) out = dec_tx['vout'][0] @@ -316,7 +316,7 @@ def run_test(self): rawtx = self.nodes[2].createrawtransaction(inputs, outputs) dec_tx = self.nodes[2].decoderawtransaction(rawtx) - assert_raises_jsonrpc(-4, "Insufficient funds", self.nodes[2].fundrawtransaction, rawtx) + assert_raises_rpc_error(-4, "Insufficient funds", self.nodes[2].fundrawtransaction, rawtx) ############################################################ #compare fee of a standard pubkeyhash transaction @@ -470,13 +470,13 @@ def run_test(self): rawtx = self.nodes[1].createrawtransaction(inputs, outputs) # fund a transaction that requires a new key for the change output # creating the key must be impossible because the wallet is locked - assert_raises_jsonrpc(-4, "Keypool ran out, please call keypoolrefill first", self.nodes[1].fundrawtransaction, rawtx) + assert_raises_rpc_error(-4, "Keypool ran out, please call keypoolrefill first", self.nodes[1].fundrawtransaction, rawtx) #refill the keypool self.nodes[1].walletpassphrase("test", 100) self.nodes[1].walletlock() - assert_raises_jsonrpc(-13, "walletpassphrase", self.nodes[1].sendtoaddress, self.nodes[0].getnewaddress(), 12) + assert_raises_rpc_error(-13, "walletpassphrase", self.nodes[1].sendtoaddress, self.nodes[0].getnewaddress(), 12) oldBalance = self.nodes[0].getbalance() diff --git a/test/functional/import-rescan.py b/test/functional/import-rescan.py index 6296a1a473f7..f78370c33ae2 100755 --- a/test/functional/import-rescan.py +++ b/test/functional/import-rescan.py @@ -19,9 +19,8 @@ happened previously. """ -from test_framework.authproxy import JSONRPCException from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import (connect_nodes, sync_blocks, assert_equal, set_node_times) +from test_framework.util import (assert_raises_rpc_error, connect_nodes, sync_blocks, assert_equal, set_node_times) import collections import enum @@ -36,21 +35,26 @@ class Variant(collections.namedtuple("Variant", "call data rescan prune")): """Helper for importing one key and verifying scanned transactions.""" + def try_rpc(self, func, *args, **kwargs): + if self.expect_disabled: + assert_raises_rpc_error(-4, "Rescan is disabled in pruned mode", func, *args, **kwargs) + else: + return func(*args, **kwargs) + def do_import(self, timestamp): """Call one key import RPC.""" if self.call == Call.single: if self.data == Data.address: - response, error = try_rpc(self.node.importaddress, self.address["address"], self.label, - self.rescan == Rescan.yes) + response = self.try_rpc(self.node.importaddress, self.address["address"], self.label, + self.rescan == Rescan.yes) elif self.data == Data.pub: - response, error = try_rpc(self.node.importpubkey, self.address["pubkey"], self.label, - self.rescan == Rescan.yes) + response = self.try_rpc(self.node.importpubkey, self.address["pubkey"], self.label, + self.rescan == Rescan.yes) elif self.data == Data.priv: - response, error = try_rpc(self.node.importprivkey, self.key, self.label, self.rescan == Rescan.yes) + response = self.try_rpc(self.node.importprivkey, self.key, self.label, self.rescan == Rescan.yes) assert_equal(response, None) - assert_equal(error, {'message': 'Rescan is disabled in pruned mode', - 'code': -4} if self.expect_disabled else None) + elif self.call == Call.multi: response = self.node.importmulti([{ "scriptPubKey": { @@ -182,13 +186,5 @@ def run_test(self): else: variant.check() - -def try_rpc(func, *args, **kwargs): - try: - return func(*args, **kwargs), None - except JSONRPCException as e: - return None, e.error - - if __name__ == "__main__": ImportRescanTest().main() diff --git a/test/functional/importmulti.py b/test/functional/importmulti.py index 6907d6407052..324f6458e305 100755 --- a/test/functional/importmulti.py +++ b/test/functional/importmulti.py @@ -448,11 +448,11 @@ def run_test (self): # Bad or missing timestamps self.log.info("Should throw on invalid or missing timestamp values") - assert_raises_message(JSONRPCException, 'Missing required timestamp field for key', + assert_raises_rpc_error(-3, 'Missing required timestamp field for key', self.nodes[1].importmulti, [{ "scriptPubKey": address['scriptPubKey'], }]) - assert_raises_message(JSONRPCException, 'Expected number or "now" timestamp value for key. got type string', + assert_raises_rpc_error(-3, 'Expected number or "now" timestamp value for key. got type string', self.nodes[1].importmulti, [{ "scriptPubKey": address['scriptPubKey'], "timestamp": "", diff --git a/test/functional/importprunedfunds.py b/test/functional/importprunedfunds.py index 3815440a3844..b1a74ba89215 100755 --- a/test/functional/importprunedfunds.py +++ b/test/functional/importprunedfunds.py @@ -67,7 +67,7 @@ def run_test(self): self.sync_all() #Import with no affiliated address - assert_raises_jsonrpc(-5, "No addresses", self.nodes[1].importprunedfunds, rawtxn1, proof1) + assert_raises_rpc_error(-5, "No addresses", self.nodes[1].importprunedfunds, rawtxn1, proof1) balance1 = self.nodes[1].getbalance("", 0, False, True) assert_equal(balance1, Decimal(0)) @@ -98,7 +98,7 @@ def run_test(self): assert_equal(address_info['ismine'], True) #Remove transactions - assert_raises_jsonrpc(-8, "Transaction does not exist in wallet.", self.nodes[1].removeprunedfunds, txnid1) + assert_raises_rpc_error(-8, "Transaction does not exist in wallet.", self.nodes[1].removeprunedfunds, txnid1) balance1 = self.nodes[1].getbalance("*", 0, False, True) assert_equal(balance1, Decimal('0.075')) diff --git a/test/functional/keypool.py b/test/functional/keypool.py index df4249c44ac2..fd71f96536c3 100755 --- a/test/functional/keypool.py +++ b/test/functional/keypool.py @@ -22,7 +22,7 @@ def run_test(self): # Keep creating keys addr = nodes[0].getnewaddress() - assert_raises_jsonrpc(-12, "Error: Keypool ran out, please call keypoolrefill first", nodes[0].getnewaddress) + assert_raises_rpc_error(-12, "Error: Keypool ran out, please call keypoolrefill first", nodes[0].getnewaddress) # put three new keys in the keypool nodes[0].walletpassphrase('test', 12000) @@ -37,7 +37,7 @@ def run_test(self): # assert that three unique addresses were returned assert(len(addr) == 3) # the next one should fail - assert_raises_jsonrpc(-12, "Keypool ran out", nodes[0].getrawchangeaddress) + assert_raises_rpc_error(-12, "Keypool ran out", nodes[0].getrawchangeaddress) # refill keypool with three new addresses nodes[0].walletpassphrase('test', 1) @@ -50,7 +50,7 @@ def run_test(self): nodes[0].generate(1) nodes[0].generate(1) nodes[0].generate(1) - assert_raises_jsonrpc(-12, "Keypool ran out", nodes[0].generate, 1) + assert_raises_rpc_error(-12, "Keypool ran out", nodes[0].generate, 1) if __name__ == '__main__': KeyPoolTest().main() diff --git a/test/functional/llmq-is-cl-conflicts.py b/test/functional/llmq-is-cl-conflicts.py index 99bd98f3e3a9..9853704a4bde 100755 --- a/test/functional/llmq-is-cl-conflicts.py +++ b/test/functional/llmq-is-cl-conflicts.py @@ -9,7 +9,7 @@ from test_framework.blocktools import get_masternode_payment, create_coinbase, create_block from test_framework.mininode import * from test_framework.test_framework import DashTestFramework -from test_framework.util import sync_blocks, p2p_port, assert_raises_jsonrpc, set_node_times +from test_framework.util import sync_blocks, p2p_port, assert_raises_rpc_error, set_node_times ''' llmq-is-cl-conflicts.py @@ -148,8 +148,8 @@ def test_chainlock_overrides_islock(self, test_block_conflict): # Lets verify that the ISLOCKs got pruned for node in self.nodes: - assert_raises_jsonrpc(-5, "No such mempool or blockchain transaction", node.getrawtransaction, rawtx1_txid, True) - assert_raises_jsonrpc(-5, "No such mempool or blockchain transaction", node.getrawtransaction, rawtx4_txid, True) + assert_raises_rpc_error(-5, "No such mempool or blockchain transaction", node.getrawtransaction, rawtx1_txid, True) + assert_raises_rpc_error(-5, "No such mempool or blockchain transaction", node.getrawtransaction, rawtx4_txid, True) rawtx = node.getrawtransaction(rawtx2_txid, True) assert(rawtx['chainlock']) assert(rawtx['instantlock']) @@ -186,7 +186,7 @@ def test_islock_overrides_nonchainlock(self): # Assert that the conflicting tx got mined and the locked TX is not valid assert(self.nodes[0].getrawtransaction(rawtx1_txid, True)['confirmations'] > 0) - assert_raises_jsonrpc(-25, "Missing inputs", self.nodes[0].sendrawtransaction, rawtx2) + assert_raises_rpc_error(-25, "Missing inputs", self.nodes[0].sendrawtransaction, rawtx2) # Send the ISLOCK, which should result in the last 2 blocks to be invalidated, even though the nodes don't know # the locked transaction yet diff --git a/test/functional/mempool_packages.py b/test/functional/mempool_packages.py index e5d6ce31cc7f..8e18f311de6b 100755 --- a/test/functional/mempool_packages.py +++ b/test/functional/mempool_packages.py @@ -115,7 +115,7 @@ def run_test(self): assert_equal(mempool[x]['descendantfees'], descendant_fees * COIN + 1000) # Adding one more transaction on to the chain should fail. - assert_raises_jsonrpc(-26, "too-long-mempool-chain", self.chain_transaction, self.nodes[0], txid, vout, value, fee, 1) + assert_raises_rpc_error(-26, "too-long-mempool-chain", self.chain_transaction, self.nodes[0], txid, vout, value, fee, 1) # Check that prioritising a tx before it's added to the mempool works # First clear the mempool by mining a block. @@ -167,7 +167,7 @@ def run_test(self): # Sending one more chained transaction will fail utxo = transaction_package.pop(0) - assert_raises_jsonrpc(-26, "too-long-mempool-chain", self.chain_transaction, self.nodes[0], utxo['txid'], utxo['vout'], utxo['amount'], fee, 10) + assert_raises_rpc_error(-26, "too-long-mempool-chain", self.chain_transaction, self.nodes[0], utxo['txid'], utxo['vout'], utxo['amount'], fee, 10) # TODO: check that node1's mempool is as expected diff --git a/test/functional/mempool_reorg.py b/test/functional/mempool_reorg.py index f75c507def26..fb9b7d77ef5f 100755 --- a/test/functional/mempool_reorg.py +++ b/test/functional/mempool_reorg.py @@ -50,14 +50,14 @@ def run_test(self): timelock_tx = timelock_tx[:-8] + hex(self.nodes[0].getblockcount() + 2)[2:] + "000000" timelock_tx = self.nodes[0].signrawtransaction(timelock_tx)["hex"] # This will raise an exception because the timelock transaction is too immature to spend - assert_raises_jsonrpc(-26, "non-final", self.nodes[0].sendrawtransaction, timelock_tx) + assert_raises_rpc_error(-26, "non-final", self.nodes[0].sendrawtransaction, timelock_tx) # Broadcast and mine spend_102 and 103: spend_102_id = self.nodes[0].sendrawtransaction(spend_102_raw) spend_103_id = self.nodes[0].sendrawtransaction(spend_103_raw) self.nodes[0].generate(1) # Time-locked transaction is still too immature to spend - assert_raises_jsonrpc(-26,'non-final', self.nodes[0].sendrawtransaction, timelock_tx) + assert_raises_rpc_error(-26,'non-final', self.nodes[0].sendrawtransaction, timelock_tx) # Create 102_1 and 103_1: spend_102_1_raw = create_tx(self.nodes[0], spend_102_id, node1_address, 499.8) diff --git a/test/functional/mempool_spendcoinbase.py b/test/functional/mempool_spendcoinbase.py index 358bf392eaaa..ea297f150a89 100755 --- a/test/functional/mempool_spendcoinbase.py +++ b/test/functional/mempool_spendcoinbase.py @@ -36,7 +36,7 @@ def run_test(self): spend_101_id = self.nodes[0].sendrawtransaction(spends_raw[0]) # coinbase at height 102 should be too immature to spend - assert_raises_jsonrpc(-26,"bad-txns-premature-spend-of-coinbase", self.nodes[0].sendrawtransaction, spends_raw[1]) + assert_raises_rpc_error(-26,"bad-txns-premature-spend-of-coinbase", self.nodes[0].sendrawtransaction, spends_raw[1]) # mempool should have just spend_101: assert_equal(self.nodes[0].getrawmempool(), [ spend_101_id ]) diff --git a/test/functional/merkle_blocks.py b/test/functional/merkle_blocks.py index 7c58654ef18a..99158bf9433e 100755 --- a/test/functional/merkle_blocks.py +++ b/test/functional/merkle_blocks.py @@ -39,7 +39,7 @@ def run_test(self): tx2 = self.nodes[0].createrawtransaction([node0utxos.pop()], {self.nodes[1].getnewaddress(): 500 - tx_fee}) txid2 = self.nodes[0].sendrawtransaction(self.nodes[0].signrawtransaction(tx2)["hex"]) # This will raise an exception because the transaction is not yet in a block - assert_raises_jsonrpc(-5, "Transaction not yet in block", self.nodes[0].gettxoutproof, [txid1]) + assert_raises_rpc_error(-5, "Transaction not yet in block", self.nodes[0].gettxoutproof, [txid1]) self.nodes[0].generate(1) blockhash = self.nodes[0].getblockhash(chain_height + 1) @@ -65,11 +65,11 @@ def run_test(self): # We can't find the block from a fully-spent tx # Doesn't apply to Dash Core - we have txindex always on - # assert_raises_jsonrpc(-5, "Transaction not yet in block", self.nodes[2].gettxoutproof, [txid_spent]) + # assert_raises_rpc_error(-5, "Transaction not yet in block", self.nodes[2].gettxoutproof, [txid_spent]) # We can get the proof if we specify the block assert_equal(self.nodes[2].verifytxoutproof(self.nodes[2].gettxoutproof([txid_spent], blockhash)), [txid_spent]) # We can't get the proof if we specify a non-existent block - assert_raises_jsonrpc(-5, "Block not found", self.nodes[2].gettxoutproof, [txid_spent], "00000000000000000000000000000000") + assert_raises_rpc_error(-5, "Block not found", self.nodes[2].gettxoutproof, [txid_spent], "00000000000000000000000000000000") # We can get the proof if the transaction is unspent assert_equal(self.nodes[2].verifytxoutproof(self.nodes[2].gettxoutproof([txid_unspent])), [txid_unspent]) # We can get the proof if we provide a list of transactions and one of them is unspent. The ordering of the list should not matter. @@ -78,7 +78,7 @@ def run_test(self): # We can always get a proof if we have a -txindex assert_equal(self.nodes[2].verifytxoutproof(self.nodes[3].gettxoutproof([txid_spent])), [txid_spent]) # We can't get a proof if we specify transactions from different blocks - assert_raises_jsonrpc(-5, "Not all transactions found in specified or retrieved block", self.nodes[2].gettxoutproof, [txid1, txid3]) + assert_raises_rpc_error(-5, "Not all transactions found in specified or retrieved block", self.nodes[2].gettxoutproof, [txid1, txid3]) if __name__ == '__main__': diff --git a/test/functional/minchainwork.py b/test/functional/minchainwork.py new file mode 100755 index 000000000000..84cd11f72274 --- /dev/null +++ b/test/functional/minchainwork.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +# Copyright (c) 2017 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test logic for setting nMinimumChainWork on command line. + +Nodes don't consider themselves out of "initial block download" until +their active chain has more work than nMinimumChainWork. + +Nodes don't download blocks from a peer unless the peer's best known block +has more work than nMinimumChainWork. + +While in initial block download, nodes won't relay blocks to their peers, so +test that this parameter functions as intended by verifying that block relay +only succeeds past a given node once its nMinimumChainWork has been exceeded. +""" + +import time + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import sync_blocks, connect_nodes, assert_equal + +# 2 hashes required per regtest block (with no difficulty adjustment) +REGTEST_WORK_PER_BLOCK = 2 + +class MinimumChainWorkTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 3 + + self.extra_args = [[], ["-minimumchainwork=0x65"], ["-minimumchainwork=0x65"]] + self.node_min_work = [0, 101, 101] + + def setup_network(self): + # Force CanDirectFetch to return false (otherwise nMinimumChainWork is ignored) + self.mocktime += 21 * 2.6 * 60 + # This test relies on the chain setup being: + # node0 <- node1 <- node2 + # Before leaving IBD, nodes prefer to download blocks from outbound + # peers, so ensure that we're mining on an outbound peer and testing + # block relay to inbound peers. + self.setup_nodes() + for i in range(self.num_nodes-1): + connect_nodes(self.nodes[i+1], i) + + def run_test(self): + # Start building a chain on node0. node2 shouldn't be able to sync until node1's + # minchainwork is exceeded + starting_chain_work = REGTEST_WORK_PER_BLOCK # Genesis block's work + self.log.info("Testing relay across node %d (minChainWork = %d)", 1, self.node_min_work[1]) + + starting_blockcount = self.nodes[2].getblockcount() + + num_blocks_to_generate = int((self.node_min_work[1] - starting_chain_work) / REGTEST_WORK_PER_BLOCK) + self.log.info("Generating %d blocks on node0", num_blocks_to_generate) + hashes = self.nodes[0].generate(num_blocks_to_generate) + + self.log.info("Node0 current chain work: %s", self.nodes[0].getblockheader(hashes[-1])['chainwork']) + + # Sleep a few seconds and verify that node2 didn't get any new blocks + # or headers. We sleep, rather than sync_blocks(node0, node1) because + # it's reasonable either way for node1 to get the blocks, or not get + # them (since they're below node1's minchainwork). + time.sleep(3) + + self.log.info("Verifying node 2 has no more blocks than before") + self.log.info("Blockcounts: %s", [n.getblockcount() for n in self.nodes]) + # Node2 shouldn't have any new headers yet, because node1 should not + # have relayed anything. + assert_equal(len(self.nodes[2].getchaintips()), 1) + assert_equal(self.nodes[2].getchaintips()[0]['height'], 0) + + assert self.nodes[1].getbestblockhash() != self.nodes[0].getbestblockhash() + assert_equal(self.nodes[2].getblockcount(), starting_blockcount) + + self.log.info("Generating one more block") + self.nodes[0].generate(1) + + self.log.info("Verifying nodes are all synced") + + # Because nodes in regtest are all manual connections (eg using + # addnode), node1 should not have disconnected node0. If not for that, + # we'd expect node1 to have disconnected node0 for serving an + # insufficient work chain, in which case we'd need to reconnect them to + # continue the test. + + self.sync_all() + self.log.info("Blockcounts: %s", [n.getblockcount() for n in self.nodes]) + +if __name__ == '__main__': + MinimumChainWorkTest().main() diff --git a/test/functional/mining.py b/test/functional/mining.py index 45fbaa145caf..fbf478ce831d 100755 --- a/test/functional/mining.py +++ b/test/functional/mining.py @@ -15,7 +15,7 @@ from test_framework.blocktools import create_coinbase from test_framework.mininode import CBlock from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_raises_jsonrpc +from test_framework.util import assert_equal, assert_raises_rpc_error def b2x(b): return b2a_hex(b).decode('ascii') @@ -68,7 +68,7 @@ def run_test(self): assert_template(node, block, None) self.log.info("submitblock: Test block decode failure") - assert_raises_jsonrpc(-22, "Block decode failed", node.submitblock, b2x(block.serialize()[:-15])) + assert_raises_rpc_error(-22, "Block decode failed", node.submitblock, b2x(block.serialize()[:-15])) self.log.info("getblocktemplate: Test bad input hash for coinbase transaction") bad_block = copy.deepcopy(block) @@ -77,10 +77,10 @@ def run_test(self): assert_template(node, bad_block, 'bad-cb-missing') self.log.info("submitblock: Test invalid coinbase transaction") - assert_raises_jsonrpc(-22, "Block does not start with a coinbase", node.submitblock, b2x(bad_block.serialize())) + assert_raises_rpc_error(-22, "Block does not start with a coinbase", node.submitblock, b2x(bad_block.serialize())) self.log.info("getblocktemplate: Test truncated final transaction") - assert_raises_jsonrpc(-22, "Block decode failed", node.getblocktemplate, {'data': b2x(block.serialize()[:-1]), 'mode': 'proposal'}) + assert_raises_rpc_error(-22, "Block decode failed", node.getblocktemplate, {'data': b2x(block.serialize()[:-1]), 'mode': 'proposal'}) self.log.info("getblocktemplate: Test duplicate transaction") bad_block = copy.deepcopy(block) @@ -107,7 +107,7 @@ def run_test(self): bad_block_sn = bytearray(block.serialize()) assert_equal(bad_block_sn[TX_COUNT_OFFSET], 1) bad_block_sn[TX_COUNT_OFFSET] += 1 - assert_raises_jsonrpc(-22, "Block decode failed", node.getblocktemplate, {'data': b2x(bad_block_sn), 'mode': 'proposal'}) + assert_raises_rpc_error(-22, "Block decode failed", node.getblocktemplate, {'data': b2x(bad_block_sn), 'mode': 'proposal'}) self.log.info("getblocktemplate: Test bad bits") bad_block = copy.deepcopy(block) diff --git a/test/functional/multiwallet.py b/test/functional/multiwallet.py index b37f7a892c63..04c912fee66d 100755 --- a/test/functional/multiwallet.py +++ b/test/functional/multiwallet.py @@ -7,9 +7,10 @@ Verify that a bitcoind node can load multiple wallet files """ import os +import shutil from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_raises_jsonrpc +from test_framework.util import assert_equal, assert_raises_rpc_error class MultiWalletTest(BitcoinTestFramework): def set_test_params(self): @@ -29,6 +30,11 @@ def run_test(self): os.mkdir(os.path.join(self.options.tmpdir, 'node0', 'regtest', 'w11')) self.assert_start_raises_init_error(0, ['-wallet=w11'], 'Error loading wallet w11. -wallet filename must be a regular file.') + # should not initialize if one wallet is a copy of another + shutil.copyfile(os.path.join(self.options.tmpdir, 'node0', 'regtest', 'w2'), + os.path.join(self.options.tmpdir, 'node0', 'regtest', 'w22')) + self.assert_start_raises_init_error(0, ['-wallet=w2', '-wallet=w22'], 'duplicates fileid') + # should not initialize if wallet file is a symlink os.symlink(os.path.join(self.options.tmpdir, 'node0', 'regtest', 'w1'), os.path.join(self.options.tmpdir, 'node0', 'regtest', 'w12')) self.assert_start_raises_init_error(0, ['-wallet=w12'], 'Error loading wallet w12. -wallet filename must be a regular file.') @@ -43,10 +49,10 @@ def run_test(self): w1.generate(1) # accessing invalid wallet fails - assert_raises_jsonrpc(-18, "Requested wallet does not exist or is not loaded", wallet_bad.getwalletinfo) + assert_raises_rpc_error(-18, "Requested wallet does not exist or is not loaded", wallet_bad.getwalletinfo) # accessing wallet RPC without using wallet endpoint fails - assert_raises_jsonrpc(-19, "Wallet file not specified", self.nodes[0].getwalletinfo) + assert_raises_rpc_error(-19, "Wallet file not specified", self.nodes[0].getwalletinfo) # check w1 wallet balance w1_info = w1.getwalletinfo() diff --git a/test/functional/net.py b/test/functional/net.py index dedb2fe7cbef..a8d8d2061386 100755 --- a/test/functional/net.py +++ b/test/functional/net.py @@ -12,7 +12,7 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - assert_raises_jsonrpc, + assert_raises_rpc_error, connect_nodes_bi, p2p_port ) @@ -84,7 +84,7 @@ def _test_getaddednodeinfo(self): assert_equal(len(added_nodes), 1) assert_equal(added_nodes[0]['addednode'], ip_port) # check that a non-existant node returns an error - assert_raises_jsonrpc(-24, "Node has not been added", + assert_raises_rpc_error(-24, "Node has not been added", self.nodes[0].getaddednodeinfo, '1.1.1.1') def _test_getpeerinfo(self): diff --git a/test/functional/nulldummy.py b/test/functional/nulldummy.py index 21a115a62ddd..6053f21453c7 100755 --- a/test/functional/nulldummy.py +++ b/test/functional/nulldummy.py @@ -66,7 +66,7 @@ def run_test(self): self.log.info("Test 2: Non-NULLDUMMY base multisig transaction should not be accepted to mempool before activation") test2tx = self.create_transaction(self.nodes[0], txid2, self.ms_address, 47) trueDummy(test2tx) - assert_raises_jsonrpc(-26, NULLDUMMY_ERROR, self.nodes[0].sendrawtransaction, bytes_to_hex_str(test2tx.serialize()), True) + assert_raises_rpc_error(-26, NULLDUMMY_ERROR, self.nodes[0].sendrawtransaction, bytes_to_hex_str(test2tx.serialize()), True) self.log.info("Test 3: Non-NULLDUMMY base transactions should be accepted in a block before activation [431]") self.block_submit(self.nodes[0], [test2tx], True) @@ -75,7 +75,7 @@ def run_test(self): test4tx = self.create_transaction(self.nodes[0], test2tx.hash, self.address, 46) test6txs=[CTransaction(test4tx)] trueDummy(test4tx) - assert_raises_jsonrpc(-26, NULLDUMMY_ERROR, self.nodes[0].sendrawtransaction, bytes_to_hex_str(test4tx.serialize()), True) + assert_raises_rpc_error(-26, NULLDUMMY_ERROR, self.nodes[0].sendrawtransaction, bytes_to_hex_str(test4tx.serialize()), True) self.block_submit(self.nodes[0], [test4tx]) self.log.info("Test 6: NULLDUMMY compliant transactions should be accepted to mempool and in block after activation [432]") diff --git a/test/functional/p2p-acceptblock.py b/test/functional/p2p-acceptblock.py index 489b6328b7db..085c8ecf7802 100755 --- a/test/functional/p2p-acceptblock.py +++ b/test/functional/p2p-acceptblock.py @@ -4,37 +4,32 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test processing of unrequested blocks. -Since behavior differs when receiving unrequested blocks from whitelisted peers -versus non-whitelisted peers, this tests the behavior of both (effectively two -separate tests running in parallel). +Setup: two nodes, node0+node1, not connected to each other. Node1 will have +nMinimumChainWork set to 0x10, so it won't process low-work unrequested blocks. -Setup: two nodes, node0 and node1, not connected to each other. Node0 does not -whitelist localhost, but node1 does. They will each be on their own chain for -this test. - -We have one NodeConn connection to each, test_node and white_node respectively. +We have one NodeConn connection to node0 called test_node, and one to node1 +called min_work_node. The test: 1. Generate one block on each node, to leave IBD. 2. Mine a new block on each tip, and deliver to each node from node's peer. - The tip should advance. + The tip should advance for node0, but node1 should skip processing due to + nMinimumChainWork. + +Node1 is unused in tests 3-7: -3. Mine a block that forks the previous block, and deliver to each node from - corresponding peer. - Node0 should not process this block (just accept the header), because it is - unrequested and doesn't have more work than the tip. - Node1 should process because this is coming from a whitelisted peer. +3. Mine a block that forks from the genesis block, and deliver to test_node. + Node0 should not process this block (just accept the header), because it + is unrequested and doesn't have more or equal work to the tip. -4. Send another block that builds on the forking block. - Node0 should process this block but be stuck on the shorter chain, because - it's missing an intermediate block. - Node1 should reorg to this longer chain. +4a,b. Send another two blocks that build on the forking block. + Node0 should process the second block but be stuck on the shorter chain, + because it's missing an intermediate block. -4b.Send 288 more blocks on the longer chain. +4c.Send 288 more blocks on the longer chain (the number of blocks ahead + we currently store). Node0 should process all but the last block (too far ahead in height). - Send all headers to Node1, and then send the last block in that chain. - Node1 should accept the block because it's coming from a whitelisted peer. 5. Send a duplicate of the block in #3 to Node0. Node0 should not process the block because it is unrequested, and stay on @@ -46,13 +41,21 @@ 7. Send Node0 the missing block again. Node0 should process and the tip should advance. + +8. Create a fork which is invalid at a height longer than the current chain + (ie to which the node will try to reorg) but which has headers built on top + of the invalid block. Check that we get disconnected if we send more headers + on the chain the node now knows to be invalid. + +9. Test Node1 is able to sync when connected to node0 (which should have sufficient + work on its chain). """ from test_framework.mininode import * from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * import time -from test_framework.blocktools import create_block, create_coinbase +from test_framework.blocktools import create_block, create_coinbase, create_transaction class AcceptBlockTest(BitcoinTestFramework): def add_options(self, parser): @@ -63,37 +66,39 @@ def add_options(self, parser): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 2 - self.extra_args = [[], ["-whitelist=127.0.0.1"]] + self.extra_args = [[], ["-minimumchainwork=0x10"]] def setup_network(self): # Node0 will be used to test behavior of processing unrequested blocks # from peers which are not whitelisted, while Node1 will be used for # the whitelisted case. + # Node2 will be used for non-whitelisted peers to test the interaction + # with nMinimumChainWork. self.setup_nodes() def run_test(self): # Setup the p2p connections and start up the network thread. - test_node = NodeConnCB() # connects to node0 (not whitelisted) - white_node = NodeConnCB() # connects to node1 (whitelisted) + test_node = NodeConnCB() # connects to node0 + min_work_node = NodeConnCB() # connects to node1 connections = [] connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], test_node)) - connections.append(NodeConn('127.0.0.1', p2p_port(1), self.nodes[1], white_node)) + connections.append(NodeConn('127.0.0.1', p2p_port(1), self.nodes[1], min_work_node)) test_node.add_connection(connections[0]) - white_node.add_connection(connections[1]) + min_work_node.add_connection(connections[1]) NetworkThread().start() # Start up network handling in another thread # Test logic begins here test_node.wait_for_verack() - white_node.wait_for_verack() + min_work_node.wait_for_verack() - # 1. Have both nodes mine a block (leave IBD) + # 1. Have nodes mine a block (leave IBD) [ n.generate(1) for n in self.nodes ] tips = [ int("0x" + n.getbestblockhash(), 0) for n in self.nodes ] # 2. Send one block that builds on each tip. - # This should be accepted. + # This should be accepted by node0 blocks_h2 = [] # the height 2 blocks on each node's chain block_time = self.mocktime + 1 for i in range(2): @@ -101,96 +106,119 @@ def run_test(self): blocks_h2[i].solve() block_time += 1 test_node.send_message(msg_block(blocks_h2[0])) - white_node.send_message(msg_block(blocks_h2[1])) + min_work_node.send_message(msg_block(blocks_h2[1])) - [ x.sync_with_ping() for x in [test_node, white_node] ] + for x in [test_node, min_work_node]: + x.sync_with_ping() assert_equal(self.nodes[0].getblockcount(), 2) - assert_equal(self.nodes[1].getblockcount(), 2) - self.log.info("First height 2 block accepted by both nodes") + assert_equal(self.nodes[1].getblockcount(), 1) + self.log.info("First height 2 block accepted by node0; correctly rejected by node1") - # 3. Send another block that builds on the original tip. - blocks_h2f = [] # Blocks at height 2 that fork off the main chain - for i in range(2): - blocks_h2f.append(create_block(tips[i], create_coinbase(2), blocks_h2[i].nTime+1)) - blocks_h2f[i].solve() - test_node.send_message(msg_block(blocks_h2f[0])) - white_node.send_message(msg_block(blocks_h2f[1])) + # 3. Send another block that builds on genesis. + block_h1f = create_block(int("0x" + self.nodes[0].getblockhash(0), 0), create_coinbase(1), block_time) + block_time += 1 + block_h1f.solve() + test_node.send_message(msg_block(block_h1f)) - [ x.sync_with_ping() for x in [test_node, white_node] ] + test_node.sync_with_ping() + tip_entry_found = False for x in self.nodes[0].getchaintips(): - if x['hash'] == blocks_h2f[0].hash: + if x['hash'] == block_h1f.hash: assert_equal(x['status'], "headers-only") + tip_entry_found = True + assert(tip_entry_found) + assert_raises_rpc_error(-1, "Block not found on disk", self.nodes[0].getblock, block_h1f.hash) - for x in self.nodes[1].getchaintips(): - if x['hash'] == blocks_h2f[1].hash: - assert_equal(x['status'], "valid-headers") + # 4. Send another two block that build on the fork. + block_h2f = create_block(block_h1f.sha256, create_coinbase(2), block_time) + block_time += 1 + block_h2f.solve() + test_node.send_message(msg_block(block_h2f)) - self.log.info("Second height 2 block accepted only from whitelisted peer") + test_node.sync_with_ping() + # Since the earlier block was not processed by node, the new block + # can't be fully validated. + tip_entry_found = False + for x in self.nodes[0].getchaintips(): + if x['hash'] == block_h2f.hash: + assert_equal(x['status'], "headers-only") + tip_entry_found = True + assert(tip_entry_found) - # 4. Now send another block that builds on the forking chain. - blocks_h3 = [] - for i in range(2): - blocks_h3.append(create_block(blocks_h2f[i].sha256, create_coinbase(3), blocks_h2f[i].nTime+1)) - blocks_h3[i].solve() - test_node.send_message(msg_block(blocks_h3[0])) - white_node.send_message(msg_block(blocks_h3[1])) + # But this block should be accepted by node since it has equal work. + self.nodes[0].getblock(block_h2f.hash) + self.log.info("Second height 2 block accepted, but not reorg'ed to") - [ x.sync_with_ping() for x in [test_node, white_node] ] - # Since the earlier block was not processed by node0, the new block + # 4b. Now send another block that builds on the forking chain. + block_h3 = create_block(block_h2f.sha256, create_coinbase(3), block_h2f.nTime+1) + block_h3.solve() + test_node.send_message(msg_block(block_h3)) + + test_node.sync_with_ping() + # Since the earlier block was not processed by node, the new block # can't be fully validated. + tip_entry_found = False for x in self.nodes[0].getchaintips(): - if x['hash'] == blocks_h3[0].hash: + if x['hash'] == block_h3.hash: assert_equal(x['status'], "headers-only") + tip_entry_found = True + assert(tip_entry_found) + self.nodes[0].getblock(block_h3.hash) + + # But this block should be accepted by node since it has more work. + self.nodes[0].getblock(block_h3.hash) + self.log.info("Unrequested more-work block accepted") + + # 4c. Now mine 288 more blocks and deliver; all should be processed but + # the last (height-too-high) on node (as long as its not missing any headers) + tip = block_h3 + all_blocks = [] + for i in range(288): + next_block = create_block(tip.sha256, create_coinbase(i + 4), tip.nTime+1) + next_block.solve() + all_blocks.append(next_block) + tip = next_block + + # Now send the block at height 5 and check that it wasn't accepted (missing header) + test_node.send_message(msg_block(all_blocks[1])) + test_node.sync_with_ping() + assert_raises_rpc_error(-5, "Block not found", self.nodes[0].getblock, all_blocks[1].hash) + assert_raises_rpc_error(-5, "Block not found", self.nodes[0].getblockheader, all_blocks[1].hash) - # But this block should be accepted by node0 since it has more work. - self.nodes[0].getblock(blocks_h3[0].hash) - self.log.info("Unrequested more-work block accepted from non-whitelisted peer") + # The block at height 5 should be accepted if we provide the missing header, though + headers_message = msg_headers() + headers_message.headers.append(CBlockHeader(all_blocks[0])) + test_node.send_message(headers_message) + test_node.send_message(msg_block(all_blocks[1])) + test_node.sync_with_ping() + self.nodes[0].getblock(all_blocks[1].hash) - # Node1 should have accepted and reorged. - assert_equal(self.nodes[1].getblockcount(), 3) - self.log.info("Successfully reorged to length 3 chain from whitelisted peer") + # Now send the blocks in all_blocks + for i in range(288): + test_node.send_message(msg_block(all_blocks[i])) + test_node.sync_with_ping() - # 4b. Now mine 288 more blocks and deliver; all should be processed but - # the last (height-too-high) on node0. Node1 should process the tip if - # we give it the headers chain leading to the tip. - tips = blocks_h3 - headers_message = msg_headers() - all_blocks = [] # node0's blocks - for j in range(2): - for i in range(288): - next_block = create_block(tips[j].sha256, create_coinbase(i + 4), tips[j].nTime+1) - next_block.solve() - if j==0: - test_node.send_message(msg_block(next_block)) - all_blocks.append(next_block) - else: - headers_message.headers.append(CBlockHeader(next_block)) - tips[j] = next_block - - self.bump_mocktime(2) - set_node_times(self.nodes, self.mocktime) # Blocks 1-287 should be accepted, block 288 should be ignored because it's too far ahead for x in all_blocks[:-1]: self.nodes[0].getblock(x.hash) - assert_raises_jsonrpc(-1, "Block not found on disk", self.nodes[0].getblock, all_blocks[-1].hash) - - headers_message.headers.pop() # Ensure the last block is unrequested - white_node.send_message(headers_message) # Send headers leading to tip - white_node.send_message(msg_block(tips[1])) # Now deliver the tip - white_node.sync_with_ping() - self.nodes[1].getblock(tips[1].hash) - self.log.info("Unrequested block far ahead of tip accepted from whitelisted peer") + assert_raises_rpc_error(-1, "Block not found on disk", self.nodes[0].getblock, all_blocks[-1].hash) # 5. Test handling of unrequested block on the node that didn't process # Should still not be processed (even though it has a child that has more # work). - test_node.send_message(msg_block(blocks_h2f[0])) - # Here, if the sleep is too short, the test could falsely succeed (if the - # node hasn't processed the block by the time the sleep returns, and then - # the node processes it and incorrectly advances the tip). - # But this would be caught later on, when we verify that an inv triggers - # a getdata request for this block. + # The node should have requested the blocks at some point, so + # disconnect/reconnect first + connections[0].disconnect_node() + test_node.wait_for_disconnect() + + test_node = NodeConnCB() # connects to node (not whitelisted) + connections[0] = NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], test_node) + test_node.add_connection(connections[0]) + + test_node.wait_for_verack() + test_node.send_message(msg_block(block_h1f)) + test_node.sync_with_ping() assert_equal(self.nodes[0].getblockcount(), 2) self.log.info("Unrequested block that would complete more-work chain was ignored") @@ -201,23 +229,100 @@ def run_test(self): with mininode_lock: # Clear state so we can check the getdata request test_node.last_message.pop("getdata", None) - test_node.send_message(msg_inv([CInv(2, blocks_h3[0].sha256)])) + test_node.send_message(msg_inv([CInv(2, block_h3.sha256)])) test_node.sync_with_ping() with mininode_lock: getdata = test_node.last_message["getdata"] # Check that the getdata includes the right block - assert_equal(getdata.inv[0].hash, blocks_h2f[0].sha256) + assert_equal(getdata.inv[0].hash, block_h1f.sha256) self.log.info("Inv at tip triggered getdata for unprocessed block") # 7. Send the missing block for the third time (now it is requested) - test_node.send_message(msg_block(blocks_h2f[0])) + test_node.send_message(msg_block(block_h1f)) test_node.sync_with_ping() assert_equal(self.nodes[0].getblockcount(), 290) + self.nodes[0].getblock(all_blocks[286].hash) + assert_equal(self.nodes[0].getbestblockhash(), all_blocks[286].hash) + assert_raises_rpc_error(-1, "Block not found on disk", self.nodes[0].getblock, all_blocks[287].hash) self.log.info("Successfully reorged to longer chain from non-whitelisted peer") + # 8. Create a chain which is invalid at a height longer than the + # current chain, but which has more blocks on top of that + block_289f = create_block(all_blocks[284].sha256, create_coinbase(289), all_blocks[284].nTime+1) + block_289f.solve() + block_290f = create_block(block_289f.sha256, create_coinbase(290), block_289f.nTime+1) + block_290f.solve() + block_291 = create_block(block_290f.sha256, create_coinbase(291), block_290f.nTime+1) + # block_291 spends a coinbase below maturity! + block_291.vtx.append(create_transaction(block_290f.vtx[0], 0, b"42", 1)) + block_291.hashMerkleRoot = block_291.calc_merkle_root() + block_291.solve() + block_292 = create_block(block_291.sha256, create_coinbase(292), block_291.nTime+1) + block_292.solve() + + # Now send all the headers on the chain and enough blocks to trigger reorg + headers_message = msg_headers() + headers_message.headers.append(CBlockHeader(block_289f)) + headers_message.headers.append(CBlockHeader(block_290f)) + headers_message.headers.append(CBlockHeader(block_291)) + headers_message.headers.append(CBlockHeader(block_292)) + test_node.send_message(headers_message) + + test_node.sync_with_ping() + tip_entry_found = False + for x in self.nodes[0].getchaintips(): + if x['hash'] == block_292.hash: + assert_equal(x['status'], "headers-only") + tip_entry_found = True + assert(tip_entry_found) + assert_raises_rpc_error(-1, "Block not found on disk", self.nodes[0].getblock, block_292.hash) + + test_node.send_message(msg_block(block_289f)) + test_node.send_message(msg_block(block_290f)) + + test_node.sync_with_ping() + self.nodes[0].getblock(block_289f.hash) + self.nodes[0].getblock(block_290f.hash) + + test_node.send_message(msg_block(block_291)) + + # At this point we've sent an obviously-bogus block, wait for full processing + # without assuming whether we will be disconnected or not + try: + # Only wait a short while so the test doesn't take forever if we do get + # disconnected + test_node.sync_with_ping(timeout=1) + except AssertionError: + test_node.wait_for_disconnect() + + test_node = NodeConnCB() # connects to node (not whitelisted) + connections[0] = NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], test_node) + test_node.add_connection(connections[0]) + + NetworkThread().start() # Start up network handling in another thread + test_node.wait_for_verack() + + # We should have failed reorg and switched back to 290 (but have block 291) + assert_equal(self.nodes[0].getblockcount(), 290) + assert_equal(self.nodes[0].getbestblockhash(), all_blocks[286].hash) + assert_equal(self.nodes[0].getblock(block_291.hash)["confirmations"], -1) + + # Now send a new header on the invalid chain, indicating we're forked off, and expect to get disconnected + block_293 = create_block(block_292.sha256, create_coinbase(293), block_292.nTime+1) + block_293.solve() + headers_message = msg_headers() + headers_message.headers.append(CBlockHeader(block_293)) + test_node.send_message(headers_message) + test_node.wait_for_disconnect() + + # 9. Connect node1 to node0 and ensure it is able to sync + connect_nodes(self.nodes[0], 1) + sync_blocks([self.nodes[0], self.nodes[1]]) + self.log.info("Successfully synced nodes 1 and 0") + [ c.disconnect_node() for c in connections ] if __name__ == '__main__': diff --git a/test/functional/p2p-instantsend.py b/test/functional/p2p-instantsend.py index c22533ece09d..6b83ee4a4277 100755 --- a/test/functional/p2p-instantsend.py +++ b/test/functional/p2p-instantsend.py @@ -6,7 +6,7 @@ from test_framework.mininode import * from test_framework.test_framework import DashTestFramework from test_framework.util import isolate_node, set_node_times, reconnect_isolated_node, assert_equal, \ - assert_raises_jsonrpc + assert_raises_rpc_error ''' InstantSendTest -- test InstantSend functionality (prevent doublespend for unconfirmed transactions) @@ -104,14 +104,14 @@ def test_mempool_doublespend(self): reconnect_isolated_node(isolated, 0) for node in self.nodes: if node is not isolated: - assert_raises_jsonrpc(-5, "No such mempool or blockchain transaction", node.getrawtransaction, dblspnd_txid) + assert_raises_rpc_error(-5, "No such mempool or blockchain transaction", node.getrawtransaction, dblspnd_txid) # instantsend to receiver. The previously isolated node should prune the doublespend TX and request the correct # TX from other nodes. receiver_addr = receiver.getnewaddress() is_id = sender.sendtoaddress(receiver_addr, 0.9) for node in self.nodes: self.wait_for_instantlock(is_id, node) - assert_raises_jsonrpc(-5, "No such mempool or blockchain transaction", isolated.getrawtransaction, dblspnd_txid) + assert_raises_rpc_error(-5, "No such mempool or blockchain transaction", isolated.getrawtransaction, dblspnd_txid) # send coins back to the controller node without waiting for confirmations receiver.sendtoaddress(self.nodes[0].getnewaddress(), 0.9, "", "", True) assert_equal(receiver.getwalletinfo()["balance"], 0) diff --git a/test/functional/prioritise_transaction.py b/test/functional/prioritise_transaction.py index e3975f169dc0..677372f5530e 100755 --- a/test/functional/prioritise_transaction.py +++ b/test/functional/prioritise_transaction.py @@ -101,7 +101,7 @@ def run_test(self): tx_id = self.nodes[0].decoderawtransaction(tx_hex)["txid"] # This will raise an exception due to min relay fee not being met - assert_raises_jsonrpc(-26, "66: min relay fee not met", self.nodes[0].sendrawtransaction, tx_hex) + assert_raises_rpc_error(-26, "66: min relay fee not met", self.nodes[0].sendrawtransaction, tx_hex) assert(tx_id not in self.nodes[0].getrawmempool()) # This is a less than 1000-byte transaction, so just set the fee diff --git a/test/functional/pruning.py b/test/functional/pruning.py index 8c37577936b7..cc2b7b5d1673 100755 --- a/test/functional/pruning.py +++ b/test/functional/pruning.py @@ -186,7 +186,7 @@ def reorg_test(self): def reorg_back(self): # Verify that a block on the old main chain fork has been pruned away - assert_raises_jsonrpc(-1, "Block not available (pruned data)", self.nodes[2].getblock, self.forkhash) + assert_raises_rpc_error(-1, "Block not available (pruned data)", self.nodes[2].getblock, self.forkhash) self.log.info("Will need to redownload block %d" % self.forkheight) # Verify that we have enough history to reorg back to the fork point @@ -233,7 +233,7 @@ def manual_test(self, node_number, use_timestamp): self.start_node(node_number, extra_args=["-litemode", "-txindex=0"]) node = self.nodes[node_number] assert_equal(node.getblockcount(), 995) - assert_raises_jsonrpc(-1, "not in prune mode", node.pruneblockchain, 500) + assert_raises_rpc_error(-1, "not in prune mode", node.pruneblockchain, 500) # now re-start in manual pruning mode self.stop_node(node_number) @@ -266,14 +266,14 @@ def has_block(index): return os.path.isfile(self.options.tmpdir + "/node{}/regtest/blocks/blk{:05}.dat".format(node_number, index)) # should not prune because chain tip of node 3 (995) < PruneAfterHeight (1000) - assert_raises_jsonrpc(-1, "Blockchain is too short for pruning", node.pruneblockchain, height(500)) + assert_raises_rpc_error(-1, "Blockchain is too short for pruning", node.pruneblockchain, height(500)) # mine 6 blocks so we are at height 1001 (i.e., above PruneAfterHeight) node.generate(6) assert_equal(node.getblockchaininfo()["blocks"], 1001) # negative heights should raise an exception - assert_raises_jsonrpc(-8, "Negative", node.pruneblockchain, -10) + assert_raises_rpc_error(-8, "Negative", node.pruneblockchain, -10) # height=100 too low to prune first block file so this is a no-op prune(100) diff --git a/test/functional/rawtransactions.py b/test/functional/rawtransactions.py index 09488eed5d7e..e3aa33aa3013 100755 --- a/test/functional/rawtransactions.py +++ b/test/functional/rawtransactions.py @@ -48,7 +48,7 @@ def run_test(self): rawtx = self.nodes[2].signrawtransaction(rawtx) # This will raise an exception since there are missing inputs - assert_raises_jsonrpc(-25, "Missing inputs", self.nodes[2].sendrawtransaction, rawtx['hex']) + assert_raises_rpc_error(-25, "Missing inputs", self.nodes[2].sendrawtransaction, rawtx['hex']) ######################### # RAW TX MULTISIG TESTS # @@ -190,13 +190,13 @@ def run_test(self): assert_equal(self.nodes[0].getrawtransaction(txHash, True)["hex"], rawTxSigned['hex']) # 6. invalid parameters - supply txid and string "Flase" - assert_raises_jsonrpc(-3,"Invalid type", self.nodes[0].getrawtransaction, txHash, "Flase") + assert_raises_rpc_error(-3,"Invalid type", self.nodes[0].getrawtransaction, txHash, "Flase") # 7. invalid parameters - supply txid and empty array - assert_raises_jsonrpc(-3,"Invalid type", self.nodes[0].getrawtransaction, txHash, []) + assert_raises_rpc_error(-3,"Invalid type", self.nodes[0].getrawtransaction, txHash, []) # 8. invalid parameters - supply txid and empty dict - assert_raises_jsonrpc(-3,"Invalid type", self.nodes[0].getrawtransaction, txHash, {}) + assert_raises_rpc_error(-3,"Invalid type", self.nodes[0].getrawtransaction, txHash, {}) inputs = [ {'txid' : "1d1d4e24ed99057e84c3f80fd8fbec79ed9e1acee37da269356ecea000000000", 'vout' : 1, 'sequence' : 1000}] outputs = { self.nodes[0].getnewaddress() : 1 } @@ -207,12 +207,12 @@ def run_test(self): # 9. invalid parameters - sequence number out of range inputs = [ {'txid' : "1d1d4e24ed99057e84c3f80fd8fbec79ed9e1acee37da269356ecea000000000", 'vout' : 1, 'sequence' : -1}] outputs = { self.nodes[0].getnewaddress() : 1 } - assert_raises_jsonrpc(-8, 'Invalid parameter, sequence number is out of range', self.nodes[0].createrawtransaction, inputs, outputs) + assert_raises_rpc_error(-8, 'Invalid parameter, sequence number is out of range', self.nodes[0].createrawtransaction, inputs, outputs) # 10. invalid parameters - sequence number out of range inputs = [ {'txid' : "1d1d4e24ed99057e84c3f80fd8fbec79ed9e1acee37da269356ecea000000000", 'vout' : 1, 'sequence' : 4294967296}] outputs = { self.nodes[0].getnewaddress() : 1 } - assert_raises_jsonrpc(-8, 'Invalid parameter, sequence number is out of range', self.nodes[0].createrawtransaction, inputs, outputs) + assert_raises_rpc_error(-8, 'Invalid parameter, sequence number is out of range', self.nodes[0].createrawtransaction, inputs, outputs) inputs = [ {'txid' : "1d1d4e24ed99057e84c3f80fd8fbec79ed9e1acee37da269356ecea000000000", 'vout' : 1, 'sequence' : 4294967294}] outputs = { self.nodes[0].getnewaddress() : 1 } diff --git a/test/functional/resendwallettransactions.py b/test/functional/resendwallettransactions.py index d6ba5913912d..d959bb4c3836 100755 --- a/test/functional/resendwallettransactions.py +++ b/test/functional/resendwallettransactions.py @@ -5,7 +5,7 @@ """Test resendwallettransactions RPC.""" from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_raises_jsonrpc +from test_framework.util import assert_equal, assert_raises_rpc_error class ResendWalletTransactionsTest(BitcoinTestFramework): def set_test_params(self): @@ -14,7 +14,7 @@ def set_test_params(self): def run_test(self): # Should raise RPC_WALLET_ERROR (-4) if walletbroadcast is disabled. - assert_raises_jsonrpc(-4, "Error: Wallet transaction broadcasting is disabled with -walletbroadcast", self.nodes[0].resendwallettransactions) + assert_raises_rpc_error(-4, "Error: Wallet transaction broadcasting is disabled with -walletbroadcast", self.nodes[0].resendwallettransactions) # Should return an empty array if there aren't unconfirmed wallet transactions. self.stop_node(0) diff --git a/test/functional/rpc_getblockstats.py b/test/functional/rpc_getblockstats.py index ff9c82baf396..bbbf7a4b3173 100755 --- a/test/functional/rpc_getblockstats.py +++ b/test/functional/rpc_getblockstats.py @@ -9,7 +9,7 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - assert_raises_jsonrpc, + assert_raises_rpc_error, ) import json import os @@ -148,9 +148,9 @@ def run_test(self): # Test invalid parameters raise the proper json exceptions tip = self.start_height + self.max_stat_pos - assert_raises_jsonrpc(-8, 'Target block height %d after current tip %d' % (tip+1, tip), + assert_raises_rpc_error(-8, 'Target block height %d after current tip %d' % (tip+1, tip), self.nodes[0].getblockstats, hash_or_height=tip+1) - assert_raises_jsonrpc(-8, 'Target block height %d is negative' % (-1), + assert_raises_rpc_error(-8, 'Target block height %d is negative' % (-1), self.nodes[0].getblockstats, hash_or_height=-1) # Make sure not valid stats aren't allowed @@ -162,18 +162,18 @@ def run_test(self): ['minfee', inv_sel_stat, 'maxfee'], ] for inv_stat in inv_stats: - assert_raises_jsonrpc(-8, 'Invalid selected statistic %s' % inv_sel_stat, + assert_raises_rpc_error(-8, 'Invalid selected statistic %s' % inv_sel_stat, self.nodes[0].getblockstats, hash_or_height=1, stats=inv_stat) # Make sure we aren't always returning inv_sel_stat as the culprit stat - assert_raises_jsonrpc(-8, 'Invalid selected statistic aaa%s' % inv_sel_stat, + assert_raises_rpc_error(-8, 'Invalid selected statistic aaa%s' % inv_sel_stat, self.nodes[0].getblockstats, hash_or_height=1, stats=['minfee' , 'aaa%s' % inv_sel_stat]) - assert_raises_jsonrpc(-8, 'One or more of the selected stats requires -txindex enabled', + assert_raises_rpc_error(-8, 'One or more of the selected stats requires -txindex enabled', self.nodes[1].getblockstats, hash_or_height=self.start_height + self.max_stat_pos) # Mainchain's genesis block shouldn't be found on regtest - assert_raises_jsonrpc(-5, 'Block not found', self.nodes[0].getblockstats, + assert_raises_rpc_error(-5, 'Block not found', self.nodes[0].getblockstats, hash_or_height='000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f') if __name__ == '__main__': diff --git a/test/functional/rpcbind_test.py b/test/functional/rpcbind_test.py index 0cf64beebd40..0e8c3fa209ee 100755 --- a/test/functional/rpcbind_test.py +++ b/test/functional/rpcbind_test.py @@ -101,7 +101,7 @@ def run_test(self): # Check that with invalid rpcallowip, we are denied self.run_allowip_test([non_loopback_ip], non_loopback_ip, defaultport) - assert_raises_jsonrpc(-342, "non-JSON HTTP response with '403 Forbidden' from server", self.run_allowip_test, ['1.1.1.1'], non_loopback_ip, defaultport) + assert_raises_rpc_error(-342, "non-JSON HTTP response with '403 Forbidden' from server", self.run_allowip_test, ['1.1.1.1'], non_loopback_ip, defaultport) if __name__ == '__main__': RPCBindTest().main() diff --git a/test/functional/rpcnamedargs.py b/test/functional/rpcnamedargs.py index da61cc66e685..0fb0a62afd02 100755 --- a/test/functional/rpcnamedargs.py +++ b/test/functional/rpcnamedargs.py @@ -7,7 +7,7 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - assert_raises_jsonrpc, + assert_raises_rpc_error, ) class NamedArgumentTest(BitcoinTestFramework): @@ -19,7 +19,7 @@ def run_test(self): h = node.help(command='getinfo') assert(h.startswith('getinfo\n')) - assert_raises_jsonrpc(-8, 'Unknown named parameter', node.help, random='getinfo') + assert_raises_rpc_error(-8, 'Unknown named parameter', node.help, random='getinfo') h = node.getblockhash(height=0) node.getblock(blockhash=h) diff --git a/test/functional/signrawtransactions.py b/test/functional/signrawtransactions.py index 17d9ab793429..96a3b6382198 100755 --- a/test/functional/signrawtransactions.py +++ b/test/functional/signrawtransactions.py @@ -82,7 +82,7 @@ def script_verification_error_test(self): assert_equal(decodedRawTx["vin"][i]["vout"], inp["vout"]) # Make sure decoderawtransaction throws if there is extra data - assert_raises(JSONRPCException, self.nodes[0].decoderawtransaction, rawTx + "00") + assert_raises_rpc_error(-22, "TX decode failed", self.nodes[0].decoderawtransaction, rawTx + "00") rawTxSigned = self.nodes[0].signrawtransaction(rawTx, scripts, privKeys) diff --git a/test/functional/test_framework/mininode.py b/test/functional/test_framework/mininode.py index 8d9f2fdfe477..ec3334f04093 100755 --- a/test/functional/test_framework/mininode.py +++ b/test/functional/test_framework/mininode.py @@ -1656,7 +1656,6 @@ def sync_with_ping(self, timeout=60): test_function = lambda: self.last_message.get("pong") and self.last_message["pong"].nonce == self.ping_counter wait_until(test_function, timeout=timeout, lock=mininode_lock) self.ping_counter += 1 - return True # The actual NodeConn class # This class provides an interface for a p2p connection to a specified node diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 457ff763e66a..c59b6277d223 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -110,8 +110,11 @@ def main(self): check_json_precision() + self.options.cachedir = os.path.abspath(self.options.cachedir) + # Set up temp directory and start logging if self.options.tmpdir: + self.options.tmpdir = os.path.abspath(self.options.tmpdir) os.makedirs(self.options.tmpdir, exist_ok=False) else: self.options.tmpdir = tempfile.mkdtemp(prefix="test") diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index 104973e14f80..f724348d556c 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -53,6 +53,8 @@ def assert_raises(exc, fun, *args, **kwds): def assert_raises_message(exc, message, fun, *args, **kwds): try: fun(*args, **kwds) + except JSONRPCException: + raise AssertionError("Use assert_raises_rpc_error() to test RPC failures") except exc as e: if message is not None and message not in e.error['message']: raise AssertionError("Expected substring not found:" + e.error['message']) @@ -85,7 +87,7 @@ def assert_raises_process_error(returncode, output, fun, *args, **kwds): else: raise AssertionError("No exception raised") -def assert_raises_jsonrpc(code, message, fun, *args, **kwds): +def assert_raises_rpc_error(code, message, fun, *args, **kwds): """Run an RPC and verify that a specific JSONRPC exception code and message is raised. Calls function `fun` with arguments `args` and `kwds`. Catches a JSONRPCException @@ -101,6 +103,13 @@ def assert_raises_jsonrpc(code, message, fun, *args, **kwds): args*: positional arguments for the function. kwds**: named arguments for the function. """ + assert try_rpc(code, message, fun, *args, **kwds), "No exception raised" + +def try_rpc(code, message, fun, *args, **kwds): + """Tries to run an rpc command. + + Test against error code and message if the rpc fails. + Returns whether a JSONRPCException was raised.""" try: fun(*args, **kwds) except JSONRPCException as e: @@ -109,10 +118,11 @@ def assert_raises_jsonrpc(code, message, fun, *args, **kwds): raise AssertionError("Unexpected JSONRPC error code %i" % e.error["code"]) if (message is not None) and (message not in e.error['message']): raise AssertionError("Expected substring not found:" + e.error['message']) + return True except Exception as e: raise AssertionError("Unexpected exception raised: " + type(e).__name__) else: - raise AssertionError("No exception raised") + return False def assert_is_hex_string(string): try: diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index f0d34f4051f3..441d7d1d7dff 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -135,6 +135,8 @@ 'bip65-cltv-p2p.py', 'uptime.py', 'resendwallettransactions.py', + 'minchainwork.py', + 'p2p-acceptblock.py', # NOTE: needs dash_hash to pass ] EXTENDED_SCRIPTS = [ @@ -162,7 +164,6 @@ 'txindex.py', 'forknotify.py', 'invalidateblock.py', - 'p2p-acceptblock.py', # NOTE: needs dash_hash to pass ] # Place EXTENDED_SCRIPTS first since it has the 3 longest running tests diff --git a/test/functional/wallet-dump.py b/test/functional/wallet-dump.py index 0407bcca38a8..cc0d8649fb68 100755 --- a/test/functional/wallet-dump.py +++ b/test/functional/wallet-dump.py @@ -7,7 +7,7 @@ import sys from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import (assert_equal, assert_raises_jsonrpc) +from test_framework.util import (assert_equal, assert_raises_rpc_error) def read_dump(file_name, addrs, hd_master_addr_old): @@ -113,7 +113,7 @@ def run_test (self): assert_equal(found_addr_rsv, 180) # keypool size # Overwriting should fail - assert_raises_jsonrpc(-8, "already exists", self.nodes[0].dumpwallet, tmpdir + "/node0/wallet.unencrypted.dump") + assert_raises_rpc_error(-8, "already exists", self.nodes[0].dumpwallet, tmpdir + "/node0/wallet.unencrypted.dump") if __name__ == '__main__': WalletDumpTest().main () diff --git a/test/functional/wallet-encryption.py b/test/functional/wallet-encryption.py index ce1e7744e903..db62e1e30f48 100755 --- a/test/functional/wallet-encryption.py +++ b/test/functional/wallet-encryption.py @@ -9,7 +9,7 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - assert_raises_jsonrpc, + assert_raises_rpc_error, ) class WalletEncryptionTest(BitcoinTestFramework): @@ -32,7 +32,7 @@ def run_test(self): self.start_node(0) # Test that the wallet is encrypted - assert_raises_jsonrpc(-13, "Please enter the wallet passphrase with walletpassphrase first", self.nodes[0].dumpprivkey, address) + assert_raises_rpc_error(-13, "Please enter the wallet passphrase with walletpassphrase first", self.nodes[0].dumpprivkey, address) # Check that walletpassphrase works self.nodes[0].walletpassphrase(passphrase, 2) @@ -40,20 +40,20 @@ def run_test(self): # Check that the timeout is right time.sleep(2) - assert_raises_jsonrpc(-13, "Please enter the wallet passphrase with walletpassphrase first", self.nodes[0].dumpprivkey, address) + assert_raises_rpc_error(-13, "Please enter the wallet passphrase with walletpassphrase first", self.nodes[0].dumpprivkey, address) # Test wrong passphrase - assert_raises_jsonrpc(-14, "wallet passphrase entered was incorrect", self.nodes[0].walletpassphrase, passphrase + "wrong", 10) + assert_raises_rpc_error(-14, "wallet passphrase entered was incorrect", self.nodes[0].walletpassphrase, passphrase + "wrong", 10) # Test walletlock self.nodes[0].walletpassphrase(passphrase, 84600) assert_equal(privkey, self.nodes[0].dumpprivkey(address)) self.nodes[0].walletlock() - assert_raises_jsonrpc(-13, "Please enter the wallet passphrase with walletpassphrase first", self.nodes[0].dumpprivkey, address) + assert_raises_rpc_error(-13, "Please enter the wallet passphrase with walletpassphrase first", self.nodes[0].dumpprivkey, address) # Test passphrase changes self.nodes[0].walletpassphrasechange(passphrase, passphrase2) - assert_raises_jsonrpc(-14, "wallet passphrase entered was incorrect", self.nodes[0].walletpassphrase, passphrase, 10) + assert_raises_rpc_error(-14, "wallet passphrase entered was incorrect", self.nodes[0].walletpassphrase, passphrase, 10) self.nodes[0].walletpassphrase(passphrase2, 10) assert_equal(privkey, self.nodes[0].dumpprivkey(address)) diff --git a/test/functional/wallet.py b/test/functional/wallet.py index 4e9737af4b1a..d4d7aad7f3af 100755 --- a/test/functional/wallet.py +++ b/test/functional/wallet.py @@ -101,7 +101,7 @@ def run_test(self): unspent_0 = self.nodes[2].listunspent()[0] unspent_0 = {"txid": unspent_0["txid"], "vout": unspent_0["vout"]} self.nodes[2].lockunspent(False, [unspent_0]) - assert_raises_jsonrpc(-4, "Insufficient funds", self.nodes[2].sendtoaddress, self.nodes[2].getnewaddress(), 200) + assert_raises_rpc_error(-4, "Insufficient funds", self.nodes[2].sendtoaddress, self.nodes[2].getnewaddress(), 200) assert_equal([unspent_0], self.nodes[2].listlockunspent()) self.nodes[2].lockunspent(True, [unspent_0]) assert_equal(len(self.nodes[2].listlockunspent()), 0) @@ -289,10 +289,10 @@ def run_test(self): assert_equal(txObj['amount'], Decimal('-0.0001')) # This will raise an exception because the amount type is wrong - assert_raises_jsonrpc(-3, "Invalid amount", self.nodes[0].sendtoaddress, self.nodes[2].getnewaddress(), "1f-4") + assert_raises_rpc_error(-3, "Invalid amount", self.nodes[0].sendtoaddress, self.nodes[2].getnewaddress(), "1f-4") # This will raise an exception since generate does not accept a string - assert_raises_jsonrpc(-1, "not an integer", self.nodes[0].generate, "2") + assert_raises_rpc_error(-1, "not an integer", self.nodes[0].generate, "2") # Import address and private key to check correct behavior of spendable unspents # 1. Send some coins to generate new UTXO @@ -425,7 +425,7 @@ def run_test(self): node0_balance = self.nodes[0].getbalance() # With walletrejectlongchains we will not create the tx and store it in our wallet. - assert_raises_jsonrpc(-4, "Transaction has too long of a mempool chain", self.nodes[0].sendtoaddress, sending_addr, node0_balance - Decimal('0.01')) + assert_raises_rpc_error(-4, "Transaction has too long of a mempool chain", self.nodes[0].sendtoaddress, sending_addr, node0_balance - Decimal('0.01')) # Verify nothing new in wallet assert_equal(total_txs, len(self.nodes[0].listtransactions("*",99999))) diff --git a/test/functional/walletbackup.py b/test/functional/walletbackup.py index 3c5ec69b3ada..430fb47c5a6d 100755 --- a/test/functional/walletbackup.py +++ b/test/functional/walletbackup.py @@ -192,6 +192,16 @@ def run_test(self): assert_equal(self.nodes[1].getbalance(), balance1) assert_equal(self.nodes[2].getbalance(), balance2) + # Backup to source wallet file must fail + sourcePaths = [ + tmpdir + "/node0/regtest/wallet.dat", + tmpdir + "/node0/./regtest/wallet.dat", + tmpdir + "/node0/regtest/", + tmpdir + "/node0/regtest"] + + for sourcePath in sourcePaths: + assert_raises_rpc_error(-4, "backup failed", self.nodes[0].backupwallet, sourcePath) + if __name__ == '__main__': WalletBackupTest().main() diff --git a/test/functional/zapwallettxes.py b/test/functional/zapwallettxes.py index 1233c82fc571..666925ac39c0 100755 --- a/test/functional/zapwallettxes.py +++ b/test/functional/zapwallettxes.py @@ -17,7 +17,7 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - assert_raises_jsonrpc, + assert_raises_rpc_error, ) from test_framework.mininode import wait_until @@ -72,7 +72,7 @@ def run_test(self): assert_equal(self.nodes[0].gettransaction(txid1)['txid'], txid1) # This will raise an exception because the unconfirmed transaction has been zapped - assert_raises_jsonrpc(-5, 'Invalid or non-wallet transaction id', self.nodes[0].gettransaction, txid2) + assert_raises_rpc_error(-5, 'Invalid or non-wallet transaction id', self.nodes[0].gettransaction, txid2) if __name__ == '__main__': ZapWalletTXesTest().main()