From 6c9551880703d5d18bc0547ec86b7e02894b80b3 Mon Sep 17 00:00:00 2001 From: Alexander Block Date: Tue, 17 Mar 2020 12:18:39 +0100 Subject: [PATCH 01/13] Bump proto version --- src/version.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/version.h b/src/version.h index 512c40260ed3..2a06f25f3bbd 100644 --- a/src/version.h +++ b/src/version.h @@ -11,7 +11,7 @@ */ -static const int PROTOCOL_VERSION = 70216; +static const int PROTOCOL_VERSION = 70217; //! initial proto version, to be increased after version/verack negotiation static const int INIT_PROTO_VERSION = 209; From 486463d6221fa8705194e950b32a51a2e4147921 Mon Sep 17 00:00:00 2001 From: Alexander Block Date: Mon, 16 Mar 2020 11:06:38 +0100 Subject: [PATCH 02/13] Add SPORK_21_QUORUM_ALL_CONNECTED --- src/spork.cpp | 1 + src/spork.h | 1 + 2 files changed, 2 insertions(+) diff --git a/src/spork.cpp b/src/spork.cpp index f64a941a999d..acbb1d5fb821 100644 --- a/src/spork.cpp +++ b/src/spork.cpp @@ -26,6 +26,7 @@ std::vector sporkDefs = { MAKE_SPORK_DEF(SPORK_17_QUORUM_DKG_ENABLED, 4070908800ULL), // OFF MAKE_SPORK_DEF(SPORK_19_CHAINLOCKS_ENABLED, 4070908800ULL), // OFF MAKE_SPORK_DEF(SPORK_20_INSTANTSEND_LLMQ_BASED, 4070908800ULL), // OFF + MAKE_SPORK_DEF(SPORK_21_QUORUM_ALL_CONNECTED, 4070908800ULL), // OFF }; CSporkManager sporkManager; diff --git a/src/spork.h b/src/spork.h index 233eee734c32..ed2118e89f0d 100644 --- a/src/spork.h +++ b/src/spork.h @@ -30,6 +30,7 @@ enum SporkId : int32_t { SPORK_17_QUORUM_DKG_ENABLED = 10016, SPORK_19_CHAINLOCKS_ENABLED = 10018, SPORK_20_INSTANTSEND_LLMQ_BASED = 10019, + SPORK_21_QUORUM_ALL_CONNECTED = 10020, SPORK_INVALID = -1, }; From 14bb62ac8ee361446c9e2007fcf102f7d02371ad Mon Sep 17 00:00:00 2001 From: Alexander Block Date: Mon, 16 Mar 2020 11:09:54 +0100 Subject: [PATCH 03/13] Connect all LLMQ members to all other members --- src/llmq/quorums_utils.cpp | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/llmq/quorums_utils.cpp b/src/llmq/quorums_utils.cpp index a2ca3ba3b7fc..2fe37726f6a8 100644 --- a/src/llmq/quorums_utils.cpp +++ b/src/llmq/quorums_utils.cpp @@ -7,6 +7,7 @@ #include #include +#include #include namespace llmq @@ -47,6 +48,22 @@ std::set CLLMQUtils::GetQuorumConnections(Consensus::LLMQType llmqType, auto mns = GetAllQuorumMembers(llmqType, pindexQuorum); std::set result; + + if (sporkManager.IsSporkActive(SPORK_21_QUORUM_ALL_CONNECTED)) { + for (auto& dmn : mns) { + // this will cause deterministic behaviour between incoming and outgoing connections. + // Each member needs a connection to all other members, so we have each member paired. The below check + // will be true on one side and false on the other side of the pairing, so we avoid having both members + // initiating the connection. + if (dmn->proTxHash < forMember) { + result.emplace(dmn->proTxHash); + } + } + return result; + } + + // TODO remove this after activation of SPORK_21_QUORUM_ALL_CONNECTED + for (size_t i = 0; i < mns.size(); i++) { auto& dmn = mns[i]; if (dmn->proTxHash == forMember) { From 8f644d18d711e60f4cdc33a142f82958a1f3aaaa Mon Sep 17 00:00:00 2001 From: Alexander Block Date: Tue, 17 Mar 2020 10:04:31 +0100 Subject: [PATCH 04/13] Implement probing of public ip/port of LLMQ members --- src/evo/mnauth.cpp | 11 +++++ src/llmq/quorums_dkgsessionhandler.cpp | 4 ++ src/llmq/quorums_utils.cpp | 38 +++++++++++++++ src/llmq/quorums_utils.h | 1 + src/masternode/masternode-meta.h | 7 ++- src/masternode/masternode-utils.cpp | 2 +- src/net.cpp | 67 ++++++++++++++++++++++---- src/net.h | 8 ++- src/net_processing.cpp | 8 ++- 9 files changed, 132 insertions(+), 14 deletions(-) diff --git a/src/evo/mnauth.cpp b/src/evo/mnauth.cpp index 06a40b09bb2b..47fe04b1bb1b 100644 --- a/src/evo/mnauth.cpp +++ b/src/evo/mnauth.cpp @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -112,6 +113,16 @@ void CMNAuth::ProcessMessage(CNode* pnode, const std::string& strCommand, CDataS return; } + if (!pnode->fInbound) { + mmetaman.GetMetaInfo(mnauth.proRegTxHash)->SetLastOutboundSuccess(GetAdjustedTime()); + if (pnode->fMasternodeProbe) { + LogPrint(BCLog::NET, "CMNAuth::ProcessMessage -- masternode probe successful for %s, disconnecting. peer=%d\n", + mnauth.proRegTxHash.ToString(), pnode->GetId()); + pnode->fDisconnect = true; + return; + } + } + connman.ForEachNode([&](CNode* pnode2) { if (pnode2->verifiedProRegTxHash == mnauth.proRegTxHash) { LogPrint(BCLog::NET, "CMNAuth::ProcessMessage -- Masternode %s has already verified as peer %d, dropping new connection. peer=%d\n", diff --git a/src/llmq/quorums_dkgsessionhandler.cpp b/src/llmq/quorums_dkgsessionhandler.cpp index c1f89dea0992..45453f12ccfc 100644 --- a/src/llmq/quorums_dkgsessionhandler.cpp +++ b/src/llmq/quorums_dkgsessionhandler.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include namespace llmq @@ -528,6 +529,9 @@ void CDKGSessionHandler::HandleDKGRound() }); CLLMQUtils::EnsureQuorumConnections(params.type, pindexQuorum, curSession->myProTxHash, gArgs.GetBoolArg("-watchquorums", DEFAULT_WATCH_QUORUMS)); + if (curSession->AreWeMember() && sporkManager.IsSporkActive(SPORK_21_QUORUM_ALL_CONNECTED)) { + CLLMQUtils::AddQuorumProbeConnections(params.type, pindexQuorum, curSession->myProTxHash); + } WaitForNextPhase(QuorumPhase_Initialized, QuorumPhase_Contribute, curQuorumHash, []{return false;}); diff --git a/src/llmq/quorums_utils.cpp b/src/llmq/quorums_utils.cpp index 2fe37726f6a8..7a84c9072b01 100644 --- a/src/llmq/quorums_utils.cpp +++ b/src/llmq/quorums_utils.cpp @@ -10,6 +10,8 @@ #include #include +#include + namespace llmq { @@ -148,6 +150,42 @@ void CLLMQUtils::EnsureQuorumConnections(Consensus::LLMQType llmqType, const CBl } } +void CLLMQUtils::AddQuorumProbeConnections(Consensus::LLMQType llmqType, const CBlockIndex *pindexQuorum, const uint256 &myProTxHash) +{ + auto members = GetAllQuorumMembers(llmqType, pindexQuorum); + auto curTime = GetAdjustedTime(); + + std::set probeConnections; + for (auto& dmn : members) { + if (dmn->proTxHash == myProTxHash) { + continue; + } + auto lastOutbound = mmetaman.GetMetaInfo(dmn->proTxHash)->GetLastOutboundSuccess(); + // re-probe after 50 minutes so that the "good connection" check in the DKG doesn't fail just because we're on + // the brink of timeout + if (curTime - lastOutbound > 50 * 60) { + probeConnections.emplace(dmn->proTxHash); + } + } + + if (!probeConnections.empty()) { + if (LogAcceptCategory(BCLog::LLMQ)) { + auto mnList = deterministicMNManager->GetListAtChainTip(); + std::string debugMsg = strprintf("CLLMQUtils::%s -- adding masternodes probes for quorum %s:\n", __func__, pindexQuorum->GetBlockHash().ToString()); + for (auto& c : probeConnections) { + auto dmn = mnList.GetValidMN(c); + if (!dmn) { + debugMsg += strprintf(" %s (not in valid MN set anymore)\n", c.ToString()); + } else { + debugMsg += strprintf(" %s (%s)\n", c.ToString(), dmn->pdmnState->addr.ToString(false)); + } + } + LogPrint(BCLog::LLMQ, debugMsg.c_str()); + } + g_connman->AddPendingProbeConnections(probeConnections); + } +} + bool CLLMQUtils::IsQuorumActive(Consensus::LLMQType llmqType, const uint256& quorumHash) { auto& params = Params().GetConsensus().llmqs.at(llmqType); diff --git a/src/llmq/quorums_utils.h b/src/llmq/quorums_utils.h index 4a047f0c0f30..ea84671ad3d2 100644 --- a/src/llmq/quorums_utils.h +++ b/src/llmq/quorums_utils.h @@ -35,6 +35,7 @@ class CLLMQUtils static std::set CalcDeterministicWatchConnections(Consensus::LLMQType llmqType, const CBlockIndex* pindexQuorum, size_t memberCount, size_t connectionCount); static void EnsureQuorumConnections(Consensus::LLMQType llmqType, const CBlockIndex* pindexQuorum, const uint256& myProTxHash, bool allowWatch); + static void AddQuorumProbeConnections(Consensus::LLMQType llmqType, const CBlockIndex* pindexQuorum, const uint256& myProTxHash); static bool IsQuorumActive(Consensus::LLMQType llmqType, const uint256& quorumHash); diff --git a/src/masternode/masternode-meta.h b/src/masternode/masternode-meta.h index 07cc0d0a8e9c..f9aa24bae3ae 100644 --- a/src/masternode/masternode-meta.h +++ b/src/masternode/masternode-meta.h @@ -34,6 +34,7 @@ class CMasternodeMetaInfo std::map mapGovernanceObjectsVotedOn; int64_t lastOutboundAttempt = 0; + int64_t lastOutboundSuccess = 0; public: CMasternodeMetaInfo() {} @@ -43,7 +44,8 @@ class CMasternodeMetaInfo nLastDsq(ref.nLastDsq), nMixingTxCount(ref.nMixingTxCount), mapGovernanceObjectsVotedOn(ref.mapGovernanceObjectsVotedOn), - lastOutboundAttempt(ref.lastOutboundAttempt) + lastOutboundAttempt(ref.lastOutboundAttempt), + lastOutboundSuccess(ref.lastOutboundSuccess) { } @@ -57,6 +59,7 @@ class CMasternodeMetaInfo READWRITE(nMixingTxCount); READWRITE(mapGovernanceObjectsVotedOn); READWRITE(lastOutboundAttempt); + READWRITE(lastOutboundSuccess); } public: @@ -73,6 +76,8 @@ class CMasternodeMetaInfo void SetLastOutboundAttempt(int64_t t) { LOCK(cs); lastOutboundAttempt = t; } int64_t GetLastOutboundAttempt() const { LOCK(cs); return lastOutboundAttempt; } + void SetLastOutboundSuccess(int64_t t) { LOCK(cs); lastOutboundSuccess = t; } + int64_t GetLastOutboundSuccess() const { LOCK(cs); return lastOutboundSuccess; } }; typedef std::shared_ptr CMasternodeMetaInfoPtr; diff --git a/src/masternode/masternode-utils.cpp b/src/masternode/masternode-utils.cpp index fdff535ae7c0..42221556dfcd 100644 --- a/src/masternode/masternode-utils.cpp +++ b/src/masternode/masternode-utils.cpp @@ -30,7 +30,7 @@ void CMasternodeUtils::ProcessMasternodeConnections(CConnman& connman) // Don't disconnect masternode connections when we have less then the desired amount of outbound nodes int nonMasternodeCount = 0; connman.ForEachNode(CConnman::AllNodes, [&](CNode* pnode) { - if (!pnode->fInbound && !pnode->fFeeler && !pnode->m_manual_connection && !pnode->fMasternode) { + if (!pnode->fInbound && !pnode->fFeeler && !pnode->m_manual_connection && !pnode->fMasternode && !pnode->fMasternodeProbe) { nonMasternodeCount++; } }); diff --git a/src/net.cpp b/src/net.cpp index 2505bbd49c58..729ec1780d50 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -1714,7 +1714,7 @@ void CConnman::ThreadDNSAddressSeed() LOCK(cs_vNodes); int nRelevant = 0; for (auto pnode : vNodes) { - nRelevant += pnode->fSuccessfullyConnected && !pnode->fFeeler && !pnode->fOneShot && !pnode->m_manual_connection && !pnode->fInbound; + nRelevant += pnode->fSuccessfullyConnected && !pnode->fFeeler && !pnode->fOneShot && !pnode->m_manual_connection && !pnode->fInbound && !pnode->fMasternodeProbe; } if (nRelevant >= 2) { LogPrintf("P2P peers available. Skipped DNS seeding.\n"); @@ -1837,7 +1837,7 @@ int CConnman::GetExtraOutboundCount() if (pnode->fMasternode) { continue; } - if (!pnode->fInbound && !pnode->m_manual_connection && !pnode->fFeeler && !pnode->fDisconnect && !pnode->fOneShot && pnode->fSuccessfullyConnected) { + if (!pnode->fInbound && !pnode->m_manual_connection && !pnode->fFeeler && !pnode->fDisconnect && !pnode->fOneShot && pnode->fSuccessfullyConnected && !pnode->fMasternodeProbe) { ++nOutbound; } } @@ -2116,11 +2116,11 @@ void CConnman::ThreadOpenMasternodeConnections() didConnect = false; std::set connectedNodes; - std::set connectedProRegTxHashes; + std::map connectedProRegTxHashes; ForEachNode([&](const CNode* pnode) { connectedNodes.emplace(pnode->addr); if (!pnode->verifiedProRegTxHash.IsNull()) { - connectedProRegTxHashes.emplace(pnode->verifiedProRegTxHash); + connectedProRegTxHashes.emplace(pnode->verifiedProRegTxHash, pnode->fInbound); } }); @@ -2135,6 +2135,7 @@ void CConnman::ThreadOpenMasternodeConnections() // NOTE: Process only one pending masternode at a time CDeterministicMNCPtr connectToDmn; + bool isProbe = false; { // don't hold lock while calling OpenMasternodeConnection as cs_main is locked deep inside LOCK2(cs_vNodes, cs_vPendingMasternodes); @@ -2172,6 +2173,41 @@ void CConnman::ThreadOpenMasternodeConnections() LogPrint(BCLog::NET, "CConnman::%s -- opening quorum connection to %s, service=%s\n", __func__, connectToDmn->proTxHash.ToString(), connectToDmn->pdmnState->addr.ToString(false)); } } + + if (!connectToDmn) { + std::vector pending; + for (auto it = masternodePendingProbes.begin(); it != masternodePendingProbes.end(); ) { + auto dmn = mnList.GetMN(*it); + if (!dmn) { + it = masternodePendingProbes.erase(it); + continue; + } + bool connectedAndOutbound = connectedProRegTxHashes.count(dmn->proTxHash) && !connectedProRegTxHashes[dmn->proTxHash]; + if (connectedAndOutbound) { + // we already have an outbound connection to this MN so there is no theed to probe it again + mmetaman.GetMetaInfo(dmn->proTxHash)->SetLastOutboundSuccess(nANow); + it = masternodePendingProbes.erase(it); + continue; + } + + ++it; + + int64_t lastAttempt = mmetaman.GetMetaInfo(dmn->proTxHash)->GetLastOutboundAttempt(); + // back off trying connecting to an address if we already tried recently + if (nANow - lastAttempt < 60) { + continue; + } + pending.emplace_back(dmn); + } + + if (!pending.empty()) { + connectToDmn = pending[GetRandInt(pending.size())]; + masternodePendingProbes.erase(connectToDmn->proTxHash); + isProbe = true; + + LogPrint(BCLog::NET, "CConnman::%s -- probing masternode %s, service=%s\n", __func__, connectToDmn->proTxHash.ToString(), connectToDmn->pdmnState->addr.ToString(false)); + } + } } if (!connectToDmn) { @@ -2182,20 +2218,24 @@ void CConnman::ThreadOpenMasternodeConnections() mmetaman.GetMetaInfo(connectToDmn->proTxHash)->SetLastOutboundAttempt(nANow); - OpenMasternodeConnection(CAddress(connectToDmn->pdmnState->addr, NODE_NETWORK)); + OpenMasternodeConnection(CAddress(connectToDmn->pdmnState->addr, NODE_NETWORK), isProbe); // should be in the list now if connection was opened - ForNode(connectToDmn->pdmnState->addr, CConnman::AllNodes, [&](CNode* pnode) { + bool connected = ForNode(connectToDmn->pdmnState->addr, CConnman::AllNodes, [&](CNode* pnode) { if (pnode->fDisconnect) { return false; } grant.MoveTo(pnode->grantMasternodeOutbound); return true; }); + if (!connected) { + // reset last outbound success + mmetaman.GetMetaInfo(connectToDmn->proTxHash)->SetLastOutboundSuccess(0); + } } } // if successful, this moves the passed grant to the constructed node -void CConnman::OpenNetworkConnection(const CAddress& addrConnect, bool fCountFailure, CSemaphoreGrant *grantOutbound, const char *pszDest, bool fOneShot, bool fFeeler, bool manual_connection, bool fConnectToMasternode) +void CConnman::OpenNetworkConnection(const CAddress& addrConnect, bool fCountFailure, CSemaphoreGrant *grantOutbound, const char *pszDest, bool fOneShot, bool fFeeler, bool manual_connection, bool fConnectToMasternode, bool fMasternodeProbe) { // // Initiate outbound network connection @@ -2235,6 +2275,8 @@ void CConnman::OpenNetworkConnection(const CAddress& addrConnect, bool fCountFai pnode->m_manual_connection = true; if (fConnectToMasternode) pnode->fMasternode = true; + if (fMasternodeProbe) + pnode->fMasternodeProbe = true; m_msgproc->InitializeNode(pnode); { @@ -2243,8 +2285,8 @@ void CConnman::OpenNetworkConnection(const CAddress& addrConnect, bool fCountFai } } -void CConnman::OpenMasternodeConnection(const CAddress &addrConnect) { - OpenNetworkConnection(addrConnect, false, nullptr, nullptr, false, false, false, true); +void CConnman::OpenMasternodeConnection(const CAddress &addrConnect, bool probe) { + OpenNetworkConnection(addrConnect, false, nullptr, nullptr, false, false, false, true, probe); } void CConnman::ThreadMessageHandler() @@ -2893,6 +2935,12 @@ bool CConnman::IsMasternodeQuorumNode(const CNode* pnode) return false; } +void CConnman::AddPendingProbeConnections(const std::set &proTxHashes) +{ + LOCK(cs_vPendingMasternodes); + masternodePendingProbes.insert(proTxHashes.begin(), proTxHashes.end()); +} + size_t CConnman::GetNodeCount(NumConnections flags) { LOCK(cs_vNodes); @@ -3194,6 +3242,7 @@ CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn nPingUsecTime = 0; fPingQueued = false; fMasternode = false; + fMasternodeProbe = false; nMinPingUsecTime = std::numeric_limits::max(); fPauseRecv = false; fPauseSend = false; diff --git a/src/net.h b/src/net.h index 461e83619b35..8e8f9edb9aae 100644 --- a/src/net.h +++ b/src/net.h @@ -204,8 +204,8 @@ class CConnman void Interrupt(); bool GetNetworkActive() const { return fNetworkActive; }; void SetNetworkActive(bool active); - void OpenNetworkConnection(const CAddress& addrConnect, bool fCountFailure, CSemaphoreGrant *grantOutbound = nullptr, const char *strDest = nullptr, bool fOneShot = false, bool fFeeler = false, bool manual_connection = false, bool fConnectToMasternode = false); - void OpenMasternodeConnection(const CAddress& addrConnect); + void OpenNetworkConnection(const CAddress& addrConnect, bool fCountFailure, CSemaphoreGrant *grantOutbound = nullptr, const char *strDest = nullptr, bool fOneShot = false, bool fFeeler = false, bool manual_connection = false, bool fConnectToMasternode = false, bool fMasternodeProbe = false); + void OpenMasternodeConnection(const CAddress& addrConnect, bool probe = false); bool CheckIncomingNonce(uint64_t nonce); struct CFullyConnectedOnly { @@ -415,6 +415,7 @@ class CConnman std::set GetMasternodeQuorumNodes(Consensus::LLMQType llmqType, const uint256& quorumHash) const; void RemoveMasternodeQuorumNodes(Consensus::LLMQType llmqType, const uint256& quorumHash); bool IsMasternodeQuorumNode(const CNode* pnode); + void AddPendingProbeConnections(const std::set& proTxHashes); size_t GetNodeCount(NumConnections num); size_t GetMaxOutboundNodeCount(); @@ -545,6 +546,7 @@ class CConnman CCriticalSection cs_vAddedNodes; std::vector vPendingMasternodes; std::map, std::set> masternodeQuorumNodes; // protected by cs_vPendingMasternodes + std::set masternodePendingProbes; mutable CCriticalSection cs_vPendingMasternodes; std::vector vNodes; std::list vNodesDisconnected; @@ -818,6 +820,8 @@ class CNode bool fSentAddr; // If 'true' this node will be disconnected on CMasternodeMan::ProcessMasternodeConnections() bool fMasternode; + // If 'true' this node will be disconnected after MNAUTH + bool fMasternodeProbe; CSemaphoreGrant grantOutbound; CSemaphoreGrant grantMasternodeOutbound; CCriticalSection cs_filter; diff --git a/src/net_processing.cpp b/src/net_processing.cpp index cd1bff022e48..af64b2a1357c 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -2013,7 +2013,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr (fLogIPs ? strprintf(", peeraddr=%s", pfrom->addr.ToString()) : "")); } - if (pfrom->nVersion >= LLMQS_PROTO_VERSION) { + if (pfrom->nVersion >= LLMQS_PROTO_VERSION && !pfrom->fMasternodeProbe) { CMNAuth::PushMNAUTH(pfrom, *connman); } @@ -2073,6 +2073,12 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr pfrom->nTimeFirstMessageReceived = GetTimeMicros(); pfrom->fFirstMessageIsMNAUTH = strCommand == NetMsgType::MNAUTH; // Note: do not break the flow here + + if (pfrom->fMasternodeProbe && !pfrom->fFirstMessageIsMNAUTH) { + LogPrint(BCLog::NET, "connection is a masternode probe but first received message is not MNAUTH, peer=%d", pfrom->GetId()); + pfrom->fDisconnect = true; + return false; + } } if (strCommand == NetMsgType::ADDR) { From 9ef2b05884a2e548ebf33005ff6f78910156e981 Mon Sep 17 00:00:00 2001 From: Alexander Block Date: Thu, 19 Mar 2020 15:23:00 +0100 Subject: [PATCH 05/13] Add masternode meta info to protx list/info --- src/masternode/masternode-meta.cpp | 18 ++++++++++++++++++ src/masternode/masternode-meta.h | 4 ++++ src/rpc/rpcevo.cpp | 5 +++++ 3 files changed, 27 insertions(+) diff --git a/src/masternode/masternode-meta.cpp b/src/masternode/masternode-meta.cpp index 6889f8900856..54489b3bf431 100644 --- a/src/masternode/masternode-meta.cpp +++ b/src/masternode/masternode-meta.cpp @@ -4,10 +4,28 @@ #include +#include "timedata.h" + CMasternodeMetaMan mmetaman; const std::string CMasternodeMetaMan::SERIALIZATION_VERSION_STRING = "CMasternodeMetaMan-Version-2"; +UniValue CMasternodeMetaInfo::ToJson() const +{ + UniValue ret(UniValue::VOBJ); + + auto now = GetAdjustedTime(); + + ret.push_back(Pair("lastDSQ", nLastDsq)); + ret.push_back(Pair("mixingTxCount", nMixingTxCount)); + ret.push_back(Pair("lastOutboundAttempt", lastOutboundAttempt)); + ret.push_back(Pair("lastOutboundAttemptElapsed", now - lastOutboundAttempt)); + ret.push_back(Pair("lastOutboundSuccess", lastOutboundSuccess)); + ret.push_back(Pair("lastOutboundSuccessElapsed", now - lastOutboundSuccess)); + + return ret; +} + void CMasternodeMetaInfo::AddGovernanceVote(const uint256& nGovernanceObjectHash) { LOCK(cs); diff --git a/src/masternode/masternode-meta.h b/src/masternode/masternode-meta.h index f9aa24bae3ae..5d2aa7422bc2 100644 --- a/src/masternode/masternode-meta.h +++ b/src/masternode/masternode-meta.h @@ -9,6 +9,8 @@ #include +#include "univalue.h" + #include class CConnman; @@ -62,6 +64,8 @@ class CMasternodeMetaInfo READWRITE(lastOutboundSuccess); } + UniValue ToJson() const; + public: const uint256& GetProTxHash() const { LOCK(cs); return proTxHash; } int64_t GetLastDsq() const { LOCK(cs); return nLastDsq; } diff --git a/src/rpc/rpcevo.cpp b/src/rpc/rpcevo.cpp index 13d7b9991142..b8c048cf4ecd 100644 --- a/src/rpc/rpcevo.cpp +++ b/src/rpc/rpcevo.cpp @@ -27,6 +27,8 @@ #include +#include "masternode/masternode-meta.h" + #ifdef ENABLE_WALLET extern UniValue signrawtransaction(const JSONRPCRequest& request); extern UniValue sendrawtransaction(const JSONRPCRequest& request); @@ -959,6 +961,9 @@ UniValue BuildDMNListEntry(CWallet* pwallet, const CDeterministicMNCPtr& dmn, bo walletObj.push_back(Pair("ownsOperatorRewardScript", CheckWalletOwnsScript(pwallet, dmn->pdmnState->scriptOperatorPayout))); o.push_back(Pair("wallet", walletObj)); + auto metaInfo = mmetaman.GetMetaInfo(dmn->proTxHash); + o.push_back(Pair("metaInfo", metaInfo->ToJson())); + return o; } From c9608bf930f6df538654a33e9ad946d996062b08 Mon Sep 17 00:00:00 2001 From: Alexander Block Date: Thu, 19 Mar 2020 15:23:19 +0100 Subject: [PATCH 06/13] Only add wallet info to protx list/info when wallet is enabled --- src/rpc/rpcevo.cpp | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/rpc/rpcevo.cpp b/src/rpc/rpcevo.cpp index b8c048cf4ecd..b3b9a252567f 100644 --- a/src/rpc/rpcevo.cpp +++ b/src/rpc/rpcevo.cpp @@ -952,14 +952,18 @@ UniValue BuildDMNListEntry(CWallet* pwallet, const CDeterministicMNCPtr& dmn, bo ownsCollateral = CheckWalletOwnsScript(pwallet, collateralTx->vout[dmn->collateralOutpoint.n].scriptPubKey); } - UniValue walletObj(UniValue::VOBJ); - walletObj.push_back(Pair("hasOwnerKey", hasOwnerKey)); - walletObj.push_back(Pair("hasOperatorKey", hasOperatorKey)); - walletObj.push_back(Pair("hasVotingKey", hasVotingKey)); - walletObj.push_back(Pair("ownsCollateral", ownsCollateral)); - walletObj.push_back(Pair("ownsPayeeScript", CheckWalletOwnsScript(pwallet, dmn->pdmnState->scriptPayout))); - walletObj.push_back(Pair("ownsOperatorRewardScript", CheckWalletOwnsScript(pwallet, dmn->pdmnState->scriptOperatorPayout))); - o.push_back(Pair("wallet", walletObj)); +#ifdef ENABLE_WALLET + if (pwallet) { + UniValue walletObj(UniValue::VOBJ); + walletObj.push_back(Pair("hasOwnerKey", hasOwnerKey)); + walletObj.push_back(Pair("hasOperatorKey", hasOperatorKey)); + walletObj.push_back(Pair("hasVotingKey", hasVotingKey)); + walletObj.push_back(Pair("ownsCollateral", ownsCollateral)); + walletObj.push_back(Pair("ownsPayeeScript", CheckWalletOwnsScript(pwallet, dmn->pdmnState->scriptPayout))); + walletObj.push_back(Pair("ownsOperatorRewardScript", CheckWalletOwnsScript(pwallet, dmn->pdmnState->scriptOperatorPayout))); + o.push_back(Pair("wallet", walletObj)); + } +#endif auto metaInfo = mmetaman.GetMetaInfo(dmn->proTxHash); o.push_back(Pair("metaInfo", metaInfo->ToJson())); From e8bbbec2599f2402f540c0fd92457c013c6da659 Mon Sep 17 00:00:00 2001 From: Alexander Block Date: Thu, 26 Mar 2020 08:38:12 +0100 Subject: [PATCH 07/13] Don't try to open masternode connections when network is disabled --- src/net.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/net.cpp b/src/net.cpp index 729ec1780d50..601d9dd19316 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -2115,6 +2115,9 @@ void CConnman::ThreadOpenMasternodeConnections() didConnect = false; + if (!fNetworkActive) + continue; + std::set connectedNodes; std::map connectedProRegTxHashes; ForEachNode([&](const CNode* pnode) { From 7df624d380ff491894f101a1abdbd50ae0aebff7 Mon Sep 17 00:00:00 2001 From: Alexander Block Date: Thu, 26 Mar 2020 08:42:54 +0100 Subject: [PATCH 08/13] Implement tests for LLMQ connection handling --- test/functional/llmq-connections.py | 116 ++++++++++++++++++ .../test_framework/test_framework.py | 10 ++ test/functional/test_runner.py | 1 + 3 files changed, 127 insertions(+) create mode 100755 test/functional/llmq-connections.py diff --git a/test/functional/llmq-connections.py b/test/functional/llmq-connections.py new file mode 100755 index 000000000000..5749fd5f9a90 --- /dev/null +++ b/test/functional/llmq-connections.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +# Copyright (c) 2015-2020 The Dash Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +from test_framework.test_framework import DashTestFramework +from test_framework.util import * + +''' +llmq-connections.py + +Checks intra quorum connections + +''' + +class LLMQConnections(DashTestFramework): + def set_test_params(self): + self.set_dash_test_params(15, 14, fast_dip3_enforcement=True) + self.set_dash_llmq_test_params(5, 3) + + def run_test(self): + self.nodes[0].spork("SPORK_17_QUORUM_DKG_ENABLED", 0) + self.wait_for_sporks_same() + + q = self.mine_quorum(expected_connections=2) + + self.log.info("checking for old intra quorum connections") + total_count = 0 + for mn in self.get_quorum_masternodes(q): + count = self.get_mn_connection_count(mn.node) + total_count += count + assert_greater_than_or_equal(count, 2) + assert(total_count < 40) + + self.check_reconnects(2) + + self.log.info("activating SPORK_21_QUORUM_ALL_CONNECTED") + self.nodes[0].spork("SPORK_21_QUORUM_ALL_CONNECTED", 0) + self.wait_for_sporks_same() + + self.log.info("mining one block and waiting for all members to connect to each other") + self.nodes[0].generate(1) + for mn in self.get_quorum_masternodes(q): + self.wait_for_mnauth(mn.node, 4) + + self.log.info("mine a new quorum and verify that all members connect to each other") + q = self.mine_quorum(expected_connections=4) + + self.log.info("checking that all MNs got probed") + for mn in self.get_quorum_masternodes(q): + wait_until(lambda: self.get_mn_probe_count(mn.node, q, False) == 4) + + self.log.info("checking that probes age") + self.bump_mocktime(60) + set_node_times(self.nodes, self.mocktime) + for mn in self.get_quorum_masternodes(q): + wait_until(lambda: self.get_mn_probe_count(mn.node, q, False) == 0) + + self.log.info("mine a new quorum and re-check probes") + q = self.mine_quorum(expected_connections=4) + for mn in self.get_quorum_masternodes(q): + wait_until(lambda: self.get_mn_probe_count(mn.node, q, True) == 4) + + self.check_reconnects(4) + + def check_reconnects(self, expected_connection_count): + self.log.info("disable and re-enable networking on all masternodes") + for mn in self.mninfo: + mn.node.setnetworkactive(False) + for mn in self.mninfo: + wait_until(lambda: len(mn.node.getpeerinfo()) == 0) + self.bump_mocktime(60) + set_node_times(self.nodes, self.mocktime) + for mn in self.mninfo: + mn.node.setnetworkactive(True) + + self.log.info("verify that all masternodes re-connected") + for q in self.nodes[0].quorum('list')['llmq_test']: + for mn in self.get_quorum_masternodes(q): + self.wait_for_mnauth(mn.node, expected_connection_count) + + # Also re-connect non-masternode connections + for i in range(1, len(self.nodes)): + connect_nodes(self.nodes[i], 0) + + + def get_mn_connection_count(self, node): + peers = node.getpeerinfo() + count = 0 + for p in peers: + if 'verified_proregtx_hash' in p and p['verified_proregtx_hash'] != '': + count += 1 + return count + + def get_mn_probe_count(self, node, q, check_peers): + count = 0 + mnList = node.protx('list', 'registered', 1) + peerList = node.getpeerinfo() + mnMap = {} + peerMap = {} + for mn in mnList: + mnMap[mn['proTxHash']] = mn + for p in peerList: + if 'verified_proregtx_hash' in p and p['verified_proregtx_hash'] != '': + peerMap[p['verified_proregtx_hash']] = p + for mn in self.get_quorum_masternodes(q): + pi = mnMap[mn.proTxHash] + if pi['metaInfo']['lastOutboundSuccessElapsed'] < 60: + count += 1 + elif check_peers and mn.proTxHash in peerMap: + count += 1 + return count + + +if __name__ == '__main__': + LLMQConnections().main() diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 52af8d5a9e1e..b1bad7b7ca31 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -930,6 +930,16 @@ def bump_time(): return new_quorum + def get_quorum_masternodes(self, q): + qi = self.nodes[0].quorum('info', 100, q) + result = [] + for m in qi['members']: + for mn in self.mninfo: + if mn.proTxHash == m['proTxHash']: + result.append(mn) + break + return result + def wait_for_mnauth(self, node, count, timeout=10): def test(): pi = node.getpeerinfo() diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 1ad88c8296c9..f4d25dd3459e 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -70,6 +70,7 @@ 'multikeysporks.py', 'llmq-signing.py', # NOTE: needs dash_hash to pass 'llmq-chainlocks.py', # NOTE: needs dash_hash to pass + 'llmq-connections.py', # NOTE: needs dash_hash to pass 'llmq-simplepose.py', # NOTE: needs dash_hash to pass 'llmq-is-cl-conflicts.py', # NOTE: needs dash_hash to pass 'llmq-is-retroactive.py', # NOTE: needs dash_hash to pass From dbaf13848dcb637cc57a8580ef986cd5c990be5c Mon Sep 17 00:00:00 2001 From: Alexander Block Date: Thu, 26 Mar 2020 16:16:35 +0100 Subject: [PATCH 09/13] Include inbound connections in output of "quorum dkgstatus" --- src/llmq/quorums_utils.cpp | 11 +++++++++-- src/llmq/quorums_utils.h | 2 +- src/rpc/rpcquorums.cpp | 2 +- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/llmq/quorums_utils.cpp b/src/llmq/quorums_utils.cpp index 7a84c9072b01..4f39ef946c16 100644 --- a/src/llmq/quorums_utils.cpp +++ b/src/llmq/quorums_utils.cpp @@ -44,13 +44,20 @@ uint256 CLLMQUtils::BuildSignHash(Consensus::LLMQType llmqType, const uint256& q return h.GetHash(); } -std::set CLLMQUtils::GetQuorumConnections(Consensus::LLMQType llmqType, const CBlockIndex* pindexQuorum, const uint256& forMember) +std::set CLLMQUtils::GetQuorumConnections(Consensus::LLMQType llmqType, const CBlockIndex* pindexQuorum, const uint256& forMember, bool onlyOutbound) { auto& params = Params().GetConsensus().llmqs.at(llmqType); auto mns = GetAllQuorumMembers(llmqType, pindexQuorum); std::set result; + if (!onlyOutbound) { + for (auto& dmn : mns) { + result.emplace(dmn->proTxHash); + } + return result; + } + if (sporkManager.IsSporkActive(SPORK_21_QUORUM_ALL_CONNECTED)) { for (auto& dmn : mns) { // this will cause deterministic behaviour between incoming and outgoing connections. @@ -125,7 +132,7 @@ void CLLMQUtils::EnsureQuorumConnections(Consensus::LLMQType llmqType, const CBl std::set connections; if (isMember) { - connections = CLLMQUtils::GetQuorumConnections(llmqType, pindexQuorum, myProTxHash); + connections = CLLMQUtils::GetQuorumConnections(llmqType, pindexQuorum, myProTxHash, true); } else { auto cindexes = CLLMQUtils::CalcDeterministicWatchConnections(llmqType, pindexQuorum, members.size(), 1); for (auto idx : cindexes) { diff --git a/src/llmq/quorums_utils.h b/src/llmq/quorums_utils.h index ea84671ad3d2..6f354781245a 100644 --- a/src/llmq/quorums_utils.h +++ b/src/llmq/quorums_utils.h @@ -31,7 +31,7 @@ class CLLMQUtils return BuildSignHash((Consensus::LLMQType)s.llmqType, s.quorumHash, s.id, s.msgHash); } - static std::set GetQuorumConnections(Consensus::LLMQType llmqType, const CBlockIndex* pindexQuorum, const uint256& forMember); + static std::set GetQuorumConnections(Consensus::LLMQType llmqType, const CBlockIndex* pindexQuorum, const uint256& forMember, bool onlyOutbound); static std::set CalcDeterministicWatchConnections(Consensus::LLMQType llmqType, const CBlockIndex* pindexQuorum, size_t memberCount, size_t connectionCount); static void EnsureQuorumConnections(Consensus::LLMQType llmqType, const CBlockIndex* pindexQuorum, const uint256& myProTxHash, bool allowWatch); diff --git a/src/rpc/rpcquorums.cpp b/src/rpc/rpcquorums.cpp index 10c868526be8..7d8b4e85b13a 100644 --- a/src/rpc/rpcquorums.cpp +++ b/src/rpc/rpcquorums.cpp @@ -185,7 +185,7 @@ UniValue quorum_dkgstatus(const JSONRPCRequest& request) if (fMasternodeMode) { const CBlockIndex* pindexQuorum = chainActive[tipHeight - (tipHeight % params.dkgInterval)]; - auto expectedConnections = llmq::CLLMQUtils::GetQuorumConnections(params.type, pindexQuorum, activeMasternodeInfo.proTxHash); + auto expectedConnections = llmq::CLLMQUtils::GetQuorumConnections(params.type, pindexQuorum, activeMasternodeInfo.proTxHash, false); std::map foundConnections; g_connman->ForEachNode([&](const CNode* pnode) { if (!pnode->verifiedProRegTxHash.IsNull() && expectedConnections.count(pnode->verifiedProRegTxHash)) { From 882b58c99096c57f4ab7f93fe3515935e779cc36 Mon Sep 17 00:00:00 2001 From: Alexander Block Date: Thu, 26 Mar 2020 16:47:48 +0100 Subject: [PATCH 10/13] Use <> instead of "" for #include --- src/masternode/masternode-meta.cpp | 2 +- src/masternode/masternode-meta.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/masternode/masternode-meta.cpp b/src/masternode/masternode-meta.cpp index 54489b3bf431..cebcd327f869 100644 --- a/src/masternode/masternode-meta.cpp +++ b/src/masternode/masternode-meta.cpp @@ -4,7 +4,7 @@ #include -#include "timedata.h" +#include CMasternodeMetaMan mmetaman; diff --git a/src/masternode/masternode-meta.h b/src/masternode/masternode-meta.h index 5d2aa7422bc2..9df2f56bf6cd 100644 --- a/src/masternode/masternode-meta.h +++ b/src/masternode/masternode-meta.h @@ -9,7 +9,7 @@ #include -#include "univalue.h" +#include #include From fdec67a55b50e3ee84995b83ae76e2dde6f27e42 Mon Sep 17 00:00:00 2001 From: Alexander Block Date: Thu, 26 Mar 2020 16:48:09 +0100 Subject: [PATCH 11/13] Wait for ping/pong after re-connecting all nodes --- test/functional/llmq-connections.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/functional/llmq-connections.py b/test/functional/llmq-connections.py index 5749fd5f9a90..a9f93ce4e42e 100755 --- a/test/functional/llmq-connections.py +++ b/test/functional/llmq-connections.py @@ -82,7 +82,11 @@ def check_reconnects(self, expected_connection_count): # Also re-connect non-masternode connections for i in range(1, len(self.nodes)): connect_nodes(self.nodes[i], 0) - + self.nodes[i].ping() + # wait for ping/pong so that we can be sure that spork propagation works + time.sleep(1) # needed to make sure we don't check before the ping is actually sent (fPingQueued might be true but SendMessages still not called) + for i in range(1, len(self.nodes)): + wait_until(lambda: all('pingwait' not in peer for peer in self.nodes[i].getpeerinfo())) def get_mn_connection_count(self, node): peers = node.getpeerinfo() From f82204db9598e9d6c5339a2e57923bd2be91a66c Mon Sep 17 00:00:00 2001 From: Alexander Block Date: Fri, 27 Mar 2020 15:09:15 +0100 Subject: [PATCH 12/13] Move intra-quorum connection calculation into local func --- src/llmq/quorums_utils.cpp | 39 ++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/src/llmq/quorums_utils.cpp b/src/llmq/quorums_utils.cpp index 4f39ef946c16..9fb5cd8d9288 100644 --- a/src/llmq/quorums_utils.cpp +++ b/src/llmq/quorums_utils.cpp @@ -73,25 +73,32 @@ std::set CLLMQUtils::GetQuorumConnections(Consensus::LLMQType llmqType, // TODO remove this after activation of SPORK_21_QUORUM_ALL_CONNECTED + auto calcOutbound = [&](size_t i, const uint256 proTxHash) { + // Connect to nodes at indexes (i+2^k)%n, where + // k: 0..max(1, floor(log2(n-1))-1) + // n: size of the quorum/ring + std::set r; + int gap = 1; + int gap_max = (int)mns.size() - 1; + int k = 0; + while ((gap_max >>= 1) || k <= 1) { + size_t idx = (i + gap) % mns.size(); + auto& otherDmn = mns[idx]; + if (otherDmn->proTxHash == proTxHash) { + continue; + } + r.emplace(otherDmn->proTxHash); + gap <<= 1; + k++; + } + return r; + }; + for (size_t i = 0; i < mns.size(); i++) { auto& dmn = mns[i]; if (dmn->proTxHash == forMember) { - // Connect to nodes at indexes (i+2^k)%n, where - // k: 0..max(1, floor(log2(n-1))-1) - // n: size of the quorum/ring - int gap = 1; - int gap_max = (int)mns.size() - 1; - int k = 0; - while ((gap_max >>= 1) || k <= 1) { - size_t idx = (i + gap) % mns.size(); - auto& otherDmn = mns[idx]; - if (otherDmn == dmn) { - continue; - } - result.emplace(otherDmn->proTxHash); - gap <<= 1; - k++; - } + auto r = calcOutbound(i, dmn->proTxHash); + result.insert(r.begin(), r.end()); // there can be no two members with the same proTxHash, so return early break; } From a09e36106e962be19c383597df724ea9bf355a01 Mon Sep 17 00:00:00 2001 From: Alexander Block Date: Fri, 27 Mar 2020 15:11:13 +0100 Subject: [PATCH 13/13] Fix onlyOutbound handling --- src/llmq/quorums_utils.cpp | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/src/llmq/quorums_utils.cpp b/src/llmq/quorums_utils.cpp index 9fb5cd8d9288..b179a78aca6c 100644 --- a/src/llmq/quorums_utils.cpp +++ b/src/llmq/quorums_utils.cpp @@ -51,20 +51,13 @@ std::set CLLMQUtils::GetQuorumConnections(Consensus::LLMQType llmqType, auto mns = GetAllQuorumMembers(llmqType, pindexQuorum); std::set result; - if (!onlyOutbound) { - for (auto& dmn : mns) { - result.emplace(dmn->proTxHash); - } - return result; - } - if (sporkManager.IsSporkActive(SPORK_21_QUORUM_ALL_CONNECTED)) { for (auto& dmn : mns) { // this will cause deterministic behaviour between incoming and outgoing connections. // Each member needs a connection to all other members, so we have each member paired. The below check // will be true on one side and false on the other side of the pairing, so we avoid having both members // initiating the connection. - if (dmn->proTxHash < forMember) { + if (!onlyOutbound || dmn->proTxHash < forMember) { result.emplace(dmn->proTxHash); } } @@ -99,10 +92,14 @@ std::set CLLMQUtils::GetQuorumConnections(Consensus::LLMQType llmqType, if (dmn->proTxHash == forMember) { auto r = calcOutbound(i, dmn->proTxHash); result.insert(r.begin(), r.end()); - // there can be no two members with the same proTxHash, so return early - break; + } else if (!onlyOutbound) { + auto r = calcOutbound(i, dmn->proTxHash); + if (r.count(forMember)) { + result.emplace(dmn->proTxHash); + } } } + return result; }