Let all LLMQ members connect to each other instead of only a few ones

src/evo/mnauth.cpp

@@ -6,6 +6,7 @@
 
 #include <masternode/activemasternode.h>
 #include <evo/deterministicmns.h>
+#include <masternode/masternode-meta.h>
 #include <masternode/masternode-sync.h>
 #include <net.h>
 #include <net_processing.h>
@@ -112,6 +113,16 @@ void CMNAuth::ProcessMessage(CNode* pnode, const std::string& strCommand, CDataS
             return;
         }
 
+        if (!pnode->fInbound) {
+            mmetaman.GetMetaInfo(mnauth.proRegTxHash)->SetLastOutboundSuccess(GetAdjustedTime());
+            if (pnode->fMasternodeProbe) {
+                LogPrint(BCLog::NET, "CMNAuth::ProcessMessage -- masternode probe successful for %s, disconnecting. peer=%d\n",
+                         mnauth.proRegTxHash.ToString(), pnode->GetId());
+                pnode->fDisconnect = true;
+                return;
+            }
+        }
+
         connman.ForEachNode([&](CNode* pnode2) {
             if (pnode2->verifiedProRegTxHash == mnauth.proRegTxHash) {
                 LogPrint(BCLog::NET, "CMNAuth::ProcessMessage -- Masternode %s has already verified as peer %d, dropping new connection. peer=%d\n",

src/llmq/quorums_dkgsessionhandler.cpp

@@ -12,6 +12,7 @@
 #include <chainparams.h>
 #include <init.h>
 #include <net_processing.h>
+#include <spork.h>
 #include <validation.h>
 
 namespace llmq
@@ -528,6 +529,9 @@ void CDKGSessionHandler::HandleDKGRound()
     });
 
     CLLMQUtils::EnsureQuorumConnections(params.type, pindexQuorum, curSession->myProTxHash, gArgs.GetBoolArg("-watchquorums", DEFAULT_WATCH_QUORUMS));
+    if (curSession->AreWeMember() && sporkManager.IsSporkActive(SPORK_21_QUORUM_ALL_CONNECTED)) {
+        CLLMQUtils::AddQuorumProbeConnections(params.type, pindexQuorum, curSession->myProTxHash);
+    }
 
     WaitForNextPhase(QuorumPhase_Initialized, QuorumPhase_Contribute, curQuorumHash, []{return false;});
 

src/llmq/quorums_utils.cpp

@@ -7,8 +7,11 @@
 
 #include <chainparams.h>
 #include <random.h>
+#include <spork.h>
 #include <validation.h>
 
+#include <masternode/masternode-meta.h>
+
 namespace llmq
 {
 
@@ -41,35 +44,62 @@ uint256 CLLMQUtils::BuildSignHash(Consensus::LLMQType llmqType, const uint256& q
     return h.GetHash();
 }
 
-std::set<uint256> CLLMQUtils::GetQuorumConnections(Consensus::LLMQType llmqType, const CBlockIndex* pindexQuorum, const uint256& forMember)
+std::set<uint256> CLLMQUtils::GetQuorumConnections(Consensus::LLMQType llmqType, const CBlockIndex* pindexQuorum, const uint256& forMember, bool onlyOutbound)
 {
     auto& params = Params().GetConsensus().llmqs.at(llmqType);
 
     auto mns = GetAllQuorumMembers(llmqType, pindexQuorum);
     std::set<uint256> result;
+
+    if (sporkManager.IsSporkActive(SPORK_21_QUORUM_ALL_CONNECTED)) {
+        for (auto& dmn : mns) {
+            // this will cause deterministic behaviour between incoming and outgoing connections.
+            // Each member needs a connection to all other members, so we have each member paired. The below check
+            // will be true on one side and false on the other side of the pairing, so we avoid having both members
+            // initiating the connection.
+            if (!onlyOutbound || dmn->proTxHash < forMember) {
+                result.emplace(dmn->proTxHash);
+            }
+        }
+        return result;
+    }
+
+    // TODO remove this after activation of SPORK_21_QUORUM_ALL_CONNECTED
+
+    auto calcOutbound = [&](size_t i, const uint256 proTxHash) {
+        // Connect to nodes at indexes (i+2^k)%n, where
+        //   k: 0..max(1, floor(log2(n-1))-1)
+        //   n: size of the quorum/ring
+        std::set<uint256> r;
+        int gap = 1;
+        int gap_max = (int)mns.size() - 1;
+        int k = 0;
+        while ((gap_max >>= 1) || k <= 1) {
+            size_t idx = (i + gap) % mns.size();
+            auto& otherDmn = mns[idx];
+            if (otherDmn->proTxHash == proTxHash) {
+                continue;
+            }
+            r.emplace(otherDmn->proTxHash);
+            gap <<= 1;
+            k++;
+        }
+        return r;
+    };
+
     for (size_t i = 0; i < mns.size(); i++) {
         auto& dmn = mns[i];
         if (dmn->proTxHash == forMember) {
-            // Connect to nodes at indexes (i+2^k)%n, where
-            //   k: 0..max(1, floor(log2(n-1))-1)
-            //   n: size of the quorum/ring
-            int gap = 1;
-            int gap_max = (int)mns.size() - 1;
-            int k = 0;
-            while ((gap_max >>= 1) || k <= 1) {
-                size_t idx = (i + gap) % mns.size();
-                auto& otherDmn = mns[idx];
-                if (otherDmn == dmn) {
-                    continue;
-                }
-                result.emplace(otherDmn->proTxHash);
-                gap <<= 1;
-                k++;
+            auto r = calcOutbound(i, dmn->proTxHash);
+            result.insert(r.begin(), r.end());
+        } else if (!onlyOutbound) {
+            auto r = calcOutbound(i, dmn->proTxHash);
+            if (r.count(forMember)) {
+                result.emplace(dmn->proTxHash);
             }
-            // there can be no two members with the same proTxHash, so return early
-            break;
         }
     }
+
     return result;
 }
 
@@ -106,7 +136,7 @@ void CLLMQUtils::EnsureQuorumConnections(Consensus::LLMQType llmqType, const CBl
 
     std::set<uint256> connections;
     if (isMember) {
-        connections = CLLMQUtils::GetQuorumConnections(llmqType, pindexQuorum, myProTxHash);
+        connections = CLLMQUtils::GetQuorumConnections(llmqType, pindexQuorum, myProTxHash, true);
     } else {
         auto cindexes = CLLMQUtils::CalcDeterministicWatchConnections(llmqType, pindexQuorum, members.size(), 1);
         for (auto idx : cindexes) {
@@ -131,6 +161,42 @@ void CLLMQUtils::EnsureQuorumConnections(Consensus::LLMQType llmqType, const CBl
     }
 }
 
+void CLLMQUtils::AddQuorumProbeConnections(Consensus::LLMQType llmqType, const CBlockIndex *pindexQuorum, const uint256 &myProTxHash)
+{
+    auto members = GetAllQuorumMembers(llmqType, pindexQuorum);
+    auto curTime = GetAdjustedTime();
+
+    std::set<uint256> probeConnections;
+    for (auto& dmn : members) {
+        if (dmn->proTxHash == myProTxHash) {
+            continue;
+        }
+        auto lastOutbound = mmetaman.GetMetaInfo(dmn->proTxHash)->GetLastOutboundSuccess();
+        // re-probe after 50 minutes so that the "good connection" check in the DKG doesn't fail just because we're on
+        // the brink of timeout
+        if (curTime - lastOutbound > 50 * 60) {
+            probeConnections.emplace(dmn->proTxHash);
+        }
+    }
+
+    if (!probeConnections.empty()) {
+        if (LogAcceptCategory(BCLog::LLMQ)) {
+            auto mnList = deterministicMNManager->GetListAtChainTip();
+            std::string debugMsg = strprintf("CLLMQUtils::%s -- adding masternodes probes for quorum %s:\n", __func__, pindexQuorum->GetBlockHash().ToString());
+            for (auto& c : probeConnections) {
+                auto dmn = mnList.GetValidMN(c);
+                if (!dmn) {
+                    debugMsg += strprintf("  %s (not in valid MN set anymore)\n", c.ToString());
+                } else {
+                    debugMsg += strprintf("  %s (%s)\n", c.ToString(), dmn->pdmnState->addr.ToString(false));
+                }
+            }
+            LogPrint(BCLog::LLMQ, debugMsg.c_str());
+        }
+        g_connman->AddPendingProbeConnections(probeConnections);
+    }
+}
+
 bool CLLMQUtils::IsQuorumActive(Consensus::LLMQType llmqType, const uint256& quorumHash)
 {
     auto& params = Params().GetConsensus().llmqs.at(llmqType);

src/llmq/quorums_utils.h

@@ -31,10 +31,11 @@ class CLLMQUtils
         return BuildSignHash((Consensus::LLMQType)s.llmqType, s.quorumHash, s.id, s.msgHash);
     }
 
-    static std::set<uint256> GetQuorumConnections(Consensus::LLMQType llmqType, const CBlockIndex* pindexQuorum, const uint256& forMember);
+    static std::set<uint256> GetQuorumConnections(Consensus::LLMQType llmqType, const CBlockIndex* pindexQuorum, const uint256& forMember, bool onlyOutbound);
     static std::set<size_t> CalcDeterministicWatchConnections(Consensus::LLMQType llmqType, const CBlockIndex* pindexQuorum, size_t memberCount, size_t connectionCount);
 
     static void EnsureQuorumConnections(Consensus::LLMQType llmqType, const CBlockIndex* pindexQuorum, const uint256& myProTxHash, bool allowWatch);
+    static void AddQuorumProbeConnections(Consensus::LLMQType llmqType, const CBlockIndex* pindexQuorum, const uint256& myProTxHash);
 
     static bool IsQuorumActive(Consensus::LLMQType llmqType, const uint256& quorumHash);
 

src/masternode/masternode-meta.cpp

@@ -4,10 +4,28 @@
 
 #include <masternode/masternode-meta.h>
 
+#include <timedata.h>
+
 CMasternodeMetaMan mmetaman;
 
 const std::string CMasternodeMetaMan::SERIALIZATION_VERSION_STRING = "CMasternodeMetaMan-Version-2";
 
+UniValue CMasternodeMetaInfo::ToJson() const
+{
+    UniValue ret(UniValue::VOBJ);
+
+    auto now = GetAdjustedTime();
+
+    ret.push_back(Pair("lastDSQ", nLastDsq));
+    ret.push_back(Pair("mixingTxCount", nMixingTxCount));
+    ret.push_back(Pair("lastOutboundAttempt", lastOutboundAttempt));
+    ret.push_back(Pair("lastOutboundAttemptElapsed", now - lastOutboundAttempt));
+    ret.push_back(Pair("lastOutboundSuccess", lastOutboundSuccess));
+    ret.push_back(Pair("lastOutboundSuccessElapsed", now - lastOutboundSuccess));
+
+    return ret;
+}
+
 void CMasternodeMetaInfo::AddGovernanceVote(const uint256& nGovernanceObjectHash)
 {
     LOCK(cs);

src/masternode/masternode-meta.h

@@ -9,6 +9,8 @@
 
 #include <evo/deterministicmns.h>
 
+#include <univalue.h>
+
 #include <memory>
 
 class CConnman;
@@ -34,6 +36,7 @@ class CMasternodeMetaInfo
     std::map<uint256, int> mapGovernanceObjectsVotedOn;
 
     int64_t lastOutboundAttempt = 0;
+    int64_t lastOutboundSuccess = 0;
 
 public:
     CMasternodeMetaInfo() {}
@@ -43,7 +46,8 @@ class CMasternodeMetaInfo
         nLastDsq(ref.nLastDsq),
         nMixingTxCount(ref.nMixingTxCount),
         mapGovernanceObjectsVotedOn(ref.mapGovernanceObjectsVotedOn),
-        lastOutboundAttempt(ref.lastOutboundAttempt)
+        lastOutboundAttempt(ref.lastOutboundAttempt),
+        lastOutboundSuccess(ref.lastOutboundSuccess)
     {
     }
 
@@ -57,8 +61,11 @@ class CMasternodeMetaInfo
         READWRITE(nMixingTxCount);
         READWRITE(mapGovernanceObjectsVotedOn);
         READWRITE(lastOutboundAttempt);
+        READWRITE(lastOutboundSuccess);
     }
 
+    UniValue ToJson() const;
+
 public:
     const uint256& GetProTxHash() const { LOCK(cs); return proTxHash; }
     int64_t GetLastDsq() const { LOCK(cs); return nLastDsq; }
@@ -73,6 +80,8 @@ class CMasternodeMetaInfo
 
     void SetLastOutboundAttempt(int64_t t) { LOCK(cs); lastOutboundAttempt = t; }
     int64_t GetLastOutboundAttempt() const { LOCK(cs); return lastOutboundAttempt; }
+    void SetLastOutboundSuccess(int64_t t) { LOCK(cs); lastOutboundSuccess = t; }
+    int64_t GetLastOutboundSuccess() const { LOCK(cs); return lastOutboundSuccess; }
 };
 typedef std::shared_ptr<CMasternodeMetaInfo> CMasternodeMetaInfoPtr;
 

src/masternode/masternode-utils.cpp

@@ -30,7 +30,7 @@ void CMasternodeUtils::ProcessMasternodeConnections(CConnman& connman)
     // Don't disconnect masternode connections when we have less then the desired amount of outbound nodes
     int nonMasternodeCount = 0;
     connman.ForEachNode(CConnman::AllNodes, [&](CNode* pnode) {
-        if (!pnode->fInbound && !pnode->fFeeler && !pnode->m_manual_connection && !pnode->fMasternode) {
+        if (!pnode->fInbound && !pnode->fFeeler && !pnode->m_manual_connection && !pnode->fMasternode && !pnode->fMasternodeProbe) {
             nonMasternodeCount++;
         }
     });