From 536529eae53df27a008b1125a5b41ab96d35410f Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Mon, 16 Sep 2019 16:40:52 +0300 Subject: [PATCH 01/77] discovery: v5 initial commit. No network, not working but compiling --- discovery/build.gradle | 12 + .../beacon/discovery/DiscoverTask.java | 70 ++++ .../beacon/discovery/DiscoveryManager.java | 96 +++++ .../discovery/DiscoveryMessageHandler.java | 7 + .../discovery/DiscoveryV5MessageHandler.java | 151 ++++++++ .../ethereum/beacon/discovery/Functions.java | 132 +++++++ .../beacon/discovery/IdentityScheme.java | 34 ++ .../beacon/discovery/MessageCode.java | 75 ++++ .../beacon/discovery/MessageHandler.java | 34 ++ .../beacon/discovery/NetworkPacket.java | 22 ++ .../beacon/discovery/NodeContext.java | 262 ++++++++++++++ .../ethereum/beacon/discovery/NodeIndex.java | 25 ++ .../ethereum/beacon/discovery/NodeStatus.java | 70 ++++ .../ethereum/beacon/discovery/NodeTable.java | 18 + .../beacon/discovery/NodeTableImpl.java | 140 +++++++ .../beacon/discovery/RefreshTask.java | 36 ++ .../beacon/discovery/enr/NodeRecord.java | 162 +++++++++ .../beacon/discovery/enr/NodeRecordInfo.java | 63 ++++ .../beacon/discovery/enr/NodeRecordV4.java | 334 +++++++++++++++++ .../beacon/discovery/enr/NodeRecordV5.java | 341 ++++++++++++++++++ .../discovery/message/DiscoveryMessage.java | 10 + .../discovery/message/DiscoveryV5Message.java | 100 +++++ .../discovery/message/FindNodeMessage.java | 69 ++++ .../discovery/message/NodesMessage.java | 103 ++++++ .../beacon/discovery/message/PingMessage.java | 61 ++++ .../beacon/discovery/message/PongMessage.java | 92 +++++ .../beacon/discovery/message/V5Message.java | 8 + .../discovery/packet/AbstractPacket.java | 16 + .../packet/AuthHeaderMessagePacket.java | 211 +++++++++++ .../discovery/packet/MessagePacket.java | 111 ++++++ .../beacon/discovery/packet/Packet.java | 14 + .../beacon/discovery/packet/RandomPacket.java | 74 ++++ .../discovery/packet/UnknownPacket.java | 56 +++ .../discovery/packet/WhoAreYouPacket.java | 112 ++++++ .../discovery/storage/AuthTagRepository.java | 57 +++ .../discovery/storage/NodeTableStorage.java | 14 + .../storage/NodeTableStorageFactory.java | 17 + .../storage/NodeTableStorageFactoryImpl.java | 52 +++ .../storage/NodeTableStorageImpl.java | 67 ++++ .../discovery/DiscoveryNoNetworkTest.java | 136 +++++++ .../beacon/discovery/NodeRecordTest.java | 103 ++++++ .../beacon/discovery/NodeTableTest.java | 111 ++++++ .../beacon/discovery/RlpExtraTest.java | 35 ++ settings.gradle | 4 +- test/src/test/resources/eth2.0-spec-tests | 2 +- .../util/bytes/ArrayWrappingBytes16.java | 59 +++ .../util/bytes/ArrayWrappingBytes33.java | 59 +++ .../pegasys/artemis/util/bytes/Bytes16.java | 187 ++++++++++ .../pegasys/artemis/util/bytes/Bytes33.java | 183 ++++++++++ .../bytes/MutableArrayWrappingBytes16.java | 39 ++ .../bytes/MutableArrayWrappingBytes33.java | 39 ++ .../artemis/util/bytes/MutableBytes16.java | 145 ++++++++ .../artemis/util/bytes/MutableBytes33.java | 145 ++++++++ .../artemis/util/bytes/WrappingBytes16.java | 62 ++++ .../artemis/util/bytes/WrappingBytes33.java | 62 ++++ versions.gradle | 1 + 56 files changed, 4698 insertions(+), 2 deletions(-) create mode 100644 discovery/build.gradle create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/DiscoverTask.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManager.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryMessageHandler.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryV5MessageHandler.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/IdentityScheme.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/MessageCode.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/MessageHandler.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/NetworkPacket.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/NodeContext.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/NodeIndex.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/NodeStatus.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/NodeTable.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/NodeTableImpl.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/RefreshTask.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecord.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordInfo.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV4.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV5.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/message/DiscoveryMessage.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/message/DiscoveryV5Message.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/message/FindNodeMessage.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/message/NodesMessage.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/message/PingMessage.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/message/PongMessage.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/message/V5Message.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/packet/AbstractPacket.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/packet/AuthHeaderMessagePacket.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/packet/MessagePacket.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/packet/Packet.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/packet/RandomPacket.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/packet/UnknownPacket.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/packet/WhoAreYouPacket.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/storage/AuthTagRepository.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorage.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactory.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactoryImpl.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageImpl.java create mode 100644 discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java create mode 100644 discovery/src/test/java/org/ethereum/beacon/discovery/NodeRecordTest.java create mode 100644 discovery/src/test/java/org/ethereum/beacon/discovery/NodeTableTest.java create mode 100644 discovery/src/test/java/org/ethereum/beacon/discovery/RlpExtraTest.java create mode 100644 types/src/main/java/tech/pegasys/artemis/util/bytes/ArrayWrappingBytes16.java create mode 100644 types/src/main/java/tech/pegasys/artemis/util/bytes/ArrayWrappingBytes33.java create mode 100644 types/src/main/java/tech/pegasys/artemis/util/bytes/Bytes16.java create mode 100644 types/src/main/java/tech/pegasys/artemis/util/bytes/Bytes33.java create mode 100644 types/src/main/java/tech/pegasys/artemis/util/bytes/MutableArrayWrappingBytes16.java create mode 100644 types/src/main/java/tech/pegasys/artemis/util/bytes/MutableArrayWrappingBytes33.java create mode 100644 types/src/main/java/tech/pegasys/artemis/util/bytes/MutableBytes16.java create mode 100644 types/src/main/java/tech/pegasys/artemis/util/bytes/MutableBytes33.java create mode 100644 types/src/main/java/tech/pegasys/artemis/util/bytes/WrappingBytes16.java create mode 100644 types/src/main/java/tech/pegasys/artemis/util/bytes/WrappingBytes33.java diff --git a/discovery/build.gradle b/discovery/build.gradle new file mode 100644 index 000000000..d2618488f --- /dev/null +++ b/discovery/build.gradle @@ -0,0 +1,12 @@ +dependencies { + implementation project(':types') + implementation project(':util') + implementation project(':db:core') + implementation project(':chain') + + implementation 'com.google.guava:guava' + implementation 'io.projectreactor:reactor-core' + implementation 'io.netty:netty-all' + implementation 'org.apache.logging.log4j:log4j-core' + implementation 'org.web3j:core' +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoverTask.java b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoverTask.java new file mode 100644 index 000000000..c3e5ffaf5 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoverTask.java @@ -0,0 +1,70 @@ +package org.ethereum.beacon.discovery; + +import org.ethereum.beacon.discovery.enr.NodeRecordInfo; +import org.ethereum.beacon.discovery.enr.NodeRecordV5; +import tech.pegasys.artemis.util.bytes.Bytes32; + +import java.util.List; +import java.util.function.Predicate; + +import static org.ethereum.beacon.discovery.NodeContext.DEFAULT_DISTANCE; +import static org.ethereum.beacon.discovery.enr.NodeRecordV5.NODE_ID_FUNCTION; + +public class DiscoverTask { + public static final int MS_IN_SECOND = 1000; + NodeTable nodeTable; + Bytes32 homeNodeId; + RefreshTask refreshTask; + + public DiscoverTask(NodeTable nodeTable, NodeRecordV5 homeNode, RefreshTask refreshTask) { + this.nodeTable = nodeTable; + this.homeNodeId = NODE_ID_FUNCTION.apply(homeNode); + this.refreshTask = refreshTask; + } + + // TODO: first batch to include dead and switch them to SLEEP + public Predicate refreshTask() { + return nodeRecord -> { + final int DISCOVER_AFTER_SECONDS = 600; + final int MAX_RETRIES = 10; + long currentTime = System.currentTimeMillis() / MS_IN_SECOND; + if (nodeRecord.getStatus() == NodeStatus.ACTIVE + && nodeRecord.getLastRetry() > currentTime - DISCOVER_AFTER_SECONDS) { + return false; // no need to rediscover + } + if (nodeRecord.getRetry() >= MAX_RETRIES) { + nodeTable.save( + new NodeRecordInfo( + nodeRecord.getNode(), nodeRecord.getLastRetry(), NodeStatus.DEAD, 0)); + return false; + } + if ((currentTime - nodeRecord.getLastRetry()) < (nodeRecord.getRetry() * nodeRecord.getRetry())) { + return false; // too early for retry + } + + return true; + }; + } + + public void run() { + List nodes = nodeTable.findClosestNodes(homeNodeId, DEFAULT_DISTANCE); + nodes.stream() + .filter(refreshTask()) + .forEach( + nodeRecord -> + refreshTask.add( + nodeRecord, + () -> nodeTable.save( + new NodeRecordInfo( + nodeRecord.getNode(), + System.currentTimeMillis() / MS_IN_SECOND, + NodeStatus.ACTIVE, + 0)), + () -> nodeTable.save( + new NodeRecordInfo( + nodeRecord.getNode(), + System.currentTimeMillis() / MS_IN_SECOND, + NodeStatus.SLEEP, + (nodeRecord.getRetry() + 1))))); + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManager.java b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManager.java new file mode 100644 index 000000000..c17e27e6a --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManager.java @@ -0,0 +1,96 @@ +package org.ethereum.beacon.discovery; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.ethereum.beacon.discovery.enr.NodeRecord; +import org.ethereum.beacon.discovery.enr.NodeRecordInfo; +import org.ethereum.beacon.discovery.enr.NodeRecordV5; +import org.ethereum.beacon.discovery.packet.UnknownPacket; +import org.ethereum.beacon.discovery.storage.AuthTagRepository; +import org.reactivestreams.Publisher; +import reactor.core.publisher.Flux; +import reactor.core.publisher.FluxSink; +import reactor.core.publisher.ReplayProcessor; +import tech.pegasys.artemis.util.bytes.Bytes32; + +import java.security.SecureRandom; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; + +public class DiscoveryManager { + private static final Logger logger = LogManager.getLogger(DiscoveryManager.class); + private final ReplayProcessor outgoingMessages = ReplayProcessor.cacheLast(); + private final FluxSink outgoingSink = outgoingMessages.sink(); + private final Bytes32 homeNodeId; + private final NodeRecordV5 homeNodeRecord; + private NodeTable nodeTable; + private Publisher incomingPackets; + private Map recentContexts = new ConcurrentHashMap<>(); // nodeId -> context + private AuthTagRepository authTagRepo; + + public DiscoveryManager( + NodeTable nodeTable, Publisher incomingPackets, Bytes32 homeNodeId) { + this.nodeTable = nodeTable; + this.incomingPackets = incomingPackets; + this.homeNodeId = homeNodeId; + this.homeNodeRecord = (NodeRecordV5) nodeTable.getHomeNode(); + this.authTagRepo = new AuthTagRepository(); + } + + public void start() { + Flux.from(incomingPackets) + .subscribe( + unknownPacket -> { + if (unknownPacket.isWhoAreYouPacket(homeNodeId)) { + Optional nodeContextOptional = + authTagRepo.get(unknownPacket.getWhoAreYouPacket().getAuthTag()); + if (nodeContextOptional.isPresent()) { + nodeContextOptional.get().addIncomingEvent(unknownPacket); + } else { + // TODO: ban or whatever + } + } else { + Bytes32 fromNodeId = unknownPacket.getSourceNodeId(homeNodeId); + getContext(fromNodeId) + .ifPresent(context -> context.addIncomingEvent(unknownPacket)); + } + }); + } + + public void connect(NodeRecord nodeRecord) { + if (!nodeTable.getNode(nodeRecord.getNodeId()).isPresent()) { + nodeTable.save(NodeRecordInfo.createDefault(nodeRecord)); + } + NodeContext context = getContext(nodeRecord.getNodeId()).get(); + context.initiate(); + } + + private Optional getContext(Bytes32 nodeId) { + NodeContext context = recentContexts.get(nodeId); + if (context == null) { + Optional nodeOptional = nodeTable.getNode(nodeId); + if (!nodeOptional.isPresent()) { + logger.trace( + () -> String.format("Couldn't find node record for nodeId %s, ignoring", nodeId)); + return Optional.empty(); + } + NodeRecordV5 nodeRecord = nodeOptional.get().getNode(); + SecureRandom random = new SecureRandom(); + context = + new NodeContext( + nodeRecord, + homeNodeRecord, + nodeTable, + authTagRepo, + packet -> outgoingSink.next(new NetworkPacket(packet, nodeRecord)), + random); + } + + return Optional.of(context); + } + + public Publisher getOutgoingMessages() { + return outgoingMessages; // TODO: link to network + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryMessageHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryMessageHandler.java new file mode 100644 index 000000000..a7e162d6d --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryMessageHandler.java @@ -0,0 +1,7 @@ +package org.ethereum.beacon.discovery; + +import org.ethereum.beacon.discovery.message.DiscoveryMessage; + +public interface DiscoveryMessageHandler { + void handleMessage(M message, NodeContext context); +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryV5MessageHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryV5MessageHandler.java new file mode 100644 index 000000000..e7e36d70a --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryV5MessageHandler.java @@ -0,0 +1,151 @@ +package org.ethereum.beacon.discovery; + +import org.ethereum.beacon.discovery.enr.NodeRecordInfo; +import org.ethereum.beacon.discovery.message.DiscoveryV5Message; +import org.ethereum.beacon.discovery.message.NodesMessage; +import org.ethereum.beacon.discovery.message.PongMessage; +import org.ethereum.beacon.discovery.packet.MessagePacket; +import tech.pegasys.artemis.util.bytes.Bytes32; +import tech.pegasys.artemis.util.bytes.BytesValue; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; + +import static org.ethereum.beacon.discovery.NodeContext.DEFAULT_DISTANCE; + +public class DiscoveryV5MessageHandler implements DiscoveryMessageHandler { + public static final int CLEANUP_DELAY = 60; + private static final int MAX_NODES_IN_MSG = 24; + private final Bytes32 homeNodeId; + private final BytesValue initiatorKey; + private final BytesValue authTag; + private final NodeTable nodeTable; + ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(); + private Map nodesCounters = new ConcurrentHashMap<>(); + private Map nodesExpirationTasks = new ConcurrentHashMap<>(); + + public DiscoveryV5MessageHandler(Bytes32 homeNodeId, BytesValue initiatorKey, BytesValue authTag, NodeTable nodeTable) { + this.homeNodeId = homeNodeId; + this.initiatorKey = initiatorKey; + this.authTag = authTag; + this.nodeTable = nodeTable; + } + + @Override + public void handleMessage(DiscoveryV5Message message, NodeContext context) { + MessageCode code = message.getCode(); + switch (code) { + case PING: + { + PongMessage responseMessage = + new PongMessage( + message.getRequestId(), + context.getNodeRecord().getSeqNumber(), + context.getNodeRecord().getIpV4address(), + context.getNodeRecord().getUdpPort()); + context.addOutgoingEvent( + MessagePacket.create( + homeNodeId, + context.getNodeRecord().getNodeId(), + authTag, + initiatorKey, + DiscoveryV5Message.from(responseMessage))); + break; + } + case PONG: + { + // FIXME: do we need to update NodeRecord from PONG response? + context.clearRequestId(message.getRequestId(), MessageCode.PING); + break; + } + case FINDNODE: + { + List nodes = + nodeTable.findClosestNodes(context.getNodeRecord().getNodeId(), DEFAULT_DISTANCE); + final AtomicInteger counter = new AtomicInteger(); + nodes.stream() + .map(NodeRecordInfo::getNode) + .collect(Collectors.groupingBy(it -> counter.getAndIncrement() / MAX_NODES_IN_MSG)) + .values() + .forEach( + c -> + context.addOutgoingEvent( + MessagePacket.create( + homeNodeId, + context.getNodeRecord().getNodeId(), + authTag, + initiatorKey, + DiscoveryV5Message.from( + new NodesMessage( + message.getRequestId(), + nodes.size() / MAX_NODES_IN_MSG, + () -> c, + nodes.size()))))); + break; + } + case NODES: + { + NodesMessage nodesMessage = (NodesMessage) message.create(); + // NODES total count handling + cleanup schedule + if (nodesCounters.containsKey(nodesMessage.getRequestId())) { + synchronized (this) { + int counter = nodesCounters.get(nodesMessage.getRequestId()) - 1; + if (counter == 0) { + cleanUp(nodesMessage.getRequestId(), context); + } else { + nodesCounters.put(nodesMessage.getRequestId(), counter); + reScheduleCleanUpDelay(nodesMessage.getRequestId(), context); + } + } + } else if (nodesMessage.getTotal() > 1) { + nodesCounters.put(nodesMessage.getRequestId(), nodesMessage.getTotal() - 1); + reScheduleCleanUpDelay(nodesMessage.getRequestId(), context); + } else { + context.clearRequestId(nodesMessage.getRequestId(), MessageCode.FINDNODE); + } + + // Parse node records + nodesMessage + .getNodeRecords() + .forEach( + nodeRecordV5 -> { + if (!nodeTable.getNode(nodeRecordV5.getNodeId()).isPresent()) { + // TODO: should we update? + nodeTable.save(NodeRecordInfo.createDefault(nodeRecordV5)); + } + }); + + break; + } + default: + { + throw new RuntimeException("Not implemented yet"); + } + } + } + + private synchronized void reScheduleCleanUpDelay(BytesValue requestId, NodeContext context) { + nodesExpirationTasks.remove(requestId).cancel(true); + ScheduledFuture future = + scheduler.schedule( + () -> { + cleanUp(requestId, context); + }, + CLEANUP_DELAY, + TimeUnit.SECONDS); + nodesExpirationTasks.put(requestId, future); + } + + private synchronized void cleanUp(BytesValue requestId, NodeContext context) { + nodesCounters.remove(requestId); + nodesExpirationTasks.remove(requestId); + context.clearRequestId(requestId, MessageCode.FINDNODE); + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java b/discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java new file mode 100644 index 000000000..d577ee70a --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java @@ -0,0 +1,132 @@ +package org.ethereum.beacon.discovery; + +import org.bouncycastle.crypto.BasicAgreement; +import org.bouncycastle.crypto.Digest; +import org.bouncycastle.crypto.agreement.ECDHBasicAgreement; +import org.bouncycastle.crypto.digests.SHA256Digest; +import org.bouncycastle.crypto.generators.HKDFBytesGenerator; +import org.bouncycastle.crypto.params.ECDomainParameters; +import org.bouncycastle.crypto.params.ECPrivateKeyParameters; +import org.bouncycastle.crypto.params.ECPublicKeyParameters; +import org.bouncycastle.crypto.params.HKDFParameters; +import org.bouncycastle.crypto.util.PrivateKeyFactory; +import org.bouncycastle.math.ec.ECCurve; +import org.ethereum.beacon.crypto.Hashes; +import org.javatuples.Triplet; +import tech.pegasys.artemis.util.bytes.Bytes32; +import tech.pegasys.artemis.util.bytes.BytesValue; + +import javax.crypto.Cipher; +import javax.crypto.spec.IvParameterSpec; +import javax.crypto.spec.SecretKeySpec; +import java.security.SecureRandom; +import java.util.Random; + +public class Functions { + private static final int RECIPIENT_KEY_LENGTH = 32; // FIXME + private static final int INITIATOR_KEY_LENGTH = 32; // FIXME + private static final int AUTH_RESP_KEY_LENGTH = 32; // FIXME + + public static Bytes32 hash(BytesValue value) { + return Hashes.sha256(value); + } + + /** Creates a signature of x using the given key */ + public static BytesValue sign(BytesValue key, BytesValue x) { + // TODO: implement + return x; + } + + /** + * AES-GCM encryption/authentication with the given `key`, `nonce` and additional authenticated + * data `ad`. Size of `key` is 16 bytes (AES-128), size of `nonce` 12 bytes. + */ + public static BytesValue aesgcm_encrypt( + BytesValue privateKey, BytesValue nonce, BytesValue message, BytesValue aad) { + try { + Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding", "BC"); + cipher.init( + Cipher.ENCRYPT_MODE, + new SecretKeySpec(privateKey.extractArray(), "AES"), + new IvParameterSpec(nonce.extractArray())); + cipher.updateAAD(aad.extractArray()); + return BytesValue.wrap(cipher.doFinal(message.extractArray())); + } catch (Exception e) { + throw new RuntimeException("No AES/GCM cipher provider", e); + } + } + + public static BytesValue aesgcm_decrypt( + BytesValue privateKey, BytesValue nonce, BytesValue encoded, BytesValue aad) { + try { + Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding", "BC"); + cipher.init( + Cipher.DECRYPT_MODE, + new SecretKeySpec(privateKey.extractArray(), "AES"), + new IvParameterSpec(nonce.extractArray())); + cipher.updateAAD(aad.extractArray()); + return BytesValue.wrap(cipher.doFinal(encoded.extractArray())); + } catch (Exception e) { + throw new RuntimeException("No AES/GCM cipher provider", e); + } + } + + /** + * The ephemeral key is used to perform Diffie-Hellman key agreement with B's static public key + * and the session keys are derived from it using the HKDF key derivation function. + * + *

+ * ephemeral-key = random private key + * ephemeral-pubkey = public key corresponding to ephemeral-key + * dest-pubkey = public key of B + * secret = agree(ephemeral-key, dest-pubkey) + * info = "discovery v5 key agreement" || node-id-A || node-id-B + * prk = HKDF-Extract(secret, id-nonce) + * initiator-key, recipient-key, auth-resp-key = HKDF-Expand(prk, info) + */ + public static Triplet hkdf_expand( + BytesValue srcNodeId, + BytesValue destNodeId, + BytesValue ephemeralKey, + BytesValue idNonce, + BytesValue destPubKey) { + try { + Digest digest = new SHA256Digest(); // FIXME: or whatever + ECPrivateKeyParameters ecdhPrivateKeyParameters = + (ECPrivateKeyParameters) (PrivateKeyFactory.createKey(ephemeralKey.extractArray())); + ECDomainParameters ecDomainParameters = ecdhPrivateKeyParameters.getParameters(); + ECCurve ecCurve = ecDomainParameters.getCurve(); + ECPublicKeyParameters ecPublicKeyParameters = + new ECPublicKeyParameters( + ecCurve.decodePoint(destPubKey.extractArray()), ecDomainParameters); + BasicAgreement agree = new ECDHBasicAgreement(); + agree.init(ecdhPrivateKeyParameters); + byte[] keyAgreement = agree.calculateAgreement(ecPublicKeyParameters).toByteArray(); + + BytesValue info = + BytesValue.wrap("discovery v5 key agreement".getBytes()) + .concat(srcNodeId) + .concat(destNodeId); + HKDFParameters hkdfParameters = + new HKDFParameters(keyAgreement, idNonce.extractArray(), info.extractArray()); + HKDFBytesGenerator hkdfBytesGenerator = new HKDFBytesGenerator(digest); + hkdfBytesGenerator.init(hkdfParameters); + // initiator-key, recipient-key, auth-resp-key + byte[] hkdfOutputBytes = + new byte[INITIATOR_KEY_LENGTH + RECIPIENT_KEY_LENGTH + AUTH_RESP_KEY_LENGTH]; + hkdfBytesGenerator.generateBytes( + hkdfOutputBytes, 0, INITIATOR_KEY_LENGTH + RECIPIENT_KEY_LENGTH + AUTH_RESP_KEY_LENGTH); + BytesValue hkdfOutput = BytesValue.wrap(hkdfOutputBytes); + BytesValue initiatorKey = hkdfOutput.slice(0, INITIATOR_KEY_LENGTH); + BytesValue recipientKey = hkdfOutput.slice(INITIATOR_KEY_LENGTH, RECIPIENT_KEY_LENGTH); + BytesValue authRespKey = hkdfOutput.slice(INITIATOR_KEY_LENGTH + RECIPIENT_KEY_LENGTH); + return Triplet.with(initiatorKey, recipientKey, authRespKey); + } catch (Exception ex) { + throw new RuntimeException(ex); + } + } + + public static Random getRandom() { + return new SecureRandom(); + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/IdentityScheme.java b/discovery/src/main/java/org/ethereum/beacon/discovery/IdentityScheme.java new file mode 100644 index 000000000..b57b28588 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/IdentityScheme.java @@ -0,0 +1,34 @@ +package org.ethereum.beacon.discovery; + +import java.util.HashMap; +import java.util.Map; + +/** + * Discovery protocol identity schemes + */ +public enum IdentityScheme { + V4("v4"), + V5("v5"); + + private static final Map nameMap = new HashMap<>(); + + static { + for (IdentityScheme scheme : IdentityScheme.values()) { + nameMap.put(scheme.name, scheme); + } + } + + private String name; + + private IdentityScheme(String name) { + this.name = name; + } + + public static IdentityScheme fromString(String name) { + return nameMap.get(name); + } + + public String stringName() { + return name; + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/MessageCode.java b/discovery/src/main/java/org/ethereum/beacon/discovery/MessageCode.java new file mode 100644 index 000000000..261191edd --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/MessageCode.java @@ -0,0 +1,75 @@ +package org.ethereum.beacon.discovery; + +import java.util.HashMap; +import java.util.Map; + +/** + * Discovery protocol message types as described in https://github.com/ethereum/devp2p/blob/master/discv5/discv5-wire.md#protocol-messages + */ +public enum MessageCode { + + /** + * PING checks whether the recipient is alive and informs it about the sender's ENR sequence + * number. + */ + PING(0x01), + + /** PONG is the reply to PING. */ + PONG(0x02), + + /** FINDNODE queries for nodes at the given logarithmic distance from the recipient's node ID. */ + FINDNODE(0x03), + + /** + * NODES is the response to a FINDNODE or TOPICQUERY message. Multiple NODES messages may be sent + * as responses to a single query. + */ + NODES(0x04), + + /** Request for {@link #TICKET} by topic. */ + REQTICKET(0x05), + + /** + * TICKET is the response to REQTICKET. It contains a ticket which can be used to register for the + * requested topic. + */ + TICKET(0x06), + + /** + * REGTOPIC registers the sender for the given topic with a ticket. The ticket must be valid and + * its waiting time must have elapsed before using the ticket. + */ + REGTOPIC(0x07), + + /** REGCONFIRMATION is the response to REGTOPIC. */ + REGCONFIRMATION(0x08), + + /** + * TOPICQUERY requests nodes in the topic queue of the given topic. The response is a NODES + * message containing node records registered for the topic. + */ + TOPICQUERY(0x09); + + private static final Map codeMap = new HashMap<>(); + + static { + for (MessageCode type : MessageCode.values()) { + codeMap.put(type.code, type); + } + } + + private int code; + + private MessageCode(int code) { + this.code = code; + } + + public static MessageCode fromNumber(int i) { + return codeMap.get(i); + } + + public byte byteCode() { + return (byte) code; + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/MessageHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/MessageHandler.java new file mode 100644 index 000000000..902f7a690 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/MessageHandler.java @@ -0,0 +1,34 @@ +package org.ethereum.beacon.discovery; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.ethereum.beacon.discovery.message.DiscoveryMessage; +import org.ethereum.beacon.discovery.message.DiscoveryV5Message; + +public class MessageHandler { + private static final Logger logger = LogManager.getLogger(MessageHandler.class); + private final DiscoveryV5MessageHandler v5MessageHandler; + + public MessageHandler(DiscoveryV5MessageHandler v5MessageHandler) { + this.v5MessageHandler = v5MessageHandler; + } + + public void handleIncoming(DiscoveryMessage message, NodeContext context) { + IdentityScheme identityScheme = message.getIdentityScheme(); + switch (identityScheme) { + case V5: + { + v5MessageHandler.handleMessage((DiscoveryV5Message) message, context); + } + default: + { + String error = + String.format( + "Message %s with identity %s received in context %s is not supported", + message, identityScheme, context); + logger.error(error); + throw new RuntimeException(error); + } + } + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/NetworkPacket.java b/discovery/src/main/java/org/ethereum/beacon/discovery/NetworkPacket.java new file mode 100644 index 000000000..0465a1316 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/NetworkPacket.java @@ -0,0 +1,22 @@ +package org.ethereum.beacon.discovery; + +import org.ethereum.beacon.discovery.enr.NodeRecord; +import org.ethereum.beacon.discovery.packet.Packet; + +public class NetworkPacket { + private final Packet packet; + private final NodeRecord nodeRecord; + + public NetworkPacket(Packet packet, NodeRecord nodeRecord) { + this.packet = packet; + this.nodeRecord = nodeRecord; + } + + public Packet getPacket() { + return packet; + } + + public NodeRecord getNodeRecord() { + return nodeRecord; + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeContext.java b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeContext.java new file mode 100644 index 000000000..e1d88d9e0 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeContext.java @@ -0,0 +1,262 @@ +package org.ethereum.beacon.discovery; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.ethereum.beacon.discovery.enr.NodeRecordV5; +import org.ethereum.beacon.discovery.message.DiscoveryV5Message; +import org.ethereum.beacon.discovery.message.FindNodeMessage; +import org.ethereum.beacon.discovery.packet.AuthHeaderMessagePacket; +import org.ethereum.beacon.discovery.packet.MessagePacket; +import org.ethereum.beacon.discovery.packet.Packet; +import org.ethereum.beacon.discovery.packet.RandomPacket; +import org.ethereum.beacon.discovery.packet.UnknownPacket; +import org.ethereum.beacon.discovery.packet.WhoAreYouPacket; +import org.ethereum.beacon.discovery.storage.AuthTagRepository; +import org.javatuples.Triplet; +import org.web3j.crypto.ECKeyPair; +import tech.pegasys.artemis.util.bytes.Bytes32; +import tech.pegasys.artemis.util.bytes.BytesValue; + +import java.security.SecureRandom; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Random; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Consumer; + +public class NodeContext { + public static final int DEFAULT_DISTANCE = 10; // FIXME: I shouldn't be here + private static final Logger logger = LogManager.getLogger(NodeContext.class); + private final NodeRecordV5 nodeRecord; + private final NodeRecordV5 homeNodeRecord; + private final Bytes32 homeNodeId; + private final AuthTagRepository authTagRepo; + private final NodeTable nodeTable; + private final Consumer outgoing; + private final Random rnd; + private List incomingEvents; + private SessionStatus status = SessionStatus.INITIAL; + private Bytes32 idNonce; + private BytesValue initiatorKey; + private BytesValue authResponseKey; + private MessageHandler messageHandler = null; + private Map requestIdReservations = new ConcurrentHashMap<>(); + + public NodeContext( + NodeRecordV5 nodeRecord, + NodeRecordV5 homeNodeRecord, + NodeTable nodeTable, + AuthTagRepository authTagRepo, + Consumer outgoing, + Random rnd) { + this.nodeRecord = nodeRecord; + this.outgoing = outgoing; + this.authTagRepo = authTagRepo; + this.nodeTable = nodeTable; + this.homeNodeRecord = homeNodeRecord; + this.homeNodeId = homeNodeRecord.getNodeId(); + this.rnd = rnd; + } + + public NodeRecordV5 getNodeRecord() { + return nodeRecord; + } + + public synchronized void addIncomingEvent(UnknownPacket packet) { + try { + logger.trace(() -> String.format("Incoming packet in context %s", this)); + switch (status) { + case INITIAL: + { + // packet it either random or message packet if session is expired + BytesValue authTag = null; + try { + RandomPacket randomPacket = packet.getRandomPacket(); + authTag = randomPacket.getAuthTag(); + } catch (Exception ex) { + // Not fatal, 1st attempt + } + // 2nd attempt + if (authTag == null) { + MessagePacket messagePacket = packet.getMessagePacket(); + authTag = messagePacket.getAuthTag(); + } + authTagRepo.put(authTag, this); + byte[] idNonceBytes = new byte[32]; + Functions.getRandom().nextBytes(idNonceBytes); + this.idNonce = Bytes32.wrap(idNonceBytes); + WhoAreYouPacket whoAreYouPacket = + WhoAreYouPacket.create( + nodeRecord.getNodeId(), authTag, idNonce, nodeRecord.getSeqNumber()); + addOutgoingEvent(whoAreYouPacket); + status = SessionStatus.WHOAREYOU_SENT; + break; + } + case RANDOM_PACKET_SENT: + { + WhoAreYouPacket whoAreYouPacket = packet.getWhoAreYouPacket(); + BytesValue authTag = authTagRepo.getTag(this).get(); + whoAreYouPacket.verify(homeNodeId, authTag); + whoAreYouPacket.getEnrSeq(); // FIXME: Their side enr seq. Do we need it? + byte[] ephemeralKeyBytes = new byte[32]; + Functions.getRandom().nextBytes(ephemeralKeyBytes); + ECKeyPair ephemeralKey = ECKeyPair.create(ephemeralKeyBytes); // TODO: generate + Triplet hkdf = + Functions.hkdf_expand( + homeNodeId, + nodeRecord.getNodeId(), + BytesValue.wrap(ephemeralKey.getPrivateKey().toByteArray()), + whoAreYouPacket.getIdNonce(), + nodeRecord.getPublicKey()); + BytesValue initiatorKey = hkdf.getValue0(); + BytesValue staticNodeKey = hkdf.getValue1(); + BytesValue authResponseKey = hkdf.getValue2(); + + AuthHeaderMessagePacket response = + AuthHeaderMessagePacket.create( + homeNodeId, + nodeRecord.getNodeId(), + authResponseKey, + whoAreYouPacket.getIdNonce(), + staticNodeKey, + homeNodeRecord, + BytesValue.wrap(ephemeralKey.getPublicKey().toByteArray()), + authTag, + initiatorKey, + DiscoveryV5Message.from( + new FindNodeMessage( + getNextRequestId(MessageCode.FINDNODE), DEFAULT_DISTANCE))); + createMessageHandler(initiatorKey, authTag); + addOutgoingEvent(response); + status = SessionStatus.AUTHENTICATED; + break; + } + case WHOAREYOU_SENT: + { + AuthHeaderMessagePacket authHeaderMessagePacket = packet.getAuthHeaderMessagePacket(); + byte[] ephemeralKeyBytes = new byte[32]; + Functions.getRandom().nextBytes(ephemeralKeyBytes); + ECKeyPair ephemeralKey = ECKeyPair.create(ephemeralKeyBytes); + Triplet hkdf = + Functions.hkdf_expand( + homeNodeId, + nodeRecord.getNodeId(), + BytesValue.wrap(ephemeralKey.getPrivateKey().toByteArray()), + idNonce, + nodeRecord.getPublicKey()); + this.initiatorKey = hkdf.getValue0(); + this.authResponseKey = hkdf.getValue2(); + authHeaderMessagePacket.decode(initiatorKey, authResponseKey); + authHeaderMessagePacket.verify(authTagRepo.getTag(this).get(), idNonce); + createMessageHandler(initiatorKey, authHeaderMessagePacket.getAuthTag()); + messageHandler.handleIncoming(authHeaderMessagePacket.getMessage(), this); + status = SessionStatus.AUTHENTICATED; + break; + } + case AUTHENTICATED: + { + MessagePacket messagePacket = packet.getMessagePacket(); + messagePacket.decode(initiatorKey); + messagePacket.verify(authTagRepo.getTag(this).get()); + messageHandler.handleIncoming(messagePacket.getMessage(), this); + break; + } + default: + { + String error = + String.format("Not expected status:%s from node: %s", status, nodeRecord); + logger.error(error); + throw new RuntimeException(error); + } + } + } catch (AssertionError ex) { + logger.info( + String.format( + "Verification not passed for message [%s] from node %s in status %s", + packet, nodeRecord, status)); + } catch (Exception ex) { + logger.info( + String.format( + "Failed to read message [%s] from node %s in status %s", packet, nodeRecord, status)); + } + } + + private void createMessageHandler(BytesValue initiatorKey, BytesValue authTag) { + this.messageHandler = new MessageHandler(new DiscoveryV5MessageHandler(homeNodeId, initiatorKey, authTag, nodeTable)); + } + + /** Sends random packet to start initiation of session with node */ + public void initiate() { + byte[] authTagBytes = new byte[12]; + rnd.nextBytes(authTagBytes); + BytesValue authTag = BytesValue.wrap(authTagBytes); + RandomPacket randomPacket = + RandomPacket.create(homeNodeId, nodeRecord.getNodeId(), authTag, new SecureRandom()); + authTagRepo.put(authTag, this); + this.addOutgoingEvent(randomPacket); + this.status = SessionStatus.RANDOM_PACKET_SENT; + } + + public synchronized void addOutgoingEvent(Packet packet) { + outgoing.accept(packet); + } + + // FIXME: size, algo + /** + * The value selected as request ID must allow for concurrent conversations. Using a timestamp can + * result in parallel conversations with the same id, so this should be avoided. Request IDs also + * prevent replay of responses. Using a simple counter would be fine if the implementation could + * ensure that restarts or even re-installs would increment the counter based on previously saved + * state in all circumstances. The easiest to implement is a random number. + * + *

Request ID is reserved for `messageCode` + */ + private synchronized BytesValue getNextRequestId(MessageCode messageCode) { + byte[] requestId = new byte[12]; + rnd.nextBytes(requestId); + BytesValue wrapped = BytesValue.wrap(requestId); + requestIdReservations.put(wrapped, messageCode); + return wrapped; + } + + public List getIncomingEvents() { + return incomingEvents; + } + + public synchronized boolean isAuthenticated() { + return SessionStatus.AUTHENTICATED.equals(status); + } + + public void cleanup() { + authTagRepo.expire(this); + } + + @Override + public String toString() { + return "NodeContext{" + + "nodeRecord=" + + nodeRecord + + ", homeNodeId=" + + homeNodeId + + ", status=" + + status + + '}'; + } + + public synchronized void clearRequestId(BytesValue requestId, MessageCode messageCode) { + assert requestIdReservations.remove(requestId).equals(messageCode); + } + + public synchronized Optional getRequestId(BytesValue requestId) { + MessageCode messageCode = requestIdReservations.get(requestId); + return messageCode == null ? Optional.empty() : Optional.of(messageCode); + } + + enum SessionStatus { + INITIAL, // other side is trying to connect, or we are initiating (before random packet is sent + WHOAREYOU_SENT, // other side is initiator, we've sent whoareyou in response + RANDOM_PACKET_SENT, // our node is initiator, we've sent random packet + AUTHENTICATED + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeIndex.java b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeIndex.java new file mode 100644 index 000000000..76cf2ed5f --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeIndex.java @@ -0,0 +1,25 @@ +package org.ethereum.beacon.discovery; + +import org.ethereum.beacon.ssz.annotation.SSZ; +import org.ethereum.beacon.ssz.annotation.SSZSerializable; +import tech.pegasys.artemis.ethereum.core.Hash32; + +import java.util.ArrayList; +import java.util.List; + +@SSZSerializable +public class NodeIndex { + @SSZ private List entries; + + public NodeIndex() { + this.entries = new ArrayList<>(); + } + + public List getEntries() { + return entries; + } + + public void setEntries(List entries) { + this.entries = entries; + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeStatus.java b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeStatus.java new file mode 100644 index 000000000..3b001ab40 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeStatus.java @@ -0,0 +1,70 @@ +package org.ethereum.beacon.discovery; + +import org.ethereum.beacon.ssz.access.SSZBasicAccessor; +import org.ethereum.beacon.ssz.access.SSZField; +import org.ethereum.beacon.ssz.access.basic.UIntPrimitive; +import org.ethereum.beacon.ssz.annotation.SSZSerializable; +import org.ethereum.beacon.ssz.visitor.SSZReader; +import tech.pegasys.artemis.util.bytes.Bytes4; + +import java.io.OutputStream; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +@SSZSerializable(basicAccessor = NodeStatus.NodeStatusAccessor.class) +public enum NodeStatus { + ACTIVE(0x01), + SLEEP(0x02), + DEAD(0x03); + + private static final Map codeMap = new HashMap<>(); + + static { + for (NodeStatus type : NodeStatus.values()) { + codeMap.put(type.code, type); + } + } + + private int code; + + private NodeStatus(int code) { + this.code = code; + } + + public static NodeStatus fromNumber(int i) { + return codeMap.get(i); + } + + public byte byteCode() { + return (byte) code; + } + + public static class NodeStatusAccessor extends UIntPrimitive { + @Override + public Set getSupportedClasses() { + return new HashSet() {{add(NodeStatus.class);}}; + } + + @Override + public int getSize(SSZField field) { + return 1; + } + + @Override + public void encode(Object value, SSZField field, OutputStream result) { + NodeStatus nodeStatus = (NodeStatus) value; + SSZField overrided = new SSZField(byte.class, field.getFieldAnnotation(), "uint", 8, field.getName(), field.getGetter()); + super.encode(nodeStatus.byteCode(), overrided, result); + } + + @Override + public Object decode(SSZField field, SSZReader reader) { + SSZField overrided = new SSZField(byte.class, field.getFieldAnnotation(), "uint", 8, field.getName(), field.getGetter()); + int code = (int) super.decode(overrided, reader); + return NodeStatus.fromNumber(code); + } + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeTable.java b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeTable.java new file mode 100644 index 000000000..ae05889df --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeTable.java @@ -0,0 +1,18 @@ +package org.ethereum.beacon.discovery; + +import org.ethereum.beacon.discovery.enr.NodeRecord; +import org.ethereum.beacon.discovery.enr.NodeRecordInfo; +import tech.pegasys.artemis.util.bytes.Bytes32; + +import java.util.List; +import java.util.Optional; + +public interface NodeTable { + void save(NodeRecordInfo node); + + Optional getNode(Bytes32 nodeId); + + List findClosestNodes(Bytes32 nodeId, int logLimit); + + NodeRecord getHomeNode(); +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeTableImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeTableImpl.java new file mode 100644 index 000000000..04a864a67 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeTableImpl.java @@ -0,0 +1,140 @@ +package org.ethereum.beacon.discovery; + +import org.ethereum.beacon.db.source.DataSource; +import org.ethereum.beacon.db.source.HoleyList; +import org.ethereum.beacon.db.source.SingleValueSource; +import org.ethereum.beacon.discovery.enr.NodeRecord; +import org.ethereum.beacon.discovery.enr.NodeRecordInfo; +import org.ethereum.beacon.discovery.enr.NodeRecordV5; +import tech.pegasys.artemis.ethereum.core.Hash32; +import tech.pegasys.artemis.util.bytes.Bytes32; +import tech.pegasys.artemis.util.bytes.BytesValue; + +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; + +public class NodeTableImpl implements NodeTable { + private static final long NUMBER_OF_INDEXES = 256; + private static final int MAXIMUM_INFO_IN_ONE_BYTE = 256; + private static final boolean START_FROM_BEGINNING = true; + private final DataSource nodeTable; + private final HoleyList indexTable; + private final Function nodeKeyFunction; + private final SingleValueSource homeNodeSource; + + public NodeTableImpl( + DataSource nodeTable, + HoleyList indexTable, + SingleValueSource homeNodeSource, + Function hashFunction) { + this.nodeTable = nodeTable; + this.indexTable = indexTable; + this.nodeKeyFunction = + nodeRecord -> { + if (!(nodeRecord instanceof NodeRecordV5)) { + throw new RuntimeException( + ""); // TODO: exception text + } + return hashFunction.apply(((NodeRecordV5) nodeRecord).getPublicKey()); + }; + this.homeNodeSource = homeNodeSource; + } + + private long getNodeIndex(Bytes32 nodeKey) { + int activeBytes = 1; + long required = NUMBER_OF_INDEXES; + while (required > 0) { + if (required == MAXIMUM_INFO_IN_ONE_BYTE) { + required = 0; + } else { + required = required / MAXIMUM_INFO_IN_ONE_BYTE; + } + + if (required > 0) { + activeBytes++; + } + } + + int start = + START_FROM_BEGINNING + ? 0 + : nodeKey.size() - activeBytes; // FIXME: check for +-1 error for end index + BytesValue active = nodeKey.slice(start, activeBytes); + BigInteger activeNumber = new BigInteger(1, active.extractArray()); // FIXME: is signum ok? + BigInteger index = + activeNumber.mod(BigInteger.valueOf(NUMBER_OF_INDEXES)); // FIXME: do we require BI here? + + return index.longValue(); + } + + @Override + public void save(NodeRecordInfo node) { + Hash32 nodeKey = nodeKeyFunction.apply(node.getNode()); + nodeTable.put(nodeKey, node); + NodeIndex activeIndex = indexTable.get(getNodeIndex(nodeKey)).orElseGet(NodeIndex::new); + List nodes = activeIndex.getEntries(); + if (!nodes.contains(nodeKey)) { + nodes.add(nodeKey); + indexTable.put(getNodeIndex(nodeKey), activeIndex); + } + } + + @Override + public Optional getNode(Bytes32 nodeId) { + return nodeTable.get(Hash32.wrap(nodeId)); + } + + @Override + public List findClosestNodes(Bytes32 nodeId, int logLimit) { + long start = getNodeIndex(nodeId); + boolean limitReached = false; + long currentIndexUp = start; + long currentIndexDown = start; + Set res = new HashSet<>(); + while (!limitReached) { + Optional upNodesOptional = indexTable.get(currentIndexUp); + Optional downNodesOptional = indexTable.get(currentIndexDown); + if (upNodesOptional.isPresent()) { + NodeIndex upNodes = upNodesOptional.get(); + for (Hash32 currentNodeId : upNodes.getEntries()) { + if (logDistance(currentNodeId, nodeId) >= logLimit) { + limitReached = true; + break; + } else { + res.add(getNode(currentNodeId).get()); + } + } + } + if (downNodesOptional.isPresent()) { + NodeIndex downNodes = downNodesOptional.get(); + for (Hash32 currentNodeId : downNodes.getEntries()) { + if (logDistance(currentNodeId, nodeId) >= logLimit) { + limitReached = true; + break; + } else { + res.add(getNode(currentNodeId).get()); + } + } + } + currentIndexUp++; // FIXME: bounds + currentIndexDown--; // FIXME: bounds + } + + return new ArrayList<>(res); + } + + private int logDistance(Bytes32 nodeId1, Bytes32 nodeId2) { + // TODO: real formula + return Math.abs((nodeId1.get(0) & 0xFF) - (nodeId2.get(0) & 0xFF)); + } + + @Override + public NodeRecord getHomeNode() { + return homeNodeSource.get().map(NodeRecordInfo::getNode).orElse(null); + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/RefreshTask.java b/discovery/src/main/java/org/ethereum/beacon/discovery/RefreshTask.java new file mode 100644 index 000000000..0f198abf9 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/RefreshTask.java @@ -0,0 +1,36 @@ +package org.ethereum.beacon.discovery; + +import com.google.common.collect.Sets; +import org.ethereum.beacon.discovery.enr.NodeRecordInfo; +import org.ethereum.beacon.schedulers.Scheduler; +import tech.pegasys.artemis.util.bytes.Bytes33; + +import java.util.Set; + +public class RefreshTask { + Scheduler scheduler; + Set currentTasks = Sets.newConcurrentHashSet(); + + public RefreshTask(Scheduler scheduler) { + this.scheduler = scheduler; + } + + public void add( + NodeRecordInfo nodeRecord, Runnable successCallback, Runnable failCallback) { + synchronized (this) { + if (currentTasks.contains(nodeRecord.getNode().getPublicKey())) { + return; + } + currentTasks.add(nodeRecord.getNode().getPublicKey()); + } + + scheduler.execute( + () -> { + // try to connect; + // if success: + // successCallback.run(); + failCallback.run(); + currentTasks.remove(nodeRecord.getNode().getPublicKey()); + }); + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecord.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecord.java new file mode 100644 index 000000000..a77a1b415 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecord.java @@ -0,0 +1,162 @@ +package org.ethereum.beacon.discovery.enr; + +import org.ethereum.beacon.discovery.IdentityScheme; +import org.ethereum.beacon.ssz.access.SSZField; +import org.ethereum.beacon.ssz.access.list.BytesValueAccessor; +import org.ethereum.beacon.ssz.annotation.SSZSerializable; +import org.ethereum.beacon.ssz.type.SSZType; +import org.web3j.rlp.RlpDecoder; +import org.web3j.rlp.RlpList; +import org.web3j.rlp.RlpString; +import org.web3j.rlp.RlpType; +import tech.pegasys.artemis.util.bytes.Bytes32; +import tech.pegasys.artemis.util.bytes.BytesValue; + +import java.math.BigInteger; +import java.util.Base64; +import java.util.List; +import java.util.function.Function; + +/** + * Ethereum Node Record + * + *

Node record as described in , NodeRecord> nodeRecordV4Creator = + fields -> { + NodeRecordV4.Builder builder = + NodeRecordV4.Builder.empty() + .withSignature(BytesValue.fromHexString(((RlpString) values.get(0)).asString())) + .withSeqNumber( + new BigInteger( + 1, + BytesValue.fromHexString(((RlpString) values.get(1)).asString()) + .extractArray()) + .longValue()); + for (int i = 4; i < values.size(); i += 2) { + builder = + builder.withKeyField( + new String(((RlpString) values.get(i)).getBytes()), + (RlpString) values.get(i + 1)); + } + + return builder.build(); + }; + Function, NodeRecord> nodeRecordV5Creator = + fields -> { + NodeRecordV5.Builder builder = + NodeRecordV5.Builder.empty() + .withSignature(BytesValue.fromHexString(((RlpString) values.get(0)).asString())) + .withSeqNumber( + new BigInteger( + 1, + BytesValue.fromHexString(((RlpString) values.get(1)).asString()) + .extractArray()) + .longValue()); + for (int i = 4; i < values.size(); i += 2) { + builder = + builder.withKeyField( + new String(((RlpString) values.get(i)).getBytes()), + (RlpString) values.get(i + 1)); + } + + return builder.build(); + }; + IdentityScheme nodeIdentity = IdentityScheme.fromString(new String(idVersion.getBytes())); + if (nodeIdentity == null) { + throw new RuntimeException( + String.format( + "Unknown node identity scheme '%s', couldn't create node record.", + idVersion.asString())); + } + switch (nodeIdentity) { + case V4: + { + return nodeRecordV4Creator.apply(values); + } + case V5: + { + return nodeRecordV5Creator.apply(values); + } + } + + return null; + } + + static NodeRecord fromBytes(BytesValue bytes) { + return fromBytes(bytes.extractArray()); + } + + IdentityScheme getIdentityScheme(); + + BytesValue serialize(); + + default String asBase64() { + return new String(Base64.getUrlEncoder().encode(serialize().extractArray())); + } + + Bytes32 getNodeId(); + + class NodeRecordAccessor extends BytesValueAccessor { + @Override + public int getChildrenCount(Object value) { + return ((NodeRecord) value).serialize().size(); + } + + @Override + public Object getChildValue(Object value, int idx) { + return ((NodeRecord) value).serialize().get(idx); + } + + @Override + public boolean isSupported(SSZField field) { + return NodeRecord.class.isAssignableFrom(field.getRawClass()); + } + + @Override + public CompositeInstanceAccessor getInstanceAccessor(SSZField compositeDescriptor) { + return this; + } + + @Override + public ListInstanceBuilder createInstanceBuilder(SSZType sszType) { + return new SimpleInstanceBuilder() { + @Override + protected Object buildImpl(List children) { + byte[] vals = new byte[children.size()]; + for (int i = 0; i < children.size(); i++) { + vals[i] = ((Number) children.get(i)).byteValue(); + } + BytesValue value = BytesValue.wrap(vals); + + return NodeRecord.fromBytes(value); + } + }; + } + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordInfo.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordInfo.java new file mode 100644 index 000000000..438669349 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordInfo.java @@ -0,0 +1,63 @@ +package org.ethereum.beacon.discovery.enr; + +import com.google.common.base.Objects; +import org.ethereum.beacon.discovery.NodeStatus; +import org.ethereum.beacon.ssz.annotation.SSZ; +import org.ethereum.beacon.ssz.annotation.SSZSerializable; + +/** + * Container for {@link NodeRecord}. Also saves all necessary data about presence of this node and + * last test of its availability + */ +@SSZSerializable +public class NodeRecordInfo { + @SSZ private final NodeRecord node; + @SSZ private final Long lastRetry; + @SSZ private final NodeStatus status; + + @SSZ(type = "uint8") + private final Integer retry; + + public NodeRecordInfo(NodeRecord node, Long lastRetry, NodeStatus status, Integer retry) { + this.node = node; + this.lastRetry = lastRetry; + this.status = status; + this.retry = retry; + } + + public static NodeRecordInfo createDefault(NodeRecord nodeRecord) { + return new NodeRecordInfo(nodeRecord, -1L, NodeStatus.ACTIVE, 0); + } + + public NodeRecordV5 getNode() { + return (NodeRecordV5) node; + } + + public Long getLastRetry() { + return lastRetry; + } + + public NodeStatus getStatus() { + return status; + } + + public Integer getRetry() { + return retry; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + NodeRecordInfo that = (NodeRecordInfo) o; + return Objects.equal(node, that.node) + && Objects.equal(lastRetry, that.lastRetry) + && status == that.status + && Objects.equal(retry, that.retry); + } + + @Override + public int hashCode() { + return Objects.hashCode(node, lastRetry, status, retry); + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV4.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV4.java new file mode 100644 index 000000000..991c79460 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV4.java @@ -0,0 +1,334 @@ +package org.ethereum.beacon.discovery.enr; + +import com.google.common.base.Objects; +import org.ethereum.beacon.crypto.Hashes; +import org.ethereum.beacon.discovery.IdentityScheme; +import org.javatuples.Pair; +import org.web3j.rlp.RlpEncoder; +import org.web3j.rlp.RlpList; +import org.web3j.rlp.RlpString; +import org.web3j.rlp.RlpType; +import tech.pegasys.artemis.util.bytes.Bytes16; +import tech.pegasys.artemis.util.bytes.Bytes32; +import tech.pegasys.artemis.util.bytes.Bytes33; +import tech.pegasys.artemis.util.bytes.Bytes4; +import tech.pegasys.artemis.util.bytes.BytesValue; + +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; + +/** Node record for V4 protocol */ +public class NodeRecordV4 implements NodeRecord { + // id name of identity scheme, e.g. “v4” + private static final IdentityScheme identityScheme = IdentityScheme.V4; + // secp256k1 compressed secp256k1 public key, 33 bytes + private Bytes33 publicKey; + // ip IPv4 address, 4 bytes + private Bytes4 ipV4address; + // tcp TCP port, big endian integer + private Integer tcpPort; + // udp UDP port, big endian integer + private Integer udpPort; + // ip6 IPv6 address, 16 bytes + private Bytes16 ipV6address; + // tcp6 IPv6-specific TCP port, big endian integer + private Integer tcpV6Port; + // udp6 IPv6-specific UDP port, big endian integer + private Integer udpV6Port; + + private Long seqNumber; + + private BytesValue signature; + + private NodeRecordV4( + Bytes33 publicKey, + Bytes4 ipV4address, + Integer tcpPort, + Integer udpPort, + Bytes16 ipV6address, + Integer tcpV6Port, + Integer udpV6Port) { + this.publicKey = publicKey; + this.ipV4address = ipV4address; + this.tcpPort = tcpPort; + this.udpPort = udpPort; + this.ipV6address = ipV6address; + this.tcpV6Port = tcpV6Port; + this.udpV6Port = udpV6Port; + } + + private NodeRecordV4() {} + + public static NodeRecordV4 fromValues( + Bytes33 publicKey, + Bytes4 ipV4address, + Integer tcpPort, + Integer udpPort, + Bytes16 ipV6address, + Integer tcpV6Port, + Integer udpV6Port) { + return new NodeRecordV4( + publicKey, ipV4address, tcpPort, udpPort, ipV6address, tcpV6Port, udpV6Port); + } + + public IdentityScheme getIdentityScheme() { + return identityScheme; + } + + public Bytes33 getPublicKey() { + return publicKey; + } + + public void setPublicKey(Bytes33 publicKey) { + this.publicKey = publicKey; + } + + public Bytes4 getIpV4address() { + return ipV4address; + } + + public void setIpV4address(Bytes4 ipV4address) { + this.ipV4address = ipV4address; + } + + public Integer getTcpPort() { + return tcpPort; + } + + public void setTcpPort(Integer tcpPort) { + this.tcpPort = tcpPort; + } + + public Integer getUdpPort() { + return udpPort; + } + + public void setUdpPort(Integer udpPort) { + this.udpPort = udpPort; + } + + public Bytes16 getIpV6address() { + return ipV6address; + } + + public void setIpV6address(Bytes16 ipV6address) { + this.ipV6address = ipV6address; + } + + public Integer getTcpV6Port() { + return tcpV6Port; + } + + public void setTcpV6Port(Integer tcpV6Port) { + this.tcpV6Port = tcpV6Port; + } + + public Integer getUdpV6Port() { + return udpV6Port; + } + + public void setUdpV6Port(Integer udpV6Port) { + this.udpV6Port = udpV6Port; + } + + public Long getSeqNumber() { + return seqNumber; + } + + public void setSeqNumber(Long seqNumber) { + this.seqNumber = seqNumber; + } + + public BytesValue getSignature() { + return signature; + } + + public void setSignature(BytesValue signature) { + this.signature = signature; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + NodeRecordV4 that = (NodeRecordV4) o; + return Objects.equal(publicKey, that.publicKey) + && Objects.equal(ipV4address, that.ipV4address) + && Objects.equal(tcpPort, that.tcpPort) + && Objects.equal(udpPort, that.udpPort) + && Objects.equal(ipV6address, that.ipV6address) + && Objects.equal(tcpV6Port, that.tcpV6Port) + && Objects.equal(udpV6Port, that.udpV6Port) + && Objects.equal(seqNumber, that.seqNumber) + && Objects.equal(signature, that.signature); + } + + @Override + public int hashCode() { + return Objects.hashCode( + publicKey, + ipV4address, + tcpPort, + udpPort, + ipV6address, + tcpV6Port, + udpV6Port, + seqNumber, + signature); + } + + @Override + public BytesValue serialize() { + assert getSignature() != null; + assert getSeqNumber() != null; + + // content = [seq, k, v, ...] + // signature = sign(content) + // record = [signature, seq, k, v, ...] + List values = new ArrayList<>(); + values.add(RlpString.create(getSignature().extractArray())); + values.add(RlpString.create(getSeqNumber())); + values.add(RlpString.create("id")); + values.add(RlpString.create(getIdentityScheme().stringName())); + if (getPublicKey() != null) { + values.add(RlpString.create("secp256k1")); + values.add(RlpString.create(getPublicKey().extractArray())); + } + if (getIpV4address() != null) { + values.add(RlpString.create("ip")); + values.add(RlpString.create(getIpV4address().extractArray())); + } + if (getTcpPort() != null) { + values.add(RlpString.create("tcp")); + values.add(RlpString.create(getTcpPort())); + } + if (getUdpPort() != null) { + values.add(RlpString.create("udp")); + values.add(RlpString.create(getUdpPort())); + } + if (getIpV6address() != null) { + values.add(RlpString.create("ip6")); + values.add(RlpString.create(getIpV6address().extractArray())); + } + if (getTcpV6Port() != null) { + values.add(RlpString.create("tcp6")); + values.add(RlpString.create(getTcpV6Port())); + } + if (getUdpV6Port() != null) { + values.add(RlpString.create("udp6")); + values.add(RlpString.create(getUdpV6Port())); + } + + byte[] bytes = RlpEncoder.encode(new RlpList(values)); + assert bytes.length <= 300; + return BytesValue.wrap(bytes); + } + + @Override + public Bytes32 getNodeId() { + return Hashes.sha256(getPublicKey()); + } + + public static class Builder { + private static final Map, Builder>> fieldFillersV4 = + new HashMap<>(); + + static { + fieldFillersV4.put( + "ip", + objects -> + objects + .getValue0() + .withIpV4Address( + BytesValue.fromHexString(((RlpString) objects.getValue1()).asString()))); + fieldFillersV4.put( + "secp256k1", + objects -> + objects + .getValue0() + .withSecp256k1( + BytesValue.fromHexString(((RlpString) objects.getValue1()).asString()))); + fieldFillersV4.put( + "udp", + objects -> + objects + .getValue0() + .withUdpPort( + new BigInteger( + 1, + BytesValue.fromHexString(((RlpString) objects.getValue1()).asString()) + .extractArray()) + .intValue())); + } + + private Bytes4 ipV4Address; + private Bytes33 secp256k1; + private Integer tcpPort; + private Integer udpPort; + private Long seqNumber; + private BytesValue signature; + + private Builder() {} + + public static Builder empty() { + return new Builder(); + } + + public Builder withIpV4Address(BytesValue ipV4Address) { + this.ipV4Address = Bytes4.wrap(ipV4Address, 0); + return this; + } + + public Builder withTcpPort(Integer port) { + this.tcpPort = port; + return this; + } + + public Builder withUdpPort(Integer port) { + this.udpPort = port; + return this; + } + + public Builder withSecp256k1(BytesValue bytes) { + this.secp256k1 = Bytes33.wrap(bytes, 0); + return this; + } + + public Builder withSeqNumber(Long seqNumber) { + this.seqNumber = seqNumber; + return this; + } + + public Builder withSignature(BytesValue signature) { + this.signature = signature; + return this; + } + + public Builder withKeyField(String key, RlpString value) { + Function, NodeRecordV4.Builder> fieldFiller = + fieldFillersV4.get(key); + if (fieldFiller == null) { + throw new RuntimeException(String.format("Couldn't find filler for V4 field '%s'", key)); + } + return fieldFiller.apply(Pair.with(this, value)); + } + + public NodeRecordV4 build() { + assert seqNumber != null; + assert secp256k1 != null; + + NodeRecordV4 nodeRecord = new NodeRecordV4(); + nodeRecord.setIpV4address(ipV4Address); + nodeRecord.setUdpPort(udpPort); + nodeRecord.setTcpPort(tcpPort); + nodeRecord.setSeqNumber(seqNumber); + nodeRecord.setSignature(signature); + nodeRecord.setPublicKey(secp256k1); + return nodeRecord; + } + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV5.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV5.java new file mode 100644 index 000000000..be8ff6439 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV5.java @@ -0,0 +1,341 @@ +package org.ethereum.beacon.discovery.enr; + +import com.google.common.base.Objects; +import org.ethereum.beacon.crypto.Hashes; +import org.ethereum.beacon.discovery.IdentityScheme; +import org.javatuples.Pair; +import org.web3j.rlp.RlpEncoder; +import org.web3j.rlp.RlpList; +import org.web3j.rlp.RlpString; +import org.web3j.rlp.RlpType; +import tech.pegasys.artemis.util.bytes.Bytes16; +import tech.pegasys.artemis.util.bytes.Bytes32; +import tech.pegasys.artemis.util.bytes.Bytes33; +import tech.pegasys.artemis.util.bytes.Bytes4; +import tech.pegasys.artemis.util.bytes.Bytes96; +import tech.pegasys.artemis.util.bytes.BytesValue; + +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; + +/** Node record for V5 protocol */ +public class NodeRecordV5 implements NodeRecord { + // id name of identity scheme, e.g. “v4” + private static final IdentityScheme identityScheme = IdentityScheme.V5; + public static Function NODE_ID_FUNCTION = + nodeRecordV5 -> Hashes.sha256(nodeRecordV5.getPublicKey()); + // secp256k1 compressed secp256k1 public key, 33 bytes + private Bytes33 publicKey; + // ip IPv4 address, 4 bytes + private Bytes4 ipV4address; + // tcp TCP port, big endian integer + private Integer tcpPort; + // udp UDP port, big endian integer + private Integer udpPort; + // ip6 IPv6 address, 16 bytes + private Bytes16 ipV6address; + // tcp6 IPv6-specific TCP port, big endian integer + private Integer tcpV6Port; + // udp6 IPv6-specific UDP port, big endian integer + private Integer udpV6Port; + private Long seqNumber; + private BytesValue signature; + + private NodeRecordV5( + Bytes33 publicKey, + Bytes4 ipV4address, + Integer tcpPort, + Integer udpPort, + Bytes16 ipV6address, + Integer tcpV6Port, + Integer udpV6Port) { + this.publicKey = publicKey; + this.ipV4address = ipV4address; + this.tcpPort = tcpPort; + this.udpPort = udpPort; + this.ipV6address = ipV6address; + this.tcpV6Port = tcpV6Port; + this.udpV6Port = udpV6Port; + } + + private NodeRecordV5() {} + + public NodeRecordV5(BytesValue bytes) { + NodeRecord.fromBytes(bytes); + } + + public static NodeRecordV5 fromValues( + Bytes33 publicKey, + Bytes4 ipV4address, + Integer tcpPort, + Integer udpPort, + Bytes16 ipV6address, + Integer tcpV6Port, + Integer udpV6Port) { + return new NodeRecordV5( + publicKey, ipV4address, tcpPort, udpPort, ipV6address, tcpV6Port, udpV6Port); + } + + public IdentityScheme getIdentityScheme() { + return identityScheme; + } + + public Bytes33 getPublicKey() { + return publicKey; + } + + public void setPublicKey(Bytes33 publicKey) { + this.publicKey = publicKey; + } + + public Bytes4 getIpV4address() { + return ipV4address; + } + + public void setIpV4address(Bytes4 ipV4address) { + this.ipV4address = ipV4address; + } + + public Integer getTcpPort() { + return tcpPort; + } + + public void setTcpPort(Integer tcpPort) { + this.tcpPort = tcpPort; + } + + public Integer getUdpPort() { + return udpPort; + } + + public void setUdpPort(Integer udpPort) { + this.udpPort = udpPort; + } + + public Bytes16 getIpV6address() { + return ipV6address; + } + + public void setIpV6address(Bytes16 ipV6address) { + this.ipV6address = ipV6address; + } + + public Integer getTcpV6Port() { + return tcpV6Port; + } + + public void setTcpV6Port(Integer tcpV6Port) { + this.tcpV6Port = tcpV6Port; + } + + public Integer getUdpV6Port() { + return udpV6Port; + } + + public void setUdpV6Port(Integer udpV6Port) { + this.udpV6Port = udpV6Port; + } + + public Long getSeqNumber() { + return seqNumber; + } + + public void setSeqNumber(Long seqNumber) { + this.seqNumber = seqNumber; + } + + public BytesValue getSignature() { + return signature; + } + + public void setSignature(BytesValue signature) { + this.signature = signature; + } + + /** Uses empty 96 bytes signature when no signature is provided\ */ + @Override + public BytesValue serialize() { + // content = [seq, k, v, ...] + // signature = sign(content) + // record = [signature, seq, k, v, ...] + List values = new ArrayList<>(); + if (getSignature() != null) { + values.add(RlpString.create(getSignature().extractArray())); + } else { + values.add(RlpString.create(Bytes96.ZERO.extractArray())); // FIXME: is it ok? + } + values.add(RlpString.create(getSeqNumber())); + values.add(RlpString.create("id")); + values.add(RlpString.create(getIdentityScheme().stringName())); + if (getPublicKey() != null) { + values.add(RlpString.create("secp256k1")); + values.add(RlpString.create(getPublicKey().extractArray())); + } + if (getIpV4address() != null) { + values.add(RlpString.create("ip")); + values.add(RlpString.create(getIpV4address().extractArray())); + } + if (getTcpPort() != null) { + values.add(RlpString.create("tcp")); + values.add(RlpString.create(getTcpPort())); + } + if (getUdpPort() != null) { + values.add(RlpString.create("udp")); + values.add(RlpString.create(getUdpPort())); + } + if (getIpV6address() != null) { + values.add(RlpString.create("ip6")); + values.add(RlpString.create(getIpV6address().extractArray())); + } + if (getTcpV6Port() != null) { + values.add(RlpString.create("tcp6")); + values.add(RlpString.create(getTcpV6Port())); + } + if (getUdpV6Port() != null) { + values.add(RlpString.create("udp6")); + values.add(RlpString.create(getUdpV6Port())); + } + + byte[] bytes = RlpEncoder.encode(new RlpList(values)); + assert bytes.length <= 300; + return BytesValue.wrap(bytes); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + NodeRecordV5 that = (NodeRecordV5) o; + return Objects.equal(publicKey, that.publicKey) + && Objects.equal(ipV4address, that.ipV4address) + && Objects.equal(tcpPort, that.tcpPort) + && Objects.equal(udpPort, that.udpPort) + && Objects.equal(ipV6address, that.ipV6address) + && Objects.equal(tcpV6Port, that.tcpV6Port) + && Objects.equal(udpV6Port, that.udpV6Port) + && Objects.equal(seqNumber, that.seqNumber) + && Objects.equal(signature, that.signature); + } + + @Override + public int hashCode() { + return Objects.hashCode( + publicKey, + ipV4address, + tcpPort, + udpPort, + ipV6address, + tcpV6Port, + udpV6Port, + seqNumber, + signature); + } + + @Override + public Bytes32 getNodeId() { + return NODE_ID_FUNCTION.apply(this); + } + + public static class Builder { + private static final Map, Builder>> fieldFillersV5 = + new HashMap<>(); + + static { + fieldFillersV5.put( + "ip", + objects -> + objects + .getValue0() + .withIpV4Address( + BytesValue.fromHexString(((RlpString) objects.getValue1()).asString()))); + fieldFillersV5.put( + "secp256k1", + objects -> + objects + .getValue0() + .withSecp256k1( + BytesValue.fromHexString(((RlpString) objects.getValue1()).asString()))); + fieldFillersV5.put( + "udp", + objects -> + objects + .getValue0() + .withUdpPort( + new BigInteger( + 1, + BytesValue.fromHexString(((RlpString) objects.getValue1()).asString()) + .extractArray()) + .intValue())); + } + + private Bytes4 ipV4Address; + private Bytes33 secp256k1; + private Integer tcpPort; + private Integer udpPort; + private Long seqNumber; + private BytesValue signature; + + private Builder() {} + + public static Builder empty() { + return new Builder(); + } + + public Builder withIpV4Address(BytesValue ipV4Address) { + this.ipV4Address = Bytes4.wrap(ipV4Address, 0); + return this; + } + + public Builder withTcpPort(Integer port) { + this.tcpPort = port; + return this; + } + + public Builder withUdpPort(Integer port) { + this.udpPort = port; + return this; + } + + public Builder withSecp256k1(BytesValue bytes) { + this.secp256k1 = Bytes33.wrap(bytes, 0); + return this; + } + + public Builder withSeqNumber(Long seqNumber) { + this.seqNumber = seqNumber; + return this; + } + + public Builder withSignature(BytesValue signature) { + this.signature = signature; + return this; + } + + public Builder withKeyField(String key, RlpString value) { + Function, NodeRecordV5.Builder> fieldFiller = + fieldFillersV5.get(key); + if (fieldFiller == null) { + throw new RuntimeException(String.format("Couldn't find filler for V4 field '%s'", key)); + } + return fieldFiller.apply(Pair.with(this, value)); + } + + public NodeRecordV5 build() { + assert seqNumber != null; + assert secp256k1 != null; + + NodeRecordV5 nodeRecord = new NodeRecordV5(); + nodeRecord.setIpV4address(ipV4Address); + nodeRecord.setUdpPort(udpPort); + nodeRecord.setTcpPort(tcpPort); + nodeRecord.setSeqNumber(seqNumber); + nodeRecord.setSignature(signature); + nodeRecord.setPublicKey(secp256k1); + return nodeRecord; + } + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/DiscoveryMessage.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/DiscoveryMessage.java new file mode 100644 index 000000000..fd05f81f4 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/DiscoveryMessage.java @@ -0,0 +1,10 @@ +package org.ethereum.beacon.discovery.message; + +import org.ethereum.beacon.discovery.IdentityScheme; +import tech.pegasys.artemis.util.bytes.BytesValue; + +public interface DiscoveryMessage { + IdentityScheme getIdentityScheme(); + + BytesValue getBytes(); +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/DiscoveryV5Message.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/DiscoveryV5Message.java new file mode 100644 index 000000000..491f1af0d --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/DiscoveryV5Message.java @@ -0,0 +1,100 @@ +package org.ethereum.beacon.discovery.message; + +import org.ethereum.beacon.discovery.IdentityScheme; +import org.ethereum.beacon.discovery.MessageCode; +import org.ethereum.beacon.discovery.enr.NodeRecord; +import org.ethereum.beacon.discovery.enr.NodeRecordV5; +import org.web3j.rlp.RlpDecoder; +import org.web3j.rlp.RlpList; +import org.web3j.rlp.RlpString; +import org.web3j.rlp.RlpType; +import tech.pegasys.artemis.util.bytes.BytesValue; + +import java.util.List; +import java.util.stream.Collectors; + +public class DiscoveryV5Message implements DiscoveryMessage { + private final BytesValue bytes; + private List payload = null; + + public DiscoveryV5Message(BytesValue bytes) { + this.bytes = bytes; + } + + @Override + public BytesValue getBytes() { + return bytes; + } + + @Override + public IdentityScheme getIdentityScheme() { + return IdentityScheme.V5; + } + + public MessageCode getCode() { + return MessageCode.fromNumber(getBytes().get(0)); + } + + private synchronized void decode() { + if (payload != null) { + return; + } + this.payload = + ((RlpList) RlpDecoder.decode(getBytes().slice(1).extractArray()).getValues().get(0)) + .getValues(); + } + + public BytesValue getRequestId() { + decode(); + return BytesValue.wrap(((RlpString) payload.get(0)).getBytes()); + } + + public V5Message create() { + decode(); + MessageCode code = MessageCode.fromNumber(getBytes().get(0)); + switch (code) { + case PING: + { + return new PingMessage( + BytesValue.wrap(((RlpString) payload.get(0)).getBytes()), + ((RlpString) payload.get(1)).asPositiveBigInteger().longValueExact()); + } + case PONG: + { + return new PongMessage( + BytesValue.wrap(((RlpString) payload.get(0)).getBytes()), + ((RlpString) payload.get(1)).asPositiveBigInteger().longValueExact(), + BytesValue.wrap(((RlpString) payload.get(2)).getBytes()), + ((RlpString) payload.get(3)).asPositiveBigInteger().intValueExact()); + } + case FINDNODE: + { + return new FindNodeMessage( + BytesValue.wrap(((RlpString) payload.get(0)).getBytes()), + ((RlpString) payload.get(1)).asPositiveBigInteger().intValueExact()); + } + case NODES: + { + RlpList nodeRecords = (RlpList) payload.get(2); + return new NodesMessage( + BytesValue.wrap(((RlpString) payload.get(0)).getBytes()), + ((RlpString) payload.get(1)).asPositiveBigInteger().intValueExact(), + () -> + nodeRecords.getValues().stream() + .map(rs -> (NodeRecordV5) (NodeRecord.fromBytes(((RlpString) rs).getBytes()))) + .collect(Collectors.toList()), + nodeRecords.getValues().size()); + } + default: + { + throw new RuntimeException( + String.format( + "Creation of discovery V5 messages from code %s is not supported", code)); + } + } + } + + public static DiscoveryV5Message from(V5Message v5Message) { + return new DiscoveryV5Message(v5Message.getBytes()); + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/FindNodeMessage.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/FindNodeMessage.java new file mode 100644 index 000000000..f61abea34 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/FindNodeMessage.java @@ -0,0 +1,69 @@ +package org.ethereum.beacon.discovery.message; + +import com.google.common.base.Objects; +import org.ethereum.beacon.discovery.MessageCode; +import org.web3j.rlp.RlpEncoder; +import org.web3j.rlp.RlpList; +import org.web3j.rlp.RlpString; +import tech.pegasys.artemis.util.bytes.Bytes1; +import tech.pegasys.artemis.util.bytes.BytesValue; + +/** + * FINDNODE queries for nodes at the given logarithmic distance from the recipient's node ID. The + * node IDs of all nodes in the response must have a shared prefix length of distance with the + * recipient's node ID. A request with distance 0 should return the recipient's current record as + * the only result. + */ +public class FindNodeMessage implements V5Message { + // Unique request id + private final BytesValue requestId; + // The requested log2 distance, a positive integer + private final Integer distance; + + public FindNodeMessage(BytesValue requestId, Integer distance) { + this.requestId = requestId; + this.distance = distance; + } + + @Override + public BytesValue getRequestId() { + return requestId; + } + + public Integer getDistance() { + return distance; + } + + @Override + public BytesValue getBytes() { + return Bytes1.intToBytes1(MessageCode.FINDNODE.byteCode()) + .concat( + BytesValue.wrap( + RlpEncoder.encode( + new RlpList( + RlpString.create(requestId.extractArray()), + RlpString.create(distance))))); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + FindNodeMessage that = (FindNodeMessage) o; + return Objects.equal(requestId, that.requestId) && + Objects.equal(distance, that.distance); + } + + @Override + public int hashCode() { + return Objects.hashCode(requestId, distance); + } + + @Override + public String toString() { + return "FindNodeMessage{" + + "requestId=" + requestId + + ", distance=" + distance + + '}'; + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/NodesMessage.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/NodesMessage.java new file mode 100644 index 000000000..7e748c178 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/NodesMessage.java @@ -0,0 +1,103 @@ +package org.ethereum.beacon.discovery.message; + +import com.google.common.base.Objects; +import org.ethereum.beacon.discovery.MessageCode; +import org.ethereum.beacon.discovery.enr.NodeRecordV5; +import org.web3j.rlp.RlpEncoder; +import org.web3j.rlp.RlpList; +import org.web3j.rlp.RlpString; +import tech.pegasys.artemis.util.bytes.Bytes1; +import tech.pegasys.artemis.util.bytes.BytesValue; + +import java.util.List; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +/** + * NODES is the response to a FINDNODE or TOPICQUERY message. Multiple NODES messages may be sent as + * responses to a single query. + */ +public class NodesMessage implements V5Message { + // Unique request id + private final BytesValue requestId; + // Total number of responses to the request + private final Integer total; + // List of nodes upon request + private final Supplier> nodeRecordsSupplier; + // Size of nodes in current response + private final Integer nodeRecordsSize; + private List nodeRecords = null; + + public NodesMessage( + BytesValue requestId, + Integer total, + Supplier> nodeRecordsSupplier, + Integer nodeRecordsSize) { + this.requestId = requestId; + this.total = total; + this.nodeRecordsSupplier = nodeRecordsSupplier; + this.nodeRecordsSize = nodeRecordsSize; + } + + @Override + public BytesValue getRequestId() { + return requestId; + } + + public Integer getTotal() { + return total; + } + + public synchronized List getNodeRecords() { + if (nodeRecords == null) { + this.nodeRecords = nodeRecordsSupplier.get(); + } + return nodeRecords; + } + + public Integer getNodeRecordsSize() { + return nodeRecordsSize; + } + + @Override + public BytesValue getBytes() { + return Bytes1.intToBytes1(MessageCode.PONG.byteCode()) + .concat( + BytesValue.wrap( + RlpEncoder.encode( + new RlpList( + RlpString.create(requestId.extractArray()), + RlpString.create(total), + new RlpList( + getNodeRecords().stream() + .map(n -> RlpString.create(n.serialize().extractArray())) + .collect(Collectors.toList())))))); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + NodesMessage that = (NodesMessage) o; + return Objects.equal(requestId, that.requestId) + && Objects.equal(total, that.total) + && Objects.equal(nodeRecordsSize, that.nodeRecordsSize); + } + + @Override + public int hashCode() { + return Objects.hashCode(requestId, total, nodeRecordsSize); + } + + @Override + public String toString() { + return "NodesMessage{" + + "requestId=" + + requestId + + ", total=" + + total + + ", nodeRecordsSize=" + + nodeRecordsSize + + '}'; + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/PingMessage.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/PingMessage.java new file mode 100644 index 000000000..be3cd726c --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/PingMessage.java @@ -0,0 +1,61 @@ +package org.ethereum.beacon.discovery.message; + +import com.google.common.base.Objects; +import org.ethereum.beacon.discovery.MessageCode; +import org.web3j.rlp.RlpEncoder; +import org.web3j.rlp.RlpList; +import org.web3j.rlp.RlpString; +import tech.pegasys.artemis.util.bytes.Bytes1; +import tech.pegasys.artemis.util.bytes.BytesValue; + +/** + * PING checks whether the recipient is alive and informs it about the sender's ENR sequence number. + */ +public class PingMessage implements V5Message { + // Unique request id + private final BytesValue requestId; + // Local ENR sequence number of sender + private final Long enrSeq; + + public PingMessage(BytesValue requestId, Long enrSeq) { + this.requestId = requestId; + this.enrSeq = enrSeq; + } + + @Override + public BytesValue getRequestId() { + return requestId; + } + + public Long getEnrSeq() { + return enrSeq; + } + + @Override + public BytesValue getBytes() { + return Bytes1.intToBytes1(MessageCode.PING.byteCode()) + .concat( + BytesValue.wrap( + RlpEncoder.encode( + new RlpList( + RlpString.create(requestId.extractArray()), RlpString.create(enrSeq))))); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + PingMessage that = (PingMessage) o; + return Objects.equal(requestId, that.requestId) && Objects.equal(enrSeq, that.enrSeq); + } + + @Override + public int hashCode() { + return Objects.hashCode(requestId, enrSeq); + } + + @Override + public String toString() { + return "PingMessage{" + "requestId=" + requestId + ", enrSeq=" + enrSeq + '}'; + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/PongMessage.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/PongMessage.java new file mode 100644 index 000000000..fe6ca72f1 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/PongMessage.java @@ -0,0 +1,92 @@ +package org.ethereum.beacon.discovery.message; + +import com.google.common.base.Objects; +import org.ethereum.beacon.discovery.MessageCode; +import org.web3j.rlp.RlpEncoder; +import org.web3j.rlp.RlpList; +import org.web3j.rlp.RlpString; +import tech.pegasys.artemis.util.bytes.Bytes1; +import tech.pegasys.artemis.util.bytes.BytesValue; + +/** PONG is the reply to PING {@link PingMessage} */ +public class PongMessage implements V5Message { + // Unique request id + private final BytesValue requestId; + // Local ENR sequence number of sender + private final Long enrSeq; + // 16 or 4 byte IP address of the intended recipient + private final BytesValue recipientIp; + // recipient UDP port, a 16-bit integer + private final Integer recipientPort; + + public PongMessage( + BytesValue requestId, + Long enrSeq, + BytesValue recipientIp, + Integer recipientPort) { + this.requestId = requestId; + this.enrSeq = enrSeq; + this.recipientIp = recipientIp; + this.recipientPort = recipientPort; + } + + @Override + public BytesValue getRequestId() { + return requestId; + } + + public Long getEnrSeq() { + return enrSeq; + } + + public BytesValue getRecipientIp() { + return recipientIp; + } + + public Integer getRecipientPort() { + return recipientPort; + } + + @Override + public BytesValue getBytes() { + return Bytes1.intToBytes1(MessageCode.PONG.byteCode()) + .concat( + BytesValue.wrap( + RlpEncoder.encode( + new RlpList( + RlpString.create(requestId.extractArray()), + RlpString.create(enrSeq), + RlpString.create(recipientIp.extractArray()), + RlpString.create(recipientPort))))); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + PongMessage that = (PongMessage) o; + return Objects.equal(requestId, that.requestId) + && Objects.equal(enrSeq, that.enrSeq) + && Objects.equal(recipientIp, that.recipientIp) + && Objects.equal(recipientPort, that.recipientPort); + } + + @Override + public int hashCode() { + return Objects.hashCode(requestId, enrSeq, recipientIp, recipientPort); + } + + @Override + public String toString() { + return "PongMessage{" + + "requestId=" + + requestId + + ", enrSeq=" + + enrSeq + + ", recipientIp=" + + recipientIp + + ", recipientPort=" + + recipientPort + + '}'; + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/V5Message.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/V5Message.java new file mode 100644 index 000000000..e96b14808 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/V5Message.java @@ -0,0 +1,8 @@ +package org.ethereum.beacon.discovery.message; + +import tech.pegasys.artemis.util.bytes.BytesValue; + +public interface V5Message { + BytesValue getRequestId(); + BytesValue getBytes(); +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/AbstractPacket.java b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/AbstractPacket.java new file mode 100644 index 000000000..4cc19d346 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/AbstractPacket.java @@ -0,0 +1,16 @@ +package org.ethereum.beacon.discovery.packet; + +import tech.pegasys.artemis.util.bytes.BytesValue; + +public abstract class AbstractPacket implements Packet { + private final BytesValue bytes; + + AbstractPacket(BytesValue bytes) { + this.bytes = bytes; + } + + @Override + public BytesValue getBytes() { + return bytes; + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/AuthHeaderMessagePacket.java b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/AuthHeaderMessagePacket.java new file mode 100644 index 000000000..5bca3ec0c --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/AuthHeaderMessagePacket.java @@ -0,0 +1,211 @@ +package org.ethereum.beacon.discovery.packet; + +import org.ethereum.beacon.discovery.Functions; +import org.ethereum.beacon.discovery.enr.NodeRecord; +import org.ethereum.beacon.discovery.enr.NodeRecordV5; +import org.ethereum.beacon.discovery.message.DiscoveryMessage; +import org.ethereum.beacon.discovery.message.DiscoveryV5Message; +import org.web3j.rlp.RlpDecoder; +import org.web3j.rlp.RlpEncoder; +import org.web3j.rlp.RlpList; +import org.web3j.rlp.RlpString; +import tech.pegasys.artemis.util.bytes.Bytes32; +import tech.pegasys.artemis.util.bytes.Bytes32s; +import tech.pegasys.artemis.util.bytes.BytesValue; + +import java.math.BigInteger; + +/** + * Used as first encrypted message sent in response to WHOAREYOU {@link WhoAreYouPacket}. Contains + * an authentication header completing the handshake. + * + *

Format: + * message-packet = tag || auth-header || message + * auth-header = [auth-tag, id-nonce, auth-scheme-name, ephemeral-pubkey, auth-response] + * auth-scheme-name = "gcm" + * + *

auth-response-pt is encrypted with a separate key, the auth-resp-key, using an all-zero nonce. + * This is safe because only one message is ever encrypted with this key. + * + *

auth-response = aesgcm_encrypt(auth-resp-key, zero-nonce, auth-response-pt, "") + * zero-nonce = 12 zero bytes + * auth-response-pt = [version, id-nonce-sig, node-record] + * version = 5 + * id-nonce-sig = sign(static-node-key, sha256("discovery-id-nonce" || id-nonce)) + * static-node-key = the private key used for node record identity + * node-record = record of sender OR [] if enr-seq in WHOAREYOU != current seq + * message = aesgcm_encrypt(initiator-key, auth-tag, message-pt, tag || auth-header) + * message-pt = message-type || message-data + * auth-tag = AES-GCM nonce, 12 random bytes unique to message + */ +public class AuthHeaderMessagePacket extends AbstractPacket { + public static final String AUTH_SCHEME_NAME = "gcm"; + private static final BytesValue DISCOVERY_ID_NONCE = + BytesValue.wrap("discovery-id-nonce".getBytes()); + private static final BytesValue ZERO_NONCE = BytesValue.wrap(new byte[12]); + private MessagePacketDecoded decoded = null; + + public AuthHeaderMessagePacket(BytesValue bytes) { + super(bytes); + } + + public static AuthHeaderMessagePacket create( + Bytes32 homeNodeId, + Bytes32 destNodeId, + BytesValue authResponseKey, + BytesValue idNonce, + BytesValue staticNodeKey, + NodeRecord nodeRecord, + BytesValue ephemeralPubkey, + BytesValue authTag, + BytesValue initiatorKey, + DiscoveryMessage message) { + BytesValue tag = Packet.createTag(homeNodeId, destNodeId); + BytesValue idNonceSig = + Functions.sign(staticNodeKey, Functions.hash(DISCOVERY_ID_NONCE.concat(idNonce))); + byte[] authResponsePt = + RlpEncoder.encode( + new RlpList( + RlpString.create(5), + RlpString.create(idNonceSig.extractArray()), + RlpString.create( + nodeRecord + .serialize() + .extractArray()) // FIXME: record of sender OR [] if enr-seq in WHOAREYOU != + // current seq + )); + BytesValue authResponse = + Functions.aesgcm_encrypt( + authResponseKey, ZERO_NONCE, BytesValue.wrap(authResponsePt), BytesValue.EMPTY); + RlpList authHeaderRlp = + new RlpList( + RlpString.create(authTag.extractArray()), + RlpString.create(idNonce.extractArray()), + RlpString.create(AUTH_SCHEME_NAME.getBytes()), + RlpString.create(ephemeralPubkey.extractArray()), + RlpString.create(authResponse.extractArray())); + BytesValue authHeader = BytesValue.wrap(RlpEncoder.encode(authHeaderRlp)); + BytesValue encryptedData = + Functions.aesgcm_encrypt(initiatorKey, authTag, message.getBytes(), tag.concat(authHeader)); + return new AuthHeaderMessagePacket(tag.concat(authHeader).concat(encryptedData)); + } + + public void verify(BytesValue expectedAuthTag, BytesValue expectedIdNonce) { + verifyDecode(); + assert expectedAuthTag.equals(getAuthTag()); + assert expectedIdNonce.equals(getIdNonce()); + // TODO: verify signature + } + + public Bytes32 getHomeNodeId(Bytes32 destNodeId) { + verifyDecode(); + return Bytes32s.xor(Functions.hash(destNodeId), decoded.tag); + } + + public BytesValue getAuthTag() { + verifyDecode(); + return decoded.authTag; + } + + public BytesValue getIdNonce() { + verifyDecode(); + return decoded.idNonce; + } + + public BytesValue getEphemeralPubkey() { + verifyDecode(); + return decoded.ephemeralPubkey; + } + + public BytesValue getIdNonceSig() { + verifyDecode(); + return decoded.idNonceSig; + } + + public NodeRecordV5 getNodeRecord() { + verifyDecode(); + return decoded.nodeRecord; + } + + public DiscoveryMessage getMessage() { + verifyDecode(); + return decoded.message; + } + + private void verifyDecode() { + if (decoded == null) { + throw new RuntimeException("You should decode packet at first!"); + } + } + + public void decode(BytesValue initiatorKey, BytesValue authResponseKey) { + if (decoded != null) { + return; + } + MessagePacketDecoded blank = new MessagePacketDecoded(); + blank.tag = Bytes32.wrap(getBytes().slice(0, 32), 0); + RlpList authHeaderParts = + (RlpList) RlpDecoder.decode(getBytes().slice(32).extractArray()).getValues().get(0); + int rlpLength = RlpEncoder.encode(authHeaderParts).length; // FIXME: bad hack + // [auth-tag, id-nonce, auth-scheme-name, ephemeral-pubkey, auth-response] + blank.authTag = BytesValue.wrap(((RlpString) authHeaderParts.getValues().get(0)).getBytes()); + blank.idNonce = BytesValue.wrap(((RlpString) authHeaderParts.getValues().get(1)).getBytes()); + assert AUTH_SCHEME_NAME.equals( + new String(((RlpString) authHeaderParts.getValues().get(2)).getBytes())); + blank.ephemeralPubkey = + BytesValue.wrap(((RlpString) authHeaderParts.getValues().get(3)).getBytes()); + BytesValue authResponse = + BytesValue.wrap(((RlpString) authHeaderParts.getValues().get(4)).getBytes()); + BytesValue authResponsePt = + Functions.aesgcm_decrypt(authResponseKey, ZERO_NONCE, authResponse, BytesValue.EMPTY); + RlpList authResponsePtParts = + (RlpList) RlpDecoder.decode(authResponsePt.extractArray()).getValues().get(0); + assert BigInteger.valueOf(5) + .equals(((RlpString) authResponsePtParts.getValues().get(0)).asPositiveBigInteger()); + blank.idNonceSig = + BytesValue.wrap(((RlpString) authResponsePtParts.getValues().get(1)).getBytes()); + blank.nodeRecord = + (NodeRecordV5) + NodeRecord.fromBytes(((RlpString) authResponsePtParts.getValues().get(2)).getBytes()); + BytesValue messageAad = blank.tag.concat(getBytes().slice(32)); + blank.message = + new DiscoveryV5Message( + Functions.aesgcm_decrypt( + initiatorKey, blank.authTag, getBytes().slice(32 + rlpLength), messageAad)); + this.decoded = blank; + } + + @Override + public String toString() { + if (decoded != null) { + return "AuthHeaderMessagePacket{" + + "tag=" + + decoded.tag + + ", authTag=" + + decoded.authTag + + ", idNonce=" + + decoded.idNonce + + ", ephemeralPubkey=" + + decoded.ephemeralPubkey + + ", idNonceSig=" + + decoded.idNonceSig + + ", nodeRecord=" + + decoded.nodeRecord + + ", message=" + + decoded.message + + '}'; + } else { + return "AuthHeaderMessagePacket{" + getBytes() + '}'; + } + } + + private static class MessagePacketDecoded { + private Bytes32 tag; + private BytesValue authTag; + private BytesValue idNonce; + private BytesValue ephemeralPubkey; + private BytesValue idNonceSig; + private NodeRecordV5 nodeRecord; + private DiscoveryMessage message; + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/MessagePacket.java b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/MessagePacket.java new file mode 100644 index 000000000..8a006a38c --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/MessagePacket.java @@ -0,0 +1,111 @@ +package org.ethereum.beacon.discovery.packet; + +import org.ethereum.beacon.discovery.Functions; +import org.ethereum.beacon.discovery.message.DiscoveryMessage; +import org.ethereum.beacon.discovery.message.DiscoveryV5Message; +import org.web3j.rlp.RlpDecoder; +import org.web3j.rlp.RlpEncoder; +import org.web3j.rlp.RlpString; +import tech.pegasys.artemis.util.bytes.Bytes32; +import tech.pegasys.artemis.util.bytes.Bytes32s; +import tech.pegasys.artemis.util.bytes.BytesValue; + +/** + * Used when handshake is completed as a {@link DiscoveryMessage} authenticated container + * + *

Format: + * message-packet = tag || rlp_bytes(auth-tag) || message + * message = aesgcm_encrypt(initiator-key, auth-tag, message-pt, tag) + * message-pt = message-type || message-data + */ +public class MessagePacket extends AbstractPacket { + private MessagePacketDecoded decoded = null; + private BytesValue initiatorKey = null; + + public MessagePacket(BytesValue bytes) { + super(bytes); + } + + public static MessagePacket create( + Bytes32 homeNodeId, + Bytes32 destNodeId, + BytesValue authTag, + BytesValue initiatorKey, + DiscoveryMessage message) { + BytesValue tag = Packet.createTag(homeNodeId, destNodeId); + byte[] authTagBytesRlp = RlpEncoder.encode(RlpString.create(authTag.extractArray())); + BytesValue authTagEncoded = BytesValue.wrap(authTagBytesRlp); + BytesValue encryptedData = + Functions.aesgcm_encrypt(initiatorKey, authTag, message.getBytes(), tag); + return new MessagePacket(tag.concat(authTagEncoded).concat(encryptedData)); + } + + public void verify(BytesValue expectedAuthTag) { + assert expectedAuthTag.equals(getAuthTag()); + } + + public Bytes32 getHomeNodeId(Bytes32 destNodeId) { + verifyDecode(); + return Bytes32s.xor(Functions.hash(destNodeId), decoded.tag); + } + + public BytesValue getAuthTag() { + if (decoded == null) { + return BytesValue.wrap( + ((RlpString) + RlpDecoder.decode(getBytes().slice(32, 13).extractArray()).getValues().get(0)) + .getBytes()); + } + return decoded.authTag; + } + + public DiscoveryMessage getMessage() { + verifyDecode(); + return decoded.message; + } + + private void verifyDecode() { + if (decoded == null) { + throw new RuntimeException("You should decode packet at first!"); + } + } + + public void decode(BytesValue initiatorKey) { + if (decoded != null) { + return; + } + MessagePacketDecoded blank = new MessagePacketDecoded(); + blank.tag = Bytes32.wrap(getBytes().slice(0, 32), 0); + blank.authTag = + BytesValue.wrap( + ((RlpString) + RlpDecoder.decode(getBytes().slice(32, 13).extractArray()).getValues().get(0)) + .getBytes()); + blank.message = + new DiscoveryV5Message( + Functions.aesgcm_decrypt(initiatorKey, blank.authTag, getBytes().slice(45), blank.tag)); + this.decoded = blank; + } + + @Override + public String toString() { + if (decoded != null) { + return "MessagePacket{" + + "tag=" + + decoded.tag + + ", authTag=" + + decoded.authTag + + ", message=" + + decoded.message + + '}'; + } else { + return "MessagePacket{" + getBytes() + '}'; + } + } + + private static class MessagePacketDecoded { + private Bytes32 tag; + private BytesValue authTag; + private DiscoveryMessage message; + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/Packet.java b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/Packet.java new file mode 100644 index 000000000..a6a93320f --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/Packet.java @@ -0,0 +1,14 @@ +package org.ethereum.beacon.discovery.packet; + +import org.ethereum.beacon.discovery.Functions; +import tech.pegasys.artemis.util.bytes.Bytes32; +import tech.pegasys.artemis.util.bytes.Bytes32s; +import tech.pegasys.artemis.util.bytes.BytesValue; + +public interface Packet { + static BytesValue createTag(Bytes32 homeNodeId, Bytes32 destNodeId) { + return Bytes32s.xor(Functions.hash(destNodeId), homeNodeId); + } + + BytesValue getBytes(); +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/RandomPacket.java b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/RandomPacket.java new file mode 100644 index 000000000..c0c5f483b --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/RandomPacket.java @@ -0,0 +1,74 @@ +package org.ethereum.beacon.discovery.packet; + +import org.ethereum.beacon.discovery.Functions; +import org.web3j.rlp.RlpDecoder; +import org.web3j.rlp.RlpEncoder; +import org.web3j.rlp.RlpString; +import tech.pegasys.artemis.util.bytes.Bytes32; +import tech.pegasys.artemis.util.bytes.Bytes32s; +import tech.pegasys.artemis.util.bytes.BytesValue; + +import java.util.Random; + +/** + * Sent if no session keys are available to initiate handshake + * + *

Format: + * random-packet = tag || rlp_bytes(auth-tag) || random-data + * auth-tag = 12 random bytes unique to message + * random-data = at least 44 bytes of random data + */ +public class RandomPacket extends AbstractPacket { + private RandomPacketDecoded decoded = null; + + public RandomPacket(BytesValue bytes) { + super(bytes); + } + + public static RandomPacket create( + Bytes32 homeNodeId, Bytes32 destNodeId, BytesValue authTag, Random rnd) { + BytesValue tag = Packet.createTag(homeNodeId, destNodeId); + byte[] authTagBytesRlp = RlpEncoder.encode(RlpString.create(authTag.extractArray())); + BytesValue authTagEncoded = BytesValue.wrap(authTagBytesRlp); + byte[] randomBytes = new byte[44]; + rnd.nextBytes(randomBytes); // at least 44 bytes of random data + return new RandomPacket(tag.concat(authTagEncoded).concat(BytesValue.wrap(randomBytes))); + } + + public Bytes32 getHomeNodeId(Bytes32 destNodeId) { + decode(); + return Bytes32s.xor(Functions.hash(destNodeId), decoded.tag); + } + + public BytesValue getAuthTag() { + decode(); + return decoded.authTag; + } + + private synchronized void decode() { + if (decoded != null) { + return; + } + RandomPacketDecoded blank = new RandomPacketDecoded(); + blank.tag = Bytes32.wrap(getBytes().slice(0, 32), 0); + blank.authTag = BytesValue.wrap(((RlpString) RlpDecoder.decode(getBytes().slice(32).extractArray()).getValues().get(0)).getBytes()); + this.decoded = blank; + } + + @Override + public String toString() { + if (decoded != null) { + return "RandomPacket{" + + "tag=" + decoded.tag + + ", authTag=" + decoded.authTag + + '}'; + } else { + return "RandomPacket{" + getBytes() + '}'; + } + } + + private static class RandomPacketDecoded { + private Bytes32 tag; + private BytesValue authTag; + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/UnknownPacket.java b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/UnknownPacket.java new file mode 100644 index 000000000..2a17c9248 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/UnknownPacket.java @@ -0,0 +1,56 @@ +package org.ethereum.beacon.discovery.packet; + +import org.ethereum.beacon.crypto.Hashes; +import tech.pegasys.artemis.util.bytes.Bytes32; +import tech.pegasys.artemis.util.bytes.Bytes32s; +import tech.pegasys.artemis.util.bytes.BytesValue; + +/** Default packet form until its goal is known */ +public class UnknownPacket extends AbstractPacket { + + public UnknownPacket(BytesValue bytes) { + super(bytes); + } + + public MessagePacket getMessagePacket() { + return new MessagePacket(getBytes()); + } + + public AuthHeaderMessagePacket getAuthHeaderMessagePacket() { + return new AuthHeaderMessagePacket(getBytes()); + } + + public RandomPacket getRandomPacket() { + return new RandomPacket(getBytes()); + } + + public WhoAreYouPacket getWhoAreYouPacket() { + return new WhoAreYouPacket(getBytes()); + } + + public boolean isWhoAreYouPacket(Bytes32 destNodeId) { + return WhoAreYouPacket.getStartMagic(destNodeId).equals(getBytes().slice(0, 32)); + } + + // tag = xor(sha256(dest-node-id), src-node-id) + // dest-node-id = 32-byte node ID of B + // src-node-id = 32-byte node ID of A + // + // The recipient can recover the sender's ID by performing the same calculation in reverse. + // + // src-node-id = xor(sha256(dest-node-id), tag) + public Bytes32 getSourceNodeId(Bytes32 destNodeId) { + assert !isWhoAreYouPacket(destNodeId); + BytesValue xorTag = getBytes().slice(0, 32); + return Bytes32s.xor(Hashes.sha256(destNodeId), Bytes32.wrap(xorTag, 0)); + } + + @Override + public String toString() { + return "UnknownPacket{" + + (getBytes().size() < 200 + ? getBytes() + : getBytes().slice(0, 190) + "..." + "(" + getBytes().size() + " bytes)") + + "}"; + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/WhoAreYouPacket.java b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/WhoAreYouPacket.java new file mode 100644 index 000000000..1774fc55f --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/WhoAreYouPacket.java @@ -0,0 +1,112 @@ +package org.ethereum.beacon.discovery.packet; + +import org.ethereum.beacon.discovery.Functions; +import org.web3j.rlp.RlpDecoder; +import org.web3j.rlp.RlpEncoder; +import org.web3j.rlp.RlpList; +import org.web3j.rlp.RlpString; +import tech.pegasys.artemis.util.bytes.Bytes32; +import tech.pegasys.artemis.util.bytes.Bytes8; +import tech.pegasys.artemis.util.bytes.BytesValue; +import tech.pegasys.artemis.util.uint.UInt64; + +/** + * The WHOAREYOU packet, used during the handshake as a response to any message received from + * unknown host + * + *

Format: + * whoareyou-packet = magic || [token, id-nonce, enr-seq] + * magic = sha256(dest-node-id || "WHOAREYOU") + * token = auth-tag of request + * id-nonce = 32 random bytes + * enr-seq = highest ENR sequence number of node A known on node B's side + */ +public class WhoAreYouPacket extends AbstractPacket { + private static final BytesValue MAGIC_BYTES = BytesValue.wrap("WHOAREYOU".getBytes()); + private WhoAreYouDecoded decoded = null; + + public WhoAreYouPacket(BytesValue bytes) { + super(bytes); + } + + public static WhoAreYouPacket create( + Bytes32 destNodeId, BytesValue authTag, Bytes32 idNonce, Long enrSeq) { + BytesValue magic = getStartMagic(destNodeId); + byte[] rlpListEncoded = + RlpEncoder.encode( + new RlpList( + RlpString.create(authTag.extractArray()), + RlpString.create(idNonce.extractArray()), + RlpString.create(enrSeq))); + return new WhoAreYouPacket(magic.concat(BytesValue.wrap(rlpListEncoded))); + } + + /** Calculates first 32 bytes of WHOAREYOU packet */ + public static BytesValue getStartMagic(Bytes32 destNodeId) { + return Functions.hash(destNodeId.concat(MAGIC_BYTES)); + } + + public BytesValue getAuthTag() { + decode(); + return decoded.authTag; + } + + public Bytes32 getIdNonce() { + decode(); + return decoded.idNonce; + } + + public Long getEnrSeq() { + decode(); + return decoded.enrSeq; + } + + public void verify(Bytes32 destNodeId, BytesValue expectedAuthTag) { + decode(); + assert Functions.hash(destNodeId.concat(MAGIC_BYTES)).equals(decoded.magic); + assert expectedAuthTag.equals(getAuthTag()); + } + + private synchronized void decode() { + if (decoded != null) { + return; + } + WhoAreYouDecoded blank = new WhoAreYouDecoded(); + blank.magic = Bytes32.wrap(getBytes().slice(0, 32), 0); + RlpList payload = + (RlpList) RlpDecoder.decode(getBytes().slice(32).extractArray()).getValues().get(0); + blank.authTag = BytesValue.wrap(((RlpString) payload.getValues().get(0)).getBytes()); + blank.idNonce = Bytes32.wrap(((RlpString) payload.getValues().get(1)).getBytes()); + blank.enrSeq = + UInt64.fromBytesLittleEndian( + Bytes8.rightPad( + BytesValue.wrap(((RlpString) payload.getValues().get(2)).getBytes()))) + .getValue(); + this.decoded = blank; + } + + @Override + public String toString() { + if (decoded != null) { + return "WhoAreYou{" + + "magic=" + + decoded.magic + + ", authTag=" + + decoded.authTag + + ", idNonce=" + + decoded.idNonce + + ", enrSeq=" + + decoded.enrSeq + + '}'; + } else { + return "WhoAreYou{" + getBytes() + '}'; + } + } + + private static class WhoAreYouDecoded { + private Bytes32 magic; + private BytesValue authTag; + private Bytes32 idNonce; + private Long enrSeq; + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/AuthTagRepository.java b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/AuthTagRepository.java new file mode 100644 index 000000000..6d88dfcbe --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/AuthTagRepository.java @@ -0,0 +1,57 @@ +package org.ethereum.beacon.discovery.storage; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.ethereum.beacon.discovery.NodeContext; +import tech.pegasys.artemis.util.bytes.BytesValue; + +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; + +/** + * In memory repository with authTags, corresponding contexts {@link NodeContext} and 2-way getters: + * {@link #get(BytesValue)} and {@link #getTag(NodeContext)} + * + *

Expired authTags should be manually removed with {@link #expire(NodeContext)} + */ +public class AuthTagRepository { + private static final Logger logger = LogManager.getLogger(AuthTagRepository.class); + private Map authTags = new ConcurrentHashMap<>(); + private Map contexts = new ConcurrentHashMap<>(); + + public synchronized void put(BytesValue authTag, NodeContext context) { + logger.trace( + () -> + String.format( + "PUT: authTag[%s] => nodeContext[%s]", + authTag, context.getNodeRecord().getNodeId())); + authTags.put(authTag, context); + contexts.put(context, authTag); + } + + public Optional get(BytesValue authTag) { + logger.trace(() -> String.format("GET: authTag[%s]", authTag)); + NodeContext context = authTags.get(authTag); + return context == null ? Optional.empty() : Optional.of(context); + } + + public Optional getTag(NodeContext context) { + logger.trace(() -> String.format("GET: context %s", context)); + BytesValue authTag = contexts.get(context); + return authTag == null ? Optional.empty() : Optional.of(authTag); + } + + public synchronized void expire(NodeContext context) { + logger.trace(() -> String.format("REMOVE: context %s", context)); + BytesValue authTag = contexts.remove(context); + logger.trace( + () -> + authTag == null + ? "Context %s not found, was not removed" + : String.format("Context %s removed with authTag[%s]", context, authTag)); + if (authTag != null) { + authTags.remove(authTag); + } + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorage.java b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorage.java new file mode 100644 index 000000000..0e1ef7d00 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorage.java @@ -0,0 +1,14 @@ +package org.ethereum.beacon.discovery.storage; + +import org.ethereum.beacon.db.source.SingleValueSource; +import org.ethereum.beacon.discovery.enr.NodeRecordInfo; +import org.ethereum.beacon.discovery.NodeTable; + +/** Stores {@link NodeTable} and home node info */ +public interface NodeTableStorage { + NodeTable get(); + + SingleValueSource getHomeNodeSource(); + + void commit(); +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactory.java b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactory.java new file mode 100644 index 000000000..7ff9252e6 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactory.java @@ -0,0 +1,17 @@ +package org.ethereum.beacon.discovery.storage; + +import org.ethereum.beacon.chain.storage.impl.SerializerFactory; +import org.ethereum.beacon.db.Database; +import org.ethereum.beacon.discovery.enr.NodeRecord; +import org.ethereum.beacon.discovery.enr.NodeRecordV5; + +import java.util.List; +import java.util.function.Supplier; + +public interface NodeTableStorageFactory { + NodeTableStorage create( + Database database, + SerializerFactory serializerFactory, + Supplier homeNodeSupplier, + Supplier> bootNodes); +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactoryImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactoryImpl.java new file mode 100644 index 000000000..aa6d5ba7a --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactoryImpl.java @@ -0,0 +1,52 @@ +package org.ethereum.beacon.discovery.storage; + +import org.ethereum.beacon.chain.storage.impl.SerializerFactory; +import org.ethereum.beacon.db.Database; +import org.ethereum.beacon.discovery.enr.NodeRecord; +import org.ethereum.beacon.discovery.enr.NodeRecordInfo; +import org.ethereum.beacon.discovery.enr.NodeRecordV5; +import org.ethereum.beacon.discovery.NodeStatus; + +import java.util.List; +import java.util.function.Supplier; + +public class NodeTableStorageFactoryImpl implements NodeTableStorageFactory { + + private boolean isStorageEmpty(NodeTableStorage nodeTableStorage) { + return nodeTableStorage.get().getHomeNode() == null; + } + + /** + * @return {@link NodeTableStorage} from `database` but if it doesn't exist, creates new one with + * home node provided by `homeNodeSupplier` and boot nodes provided with `bootNodesSupplier`. + * Uses `serializerFactory` for node records serialization. + */ + @Override + public NodeTableStorage create( + Database database, + SerializerFactory serializerFactory, + Supplier homeNodeSupplier, + Supplier> bootNodesSupplier) { + NodeTableStorage nodeTableStorage = new NodeTableStorageImpl(database, serializerFactory); + + // Init storage with boot nodes if its empty + if (isStorageEmpty(nodeTableStorage)) { + nodeTableStorage + .getHomeNodeSource() + .set(NodeRecordInfo.createDefault(homeNodeSupplier.get())); + bootNodesSupplier + .get() + .forEach( + nodeRecord -> { + if (!(nodeRecord instanceof NodeRecordV5)) { + throw new RuntimeException("Only V5 node records are supported as boot nodes"); + } + NodeRecordInfo nodeRecordInfo = NodeRecordInfo.createDefault(nodeRecord); + nodeTableStorage.get().save(nodeRecordInfo); + }); + } + ; + + return nodeTableStorage; + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageImpl.java new file mode 100644 index 000000000..1c9fe62f4 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageImpl.java @@ -0,0 +1,67 @@ +package org.ethereum.beacon.discovery.storage; + +import org.ethereum.beacon.chain.storage.impl.SerializerFactory; +import org.ethereum.beacon.crypto.Hashes; +import org.ethereum.beacon.db.Database; +import org.ethereum.beacon.db.source.CodecSource; +import org.ethereum.beacon.db.source.DataSource; +import org.ethereum.beacon.db.source.HoleyList; +import org.ethereum.beacon.db.source.SingleValueSource; +import org.ethereum.beacon.db.source.impl.DataSourceList; +import org.ethereum.beacon.discovery.NodeIndex; +import org.ethereum.beacon.discovery.enr.NodeRecordInfo; +import org.ethereum.beacon.discovery.NodeTable; +import org.ethereum.beacon.discovery.NodeTableImpl; +import tech.pegasys.artemis.ethereum.core.Hash32; +import tech.pegasys.artemis.util.bytes.BytesValue; + +/** Creates NodeTableStorage containing NodeTable with indexes */ +public class NodeTableStorageImpl implements NodeTableStorage { + public static final String NODE_TABLE_STORAGE_NAME = "node-table"; + public static final String INDEXES_STORAGE_NAME = "node-table-index"; + private static final Hash32 HOME_NODE_KEY = + Hash32.wrap(Hashes.sha256(BytesValue.wrap("HOME_NODE".getBytes()))); + private final DataSource nodeTableSource; + private final DataSource nodeIndexesSource; + private final SingleValueSource homeNodeSource; + private final NodeTable nodeTable; + + public NodeTableStorageImpl(Database database, SerializerFactory serializerFactory) { + DataSource nodeTableSource = + database.createStorage(NODE_TABLE_STORAGE_NAME); + this.nodeTableSource = nodeTableSource; + DataSource nodeIndexesSource = + database.createStorage(INDEXES_STORAGE_NAME); + this.nodeIndexesSource = nodeIndexesSource; + + DataSource nodeTable = + new CodecSource<>( + nodeTableSource, + key -> key, + serializerFactory.getSerializer(NodeRecordInfo.class), + serializerFactory.getDeserializer(NodeRecordInfo.class)); + HoleyList nodeIndexesTable = + new DataSourceList<>( + nodeIndexesSource, + serializerFactory.getSerializer(NodeIndex.class), + serializerFactory.getDeserializer(NodeIndex.class)); + this.homeNodeSource = SingleValueSource.fromDataSource(nodeTable, HOME_NODE_KEY); + this.nodeTable = new NodeTableImpl(nodeTable, nodeIndexesTable, homeNodeSource, Hashes::sha256); + } + + @Override + public NodeTable get() { + return nodeTable; + } + + @Override + public SingleValueSource getHomeNodeSource() { + return homeNodeSource; + } + + @Override + public void commit() { + nodeTableSource.flush(); + nodeIndexesSource.flush(); + } +} diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java new file mode 100644 index 000000000..3548afceb --- /dev/null +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java @@ -0,0 +1,136 @@ +package org.ethereum.beacon.discovery; + +import org.ethereum.beacon.chain.storage.impl.SerializerFactory; +import org.ethereum.beacon.consensus.BeaconChainSpec; +import org.ethereum.beacon.db.Database; +import org.ethereum.beacon.discovery.enr.NodeRecord; +import org.ethereum.beacon.discovery.enr.NodeRecordV5; +import org.ethereum.beacon.discovery.message.DiscoveryMessage; +import org.ethereum.beacon.discovery.message.DiscoveryV5Message; +import org.ethereum.beacon.discovery.message.FindNodeMessage; +import org.ethereum.beacon.discovery.message.NodesMessage; +import org.ethereum.beacon.discovery.packet.AuthHeaderMessagePacket; +import org.ethereum.beacon.discovery.packet.MessagePacket; +import org.ethereum.beacon.discovery.packet.RandomPacket; +import org.ethereum.beacon.discovery.packet.UnknownPacket; +import org.ethereum.beacon.discovery.packet.WhoAreYouPacket; +import org.ethereum.beacon.discovery.storage.NodeTableStorage; +import org.ethereum.beacon.discovery.storage.NodeTableStorageFactoryImpl; +import org.ethereum.beacon.schedulers.Schedulers; +import org.ethereum.beacon.stream.SimpleProcessor; +import org.junit.Test; +import reactor.core.publisher.Flux; +import tech.pegasys.artemis.util.bytes.BytesValue; + +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import static org.ethereum.beacon.discovery.NodeContext.DEFAULT_DISTANCE; +import static org.ethereum.beacon.discovery.enr.NodeRecordV5.NODE_ID_FUNCTION; + +public class DiscoveryNoNetworkTest { + @Test + public void test() throws Exception { + // 1) start 2 nodes + NodeRecordV5 nodeRecord1 = NodeRecordV5.Builder.empty() + .withIpV4Address(BytesValue.wrap(InetAddress.getByName("127.0.0.1").getAddress())) + .withSeqNumber(1L) + .withUdpPort(30303) + .withSecp256k1(BytesValue.fromHexString("0bfb48004b1698f05872cf18b1f278998ad8f7d4c135aa41f83744e7b850ab6b98")) + .build(); + NodeRecordV5 nodeRecord2 = NodeRecordV5.Builder.empty() + .withIpV4Address(BytesValue.wrap(InetAddress.getByName("192.168.0.1").getAddress())) + .withSeqNumber(1L) + .withUdpPort(30303) + .withSecp256k1(BytesValue.fromHexString("7ef3502240a42891771de732f5ee6bee3eb881939edf3e6008c0d07b502756e426")) + .build(); + NodeTableStorageFactoryImpl nodeTableStorageFactory = new NodeTableStorageFactoryImpl(); + Database database1 = Database.inMemoryDB(); + Database database2 = Database.inMemoryDB(); + SerializerFactory serializerFactory = + SerializerFactory.createSSZ(BeaconChainSpec.DEFAULT_CONSTANTS); + NodeTableStorage nodeTableStorage1 = nodeTableStorageFactory.create(database1, serializerFactory, () -> nodeRecord1, () -> new ArrayList() {{add(nodeRecord2);}}); + NodeTableStorage nodeTableStorage2 = nodeTableStorageFactory.create(database2, serializerFactory, () -> nodeRecord2, () -> new ArrayList() {{add(nodeRecord1);}}); + SimpleProcessor from1to2 = new SimpleProcessor<>(Schedulers.createDefault().newSingleThreadDaemon("from1to2-thread"), "from1to2"); + SimpleProcessor from2to1 = new SimpleProcessor<>(Schedulers.createDefault().newSingleThreadDaemon("from2to1-thread"), "from2to1"); + DiscoveryManager discoveryManager1 = new DiscoveryManager( + nodeTableStorage1.get(), + from2to1, + NODE_ID_FUNCTION.apply(nodeRecord1) + ); + DiscoveryManager discoveryManager2 = new DiscoveryManager( + nodeTableStorage2.get(), + from1to2, + NODE_ID_FUNCTION.apply(nodeRecord2) + ); + + discoveryManager1.start(); + discoveryManager2.start(); + // 2) Link outgoing of each one with incoming of another + Flux.from(discoveryManager1.getOutgoingMessages()).subscribe(t -> from1to2.onNext(new UnknownPacket(t.getPacket().getBytes()))); + Flux.from(discoveryManager2.getOutgoingMessages()).subscribe(t -> from2to1.onNext(new UnknownPacket(t.getPacket().getBytes()))); + + // 3) Expect standard 1 => 2 dialog + CountDownLatch randomSent1to2 = new CountDownLatch(1); + CountDownLatch authPacketSent1to2 = new CountDownLatch(1); + CountDownLatch whoareyouSent2to1 = new CountDownLatch(1); + CountDownLatch findNodeSent2to1 = new CountDownLatch(1); + CountDownLatch nodesSent1to2 = new CountDownLatch(1); + Flux.from(from1to2).subscribe(networkPacket -> { + // 1 -> 2 random + if (randomSent1to2.getCount() != 0) { + RandomPacket randomPacket = networkPacket.getRandomPacket(); + System.out.println("1 => 2: " + randomPacket); + randomSent1to2.countDown(); + } else if (authPacketSent1to2.getCount() != 0) { + // 1 -> 2 auth packet with FINDNODES + AuthHeaderMessagePacket authHeaderMessagePacket = networkPacket.getAuthHeaderMessagePacket(); + System.out.println("1 => 2: " + authHeaderMessagePacket); + authPacketSent1to2.countDown(); + } else { + // 1 -> 2 NODES packet + MessagePacket messagePacket = networkPacket.getMessagePacket(); + System.out.println("1 => 2: " + messagePacket); + DiscoveryMessage discoveryMessage = messagePacket.getMessage(); + assert IdentityScheme.V5.equals(discoveryMessage.getIdentityScheme()); + DiscoveryV5Message discoveryV5Message = (DiscoveryV5Message) discoveryMessage; + NodesMessage nodesMessage = (NodesMessage) discoveryV5Message.create(); + assert 1 == nodesMessage.getTotal(); + assert 1 == nodesMessage.getNodeRecordsSize(); + assert 1 == nodesMessage.getNodeRecords().size(); + nodesSent1to2.countDown(); + } + }); + Flux.from(from2to1).subscribe(networkPacket -> { + // 2 -> 1 whoareyou + if (whoareyouSent2to1.getCount() != 0) { + WhoAreYouPacket whoAreYouPacket = networkPacket.getWhoAreYouPacket(); + System.out.println("2 => 1: " + whoAreYouPacket); + whoareyouSent2to1.countDown(); + } else { + // 2 -> 1 findNode + MessagePacket messagePacket = networkPacket.getMessagePacket(); + System.out.println("2 => 1: " + messagePacket); + DiscoveryMessage discoveryMessage = messagePacket.getMessage(); + assert IdentityScheme.V5.equals(discoveryMessage.getIdentityScheme()); + DiscoveryV5Message discoveryV5Message = (DiscoveryV5Message) discoveryMessage; + FindNodeMessage findNodeMessage = (FindNodeMessage) discoveryV5Message.create(); + assert DEFAULT_DISTANCE == findNodeMessage.getDistance(); + findNodeSent2to1.countDown(); + } + }); + + // 4) fire 1 to 2 dialog + discoveryManager1.connect(nodeRecord2); + + assert randomSent1to2.await(1, TimeUnit.SECONDS); + assert whoareyouSent2to1.await(1, TimeUnit.SECONDS); + assert authPacketSent1to2.await(1, TimeUnit.SECONDS); + assert findNodeSent2to1.await(1, TimeUnit.SECONDS); + assert nodesSent1to2.await(1, TimeUnit.SECONDS); + } + + // TODO: discovery tasks are emitted from time to time as they should +} diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/NodeRecordTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/NodeRecordTest.java new file mode 100644 index 000000000..808dc9ca2 --- /dev/null +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/NodeRecordTest.java @@ -0,0 +1,103 @@ +package org.ethereum.beacon.discovery; + +import org.ethereum.beacon.discovery.enr.NodeRecord; +import org.ethereum.beacon.discovery.enr.NodeRecordV4; +import org.ethereum.beacon.discovery.enr.NodeRecordV5; +import org.junit.Test; +import tech.pegasys.artemis.util.bytes.Bytes33; +import tech.pegasys.artemis.util.bytes.Bytes4; +import tech.pegasys.artemis.util.bytes.BytesValue; + +import java.net.InetAddress; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +/** + * ENR serialization/deserialization test + * + *

ENR - Ethereum Node Record, according to https://eips.ethereum.org/EIPS/eip-778 + */ +public class NodeRecordTest { + + @Test + public void testLocalhostV4() throws Exception { + final String expectedHost = "127.0.0.1"; + final Integer expectedUdpPort = 30303; + final Integer expectedTcpPort = null; + final Long expectedSeqNumber = 1L; + final Bytes33 expectedPublicKey = + Bytes33.fromHexString("03ca634cae0d49acb401d8a4c6b6fe8c55b70d115bf400769cc1400f3258cd3138"); + final BytesValue expectedSignature = + BytesValue.fromHexString( + "7098ad865b00a582051940cb9cf36836572411a47278783077011599ed5cd16b76f2635f4e234738f30813a89eb9137e3e3df5266e3a1f11df72ecf1145ccb9c"); + + final String localhostEnr = + "-IS4QHCYrYZbAKWCBRlAy5zzaDZXJBGkcnh4MHcBFZntXNFrdvJjX04jRzjzCBOonrkTfj499SZuOh8R33Ls8RRcy5wBgmlkgnY0gmlwhH8AAAGJc2VjcDI1NmsxoQPKY0yuDUmstAHYpMa2_oxVtw0RW_QAdpzBQA8yWM0xOIN1ZHCCdl8"; + NodeRecord nodeRecord = NodeRecord.fromBase64(localhostEnr); + + assertEquals(IdentityScheme.V4, nodeRecord.getIdentityScheme()); + NodeRecordV4 nodeRecordV4 = (NodeRecordV4) nodeRecord; + assertArrayEquals( + InetAddress.getByName(expectedHost).getAddress(), + nodeRecordV4.getIpV4address().extractArray()); + assertEquals(expectedUdpPort, nodeRecordV4.getUdpPort()); + assertEquals(expectedTcpPort, nodeRecordV4.getTcpPort()); + assertEquals(expectedSeqNumber, nodeRecordV4.getSeqNumber()); + assertEquals(expectedPublicKey, nodeRecordV4.getPublicKey()); + assertEquals(expectedSignature, nodeRecordV4.getSignature()); + + String localhostEnrRestored = nodeRecordV4.asBase64(); + // The order of fields is not strict so we don't compare strings + NodeRecord nodeRecordRestored = NodeRecord.fromBase64(localhostEnrRestored); + + assertEquals(IdentityScheme.V4, nodeRecordRestored.getIdentityScheme()); + NodeRecordV4 nodeRecordV4Restored = (NodeRecordV4) nodeRecordRestored; + assertArrayEquals( + InetAddress.getByName(expectedHost).getAddress(), + nodeRecordV4Restored.getIpV4address().extractArray()); + assertEquals(expectedUdpPort, nodeRecordV4Restored.getUdpPort()); + assertEquals(expectedTcpPort, nodeRecordV4Restored.getTcpPort()); + assertEquals(expectedSeqNumber, nodeRecordV4Restored.getSeqNumber()); + assertEquals(expectedPublicKey, nodeRecordV4Restored.getPublicKey()); + assertEquals(expectedSignature, nodeRecordV4Restored.getSignature()); + } + + @Test + public void testLocalhostV5() throws Exception { + final String expectedHost = "127.0.0.1"; + final Integer expectedUdpPort = 30303; + final Integer expectedTcpPort = null; + final Long expectedSeqNumber = 1L; + final Bytes33 expectedPublicKey = + Bytes33.fromHexString("03ca634cae0d49acb401d8a4c6b6fe8c55b70d115bf400769cc1400f3258cd3138"); + final BytesValue expectedSignature = + BytesValue.fromHexString( + "7098ad865b00a582051940cb9cf36836572411a47278783077011599ed5cd16b76f2635f4e234738f30813a89eb9137e3e3df5266e3a1f11df72ecf1145ccb9c"); + + NodeRecordV5 nodeRecordV5 = + NodeRecordV5.fromValues( + expectedPublicKey, + Bytes4.wrap(InetAddress.getByName(expectedHost).getAddress()), + expectedTcpPort, + expectedUdpPort, + null, + null, + null); + nodeRecordV5.setSeqNumber(expectedSeqNumber); + nodeRecordV5.setSignature(expectedSignature); + + String enrV5 = nodeRecordV5.asBase64(); + NodeRecordV5 nodeRecordV5Restored = (NodeRecordV5) NodeRecord.fromBase64(enrV5); + assertEquals(IdentityScheme.V5, nodeRecordV5Restored.getIdentityScheme()); + assertArrayEquals( + InetAddress.getByName(expectedHost).getAddress(), + nodeRecordV5Restored.getIpV4address().extractArray()); + assertEquals(expectedUdpPort, nodeRecordV5Restored.getUdpPort()); + assertEquals(expectedTcpPort, nodeRecordV5Restored.getTcpPort()); + assertEquals(expectedSeqNumber, nodeRecordV5Restored.getSeqNumber()); + assertEquals(expectedPublicKey, nodeRecordV5Restored.getPublicKey()); + assertEquals(expectedSignature, nodeRecordV5Restored.getSignature()); + } +} diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/NodeTableTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/NodeTableTest.java new file mode 100644 index 000000000..3eb8dd90c --- /dev/null +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/NodeTableTest.java @@ -0,0 +1,111 @@ +package org.ethereum.beacon.discovery; + +import org.ethereum.beacon.chain.storage.impl.SerializerFactory; +import org.ethereum.beacon.consensus.BeaconChainSpec; +import org.ethereum.beacon.db.Database; +import org.ethereum.beacon.discovery.enr.NodeRecord; +import org.ethereum.beacon.discovery.enr.NodeRecordInfo; +import org.ethereum.beacon.discovery.enr.NodeRecordV5; +import org.ethereum.beacon.discovery.storage.NodeTableStorage; +import org.ethereum.beacon.discovery.storage.NodeTableStorageFactoryImpl; +import org.junit.Test; +import tech.pegasys.artemis.util.bytes.BytesValue; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.function.Supplier; + +import static org.ethereum.beacon.discovery.enr.NodeRecordV5.NODE_ID_FUNCTION; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class NodeTableTest { + + private Supplier homeNodeSupplier = () -> { + try { + return NodeRecordV5.Builder.empty() + .withIpV4Address(BytesValue.wrap(InetAddress.getByName("127.0.0.1").getAddress())) + .withSeqNumber(1L) + .withUdpPort(30303) + .withSecp256k1(BytesValue.fromHexString("0bfb48004b1698f05872cf18b1f278998ad8f7d4c135aa41f83744e7b850ab6b98")) + .build(); + } catch (UnknownHostException e) { + throw new RuntimeException(e); + } + }; + + @Test + public void testCreate() throws Exception { + final String localhostEnr = + "-IS4QHCYrYZbAKWCBRlAy5zzaDZXJBGkcnh4MHcBFZntXNFrdvJjX04jRzjzCBOonrkTfj499SZuOh8R33Ls8RRcy5wBgmlkgnY1iXNlY3AyNTZrMaEDymNMrg1JrLQB2KTGtv6MVbcNEVv0AHacwUAPMljNMTiCaXCEfwAAAYN1ZHCCdl8="; + NodeRecordV5 nodeRecordV5 = (NodeRecordV5) NodeRecord.fromBase64(localhostEnr); + NodeTableStorageFactoryImpl nodeTableStorageFactory = new NodeTableStorageFactoryImpl(); + Database database = Database.inMemoryDB(); + SerializerFactory serializerFactory = + SerializerFactory.createSSZ(BeaconChainSpec.DEFAULT_CONSTANTS); + NodeTableStorage nodeTableStorage = + nodeTableStorageFactory.create( + database, + serializerFactory, + homeNodeSupplier, + () -> { + List nodes = new ArrayList<>(); + nodes.add(nodeRecordV5); + return nodes; + }); + Optional extendedEnr = + nodeTableStorage.get().getNode(NODE_ID_FUNCTION.apply(nodeRecordV5)); + assertTrue(extendedEnr.isPresent()); + NodeRecordInfo nodeRecord = extendedEnr.get(); + assertEquals(nodeRecordV5.getPublicKey(), nodeRecord.getNode().getPublicKey()); + assertEquals(nodeTableStorage.get().getHomeNode().getNodeId(), NODE_ID_FUNCTION.apply(homeNodeSupplier.get())); + } + + @Test + public void testFind() throws Exception { + final String localhostEnr = + "-IS4QHCYrYZbAKWCBRlAy5zzaDZXJBGkcnh4MHcBFZntXNFrdvJjX04jRzjzCBOonrkTfj499SZuOh8R33Ls8RRcy5wBgmlkgnY1iXNlY3AyNTZrMaEDymNMrg1JrLQB2KTGtv6MVbcNEVv0AHacwUAPMljNMTiCaXCEfwAAAYN1ZHCCdl8="; + NodeRecordV5 localHostNode = (NodeRecordV5) NodeRecord.fromBase64(localhostEnr); + NodeTableStorageFactoryImpl nodeTableStorageFactory = new NodeTableStorageFactoryImpl(); + Database database = Database.inMemoryDB(); + SerializerFactory serializerFactory = + SerializerFactory.createSSZ(BeaconChainSpec.DEFAULT_CONSTANTS); + NodeTableStorage nodeTableStorage = + nodeTableStorageFactory.create( + database, + serializerFactory, + homeNodeSupplier, + () -> { + List nodes = new ArrayList<>(); + nodes.add(localHostNode); + return nodes; + }); + + // node is adjusted to be close to localhostEnr + NodeRecordV5 closestNode = NodeRecordV5.Builder.empty() + .withIpV4Address(BytesValue.wrap(InetAddress.getByName("127.0.0.2").getAddress())) + .withSeqNumber(1L) + .withUdpPort(30303) + .withSecp256k1(BytesValue.fromHexString("aafb48004b1698f05872cf18b1f278998ad8f7d4c135aa41f83744e7b850ab6b98")) + .build(); + nodeTableStorage.get().save(new NodeRecordInfo(closestNode, -1L, NodeStatus.ACTIVE, 0)); + assertEquals(nodeTableStorage.get().getNode(NODE_ID_FUNCTION.apply(closestNode)).get().getNode().getPublicKey(), closestNode.getPublicKey()); + NodeRecordV5 farNode = NodeRecordV5.Builder.empty() + .withIpV4Address(BytesValue.wrap(InetAddress.getByName("127.0.0.3").getAddress())) + .withSeqNumber(1L) + .withUdpPort(30303) + .withSecp256k1(BytesValue.fromHexString("bafb48004b1698f05872cf18b1f278998ad8f7d4c135aa41f83744e7b850ab6b98")) + .build(); + nodeTableStorage.get().save(new NodeRecordInfo(farNode, -1L, NodeStatus.ACTIVE, 0)); + List closestNodes = nodeTableStorage.get().findClosestNodes(NODE_ID_FUNCTION.apply(closestNode), 1); + assertEquals(2, closestNodes.size()); + assertEquals(closestNodes.get(0).getNode().getPublicKey(), closestNode.getPublicKey()); + assertEquals(closestNodes.get(1).getNode().getPublicKey(), localHostNode.getPublicKey()); + List farNodes = nodeTableStorage.get().findClosestNodes(NODE_ID_FUNCTION.apply(farNode), 1); + assertEquals(1, farNodes.size()); + assertEquals(farNodes.get(0).getNode().getPublicKey(), farNode.getPublicKey()); + } +} diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/RlpExtraTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/RlpExtraTest.java new file mode 100644 index 000000000..5671007da --- /dev/null +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/RlpExtraTest.java @@ -0,0 +1,35 @@ +package org.ethereum.beacon.discovery; + +import org.junit.Test; +import org.web3j.rlp.RlpDecoder; +import org.web3j.rlp.RlpEncoder; +import org.web3j.rlp.RlpList; +import org.web3j.rlp.RlpString; + +import static org.junit.Assert.assertEquals; + +public class RlpExtraTest { + private static final String ABC = "abc"; + private static final int INT = 12345; + + /** + * Testing what could we do when rlp is appended with some extra data, how could we split rlp and + * that data + */ + @Test + public void testMakeRlpExtra() { + RlpList rlpList = new RlpList(RlpString.create(ABC), RlpString.create(INT)); + byte[] encoded = RlpEncoder.encode(rlpList); + byte[] encodedPlus = new byte[encoded.length + 2]; + System.arraycopy(encoded, 0, encodedPlus, 0, encoded.length); + encodedPlus[encodedPlus.length - 2] = 0x12; + encodedPlus[encodedPlus.length - 1] = 0x34; + RlpList rlpList1 = RlpDecoder.decode(encodedPlus); + RlpList rlpList2 = ((RlpList) rlpList1.getValues().get(0)); + assertEquals(ABC, new String(((RlpString) rlpList2.getValues().get(0)).getBytes())); + assertEquals(INT, ((RlpString) rlpList2.getValues().get(1)).asPositiveBigInteger().intValue()); + // but what else could we do? + int length = RlpEncoder.encode(rlpList2).length; + assertEquals(length + 2, encodedPlus.length); + } +} diff --git a/settings.gradle b/settings.gradle index 2318a5e94..b3f574263 100644 --- a/settings.gradle +++ b/settings.gradle @@ -9,6 +9,8 @@ include 'core' include 'crypto' // DB persistence core interfaces include 'db:core' +// Discovery v5 protocol implementation +include 'discovery' // PoW (Proof of Work) interfaces etc include 'pow:core' // PoW made with EthereumJ @@ -43,5 +45,5 @@ include 'validator:core' include 'validator:embedded' // Validator server-side api include 'validator:server' -// Wire API mock +// Wire API include 'wire' diff --git a/test/src/test/resources/eth2.0-spec-tests b/test/src/test/resources/eth2.0-spec-tests index aaa1673f5..ae6dd9011 160000 --- a/test/src/test/resources/eth2.0-spec-tests +++ b/test/src/test/resources/eth2.0-spec-tests @@ -1 +1 @@ -Subproject commit aaa1673f508103e11304833e0456e4149f880065 +Subproject commit ae6dd9011df05fab8c7e651c09cf9c940973bf81 diff --git a/types/src/main/java/tech/pegasys/artemis/util/bytes/ArrayWrappingBytes16.java b/types/src/main/java/tech/pegasys/artemis/util/bytes/ArrayWrappingBytes16.java new file mode 100644 index 000000000..ee0b316da --- /dev/null +++ b/types/src/main/java/tech/pegasys/artemis/util/bytes/ArrayWrappingBytes16.java @@ -0,0 +1,59 @@ +/* + * Copyright 2018 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.artemis.util.bytes; + +import static com.google.common.base.Preconditions.checkArgument; + +/** + * An implementation of {@link Bytes16} backed by a byte array ({@code byte[]}). + */ +class ArrayWrappingBytes16 extends ArrayWrappingBytesValue implements Bytes16 { + + ArrayWrappingBytes16(byte[] bytes) { + this(checkLength(bytes), 0); + } + + ArrayWrappingBytes16(byte[] bytes, int offset) { + super(checkLength(bytes, offset), offset, SIZE); + } + + // Ensures a proper error message. + private static byte[] checkLength(byte[] bytes) { + checkArgument(bytes.length == SIZE, "Expected %s bytes but got %s", SIZE, bytes.length); + return bytes; + } + + // Ensures a proper error message. + private static byte[] checkLength(byte[] bytes, int offset) { + checkArgument(bytes.length - offset >= SIZE, + "Expected at least %s bytes from offset %s but got only %s", SIZE, offset, + bytes.length - offset); + return bytes; + } + + @Override + public Bytes16 copy() { + // Because MutableArrayWrappingBytesValue overrides this, we know we are immutable. We may + // retain more than necessary however. + if (offset == 0 && length == bytes.length) + return this; + + return new ArrayWrappingBytes16(arrayCopy()); + } + + @Override + public MutableBytes16 mutableCopy() { + return new MutableArrayWrappingBytes16(arrayCopy()); + } +} diff --git a/types/src/main/java/tech/pegasys/artemis/util/bytes/ArrayWrappingBytes33.java b/types/src/main/java/tech/pegasys/artemis/util/bytes/ArrayWrappingBytes33.java new file mode 100644 index 000000000..ce49d34ff --- /dev/null +++ b/types/src/main/java/tech/pegasys/artemis/util/bytes/ArrayWrappingBytes33.java @@ -0,0 +1,59 @@ +/* + * Copyright 2018 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.artemis.util.bytes; + +import static com.google.common.base.Preconditions.checkArgument; + +/** + * An implementation of {@link Bytes33} backed by a byte array ({@code byte[]}). + */ +class ArrayWrappingBytes33 extends ArrayWrappingBytesValue implements Bytes33 { + + ArrayWrappingBytes33(byte[] bytes) { + this(checkLength(bytes), 0); + } + + ArrayWrappingBytes33(byte[] bytes, int offset) { + super(checkLength(bytes, offset), offset, SIZE); + } + + // Ensures a proper error message. + private static byte[] checkLength(byte[] bytes) { + checkArgument(bytes.length == SIZE, "Expected %s bytes but got %s", SIZE, bytes.length); + return bytes; + } + + // Ensures a proper error message. + private static byte[] checkLength(byte[] bytes, int offset) { + checkArgument(bytes.length - offset >= SIZE, + "Expected at least %s bytes from offset %s but got only %s", SIZE, offset, + bytes.length - offset); + return bytes; + } + + @Override + public Bytes33 copy() { + // Because MutableArrayWrappingBytesValue overrides this, we know we are immutable. We may + // retain more than necessary however. + if (offset == 0 && length == bytes.length) + return this; + + return new ArrayWrappingBytes33(arrayCopy()); + } + + @Override + public MutableBytes33 mutableCopy() { + return new MutableArrayWrappingBytes33(arrayCopy()); + } +} diff --git a/types/src/main/java/tech/pegasys/artemis/util/bytes/Bytes16.java b/types/src/main/java/tech/pegasys/artemis/util/bytes/Bytes16.java new file mode 100644 index 000000000..d084dcfc5 --- /dev/null +++ b/types/src/main/java/tech/pegasys/artemis/util/bytes/Bytes16.java @@ -0,0 +1,187 @@ +/* + * Copyright 2018 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.artemis.util.bytes; + +import tech.pegasys.artemis.util.uint.Int256; +import tech.pegasys.artemis.util.uint.UInt256; +import tech.pegasys.artemis.util.uint.UInt256Bytes; + +import java.util.Random; + +import static com.google.common.base.Preconditions.checkArgument; + +/** + * A {@link BytesValue} that is guaranteed to contain exactly 16 bytes. + */ +public interface Bytes16 extends BytesValue { + int SIZE = 16; + Bytes16 ZERO = wrap(new byte[16]); + + /** + * Wraps the provided byte array, which must be of length 16, as a {@link Bytes16}. + * + *

Note that value is not copied, only wrapped, and thus any future update to {@code value} + * will be reflected in the returned value. + * + * @param bytes The bytes to wrap. + * @return A {@link Bytes16} wrapping {@code value}. + * @throws IllegalArgumentException if {@code value.length != 16}. + */ + static Bytes16 wrap(byte[] bytes) { + checkArgument(bytes.length == SIZE, "Expected %s bytes but got %s", SIZE, bytes.length); + return wrap(bytes, 0); + } + + /** + * Wraps a slice/sub-part of the provided array as a {@link Bytes16}. + * + *

Note that value is not copied, only wrapped, and thus any future update to {@code value} + * within the wrapped parts will be reflected in the returned value. + * + * @param bytes The bytes to wrap. + * @param offset The index (inclusive) in {@code value} of the first byte exposed by the returned + * value. In other words, you will have {@code wrap(value, i).get(0) == value[i]}. + * @return A {@link Bytes16} that exposes the bytes of {@code value} from {@code offset} + * (inclusive) to {@code offset + 16} (exclusive). + * @throws IndexOutOfBoundsException if {@code offset < 0 || (value.length > 0 && offset >= + * value.length)}. + * @throws IllegalArgumentException if {@code length < 0 || offset + 16 > value.length}. + */ + static Bytes16 wrap(byte[] bytes, int offset) { + return new ArrayWrappingBytes16(bytes, offset); + } + + /** + * Wraps a slice/sub-part of the provided value as a {@link Bytes16}. + * + *

Note that value is not copied, only wrapped, and thus any future update to {@code value} + * within the wrapped parts will be reflected in the returned value. + * + * @param bytes The bytes to wrap. + * @param offset The index (inclusive) in {@code value} of the first byte exposed by the returned + * value. In other words, you will have {@code wrap(value, i).get(0) == value.get(i)}. + * @return A {@link Bytes16} that exposes the bytes of {@code value} from {@code offset} + * (inclusive) to {@code offset + 16} (exclusive). + * @throws IndexOutOfBoundsException if {@code offset < 0 || (value.size() > 0 && offset >= + * value.size())}. + * @throws IllegalArgumentException if {@code length < 0 || offset + 16 > value.size()}. + */ + static Bytes16 wrap(BytesValue bytes, int offset) { + BytesValue slice = bytes.slice(offset, Bytes16.SIZE); + return slice instanceof Bytes16 ? (Bytes16) slice : new WrappingBytes16(slice); + } + + /** + * Left pad a {@link BytesValue} with zero bytes to create a {@link Bytes16} + * + * @param value The bytes value pad. + * @return A {@link Bytes16} that exposes the left-padded bytes of {@code value}. + * @throws IllegalArgumentException if {@code value.size() > 16}. + */ + static Bytes16 leftPad(BytesValue value) { + checkArgument( + value.size() <= SIZE, "Expected at most %s bytes but got only %s", SIZE, value.size()); + + MutableBytes16 bytes = MutableBytes16.create(); + value.copyTo(bytes, SIZE - value.size()); + return bytes; + } + + /** + * Right pad a {@link BytesValue} with zero bytes to create a {@link Bytes16} + * + * @param value The bytes value pad. + * @return A {@link Bytes16} that exposes the right-padded bytes of {@code value}. + * @throws IllegalArgumentException if {@code value.size() > 16}. + */ + static Bytes16 rightPad(BytesValue value) { + checkArgument( + value.size() <= SIZE, "Expected at most %s bytes but got only %s", SIZE, value.size()); + + MutableBytes16 bytes = MutableBytes16.create(); + value.copyTo(bytes, 0); + return bytes; + } + + /** + * Parse an hexadecimal string into a {@link Bytes16}. + * + *

This method is lenient in that {@code str} may of an odd length, in which case it will + * behave exactly as if it had an additional 0 in front. + * + * @param str The hexadecimal string to parse, which may or may not start with "0x". That + * representation may contain less than 16 bytes, in which case the result is left padded with + * zeros (see {@link #fromHexStringStrict} if this is not what you want). + * @return The value corresponding to {@code str}. + * @throws IllegalArgumentException if {@code str} does not correspond to valid hexadecimal + * representation or contains more than 16 bytes. + */ + static Bytes16 fromHexStringLenient(String str) { + return wrap(BytesValues.fromRawHexString(str, SIZE, true)); + } + + /** + * Parse an hexadecimal string into a {@link Bytes16}. + * + *

This method is strict in that {@code str} must of an even length. + * + * @param str The hexadecimal string to parse, which may or may not start with "0x". That + * representation may contain less than 16 bytes, in which case the result is left padded with + * zeros (see {@link #fromHexStringStrict} if this is not what you want). + * @return The value corresponding to {@code str}. + * @throws IllegalArgumentException if {@code str} does not correspond to valid hexadecimal + * representation, is of an odd length, or contains more than 16 bytes. + */ + static Bytes16 fromHexString(String str) { + return wrap(BytesValues.fromRawHexString(str, SIZE, false)); + } + + /** + * Parse an hexadecimal string into a {@link Bytes16}. + * + *

This method is extra strict in that {@code str} must of an even length and the provided + * representation must have exactly 16 bytes. + * + * @param str The hexadecimal string to parse, which may or may not start with "0x". + * @return The value corresponding to {@code str}. + * @throws IllegalArgumentException if {@code str} does not correspond to valid hexadecimal + * representation, is of an odd length or does not contain exactly 16 bytes. + */ + static Bytes16 fromHexStringStrict(String str) { + return wrap(BytesValues.fromRawHexString(str, -1, false)); + } + + /** + * Constructs a randomly generated value. + * + * @param rnd random number generator. + * @return random value. + */ + static Bytes16 random(Random rnd) { + byte[] randomBytes = new byte[SIZE]; + rnd.nextBytes(randomBytes); + return wrap(randomBytes); + } + + @Override + default int size() { + return SIZE; + } + + @Override + Bytes16 copy(); + + @Override + MutableBytes16 mutableCopy(); +} diff --git a/types/src/main/java/tech/pegasys/artemis/util/bytes/Bytes33.java b/types/src/main/java/tech/pegasys/artemis/util/bytes/Bytes33.java new file mode 100644 index 000000000..e99288094 --- /dev/null +++ b/types/src/main/java/tech/pegasys/artemis/util/bytes/Bytes33.java @@ -0,0 +1,183 @@ +/* + * Copyright 2018 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.artemis.util.bytes; + +import java.util.Random; + +import static com.google.common.base.Preconditions.checkArgument; + +/** + * A {@link BytesValue} that is guaranteed to contain exactly 33 bytes. + */ +public interface Bytes33 extends BytesValue { + int SIZE = 33; + Bytes33 ZERO = wrap(new byte[33]); + + /** + * Wraps the provided byte array, which must be of length 33, as a {@link Bytes33}. + * + *

Note that value is not copied, only wrapped, and thus any future update to {@code value} + * will be reflected in the returned value. + * + * @param bytes The bytes to wrap. + * @return A {@link Bytes33} wrapping {@code value}. + * @throws IllegalArgumentException if {@code value.length != 33}. + */ + static Bytes33 wrap(byte[] bytes) { + checkArgument(bytes.length == SIZE, "Expected %s bytes but got %s", SIZE, bytes.length); + return wrap(bytes, 0); + } + + /** + * Wraps a slice/sub-part of the provided array as a {@link Bytes33}. + * + *

Note that value is not copied, only wrapped, and thus any future update to {@code value} + * within the wrapped parts will be reflected in the returned value. + * + * @param bytes The bytes to wrap. + * @param offset The index (inclusive) in {@code value} of the first byte exposed by the returned + * value. In other words, you will have {@code wrap(value, i).get(0) == value[i]}. + * @return A {@link Bytes33} that exposes the bytes of {@code value} from {@code offset} + * (inclusive) to {@code offset + 33} (exclusive). + * @throws IndexOutOfBoundsException if {@code offset < 0 || (value.length > 0 && offset >= + * value.length)}. + * @throws IllegalArgumentException if {@code length < 0 || offset + 33 > value.length}. + */ + static Bytes33 wrap(byte[] bytes, int offset) { + return new ArrayWrappingBytes33(bytes, offset); + } + + /** + * Wraps a slice/sub-part of the provided value as a {@link Bytes33}. + * + *

Note that value is not copied, only wrapped, and thus any future update to {@code value} + * within the wrapped parts will be reflected in the returned value. + * + * @param bytes The bytes to wrap. + * @param offset The index (inclusive) in {@code value} of the first byte exposed by the returned + * value. In other words, you will have {@code wrap(value, i).get(0) == value.get(i)}. + * @return A {@link Bytes33} that exposes the bytes of {@code value} from {@code offset} + * (inclusive) to {@code offset + 33} (exclusive). + * @throws IndexOutOfBoundsException if {@code offset < 0 || (value.size() > 0 && offset >= + * value.size())}. + * @throws IllegalArgumentException if {@code length < 0 || offset + 33 > value.size()}. + */ + static Bytes33 wrap(BytesValue bytes, int offset) { + BytesValue slice = bytes.slice(offset, Bytes33.SIZE); + return slice instanceof Bytes33 ? (Bytes33) slice : new WrappingBytes33(slice); + } + + /** + * Left pad a {@link BytesValue} with zero bytes to create a {@link Bytes33} + * + * @param value The bytes value pad. + * @return A {@link Bytes33} that exposes the left-padded bytes of {@code value}. + * @throws IllegalArgumentException if {@code value.size() > 33}. + */ + static Bytes33 leftPad(BytesValue value) { + checkArgument( + value.size() <= SIZE, "Expected at most %s bytes but got only %s", SIZE, value.size()); + + MutableBytes33 bytes = MutableBytes33.create(); + value.copyTo(bytes, SIZE - value.size()); + return bytes; + } + + /** + * Right pad a {@link BytesValue} with zero bytes to create a {@link Bytes33} + * + * @param value The bytes value pad. + * @return A {@link Bytes33} that exposes the right-padded bytes of {@code value}. + * @throws IllegalArgumentException if {@code value.size() > 33}. + */ + static Bytes33 rightPad(BytesValue value) { + checkArgument( + value.size() <= SIZE, "Expected at most %s bytes but got only %s", SIZE, value.size()); + + MutableBytes33 bytes = MutableBytes33.create(); + value.copyTo(bytes, 0); + return bytes; + } + + /** + * Parse an hexadecimal string into a {@link Bytes33}. + * + *

This method is lenient in that {@code str} may of an odd length, in which case it will + * behave exactly as if it had an additional 0 in front. + * + * @param str The hexadecimal string to parse, which may or may not start with "0x". That + * representation may contain less than 33 bytes, in which case the result is left padded with + * zeros (see {@link #fromHexStringStrict} if this is not what you want). + * @return The value corresponding to {@code str}. + * @throws IllegalArgumentException if {@code str} does not correspond to valid hexadecimal + * representation or contains more than 33 bytes. + */ + static Bytes33 fromHexStringLenient(String str) { + return wrap(BytesValues.fromRawHexString(str, SIZE, true)); + } + + /** + * Parse an hexadecimal string into a {@link Bytes33}. + * + *

This method is strict in that {@code str} must of an even length. + * + * @param str The hexadecimal string to parse, which may or may not start with "0x". That + * representation may contain less than 33 bytes, in which case the result is left padded with + * zeros (see {@link #fromHexStringStrict} if this is not what you want). + * @return The value corresponding to {@code str}. + * @throws IllegalArgumentException if {@code str} does not correspond to valid hexadecimal + * representation, is of an odd length, or contains more than 33 bytes. + */ + static Bytes33 fromHexString(String str) { + return wrap(BytesValues.fromRawHexString(str, SIZE, false)); + } + + /** + * Parse an hexadecimal string into a {@link Bytes33}. + * + *

This method is extra strict in that {@code str} must of an even length and the provided + * representation must have exactly 33 bytes. + * + * @param str The hexadecimal string to parse, which may or may not start with "0x". + * @return The value corresponding to {@code str}. + * @throws IllegalArgumentException if {@code str} does not correspond to valid hexadecimal + * representation, is of an odd length or does not contain exactly 33 bytes. + */ + static Bytes33 fromHexStringStrict(String str) { + return wrap(BytesValues.fromRawHexString(str, -1, false)); + } + + /** + * Constructs a randomly generated value. + * + * @param rnd random number generator. + * @return random value. + */ + static Bytes33 random(Random rnd) { + byte[] randomBytes = new byte[SIZE]; + rnd.nextBytes(randomBytes); + return wrap(randomBytes); + } + + @Override + default int size() { + return SIZE; + } + + @Override + Bytes33 copy(); + + @Override + MutableBytes33 mutableCopy(); +} diff --git a/types/src/main/java/tech/pegasys/artemis/util/bytes/MutableArrayWrappingBytes16.java b/types/src/main/java/tech/pegasys/artemis/util/bytes/MutableArrayWrappingBytes16.java new file mode 100644 index 000000000..3916dbaaf --- /dev/null +++ b/types/src/main/java/tech/pegasys/artemis/util/bytes/MutableArrayWrappingBytes16.java @@ -0,0 +1,39 @@ +/* + * Copyright 2018 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.artemis.util.bytes; + +/** + * An implementation of {@link MutableBytes16} backed by a byte array ({@code byte[]}). + */ +class MutableArrayWrappingBytes16 extends MutableArrayWrappingBytesValue implements MutableBytes16 { + + MutableArrayWrappingBytes16(byte[] bytes) { + this(bytes, 0); + } + + MutableArrayWrappingBytes16(byte[] bytes, int offset) { + super(bytes, offset, SIZE); + } + + @Override + public Bytes16 copy() { + // We *must* override this method because ArrayWrappingBytes16 assumes that it is the case. + return new ArrayWrappingBytes16(arrayCopy()); + } + + @Override + public MutableBytes16 mutableCopy() { + return new MutableArrayWrappingBytes16(arrayCopy()); + } +} diff --git a/types/src/main/java/tech/pegasys/artemis/util/bytes/MutableArrayWrappingBytes33.java b/types/src/main/java/tech/pegasys/artemis/util/bytes/MutableArrayWrappingBytes33.java new file mode 100644 index 000000000..c133cc1d2 --- /dev/null +++ b/types/src/main/java/tech/pegasys/artemis/util/bytes/MutableArrayWrappingBytes33.java @@ -0,0 +1,39 @@ +/* + * Copyright 2018 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.artemis.util.bytes; + +/** + * An implementation of {@link MutableBytes33} backed by a byte array ({@code byte[]}). + */ +class MutableArrayWrappingBytes33 extends MutableArrayWrappingBytesValue implements MutableBytes33 { + + MutableArrayWrappingBytes33(byte[] bytes) { + this(bytes, 0); + } + + MutableArrayWrappingBytes33(byte[] bytes, int offset) { + super(bytes, offset, SIZE); + } + + @Override + public Bytes33 copy() { + // We *must* override this method because ArrayWrappingBytes33 assumes that it is the case. + return new ArrayWrappingBytes33(arrayCopy()); + } + + @Override + public MutableBytes33 mutableCopy() { + return new MutableArrayWrappingBytes33(arrayCopy()); + } +} diff --git a/types/src/main/java/tech/pegasys/artemis/util/bytes/MutableBytes16.java b/types/src/main/java/tech/pegasys/artemis/util/bytes/MutableBytes16.java new file mode 100644 index 000000000..4edb6b540 --- /dev/null +++ b/types/src/main/java/tech/pegasys/artemis/util/bytes/MutableBytes16.java @@ -0,0 +1,145 @@ +/* + * Copyright 2018 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.artemis.util.bytes; + +import java.security.MessageDigest; + +import static com.google.common.base.Preconditions.checkArgument; + +/** + * A mutable {@link Bytes16}, that is a mutable {@link BytesValue} of exactly 16 bytes. + */ +public interface MutableBytes16 extends MutableBytesValue, Bytes16 { + + /** + * Wraps a 16 bytes array as a mutable 16 bytes value. + * + *

+ * This method behave exactly as {@link Bytes16#wrap(byte[])} except that the result is a mutable. + * + * @param value The value to wrap. + * @return A {@link MutableBytes16} wrapping {@code value}. + * @throws IllegalArgumentException if {@code value.length != 16}. + */ + static MutableBytes16 wrap(byte[] value) { + return new MutableArrayWrappingBytes16(value); + } + + /** + * Creates a new mutable 16 bytes value. + * + * @return A newly allocated {@link MutableBytesValue}. + */ + static MutableBytes16 create() { + return new MutableArrayWrappingBytes16(new byte[SIZE]); + } + + /** + * Wraps an existing {@link MutableBytesValue} of size 16 as a mutable 16 bytes value. + * + *

+ * This method does no copy the provided bytes and so any mutation on {@code value} will also be + * reflected in the value returned by this method. If a copy is desirable, this can be simply + * achieved with calling {@link BytesValue#copyTo(MutableBytesValue)} with a newly created + * {@link MutableBytes16} as destination to the copy. + * + * @param value The value to wrap. + * @return A {@link MutableBytes16} wrapping {@code value}. + * @throws IllegalArgumentException if {@code value.size() != 16}. + */ + static MutableBytes16 wrap(MutableBytesValue value) { + checkArgument(value.size() == SIZE, "Expected %s bytes but got %s", SIZE, value.size()); + return new MutableBytes16() { + @Override + public void set(int i, byte b) { + value.set(i, b); + } + + @Override + public MutableBytesValue mutableSlice(int i, int length) { + return value.mutableSlice(i, length); + } + + @Override + public byte get(int i) { + return value.get(i); + } + + @Override + public BytesValue slice(int index) { + return value.slice(index); + } + + @Override + public BytesValue slice(int index, int length) { + return value.slice(index, length); + } + + @Override + public Bytes16 copy() { + return Bytes16.wrap(value.extractArray()); + } + + @Override + public MutableBytes16 mutableCopy() { + return MutableBytes16.wrap(value.extractArray()); + } + + @Override + public void copyTo(MutableBytesValue destination) { + value.copyTo(destination); + } + + @Override + public void copyTo(MutableBytesValue destination, int destinationOffset) { + value.copyTo(destination, destinationOffset); + } + + @Override + public int commonPrefixLength(BytesValue other) { + return value.commonPrefixLength(other); + } + + @Override + public BytesValue commonPrefix(BytesValue other) { + return value.commonPrefix(other); + } + + @Override + public void update(MessageDigest digest) { + value.update(digest); + } + + @Override + public boolean isZero() { + return value.isZero(); + } + + @Override + public boolean equals(Object other) { + return value.equals(other); + } + + @Override + public int hashCode() { + return value.hashCode(); + } + + @Override + public String toString() { + return value.toString(); + } + }; + } +} diff --git a/types/src/main/java/tech/pegasys/artemis/util/bytes/MutableBytes33.java b/types/src/main/java/tech/pegasys/artemis/util/bytes/MutableBytes33.java new file mode 100644 index 000000000..7d536fde9 --- /dev/null +++ b/types/src/main/java/tech/pegasys/artemis/util/bytes/MutableBytes33.java @@ -0,0 +1,145 @@ +/* + * Copyright 2018 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.artemis.util.bytes; + +import java.security.MessageDigest; + +import static com.google.common.base.Preconditions.checkArgument; + +/** + * A mutable {@link Bytes33}, that is a mutable {@link BytesValue} of exactly 33 bytes. + */ +public interface MutableBytes33 extends MutableBytesValue, Bytes33 { + + /** + * Wraps a 33 bytes array as a mutable 33 bytes value. + * + *

+ * This method behave exactly as {@link Bytes33#wrap(byte[])} except that the result is a mutable. + * + * @param value The value to wrap. + * @return A {@link MutableBytes33} wrapping {@code value}. + * @throws IllegalArgumentException if {@code value.length != 33}. + */ + static MutableBytes33 wrap(byte[] value) { + return new MutableArrayWrappingBytes33(value); + } + + /** + * Creates a new mutable 33 bytes value. + * + * @return A newly allocated {@link MutableBytesValue}. + */ + static MutableBytes33 create() { + return new MutableArrayWrappingBytes33(new byte[SIZE]); + } + + /** + * Wraps an existing {@link MutableBytesValue} of size 33 as a mutable 33 bytes value. + * + *

+ * This method does no copy the provided bytes and so any mutation on {@code value} will also be + * reflected in the value returned by this method. If a copy is desirable, this can be simply + * achieved with calling {@link BytesValue#copyTo(MutableBytesValue)} with a newly created + * {@link MutableBytes33} as destination to the copy. + * + * @param value The value to wrap. + * @return A {@link MutableBytes33} wrapping {@code value}. + * @throws IllegalArgumentException if {@code value.size() != 33}. + */ + static MutableBytes33 wrap(MutableBytesValue value) { + checkArgument(value.size() == SIZE, "Expected %s bytes but got %s", SIZE, value.size()); + return new MutableBytes33() { + @Override + public void set(int i, byte b) { + value.set(i, b); + } + + @Override + public MutableBytesValue mutableSlice(int i, int length) { + return value.mutableSlice(i, length); + } + + @Override + public byte get(int i) { + return value.get(i); + } + + @Override + public BytesValue slice(int index) { + return value.slice(index); + } + + @Override + public BytesValue slice(int index, int length) { + return value.slice(index, length); + } + + @Override + public Bytes33 copy() { + return Bytes33.wrap(value.extractArray()); + } + + @Override + public MutableBytes33 mutableCopy() { + return MutableBytes33.wrap(value.extractArray()); + } + + @Override + public void copyTo(MutableBytesValue destination) { + value.copyTo(destination); + } + + @Override + public void copyTo(MutableBytesValue destination, int destinationOffset) { + value.copyTo(destination, destinationOffset); + } + + @Override + public int commonPrefixLength(BytesValue other) { + return value.commonPrefixLength(other); + } + + @Override + public BytesValue commonPrefix(BytesValue other) { + return value.commonPrefix(other); + } + + @Override + public void update(MessageDigest digest) { + value.update(digest); + } + + @Override + public boolean isZero() { + return value.isZero(); + } + + @Override + public boolean equals(Object other) { + return value.equals(other); + } + + @Override + public int hashCode() { + return value.hashCode(); + } + + @Override + public String toString() { + return value.toString(); + } + }; + } +} diff --git a/types/src/main/java/tech/pegasys/artemis/util/bytes/WrappingBytes16.java b/types/src/main/java/tech/pegasys/artemis/util/bytes/WrappingBytes16.java new file mode 100644 index 000000000..c547f10a7 --- /dev/null +++ b/types/src/main/java/tech/pegasys/artemis/util/bytes/WrappingBytes16.java @@ -0,0 +1,62 @@ +/* + * Copyright 2018 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.artemis.util.bytes; + +import static com.google.common.base.Preconditions.checkArgument; + +/** + * A simple class to wrap another {@link BytesValue} of exactly 16 bytes as a {@link Bytes16}. + */ +class WrappingBytes16 extends AbstractBytesValue implements Bytes16 { + + private final BytesValue value; + + WrappingBytes16(BytesValue value) { + checkArgument(value.size() == SIZE, "Expected value to be %s bytes, but is %s bytes", SIZE, + value.size()); + this.value = value; + } + + @Override + public byte get(int i) { + return value.get(i); + } + + @Override + public BytesValue slice(int index, int length) { + return value.slice(index, length); + } + + @Override + public MutableBytes16 mutableCopy() { + MutableBytes16 copy = MutableBytes16.create(); + value.copyTo(copy); + return copy; + } + + @Override + public Bytes16 copy() { + return mutableCopy(); + } + + @Override + public byte[] getArrayUnsafe() { + return value.getArrayUnsafe(); + } + + @Override + public int size() { + return value.size(); + } +} diff --git a/types/src/main/java/tech/pegasys/artemis/util/bytes/WrappingBytes33.java b/types/src/main/java/tech/pegasys/artemis/util/bytes/WrappingBytes33.java new file mode 100644 index 000000000..94f1a3199 --- /dev/null +++ b/types/src/main/java/tech/pegasys/artemis/util/bytes/WrappingBytes33.java @@ -0,0 +1,62 @@ +/* + * Copyright 2018 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.artemis.util.bytes; + +import static com.google.common.base.Preconditions.checkArgument; + +/** + * A simple class to wrap another {@link BytesValue} of exactly 33 bytes as a {@link Bytes33}. + */ +class WrappingBytes33 extends AbstractBytesValue implements Bytes33 { + + private final BytesValue value; + + WrappingBytes33(BytesValue value) { + checkArgument(value.size() == SIZE, "Expected value to be %s bytes, but is %s bytes", SIZE, + value.size()); + this.value = value; + } + + @Override + public byte get(int i) { + return value.get(i); + } + + @Override + public BytesValue slice(int index, int length) { + return value.slice(index, length); + } + + @Override + public MutableBytes33 mutableCopy() { + MutableBytes33 copy = MutableBytes33.create(); + value.copyTo(copy); + return copy; + } + + @Override + public Bytes33 copy() { + return mutableCopy(); + } + + @Override + public byte[] getArrayUnsafe() { + return value.getArrayUnsafe(); + } + + @Override + public int size() { + return value.size(); + } +} diff --git a/versions.gradle b/versions.gradle index 6e33972a9..5f67669a7 100644 --- a/versions.gradle +++ b/versions.gradle @@ -11,6 +11,7 @@ dependencyManagement { dependency "org.apache.logging.log4j:log4j-api:${log4j2Version}" dependency "org.apache.logging.log4j:log4j-core:${log4j2Version}" dependency 'org.ethereum:ethereumj-core:1+' + dependency "org.web3j:core:4.2.0" dependency 'org.bouncycastle:bcprov-jdk15on:1.60' dependency 'org.miracl.milagro.amcl:milagro-crypto-java:0.4.0' From c04072961f936045f2f2b7ecc47cc15a308fb43f Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Tue, 17 Sep 2019 22:21:51 +0300 Subject: [PATCH 02/77] discovery: network implementation using Netty added --- .../beacon/discovery/DiscoveryManager.java | 101 +--------- .../discovery/DiscoveryManagerImpl.java | 123 ++++++++++++ ...etworkPacket.java => NetworkPacketV5.java} | 10 +- .../network/DatagramToBytesValue.java | 21 ++ .../discovery/network/DiscoveryClient.java | 54 ++++++ .../discovery/network/DiscoveryServer.java | 22 +++ .../network/DiscoveryServerImpl.java | 146 ++++++++++++++ .../network/IncomingMessageSink.java | 28 +++ .../discovery/DiscoveryNetworkTest.java | 163 ++++++++++++++++ .../discovery/DiscoveryNoNetworkTest.java | 179 +++++++++++------- .../mock/DiscoveryManagerNoNetwork.java | 109 +++++++++++ discovery/src/test/resources/log4j2.xml | 23 +++ 12 files changed, 809 insertions(+), 170 deletions(-) create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java rename discovery/src/main/java/org/ethereum/beacon/discovery/{NetworkPacket.java => NetworkPacketV5.java} (53%) create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/network/DatagramToBytesValue.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/network/DiscoveryClient.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/network/DiscoveryServer.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/network/DiscoveryServerImpl.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/network/IncomingMessageSink.java create mode 100644 discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java create mode 100644 discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java create mode 100644 discovery/src/test/resources/log4j2.xml diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManager.java b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManager.java index c17e27e6a..349b11619 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManager.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManager.java @@ -1,96 +1,11 @@ package org.ethereum.beacon.discovery; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.ethereum.beacon.discovery.enr.NodeRecord; -import org.ethereum.beacon.discovery.enr.NodeRecordInfo; -import org.ethereum.beacon.discovery.enr.NodeRecordV5; -import org.ethereum.beacon.discovery.packet.UnknownPacket; -import org.ethereum.beacon.discovery.storage.AuthTagRepository; -import org.reactivestreams.Publisher; -import reactor.core.publisher.Flux; -import reactor.core.publisher.FluxSink; -import reactor.core.publisher.ReplayProcessor; -import tech.pegasys.artemis.util.bytes.Bytes32; - -import java.security.SecureRandom; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; - -public class DiscoveryManager { - private static final Logger logger = LogManager.getLogger(DiscoveryManager.class); - private final ReplayProcessor outgoingMessages = ReplayProcessor.cacheLast(); - private final FluxSink outgoingSink = outgoingMessages.sink(); - private final Bytes32 homeNodeId; - private final NodeRecordV5 homeNodeRecord; - private NodeTable nodeTable; - private Publisher incomingPackets; - private Map recentContexts = new ConcurrentHashMap<>(); // nodeId -> context - private AuthTagRepository authTagRepo; - - public DiscoveryManager( - NodeTable nodeTable, Publisher incomingPackets, Bytes32 homeNodeId) { - this.nodeTable = nodeTable; - this.incomingPackets = incomingPackets; - this.homeNodeId = homeNodeId; - this.homeNodeRecord = (NodeRecordV5) nodeTable.getHomeNode(); - this.authTagRepo = new AuthTagRepository(); - } - - public void start() { - Flux.from(incomingPackets) - .subscribe( - unknownPacket -> { - if (unknownPacket.isWhoAreYouPacket(homeNodeId)) { - Optional nodeContextOptional = - authTagRepo.get(unknownPacket.getWhoAreYouPacket().getAuthTag()); - if (nodeContextOptional.isPresent()) { - nodeContextOptional.get().addIncomingEvent(unknownPacket); - } else { - // TODO: ban or whatever - } - } else { - Bytes32 fromNodeId = unknownPacket.getSourceNodeId(homeNodeId); - getContext(fromNodeId) - .ifPresent(context -> context.addIncomingEvent(unknownPacket)); - } - }); - } - - public void connect(NodeRecord nodeRecord) { - if (!nodeTable.getNode(nodeRecord.getNodeId()).isPresent()) { - nodeTable.save(NodeRecordInfo.createDefault(nodeRecord)); - } - NodeContext context = getContext(nodeRecord.getNodeId()).get(); - context.initiate(); - } - - private Optional getContext(Bytes32 nodeId) { - NodeContext context = recentContexts.get(nodeId); - if (context == null) { - Optional nodeOptional = nodeTable.getNode(nodeId); - if (!nodeOptional.isPresent()) { - logger.trace( - () -> String.format("Couldn't find node record for nodeId %s, ignoring", nodeId)); - return Optional.empty(); - } - NodeRecordV5 nodeRecord = nodeOptional.get().getNode(); - SecureRandom random = new SecureRandom(); - context = - new NodeContext( - nodeRecord, - homeNodeRecord, - nodeTable, - authTagRepo, - packet -> outgoingSink.next(new NetworkPacket(packet, nodeRecord)), - random); - } - - return Optional.of(context); - } - - public Publisher getOutgoingMessages() { - return outgoingMessages; // TODO: link to network - } +/** + * Discovery Manager, top interface for discovery mechanism as described at https://github.com/ethereum/devp2p/blob/master/discv5/discv5.md + */ +public interface DiscoveryManager { + void start(); + + void stop(); } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java new file mode 100644 index 000000000..39a32c553 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java @@ -0,0 +1,123 @@ +package org.ethereum.beacon.discovery; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.ethereum.beacon.discovery.enr.NodeRecord; +import org.ethereum.beacon.discovery.enr.NodeRecordInfo; +import org.ethereum.beacon.discovery.enr.NodeRecordV5; +import org.ethereum.beacon.discovery.network.DiscoveryClient; +import org.ethereum.beacon.discovery.network.DiscoveryServer; +import org.ethereum.beacon.discovery.network.DiscoveryServerImpl; +import org.ethereum.beacon.discovery.packet.UnknownPacket; +import org.ethereum.beacon.discovery.storage.AuthTagRepository; +import org.ethereum.beacon.schedulers.Scheduler; +import org.reactivestreams.Publisher; +import reactor.core.publisher.Flux; +import reactor.core.publisher.FluxSink; +import reactor.core.publisher.ReplayProcessor; +import tech.pegasys.artemis.util.bytes.Bytes32; + +import java.security.SecureRandom; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; + +public class DiscoveryManagerImpl implements DiscoveryManager { + private static final Logger logger = LogManager.getLogger(DiscoveryManagerImpl.class); + private final ReplayProcessor outgoingMessages = ReplayProcessor.cacheLast(); + private final FluxSink outgoingSink = outgoingMessages.sink(); + private final Bytes32 homeNodeId; + private final NodeRecordV5 homeNodeRecord; + private final NodeTable nodeTable; + private final Map recentContexts = + new ConcurrentHashMap<>(); // nodeId -> context + private final AuthTagRepository authTagRepo; + private final DiscoveryServer discoveryServer; + private final CompletableFuture discoveryClientAssigned; + private final Scheduler scheduler; + private DiscoveryClient discoveryClient; + + public DiscoveryManagerImpl( + NodeTable nodeTable, NodeRecordV5 homeNode, Scheduler serverScheduler) { + this.nodeTable = nodeTable; + this.homeNodeId = homeNode.getNodeId(); + this.homeNodeRecord = (NodeRecordV5) nodeTable.getHomeNode(); + this.authTagRepo = new AuthTagRepository(); + this.scheduler = serverScheduler; + this.discoveryServer = + new DiscoveryServerImpl(homeNodeRecord.getIpV4address(), homeNodeRecord.getUdpPort()); + discoveryClientAssigned = + discoveryServer.useDatagramChannel( + nioDatagramChannel -> + discoveryClient = new DiscoveryClient(nioDatagramChannel, outgoingMessages)); + } + + @Override + public void start() { + Flux.from(discoveryServer.getIncomingPackets()) + .map(UnknownPacket::new) + .subscribe( + unknownPacket -> { + if (unknownPacket.isWhoAreYouPacket(homeNodeId)) { + Optional nodeContextOptional = + authTagRepo.get(unknownPacket.getWhoAreYouPacket().getAuthTag()); + if (nodeContextOptional.isPresent()) { + nodeContextOptional.get().addIncomingEvent(unknownPacket); + } else { + // TODO: ban or whatever + } + } else { + Bytes32 fromNodeId = unknownPacket.getSourceNodeId(homeNodeId); + getContext(fromNodeId) + .ifPresent(context -> context.addIncomingEvent(unknownPacket)); + } + }); + discoveryServer.start(scheduler); + discoveryClientAssigned.join(); + } + + @Override + public void stop() { + discoveryServer.stop(); + } + + @VisibleForTesting + void connect(NodeRecord nodeRecord) { + if (!nodeTable.getNode(nodeRecord.getNodeId()).isPresent()) { + nodeTable.save(NodeRecordInfo.createDefault(nodeRecord)); + } + NodeContext context = getContext(nodeRecord.getNodeId()).get(); + context.initiate(); + } + + private Optional getContext(Bytes32 nodeId) { + NodeContext context = recentContexts.get(nodeId); + if (context == null) { + Optional nodeOptional = nodeTable.getNode(nodeId); + if (!nodeOptional.isPresent()) { + logger.trace( + () -> String.format("Couldn't find node record for nodeId %s, ignoring", nodeId)); + return Optional.empty(); + } + NodeRecordV5 nodeRecord = nodeOptional.get().getNode(); + SecureRandom random = new SecureRandom(); + context = + new NodeContext( + nodeRecord, + homeNodeRecord, + nodeTable, + authTagRepo, + packet -> outgoingSink.next(new NetworkPacketV5(packet, nodeRecord)), + random); + } + + return Optional.of(context); + } + + @VisibleForTesting + Publisher getOutgoingMessages() { + return outgoingMessages; + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/NetworkPacket.java b/discovery/src/main/java/org/ethereum/beacon/discovery/NetworkPacketV5.java similarity index 53% rename from discovery/src/main/java/org/ethereum/beacon/discovery/NetworkPacket.java rename to discovery/src/main/java/org/ethereum/beacon/discovery/NetworkPacketV5.java index 0465a1316..8cb61c224 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/NetworkPacket.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/NetworkPacketV5.java @@ -1,13 +1,13 @@ package org.ethereum.beacon.discovery; -import org.ethereum.beacon.discovery.enr.NodeRecord; +import org.ethereum.beacon.discovery.enr.NodeRecordV5; import org.ethereum.beacon.discovery.packet.Packet; -public class NetworkPacket { +public class NetworkPacketV5 { private final Packet packet; - private final NodeRecord nodeRecord; + private final NodeRecordV5 nodeRecord; - public NetworkPacket(Packet packet, NodeRecord nodeRecord) { + public NetworkPacketV5(Packet packet, NodeRecordV5 nodeRecord) { this.packet = packet; this.nodeRecord = nodeRecord; } @@ -16,7 +16,7 @@ public Packet getPacket() { return packet; } - public NodeRecord getNodeRecord() { + public NodeRecordV5 getNodeRecord() { return nodeRecord; } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/network/DatagramToBytesValue.java b/discovery/src/main/java/org/ethereum/beacon/discovery/network/DatagramToBytesValue.java new file mode 100644 index 000000000..4be03ad37 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/network/DatagramToBytesValue.java @@ -0,0 +1,21 @@ +package org.ethereum.beacon.discovery.network; + +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.socket.DatagramPacket; +import io.netty.handler.codec.MessageToMessageDecoder; +import tech.pegasys.artemis.util.bytes.BytesValue; + +import java.util.List; + +/** UDP Packet -> BytesValue converter with default Netty interface */ +public class DatagramToBytesValue extends MessageToMessageDecoder { + @Override + protected void decode(ChannelHandlerContext ctx, DatagramPacket msg, List out) + throws Exception { + ByteBuf buf = msg.content(); + byte[] data = new byte[buf.readableBytes()]; + buf.readBytes(data); + out.add(BytesValue.wrap(data)); + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/network/DiscoveryClient.java b/discovery/src/main/java/org/ethereum/beacon/discovery/network/DiscoveryClient.java new file mode 100644 index 000000000..dd6c539fe --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/network/DiscoveryClient.java @@ -0,0 +1,54 @@ +package org.ethereum.beacon.discovery.network; + +import io.netty.buffer.Unpooled; +import io.netty.channel.socket.DatagramPacket; +import io.netty.channel.socket.nio.NioDatagramChannel; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.ethereum.beacon.discovery.NetworkPacketV5; +import org.ethereum.beacon.discovery.enr.NodeRecordV5; +import org.reactivestreams.Publisher; +import reactor.core.publisher.Flux; +import tech.pegasys.artemis.util.bytes.BytesValue; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.UnknownHostException; + +/** Discovery UDP client */ +public class DiscoveryClient { + private static final Logger logger = LogManager.getLogger(DiscoveryClient.class); + private final NioDatagramChannel channel; + + /** + * Constructs UDP client using + * + * @param channel Netty UDP datagram channel + * @param outgoingStream Stream of outgoing packets, client will forward them to the channel + */ + public DiscoveryClient(NioDatagramChannel channel, Publisher outgoingStream) { + this.channel = channel; + Flux.from(outgoingStream) + .subscribe( + networkPacket -> + send(networkPacket.getPacket().getBytes(), networkPacket.getNodeRecord())); + } + + private void send(BytesValue data, NodeRecordV5 nodeRecord) { + InetSocketAddress address; + try { + address = + new InetSocketAddress( + InetAddress.getByAddress(nodeRecord.getIpV4address().extractArray()), + nodeRecord.getUdpPort()); + } catch (UnknownHostException e) { + String error = String.format("Failed to resolve host for node record: %s", nodeRecord); + logger.error(error); + throw new RuntimeException(error); + } + DatagramPacket packet = new DatagramPacket(Unpooled.copiedBuffer(data.extractArray()), address); + logger.trace(() -> String.format("Sending packet %s", packet)); + channel.write(packet); + channel.flush(); + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/network/DiscoveryServer.java b/discovery/src/main/java/org/ethereum/beacon/discovery/network/DiscoveryServer.java new file mode 100644 index 000000000..31970097d --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/network/DiscoveryServer.java @@ -0,0 +1,22 @@ +package org.ethereum.beacon.discovery.network; + +import io.netty.channel.socket.nio.NioDatagramChannel; +import org.ethereum.beacon.schedulers.Scheduler; +import org.reactivestreams.Publisher; +import tech.pegasys.artemis.util.bytes.BytesValue; + +import java.util.concurrent.CompletableFuture; +import java.util.function.Consumer; + +/** Discovery server which listens to the messages according to setup */ +public interface DiscoveryServer { + void start(Scheduler scheduler); + + void stop(); + + /** Raw incoming packets stream */ + Publisher getIncomingPackets(); + + /** Provides safe way to use internal datagram channel for other services */ + CompletableFuture useDatagramChannel(Consumer consumer); +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/network/DiscoveryServerImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/network/DiscoveryServerImpl.java new file mode 100644 index 000000000..56725a30a --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/network/DiscoveryServerImpl.java @@ -0,0 +1,146 @@ +package org.ethereum.beacon.discovery.network; + +import io.netty.bootstrap.Bootstrap; +import io.netty.channel.Channel; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.nio.NioDatagramChannel; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.ethereum.beacon.schedulers.RunnableEx; +import org.ethereum.beacon.schedulers.Scheduler; +import org.reactivestreams.Publisher; +import reactor.core.publisher.FluxSink; +import reactor.core.publisher.ReplayProcessor; +import tech.pegasys.artemis.util.bytes.Bytes4; +import tech.pegasys.artemis.util.bytes.BytesValue; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Consumer; + +public class DiscoveryServerImpl implements DiscoveryServer { + private static final int RECREATION_TIMEOUT = 5000; + private static final int STOPPING_TIMEOUT = 10000; + private static final Logger logger = LogManager.getLogger(DiscoveryServerImpl.class); + private final ReplayProcessor incomingPackets = ReplayProcessor.cacheLast(); + private final FluxSink incomingSink = incomingPackets.sink(); + private final Integer udpListenPort; + private final String udpListenHost; + private AtomicBoolean listen = new AtomicBoolean(true); + private Channel channel; + private NioDatagramChannel datagramChannel; + private Set> datagramChannelUsageQueue = new HashSet<>(); + + public DiscoveryServerImpl(Scheduler scheduler, String udpListenHost, Integer udpListenPort) { + this.udpListenHost = udpListenHost; + this.udpListenPort = udpListenPort; + } + + public DiscoveryServerImpl(Bytes4 udpListenHost, Integer udpListenPort) { + try { + this.udpListenHost = InetAddress.getByAddress(udpListenHost.extractArray()).getHostAddress(); + } catch (UnknownHostException e) { + throw new RuntimeException(e); + } + this.udpListenPort = udpListenPort; + } + + @Override + public void start(Scheduler scheduler) { + logger.info(String.format("Starting discovery server on UDP port %s", udpListenPort)); + scheduler.execute( + new RunnableEx() { + @Override + public void run() throws Exception { + serverLoop(); + } + }); + } + + private void serverLoop() { + NioEventLoopGroup group = new NioEventLoopGroup(1); + try { + while (listen.get()) { + Bootstrap b = new Bootstrap(); + b.group(group) + .channel(NioDatagramChannel.class) + .handler( + new ChannelInitializer() { + @Override + public void initChannel(NioDatagramChannel ch) throws Exception { + ch.pipeline() + .addLast(new DatagramToBytesValue()) + .addLast(new IncomingMessageSink(incomingSink)); + synchronized (DiscoveryServerImpl.class) { + datagramChannel = ch; + datagramChannelUsageQueue.forEach( + nioDatagramChannelConsumer -> nioDatagramChannelConsumer.accept(ch)); + } + } + }); + + channel = b.bind(udpListenHost, udpListenPort).sync().channel(); + channel.closeFuture().sync(); + + if (!listen.get()) { + logger.info("Shutting down discovery server"); + break; + } + logger.error("Discovery server closed. Trying to restore after %s seconds delay"); + Thread.sleep(RECREATION_TIMEOUT); + } + } catch (Exception e) { + logger.error("Can't start discovery server", e); + } finally { + try { + group.shutdownGracefully().sync(); + } catch (Exception ex) { + logger.error("Failed to shutdown discovery sever thread group", ex); + } + } + } + + @Override + public Publisher getIncomingPackets() { + return incomingPackets; + } + + public synchronized CompletableFuture useDatagramChannel( + Consumer consumer) { + CompletableFuture usage = new CompletableFuture<>(); + if (datagramChannel != null) { + consumer.accept(datagramChannel); + usage.complete(null); + } else { + datagramChannelUsageQueue.add( + nioDatagramChannel -> { + consumer.accept(nioDatagramChannel); + usage.complete(null); + }); + } + + return usage; + } + + @Override + public void stop() { + if (listen.get()) { + logger.info("Stopping discovery server"); + listen.set(false); + if (channel != null) { + try { + channel.close().await(STOPPING_TIMEOUT); + } catch (InterruptedException ex) { + logger.error("Failed to stop discovery server", ex); + } + } + } else { + logger.warn("An attempt to stop already stopping/stopped discovery server"); + } + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/network/IncomingMessageSink.java b/discovery/src/main/java/org/ethereum/beacon/discovery/network/IncomingMessageSink.java new file mode 100644 index 000000000..35775b6ea --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/network/IncomingMessageSink.java @@ -0,0 +1,28 @@ +package org.ethereum.beacon.discovery.network; + +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.SimpleChannelInboundHandler; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import reactor.core.publisher.FluxSink; +import tech.pegasys.artemis.util.bytes.BytesValue; + +/** + * Netty interface handler for incoming packets in form of raw bytes data wrapped as {@link + * BytesValue} Implementation forwards all incoming packets in {@link FluxSink} provided via + * constructor, so it could be later linked to processor to form incoming messages stream + */ +public class IncomingMessageSink extends SimpleChannelInboundHandler { + private static final Logger logger = LogManager.getLogger(IncomingMessageSink.class); + private final FluxSink bytesValueSink; + + public IncomingMessageSink(FluxSink bytesValueSink) { + this.bytesValueSink = bytesValueSink; + } + + @Override + protected void channelRead0(ChannelHandlerContext ctx, BytesValue msg) throws Exception { + logger.trace(() -> String.format("Incoming packet %s in context %s", msg, ctx)); + bytesValueSink.next(msg); + } +} diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java new file mode 100644 index 000000000..00a1a9c5a --- /dev/null +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java @@ -0,0 +1,163 @@ +package org.ethereum.beacon.discovery; + +import org.ethereum.beacon.chain.storage.impl.SerializerFactory; +import org.ethereum.beacon.consensus.BeaconChainSpec; +import org.ethereum.beacon.db.Database; +import org.ethereum.beacon.discovery.enr.NodeRecord; +import org.ethereum.beacon.discovery.enr.NodeRecordV5; +import org.ethereum.beacon.discovery.message.DiscoveryMessage; +import org.ethereum.beacon.discovery.message.DiscoveryV5Message; +import org.ethereum.beacon.discovery.message.FindNodeMessage; +import org.ethereum.beacon.discovery.message.NodesMessage; +import org.ethereum.beacon.discovery.packet.AuthHeaderMessagePacket; +import org.ethereum.beacon.discovery.packet.MessagePacket; +import org.ethereum.beacon.discovery.packet.RandomPacket; +import org.ethereum.beacon.discovery.packet.UnknownPacket; +import org.ethereum.beacon.discovery.packet.WhoAreYouPacket; +import org.ethereum.beacon.discovery.storage.NodeTableStorage; +import org.ethereum.beacon.discovery.storage.NodeTableStorageFactoryImpl; +import org.ethereum.beacon.schedulers.Schedulers; +import org.junit.Test; +import reactor.core.publisher.Flux; +import tech.pegasys.artemis.util.bytes.BytesValue; + +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import static org.ethereum.beacon.discovery.NodeContext.DEFAULT_DISTANCE; + +/** Same as {@link DiscoveryNoNetworkTest} but using real network */ +public class DiscoveryNetworkTest { + @Test + public void test() throws Exception { + // 1) start 2 nodes + NodeRecordV5 nodeRecord1 = + NodeRecordV5.Builder.empty() + .withIpV4Address(BytesValue.wrap(InetAddress.getByName("127.0.0.1").getAddress())) + .withSeqNumber(1L) + .withUdpPort(30303) + .withSecp256k1( + BytesValue.fromHexString( + "0bfb48004b1698f05872cf18b1f278998ad8f7d4c135aa41f83744e7b850ab6b98")) + .build(); + NodeRecordV5 nodeRecord2 = + NodeRecordV5.Builder.empty() + .withIpV4Address(BytesValue.wrap(InetAddress.getByName("127.0.0.1").getAddress())) + .withSeqNumber(1L) + .withUdpPort(30304) + .withSecp256k1( + BytesValue.fromHexString( + "7ef3502240a42891771de732f5ee6bee3eb881939edf3e6008c0d07b502756e426")) + .build(); + NodeTableStorageFactoryImpl nodeTableStorageFactory = new NodeTableStorageFactoryImpl(); + Database database1 = Database.inMemoryDB(); + Database database2 = Database.inMemoryDB(); + SerializerFactory serializerFactory = + SerializerFactory.createSSZ(BeaconChainSpec.DEFAULT_CONSTANTS); + NodeTableStorage nodeTableStorage1 = + nodeTableStorageFactory.create( + database1, + serializerFactory, + () -> nodeRecord1, + () -> + new ArrayList() { + { + add(nodeRecord2); + } + }); + NodeTableStorage nodeTableStorage2 = + nodeTableStorageFactory.create( + database2, + serializerFactory, + () -> nodeRecord2, + () -> + new ArrayList() { + { + add(nodeRecord1); + } + }); + DiscoveryManagerImpl discoveryManager1 = + new DiscoveryManagerImpl( + nodeTableStorage1.get(), + nodeRecord1, + Schedulers.createDefault().newSingleThreadDaemon("server-1")); + DiscoveryManagerImpl discoveryManager2 = + new DiscoveryManagerImpl( + nodeTableStorage2.get(), + nodeRecord2, + Schedulers.createDefault().newSingleThreadDaemon("server-2")); + + discoveryManager1.start(); + discoveryManager2.start(); + + // 3) Expect standard 1 => 2 dialog + CountDownLatch randomSent1to2 = new CountDownLatch(1); + CountDownLatch authPacketSent1to2 = new CountDownLatch(1); + CountDownLatch whoareyouSent2to1 = new CountDownLatch(1); + CountDownLatch findNodeSent2to1 = new CountDownLatch(1); + CountDownLatch nodesSent1to2 = new CountDownLatch(1); + Flux.from(discoveryManager1.getOutgoingMessages()) + .map(p -> new UnknownPacket(p.getPacket().getBytes())) + .subscribe( + networkPacket -> { + // 1 -> 2 random + if (randomSent1to2.getCount() != 0) { + RandomPacket randomPacket = networkPacket.getRandomPacket(); + System.out.println("1 => 2: " + randomPacket); + randomSent1to2.countDown(); + } else if (authPacketSent1to2.getCount() != 0) { + // 1 -> 2 auth packet with FINDNODES + AuthHeaderMessagePacket authHeaderMessagePacket = + networkPacket.getAuthHeaderMessagePacket(); + System.out.println("1 => 2: " + authHeaderMessagePacket); + authPacketSent1to2.countDown(); + } else { + // 1 -> 2 NODES packet + MessagePacket messagePacket = networkPacket.getMessagePacket(); + System.out.println("1 => 2: " + messagePacket); + DiscoveryMessage discoveryMessage = messagePacket.getMessage(); + assert IdentityScheme.V5.equals(discoveryMessage.getIdentityScheme()); + DiscoveryV5Message discoveryV5Message = (DiscoveryV5Message) discoveryMessage; + NodesMessage nodesMessage = (NodesMessage) discoveryV5Message.create(); + assert 1 == nodesMessage.getTotal(); + assert 1 == nodesMessage.getNodeRecordsSize(); + assert 1 == nodesMessage.getNodeRecords().size(); + nodesSent1to2.countDown(); + } + }); + Flux.from(discoveryManager2.getOutgoingMessages()) + .map(p -> new UnknownPacket(p.getPacket().getBytes())) + .subscribe( + networkPacket -> { + // 2 -> 1 whoareyou + if (whoareyouSent2to1.getCount() != 0) { + WhoAreYouPacket whoAreYouPacket = networkPacket.getWhoAreYouPacket(); + System.out.println("2 => 1: " + whoAreYouPacket); + whoareyouSent2to1.countDown(); + } else { + // 2 -> 1 findNode + MessagePacket messagePacket = networkPacket.getMessagePacket(); + System.out.println("2 => 1: " + messagePacket); + DiscoveryMessage discoveryMessage = messagePacket.getMessage(); + assert IdentityScheme.V5.equals(discoveryMessage.getIdentityScheme()); + DiscoveryV5Message discoveryV5Message = (DiscoveryV5Message) discoveryMessage; + FindNodeMessage findNodeMessage = (FindNodeMessage) discoveryV5Message.create(); + assert DEFAULT_DISTANCE == findNodeMessage.getDistance(); + findNodeSent2to1.countDown(); + } + }); + + // 4) fire 1 to 2 dialog + discoveryManager1.connect(nodeRecord2); + + assert randomSent1to2.await(1, TimeUnit.SECONDS); + assert whoareyouSent2to1.await(1, TimeUnit.SECONDS); + assert authPacketSent1to2.await(1, TimeUnit.SECONDS); + assert findNodeSent2to1.await(1, TimeUnit.SECONDS); + assert nodesSent1to2.await(1, TimeUnit.SECONDS); + } + + // TODO: discovery tasks are emitted from time to time as they should +} diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java index 3548afceb..1f9d1e97a 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java @@ -9,6 +9,7 @@ import org.ethereum.beacon.discovery.message.DiscoveryV5Message; import org.ethereum.beacon.discovery.message.FindNodeMessage; import org.ethereum.beacon.discovery.message.NodesMessage; +import org.ethereum.beacon.discovery.mock.DiscoveryManagerNoNetwork; import org.ethereum.beacon.discovery.packet.AuthHeaderMessagePacket; import org.ethereum.beacon.discovery.packet.MessagePacket; import org.ethereum.beacon.discovery.packet.RandomPacket; @@ -28,49 +29,78 @@ import java.util.concurrent.TimeUnit; import static org.ethereum.beacon.discovery.NodeContext.DEFAULT_DISTANCE; -import static org.ethereum.beacon.discovery.enr.NodeRecordV5.NODE_ID_FUNCTION; +/** + * Discovery test without real network, instead outgoing stream of each peer is connected with + * incoming of another and vice versa + */ public class DiscoveryNoNetworkTest { @Test public void test() throws Exception { // 1) start 2 nodes - NodeRecordV5 nodeRecord1 = NodeRecordV5.Builder.empty() - .withIpV4Address(BytesValue.wrap(InetAddress.getByName("127.0.0.1").getAddress())) - .withSeqNumber(1L) - .withUdpPort(30303) - .withSecp256k1(BytesValue.fromHexString("0bfb48004b1698f05872cf18b1f278998ad8f7d4c135aa41f83744e7b850ab6b98")) - .build(); - NodeRecordV5 nodeRecord2 = NodeRecordV5.Builder.empty() - .withIpV4Address(BytesValue.wrap(InetAddress.getByName("192.168.0.1").getAddress())) - .withSeqNumber(1L) - .withUdpPort(30303) - .withSecp256k1(BytesValue.fromHexString("7ef3502240a42891771de732f5ee6bee3eb881939edf3e6008c0d07b502756e426")) - .build(); + NodeRecordV5 nodeRecord1 = + NodeRecordV5.Builder.empty() + .withIpV4Address(BytesValue.wrap(InetAddress.getByName("127.0.0.1").getAddress())) + .withSeqNumber(1L) + .withUdpPort(30303) + .withSecp256k1( + BytesValue.fromHexString( + "0bfb48004b1698f05872cf18b1f278998ad8f7d4c135aa41f83744e7b850ab6b98")) + .build(); + NodeRecordV5 nodeRecord2 = + NodeRecordV5.Builder.empty() + .withIpV4Address(BytesValue.wrap(InetAddress.getByName("192.168.0.1").getAddress())) + .withSeqNumber(1L) + .withUdpPort(30303) + .withSecp256k1( + BytesValue.fromHexString( + "7ef3502240a42891771de732f5ee6bee3eb881939edf3e6008c0d07b502756e426")) + .build(); NodeTableStorageFactoryImpl nodeTableStorageFactory = new NodeTableStorageFactoryImpl(); Database database1 = Database.inMemoryDB(); Database database2 = Database.inMemoryDB(); SerializerFactory serializerFactory = SerializerFactory.createSSZ(BeaconChainSpec.DEFAULT_CONSTANTS); - NodeTableStorage nodeTableStorage1 = nodeTableStorageFactory.create(database1, serializerFactory, () -> nodeRecord1, () -> new ArrayList() {{add(nodeRecord2);}}); - NodeTableStorage nodeTableStorage2 = nodeTableStorageFactory.create(database2, serializerFactory, () -> nodeRecord2, () -> new ArrayList() {{add(nodeRecord1);}}); - SimpleProcessor from1to2 = new SimpleProcessor<>(Schedulers.createDefault().newSingleThreadDaemon("from1to2-thread"), "from1to2"); - SimpleProcessor from2to1 = new SimpleProcessor<>(Schedulers.createDefault().newSingleThreadDaemon("from2to1-thread"), "from2to1"); - DiscoveryManager discoveryManager1 = new DiscoveryManager( - nodeTableStorage1.get(), - from2to1, - NODE_ID_FUNCTION.apply(nodeRecord1) - ); - DiscoveryManager discoveryManager2 = new DiscoveryManager( - nodeTableStorage2.get(), - from1to2, - NODE_ID_FUNCTION.apply(nodeRecord2) - ); + NodeTableStorage nodeTableStorage1 = + nodeTableStorageFactory.create( + database1, + serializerFactory, + () -> nodeRecord1, + () -> + new ArrayList() { + { + add(nodeRecord2); + } + }); + NodeTableStorage nodeTableStorage2 = + nodeTableStorageFactory.create( + database2, + serializerFactory, + () -> nodeRecord2, + () -> + new ArrayList() { + { + add(nodeRecord1); + } + }); + SimpleProcessor from1to2 = + new SimpleProcessor<>( + Schedulers.createDefault().newSingleThreadDaemon("from1to2-thread"), "from1to2"); + SimpleProcessor from2to1 = + new SimpleProcessor<>( + Schedulers.createDefault().newSingleThreadDaemon("from2to1-thread"), "from2to1"); + DiscoveryManagerNoNetwork discoveryManager1 = + new DiscoveryManagerNoNetwork(nodeTableStorage1.get(), nodeRecord1, from2to1); + DiscoveryManagerNoNetwork discoveryManager2 = + new DiscoveryManagerNoNetwork(nodeTableStorage2.get(), nodeRecord2, from1to2); discoveryManager1.start(); discoveryManager2.start(); // 2) Link outgoing of each one with incoming of another - Flux.from(discoveryManager1.getOutgoingMessages()).subscribe(t -> from1to2.onNext(new UnknownPacket(t.getPacket().getBytes()))); - Flux.from(discoveryManager2.getOutgoingMessages()).subscribe(t -> from2to1.onNext(new UnknownPacket(t.getPacket().getBytes()))); + Flux.from(discoveryManager1.getOutgoingMessages()) + .subscribe(t -> from1to2.onNext(new UnknownPacket(t.getPacket().getBytes()))); + Flux.from(discoveryManager2.getOutgoingMessages()) + .subscribe(t -> from2to1.onNext(new UnknownPacket(t.getPacket().getBytes()))); // 3) Expect standard 1 => 2 dialog CountDownLatch randomSent1to2 = new CountDownLatch(1); @@ -78,49 +108,54 @@ public void test() throws Exception { CountDownLatch whoareyouSent2to1 = new CountDownLatch(1); CountDownLatch findNodeSent2to1 = new CountDownLatch(1); CountDownLatch nodesSent1to2 = new CountDownLatch(1); - Flux.from(from1to2).subscribe(networkPacket -> { - // 1 -> 2 random - if (randomSent1to2.getCount() != 0) { - RandomPacket randomPacket = networkPacket.getRandomPacket(); - System.out.println("1 => 2: " + randomPacket); - randomSent1to2.countDown(); - } else if (authPacketSent1to2.getCount() != 0) { - // 1 -> 2 auth packet with FINDNODES - AuthHeaderMessagePacket authHeaderMessagePacket = networkPacket.getAuthHeaderMessagePacket(); - System.out.println("1 => 2: " + authHeaderMessagePacket); - authPacketSent1to2.countDown(); - } else { - // 1 -> 2 NODES packet - MessagePacket messagePacket = networkPacket.getMessagePacket(); - System.out.println("1 => 2: " + messagePacket); - DiscoveryMessage discoveryMessage = messagePacket.getMessage(); - assert IdentityScheme.V5.equals(discoveryMessage.getIdentityScheme()); - DiscoveryV5Message discoveryV5Message = (DiscoveryV5Message) discoveryMessage; - NodesMessage nodesMessage = (NodesMessage) discoveryV5Message.create(); - assert 1 == nodesMessage.getTotal(); - assert 1 == nodesMessage.getNodeRecordsSize(); - assert 1 == nodesMessage.getNodeRecords().size(); - nodesSent1to2.countDown(); - } - }); - Flux.from(from2to1).subscribe(networkPacket -> { - // 2 -> 1 whoareyou - if (whoareyouSent2to1.getCount() != 0) { - WhoAreYouPacket whoAreYouPacket = networkPacket.getWhoAreYouPacket(); - System.out.println("2 => 1: " + whoAreYouPacket); - whoareyouSent2to1.countDown(); - } else { - // 2 -> 1 findNode - MessagePacket messagePacket = networkPacket.getMessagePacket(); - System.out.println("2 => 1: " + messagePacket); - DiscoveryMessage discoveryMessage = messagePacket.getMessage(); - assert IdentityScheme.V5.equals(discoveryMessage.getIdentityScheme()); - DiscoveryV5Message discoveryV5Message = (DiscoveryV5Message) discoveryMessage; - FindNodeMessage findNodeMessage = (FindNodeMessage) discoveryV5Message.create(); - assert DEFAULT_DISTANCE == findNodeMessage.getDistance(); - findNodeSent2to1.countDown(); - } - }); + Flux.from(from1to2) + .subscribe( + networkPacket -> { + // 1 -> 2 random + if (randomSent1to2.getCount() != 0) { + RandomPacket randomPacket = networkPacket.getRandomPacket(); + System.out.println("1 => 2: " + randomPacket); + randomSent1to2.countDown(); + } else if (authPacketSent1to2.getCount() != 0) { + // 1 -> 2 auth packet with FINDNODES + AuthHeaderMessagePacket authHeaderMessagePacket = + networkPacket.getAuthHeaderMessagePacket(); + System.out.println("1 => 2: " + authHeaderMessagePacket); + authPacketSent1to2.countDown(); + } else { + // 1 -> 2 NODES packet + MessagePacket messagePacket = networkPacket.getMessagePacket(); + System.out.println("1 => 2: " + messagePacket); + DiscoveryMessage discoveryMessage = messagePacket.getMessage(); + assert IdentityScheme.V5.equals(discoveryMessage.getIdentityScheme()); + DiscoveryV5Message discoveryV5Message = (DiscoveryV5Message) discoveryMessage; + NodesMessage nodesMessage = (NodesMessage) discoveryV5Message.create(); + assert 1 == nodesMessage.getTotal(); + assert 1 == nodesMessage.getNodeRecordsSize(); + assert 1 == nodesMessage.getNodeRecords().size(); + nodesSent1to2.countDown(); + } + }); + Flux.from(from2to1) + .subscribe( + networkPacket -> { + // 2 -> 1 whoareyou + if (whoareyouSent2to1.getCount() != 0) { + WhoAreYouPacket whoAreYouPacket = networkPacket.getWhoAreYouPacket(); + System.out.println("2 => 1: " + whoAreYouPacket); + whoareyouSent2to1.countDown(); + } else { + // 2 -> 1 findNode + MessagePacket messagePacket = networkPacket.getMessagePacket(); + System.out.println("2 => 1: " + messagePacket); + DiscoveryMessage discoveryMessage = messagePacket.getMessage(); + assert IdentityScheme.V5.equals(discoveryMessage.getIdentityScheme()); + DiscoveryV5Message discoveryV5Message = (DiscoveryV5Message) discoveryMessage; + FindNodeMessage findNodeMessage = (FindNodeMessage) discoveryV5Message.create(); + assert DEFAULT_DISTANCE == findNodeMessage.getDistance(); + findNodeSent2to1.countDown(); + } + }); // 4) fire 1 to 2 dialog discoveryManager1.connect(nodeRecord2); diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java b/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java new file mode 100644 index 000000000..f69c27bc3 --- /dev/null +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java @@ -0,0 +1,109 @@ +package org.ethereum.beacon.discovery.mock; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.ethereum.beacon.discovery.DiscoveryManager; +import org.ethereum.beacon.discovery.NetworkPacketV5; +import org.ethereum.beacon.discovery.NodeContext; +import org.ethereum.beacon.discovery.NodeTable; +import org.ethereum.beacon.discovery.enr.NodeRecord; +import org.ethereum.beacon.discovery.enr.NodeRecordInfo; +import org.ethereum.beacon.discovery.enr.NodeRecordV5; +import org.ethereum.beacon.discovery.packet.UnknownPacket; +import org.ethereum.beacon.discovery.storage.AuthTagRepository; +import org.reactivestreams.Publisher; +import reactor.core.publisher.Flux; +import reactor.core.publisher.FluxSink; +import reactor.core.publisher.ReplayProcessor; +import tech.pegasys.artemis.util.bytes.Bytes32; + +import java.security.SecureRandom; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Implementation of {@link DiscoveryManager} without network as an opposite to Netty network + * implementation {@link org.ethereum.beacon.discovery.DiscoveryManagerImpl} Outgoing packets could + * be obtained from `outgoingMessages` publisher, using {@link #getOutgoingMessages()}, incoming + * packets could be provided through the constructor parameter `incomingPackets` + */ +public class DiscoveryManagerNoNetwork implements DiscoveryManager { + private static final Logger logger = LogManager.getLogger(DiscoveryManager.class); + private final ReplayProcessor outgoingMessages = ReplayProcessor.cacheLast(); + private final FluxSink outgoingSink = outgoingMessages.sink(); + private final Bytes32 homeNodeId; + private final NodeRecordV5 homeNodeRecord; + private NodeTable nodeTable; + private Publisher incomingPackets; + private Map recentContexts = new ConcurrentHashMap<>(); // nodeId -> context + private AuthTagRepository authTagRepo; + + public DiscoveryManagerNoNetwork( + NodeTable nodeTable, NodeRecordV5 homeNode, Publisher incomingPackets) { + this.nodeTable = nodeTable; + this.incomingPackets = incomingPackets; + this.homeNodeId = homeNode.getNodeId(); + this.homeNodeRecord = (NodeRecordV5) nodeTable.getHomeNode(); + this.authTagRepo = new AuthTagRepository(); + } + + public void start() { + Flux.from(incomingPackets) + .subscribe( + unknownPacket -> { + if (unknownPacket.isWhoAreYouPacket(homeNodeId)) { + Optional nodeContextOptional = + authTagRepo.get(unknownPacket.getWhoAreYouPacket().getAuthTag()); + if (nodeContextOptional.isPresent()) { + nodeContextOptional.get().addIncomingEvent(unknownPacket); + } else { + // TODO: ban or whatever + } + } else { + Bytes32 fromNodeId = unknownPacket.getSourceNodeId(homeNodeId); + getContext(fromNodeId) + .ifPresent(context -> context.addIncomingEvent(unknownPacket)); + } + }); + } + + @Override + public void stop() {} + + public void connect(NodeRecord nodeRecord) { + if (!nodeTable.getNode(nodeRecord.getNodeId()).isPresent()) { + nodeTable.save(NodeRecordInfo.createDefault(nodeRecord)); + } + NodeContext context = getContext(nodeRecord.getNodeId()).get(); + context.initiate(); + } + + private Optional getContext(Bytes32 nodeId) { + NodeContext context = recentContexts.get(nodeId); + if (context == null) { + Optional nodeOptional = nodeTable.getNode(nodeId); + if (!nodeOptional.isPresent()) { + logger.trace( + () -> String.format("Couldn't find node record for nodeId %s, ignoring", nodeId)); + return Optional.empty(); + } + NodeRecordV5 nodeRecord = nodeOptional.get().getNode(); + SecureRandom random = new SecureRandom(); + context = + new NodeContext( + nodeRecord, + homeNodeRecord, + nodeTable, + authTagRepo, + packet -> outgoingSink.next(new NetworkPacketV5(packet, nodeRecord)), + random); + } + + return Optional.of(context); + } + + public Publisher getOutgoingMessages() { + return outgoingMessages; + } +} diff --git a/discovery/src/test/resources/log4j2.xml b/discovery/src/test/resources/log4j2.xml new file mode 100644 index 000000000..6e8add2a1 --- /dev/null +++ b/discovery/src/test/resources/log4j2.xml @@ -0,0 +1,23 @@ + + + + + + + + %d{HH:mm:ss.SSS} %-5level - %msg%n + + + + + + + + + + + + + + + From d3f2051ff3e2e05a57d00cf24e6bd0891b5f22e2 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Wed, 18 Sep 2019 11:34:52 +0300 Subject: [PATCH 03/77] discovery: call top network abstraction a parcel to avoid confusing with packet --- .../beacon/discovery/DiscoveryManager.java | 2 +- .../discovery/DiscoveryManagerImpl.java | 10 ++++++---- .../discovery/network/DiscoveryClient.java | 19 +++++++++++++------ .../discovery/network/NetworkParcel.java | 16 ++++++++++++++++ .../NetworkParcelV5.java} | 11 +++++++---- .../mock/DiscoveryManagerNoNetwork.java | 11 ++++++----- 6 files changed, 49 insertions(+), 20 deletions(-) create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/network/NetworkParcel.java rename discovery/src/main/java/org/ethereum/beacon/discovery/{NetworkPacketV5.java => network/NetworkParcelV5.java} (54%) diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManager.java b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManager.java index 349b11619..9d40a8706 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManager.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManager.java @@ -1,7 +1,7 @@ package org.ethereum.beacon.discovery; /** - * Discovery Manager, top interface for discovery mechanism as described at https://github.com/ethereum/devp2p/blob/master/discv5/discv5.md */ public interface DiscoveryManager { diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java index 39a32c553..fdee6f409 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java @@ -9,6 +9,8 @@ import org.ethereum.beacon.discovery.network.DiscoveryClient; import org.ethereum.beacon.discovery.network.DiscoveryServer; import org.ethereum.beacon.discovery.network.DiscoveryServerImpl; +import org.ethereum.beacon.discovery.network.NetworkParcel; +import org.ethereum.beacon.discovery.network.NetworkParcelV5; import org.ethereum.beacon.discovery.packet.UnknownPacket; import org.ethereum.beacon.discovery.storage.AuthTagRepository; import org.ethereum.beacon.schedulers.Scheduler; @@ -26,8 +28,8 @@ public class DiscoveryManagerImpl implements DiscoveryManager { private static final Logger logger = LogManager.getLogger(DiscoveryManagerImpl.class); - private final ReplayProcessor outgoingMessages = ReplayProcessor.cacheLast(); - private final FluxSink outgoingSink = outgoingMessages.sink(); + private final ReplayProcessor outgoingMessages = ReplayProcessor.cacheLast(); + private final FluxSink outgoingSink = outgoingMessages.sink(); private final Bytes32 homeNodeId; private final NodeRecordV5 homeNodeRecord; private final NodeTable nodeTable; @@ -109,7 +111,7 @@ private Optional getContext(Bytes32 nodeId) { homeNodeRecord, nodeTable, authTagRepo, - packet -> outgoingSink.next(new NetworkPacketV5(packet, nodeRecord)), + packet -> outgoingSink.next(new NetworkParcelV5(packet, nodeRecord)), random); } @@ -117,7 +119,7 @@ private Optional getContext(Bytes32 nodeId) { } @VisibleForTesting - Publisher getOutgoingMessages() { + Publisher getOutgoingMessages() { return outgoingMessages; } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/network/DiscoveryClient.java b/discovery/src/main/java/org/ethereum/beacon/discovery/network/DiscoveryClient.java index dd6c539fe..67e1abd24 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/network/DiscoveryClient.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/network/DiscoveryClient.java @@ -5,7 +5,7 @@ import io.netty.channel.socket.nio.NioDatagramChannel; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.ethereum.beacon.discovery.NetworkPacketV5; +import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.enr.NodeRecordV5; import org.reactivestreams.Publisher; import reactor.core.publisher.Flux; @@ -26,7 +26,7 @@ public class DiscoveryClient { * @param channel Netty UDP datagram channel * @param outgoingStream Stream of outgoing packets, client will forward them to the channel */ - public DiscoveryClient(NioDatagramChannel channel, Publisher outgoingStream) { + public DiscoveryClient(NioDatagramChannel channel, Publisher outgoingStream) { this.channel = channel; Flux.from(outgoingStream) .subscribe( @@ -34,15 +34,22 @@ public DiscoveryClient(NioDatagramChannel channel, Publisher ou send(networkPacket.getPacket().getBytes(), networkPacket.getNodeRecord())); } - private void send(BytesValue data, NodeRecordV5 nodeRecord) { + private void send(BytesValue data, NodeRecord recipient) { + if (!(recipient instanceof NodeRecordV5)) { + String error = + String.format( + "Accepts only V5 versions of recipient's node records. Got %s instead", recipient); + logger.error(error); + throw new RuntimeException(error); + } InetSocketAddress address; try { address = new InetSocketAddress( - InetAddress.getByAddress(nodeRecord.getIpV4address().extractArray()), - nodeRecord.getUdpPort()); + InetAddress.getByAddress(((NodeRecordV5) recipient).getIpV4address().extractArray()), + ((NodeRecordV5) recipient).getUdpPort()); } catch (UnknownHostException e) { - String error = String.format("Failed to resolve host for node record: %s", nodeRecord); + String error = String.format("Failed to resolve host for node record: %s", recipient); logger.error(error); throw new RuntimeException(error); } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/network/NetworkParcel.java b/discovery/src/main/java/org/ethereum/beacon/discovery/network/NetworkParcel.java new file mode 100644 index 000000000..339958dfc --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/network/NetworkParcel.java @@ -0,0 +1,16 @@ +package org.ethereum.beacon.discovery.network; + +import org.ethereum.beacon.discovery.enr.NodeRecord; +import org.ethereum.beacon.discovery.packet.Packet; + +/** + * Abstraction on the top of the {@link Packet}. + * + *

Stores `packet` and associated node record. Record could be a sender or recipient, depends on + * context. + */ +public interface NetworkParcel { + Packet getPacket(); + + NodeRecord getNodeRecord(); +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/NetworkPacketV5.java b/discovery/src/main/java/org/ethereum/beacon/discovery/network/NetworkParcelV5.java similarity index 54% rename from discovery/src/main/java/org/ethereum/beacon/discovery/NetworkPacketV5.java rename to discovery/src/main/java/org/ethereum/beacon/discovery/network/NetworkParcelV5.java index 8cb61c224..8d36e8178 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/NetworkPacketV5.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/network/NetworkParcelV5.java @@ -1,22 +1,25 @@ -package org.ethereum.beacon.discovery; +package org.ethereum.beacon.discovery.network; +import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.enr.NodeRecordV5; import org.ethereum.beacon.discovery.packet.Packet; -public class NetworkPacketV5 { +public class NetworkParcelV5 implements NetworkParcel { private final Packet packet; private final NodeRecordV5 nodeRecord; - public NetworkPacketV5(Packet packet, NodeRecordV5 nodeRecord) { + public NetworkParcelV5(Packet packet, NodeRecordV5 nodeRecord) { this.packet = packet; this.nodeRecord = nodeRecord; } + @Override public Packet getPacket() { return packet; } - public NodeRecordV5 getNodeRecord() { + @Override + public NodeRecord getNodeRecord() { return nodeRecord; } } diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java b/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java index f69c27bc3..2ec4b02ed 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java @@ -3,12 +3,13 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.ethereum.beacon.discovery.DiscoveryManager; -import org.ethereum.beacon.discovery.NetworkPacketV5; import org.ethereum.beacon.discovery.NodeContext; import org.ethereum.beacon.discovery.NodeTable; import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.enr.NodeRecordInfo; import org.ethereum.beacon.discovery.enr.NodeRecordV5; +import org.ethereum.beacon.discovery.network.NetworkParcel; +import org.ethereum.beacon.discovery.network.NetworkParcelV5; import org.ethereum.beacon.discovery.packet.UnknownPacket; import org.ethereum.beacon.discovery.storage.AuthTagRepository; import org.reactivestreams.Publisher; @@ -30,8 +31,8 @@ */ public class DiscoveryManagerNoNetwork implements DiscoveryManager { private static final Logger logger = LogManager.getLogger(DiscoveryManager.class); - private final ReplayProcessor outgoingMessages = ReplayProcessor.cacheLast(); - private final FluxSink outgoingSink = outgoingMessages.sink(); + private final ReplayProcessor outgoingMessages = ReplayProcessor.cacheLast(); + private final FluxSink outgoingSink = outgoingMessages.sink(); private final Bytes32 homeNodeId; private final NodeRecordV5 homeNodeRecord; private NodeTable nodeTable; @@ -96,14 +97,14 @@ private Optional getContext(Bytes32 nodeId) { homeNodeRecord, nodeTable, authTagRepo, - packet -> outgoingSink.next(new NetworkPacketV5(packet, nodeRecord)), + packet -> outgoingSink.next(new NetworkParcelV5(packet, nodeRecord)), random); } return Optional.of(context); } - public Publisher getOutgoingMessages() { + public Publisher getOutgoingMessages() { return outgoingMessages; } } From fe91d72d1e6fbd3262268e8c38f4698e238e9bd0 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Wed, 18 Sep 2019 12:36:18 +0300 Subject: [PATCH 04/77] discovery: add draft for sign() method and signature recovery --- .../ethereum/beacon/discovery/Functions.java | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java b/discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java index d577ee70a..5189649c5 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java @@ -13,12 +13,16 @@ import org.bouncycastle.math.ec.ECCurve; import org.ethereum.beacon.crypto.Hashes; import org.javatuples.Triplet; +import org.web3j.crypto.ECDSASignature; +import org.web3j.crypto.ECKeyPair; +import org.web3j.crypto.Sign; import tech.pegasys.artemis.util.bytes.Bytes32; import tech.pegasys.artemis.util.bytes.BytesValue; import javax.crypto.Cipher; import javax.crypto.spec.IvParameterSpec; import javax.crypto.spec.SecretKeySpec; +import java.math.BigInteger; import java.security.SecureRandom; import java.util.Random; @@ -33,8 +37,20 @@ public static Bytes32 hash(BytesValue value) { /** Creates a signature of x using the given key */ public static BytesValue sign(BytesValue key, BytesValue x) { - // TODO: implement - return x; + ECDSASignature signature = ECKeyPair.create(key.extractArray()).sign(x.extractArray()); + Bytes32 r = Bytes32.wrap(signature.r.toByteArray()); + Bytes32 s = Bytes32.wrap(signature.s.toByteArray()); + return r.concat(s); + } + + public static BytesValue recoverFromSignature(BytesValue signature, BytesValue x) { + assert 64 == signature.size(); + BigInteger r = new BigInteger(signature.slice(0, 32).extractArray()); + BigInteger s = new BigInteger(signature.slice(32).extractArray()); + ECDSASignature ecdsaSignature = new ECDSASignature(r, s); + // FIXME: recId, which number should it use? + BigInteger pubKey = Sign.recoverFromSignature(0, ecdsaSignature, x.extractArray()); + return BytesValue.wrap(pubKey.toByteArray()); } /** From d0394f20ab8cf85706d81e7c5d0938f0e93a1d58 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Wed, 18 Sep 2019 16:55:58 +0300 Subject: [PATCH 05/77] discovery: cleanup DiscoveryServer interface --- .../discovery/DiscoveryManagerImpl.java | 10 ++++++---- .../discovery/network/DiscoveryServer.java | 9 +-------- .../network/DiscoveryServerImpl.java | 2 +- .../network/NettyDiscoveryServer.java | 20 +++++++++++++++++++ 4 files changed, 28 insertions(+), 13 deletions(-) create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/network/NettyDiscoveryServer.java diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java index fdee6f409..1781e6fd3 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java @@ -9,6 +9,7 @@ import org.ethereum.beacon.discovery.network.DiscoveryClient; import org.ethereum.beacon.discovery.network.DiscoveryServer; import org.ethereum.beacon.discovery.network.DiscoveryServerImpl; +import org.ethereum.beacon.discovery.network.NettyDiscoveryServer; import org.ethereum.beacon.discovery.network.NetworkParcel; import org.ethereum.beacon.discovery.network.NetworkParcelV5; import org.ethereum.beacon.discovery.packet.UnknownPacket; @@ -50,10 +51,11 @@ public DiscoveryManagerImpl( this.scheduler = serverScheduler; this.discoveryServer = new DiscoveryServerImpl(homeNodeRecord.getIpV4address(), homeNodeRecord.getUdpPort()); - discoveryClientAssigned = - discoveryServer.useDatagramChannel( - nioDatagramChannel -> - discoveryClient = new DiscoveryClient(nioDatagramChannel, outgoingMessages)); + this.discoveryClientAssigned = + ((NettyDiscoveryServer) discoveryServer) + .useDatagramChannel( + nioDatagramChannel -> + discoveryClient = new DiscoveryClient(nioDatagramChannel, outgoingMessages)); } @Override diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/network/DiscoveryServer.java b/discovery/src/main/java/org/ethereum/beacon/discovery/network/DiscoveryServer.java index 31970097d..985b2d9e7 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/network/DiscoveryServer.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/network/DiscoveryServer.java @@ -1,14 +1,10 @@ package org.ethereum.beacon.discovery.network; -import io.netty.channel.socket.nio.NioDatagramChannel; import org.ethereum.beacon.schedulers.Scheduler; import org.reactivestreams.Publisher; import tech.pegasys.artemis.util.bytes.BytesValue; -import java.util.concurrent.CompletableFuture; -import java.util.function.Consumer; - -/** Discovery server which listens to the messages according to setup */ +/** Discovery server which listens to incoming messages according to setup */ public interface DiscoveryServer { void start(Scheduler scheduler); @@ -16,7 +12,4 @@ public interface DiscoveryServer { /** Raw incoming packets stream */ Publisher getIncomingPackets(); - - /** Provides safe way to use internal datagram channel for other services */ - CompletableFuture useDatagramChannel(Consumer consumer); } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/network/DiscoveryServerImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/network/DiscoveryServerImpl.java index 56725a30a..1ef045031 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/network/DiscoveryServerImpl.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/network/DiscoveryServerImpl.java @@ -23,7 +23,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; -public class DiscoveryServerImpl implements DiscoveryServer { +public class DiscoveryServerImpl implements NettyDiscoveryServer { private static final int RECREATION_TIMEOUT = 5000; private static final int STOPPING_TIMEOUT = 10000; private static final Logger logger = LogManager.getLogger(DiscoveryServerImpl.class); diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/network/NettyDiscoveryServer.java b/discovery/src/main/java/org/ethereum/beacon/discovery/network/NettyDiscoveryServer.java new file mode 100644 index 000000000..f3a77e1b8 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/network/NettyDiscoveryServer.java @@ -0,0 +1,20 @@ +package org.ethereum.beacon.discovery.network; + +import io.netty.channel.socket.nio.NioDatagramChannel; + +import java.util.concurrent.CompletableFuture; +import java.util.function.Consumer; + +public interface NettyDiscoveryServer extends DiscoveryServer { + + /** + * Provides safe way to use internal datagram channel for other services. Channel is usually not + * available on request, instead it will be available at some time in the future. After server + * starts and some events happened. Return future is fired when it finally happens. + * + * @param consumer Consumer which needs Netty datagram channel. For example, UDP client + * application could share the channel with the server + * @return Future which is completed when channel is provided to consumer + */ + CompletableFuture useDatagramChannel(Consumer consumer); +} From c5fa4fa1ef23a1436e1af10b04b776096e1c0a31 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Wed, 18 Sep 2019 18:11:46 +0300 Subject: [PATCH 06/77] discovery: enums cleanup --- .../main/java/org/ethereum/beacon/discovery/MessageCode.java | 2 +- .../src/main/java/org/ethereum/beacon/discovery/NodeStatus.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/MessageCode.java b/discovery/src/main/java/org/ethereum/beacon/discovery/MessageCode.java index 261191edd..ae528c6af 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/MessageCode.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/MessageCode.java @@ -61,7 +61,7 @@ public enum MessageCode { private int code; - private MessageCode(int code) { + MessageCode(int code) { this.code = code; } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeStatus.java b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeStatus.java index 3b001ab40..481056e20 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeStatus.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeStatus.java @@ -30,7 +30,7 @@ public enum NodeStatus { private int code; - private NodeStatus(int code) { + NodeStatus(int code) { this.code = code; } From 88c540ec626e1e2cc82b3d35dca005acf3988e85 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Wed, 18 Sep 2019 18:34:41 +0300 Subject: [PATCH 07/77] discovery: resolve all NodeTable TODOs --- .../ethereum/beacon/discovery/Functions.java | 15 +++++ .../beacon/discovery/NodeTableImpl.java | 64 +++++++++---------- .../storage/NodeTableStorageImpl.java | 2 +- .../beacon/discovery/FunctionsTest.java | 27 ++++++++ .../beacon/discovery/NodeTableTest.java | 26 +++++++- 5 files changed, 95 insertions(+), 39 deletions(-) create mode 100644 discovery/src/test/java/org/ethereum/beacon/discovery/FunctionsTest.java diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java b/discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java index 5189649c5..e574bf7cd 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java @@ -17,6 +17,7 @@ import org.web3j.crypto.ECKeyPair; import org.web3j.crypto.Sign; import tech.pegasys.artemis.util.bytes.Bytes32; +import tech.pegasys.artemis.util.bytes.Bytes32s; import tech.pegasys.artemis.util.bytes.BytesValue; import javax.crypto.Cipher; @@ -145,4 +146,18 @@ public static Triplet hkdf_expand( public static Random getRandom() { return new SecureRandom(); } + + public static int logDistance(Bytes32 nodeId1, Bytes32 nodeId2) { + BigInteger distance = new BigInteger(1, Bytes32s.xor(nodeId1, nodeId2).extractArray()); + return log2(distance.doubleValue()); + } + + /** + * Logarithm with base 2. See https://stackoverflow.com/a/3305400 for + * implementation details. + */ + private static int log2(double x) { + return (int) Math.floor((Math.log(x) / Math.log(2.0) + 1e-10)); + } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeTableImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeTableImpl.java index 04a864a67..0b6b2022e 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeTableImpl.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeTableImpl.java @@ -1,11 +1,13 @@ package org.ethereum.beacon.discovery; +import com.google.common.annotations.VisibleForTesting; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.ethereum.beacon.db.source.DataSource; import org.ethereum.beacon.db.source.HoleyList; import org.ethereum.beacon.db.source.SingleValueSource; import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.enr.NodeRecordInfo; -import org.ethereum.beacon.discovery.enr.NodeRecordV5; import tech.pegasys.artemis.ethereum.core.Hash32; import tech.pegasys.artemis.util.bytes.Bytes32; import tech.pegasys.artemis.util.bytes.BytesValue; @@ -16,36 +18,27 @@ import java.util.List; import java.util.Optional; import java.util.Set; -import java.util.function.Function; public class NodeTableImpl implements NodeTable { - private static final long NUMBER_OF_INDEXES = 256; + static final long NUMBER_OF_INDEXES = 256; + private static final Logger logger = LogManager.getLogger(NodeTableImpl.class); private static final int MAXIMUM_INFO_IN_ONE_BYTE = 256; private static final boolean START_FROM_BEGINNING = true; private final DataSource nodeTable; private final HoleyList indexTable; - private final Function nodeKeyFunction; private final SingleValueSource homeNodeSource; public NodeTableImpl( DataSource nodeTable, HoleyList indexTable, - SingleValueSource homeNodeSource, - Function hashFunction) { + SingleValueSource homeNodeSource) { this.nodeTable = nodeTable; this.indexTable = indexTable; - this.nodeKeyFunction = - nodeRecord -> { - if (!(nodeRecord instanceof NodeRecordV5)) { - throw new RuntimeException( - ""); // TODO: exception text - } - return hashFunction.apply(((NodeRecordV5) nodeRecord).getPublicKey()); - }; this.homeNodeSource = homeNodeSource; } - private long getNodeIndex(Bytes32 nodeKey) { + @VisibleForTesting + static long getNodeIndex(Bytes32 nodeKey) { int activeBytes = 1; long required = NUMBER_OF_INDEXES; while (required > 0) { @@ -60,21 +53,18 @@ private long getNodeIndex(Bytes32 nodeKey) { } } - int start = - START_FROM_BEGINNING - ? 0 - : nodeKey.size() - activeBytes; // FIXME: check for +-1 error for end index + int start = START_FROM_BEGINNING ? 0 : nodeKey.size() - activeBytes; BytesValue active = nodeKey.slice(start, activeBytes); - BigInteger activeNumber = new BigInteger(1, active.extractArray()); // FIXME: is signum ok? - BigInteger index = - activeNumber.mod(BigInteger.valueOf(NUMBER_OF_INDEXES)); // FIXME: do we require BI here? + BigInteger activeNumber = new BigInteger(1, active.extractArray()); + // XXX: could be optimized for small NUMBER_OF_INDEXES + BigInteger index = activeNumber.mod(BigInteger.valueOf(NUMBER_OF_INDEXES)); return index.longValue(); } @Override public void save(NodeRecordInfo node) { - Hash32 nodeKey = nodeKeyFunction.apply(node.getNode()); + Hash32 nodeKey = Hash32.wrap(node.getNode().getNodeId()); nodeTable.put(nodeKey, node); NodeIndex activeIndex = indexTable.get(getNodeIndex(nodeKey)).orElseGet(NodeIndex::new); List nodes = activeIndex.getEntries(); @@ -97,12 +87,18 @@ public List findClosestNodes(Bytes32 nodeId, int logLimit) { long currentIndexDown = start; Set res = new HashSet<>(); while (!limitReached) { - Optional upNodesOptional = indexTable.get(currentIndexUp); - Optional downNodesOptional = indexTable.get(currentIndexDown); + Optional upNodesOptional = + currentIndexUp >= NUMBER_OF_INDEXES ? Optional.empty() : indexTable.get(currentIndexUp); + Optional downNodesOptional = + currentIndexDown < 0 ? Optional.empty() : indexTable.get(currentIndexDown); + if (currentIndexUp >= NUMBER_OF_INDEXES && currentIndexDown < 0) { + // Bounds are reached from both top and bottom + break; + } if (upNodesOptional.isPresent()) { NodeIndex upNodes = upNodesOptional.get(); for (Hash32 currentNodeId : upNodes.getEntries()) { - if (logDistance(currentNodeId, nodeId) >= logLimit) { + if (Functions.logDistance(currentNodeId, nodeId) >= logLimit) { limitReached = true; break; } else { @@ -112,8 +108,11 @@ public List findClosestNodes(Bytes32 nodeId, int logLimit) { } if (downNodesOptional.isPresent()) { NodeIndex downNodes = downNodesOptional.get(); - for (Hash32 currentNodeId : downNodes.getEntries()) { - if (logDistance(currentNodeId, nodeId) >= logLimit) { + List entries = downNodes.getEntries(); + // XXX: iterate in reverse order to reach logDistance limit from the right side + for (int i = entries.size() - 1; i >= 0; i--) { + Hash32 currentNodeId = entries.get(i); + if (Functions.logDistance(currentNodeId, nodeId) >= logLimit) { limitReached = true; break; } else { @@ -121,18 +120,13 @@ public List findClosestNodes(Bytes32 nodeId, int logLimit) { } } } - currentIndexUp++; // FIXME: bounds - currentIndexDown--; // FIXME: bounds + currentIndexUp++; + currentIndexDown--; } return new ArrayList<>(res); } - private int logDistance(Bytes32 nodeId1, Bytes32 nodeId2) { - // TODO: real formula - return Math.abs((nodeId1.get(0) & 0xFF) - (nodeId2.get(0) & 0xFF)); - } - @Override public NodeRecord getHomeNode() { return homeNodeSource.get().map(NodeRecordInfo::getNode).orElse(null); diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageImpl.java index 1c9fe62f4..b2bb07b3a 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageImpl.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageImpl.java @@ -46,7 +46,7 @@ public NodeTableStorageImpl(Database database, SerializerFactory serializerFacto serializerFactory.getSerializer(NodeIndex.class), serializerFactory.getDeserializer(NodeIndex.class)); this.homeNodeSource = SingleValueSource.fromDataSource(nodeTable, HOME_NODE_KEY); - this.nodeTable = new NodeTableImpl(nodeTable, nodeIndexesTable, homeNodeSource, Hashes::sha256); + this.nodeTable = new NodeTableImpl(nodeTable, nodeIndexesTable, homeNodeSource); } @Override diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/FunctionsTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/FunctionsTest.java new file mode 100644 index 000000000..fd2eac170 --- /dev/null +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/FunctionsTest.java @@ -0,0 +1,27 @@ +package org.ethereum.beacon.discovery; + +import org.junit.Test; +import tech.pegasys.artemis.util.bytes.Bytes32; + +import static org.junit.Assert.assertEquals; + +public class FunctionsTest { + @Test + public void testLogDistance() { + Bytes32 nodeId0 = Bytes32.fromHexString("0000000000000000000000000000000000000000000000000000000000000000"); + Bytes32 nodeId1a = Bytes32.fromHexString("0000000000000000000000000000000000000000000000000000000000000001"); + Bytes32 nodeId1b = Bytes32.fromHexString("1000000000000000000000000000000000000000000000000000000000000000"); + Bytes32 nodeId1s = Bytes32.fromHexString("1111111111111111111111111111111111111111111111111111111111111111"); + Bytes32 nodeId9s = Bytes32.fromHexString("9999999999999999999999999999999999999999999999999999999999999999"); + Bytes32 nodeIdfs = Bytes32.fromHexString("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"); + assertEquals(0, Functions.logDistance(nodeId0, nodeId1a)); + // So it's big endian + assertEquals(252, Functions.logDistance(nodeId0, nodeId1b)); + assertEquals(252, Functions.logDistance(nodeId0, nodeId1s)); + assertEquals(255, Functions.logDistance(nodeId0, nodeId9s)); + // maximum distance + assertEquals(256, Functions.logDistance(nodeId0, nodeIdfs)); + // logDistance is not an additive function + assertEquals(255, Functions.logDistance(nodeId1s, nodeIdfs)); + } +} diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/NodeTableTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/NodeTableTest.java index 3eb8dd90c..50456f3dc 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/NodeTableTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/NodeTableTest.java @@ -9,6 +9,7 @@ import org.ethereum.beacon.discovery.storage.NodeTableStorage; import org.ethereum.beacon.discovery.storage.NodeTableStorageFactoryImpl; import org.junit.Test; +import tech.pegasys.artemis.util.bytes.Bytes32; import tech.pegasys.artemis.util.bytes.BytesValue; import java.net.InetAddress; @@ -100,12 +101,31 @@ public void testFind() throws Exception { .withSecp256k1(BytesValue.fromHexString("bafb48004b1698f05872cf18b1f278998ad8f7d4c135aa41f83744e7b850ab6b98")) .build(); nodeTableStorage.get().save(new NodeRecordInfo(farNode, -1L, NodeStatus.ACTIVE, 0)); - List closestNodes = nodeTableStorage.get().findClosestNodes(NODE_ID_FUNCTION.apply(closestNode), 1); + List closestNodes = nodeTableStorage.get().findClosestNodes(NODE_ID_FUNCTION.apply(closestNode), 252); assertEquals(2, closestNodes.size()); - assertEquals(closestNodes.get(0).getNode().getPublicKey(), closestNode.getPublicKey()); - assertEquals(closestNodes.get(1).getNode().getPublicKey(), localHostNode.getPublicKey()); + assertEquals(closestNodes.get(0).getNode().getPublicKey(), localHostNode.getPublicKey()); + assertEquals(closestNodes.get(1).getNode().getPublicKey(), closestNode.getPublicKey()); List farNodes = nodeTableStorage.get().findClosestNodes(NODE_ID_FUNCTION.apply(farNode), 1); assertEquals(1, farNodes.size()); assertEquals(farNodes.get(0).getNode().getPublicKey(), farNode.getPublicKey()); } + + /** + * Verifies that calculated index number is in range of [0, {@link NodeTableImpl#NUMBER_OF_INDEXES}) + */ + @Test + public void testIndexCalculation() { + Bytes32 nodeId0 = Bytes32.fromHexString("0000000000000000000000000000000000000000000000000000000000000000"); + Bytes32 nodeId1a = Bytes32.fromHexString("0000000000000000000000000000000000000000000000000000000000000001"); + Bytes32 nodeId1b = Bytes32.fromHexString("1000000000000000000000000000000000000000000000000000000000000000"); + Bytes32 nodeId1s = Bytes32.fromHexString("1111111111111111111111111111111111111111111111111111111111111111"); + Bytes32 nodeId9s = Bytes32.fromHexString("9999999999999999999999999999999999999999999999999999999999999999"); + Bytes32 nodeIdfs = Bytes32.fromHexString("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"); + assertEquals(0, NodeTableImpl.getNodeIndex(nodeId0)); + assertEquals(0, NodeTableImpl.getNodeIndex(nodeId1a)); + assertEquals(16, NodeTableImpl.getNodeIndex(nodeId1b)); + assertEquals(17, NodeTableImpl.getNodeIndex(nodeId1s)); + assertEquals(153, NodeTableImpl.getNodeIndex(nodeId9s)); + assertEquals(255, NodeTableImpl.getNodeIndex(nodeIdfs)); + } } From f7c6f1878af2bc28caf7d63a65723e143fa9fadd Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Wed, 18 Sep 2019 19:57:56 +0300 Subject: [PATCH 08/77] discovery: organize structures by usage --- .../beacon/discovery/DiscoverTask.java | 2 ++ .../discovery/DiscoveryManagerImpl.java | 1 + .../discovery/DiscoveryV5MessageHandler.java | 2 ++ .../ethereum/beacon/discovery/Functions.java | 10 +++++++ .../beacon/discovery/NodeContext.java | 2 ++ .../beacon/discovery/enr/NodeRecordInfo.java | 1 - .../discovery/{ => enr}/NodeStatus.java | 29 ++++++++++++++----- .../discovery/message/DiscoveryV5Message.java | 9 +++--- .../discovery/message/FindNodeMessage.java | 12 ++------ .../discovery/{ => message}/MessageCode.java | 2 +- .../discovery/message/NodesMessage.java | 1 - .../beacon/discovery/message/PingMessage.java | 1 - .../beacon/discovery/message/PongMessage.java | 6 +--- .../discovery/{ => storage}/NodeIndex.java | 3 +- .../discovery/{ => storage}/NodeTable.java | 7 ++++- .../{ => storage}/NodeTableImpl.java | 11 ++++++- .../discovery/storage/NodeTableStorage.java | 1 - .../storage/NodeTableStorageFactoryImpl.java | 1 - .../storage/NodeTableStorageImpl.java | 3 -- .../beacon/discovery/NodeTableTest.java | 2 ++ .../mock/DiscoveryManagerNoNetwork.java | 2 +- 21 files changed, 69 insertions(+), 39 deletions(-) rename discovery/src/main/java/org/ethereum/beacon/discovery/{ => enr}/NodeStatus.java (71%) rename discovery/src/main/java/org/ethereum/beacon/discovery/{ => message}/MessageCode.java (97%) rename discovery/src/main/java/org/ethereum/beacon/discovery/{ => storage}/NodeIndex.java (84%) rename discovery/src/main/java/org/ethereum/beacon/discovery/{ => storage}/NodeTable.java (62%) rename discovery/src/main/java/org/ethereum/beacon/discovery/{ => storage}/NodeTableImpl.java (90%) diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoverTask.java b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoverTask.java index c3e5ffaf5..baa1f59b7 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoverTask.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoverTask.java @@ -2,6 +2,8 @@ import org.ethereum.beacon.discovery.enr.NodeRecordInfo; import org.ethereum.beacon.discovery.enr.NodeRecordV5; +import org.ethereum.beacon.discovery.enr.NodeStatus; +import org.ethereum.beacon.discovery.storage.NodeTable; import tech.pegasys.artemis.util.bytes.Bytes32; import java.util.List; diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java index 1781e6fd3..bcc0c4c59 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java @@ -14,6 +14,7 @@ import org.ethereum.beacon.discovery.network.NetworkParcelV5; import org.ethereum.beacon.discovery.packet.UnknownPacket; import org.ethereum.beacon.discovery.storage.AuthTagRepository; +import org.ethereum.beacon.discovery.storage.NodeTable; import org.ethereum.beacon.schedulers.Scheduler; import org.reactivestreams.Publisher; import reactor.core.publisher.Flux; diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryV5MessageHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryV5MessageHandler.java index e7e36d70a..1c3008f8d 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryV5MessageHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryV5MessageHandler.java @@ -2,9 +2,11 @@ import org.ethereum.beacon.discovery.enr.NodeRecordInfo; import org.ethereum.beacon.discovery.message.DiscoveryV5Message; +import org.ethereum.beacon.discovery.message.MessageCode; import org.ethereum.beacon.discovery.message.NodesMessage; import org.ethereum.beacon.discovery.message.PongMessage; import org.ethereum.beacon.discovery.packet.MessagePacket; +import org.ethereum.beacon.discovery.storage.NodeTable; import tech.pegasys.artemis.util.bytes.Bytes32; import tech.pegasys.artemis.util.bytes.BytesValue; diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java b/discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java index e574bf7cd..24f33c5ec 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java @@ -147,6 +147,16 @@ public static Random getRandom() { return new SecureRandom(); } + /** + * The 'distance' between two node IDs is the bitwise XOR of the IDs, taken as the number. + * + *

distance(n₁, n₂) = n₁ XOR n₂ + * + *

In many situations, the logarithmic distance (i.e. length of common prefix in bits) is used + * in place of the actual distance. + * + *

logdistance(n₁, n₂) = log2(distance(n₁, n₂)) + */ public static int logDistance(Bytes32 nodeId1, Bytes32 nodeId2) { BigInteger distance = new BigInteger(1, Bytes32s.xor(nodeId1, nodeId2).extractArray()); return log2(distance.doubleValue()); diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeContext.java b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeContext.java index e1d88d9e0..d61b605b4 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeContext.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeContext.java @@ -5,6 +5,7 @@ import org.ethereum.beacon.discovery.enr.NodeRecordV5; import org.ethereum.beacon.discovery.message.DiscoveryV5Message; import org.ethereum.beacon.discovery.message.FindNodeMessage; +import org.ethereum.beacon.discovery.message.MessageCode; import org.ethereum.beacon.discovery.packet.AuthHeaderMessagePacket; import org.ethereum.beacon.discovery.packet.MessagePacket; import org.ethereum.beacon.discovery.packet.Packet; @@ -12,6 +13,7 @@ import org.ethereum.beacon.discovery.packet.UnknownPacket; import org.ethereum.beacon.discovery.packet.WhoAreYouPacket; import org.ethereum.beacon.discovery.storage.AuthTagRepository; +import org.ethereum.beacon.discovery.storage.NodeTable; import org.javatuples.Triplet; import org.web3j.crypto.ECKeyPair; import tech.pegasys.artemis.util.bytes.Bytes32; diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordInfo.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordInfo.java index 438669349..c80d0da3b 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordInfo.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordInfo.java @@ -1,7 +1,6 @@ package org.ethereum.beacon.discovery.enr; import com.google.common.base.Objects; -import org.ethereum.beacon.discovery.NodeStatus; import org.ethereum.beacon.ssz.annotation.SSZ; import org.ethereum.beacon.ssz.annotation.SSZSerializable; diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeStatus.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeStatus.java similarity index 71% rename from discovery/src/main/java/org/ethereum/beacon/discovery/NodeStatus.java rename to discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeStatus.java index 481056e20..a1064bd9d 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeStatus.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeStatus.java @@ -1,14 +1,11 @@ -package org.ethereum.beacon.discovery; +package org.ethereum.beacon.discovery.enr; -import org.ethereum.beacon.ssz.access.SSZBasicAccessor; import org.ethereum.beacon.ssz.access.SSZField; import org.ethereum.beacon.ssz.access.basic.UIntPrimitive; import org.ethereum.beacon.ssz.annotation.SSZSerializable; import org.ethereum.beacon.ssz.visitor.SSZReader; -import tech.pegasys.artemis.util.bytes.Bytes4; import java.io.OutputStream; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -45,7 +42,11 @@ public byte byteCode() { public static class NodeStatusAccessor extends UIntPrimitive { @Override public Set getSupportedClasses() { - return new HashSet() {{add(NodeStatus.class);}}; + return new HashSet() { + { + add(NodeStatus.class); + } + }; } @Override @@ -56,13 +57,27 @@ public int getSize(SSZField field) { @Override public void encode(Object value, SSZField field, OutputStream result) { NodeStatus nodeStatus = (NodeStatus) value; - SSZField overrided = new SSZField(byte.class, field.getFieldAnnotation(), "uint", 8, field.getName(), field.getGetter()); + SSZField overrided = + new SSZField( + byte.class, + field.getFieldAnnotation(), + "uint", + 8, + field.getName(), + field.getGetter()); super.encode(nodeStatus.byteCode(), overrided, result); } @Override public Object decode(SSZField field, SSZReader reader) { - SSZField overrided = new SSZField(byte.class, field.getFieldAnnotation(), "uint", 8, field.getName(), field.getGetter()); + SSZField overrided = + new SSZField( + byte.class, + field.getFieldAnnotation(), + "uint", + 8, + field.getName(), + field.getGetter()); int code = (int) super.decode(overrided, reader); return NodeStatus.fromNumber(code); } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/DiscoveryV5Message.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/DiscoveryV5Message.java index 491f1af0d..71e87f6f3 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/message/DiscoveryV5Message.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/DiscoveryV5Message.java @@ -1,7 +1,6 @@ package org.ethereum.beacon.discovery.message; import org.ethereum.beacon.discovery.IdentityScheme; -import org.ethereum.beacon.discovery.MessageCode; import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.enr.NodeRecordV5; import org.web3j.rlp.RlpDecoder; @@ -21,6 +20,10 @@ public DiscoveryV5Message(BytesValue bytes) { this.bytes = bytes; } + public static DiscoveryV5Message from(V5Message v5Message) { + return new DiscoveryV5Message(v5Message.getBytes()); + } + @Override public BytesValue getBytes() { return bytes; @@ -93,8 +96,4 @@ public V5Message create() { } } } - - public static DiscoveryV5Message from(V5Message v5Message) { - return new DiscoveryV5Message(v5Message.getBytes()); - } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/FindNodeMessage.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/FindNodeMessage.java index f61abea34..7f2b0f71b 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/message/FindNodeMessage.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/FindNodeMessage.java @@ -1,7 +1,6 @@ package org.ethereum.beacon.discovery.message; import com.google.common.base.Objects; -import org.ethereum.beacon.discovery.MessageCode; import org.web3j.rlp.RlpEncoder; import org.web3j.rlp.RlpList; import org.web3j.rlp.RlpString; @@ -41,8 +40,7 @@ public BytesValue getBytes() { BytesValue.wrap( RlpEncoder.encode( new RlpList( - RlpString.create(requestId.extractArray()), - RlpString.create(distance))))); + RlpString.create(requestId.extractArray()), RlpString.create(distance))))); } @Override @@ -50,8 +48,7 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; FindNodeMessage that = (FindNodeMessage) o; - return Objects.equal(requestId, that.requestId) && - Objects.equal(distance, that.distance); + return Objects.equal(requestId, that.requestId) && Objects.equal(distance, that.distance); } @Override @@ -61,9 +58,6 @@ public int hashCode() { @Override public String toString() { - return "FindNodeMessage{" + - "requestId=" + requestId + - ", distance=" + distance + - '}'; + return "FindNodeMessage{" + "requestId=" + requestId + ", distance=" + distance + '}'; } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/MessageCode.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/MessageCode.java similarity index 97% rename from discovery/src/main/java/org/ethereum/beacon/discovery/MessageCode.java rename to discovery/src/main/java/org/ethereum/beacon/discovery/message/MessageCode.java index ae528c6af..6c06118b2 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/MessageCode.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/MessageCode.java @@ -1,4 +1,4 @@ -package org.ethereum.beacon.discovery; +package org.ethereum.beacon.discovery.message; import java.util.HashMap; import java.util.Map; diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/NodesMessage.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/NodesMessage.java index 7e748c178..bed61fea7 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/message/NodesMessage.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/NodesMessage.java @@ -1,7 +1,6 @@ package org.ethereum.beacon.discovery.message; import com.google.common.base.Objects; -import org.ethereum.beacon.discovery.MessageCode; import org.ethereum.beacon.discovery.enr.NodeRecordV5; import org.web3j.rlp.RlpEncoder; import org.web3j.rlp.RlpList; diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/PingMessage.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/PingMessage.java index be3cd726c..2033602db 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/message/PingMessage.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/PingMessage.java @@ -1,7 +1,6 @@ package org.ethereum.beacon.discovery.message; import com.google.common.base.Objects; -import org.ethereum.beacon.discovery.MessageCode; import org.web3j.rlp.RlpEncoder; import org.web3j.rlp.RlpList; import org.web3j.rlp.RlpString; diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/PongMessage.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/PongMessage.java index fe6ca72f1..ce988bc9c 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/message/PongMessage.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/PongMessage.java @@ -1,7 +1,6 @@ package org.ethereum.beacon.discovery.message; import com.google.common.base.Objects; -import org.ethereum.beacon.discovery.MessageCode; import org.web3j.rlp.RlpEncoder; import org.web3j.rlp.RlpList; import org.web3j.rlp.RlpString; @@ -20,10 +19,7 @@ public class PongMessage implements V5Message { private final Integer recipientPort; public PongMessage( - BytesValue requestId, - Long enrSeq, - BytesValue recipientIp, - Integer recipientPort) { + BytesValue requestId, Long enrSeq, BytesValue recipientIp, Integer recipientPort) { this.requestId = requestId; this.enrSeq = enrSeq; this.recipientIp = recipientIp; diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeIndex.java b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeIndex.java similarity index 84% rename from discovery/src/main/java/org/ethereum/beacon/discovery/NodeIndex.java rename to discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeIndex.java index 76cf2ed5f..28d08dc14 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeIndex.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeIndex.java @@ -1,4 +1,4 @@ -package org.ethereum.beacon.discovery; +package org.ethereum.beacon.discovery.storage; import org.ethereum.beacon.ssz.annotation.SSZ; import org.ethereum.beacon.ssz.annotation.SSZSerializable; @@ -7,6 +7,7 @@ import java.util.ArrayList; import java.util.List; +/** Node Index. Stores several node keys. */ @SSZSerializable public class NodeIndex { @SSZ private List entries; diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeTable.java b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTable.java similarity index 62% rename from discovery/src/main/java/org/ethereum/beacon/discovery/NodeTable.java rename to discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTable.java index ae05889df..9a3a6f2b9 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeTable.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTable.java @@ -1,4 +1,4 @@ -package org.ethereum.beacon.discovery; +package org.ethereum.beacon.discovery.storage; import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.enr.NodeRecordInfo; @@ -7,11 +7,16 @@ import java.util.List; import java.util.Optional; +/** + * Stores Ethereum Node Records in {@link NodeRecordInfo} containers. Also stores home node as node + * record. + */ public interface NodeTable { void save(NodeRecordInfo node); Optional getNode(Bytes32 nodeId); + /** Returns list of nodes including `nodeId` (if it's found) in logLimit distance from it. */ List findClosestNodes(Bytes32 nodeId, int logLimit); NodeRecord getHomeNode(); diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeTableImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableImpl.java similarity index 90% rename from discovery/src/main/java/org/ethereum/beacon/discovery/NodeTableImpl.java rename to discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableImpl.java index 0b6b2022e..c754b277f 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeTableImpl.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableImpl.java @@ -1,4 +1,4 @@ -package org.ethereum.beacon.discovery; +package org.ethereum.beacon.discovery.storage; import com.google.common.annotations.VisibleForTesting; import org.apache.logging.log4j.LogManager; @@ -6,6 +6,7 @@ import org.ethereum.beacon.db.source.DataSource; import org.ethereum.beacon.db.source.HoleyList; import org.ethereum.beacon.db.source.SingleValueSource; +import org.ethereum.beacon.discovery.Functions; import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.enr.NodeRecordInfo; import tech.pegasys.artemis.ethereum.core.Hash32; @@ -19,6 +20,10 @@ import java.util.Optional; import java.util.Set; +/** + * Stores Ethereum Node Records in {@link NodeRecordInfo} containers. Also stores home node as node + * record. Uses indexes, {@link NodeIndex} for quick access to nodes that are close to others. + */ public class NodeTableImpl implements NodeTable { static final long NUMBER_OF_INDEXES = 256; private static final Logger logger = LogManager.getLogger(NodeTableImpl.class); @@ -79,6 +84,10 @@ public Optional getNode(Bytes32 nodeId) { return nodeTable.get(Hash32.wrap(nodeId)); } + /** + * Returns list of nodes including `nodeId` (if it's found) in logLimit distance from it. Uses + * {@link Functions#logDistance(Bytes32, Bytes32)} as distance function. + */ @Override public List findClosestNodes(Bytes32 nodeId, int logLimit) { long start = getNodeIndex(nodeId); diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorage.java b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorage.java index 0e1ef7d00..2a636d0da 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorage.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorage.java @@ -2,7 +2,6 @@ import org.ethereum.beacon.db.source.SingleValueSource; import org.ethereum.beacon.discovery.enr.NodeRecordInfo; -import org.ethereum.beacon.discovery.NodeTable; /** Stores {@link NodeTable} and home node info */ public interface NodeTableStorage { diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactoryImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactoryImpl.java index aa6d5ba7a..449ddc2ff 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactoryImpl.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactoryImpl.java @@ -5,7 +5,6 @@ import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.enr.NodeRecordInfo; import org.ethereum.beacon.discovery.enr.NodeRecordV5; -import org.ethereum.beacon.discovery.NodeStatus; import java.util.List; import java.util.function.Supplier; diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageImpl.java index b2bb07b3a..b41f61c3d 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageImpl.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageImpl.java @@ -8,10 +8,7 @@ import org.ethereum.beacon.db.source.HoleyList; import org.ethereum.beacon.db.source.SingleValueSource; import org.ethereum.beacon.db.source.impl.DataSourceList; -import org.ethereum.beacon.discovery.NodeIndex; import org.ethereum.beacon.discovery.enr.NodeRecordInfo; -import org.ethereum.beacon.discovery.NodeTable; -import org.ethereum.beacon.discovery.NodeTableImpl; import tech.pegasys.artemis.ethereum.core.Hash32; import tech.pegasys.artemis.util.bytes.BytesValue; diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/NodeTableTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/NodeTableTest.java index 50456f3dc..472fb10fd 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/NodeTableTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/NodeTableTest.java @@ -6,6 +6,8 @@ import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.enr.NodeRecordInfo; import org.ethereum.beacon.discovery.enr.NodeRecordV5; +import org.ethereum.beacon.discovery.enr.NodeStatus; +import org.ethereum.beacon.discovery.storage.NodeTableImpl; import org.ethereum.beacon.discovery.storage.NodeTableStorage; import org.ethereum.beacon.discovery.storage.NodeTableStorageFactoryImpl; import org.junit.Test; diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java b/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java index 2ec4b02ed..ebb2b0480 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java @@ -4,7 +4,7 @@ import org.apache.logging.log4j.Logger; import org.ethereum.beacon.discovery.DiscoveryManager; import org.ethereum.beacon.discovery.NodeContext; -import org.ethereum.beacon.discovery.NodeTable; +import org.ethereum.beacon.discovery.storage.NodeTable; import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.enr.NodeRecordInfo; import org.ethereum.beacon.discovery.enr.NodeRecordV5; From 29552750f3e92f80c9774b824a739c4cd25c5a56 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Wed, 18 Sep 2019 19:59:50 +0300 Subject: [PATCH 09/77] discovery: fix NodeTable test --- .../ethereum/beacon/discovery/{ => storage}/NodeTableTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename discovery/src/test/java/org/ethereum/beacon/discovery/{ => storage}/NodeTableTest.java (99%) diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/NodeTableTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeTableTest.java similarity index 99% rename from discovery/src/test/java/org/ethereum/beacon/discovery/NodeTableTest.java rename to discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeTableTest.java index 472fb10fd..23a70771f 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/NodeTableTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeTableTest.java @@ -1,4 +1,4 @@ -package org.ethereum.beacon.discovery; +package org.ethereum.beacon.discovery.storage; import org.ethereum.beacon.chain.storage.impl.SerializerFactory; import org.ethereum.beacon.consensus.BeaconChainSpec; From 19a8729f68862d8e5ee1d2c6cba8779020a9e761 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Wed, 18 Sep 2019 20:38:31 +0300 Subject: [PATCH 10/77] discovery: add cleanup of old contexts --- .../discovery/DiscoveryManagerImpl.java | 11 ++++++ .../discovery/DiscoveryV5MessageHandler.java | 32 +++++++--------- .../beacon/util/ExpirationScheduler.java | 37 +++++++++++++++++++ 3 files changed, 62 insertions(+), 18 deletions(-) create mode 100644 util/src/main/java/org/ethereum/beacon/util/ExpirationScheduler.java diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java index bcc0c4c59..98a6efa6d 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java @@ -16,6 +16,7 @@ import org.ethereum.beacon.discovery.storage.AuthTagRepository; import org.ethereum.beacon.discovery.storage.NodeTable; import org.ethereum.beacon.schedulers.Scheduler; +import org.ethereum.beacon.util.ExpirationScheduler; import org.reactivestreams.Publisher; import reactor.core.publisher.Flux; import reactor.core.publisher.FluxSink; @@ -27,8 +28,10 @@ import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; public class DiscoveryManagerImpl implements DiscoveryManager { + private static final int CLEANUP_DELAY_SECONDS = 180; private static final Logger logger = LogManager.getLogger(DiscoveryManagerImpl.class); private final ReplayProcessor outgoingMessages = ReplayProcessor.cacheLast(); private final FluxSink outgoingSink = outgoingMessages.sink(); @@ -37,6 +40,8 @@ public class DiscoveryManagerImpl implements DiscoveryManager { private final NodeTable nodeTable; private final Map recentContexts = new ConcurrentHashMap<>(); // nodeId -> context + private ExpirationScheduler contextExpirationScheduler = + new ExpirationScheduler<>(CLEANUP_DELAY_SECONDS, TimeUnit.SECONDS); private final AuthTagRepository authTagRepo; private final DiscoveryServer discoveryServer; private final CompletableFuture discoveryClientAssigned; @@ -116,8 +121,14 @@ private Optional getContext(Bytes32 nodeId) { authTagRepo, packet -> outgoingSink.next(new NetworkParcelV5(packet, nodeRecord)), random); + recentContexts.put(nodeId, context); } + final NodeContext contextBackup = context; + contextExpirationScheduler.put(context.getNodeRecord().getNodeId(), () -> { + recentContexts.remove(contextBackup.getNodeRecord().getNodeId()); + contextBackup.cleanup(); + }); return Optional.of(context); } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryV5MessageHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryV5MessageHandler.java index 1c3008f8d..7bc84e6d1 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryV5MessageHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryV5MessageHandler.java @@ -7,14 +7,13 @@ import org.ethereum.beacon.discovery.message.PongMessage; import org.ethereum.beacon.discovery.packet.MessagePacket; import org.ethereum.beacon.discovery.storage.NodeTable; +import org.ethereum.beacon.util.ExpirationScheduler; import tech.pegasys.artemis.util.bytes.Bytes32; import tech.pegasys.artemis.util.bytes.BytesValue; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -23,17 +22,19 @@ import static org.ethereum.beacon.discovery.NodeContext.DEFAULT_DISTANCE; public class DiscoveryV5MessageHandler implements DiscoveryMessageHandler { - public static final int CLEANUP_DELAY = 60; + private static final int CLEANUP_DELAY_SECONDS = 60; private static final int MAX_NODES_IN_MSG = 24; private final Bytes32 homeNodeId; private final BytesValue initiatorKey; private final BytesValue authTag; private final NodeTable nodeTable; - ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(); + private ExpirationScheduler nodeExpirationScheduler = + new ExpirationScheduler<>(CLEANUP_DELAY_SECONDS, TimeUnit.SECONDS); private Map nodesCounters = new ConcurrentHashMap<>(); private Map nodesExpirationTasks = new ConcurrentHashMap<>(); - public DiscoveryV5MessageHandler(Bytes32 homeNodeId, BytesValue initiatorKey, BytesValue authTag, NodeTable nodeTable) { + public DiscoveryV5MessageHandler( + Bytes32 homeNodeId, BytesValue initiatorKey, BytesValue authTag, NodeTable nodeTable) { this.homeNodeId = homeNodeId; this.initiatorKey = initiatorKey; this.authTag = authTag; @@ -103,12 +104,12 @@ public void handleMessage(DiscoveryV5Message message, NodeContext context) { cleanUp(nodesMessage.getRequestId(), context); } else { nodesCounters.put(nodesMessage.getRequestId(), counter); - reScheduleCleanUpDelay(nodesMessage.getRequestId(), context); + updateExpiration(nodesMessage.getRequestId(), context); } } } else if (nodesMessage.getTotal() > 1) { nodesCounters.put(nodesMessage.getRequestId(), nodesMessage.getTotal() - 1); - reScheduleCleanUpDelay(nodesMessage.getRequestId(), context); + updateExpiration(nodesMessage.getRequestId(), context); } else { context.clearRequestId(nodesMessage.getRequestId(), MessageCode.FINDNODE); } @@ -133,21 +134,16 @@ public void handleMessage(DiscoveryV5Message message, NodeContext context) { } } - private synchronized void reScheduleCleanUpDelay(BytesValue requestId, NodeContext context) { - nodesExpirationTasks.remove(requestId).cancel(true); - ScheduledFuture future = - scheduler.schedule( - () -> { - cleanUp(requestId, context); - }, - CLEANUP_DELAY, - TimeUnit.SECONDS); - nodesExpirationTasks.put(requestId, future); + private synchronized void updateExpiration(BytesValue requestId, NodeContext context) { + nodeExpirationScheduler.put( + requestId, + () -> { + cleanUp(requestId, context); + }); } private synchronized void cleanUp(BytesValue requestId, NodeContext context) { nodesCounters.remove(requestId); - nodesExpirationTasks.remove(requestId); context.clearRequestId(requestId, MessageCode.FINDNODE); } } diff --git a/util/src/main/java/org/ethereum/beacon/util/ExpirationScheduler.java b/util/src/main/java/org/ethereum/beacon/util/ExpirationScheduler.java new file mode 100644 index 000000000..a14403736 --- /dev/null +++ b/util/src/main/java/org/ethereum/beacon/util/ExpirationScheduler.java @@ -0,0 +1,37 @@ +package org.ethereum.beacon.util; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +/** + * Schedules `runnable` in delay which is set by constructor. When runnable is renewed by putting it + * in map again, old task is cancelled and removed. Task are equalled by the + */ +public class ExpirationScheduler { + private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(); + private final long delay; + private final TimeUnit timeUnit; + private Map expirationTasks = new ConcurrentHashMap<>(); + + public ExpirationScheduler(long delay, TimeUnit timeUnit) { + this.delay = delay; + this.timeUnit = timeUnit; + } + + public void put(Key key, Runnable runnable) { + expirationTasks.remove(key).cancel(true); + ScheduledFuture future = + scheduler.schedule( + () -> { + runnable.run(); + expirationTasks.remove(key); + }, + delay, + timeUnit); + expirationTasks.put(key, future); + } +} From 82854c6d8f65b5794e77264639c3858df8f49895 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Wed, 18 Sep 2019 21:20:54 +0300 Subject: [PATCH 11/77] discovery: refactor message handling with context --- ...er.java => DiscoveryMessageProcessor.java} | 4 +- .../discovery/DiscoveryV5MessageHandler.java | 149 ------------------ .../DiscoveryV5MessageProcessor.java | 39 +++++ .../beacon/discovery/MessageHandler.java | 34 ---- .../beacon/discovery/MessageProcessor.java | 33 ++++ .../beacon/discovery/NodeContext.java | 34 +++- .../message/handler/FindNodeHandler.java | 47 ++++++ .../message/handler/MessageHandler.java | 7 + .../message/handler/NodesHandler.java | 68 ++++++++ .../message/handler/PingHandler.java | 26 +++ .../message/handler/PongHandler.java | 15 ++ 11 files changed, 265 insertions(+), 191 deletions(-) rename discovery/src/main/java/org/ethereum/beacon/discovery/{DiscoveryMessageHandler.java => DiscoveryMessageProcessor.java} (58%) delete mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryV5MessageHandler.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryV5MessageProcessor.java delete mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/MessageHandler.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/MessageProcessor.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/FindNodeHandler.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/MessageHandler.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/NodesHandler.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/PingHandler.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/PongHandler.java diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryMessageHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryMessageProcessor.java similarity index 58% rename from discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryMessageHandler.java rename to discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryMessageProcessor.java index a7e162d6d..9d26de022 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryMessageHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryMessageProcessor.java @@ -2,6 +2,8 @@ import org.ethereum.beacon.discovery.message.DiscoveryMessage; -public interface DiscoveryMessageHandler { +public interface DiscoveryMessageProcessor { + IdentityScheme getSupportedIdentity(); + void handleMessage(M message, NodeContext context); } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryV5MessageHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryV5MessageHandler.java deleted file mode 100644 index 7bc84e6d1..000000000 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryV5MessageHandler.java +++ /dev/null @@ -1,149 +0,0 @@ -package org.ethereum.beacon.discovery; - -import org.ethereum.beacon.discovery.enr.NodeRecordInfo; -import org.ethereum.beacon.discovery.message.DiscoveryV5Message; -import org.ethereum.beacon.discovery.message.MessageCode; -import org.ethereum.beacon.discovery.message.NodesMessage; -import org.ethereum.beacon.discovery.message.PongMessage; -import org.ethereum.beacon.discovery.packet.MessagePacket; -import org.ethereum.beacon.discovery.storage.NodeTable; -import org.ethereum.beacon.util.ExpirationScheduler; -import tech.pegasys.artemis.util.bytes.Bytes32; -import tech.pegasys.artemis.util.bytes.BytesValue; - -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Collectors; - -import static org.ethereum.beacon.discovery.NodeContext.DEFAULT_DISTANCE; - -public class DiscoveryV5MessageHandler implements DiscoveryMessageHandler { - private static final int CLEANUP_DELAY_SECONDS = 60; - private static final int MAX_NODES_IN_MSG = 24; - private final Bytes32 homeNodeId; - private final BytesValue initiatorKey; - private final BytesValue authTag; - private final NodeTable nodeTable; - private ExpirationScheduler nodeExpirationScheduler = - new ExpirationScheduler<>(CLEANUP_DELAY_SECONDS, TimeUnit.SECONDS); - private Map nodesCounters = new ConcurrentHashMap<>(); - private Map nodesExpirationTasks = new ConcurrentHashMap<>(); - - public DiscoveryV5MessageHandler( - Bytes32 homeNodeId, BytesValue initiatorKey, BytesValue authTag, NodeTable nodeTable) { - this.homeNodeId = homeNodeId; - this.initiatorKey = initiatorKey; - this.authTag = authTag; - this.nodeTable = nodeTable; - } - - @Override - public void handleMessage(DiscoveryV5Message message, NodeContext context) { - MessageCode code = message.getCode(); - switch (code) { - case PING: - { - PongMessage responseMessage = - new PongMessage( - message.getRequestId(), - context.getNodeRecord().getSeqNumber(), - context.getNodeRecord().getIpV4address(), - context.getNodeRecord().getUdpPort()); - context.addOutgoingEvent( - MessagePacket.create( - homeNodeId, - context.getNodeRecord().getNodeId(), - authTag, - initiatorKey, - DiscoveryV5Message.from(responseMessage))); - break; - } - case PONG: - { - // FIXME: do we need to update NodeRecord from PONG response? - context.clearRequestId(message.getRequestId(), MessageCode.PING); - break; - } - case FINDNODE: - { - List nodes = - nodeTable.findClosestNodes(context.getNodeRecord().getNodeId(), DEFAULT_DISTANCE); - final AtomicInteger counter = new AtomicInteger(); - nodes.stream() - .map(NodeRecordInfo::getNode) - .collect(Collectors.groupingBy(it -> counter.getAndIncrement() / MAX_NODES_IN_MSG)) - .values() - .forEach( - c -> - context.addOutgoingEvent( - MessagePacket.create( - homeNodeId, - context.getNodeRecord().getNodeId(), - authTag, - initiatorKey, - DiscoveryV5Message.from( - new NodesMessage( - message.getRequestId(), - nodes.size() / MAX_NODES_IN_MSG, - () -> c, - nodes.size()))))); - break; - } - case NODES: - { - NodesMessage nodesMessage = (NodesMessage) message.create(); - // NODES total count handling + cleanup schedule - if (nodesCounters.containsKey(nodesMessage.getRequestId())) { - synchronized (this) { - int counter = nodesCounters.get(nodesMessage.getRequestId()) - 1; - if (counter == 0) { - cleanUp(nodesMessage.getRequestId(), context); - } else { - nodesCounters.put(nodesMessage.getRequestId(), counter); - updateExpiration(nodesMessage.getRequestId(), context); - } - } - } else if (nodesMessage.getTotal() > 1) { - nodesCounters.put(nodesMessage.getRequestId(), nodesMessage.getTotal() - 1); - updateExpiration(nodesMessage.getRequestId(), context); - } else { - context.clearRequestId(nodesMessage.getRequestId(), MessageCode.FINDNODE); - } - - // Parse node records - nodesMessage - .getNodeRecords() - .forEach( - nodeRecordV5 -> { - if (!nodeTable.getNode(nodeRecordV5.getNodeId()).isPresent()) { - // TODO: should we update? - nodeTable.save(NodeRecordInfo.createDefault(nodeRecordV5)); - } - }); - - break; - } - default: - { - throw new RuntimeException("Not implemented yet"); - } - } - } - - private synchronized void updateExpiration(BytesValue requestId, NodeContext context) { - nodeExpirationScheduler.put( - requestId, - () -> { - cleanUp(requestId, context); - }); - } - - private synchronized void cleanUp(BytesValue requestId, NodeContext context) { - nodesCounters.remove(requestId); - context.clearRequestId(requestId, MessageCode.FINDNODE); - } -} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryV5MessageProcessor.java b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryV5MessageProcessor.java new file mode 100644 index 000000000..5225725bd --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryV5MessageProcessor.java @@ -0,0 +1,39 @@ +package org.ethereum.beacon.discovery; + +import org.ethereum.beacon.discovery.message.DiscoveryV5Message; +import org.ethereum.beacon.discovery.message.MessageCode; +import org.ethereum.beacon.discovery.message.handler.FindNodeHandler; +import org.ethereum.beacon.discovery.message.handler.MessageHandler; +import org.ethereum.beacon.discovery.message.handler.NodesHandler; +import org.ethereum.beacon.discovery.message.handler.PingHandler; +import org.ethereum.beacon.discovery.message.handler.PongHandler; + +import java.util.HashMap; +import java.util.Map; + +public class DiscoveryV5MessageProcessor implements DiscoveryMessageProcessor { + private static final int MAX_NODES_IN_MSG = 24; + private final Map messageHandlers = new HashMap<>(); + + public DiscoveryV5MessageProcessor() { + messageHandlers.put(MessageCode.PING, new PingHandler()); + messageHandlers.put(MessageCode.PONG, new PongHandler()); + messageHandlers.put(MessageCode.FINDNODE, new FindNodeHandler(MAX_NODES_IN_MSG)); + messageHandlers.put(MessageCode.NODES, new NodesHandler()); + } + + @Override + public IdentityScheme getSupportedIdentity() { + return IdentityScheme.V5; + } + + @Override + public void handleMessage(DiscoveryV5Message message, NodeContext context) { + MessageCode code = message.getCode(); + MessageHandler messageHandler = messageHandlers.get(code); + if (messageHandler == null) { + throw new RuntimeException("Not implemented yet"); + } + messageHandler.handle(message.create(), context); + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/MessageHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/MessageHandler.java deleted file mode 100644 index 902f7a690..000000000 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/MessageHandler.java +++ /dev/null @@ -1,34 +0,0 @@ -package org.ethereum.beacon.discovery; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.ethereum.beacon.discovery.message.DiscoveryMessage; -import org.ethereum.beacon.discovery.message.DiscoveryV5Message; - -public class MessageHandler { - private static final Logger logger = LogManager.getLogger(MessageHandler.class); - private final DiscoveryV5MessageHandler v5MessageHandler; - - public MessageHandler(DiscoveryV5MessageHandler v5MessageHandler) { - this.v5MessageHandler = v5MessageHandler; - } - - public void handleIncoming(DiscoveryMessage message, NodeContext context) { - IdentityScheme identityScheme = message.getIdentityScheme(); - switch (identityScheme) { - case V5: - { - v5MessageHandler.handleMessage((DiscoveryV5Message) message, context); - } - default: - { - String error = - String.format( - "Message %s with identity %s received in context %s is not supported", - message, identityScheme, context); - logger.error(error); - throw new RuntimeException(error); - } - } - } -} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/MessageProcessor.java b/discovery/src/main/java/org/ethereum/beacon/discovery/MessageProcessor.java new file mode 100644 index 000000000..37efef666 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/MessageProcessor.java @@ -0,0 +1,33 @@ +package org.ethereum.beacon.discovery; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.ethereum.beacon.discovery.message.DiscoveryMessage; + +import java.util.HashMap; +import java.util.Map; + +public class MessageProcessor { + private static final Logger logger = LogManager.getLogger(MessageProcessor.class); + private final Map messageHandlers = new HashMap<>(); + + public MessageProcessor(DiscoveryMessageProcessor... messageHandlers) { + for (int i = 0; i < messageHandlers.length; ++i) { + this.messageHandlers.put(messageHandlers[i].getSupportedIdentity(), messageHandlers[i]); + } + } + + public void handleIncoming(DiscoveryMessage message, NodeContext context) { + IdentityScheme identityScheme = message.getIdentityScheme(); + DiscoveryMessageProcessor messageHandler = messageHandlers.get(identityScheme); + if (messageHandler == null) { + String error = + String.format( + "Message %s with identity %s received in context %s is not supported", + message, identityScheme, context); + logger.error(error); + throw new RuntimeException(error); + } + messageHandler.handleMessage(message, context); + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeContext.java b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeContext.java index d61b605b4..829e5a3df 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeContext.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeContext.java @@ -3,6 +3,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.ethereum.beacon.discovery.enr.NodeRecordV5; +import org.ethereum.beacon.discovery.message.DiscoveryMessage; import org.ethereum.beacon.discovery.message.DiscoveryV5Message; import org.ethereum.beacon.discovery.message.FindNodeMessage; import org.ethereum.beacon.discovery.message.MessageCode; @@ -42,7 +43,7 @@ public class NodeContext { private Bytes32 idNonce; private BytesValue initiatorKey; private BytesValue authResponseKey; - private MessageHandler messageHandler = null; + private MessageProcessor messageProcessor = null; private Map requestIdReservations = new ConcurrentHashMap<>(); public NodeContext( @@ -129,7 +130,6 @@ public synchronized void addIncomingEvent(UnknownPacket packet) { DiscoveryV5Message.from( new FindNodeMessage( getNextRequestId(MessageCode.FINDNODE), DEFAULT_DISTANCE))); - createMessageHandler(initiatorKey, authTag); addOutgoingEvent(response); status = SessionStatus.AUTHENTICATED; break; @@ -151,8 +151,7 @@ public synchronized void addIncomingEvent(UnknownPacket packet) { this.authResponseKey = hkdf.getValue2(); authHeaderMessagePacket.decode(initiatorKey, authResponseKey); authHeaderMessagePacket.verify(authTagRepo.getTag(this).get(), idNonce); - createMessageHandler(initiatorKey, authHeaderMessagePacket.getAuthTag()); - messageHandler.handleIncoming(authHeaderMessagePacket.getMessage(), this); + handleMessage(authHeaderMessagePacket.getMessage()); status = SessionStatus.AUTHENTICATED; break; } @@ -161,7 +160,7 @@ public synchronized void addIncomingEvent(UnknownPacket packet) { MessagePacket messagePacket = packet.getMessagePacket(); messagePacket.decode(initiatorKey); messagePacket.verify(authTagRepo.getTag(this).get()); - messageHandler.handleIncoming(messagePacket.getMessage(), this); + handleMessage(messagePacket.getMessage()); break; } default: @@ -184,8 +183,13 @@ public synchronized void addIncomingEvent(UnknownPacket packet) { } } - private void createMessageHandler(BytesValue initiatorKey, BytesValue authTag) { - this.messageHandler = new MessageHandler(new DiscoveryV5MessageHandler(homeNodeId, initiatorKey, authTag, nodeTable)); + private void handleMessage(DiscoveryMessage message) { + synchronized (this) { + if (messageProcessor == null) { + this.messageProcessor = new MessageProcessor(new DiscoveryV5MessageProcessor()); + } + } + messageProcessor.handleIncoming(message, this); } /** Sends random packet to start initiation of session with node */ @@ -234,6 +238,18 @@ public void cleanup() { authTagRepo.expire(this); } + public Optional getAuthTag() { + return authTagRepo.getTag(this); + } + + public Bytes32 getHomeNodeId() { + return homeNodeId; + } + + public BytesValue getInitiatorKey() { + return initiatorKey; + } + @Override public String toString() { return "NodeContext{" @@ -255,6 +271,10 @@ public synchronized Optional getRequestId(BytesValue requestId) { return messageCode == null ? Optional.empty() : Optional.of(messageCode); } + public NodeTable getNodeTable() { + return nodeTable; + } + enum SessionStatus { INITIAL, // other side is trying to connect, or we are initiating (before random packet is sent WHOAREYOU_SENT, // other side is initiator, we've sent whoareyou in response diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/FindNodeHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/FindNodeHandler.java new file mode 100644 index 000000000..9e37d24aa --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/FindNodeHandler.java @@ -0,0 +1,47 @@ +package org.ethereum.beacon.discovery.message.handler; + +import org.ethereum.beacon.discovery.NodeContext; +import org.ethereum.beacon.discovery.enr.NodeRecordInfo; +import org.ethereum.beacon.discovery.message.DiscoveryV5Message; +import org.ethereum.beacon.discovery.message.FindNodeMessage; +import org.ethereum.beacon.discovery.message.NodesMessage; +import org.ethereum.beacon.discovery.packet.MessagePacket; + +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; + +import static org.ethereum.beacon.discovery.NodeContext.DEFAULT_DISTANCE; + +public class FindNodeHandler implements MessageHandler { + private final int maxNodesInMsg; + + public FindNodeHandler(int maxNodesInMsg) { + this.maxNodesInMsg = maxNodesInMsg; + } + + @Override + public void handle(FindNodeMessage message, NodeContext context) { + List nodes = + context.getNodeTable().findClosestNodes(context.getNodeRecord().getNodeId(), DEFAULT_DISTANCE); + final AtomicInteger counter = new AtomicInteger(); + nodes.stream() + .map(NodeRecordInfo::getNode) + .collect(Collectors.groupingBy(it -> counter.getAndIncrement() / maxNodesInMsg)) + .values() + .forEach( + c -> + context.addOutgoingEvent( + MessagePacket.create( + context.getHomeNodeId(), + context.getNodeRecord().getNodeId(), + context.getAuthTag().get(), + context.getInitiatorKey(), + DiscoveryV5Message.from( + new NodesMessage( + message.getRequestId(), + nodes.size() / maxNodesInMsg, + () -> c, + nodes.size()))))); + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/MessageHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/MessageHandler.java new file mode 100644 index 000000000..011acd4ad --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/MessageHandler.java @@ -0,0 +1,7 @@ +package org.ethereum.beacon.discovery.message.handler; + +import org.ethereum.beacon.discovery.NodeContext; + +public interface MessageHandler { + void handle(Message message, NodeContext context); +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/NodesHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/NodesHandler.java new file mode 100644 index 000000000..dc1b1e66f --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/NodesHandler.java @@ -0,0 +1,68 @@ +package org.ethereum.beacon.discovery.message.handler; + +import org.ethereum.beacon.discovery.NodeContext; +import org.ethereum.beacon.discovery.enr.NodeRecordInfo; +import org.ethereum.beacon.discovery.message.DiscoveryV5Message; +import org.ethereum.beacon.discovery.message.FindNodeMessage; +import org.ethereum.beacon.discovery.message.MessageCode; +import org.ethereum.beacon.discovery.message.NodesMessage; +import org.ethereum.beacon.discovery.message.PongMessage; +import org.ethereum.beacon.util.ExpirationScheduler; +import tech.pegasys.artemis.util.bytes.BytesValue; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; + +public class NodesHandler implements MessageHandler { + private static final int CLEANUP_DELAY_SECONDS = 60; + private ExpirationScheduler nodeExpirationScheduler = + new ExpirationScheduler<>(CLEANUP_DELAY_SECONDS, TimeUnit.SECONDS); + private Map nodesCounters = new ConcurrentHashMap<>(); + + @Override + public void handle(NodesMessage message, NodeContext context) { + // NODES total count handling + cleanup schedule + if (nodesCounters.containsKey(message.getRequestId())) { + synchronized (this) { + int counter = nodesCounters.get(message.getRequestId()) - 1; + if (counter == 0) { + cleanUp(message.getRequestId(), context); + } else { + nodesCounters.put(message.getRequestId(), counter); + updateExpiration(message.getRequestId(), context); + } + } + } else if (message.getTotal() > 1) { + nodesCounters.put(message.getRequestId(), message.getTotal() - 1); + updateExpiration(message.getRequestId(), context); + } else { + context.clearRequestId(message.getRequestId(), MessageCode.FINDNODE); + } + + // Parse node records + message + .getNodeRecords() + .forEach( + nodeRecordV5 -> { + if (!context.getNodeTable().getNode(nodeRecordV5.getNodeId()).isPresent()) { + // TODO: should we update-merge? + context.getNodeTable().save(NodeRecordInfo.createDefault(nodeRecordV5)); + } + }); + } + + + private synchronized void updateExpiration(BytesValue requestId, NodeContext context) { + nodeExpirationScheduler.put( + requestId, + () -> { + cleanUp(requestId, context); + }); + } + + private synchronized void cleanUp(BytesValue requestId, NodeContext context) { + nodesCounters.remove(requestId); + context.clearRequestId(requestId, MessageCode.FINDNODE); + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/PingHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/PingHandler.java new file mode 100644 index 000000000..68e3941cd --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/PingHandler.java @@ -0,0 +1,26 @@ +package org.ethereum.beacon.discovery.message.handler; + +import org.ethereum.beacon.discovery.NodeContext; +import org.ethereum.beacon.discovery.message.DiscoveryV5Message; +import org.ethereum.beacon.discovery.message.PingMessage; +import org.ethereum.beacon.discovery.message.PongMessage; +import org.ethereum.beacon.discovery.packet.MessagePacket; + +public class PingHandler implements MessageHandler { + @Override + public void handle(PingMessage message, NodeContext context) { + PongMessage responseMessage = + new PongMessage( + message.getRequestId(), + context.getNodeRecord().getSeqNumber(), + context.getNodeRecord().getIpV4address(), + context.getNodeRecord().getUdpPort()); + context.addOutgoingEvent( + MessagePacket.create( + context.getHomeNodeId(), + context.getNodeRecord().getNodeId(), + context.getAuthTag().get(), + context.getInitiatorKey(), + DiscoveryV5Message.from(responseMessage))); + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/PongHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/PongHandler.java new file mode 100644 index 000000000..c568f90aa --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/PongHandler.java @@ -0,0 +1,15 @@ +package org.ethereum.beacon.discovery.message.handler; + +import org.ethereum.beacon.discovery.NodeContext; +import org.ethereum.beacon.discovery.message.DiscoveryV5Message; +import org.ethereum.beacon.discovery.message.MessageCode; +import org.ethereum.beacon.discovery.message.PingMessage; +import org.ethereum.beacon.discovery.message.PongMessage; +import org.ethereum.beacon.discovery.packet.MessagePacket; + +public class PongHandler implements MessageHandler { + @Override + public void handle(PongMessage message, NodeContext context) { + context.clearRequestId(message.getRequestId(), MessageCode.PING); + } +} From 5f48ed78c52f871469bccb1c4a6b77a372aef0e0 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Thu, 19 Sep 2019 12:25:39 +0300 Subject: [PATCH 12/77] discovery: refactor packet handling --- .../beacon/discovery/NodeContext.java | 224 +++++++----------- .../beacon/discovery/enr/NodeRecordInfo.java | 10 + .../beacon/discovery/enr/NodeRecordV4.java | 9 + .../beacon/discovery/enr/NodeRecordV5.java | 9 + .../AuthHeaderMessagePacketHandler.java | 54 +++++ .../packet/handler/MessagePacketHandler.java | 37 +++ .../packet/handler/PacketHandler.java | 7 + .../packet/handler/UnknownPacketHandler.java | 65 +++++ .../handler/WhoAreYouPacketHandler.java | 76 ++++++ 9 files changed, 358 insertions(+), 133 deletions(-) create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/packet/handler/AuthHeaderMessagePacketHandler.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/packet/handler/MessagePacketHandler.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/packet/handler/PacketHandler.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/packet/handler/UnknownPacketHandler.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/packet/handler/WhoAreYouPacketHandler.java diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeContext.java b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeContext.java index 829e5a3df..e14938371 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeContext.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeContext.java @@ -2,10 +2,9 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.enr.NodeRecordV5; import org.ethereum.beacon.discovery.message.DiscoveryMessage; -import org.ethereum.beacon.discovery.message.DiscoveryV5Message; -import org.ethereum.beacon.discovery.message.FindNodeMessage; import org.ethereum.beacon.discovery.message.MessageCode; import org.ethereum.beacon.discovery.packet.AuthHeaderMessagePacket; import org.ethereum.beacon.discovery.packet.MessagePacket; @@ -13,15 +12,18 @@ import org.ethereum.beacon.discovery.packet.RandomPacket; import org.ethereum.beacon.discovery.packet.UnknownPacket; import org.ethereum.beacon.discovery.packet.WhoAreYouPacket; +import org.ethereum.beacon.discovery.packet.handler.AuthHeaderMessagePacketHandler; +import org.ethereum.beacon.discovery.packet.handler.MessagePacketHandler; +import org.ethereum.beacon.discovery.packet.handler.PacketHandler; +import org.ethereum.beacon.discovery.packet.handler.UnknownPacketHandler; +import org.ethereum.beacon.discovery.packet.handler.WhoAreYouPacketHandler; import org.ethereum.beacon.discovery.storage.AuthTagRepository; import org.ethereum.beacon.discovery.storage.NodeTable; -import org.javatuples.Triplet; -import org.web3j.crypto.ECKeyPair; import tech.pegasys.artemis.util.bytes.Bytes32; import tech.pegasys.artemis.util.bytes.BytesValue; import java.security.SecureRandom; -import java.util.List; +import java.util.HashMap; import java.util.Map; import java.util.Optional; import java.util.Random; @@ -38,11 +40,10 @@ public class NodeContext { private final NodeTable nodeTable; private final Consumer outgoing; private final Random rnd; - private List incomingEvents; + private final Map packetHandlers = new HashMap<>(); private SessionStatus status = SessionStatus.INITIAL; private Bytes32 idNonce; private BytesValue initiatorKey; - private BytesValue authResponseKey; private MessageProcessor messageProcessor = null; private Map requestIdReservations = new ConcurrentHashMap<>(); @@ -60,130 +61,59 @@ public NodeContext( this.homeNodeRecord = homeNodeRecord; this.homeNodeId = homeNodeRecord.getNodeId(); this.rnd = rnd; + packetHandlers.put(UnknownPacket.class, new UnknownPacketHandler(this, logger)); + packetHandlers.put(WhoAreYouPacket.class, new WhoAreYouPacketHandler(this, logger)); + packetHandlers.put( + AuthHeaderMessagePacket.class, new AuthHeaderMessagePacketHandler(this, logger)); + packetHandlers.put(MessagePacket.class, new MessagePacketHandler(this, logger)); } public NodeRecordV5 getNodeRecord() { return nodeRecord; } - public synchronized void addIncomingEvent(UnknownPacket packet) { - try { - logger.trace(() -> String.format("Incoming packet in context %s", this)); - switch (status) { - case INITIAL: - { - // packet it either random or message packet if session is expired - BytesValue authTag = null; - try { - RandomPacket randomPacket = packet.getRandomPacket(); - authTag = randomPacket.getAuthTag(); - } catch (Exception ex) { - // Not fatal, 1st attempt - } - // 2nd attempt - if (authTag == null) { - MessagePacket messagePacket = packet.getMessagePacket(); - authTag = messagePacket.getAuthTag(); - } - authTagRepo.put(authTag, this); - byte[] idNonceBytes = new byte[32]; - Functions.getRandom().nextBytes(idNonceBytes); - this.idNonce = Bytes32.wrap(idNonceBytes); - WhoAreYouPacket whoAreYouPacket = - WhoAreYouPacket.create( - nodeRecord.getNodeId(), authTag, idNonce, nodeRecord.getSeqNumber()); - addOutgoingEvent(whoAreYouPacket); - status = SessionStatus.WHOAREYOU_SENT; - break; + public void addIncomingEvent(UnknownPacket packet) { + logger.trace(() -> String.format("Incoming packet in context %s", this)); + switch (status) { + case INITIAL: + { + if (packetHandlers.get(UnknownPacket.class).handle(packet)) { + setStatus(SessionStatus.WHOAREYOU_SENT); } - case RANDOM_PACKET_SENT: - { - WhoAreYouPacket whoAreYouPacket = packet.getWhoAreYouPacket(); - BytesValue authTag = authTagRepo.getTag(this).get(); - whoAreYouPacket.verify(homeNodeId, authTag); - whoAreYouPacket.getEnrSeq(); // FIXME: Their side enr seq. Do we need it? - byte[] ephemeralKeyBytes = new byte[32]; - Functions.getRandom().nextBytes(ephemeralKeyBytes); - ECKeyPair ephemeralKey = ECKeyPair.create(ephemeralKeyBytes); // TODO: generate - Triplet hkdf = - Functions.hkdf_expand( - homeNodeId, - nodeRecord.getNodeId(), - BytesValue.wrap(ephemeralKey.getPrivateKey().toByteArray()), - whoAreYouPacket.getIdNonce(), - nodeRecord.getPublicKey()); - BytesValue initiatorKey = hkdf.getValue0(); - BytesValue staticNodeKey = hkdf.getValue1(); - BytesValue authResponseKey = hkdf.getValue2(); - - AuthHeaderMessagePacket response = - AuthHeaderMessagePacket.create( - homeNodeId, - nodeRecord.getNodeId(), - authResponseKey, - whoAreYouPacket.getIdNonce(), - staticNodeKey, - homeNodeRecord, - BytesValue.wrap(ephemeralKey.getPublicKey().toByteArray()), - authTag, - initiatorKey, - DiscoveryV5Message.from( - new FindNodeMessage( - getNextRequestId(MessageCode.FINDNODE), DEFAULT_DISTANCE))); - addOutgoingEvent(response); - status = SessionStatus.AUTHENTICATED; - break; - } - case WHOAREYOU_SENT: - { - AuthHeaderMessagePacket authHeaderMessagePacket = packet.getAuthHeaderMessagePacket(); - byte[] ephemeralKeyBytes = new byte[32]; - Functions.getRandom().nextBytes(ephemeralKeyBytes); - ECKeyPair ephemeralKey = ECKeyPair.create(ephemeralKeyBytes); - Triplet hkdf = - Functions.hkdf_expand( - homeNodeId, - nodeRecord.getNodeId(), - BytesValue.wrap(ephemeralKey.getPrivateKey().toByteArray()), - idNonce, - nodeRecord.getPublicKey()); - this.initiatorKey = hkdf.getValue0(); - this.authResponseKey = hkdf.getValue2(); - authHeaderMessagePacket.decode(initiatorKey, authResponseKey); - authHeaderMessagePacket.verify(authTagRepo.getTag(this).get(), idNonce); - handleMessage(authHeaderMessagePacket.getMessage()); - status = SessionStatus.AUTHENTICATED; - break; - } - case AUTHENTICATED: - { - MessagePacket messagePacket = packet.getMessagePacket(); - messagePacket.decode(initiatorKey); - messagePacket.verify(authTagRepo.getTag(this).get()); - handleMessage(messagePacket.getMessage()); - break; + break; + } + case RANDOM_PACKET_SENT: + { + WhoAreYouPacket whoAreYouPacket = packet.getWhoAreYouPacket(); + if (packetHandlers.get(WhoAreYouPacket.class).handle(whoAreYouPacket)) { + setStatus(SessionStatus.AUTHENTICATED); } - default: - { - String error = - String.format("Not expected status:%s from node: %s", status, nodeRecord); - logger.error(error); - throw new RuntimeException(error); + break; + } + case WHOAREYOU_SENT: + { + AuthHeaderMessagePacket authHeaderMessagePacket = packet.getAuthHeaderMessagePacket(); + if (packetHandlers.get(AuthHeaderMessagePacket.class).handle(authHeaderMessagePacket)) { + setStatus(SessionStatus.AUTHENTICATED); } - } - } catch (AssertionError ex) { - logger.info( - String.format( - "Verification not passed for message [%s] from node %s in status %s", - packet, nodeRecord, status)); - } catch (Exception ex) { - logger.info( - String.format( - "Failed to read message [%s] from node %s in status %s", packet, nodeRecord, status)); + break; + } + case AUTHENTICATED: + { + MessagePacket messagePacket = packet.getMessagePacket(); + packetHandlers.get(MessagePacket.class).handle(messagePacket); + break; + } + default: + { + String error = String.format("Not expected status:%s from node: %s", status, nodeRecord); + logger.error(error); + throw new RuntimeException(error); + } } } - private void handleMessage(DiscoveryMessage message) { + public void handleMessage(DiscoveryMessage message) { synchronized (this) { if (messageProcessor == null) { this.messageProcessor = new MessageProcessor(new DiscoveryV5MessageProcessor()); @@ -218,7 +148,7 @@ public synchronized void addOutgoingEvent(Packet packet) { * *

Request ID is reserved for `messageCode` */ - private synchronized BytesValue getNextRequestId(MessageCode messageCode) { + public synchronized BytesValue getNextRequestId(MessageCode messageCode) { byte[] requestId = new byte[12]; rnd.nextBytes(requestId); BytesValue wrapped = BytesValue.wrap(requestId); @@ -226,10 +156,6 @@ private synchronized BytesValue getNextRequestId(MessageCode messageCode) { return wrapped; } - public List getIncomingEvents() { - return incomingEvents; - } - public synchronized boolean isAuthenticated() { return SessionStatus.AUTHENTICATED.equals(status); } @@ -242,6 +168,10 @@ public Optional getAuthTag() { return authTagRepo.getTag(this); } + public void setAuthTag(BytesValue authTag) { + authTagRepo.put(authTag, this); + } + public Bytes32 getHomeNodeId() { return homeNodeId; } @@ -250,6 +180,35 @@ public BytesValue getInitiatorKey() { return initiatorKey; } + public void setInitiatorKey(BytesValue initiatorKey) { + this.initiatorKey = initiatorKey; + } + + public synchronized void clearRequestId(BytesValue requestId, MessageCode messageCode) { + assert requestIdReservations.remove(requestId).equals(messageCode); + } + + public synchronized Optional getRequestId(BytesValue requestId) { + MessageCode messageCode = requestIdReservations.get(requestId); + return messageCode == null ? Optional.empty() : Optional.of(messageCode); + } + + public NodeTable getNodeTable() { + return nodeTable; + } + + public synchronized Bytes32 getIdNonce() { + return idNonce; + } + + public synchronized void setIdNonce(Bytes32 idNonce) { + this.idNonce = idNonce; + } + + public NodeRecord getHomeNodeRecord() { + return homeNodeRecord; + } + @Override public String toString() { return "NodeContext{" @@ -262,17 +221,16 @@ public String toString() { + '}'; } - public synchronized void clearRequestId(BytesValue requestId, MessageCode messageCode) { - assert requestIdReservations.remove(requestId).equals(messageCode); + public synchronized SessionStatus getStatus() { + return status; } - public synchronized Optional getRequestId(BytesValue requestId) { - MessageCode messageCode = requestIdReservations.get(requestId); - return messageCode == null ? Optional.empty() : Optional.of(messageCode); - } - - public NodeTable getNodeTable() { - return nodeTable; + private synchronized void setStatus(SessionStatus newStatus) { + logger.debug( + () -> + String.format( + "Switching status of node %s from %s to %s", nodeRecord, status, newStatus)); + this.status = newStatus; } enum SessionStatus { diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordInfo.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordInfo.java index c80d0da3b..3147e0690 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordInfo.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordInfo.java @@ -59,4 +59,14 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hashCode(node, lastRetry, status, retry); } + + @Override + public String toString() { + return "NodeRecordInfo{" + + "node=" + node + + ", lastRetry=" + lastRetry + + ", status=" + status + + ", retry=" + retry + + '}'; + } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV4.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV4.java index 991c79460..6fc781cd5 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV4.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV4.java @@ -331,4 +331,13 @@ public NodeRecordV4 build() { return nodeRecord; } } + + @Override + public String toString() { + return "NodeRecordV4{" + + "publicKey=" + publicKey + + ", ipV4address=" + ipV4address + + ", udpPort=" + udpPort + + '}'; + } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV5.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV5.java index be8ff6439..9eec6bcbc 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV5.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV5.java @@ -338,4 +338,13 @@ public NodeRecordV5 build() { return nodeRecord; } } + + @Override + public String toString() { + return "NodeRecordV5{" + + "publicKey=" + publicKey + + ", ipV4address=" + ipV4address + + ", udpPort=" + udpPort + + '}'; + } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/handler/AuthHeaderMessagePacketHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/handler/AuthHeaderMessagePacketHandler.java new file mode 100644 index 000000000..e7dfc6607 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/handler/AuthHeaderMessagePacketHandler.java @@ -0,0 +1,54 @@ +package org.ethereum.beacon.discovery.packet.handler; + +import org.apache.logging.log4j.Logger; +import org.ethereum.beacon.discovery.Functions; +import org.ethereum.beacon.discovery.NodeContext; +import org.ethereum.beacon.discovery.packet.AuthHeaderMessagePacket; +import org.javatuples.Triplet; +import org.web3j.crypto.ECKeyPair; +import tech.pegasys.artemis.util.bytes.BytesValue; + +public class AuthHeaderMessagePacketHandler implements PacketHandler { + private final NodeContext context; + private final Logger logger; + + public AuthHeaderMessagePacketHandler(NodeContext context, Logger logger) { + this.context = context; + this.logger = logger; + } + + @Override + public boolean handle(AuthHeaderMessagePacket packet) { + try { + byte[] ephemeralKeyBytes = new byte[32]; + Functions.getRandom().nextBytes(ephemeralKeyBytes); + ECKeyPair ephemeralKey = ECKeyPair.create(ephemeralKeyBytes); + Triplet hkdf = + Functions.hkdf_expand( + context.getHomeNodeId(), + context.getNodeRecord().getNodeId(), + BytesValue.wrap(ephemeralKey.getPrivateKey().toByteArray()), + context.getIdNonce(), + context.getNodeRecord().getPublicKey()); + BytesValue initiatorKey = hkdf.getValue0(); + context.setInitiatorKey(initiatorKey); + BytesValue authResponseKey = hkdf.getValue2(); + packet.decode(initiatorKey, authResponseKey); + packet.verify(context.getAuthTag().get(), context.getIdNonce()); + context.handleMessage(packet.getMessage()); + } catch (AssertionError ex) { + logger.info( + String.format( + "Verification not passed for message [%s] from node %s in status %s", + packet, context.getNodeRecord(), context.getStatus())); + } catch (Exception ex) { + String error = + String.format( + "Failed to read message [%s] from node %s in status %s", + packet, context.getNodeRecord(), context.getStatus()); + logger.error(error, ex); + return false; + } + return true; + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/handler/MessagePacketHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/handler/MessagePacketHandler.java new file mode 100644 index 000000000..23cc49003 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/handler/MessagePacketHandler.java @@ -0,0 +1,37 @@ +package org.ethereum.beacon.discovery.packet.handler; + +import org.apache.logging.log4j.Logger; +import org.ethereum.beacon.discovery.NodeContext; +import org.ethereum.beacon.discovery.packet.MessagePacket; + +public class MessagePacketHandler implements PacketHandler { + private final NodeContext context; + private final Logger logger; + + public MessagePacketHandler(NodeContext context, Logger logger) { + this.context = context; + this.logger = logger; + } + + @Override + public boolean handle(MessagePacket packet) { + try { + packet.decode(context.getInitiatorKey()); + packet.verify(context.getAuthTag().get()); + context.handleMessage(packet.getMessage()); + } catch (AssertionError ex) { + logger.info( + String.format( + "Verification not passed for message [%s] from node %s in status %s", + packet, context.getNodeRecord(), context.getStatus())); + } catch (Exception ex) { + String error = + String.format( + "Failed to read message [%s] from node %s in status %s", + packet, context.getNodeRecord(), context.getStatus()); + logger.error(error, ex); + return false; + } + return true; + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/handler/PacketHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/handler/PacketHandler.java new file mode 100644 index 000000000..b2b1e4a5b --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/handler/PacketHandler.java @@ -0,0 +1,7 @@ +package org.ethereum.beacon.discovery.packet.handler; + +import org.ethereum.beacon.discovery.packet.Packet; + +public interface PacketHandler

{ + boolean handle(P packet); +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/handler/UnknownPacketHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/handler/UnknownPacketHandler.java new file mode 100644 index 000000000..de7c4b8bd --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/handler/UnknownPacketHandler.java @@ -0,0 +1,65 @@ +package org.ethereum.beacon.discovery.packet.handler; + +import org.apache.logging.log4j.Logger; +import org.ethereum.beacon.discovery.Functions; +import org.ethereum.beacon.discovery.NodeContext; +import org.ethereum.beacon.discovery.packet.MessagePacket; +import org.ethereum.beacon.discovery.packet.RandomPacket; +import org.ethereum.beacon.discovery.packet.UnknownPacket; +import org.ethereum.beacon.discovery.packet.WhoAreYouPacket; +import tech.pegasys.artemis.util.bytes.Bytes32; +import tech.pegasys.artemis.util.bytes.BytesValue; + +public class UnknownPacketHandler implements PacketHandler { + private final NodeContext context; + private final Logger logger; + + public UnknownPacketHandler(NodeContext context, Logger logger) { + this.context = context; + this.logger = logger; + } + + @Override + public boolean handle(UnknownPacket packet) { + try { + // packet it either random or message packet if session is expired + BytesValue authTag = null; + try { + RandomPacket randomPacket = packet.getRandomPacket(); + authTag = randomPacket.getAuthTag(); + } catch (Exception ex) { + // Not fatal, 1st attempt + } + // 2nd attempt + if (authTag == null) { + MessagePacket messagePacket = packet.getMessagePacket(); + authTag = messagePacket.getAuthTag(); + } + context.setAuthTag(authTag); + byte[] idNonceBytes = new byte[32]; + Functions.getRandom().nextBytes(idNonceBytes); + Bytes32 idNonce = Bytes32.wrap(idNonceBytes); + context.setIdNonce(idNonce); + WhoAreYouPacket whoAreYouPacket = + WhoAreYouPacket.create( + context.getNodeRecord().getNodeId(), + authTag, + idNonce, + context.getNodeRecord().getSeqNumber()); + context.addOutgoingEvent(whoAreYouPacket); + } catch (AssertionError ex) { + logger.info( + String.format( + "Verification not passed for message [%s] from node %s in status %s", + packet, context.getNodeRecord(), context.getStatus())); + } catch (Exception ex) { + String error = + String.format( + "Failed to read message [%s] from node %s in status %s", + packet, context.getNodeRecord(), context.getStatus()); + logger.error(error, ex); + return false; + } + return true; + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/handler/WhoAreYouPacketHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/handler/WhoAreYouPacketHandler.java new file mode 100644 index 000000000..a176002bc --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/handler/WhoAreYouPacketHandler.java @@ -0,0 +1,76 @@ +package org.ethereum.beacon.discovery.packet.handler; + +import org.apache.logging.log4j.Logger; +import org.ethereum.beacon.discovery.Functions; +import org.ethereum.beacon.discovery.NodeContext; +import org.ethereum.beacon.discovery.message.DiscoveryV5Message; +import org.ethereum.beacon.discovery.message.FindNodeMessage; +import org.ethereum.beacon.discovery.message.MessageCode; +import org.ethereum.beacon.discovery.packet.AuthHeaderMessagePacket; +import org.ethereum.beacon.discovery.packet.WhoAreYouPacket; +import org.javatuples.Triplet; +import org.web3j.crypto.ECKeyPair; +import tech.pegasys.artemis.util.bytes.BytesValue; + +import static org.ethereum.beacon.discovery.NodeContext.DEFAULT_DISTANCE; + +public class WhoAreYouPacketHandler implements PacketHandler { + private final NodeContext context; + private final Logger logger; + + public WhoAreYouPacketHandler(NodeContext context, Logger logger) { + this.context = context; + this.logger = logger; + } + + @Override + public boolean handle(WhoAreYouPacket packet) { + try { + BytesValue authTag = context.getAuthTag().get(); + packet.verify(context.getHomeNodeId(), authTag); + packet.getEnrSeq(); // FIXME: Their side enr seq. Do we need it? + byte[] ephemeralKeyBytes = new byte[32]; + Functions.getRandom().nextBytes(ephemeralKeyBytes); + ECKeyPair ephemeralKey = ECKeyPair.create(ephemeralKeyBytes); // TODO: generate + Triplet hkdf = + Functions.hkdf_expand( + context.getHomeNodeId(), + context.getNodeRecord().getNodeId(), + BytesValue.wrap(ephemeralKey.getPrivateKey().toByteArray()), + packet.getIdNonce(), + context.getNodeRecord().getPublicKey()); + BytesValue initiatorKey = hkdf.getValue0(); + BytesValue staticNodeKey = hkdf.getValue1(); + BytesValue authResponseKey = hkdf.getValue2(); + + AuthHeaderMessagePacket response = + AuthHeaderMessagePacket.create( + context.getHomeNodeId(), + context.getNodeRecord().getNodeId(), + authResponseKey, + packet.getIdNonce(), + staticNodeKey, + context.getHomeNodeRecord(), + BytesValue.wrap(ephemeralKey.getPublicKey().toByteArray()), + authTag, + initiatorKey, + DiscoveryV5Message.from( + new FindNodeMessage( + context.getNextRequestId(MessageCode.FINDNODE), DEFAULT_DISTANCE))); + context.addOutgoingEvent(response); + } catch (AssertionError ex) { + logger.info( + String.format( + "Verification not passed for message [%s] from node %s in status %s", + packet, context.getNodeRecord(), context.getStatus())); + } catch (Exception ex) { + String error = + String.format( + "Failed to read message [%s] from node %s in status %s", + packet, context.getNodeRecord(), context.getStatus()); + logger.error(error, ex); + return false; + } + return true; + } +} From 39c5f35889985a49043f05934ab2b55cdb39a249 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Sat, 21 Sep 2019 15:33:55 +0300 Subject: [PATCH 13/77] discovery: add ExpirationScheduler test + fix bug --- .../beacon/util/ExpirationScheduler.java | 6 +- .../beacon/util/ExpirationSchedulerTest.java | 66 +++++++++++++++++++ 2 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 util/src/test/java/org/ethereum/beacon/util/ExpirationSchedulerTest.java diff --git a/util/src/main/java/org/ethereum/beacon/util/ExpirationScheduler.java b/util/src/main/java/org/ethereum/beacon/util/ExpirationScheduler.java index a14403736..9a71322c3 100644 --- a/util/src/main/java/org/ethereum/beacon/util/ExpirationScheduler.java +++ b/util/src/main/java/org/ethereum/beacon/util/ExpirationScheduler.java @@ -23,7 +23,11 @@ public ExpirationScheduler(long delay, TimeUnit timeUnit) { } public void put(Key key, Runnable runnable) { - expirationTasks.remove(key).cancel(true); + synchronized (this) { + if (expirationTasks.containsKey(key)) { + expirationTasks.remove(key).cancel(true); + } + } ScheduledFuture future = scheduler.schedule( () -> { diff --git a/util/src/test/java/org/ethereum/beacon/util/ExpirationSchedulerTest.java b/util/src/test/java/org/ethereum/beacon/util/ExpirationSchedulerTest.java new file mode 100644 index 000000000..093ff6852 --- /dev/null +++ b/util/src/test/java/org/ethereum/beacon/util/ExpirationSchedulerTest.java @@ -0,0 +1,66 @@ +package org.ethereum.beacon.util; + +import org.junit.Test; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +public class ExpirationSchedulerTest { + @Test + public void test() throws Exception { + ExpirationScheduler scheduler1 = new ExpirationScheduler(500, TimeUnit.MILLISECONDS); + ExpirationScheduler scheduler2 = new ExpirationScheduler(500, TimeUnit.MILLISECONDS); + CountDownLatch first = new CountDownLatch(1); + CountDownLatch second = new CountDownLatch(1); + CountDownLatch third = new CountDownLatch(1); + CountDownLatch fourth = new CountDownLatch(1); + CountDownLatch fifth = new CountDownLatch(1); + scheduler1.put(5, fifth::countDown); + Thread.sleep(100); + scheduler1.put( + 1, + () -> { + first.countDown(); + assert second.getCount() == 1; + assert third.getCount() == 1; + assert fourth.getCount() == 1; + assert fifth.getCount() == 1; + }); + scheduler2.put( + 2, + () -> { + second.countDown(); + assert first.getCount() == 0; + assert third.getCount() == 1; + assert fourth.getCount() == 1; + assert fifth.getCount() == 1; + }); + scheduler2.put( + 3, + () -> { + third.countDown(); + assert first.getCount() == 0; + assert second.getCount() == 0; + assert fourth.getCount() == 1; + assert fifth.getCount() == 1; + }); + scheduler2.put( + 4, + () -> { + fourth.countDown(); + assert first.getCount() == 0; + assert second.getCount() == 0; + assert third.getCount() == 0; + assert fifth.getCount() == 1; + }); + + Thread.sleep(50); + scheduler1.put(5, fifth::countDown); + + assert first.await(1, TimeUnit.SECONDS); + assert second.await(100, TimeUnit.MILLISECONDS); + assert third.await(100, TimeUnit.MILLISECONDS); + assert fourth.await(100, TimeUnit.MILLISECONDS); + assert fifth.await(100, TimeUnit.MILLISECONDS); + } +} From 5a78fa17347725cb2e2bad2f23b52721565d0452 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Mon, 23 Sep 2019 14:31:19 +0300 Subject: [PATCH 14/77] util: add javadoc to ExpirationScheduler --- .../java/org/ethereum/beacon/util/ExpirationScheduler.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/util/src/main/java/org/ethereum/beacon/util/ExpirationScheduler.java b/util/src/main/java/org/ethereum/beacon/util/ExpirationScheduler.java index 9a71322c3..28dc81aa0 100644 --- a/util/src/main/java/org/ethereum/beacon/util/ExpirationScheduler.java +++ b/util/src/main/java/org/ethereum/beacon/util/ExpirationScheduler.java @@ -22,6 +22,12 @@ public ExpirationScheduler(long delay, TimeUnit timeUnit) { this.timeUnit = timeUnit; } + /** + * Puts scheduled task and renews (cancelling old) timeout for the task associated with the key + * + * @param key Task key + * @param runnable Task + */ public void put(Key key, Runnable runnable) { synchronized (this) { if (expirationTasks.containsKey(key)) { From e09ddc73e8d0e8404ea043c43e359981bca6ee8d Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Mon, 23 Sep 2019 14:56:49 +0300 Subject: [PATCH 15/77] discovery: recurrent task fixes and refactoring --- .../beacon/discovery/DiscoverTask.java | 72 ---------- .../beacon/discovery/DiscoveryManager.java | 13 ++ .../discovery/DiscoveryManagerImpl.java | 5 +- .../discovery/DiscoveryTaskManager.java | 127 ++++++++++++++++++ .../beacon/discovery/NodeConnectTasks.java | 64 +++++++++ .../beacon/discovery/NodeContext.java | 45 ++++++- .../beacon/discovery/RefreshTask.java | 36 ----- .../mock/DiscoveryManagerNoNetwork.java | 5 +- 8 files changed, 252 insertions(+), 115 deletions(-) delete mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/DiscoverTask.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryTaskManager.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/NodeConnectTasks.java delete mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/RefreshTask.java diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoverTask.java b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoverTask.java deleted file mode 100644 index baa1f59b7..000000000 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoverTask.java +++ /dev/null @@ -1,72 +0,0 @@ -package org.ethereum.beacon.discovery; - -import org.ethereum.beacon.discovery.enr.NodeRecordInfo; -import org.ethereum.beacon.discovery.enr.NodeRecordV5; -import org.ethereum.beacon.discovery.enr.NodeStatus; -import org.ethereum.beacon.discovery.storage.NodeTable; -import tech.pegasys.artemis.util.bytes.Bytes32; - -import java.util.List; -import java.util.function.Predicate; - -import static org.ethereum.beacon.discovery.NodeContext.DEFAULT_DISTANCE; -import static org.ethereum.beacon.discovery.enr.NodeRecordV5.NODE_ID_FUNCTION; - -public class DiscoverTask { - public static final int MS_IN_SECOND = 1000; - NodeTable nodeTable; - Bytes32 homeNodeId; - RefreshTask refreshTask; - - public DiscoverTask(NodeTable nodeTable, NodeRecordV5 homeNode, RefreshTask refreshTask) { - this.nodeTable = nodeTable; - this.homeNodeId = NODE_ID_FUNCTION.apply(homeNode); - this.refreshTask = refreshTask; - } - - // TODO: first batch to include dead and switch them to SLEEP - public Predicate refreshTask() { - return nodeRecord -> { - final int DISCOVER_AFTER_SECONDS = 600; - final int MAX_RETRIES = 10; - long currentTime = System.currentTimeMillis() / MS_IN_SECOND; - if (nodeRecord.getStatus() == NodeStatus.ACTIVE - && nodeRecord.getLastRetry() > currentTime - DISCOVER_AFTER_SECONDS) { - return false; // no need to rediscover - } - if (nodeRecord.getRetry() >= MAX_RETRIES) { - nodeTable.save( - new NodeRecordInfo( - nodeRecord.getNode(), nodeRecord.getLastRetry(), NodeStatus.DEAD, 0)); - return false; - } - if ((currentTime - nodeRecord.getLastRetry()) < (nodeRecord.getRetry() * nodeRecord.getRetry())) { - return false; // too early for retry - } - - return true; - }; - } - - public void run() { - List nodes = nodeTable.findClosestNodes(homeNodeId, DEFAULT_DISTANCE); - nodes.stream() - .filter(refreshTask()) - .forEach( - nodeRecord -> - refreshTask.add( - nodeRecord, - () -> nodeTable.save( - new NodeRecordInfo( - nodeRecord.getNode(), - System.currentTimeMillis() / MS_IN_SECOND, - NodeStatus.ACTIVE, - 0)), - () -> nodeTable.save( - new NodeRecordInfo( - nodeRecord.getNode(), - System.currentTimeMillis() / MS_IN_SECOND, - NodeStatus.SLEEP, - (nodeRecord.getRetry() + 1))))); - } -} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManager.java b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManager.java index 9d40a8706..ac87849b1 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManager.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManager.java @@ -1,5 +1,9 @@ package org.ethereum.beacon.discovery; +import org.ethereum.beacon.discovery.enr.NodeRecord; + +import java.util.concurrent.CompletableFuture; + /** * Discovery Manager, top interface for peer discovery mechanism as described at https://github.com/ethereum/devp2p/blob/master/discv5/discv5.md @@ -8,4 +12,13 @@ public interface DiscoveryManager { void start(); void stop(); + + /** + * Initiates auth handshake with node, sending some message and receiving reply. + * + * @param nodeRecord Ethereum Node record + * @return Future which is fired when reply is received or fails in timeout/not successful + * handshake/bad message exchange. + */ + CompletableFuture connect(NodeRecord nodeRecord); } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java index 98a6efa6d..90a129564 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java @@ -93,13 +93,12 @@ public void stop() { discoveryServer.stop(); } - @VisibleForTesting - void connect(NodeRecord nodeRecord) { + public CompletableFuture connect(NodeRecord nodeRecord) { if (!nodeTable.getNode(nodeRecord.getNodeId()).isPresent()) { nodeTable.save(NodeRecordInfo.createDefault(nodeRecord)); } NodeContext context = getContext(nodeRecord.getNodeId()).get(); - context.initiate(); + return context.initiate(); } private Optional getContext(Bytes32 nodeId) { diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryTaskManager.java b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryTaskManager.java new file mode 100644 index 000000000..e87085956 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryTaskManager.java @@ -0,0 +1,127 @@ +package org.ethereum.beacon.discovery; + +import org.ethereum.beacon.discovery.enr.NodeRecordInfo; +import org.ethereum.beacon.discovery.enr.NodeRecordV5; +import org.ethereum.beacon.discovery.enr.NodeStatus; +import org.ethereum.beacon.discovery.storage.NodeTable; +import org.ethereum.beacon.schedulers.Scheduler; +import tech.pegasys.artemis.util.bytes.Bytes32; + +import java.time.Duration; +import java.util.List; +import java.util.function.Predicate; +import java.util.stream.Stream; + +import static org.ethereum.beacon.discovery.NodeContext.DEFAULT_DISTANCE; +import static org.ethereum.beacon.discovery.enr.NodeRecordV5.NODE_ID_FUNCTION; + +/** Manages recurrent node check task(s) */ +public class DiscoveryTaskManager { + private static final int MS_IN_SECOND = 1000; + private static final int STATUS_EXPIRATION_SECONDS = 600; + private static final int TASK_INTERVAL_SECONDS = 10; + private static final int RETRY_TIMEOUT_SECONDS = 60; + private static final int MAX_RETRIES = 10; + private final Scheduler scheduler; + private final Bytes32 homeNodeId; + private final NodeConnectTasks nodeConnectTasks; + private NodeTable nodeTable; + /** + * Checks whether {@link org.ethereum.beacon.discovery.enr.NodeRecord} is ready for alive status + * check. Plus, marks records as DEAD if there were a lot of unsuccessful retries to get reply + * from node. + * + *

We don't need to recheck the node if + * + *

    + *
  • Node is ACTIVE and last connection retry was not too much time ago + *
  • Number of unsuccessful retries exceeds settings + *
  • Node is not ACTIVE but last connection retry was "seconds ago" + *
+ * + *

In all other cases method returns true, meaning node is ready for ping check + */ + private final Predicate READY_FOR_PING = + nodeRecord -> { + long currentTime = System.currentTimeMillis() / MS_IN_SECOND; + if (nodeRecord.getStatus() == NodeStatus.ACTIVE + && nodeRecord.getLastRetry() > currentTime - STATUS_EXPIRATION_SECONDS) { + return false; // no need to rediscover + } + if (nodeRecord.getRetry() >= MAX_RETRIES) { + nodeTable.save( + new NodeRecordInfo( + nodeRecord.getNode(), nodeRecord.getLastRetry(), NodeStatus.DEAD, 0)); + return false; + } + if ((currentTime - nodeRecord.getLastRetry()) + < (nodeRecord.getRetry() * nodeRecord.getRetry())) { + return false; // too early for retry + } + + return true; + }; + private boolean resetDead = false; + + /** + * @param discoveryManager Discovery manager + * @param nodeTable Ethereum node records storage + * @param homeNode Home node + * @param scheduler scheduler to run recurrent tasks on + * @param resetDead Whether to reset dead status of the nodes. If set to true, resets its status + * at startup and sets number of used retries to 0 + */ + public DiscoveryTaskManager( + DiscoveryManager discoveryManager, + NodeTable nodeTable, + NodeRecordV5 homeNode, + Scheduler scheduler, + boolean resetDead) { + this.scheduler = scheduler; + this.nodeTable = nodeTable; + this.homeNodeId = NODE_ID_FUNCTION.apply(homeNode); + this.nodeConnectTasks = new NodeConnectTasks(discoveryManager, scheduler, Duration.ofSeconds(RETRY_TIMEOUT_SECONDS)); + this.resetDead = resetDead; + } + + public void start() { + scheduler.executeAtFixedRate( + Duration.ZERO, Duration.ofSeconds(TASK_INTERVAL_SECONDS), this::recurrentTask); + } + + private void recurrentTask() { + List nodes = nodeTable.findClosestNodes(homeNodeId, DEFAULT_DISTANCE); + Stream closestNodes = nodes.stream(); + if (resetDead) { + closestNodes = + closestNodes.map( + nodeRecordInfo -> + new NodeRecordInfo( + nodeRecordInfo.getNode(), + nodeRecordInfo.getLastRetry(), + NodeStatus.SLEEP, + 0)); + resetDead = false; + } + closestNodes + .filter(READY_FOR_PING) + .forEach( + nodeRecord -> + nodeConnectTasks.add( + nodeRecord, + () -> + nodeTable.save( + new NodeRecordInfo( + nodeRecord.getNode(), + System.currentTimeMillis() / MS_IN_SECOND, + NodeStatus.ACTIVE, + 0)), + () -> + nodeTable.save( + new NodeRecordInfo( + nodeRecord.getNode(), + System.currentTimeMillis() / MS_IN_SECOND, + NodeStatus.SLEEP, + (nodeRecord.getRetry() + 1))))); + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeConnectTasks.java b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeConnectTasks.java new file mode 100644 index 000000000..0c2d5025e --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeConnectTasks.java @@ -0,0 +1,64 @@ +package org.ethereum.beacon.discovery; + +import com.google.common.collect.Sets; +import org.ethereum.beacon.discovery.enr.NodeRecord; +import org.ethereum.beacon.discovery.enr.NodeRecordInfo; +import org.ethereum.beacon.schedulers.Scheduler; +import org.ethereum.beacon.util.ExpirationScheduler; +import tech.pegasys.artemis.util.bytes.Bytes32; + +import java.time.Duration; +import java.time.temporal.ChronoUnit; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; + +import static org.ethereum.beacon.discovery.enr.NodeRecordV5.NODE_ID_FUNCTION; + +/** + * Executes tasks {@link DiscoveryManager#connect(NodeRecord)} for all NodeRecords added via {@link + * #add(NodeRecordInfo, Runnable, Runnable)}. Tasks is called failed if timeout is reached and reply + * from node is not received. + */ +public class NodeConnectTasks { + private final Scheduler scheduler; + private final DiscoveryManager discoveryManager; + private final Set currentTasks = Sets.newConcurrentHashSet(); + private final ExpirationScheduler taskTimeouts; + + public NodeConnectTasks( + DiscoveryManager discoveryManager, Scheduler scheduler, Duration timeout) { + this.discoveryManager = discoveryManager; + this.scheduler = scheduler; + this.taskTimeouts = + new ExpirationScheduler<>(timeout.get(ChronoUnit.MILLIS), TimeUnit.MILLISECONDS); + } + + public void add(NodeRecordInfo nodeRecordInfo, Runnable successCallback, Runnable failCallback) { + synchronized (this) { + if (currentTasks.contains(NODE_ID_FUNCTION.apply(nodeRecordInfo.getNode()))) { + return; + } + currentTasks.add(NODE_ID_FUNCTION.apply(nodeRecordInfo.getNode())); + } + + scheduler.execute( + () -> { + CompletableFuture retry = discoveryManager.connect(nodeRecordInfo.getNode()); + taskTimeouts.put( + NODE_ID_FUNCTION.apply(nodeRecordInfo.getNode()), + () -> + retry.completeExceptionally(new RuntimeException("Timeout for node check task"))); + retry.whenComplete( + (aVoid, throwable) -> { + if (throwable != null) { + failCallback.run(); + currentTasks.remove(NODE_ID_FUNCTION.apply(nodeRecordInfo.getNode())); + } else { + successCallback.run(); + currentTasks.remove(NODE_ID_FUNCTION.apply(nodeRecordInfo.getNode())); + } + }); + }); + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeContext.java b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeContext.java index e14938371..6fa7d1a4d 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeContext.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeContext.java @@ -27,6 +27,7 @@ import java.util.Map; import java.util.Optional; import java.util.Random; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.function.Consumer; @@ -46,6 +47,7 @@ public class NodeContext { private BytesValue initiatorKey; private MessageProcessor messageProcessor = null; private Map requestIdReservations = new ConcurrentHashMap<>(); + private CompletableFuture connectFuture = null; public NodeContext( NodeRecordV5 nodeRecord, @@ -79,6 +81,8 @@ public void addIncomingEvent(UnknownPacket packet) { { if (packetHandlers.get(UnknownPacket.class).handle(packet)) { setStatus(SessionStatus.WHOAREYOU_SENT); + } else { + failConnectFuture(); } break; } @@ -87,6 +91,8 @@ public void addIncomingEvent(UnknownPacket packet) { WhoAreYouPacket whoAreYouPacket = packet.getWhoAreYouPacket(); if (packetHandlers.get(WhoAreYouPacket.class).handle(whoAreYouPacket)) { setStatus(SessionStatus.AUTHENTICATED); + } else { + failConnectFuture(); } break; } @@ -95,13 +101,20 @@ public void addIncomingEvent(UnknownPacket packet) { AuthHeaderMessagePacket authHeaderMessagePacket = packet.getAuthHeaderMessagePacket(); if (packetHandlers.get(AuthHeaderMessagePacket.class).handle(authHeaderMessagePacket)) { setStatus(SessionStatus.AUTHENTICATED); + completeConnectFuture(); + } else { + failConnectFuture(); } break; } case AUTHENTICATED: { MessagePacket messagePacket = packet.getMessagePacket(); - packetHandlers.get(MessagePacket.class).handle(messagePacket); + if (packetHandlers.get(MessagePacket.class).handle(messagePacket)) { + completeConnectFuture(); + } else { + failConnectFuture(); + } break; } default: @@ -122,8 +135,35 @@ public void handleMessage(DiscoveryMessage message) { messageProcessor.handleIncoming(message, this); } + private void completeConnectFuture() { + if (connectFuture != null) { + connectFuture.complete(null); + connectFuture = null; + } + } + + private void failConnectFuture() { + if (connectFuture != null) { + connectFuture.completeExceptionally( + new RuntimeException( + String.format("Peer message initiation failed for %s", this.getNodeRecord()))); + connectFuture = null; + } + } + /** Sends random packet to start initiation of session with node */ - public void initiate() { + public CompletableFuture initiate() { + CompletableFuture connectFuture = new CompletableFuture<>(); + if (this.connectFuture != null) { + connectFuture.completeExceptionally( + new RuntimeException( + String.format( + "Only one simultaneous handshake initiation allowed per peer. Got second for %s", + this.getNodeRecord()))); + } + if (status == SessionStatus.AUTHENTICATED) { + completeConnectFuture(); + } byte[] authTagBytes = new byte[12]; rnd.nextBytes(authTagBytes); BytesValue authTag = BytesValue.wrap(authTagBytes); @@ -132,6 +172,7 @@ public void initiate() { authTagRepo.put(authTag, this); this.addOutgoingEvent(randomPacket); this.status = SessionStatus.RANDOM_PACKET_SENT; + return connectFuture; } public synchronized void addOutgoingEvent(Packet packet) { diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/RefreshTask.java b/discovery/src/main/java/org/ethereum/beacon/discovery/RefreshTask.java deleted file mode 100644 index 0f198abf9..000000000 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/RefreshTask.java +++ /dev/null @@ -1,36 +0,0 @@ -package org.ethereum.beacon.discovery; - -import com.google.common.collect.Sets; -import org.ethereum.beacon.discovery.enr.NodeRecordInfo; -import org.ethereum.beacon.schedulers.Scheduler; -import tech.pegasys.artemis.util.bytes.Bytes33; - -import java.util.Set; - -public class RefreshTask { - Scheduler scheduler; - Set currentTasks = Sets.newConcurrentHashSet(); - - public RefreshTask(Scheduler scheduler) { - this.scheduler = scheduler; - } - - public void add( - NodeRecordInfo nodeRecord, Runnable successCallback, Runnable failCallback) { - synchronized (this) { - if (currentTasks.contains(nodeRecord.getNode().getPublicKey())) { - return; - } - currentTasks.add(nodeRecord.getNode().getPublicKey()); - } - - scheduler.execute( - () -> { - // try to connect; - // if success: - // successCallback.run(); - failCallback.run(); - currentTasks.remove(nodeRecord.getNode().getPublicKey()); - }); - } -} diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java b/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java index ebb2b0480..72fb8e534 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java @@ -21,6 +21,7 @@ import java.security.SecureRandom; import java.util.Map; import java.util.Optional; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; /** @@ -72,12 +73,12 @@ public void start() { @Override public void stop() {} - public void connect(NodeRecord nodeRecord) { + public CompletableFuture connect(NodeRecord nodeRecord) { if (!nodeTable.getNode(nodeRecord.getNodeId()).isPresent()) { nodeTable.save(NodeRecordInfo.createDefault(nodeRecord)); } NodeContext context = getContext(nodeRecord.getNodeId()).get(); - context.initiate(); + return context.initiate(); } private Optional getContext(Bytes32 nodeId) { From 1e775eabc3da1e62c1736c0c03e294e5937b844a Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Wed, 25 Sep 2019 16:47:00 +0300 Subject: [PATCH 16/77] discovery: noderecordv4, v5 refactoring. Typo fixes, redundant conversions cleanup --- .../beacon/discovery/enr/NodeRecord.java | 57 +++---------------- .../beacon/discovery/enr/NodeRecordV4.java | 57 ++++++++++++------- .../beacon/discovery/enr/NodeRecordV5.java | 53 ++++++++++------- 3 files changed, 78 insertions(+), 89 deletions(-) diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecord.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecord.java index a77a1b415..823010ba5 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecord.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecord.java @@ -12,15 +12,13 @@ import tech.pegasys.artemis.util.bytes.Bytes32; import tech.pegasys.artemis.util.bytes.BytesValue; -import java.math.BigInteger; import java.util.Base64; import java.util.List; -import java.util.function.Function; /** * Ethereum Node Record * - *

Node record as described in EIP-778 */ @SSZSerializable(listAccessor = NodeRecord.NodeRecordAccessor.class) public interface NodeRecord { @@ -45,48 +43,8 @@ static NodeRecord fromBytes(byte[] bytes) { "Unable to deserialize ENR with no id field at 2-3 records, [%s]", BytesValue.wrap(bytes))); } - RlpString idVersion = (RlpString) values.get(3); - - Function, NodeRecord> nodeRecordV4Creator = - fields -> { - NodeRecordV4.Builder builder = - NodeRecordV4.Builder.empty() - .withSignature(BytesValue.fromHexString(((RlpString) values.get(0)).asString())) - .withSeqNumber( - new BigInteger( - 1, - BytesValue.fromHexString(((RlpString) values.get(1)).asString()) - .extractArray()) - .longValue()); - for (int i = 4; i < values.size(); i += 2) { - builder = - builder.withKeyField( - new String(((RlpString) values.get(i)).getBytes()), - (RlpString) values.get(i + 1)); - } - - return builder.build(); - }; - Function, NodeRecord> nodeRecordV5Creator = - fields -> { - NodeRecordV5.Builder builder = - NodeRecordV5.Builder.empty() - .withSignature(BytesValue.fromHexString(((RlpString) values.get(0)).asString())) - .withSeqNumber( - new BigInteger( - 1, - BytesValue.fromHexString(((RlpString) values.get(1)).asString()) - .extractArray()) - .longValue()); - for (int i = 4; i < values.size(); i += 2) { - builder = - builder.withKeyField( - new String(((RlpString) values.get(i)).getBytes()), - (RlpString) values.get(i + 1)); - } - return builder.build(); - }; + RlpString idVersion = (RlpString) values.get(3); IdentityScheme nodeIdentity = IdentityScheme.fromString(new String(idVersion.getBytes())); if (nodeIdentity == null) { throw new RuntimeException( @@ -97,15 +55,18 @@ static NodeRecord fromBytes(byte[] bytes) { switch (nodeIdentity) { case V4: { - return nodeRecordV4Creator.apply(values); + return NodeRecordV4.fromRlpList(values); } case V5: { - return nodeRecordV5Creator.apply(values); + return NodeRecordV5.fromRlpList(values); + } + default: + { + throw new RuntimeException( + String.format("Builder for identity %s not found", nodeIdentity)); } } - - return null; } static NodeRecord fromBytes(BytesValue bytes) { diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV4.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV4.java index 6fc781cd5..fca13f203 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV4.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV4.java @@ -14,7 +14,6 @@ import tech.pegasys.artemis.util.bytes.Bytes4; import tech.pegasys.artemis.util.bytes.BytesValue; -import java.math.BigInteger; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -25,6 +24,21 @@ public class NodeRecordV4 implements NodeRecord { // id name of identity scheme, e.g. “v4” private static final IdentityScheme identityScheme = IdentityScheme.V4; + private static final Function, NodeRecordV4> nodeRecordV4Creator = + fields -> { + NodeRecordV4.Builder builder = + NodeRecordV4.Builder.empty() + .withSignature(BytesValue.wrap(((RlpString) fields.get(0)).getBytes())) + .withSeqNumber(((RlpString) fields.get(1)).asPositiveBigInteger().longValue()); + for (int i = 4; i < fields.size(); i += 2) { + builder = + builder.withKeyField( + new String(((RlpString) fields.get(i)).getBytes()), + (RlpString) fields.get(i + 1)); + } + + return builder.build(); + }; // secp256k1 compressed secp256k1 public key, 33 bytes private Bytes33 publicKey; // ip IPv4 address, 4 bytes @@ -39,9 +53,7 @@ public class NodeRecordV4 implements NodeRecord { private Integer tcpV6Port; // udp6 IPv6-specific UDP port, big endian integer private Integer udpV6Port; - private Long seqNumber; - private BytesValue signature; private NodeRecordV4( @@ -75,6 +87,10 @@ public static NodeRecordV4 fromValues( publicKey, ipV4address, tcpPort, udpPort, ipV6address, tcpV6Port, udpV6Port); } + public static NodeRecordV4 fromRlpList(List values) { + return nodeRecordV4Creator.apply(values); + } + public IdentityScheme getIdentityScheme() { return identityScheme; } @@ -183,7 +199,7 @@ public int hashCode() { @Override public BytesValue serialize() { - assert getSignature() != null; + assert getSignature() != null; assert getSeqNumber() != null; // content = [seq, k, v, ...] @@ -233,6 +249,18 @@ public Bytes32 getNodeId() { return Hashes.sha256(getPublicKey()); } + @Override + public String toString() { + return "NodeRecordV4{" + + "publicKey=" + + publicKey + + ", ipV4address=" + + ipV4address + + ", udpPort=" + + udpPort + + '}'; + } + public static class Builder { private static final Map, Builder>> fieldFillersV4 = new HashMap<>(); @@ -243,26 +271,20 @@ public static class Builder { objects -> objects .getValue0() - .withIpV4Address( - BytesValue.fromHexString(((RlpString) objects.getValue1()).asString()))); + .withIpV4Address(BytesValue.wrap(((RlpString) objects.getValue1()).getBytes()))); fieldFillersV4.put( "secp256k1", objects -> objects .getValue0() - .withSecp256k1( - BytesValue.fromHexString(((RlpString) objects.getValue1()).asString()))); + .withSecp256k1(BytesValue.wrap(((RlpString) objects.getValue1()).getBytes()))); fieldFillersV4.put( "udp", objects -> objects .getValue0() .withUdpPort( - new BigInteger( - 1, - BytesValue.fromHexString(((RlpString) objects.getValue1()).asString()) - .extractArray()) - .intValue())); + ((RlpString) objects.getValue1()).asPositiveBigInteger().intValue())); } private Bytes4 ipV4Address; @@ -331,13 +353,4 @@ public NodeRecordV4 build() { return nodeRecord; } } - - @Override - public String toString() { - return "NodeRecordV4{" + - "publicKey=" + publicKey + - ", ipV4address=" + ipV4address + - ", udpPort=" + udpPort + - '}'; - } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV5.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV5.java index 9eec6bcbc..909bacb3d 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV5.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV5.java @@ -15,7 +15,6 @@ import tech.pegasys.artemis.util.bytes.Bytes96; import tech.pegasys.artemis.util.bytes.BytesValue; -import java.math.BigInteger; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -26,6 +25,21 @@ public class NodeRecordV5 implements NodeRecord { // id name of identity scheme, e.g. “v4” private static final IdentityScheme identityScheme = IdentityScheme.V5; + private static final Function, NodeRecordV5> nodeRecordV5Creator = + fields -> { + NodeRecordV5.Builder builder = + NodeRecordV5.Builder.empty() + .withSignature(BytesValue.wrap(((RlpString) fields.get(0)).getBytes())) + .withSeqNumber(((RlpString) fields.get(1)).asPositiveBigInteger().longValue()); + for (int i = 4; i < fields.size(); i += 2) { + builder = + builder.withKeyField( + new String(((RlpString) fields.get(i)).getBytes()), + (RlpString) fields.get(i + 1)); + } + + return builder.build(); + }; public static Function NODE_ID_FUNCTION = nodeRecordV5 -> Hashes.sha256(nodeRecordV5.getPublicKey()); // secp256k1 compressed secp256k1 public key, 33 bytes @@ -80,6 +94,10 @@ public static NodeRecordV5 fromValues( publicKey, ipV4address, tcpPort, udpPort, ipV6address, tcpV6Port, udpV6Port); } + public static NodeRecordV5 fromRlpList(List values) { + return nodeRecordV5Creator.apply(values); + } + public IdentityScheme getIdentityScheme() { return identityScheme; } @@ -240,6 +258,18 @@ public Bytes32 getNodeId() { return NODE_ID_FUNCTION.apply(this); } + @Override + public String toString() { + return "NodeRecordV5{" + + "publicKey=" + + publicKey + + ", ipV4address=" + + ipV4address + + ", udpPort=" + + udpPort + + '}'; + } + public static class Builder { private static final Map, Builder>> fieldFillersV5 = new HashMap<>(); @@ -250,26 +280,20 @@ public static class Builder { objects -> objects .getValue0() - .withIpV4Address( - BytesValue.fromHexString(((RlpString) objects.getValue1()).asString()))); + .withIpV4Address(BytesValue.wrap(((RlpString) objects.getValue1()).getBytes()))); fieldFillersV5.put( "secp256k1", objects -> objects .getValue0() - .withSecp256k1( - BytesValue.fromHexString(((RlpString) objects.getValue1()).asString()))); + .withSecp256k1(BytesValue.wrap(((RlpString) objects.getValue1()).getBytes()))); fieldFillersV5.put( "udp", objects -> objects .getValue0() .withUdpPort( - new BigInteger( - 1, - BytesValue.fromHexString(((RlpString) objects.getValue1()).asString()) - .extractArray()) - .intValue())); + ((RlpString) objects.getValue1()).asPositiveBigInteger().intValue())); } private Bytes4 ipV4Address; @@ -338,13 +362,4 @@ public NodeRecordV5 build() { return nodeRecord; } } - - @Override - public String toString() { - return "NodeRecordV5{" + - "publicKey=" + publicKey + - ", ipV4address=" + ipV4address + - ", udpPort=" + udpPort + - '}'; - } } From 840ac962a84531e22b765a601e30ab534ebc3bf5 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Wed, 25 Sep 2019 17:15:06 +0300 Subject: [PATCH 17/77] discovery: NodeRecord: add guaranteed fields to the interface, fix seq size --- .../beacon/discovery/enr/NodeRecord.java | 10 ++ .../beacon/discovery/enr/NodeRecordV4.java | 58 ++++++---- .../beacon/discovery/enr/NodeRecordV5.java | 55 +++++++--- .../discovery/message/DiscoveryV5Message.java | 5 +- .../beacon/discovery/message/PongMessage.java | 9 +- .../message/handler/PingHandler.java | 2 +- .../discovery/packet/WhoAreYouPacket.java | 4 +- .../packet/handler/UnknownPacketHandler.java | 2 +- .../discovery/DiscoveryNetworkTest.java | 5 +- .../discovery/DiscoveryNoNetworkTest.java | 5 +- .../beacon/discovery/NodeRecordTest.java | 18 ++-- .../discovery/storage/NodeTableTest.java | 101 +++++++++++------- 12 files changed, 182 insertions(+), 92 deletions(-) diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecord.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecord.java index 823010ba5..a1b60df40 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecord.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecord.java @@ -11,6 +11,7 @@ import org.web3j.rlp.RlpType; import tech.pegasys.artemis.util.bytes.Bytes32; import tech.pegasys.artemis.util.bytes.BytesValue; +import tech.pegasys.artemis.util.uint.UInt64; import java.util.Base64; import java.util.List; @@ -73,8 +74,17 @@ static NodeRecord fromBytes(BytesValue bytes) { return fromBytes(bytes.extractArray()); } + /** Every {@link IdentityScheme} links with its own implementation */ IdentityScheme getIdentityScheme(); + BytesValue getSignature(); + + /** + * @return The sequence number, a 64-bit unsigned integer. Nodes should increase the number + * whenever the record changes and republish the record. + */ + UInt64 getSeq(); + BytesValue serialize(); default String asBase64() { diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV4.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV4.java index fca13f203..87b002268 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV4.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV4.java @@ -12,7 +12,9 @@ import tech.pegasys.artemis.util.bytes.Bytes32; import tech.pegasys.artemis.util.bytes.Bytes33; import tech.pegasys.artemis.util.bytes.Bytes4; +import tech.pegasys.artemis.util.bytes.Bytes8; import tech.pegasys.artemis.util.bytes.BytesValue; +import tech.pegasys.artemis.util.uint.UInt64; import java.util.ArrayList; import java.util.HashMap; @@ -29,7 +31,9 @@ public class NodeRecordV4 implements NodeRecord { NodeRecordV4.Builder builder = NodeRecordV4.Builder.empty() .withSignature(BytesValue.wrap(((RlpString) fields.get(0)).getBytes())) - .withSeqNumber(((RlpString) fields.get(1)).asPositiveBigInteger().longValue()); + .withSeq( + UInt64.fromBytesBigEndian( + Bytes8.leftPad(BytesValue.wrap(((RlpString) fields.get(1)).getBytes())))); for (int i = 4; i < fields.size(); i += 2) { builder = builder.withKeyField( @@ -53,7 +57,10 @@ public class NodeRecordV4 implements NodeRecord { private Integer tcpV6Port; // udp6 IPv6-specific UDP port, big endian integer private Integer udpV6Port; - private Long seqNumber; + // seq The sequence number, a 64-bit unsigned integer. Nodes should increase the number whenever + // the record changes and republish the record. + private UInt64 seq; + // Signature private BytesValue signature; private NodeRecordV4( @@ -63,7 +70,9 @@ private NodeRecordV4( Integer udpPort, Bytes16 ipV6address, Integer tcpV6Port, - Integer udpV6Port) { + Integer udpV6Port, + UInt64 seq, + BytesValue signature) { this.publicKey = publicKey; this.ipV4address = ipV4address; this.tcpPort = tcpPort; @@ -71,6 +80,8 @@ private NodeRecordV4( this.ipV6address = ipV6address; this.tcpV6Port = tcpV6Port; this.udpV6Port = udpV6Port; + this.seq = seq; + this.signature = signature; } private NodeRecordV4() {} @@ -82,9 +93,19 @@ public static NodeRecordV4 fromValues( Integer udpPort, Bytes16 ipV6address, Integer tcpV6Port, - Integer udpV6Port) { + Integer udpV6Port, + UInt64 seq, + BytesValue signature) { return new NodeRecordV4( - publicKey, ipV4address, tcpPort, udpPort, ipV6address, tcpV6Port, udpV6Port); + publicKey, + ipV4address, + tcpPort, + udpPort, + ipV6address, + tcpV6Port, + udpV6Port, + seq, + signature); } public static NodeRecordV4 fromRlpList(List values) { @@ -151,14 +172,15 @@ public void setUdpV6Port(Integer udpV6Port) { this.udpV6Port = udpV6Port; } - public Long getSeqNumber() { - return seqNumber; + public UInt64 getSeq() { + return seq; } - public void setSeqNumber(Long seqNumber) { - this.seqNumber = seqNumber; + public void setSeq(UInt64 seq) { + this.seq = seq; } + @Override public BytesValue getSignature() { return signature; } @@ -179,7 +201,7 @@ public boolean equals(Object o) { && Objects.equal(ipV6address, that.ipV6address) && Objects.equal(tcpV6Port, that.tcpV6Port) && Objects.equal(udpV6Port, that.udpV6Port) - && Objects.equal(seqNumber, that.seqNumber) + && Objects.equal(seq, that.seq) && Objects.equal(signature, that.signature); } @@ -193,21 +215,21 @@ public int hashCode() { ipV6address, tcpV6Port, udpV6Port, - seqNumber, + seq, signature); } @Override public BytesValue serialize() { assert getSignature() != null; - assert getSeqNumber() != null; + assert getSeq() != null; // content = [seq, k, v, ...] // signature = sign(content) // record = [signature, seq, k, v, ...] List values = new ArrayList<>(); values.add(RlpString.create(getSignature().extractArray())); - values.add(RlpString.create(getSeqNumber())); + values.add(RlpString.create(getSeq().toBI())); values.add(RlpString.create("id")); values.add(RlpString.create(getIdentityScheme().stringName())); if (getPublicKey() != null) { @@ -291,7 +313,7 @@ public static class Builder { private Bytes33 secp256k1; private Integer tcpPort; private Integer udpPort; - private Long seqNumber; + private UInt64 seq; private BytesValue signature; private Builder() {} @@ -320,8 +342,8 @@ public Builder withSecp256k1(BytesValue bytes) { return this; } - public Builder withSeqNumber(Long seqNumber) { - this.seqNumber = seqNumber; + public Builder withSeq(UInt64 seq) { + this.seq = seq; return this; } @@ -340,14 +362,14 @@ public Builder withKeyField(String key, RlpString value) { } public NodeRecordV4 build() { - assert seqNumber != null; + assert seq != null; assert secp256k1 != null; NodeRecordV4 nodeRecord = new NodeRecordV4(); nodeRecord.setIpV4address(ipV4Address); nodeRecord.setUdpPort(udpPort); nodeRecord.setTcpPort(tcpPort); - nodeRecord.setSeqNumber(seqNumber); + nodeRecord.setSeq(seq); nodeRecord.setSignature(signature); nodeRecord.setPublicKey(secp256k1); return nodeRecord; diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV5.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV5.java index 909bacb3d..b82f1c9f1 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV5.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV5.java @@ -12,8 +12,10 @@ import tech.pegasys.artemis.util.bytes.Bytes32; import tech.pegasys.artemis.util.bytes.Bytes33; import tech.pegasys.artemis.util.bytes.Bytes4; +import tech.pegasys.artemis.util.bytes.Bytes8; import tech.pegasys.artemis.util.bytes.Bytes96; import tech.pegasys.artemis.util.bytes.BytesValue; +import tech.pegasys.artemis.util.uint.UInt64; import java.util.ArrayList; import java.util.HashMap; @@ -30,7 +32,9 @@ public class NodeRecordV5 implements NodeRecord { NodeRecordV5.Builder builder = NodeRecordV5.Builder.empty() .withSignature(BytesValue.wrap(((RlpString) fields.get(0)).getBytes())) - .withSeqNumber(((RlpString) fields.get(1)).asPositiveBigInteger().longValue()); + .withSeq( + UInt64.fromBytesBigEndian( + Bytes8.leftPad(BytesValue.wrap(((RlpString) fields.get(1)).getBytes())))); for (int i = 4; i < fields.size(); i += 2) { builder = builder.withKeyField( @@ -56,7 +60,10 @@ public class NodeRecordV5 implements NodeRecord { private Integer tcpV6Port; // udp6 IPv6-specific UDP port, big endian integer private Integer udpV6Port; - private Long seqNumber; + // seq The sequence number, a 64-bit unsigned integer. Nodes should increase the number whenever + // the record changes and republish the record. + private UInt64 seq; + // Signature private BytesValue signature; private NodeRecordV5( @@ -66,7 +73,9 @@ private NodeRecordV5( Integer udpPort, Bytes16 ipV6address, Integer tcpV6Port, - Integer udpV6Port) { + Integer udpV6Port, + UInt64 seq, + BytesValue signature) { this.publicKey = publicKey; this.ipV4address = ipV4address; this.tcpPort = tcpPort; @@ -74,6 +83,8 @@ private NodeRecordV5( this.ipV6address = ipV6address; this.tcpV6Port = tcpV6Port; this.udpV6Port = udpV6Port; + this.seq = seq; + this.signature = signature; } private NodeRecordV5() {} @@ -89,9 +100,19 @@ public static NodeRecordV5 fromValues( Integer udpPort, Bytes16 ipV6address, Integer tcpV6Port, - Integer udpV6Port) { + Integer udpV6Port, + UInt64 seq, + BytesValue signature) { return new NodeRecordV5( - publicKey, ipV4address, tcpPort, udpPort, ipV6address, tcpV6Port, udpV6Port); + publicKey, + ipV4address, + tcpPort, + udpPort, + ipV6address, + tcpV6Port, + udpV6Port, + seq, + signature); } public static NodeRecordV5 fromRlpList(List values) { @@ -158,14 +179,16 @@ public void setUdpV6Port(Integer udpV6Port) { this.udpV6Port = udpV6Port; } - public Long getSeqNumber() { - return seqNumber; + @Override + public UInt64 getSeq() { + return seq; } - public void setSeqNumber(Long seqNumber) { - this.seqNumber = seqNumber; + public void setSeq(UInt64 seq) { + this.seq = seq; } + @Override public BytesValue getSignature() { return signature; } @@ -186,7 +209,7 @@ public BytesValue serialize() { } else { values.add(RlpString.create(Bytes96.ZERO.extractArray())); // FIXME: is it ok? } - values.add(RlpString.create(getSeqNumber())); + values.add(RlpString.create(getSeq().toBI())); values.add(RlpString.create("id")); values.add(RlpString.create(getIdentityScheme().stringName())); if (getPublicKey() != null) { @@ -235,7 +258,7 @@ public boolean equals(Object o) { && Objects.equal(ipV6address, that.ipV6address) && Objects.equal(tcpV6Port, that.tcpV6Port) && Objects.equal(udpV6Port, that.udpV6Port) - && Objects.equal(seqNumber, that.seqNumber) + && Objects.equal(seq, that.seq) && Objects.equal(signature, that.signature); } @@ -249,7 +272,7 @@ public int hashCode() { ipV6address, tcpV6Port, udpV6Port, - seqNumber, + seq, signature); } @@ -300,7 +323,7 @@ public static class Builder { private Bytes33 secp256k1; private Integer tcpPort; private Integer udpPort; - private Long seqNumber; + private UInt64 seqNumber; private BytesValue signature; private Builder() {} @@ -329,8 +352,8 @@ public Builder withSecp256k1(BytesValue bytes) { return this; } - public Builder withSeqNumber(Long seqNumber) { - this.seqNumber = seqNumber; + public Builder withSeq(UInt64 seq) { + this.seqNumber = seq; return this; } @@ -356,7 +379,7 @@ public NodeRecordV5 build() { nodeRecord.setIpV4address(ipV4Address); nodeRecord.setUdpPort(udpPort); nodeRecord.setTcpPort(tcpPort); - nodeRecord.setSeqNumber(seqNumber); + nodeRecord.setSeq(seqNumber); nodeRecord.setSignature(signature); nodeRecord.setPublicKey(secp256k1); return nodeRecord; diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/DiscoveryV5Message.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/DiscoveryV5Message.java index 71e87f6f3..840482683 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/message/DiscoveryV5Message.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/DiscoveryV5Message.java @@ -7,7 +7,9 @@ import org.web3j.rlp.RlpList; import org.web3j.rlp.RlpString; import org.web3j.rlp.RlpType; +import tech.pegasys.artemis.util.bytes.Bytes8; import tech.pegasys.artemis.util.bytes.BytesValue; +import tech.pegasys.artemis.util.uint.UInt64; import java.util.List; import java.util.stream.Collectors; @@ -66,7 +68,8 @@ public V5Message create() { { return new PongMessage( BytesValue.wrap(((RlpString) payload.get(0)).getBytes()), - ((RlpString) payload.get(1)).asPositiveBigInteger().longValueExact(), + UInt64.fromBytesBigEndian( + Bytes8.leftPad(BytesValue.wrap(((RlpString) payload.get(1)).getBytes()))), BytesValue.wrap(((RlpString) payload.get(2)).getBytes()), ((RlpString) payload.get(3)).asPositiveBigInteger().intValueExact()); } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/PongMessage.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/PongMessage.java index ce988bc9c..d42dbd7d7 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/message/PongMessage.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/PongMessage.java @@ -6,20 +6,21 @@ import org.web3j.rlp.RlpString; import tech.pegasys.artemis.util.bytes.Bytes1; import tech.pegasys.artemis.util.bytes.BytesValue; +import tech.pegasys.artemis.util.uint.UInt64; /** PONG is the reply to PING {@link PingMessage} */ public class PongMessage implements V5Message { // Unique request id private final BytesValue requestId; // Local ENR sequence number of sender - private final Long enrSeq; + private final UInt64 enrSeq; // 16 or 4 byte IP address of the intended recipient private final BytesValue recipientIp; // recipient UDP port, a 16-bit integer private final Integer recipientPort; public PongMessage( - BytesValue requestId, Long enrSeq, BytesValue recipientIp, Integer recipientPort) { + BytesValue requestId, UInt64 enrSeq, BytesValue recipientIp, Integer recipientPort) { this.requestId = requestId; this.enrSeq = enrSeq; this.recipientIp = recipientIp; @@ -31,7 +32,7 @@ public BytesValue getRequestId() { return requestId; } - public Long getEnrSeq() { + public UInt64 getEnrSeq() { return enrSeq; } @@ -51,7 +52,7 @@ public BytesValue getBytes() { RlpEncoder.encode( new RlpList( RlpString.create(requestId.extractArray()), - RlpString.create(enrSeq), + RlpString.create(enrSeq.toBI()), RlpString.create(recipientIp.extractArray()), RlpString.create(recipientPort))))); } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/PingHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/PingHandler.java index 68e3941cd..70260eb31 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/PingHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/PingHandler.java @@ -12,7 +12,7 @@ public void handle(PingMessage message, NodeContext context) { PongMessage responseMessage = new PongMessage( message.getRequestId(), - context.getNodeRecord().getSeqNumber(), + context.getNodeRecord().getSeq(), context.getNodeRecord().getIpV4address(), context.getNodeRecord().getUdpPort()); context.addOutgoingEvent( diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/WhoAreYouPacket.java b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/WhoAreYouPacket.java index 1774fc55f..d0476ac6e 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/WhoAreYouPacket.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/WhoAreYouPacket.java @@ -30,14 +30,14 @@ public WhoAreYouPacket(BytesValue bytes) { } public static WhoAreYouPacket create( - Bytes32 destNodeId, BytesValue authTag, Bytes32 idNonce, Long enrSeq) { + Bytes32 destNodeId, BytesValue authTag, Bytes32 idNonce, UInt64 enrSeq) { BytesValue magic = getStartMagic(destNodeId); byte[] rlpListEncoded = RlpEncoder.encode( new RlpList( RlpString.create(authTag.extractArray()), RlpString.create(idNonce.extractArray()), - RlpString.create(enrSeq))); + RlpString.create(enrSeq.toBI()))); return new WhoAreYouPacket(magic.concat(BytesValue.wrap(rlpListEncoded))); } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/handler/UnknownPacketHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/handler/UnknownPacketHandler.java index de7c4b8bd..a84955e39 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/handler/UnknownPacketHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/handler/UnknownPacketHandler.java @@ -45,7 +45,7 @@ public boolean handle(UnknownPacket packet) { context.getNodeRecord().getNodeId(), authTag, idNonce, - context.getNodeRecord().getSeqNumber()); + context.getNodeRecord().getSeq()); context.addOutgoingEvent(whoAreYouPacket); } catch (AssertionError ex) { logger.info( diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java index 00a1a9c5a..028501ef8 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java @@ -20,6 +20,7 @@ import org.junit.Test; import reactor.core.publisher.Flux; import tech.pegasys.artemis.util.bytes.BytesValue; +import tech.pegasys.artemis.util.uint.UInt64; import java.net.InetAddress; import java.util.ArrayList; @@ -36,7 +37,7 @@ public void test() throws Exception { NodeRecordV5 nodeRecord1 = NodeRecordV5.Builder.empty() .withIpV4Address(BytesValue.wrap(InetAddress.getByName("127.0.0.1").getAddress())) - .withSeqNumber(1L) + .withSeq(UInt64.valueOf(1)) .withUdpPort(30303) .withSecp256k1( BytesValue.fromHexString( @@ -45,7 +46,7 @@ public void test() throws Exception { NodeRecordV5 nodeRecord2 = NodeRecordV5.Builder.empty() .withIpV4Address(BytesValue.wrap(InetAddress.getByName("127.0.0.1").getAddress())) - .withSeqNumber(1L) + .withSeq(UInt64.valueOf(1)) .withUdpPort(30304) .withSecp256k1( BytesValue.fromHexString( diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java index 1f9d1e97a..132c09458 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java @@ -22,6 +22,7 @@ import org.junit.Test; import reactor.core.publisher.Flux; import tech.pegasys.artemis.util.bytes.BytesValue; +import tech.pegasys.artemis.util.uint.UInt64; import java.net.InetAddress; import java.util.ArrayList; @@ -41,7 +42,7 @@ public void test() throws Exception { NodeRecordV5 nodeRecord1 = NodeRecordV5.Builder.empty() .withIpV4Address(BytesValue.wrap(InetAddress.getByName("127.0.0.1").getAddress())) - .withSeqNumber(1L) + .withSeq(UInt64.valueOf(1)) .withUdpPort(30303) .withSecp256k1( BytesValue.fromHexString( @@ -50,7 +51,7 @@ public void test() throws Exception { NodeRecordV5 nodeRecord2 = NodeRecordV5.Builder.empty() .withIpV4Address(BytesValue.wrap(InetAddress.getByName("192.168.0.1").getAddress())) - .withSeqNumber(1L) + .withSeq(UInt64.valueOf(1)) .withUdpPort(30303) .withSecp256k1( BytesValue.fromHexString( diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/NodeRecordTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/NodeRecordTest.java index 808dc9ca2..41e03fe0a 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/NodeRecordTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/NodeRecordTest.java @@ -6,7 +6,9 @@ import org.junit.Test; import tech.pegasys.artemis.util.bytes.Bytes33; import tech.pegasys.artemis.util.bytes.Bytes4; +import tech.pegasys.artemis.util.bytes.Bytes96; import tech.pegasys.artemis.util.bytes.BytesValue; +import tech.pegasys.artemis.util.uint.UInt64; import java.net.InetAddress; @@ -26,7 +28,7 @@ public void testLocalhostV4() throws Exception { final String expectedHost = "127.0.0.1"; final Integer expectedUdpPort = 30303; final Integer expectedTcpPort = null; - final Long expectedSeqNumber = 1L; + final UInt64 expectedSeqNumber = UInt64.valueOf(1); final Bytes33 expectedPublicKey = Bytes33.fromHexString("03ca634cae0d49acb401d8a4c6b6fe8c55b70d115bf400769cc1400f3258cd3138"); final BytesValue expectedSignature = @@ -44,7 +46,7 @@ public void testLocalhostV4() throws Exception { nodeRecordV4.getIpV4address().extractArray()); assertEquals(expectedUdpPort, nodeRecordV4.getUdpPort()); assertEquals(expectedTcpPort, nodeRecordV4.getTcpPort()); - assertEquals(expectedSeqNumber, nodeRecordV4.getSeqNumber()); + assertEquals(expectedSeqNumber, nodeRecordV4.getSeq()); assertEquals(expectedPublicKey, nodeRecordV4.getPublicKey()); assertEquals(expectedSignature, nodeRecordV4.getSignature()); @@ -59,7 +61,7 @@ public void testLocalhostV4() throws Exception { nodeRecordV4Restored.getIpV4address().extractArray()); assertEquals(expectedUdpPort, nodeRecordV4Restored.getUdpPort()); assertEquals(expectedTcpPort, nodeRecordV4Restored.getTcpPort()); - assertEquals(expectedSeqNumber, nodeRecordV4Restored.getSeqNumber()); + assertEquals(expectedSeqNumber, nodeRecordV4Restored.getSeq()); assertEquals(expectedPublicKey, nodeRecordV4Restored.getPublicKey()); assertEquals(expectedSignature, nodeRecordV4Restored.getSignature()); } @@ -69,7 +71,7 @@ public void testLocalhostV5() throws Exception { final String expectedHost = "127.0.0.1"; final Integer expectedUdpPort = 30303; final Integer expectedTcpPort = null; - final Long expectedSeqNumber = 1L; + final UInt64 expectedSeqNumber = UInt64.valueOf(1); final Bytes33 expectedPublicKey = Bytes33.fromHexString("03ca634cae0d49acb401d8a4c6b6fe8c55b70d115bf400769cc1400f3258cd3138"); final BytesValue expectedSignature = @@ -84,8 +86,10 @@ public void testLocalhostV5() throws Exception { expectedUdpPort, null, null, - null); - nodeRecordV5.setSeqNumber(expectedSeqNumber); + null, + UInt64.valueOf(1), + Bytes96.ZERO); + nodeRecordV5.setSeq(expectedSeqNumber); nodeRecordV5.setSignature(expectedSignature); String enrV5 = nodeRecordV5.asBase64(); @@ -96,7 +100,7 @@ public void testLocalhostV5() throws Exception { nodeRecordV5Restored.getIpV4address().extractArray()); assertEquals(expectedUdpPort, nodeRecordV5Restored.getUdpPort()); assertEquals(expectedTcpPort, nodeRecordV5Restored.getTcpPort()); - assertEquals(expectedSeqNumber, nodeRecordV5Restored.getSeqNumber()); + assertEquals(expectedSeqNumber, nodeRecordV5Restored.getSeq()); assertEquals(expectedPublicKey, nodeRecordV5Restored.getPublicKey()); assertEquals(expectedSignature, nodeRecordV5Restored.getSignature()); } diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeTableTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeTableTest.java index 23a70771f..be078837e 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeTableTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeTableTest.java @@ -7,12 +7,10 @@ import org.ethereum.beacon.discovery.enr.NodeRecordInfo; import org.ethereum.beacon.discovery.enr.NodeRecordV5; import org.ethereum.beacon.discovery.enr.NodeStatus; -import org.ethereum.beacon.discovery.storage.NodeTableImpl; -import org.ethereum.beacon.discovery.storage.NodeTableStorage; -import org.ethereum.beacon.discovery.storage.NodeTableStorageFactoryImpl; import org.junit.Test; import tech.pegasys.artemis.util.bytes.Bytes32; import tech.pegasys.artemis.util.bytes.BytesValue; +import tech.pegasys.artemis.util.uint.UInt64; import java.net.InetAddress; import java.net.UnknownHostException; @@ -27,18 +25,21 @@ public class NodeTableTest { - private Supplier homeNodeSupplier = () -> { - try { - return NodeRecordV5.Builder.empty() - .withIpV4Address(BytesValue.wrap(InetAddress.getByName("127.0.0.1").getAddress())) - .withSeqNumber(1L) - .withUdpPort(30303) - .withSecp256k1(BytesValue.fromHexString("0bfb48004b1698f05872cf18b1f278998ad8f7d4c135aa41f83744e7b850ab6b98")) - .build(); - } catch (UnknownHostException e) { - throw new RuntimeException(e); - } - }; + private Supplier homeNodeSupplier = + () -> { + try { + return NodeRecordV5.Builder.empty() + .withIpV4Address(BytesValue.wrap(InetAddress.getByName("127.0.0.1").getAddress())) + .withSeq(UInt64.valueOf(1)) + .withUdpPort(30303) + .withSecp256k1( + BytesValue.fromHexString( + "0bfb48004b1698f05872cf18b1f278998ad8f7d4c135aa41f83744e7b850ab6b98")) + .build(); + } catch (UnknownHostException e) { + throw new RuntimeException(e); + } + }; @Test public void testCreate() throws Exception { @@ -64,7 +65,9 @@ public void testCreate() throws Exception { assertTrue(extendedEnr.isPresent()); NodeRecordInfo nodeRecord = extendedEnr.get(); assertEquals(nodeRecordV5.getPublicKey(), nodeRecord.getNode().getPublicKey()); - assertEquals(nodeTableStorage.get().getHomeNode().getNodeId(), NODE_ID_FUNCTION.apply(homeNodeSupplier.get())); + assertEquals( + nodeTableStorage.get().getHomeNode().getNodeId(), + NODE_ID_FUNCTION.apply(homeNodeSupplier.get())); } @Test @@ -88,41 +91,63 @@ public void testFind() throws Exception { }); // node is adjusted to be close to localhostEnr - NodeRecordV5 closestNode = NodeRecordV5.Builder.empty() - .withIpV4Address(BytesValue.wrap(InetAddress.getByName("127.0.0.2").getAddress())) - .withSeqNumber(1L) - .withUdpPort(30303) - .withSecp256k1(BytesValue.fromHexString("aafb48004b1698f05872cf18b1f278998ad8f7d4c135aa41f83744e7b850ab6b98")) - .build(); + NodeRecordV5 closestNode = + NodeRecordV5.Builder.empty() + .withIpV4Address(BytesValue.wrap(InetAddress.getByName("127.0.0.2").getAddress())) + .withSeq(UInt64.valueOf(1)) + .withUdpPort(30303) + .withSecp256k1( + BytesValue.fromHexString( + "aafb48004b1698f05872cf18b1f278998ad8f7d4c135aa41f83744e7b850ab6b98")) + .build(); nodeTableStorage.get().save(new NodeRecordInfo(closestNode, -1L, NodeStatus.ACTIVE, 0)); - assertEquals(nodeTableStorage.get().getNode(NODE_ID_FUNCTION.apply(closestNode)).get().getNode().getPublicKey(), closestNode.getPublicKey()); - NodeRecordV5 farNode = NodeRecordV5.Builder.empty() - .withIpV4Address(BytesValue.wrap(InetAddress.getByName("127.0.0.3").getAddress())) - .withSeqNumber(1L) - .withUdpPort(30303) - .withSecp256k1(BytesValue.fromHexString("bafb48004b1698f05872cf18b1f278998ad8f7d4c135aa41f83744e7b850ab6b98")) - .build(); + assertEquals( + nodeTableStorage + .get() + .getNode(NODE_ID_FUNCTION.apply(closestNode)) + .get() + .getNode() + .getPublicKey(), + closestNode.getPublicKey()); + NodeRecordV5 farNode = + NodeRecordV5.Builder.empty() + .withIpV4Address(BytesValue.wrap(InetAddress.getByName("127.0.0.3").getAddress())) + .withSeq(UInt64.valueOf(1)) + .withUdpPort(30303) + .withSecp256k1( + BytesValue.fromHexString( + "bafb48004b1698f05872cf18b1f278998ad8f7d4c135aa41f83744e7b850ab6b98")) + .build(); nodeTableStorage.get().save(new NodeRecordInfo(farNode, -1L, NodeStatus.ACTIVE, 0)); - List closestNodes = nodeTableStorage.get().findClosestNodes(NODE_ID_FUNCTION.apply(closestNode), 252); + List closestNodes = + nodeTableStorage.get().findClosestNodes(NODE_ID_FUNCTION.apply(closestNode), 252); assertEquals(2, closestNodes.size()); assertEquals(closestNodes.get(0).getNode().getPublicKey(), localHostNode.getPublicKey()); assertEquals(closestNodes.get(1).getNode().getPublicKey(), closestNode.getPublicKey()); - List farNodes = nodeTableStorage.get().findClosestNodes(NODE_ID_FUNCTION.apply(farNode), 1); + List farNodes = + nodeTableStorage.get().findClosestNodes(NODE_ID_FUNCTION.apply(farNode), 1); assertEquals(1, farNodes.size()); assertEquals(farNodes.get(0).getNode().getPublicKey(), farNode.getPublicKey()); } /** - * Verifies that calculated index number is in range of [0, {@link NodeTableImpl#NUMBER_OF_INDEXES}) + * Verifies that calculated index number is in range of [0, {@link + * NodeTableImpl#NUMBER_OF_INDEXES}) */ @Test public void testIndexCalculation() { - Bytes32 nodeId0 = Bytes32.fromHexString("0000000000000000000000000000000000000000000000000000000000000000"); - Bytes32 nodeId1a = Bytes32.fromHexString("0000000000000000000000000000000000000000000000000000000000000001"); - Bytes32 nodeId1b = Bytes32.fromHexString("1000000000000000000000000000000000000000000000000000000000000000"); - Bytes32 nodeId1s = Bytes32.fromHexString("1111111111111111111111111111111111111111111111111111111111111111"); - Bytes32 nodeId9s = Bytes32.fromHexString("9999999999999999999999999999999999999999999999999999999999999999"); - Bytes32 nodeIdfs = Bytes32.fromHexString("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"); + Bytes32 nodeId0 = + Bytes32.fromHexString("0000000000000000000000000000000000000000000000000000000000000000"); + Bytes32 nodeId1a = + Bytes32.fromHexString("0000000000000000000000000000000000000000000000000000000000000001"); + Bytes32 nodeId1b = + Bytes32.fromHexString("1000000000000000000000000000000000000000000000000000000000000000"); + Bytes32 nodeId1s = + Bytes32.fromHexString("1111111111111111111111111111111111111111111111111111111111111111"); + Bytes32 nodeId9s = + Bytes32.fromHexString("9999999999999999999999999999999999999999999999999999999999999999"); + Bytes32 nodeIdfs = + Bytes32.fromHexString("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"); assertEquals(0, NodeTableImpl.getNodeIndex(nodeId0)); assertEquals(0, NodeTableImpl.getNodeIndex(nodeId1a)); assertEquals(16, NodeTableImpl.getNodeIndex(nodeId1b)); From 59180c9b5743cfbc8fe5ab55971d62e298216e41 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Wed, 25 Sep 2019 17:19:23 +0300 Subject: [PATCH 18/77] util: remove Bytes33 type --- .../beacon/discovery/enr/NodeRecordV4.java | 15 +- .../beacon/discovery/enr/NodeRecordV5.java | 15 +- .../beacon/discovery/NodeRecordTest.java | 11 +- .../util/bytes/ArrayWrappingBytes33.java | 59 ------ .../pegasys/artemis/util/bytes/Bytes33.java | 183 ------------------ .../bytes/MutableArrayWrappingBytes33.java | 39 ---- .../artemis/util/bytes/MutableBytes33.java | 145 -------------- .../artemis/util/bytes/WrappingBytes33.java | 62 ------ 8 files changed, 20 insertions(+), 509 deletions(-) delete mode 100644 types/src/main/java/tech/pegasys/artemis/util/bytes/ArrayWrappingBytes33.java delete mode 100644 types/src/main/java/tech/pegasys/artemis/util/bytes/Bytes33.java delete mode 100644 types/src/main/java/tech/pegasys/artemis/util/bytes/MutableArrayWrappingBytes33.java delete mode 100644 types/src/main/java/tech/pegasys/artemis/util/bytes/MutableBytes33.java delete mode 100644 types/src/main/java/tech/pegasys/artemis/util/bytes/WrappingBytes33.java diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV4.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV4.java index 87b002268..c4f795d8d 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV4.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV4.java @@ -10,7 +10,6 @@ import org.web3j.rlp.RlpType; import tech.pegasys.artemis.util.bytes.Bytes16; import tech.pegasys.artemis.util.bytes.Bytes32; -import tech.pegasys.artemis.util.bytes.Bytes33; import tech.pegasys.artemis.util.bytes.Bytes4; import tech.pegasys.artemis.util.bytes.Bytes8; import tech.pegasys.artemis.util.bytes.BytesValue; @@ -44,7 +43,7 @@ public class NodeRecordV4 implements NodeRecord { return builder.build(); }; // secp256k1 compressed secp256k1 public key, 33 bytes - private Bytes33 publicKey; + private BytesValue publicKey; // ip IPv4 address, 4 bytes private Bytes4 ipV4address; // tcp TCP port, big endian integer @@ -64,7 +63,7 @@ public class NodeRecordV4 implements NodeRecord { private BytesValue signature; private NodeRecordV4( - Bytes33 publicKey, + BytesValue publicKey, Bytes4 ipV4address, Integer tcpPort, Integer udpPort, @@ -87,7 +86,7 @@ private NodeRecordV4( private NodeRecordV4() {} public static NodeRecordV4 fromValues( - Bytes33 publicKey, + BytesValue publicKey, Bytes4 ipV4address, Integer tcpPort, Integer udpPort, @@ -116,11 +115,11 @@ public IdentityScheme getIdentityScheme() { return identityScheme; } - public Bytes33 getPublicKey() { + public BytesValue getPublicKey() { return publicKey; } - public void setPublicKey(Bytes33 publicKey) { + public void setPublicKey(BytesValue publicKey) { this.publicKey = publicKey; } @@ -310,7 +309,7 @@ public static class Builder { } private Bytes4 ipV4Address; - private Bytes33 secp256k1; + private BytesValue secp256k1; private Integer tcpPort; private Integer udpPort; private UInt64 seq; @@ -338,7 +337,7 @@ public Builder withUdpPort(Integer port) { } public Builder withSecp256k1(BytesValue bytes) { - this.secp256k1 = Bytes33.wrap(bytes, 0); + this.secp256k1 = bytes; return this; } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV5.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV5.java index b82f1c9f1..7b853a39f 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV5.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV5.java @@ -10,7 +10,6 @@ import org.web3j.rlp.RlpType; import tech.pegasys.artemis.util.bytes.Bytes16; import tech.pegasys.artemis.util.bytes.Bytes32; -import tech.pegasys.artemis.util.bytes.Bytes33; import tech.pegasys.artemis.util.bytes.Bytes4; import tech.pegasys.artemis.util.bytes.Bytes8; import tech.pegasys.artemis.util.bytes.Bytes96; @@ -47,7 +46,7 @@ public class NodeRecordV5 implements NodeRecord { public static Function NODE_ID_FUNCTION = nodeRecordV5 -> Hashes.sha256(nodeRecordV5.getPublicKey()); // secp256k1 compressed secp256k1 public key, 33 bytes - private Bytes33 publicKey; + private BytesValue publicKey; // ip IPv4 address, 4 bytes private Bytes4 ipV4address; // tcp TCP port, big endian integer @@ -67,7 +66,7 @@ public class NodeRecordV5 implements NodeRecord { private BytesValue signature; private NodeRecordV5( - Bytes33 publicKey, + BytesValue publicKey, Bytes4 ipV4address, Integer tcpPort, Integer udpPort, @@ -94,7 +93,7 @@ public NodeRecordV5(BytesValue bytes) { } public static NodeRecordV5 fromValues( - Bytes33 publicKey, + BytesValue publicKey, Bytes4 ipV4address, Integer tcpPort, Integer udpPort, @@ -123,11 +122,11 @@ public IdentityScheme getIdentityScheme() { return identityScheme; } - public Bytes33 getPublicKey() { + public BytesValue getPublicKey() { return publicKey; } - public void setPublicKey(Bytes33 publicKey) { + public void setPublicKey(BytesValue publicKey) { this.publicKey = publicKey; } @@ -320,7 +319,7 @@ public static class Builder { } private Bytes4 ipV4Address; - private Bytes33 secp256k1; + private BytesValue secp256k1; private Integer tcpPort; private Integer udpPort; private UInt64 seqNumber; @@ -348,7 +347,7 @@ public Builder withUdpPort(Integer port) { } public Builder withSecp256k1(BytesValue bytes) { - this.secp256k1 = Bytes33.wrap(bytes, 0); + this.secp256k1 = bytes; return this; } diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/NodeRecordTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/NodeRecordTest.java index 41e03fe0a..fd9f3c697 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/NodeRecordTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/NodeRecordTest.java @@ -4,7 +4,6 @@ import org.ethereum.beacon.discovery.enr.NodeRecordV4; import org.ethereum.beacon.discovery.enr.NodeRecordV5; import org.junit.Test; -import tech.pegasys.artemis.util.bytes.Bytes33; import tech.pegasys.artemis.util.bytes.Bytes4; import tech.pegasys.artemis.util.bytes.Bytes96; import tech.pegasys.artemis.util.bytes.BytesValue; @@ -29,8 +28,9 @@ public void testLocalhostV4() throws Exception { final Integer expectedUdpPort = 30303; final Integer expectedTcpPort = null; final UInt64 expectedSeqNumber = UInt64.valueOf(1); - final Bytes33 expectedPublicKey = - Bytes33.fromHexString("03ca634cae0d49acb401d8a4c6b6fe8c55b70d115bf400769cc1400f3258cd3138"); + final BytesValue expectedPublicKey = + BytesValue.fromHexString( + "03ca634cae0d49acb401d8a4c6b6fe8c55b70d115bf400769cc1400f3258cd3138"); final BytesValue expectedSignature = BytesValue.fromHexString( "7098ad865b00a582051940cb9cf36836572411a47278783077011599ed5cd16b76f2635f4e234738f30813a89eb9137e3e3df5266e3a1f11df72ecf1145ccb9c"); @@ -72,8 +72,9 @@ public void testLocalhostV5() throws Exception { final Integer expectedUdpPort = 30303; final Integer expectedTcpPort = null; final UInt64 expectedSeqNumber = UInt64.valueOf(1); - final Bytes33 expectedPublicKey = - Bytes33.fromHexString("03ca634cae0d49acb401d8a4c6b6fe8c55b70d115bf400769cc1400f3258cd3138"); + final BytesValue expectedPublicKey = + BytesValue.fromHexString( + "03ca634cae0d49acb401d8a4c6b6fe8c55b70d115bf400769cc1400f3258cd3138"); final BytesValue expectedSignature = BytesValue.fromHexString( "7098ad865b00a582051940cb9cf36836572411a47278783077011599ed5cd16b76f2635f4e234738f30813a89eb9137e3e3df5266e3a1f11df72ecf1145ccb9c"); diff --git a/types/src/main/java/tech/pegasys/artemis/util/bytes/ArrayWrappingBytes33.java b/types/src/main/java/tech/pegasys/artemis/util/bytes/ArrayWrappingBytes33.java deleted file mode 100644 index ce49d34ff..000000000 --- a/types/src/main/java/tech/pegasys/artemis/util/bytes/ArrayWrappingBytes33.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright 2018 ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -package tech.pegasys.artemis.util.bytes; - -import static com.google.common.base.Preconditions.checkArgument; - -/** - * An implementation of {@link Bytes33} backed by a byte array ({@code byte[]}). - */ -class ArrayWrappingBytes33 extends ArrayWrappingBytesValue implements Bytes33 { - - ArrayWrappingBytes33(byte[] bytes) { - this(checkLength(bytes), 0); - } - - ArrayWrappingBytes33(byte[] bytes, int offset) { - super(checkLength(bytes, offset), offset, SIZE); - } - - // Ensures a proper error message. - private static byte[] checkLength(byte[] bytes) { - checkArgument(bytes.length == SIZE, "Expected %s bytes but got %s", SIZE, bytes.length); - return bytes; - } - - // Ensures a proper error message. - private static byte[] checkLength(byte[] bytes, int offset) { - checkArgument(bytes.length - offset >= SIZE, - "Expected at least %s bytes from offset %s but got only %s", SIZE, offset, - bytes.length - offset); - return bytes; - } - - @Override - public Bytes33 copy() { - // Because MutableArrayWrappingBytesValue overrides this, we know we are immutable. We may - // retain more than necessary however. - if (offset == 0 && length == bytes.length) - return this; - - return new ArrayWrappingBytes33(arrayCopy()); - } - - @Override - public MutableBytes33 mutableCopy() { - return new MutableArrayWrappingBytes33(arrayCopy()); - } -} diff --git a/types/src/main/java/tech/pegasys/artemis/util/bytes/Bytes33.java b/types/src/main/java/tech/pegasys/artemis/util/bytes/Bytes33.java deleted file mode 100644 index e99288094..000000000 --- a/types/src/main/java/tech/pegasys/artemis/util/bytes/Bytes33.java +++ /dev/null @@ -1,183 +0,0 @@ -/* - * Copyright 2018 ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -package tech.pegasys.artemis.util.bytes; - -import java.util.Random; - -import static com.google.common.base.Preconditions.checkArgument; - -/** - * A {@link BytesValue} that is guaranteed to contain exactly 33 bytes. - */ -public interface Bytes33 extends BytesValue { - int SIZE = 33; - Bytes33 ZERO = wrap(new byte[33]); - - /** - * Wraps the provided byte array, which must be of length 33, as a {@link Bytes33}. - * - *

Note that value is not copied, only wrapped, and thus any future update to {@code value} - * will be reflected in the returned value. - * - * @param bytes The bytes to wrap. - * @return A {@link Bytes33} wrapping {@code value}. - * @throws IllegalArgumentException if {@code value.length != 33}. - */ - static Bytes33 wrap(byte[] bytes) { - checkArgument(bytes.length == SIZE, "Expected %s bytes but got %s", SIZE, bytes.length); - return wrap(bytes, 0); - } - - /** - * Wraps a slice/sub-part of the provided array as a {@link Bytes33}. - * - *

Note that value is not copied, only wrapped, and thus any future update to {@code value} - * within the wrapped parts will be reflected in the returned value. - * - * @param bytes The bytes to wrap. - * @param offset The index (inclusive) in {@code value} of the first byte exposed by the returned - * value. In other words, you will have {@code wrap(value, i).get(0) == value[i]}. - * @return A {@link Bytes33} that exposes the bytes of {@code value} from {@code offset} - * (inclusive) to {@code offset + 33} (exclusive). - * @throws IndexOutOfBoundsException if {@code offset < 0 || (value.length > 0 && offset >= - * value.length)}. - * @throws IllegalArgumentException if {@code length < 0 || offset + 33 > value.length}. - */ - static Bytes33 wrap(byte[] bytes, int offset) { - return new ArrayWrappingBytes33(bytes, offset); - } - - /** - * Wraps a slice/sub-part of the provided value as a {@link Bytes33}. - * - *

Note that value is not copied, only wrapped, and thus any future update to {@code value} - * within the wrapped parts will be reflected in the returned value. - * - * @param bytes The bytes to wrap. - * @param offset The index (inclusive) in {@code value} of the first byte exposed by the returned - * value. In other words, you will have {@code wrap(value, i).get(0) == value.get(i)}. - * @return A {@link Bytes33} that exposes the bytes of {@code value} from {@code offset} - * (inclusive) to {@code offset + 33} (exclusive). - * @throws IndexOutOfBoundsException if {@code offset < 0 || (value.size() > 0 && offset >= - * value.size())}. - * @throws IllegalArgumentException if {@code length < 0 || offset + 33 > value.size()}. - */ - static Bytes33 wrap(BytesValue bytes, int offset) { - BytesValue slice = bytes.slice(offset, Bytes33.SIZE); - return slice instanceof Bytes33 ? (Bytes33) slice : new WrappingBytes33(slice); - } - - /** - * Left pad a {@link BytesValue} with zero bytes to create a {@link Bytes33} - * - * @param value The bytes value pad. - * @return A {@link Bytes33} that exposes the left-padded bytes of {@code value}. - * @throws IllegalArgumentException if {@code value.size() > 33}. - */ - static Bytes33 leftPad(BytesValue value) { - checkArgument( - value.size() <= SIZE, "Expected at most %s bytes but got only %s", SIZE, value.size()); - - MutableBytes33 bytes = MutableBytes33.create(); - value.copyTo(bytes, SIZE - value.size()); - return bytes; - } - - /** - * Right pad a {@link BytesValue} with zero bytes to create a {@link Bytes33} - * - * @param value The bytes value pad. - * @return A {@link Bytes33} that exposes the right-padded bytes of {@code value}. - * @throws IllegalArgumentException if {@code value.size() > 33}. - */ - static Bytes33 rightPad(BytesValue value) { - checkArgument( - value.size() <= SIZE, "Expected at most %s bytes but got only %s", SIZE, value.size()); - - MutableBytes33 bytes = MutableBytes33.create(); - value.copyTo(bytes, 0); - return bytes; - } - - /** - * Parse an hexadecimal string into a {@link Bytes33}. - * - *

This method is lenient in that {@code str} may of an odd length, in which case it will - * behave exactly as if it had an additional 0 in front. - * - * @param str The hexadecimal string to parse, which may or may not start with "0x". That - * representation may contain less than 33 bytes, in which case the result is left padded with - * zeros (see {@link #fromHexStringStrict} if this is not what you want). - * @return The value corresponding to {@code str}. - * @throws IllegalArgumentException if {@code str} does not correspond to valid hexadecimal - * representation or contains more than 33 bytes. - */ - static Bytes33 fromHexStringLenient(String str) { - return wrap(BytesValues.fromRawHexString(str, SIZE, true)); - } - - /** - * Parse an hexadecimal string into a {@link Bytes33}. - * - *

This method is strict in that {@code str} must of an even length. - * - * @param str The hexadecimal string to parse, which may or may not start with "0x". That - * representation may contain less than 33 bytes, in which case the result is left padded with - * zeros (see {@link #fromHexStringStrict} if this is not what you want). - * @return The value corresponding to {@code str}. - * @throws IllegalArgumentException if {@code str} does not correspond to valid hexadecimal - * representation, is of an odd length, or contains more than 33 bytes. - */ - static Bytes33 fromHexString(String str) { - return wrap(BytesValues.fromRawHexString(str, SIZE, false)); - } - - /** - * Parse an hexadecimal string into a {@link Bytes33}. - * - *

This method is extra strict in that {@code str} must of an even length and the provided - * representation must have exactly 33 bytes. - * - * @param str The hexadecimal string to parse, which may or may not start with "0x". - * @return The value corresponding to {@code str}. - * @throws IllegalArgumentException if {@code str} does not correspond to valid hexadecimal - * representation, is of an odd length or does not contain exactly 33 bytes. - */ - static Bytes33 fromHexStringStrict(String str) { - return wrap(BytesValues.fromRawHexString(str, -1, false)); - } - - /** - * Constructs a randomly generated value. - * - * @param rnd random number generator. - * @return random value. - */ - static Bytes33 random(Random rnd) { - byte[] randomBytes = new byte[SIZE]; - rnd.nextBytes(randomBytes); - return wrap(randomBytes); - } - - @Override - default int size() { - return SIZE; - } - - @Override - Bytes33 copy(); - - @Override - MutableBytes33 mutableCopy(); -} diff --git a/types/src/main/java/tech/pegasys/artemis/util/bytes/MutableArrayWrappingBytes33.java b/types/src/main/java/tech/pegasys/artemis/util/bytes/MutableArrayWrappingBytes33.java deleted file mode 100644 index c133cc1d2..000000000 --- a/types/src/main/java/tech/pegasys/artemis/util/bytes/MutableArrayWrappingBytes33.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2018 ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -package tech.pegasys.artemis.util.bytes; - -/** - * An implementation of {@link MutableBytes33} backed by a byte array ({@code byte[]}). - */ -class MutableArrayWrappingBytes33 extends MutableArrayWrappingBytesValue implements MutableBytes33 { - - MutableArrayWrappingBytes33(byte[] bytes) { - this(bytes, 0); - } - - MutableArrayWrappingBytes33(byte[] bytes, int offset) { - super(bytes, offset, SIZE); - } - - @Override - public Bytes33 copy() { - // We *must* override this method because ArrayWrappingBytes33 assumes that it is the case. - return new ArrayWrappingBytes33(arrayCopy()); - } - - @Override - public MutableBytes33 mutableCopy() { - return new MutableArrayWrappingBytes33(arrayCopy()); - } -} diff --git a/types/src/main/java/tech/pegasys/artemis/util/bytes/MutableBytes33.java b/types/src/main/java/tech/pegasys/artemis/util/bytes/MutableBytes33.java deleted file mode 100644 index 7d536fde9..000000000 --- a/types/src/main/java/tech/pegasys/artemis/util/bytes/MutableBytes33.java +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Copyright 2018 ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -package tech.pegasys.artemis.util.bytes; - -import java.security.MessageDigest; - -import static com.google.common.base.Preconditions.checkArgument; - -/** - * A mutable {@link Bytes33}, that is a mutable {@link BytesValue} of exactly 33 bytes. - */ -public interface MutableBytes33 extends MutableBytesValue, Bytes33 { - - /** - * Wraps a 33 bytes array as a mutable 33 bytes value. - * - *

- * This method behave exactly as {@link Bytes33#wrap(byte[])} except that the result is a mutable. - * - * @param value The value to wrap. - * @return A {@link MutableBytes33} wrapping {@code value}. - * @throws IllegalArgumentException if {@code value.length != 33}. - */ - static MutableBytes33 wrap(byte[] value) { - return new MutableArrayWrappingBytes33(value); - } - - /** - * Creates a new mutable 33 bytes value. - * - * @return A newly allocated {@link MutableBytesValue}. - */ - static MutableBytes33 create() { - return new MutableArrayWrappingBytes33(new byte[SIZE]); - } - - /** - * Wraps an existing {@link MutableBytesValue} of size 33 as a mutable 33 bytes value. - * - *

- * This method does no copy the provided bytes and so any mutation on {@code value} will also be - * reflected in the value returned by this method. If a copy is desirable, this can be simply - * achieved with calling {@link BytesValue#copyTo(MutableBytesValue)} with a newly created - * {@link MutableBytes33} as destination to the copy. - * - * @param value The value to wrap. - * @return A {@link MutableBytes33} wrapping {@code value}. - * @throws IllegalArgumentException if {@code value.size() != 33}. - */ - static MutableBytes33 wrap(MutableBytesValue value) { - checkArgument(value.size() == SIZE, "Expected %s bytes but got %s", SIZE, value.size()); - return new MutableBytes33() { - @Override - public void set(int i, byte b) { - value.set(i, b); - } - - @Override - public MutableBytesValue mutableSlice(int i, int length) { - return value.mutableSlice(i, length); - } - - @Override - public byte get(int i) { - return value.get(i); - } - - @Override - public BytesValue slice(int index) { - return value.slice(index); - } - - @Override - public BytesValue slice(int index, int length) { - return value.slice(index, length); - } - - @Override - public Bytes33 copy() { - return Bytes33.wrap(value.extractArray()); - } - - @Override - public MutableBytes33 mutableCopy() { - return MutableBytes33.wrap(value.extractArray()); - } - - @Override - public void copyTo(MutableBytesValue destination) { - value.copyTo(destination); - } - - @Override - public void copyTo(MutableBytesValue destination, int destinationOffset) { - value.copyTo(destination, destinationOffset); - } - - @Override - public int commonPrefixLength(BytesValue other) { - return value.commonPrefixLength(other); - } - - @Override - public BytesValue commonPrefix(BytesValue other) { - return value.commonPrefix(other); - } - - @Override - public void update(MessageDigest digest) { - value.update(digest); - } - - @Override - public boolean isZero() { - return value.isZero(); - } - - @Override - public boolean equals(Object other) { - return value.equals(other); - } - - @Override - public int hashCode() { - return value.hashCode(); - } - - @Override - public String toString() { - return value.toString(); - } - }; - } -} diff --git a/types/src/main/java/tech/pegasys/artemis/util/bytes/WrappingBytes33.java b/types/src/main/java/tech/pegasys/artemis/util/bytes/WrappingBytes33.java deleted file mode 100644 index 94f1a3199..000000000 --- a/types/src/main/java/tech/pegasys/artemis/util/bytes/WrappingBytes33.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright 2018 ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -package tech.pegasys.artemis.util.bytes; - -import static com.google.common.base.Preconditions.checkArgument; - -/** - * A simple class to wrap another {@link BytesValue} of exactly 33 bytes as a {@link Bytes33}. - */ -class WrappingBytes33 extends AbstractBytesValue implements Bytes33 { - - private final BytesValue value; - - WrappingBytes33(BytesValue value) { - checkArgument(value.size() == SIZE, "Expected value to be %s bytes, but is %s bytes", SIZE, - value.size()); - this.value = value; - } - - @Override - public byte get(int i) { - return value.get(i); - } - - @Override - public BytesValue slice(int index, int length) { - return value.slice(index, length); - } - - @Override - public MutableBytes33 mutableCopy() { - MutableBytes33 copy = MutableBytes33.create(); - value.copyTo(copy); - return copy; - } - - @Override - public Bytes33 copy() { - return mutableCopy(); - } - - @Override - public byte[] getArrayUnsafe() { - return value.getArrayUnsafe(); - } - - @Override - public int size() { - return value.size(); - } -} From 930e994418ea21152cf2c584c55396e9280cf5ba Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Wed, 25 Sep 2019 22:08:16 +0300 Subject: [PATCH 19/77] discovery: make all node record fields accessible. Remove ssz usage for serialization --- .../beacon/discovery/enr/NodeRecord.java | 62 ++----- .../beacon/discovery/enr/NodeRecordInfo.java | 40 +++- .../beacon/discovery/enr/NodeRecordV4.java | 172 +++++++++-------- .../beacon/discovery/enr/NodeRecordV5.java | 174 +++++++++--------- .../beacon/discovery/enr/NodeStatus.java | 53 ------ .../beacon/discovery/storage/NodeIndex.java | 32 +++- .../storage/NodeSerializerFactory.java | 31 ++++ .../discovery/storage/NodeTableStorage.java | 3 + .../discovery/DiscoveryNetworkTest.java | 9 +- .../discovery/DiscoveryNoNetworkTest.java | 9 +- .../discovery/storage/NodeTableTest.java | 15 +- 11 files changed, 290 insertions(+), 310 deletions(-) create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeSerializerFactory.java diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecord.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecord.java index a1b60df40..d63598bca 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecord.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecord.java @@ -1,10 +1,6 @@ package org.ethereum.beacon.discovery.enr; import org.ethereum.beacon.discovery.IdentityScheme; -import org.ethereum.beacon.ssz.access.SSZField; -import org.ethereum.beacon.ssz.access.list.BytesValueAccessor; -import org.ethereum.beacon.ssz.annotation.SSZSerializable; -import org.ethereum.beacon.ssz.type.SSZType; import org.web3j.rlp.RlpDecoder; import org.web3j.rlp.RlpList; import org.web3j.rlp.RlpString; @@ -15,14 +11,28 @@ import java.util.Base64; import java.util.List; +import java.util.Map; /** * Ethereum Node Record * *

Node record as described in EIP-778 */ -@SSZSerializable(listAccessor = NodeRecord.NodeRecordAccessor.class) public interface NodeRecord { + // Compressed secp256k1 public key, 33 bytes + String FIELD_PKEY_SECP256K1 = "secp256k1"; + // IPv4 address + String FIELD_IP_V4 = "ip"; + // TCP port, integer + String FIELD_TCP_V4 = "tcp"; + // UDP port, integer + String FIELD_UDP_V4 = "udp"; + // IPv6 address + String FIELD_IP_V6 = "ip6"; + // IPv6-specific TCP port + String FIELD_TCP_V6 = "tcp6"; + // IPv6-specific UDP port + String FIELD_UDP_V6 = "udp6"; static NodeRecord fromBase64(String enrBase64) { return fromBytes(Base64.getUrlDecoder().decode(enrBase64)); @@ -93,41 +103,9 @@ default String asBase64() { Bytes32 getNodeId(); - class NodeRecordAccessor extends BytesValueAccessor { - @Override - public int getChildrenCount(Object value) { - return ((NodeRecord) value).serialize().size(); - } - - @Override - public Object getChildValue(Object value, int idx) { - return ((NodeRecord) value).serialize().get(idx); - } - - @Override - public boolean isSupported(SSZField field) { - return NodeRecord.class.isAssignableFrom(field.getRawClass()); - } - - @Override - public CompositeInstanceAccessor getInstanceAccessor(SSZField compositeDescriptor) { - return this; - } - - @Override - public ListInstanceBuilder createInstanceBuilder(SSZType sszType) { - return new SimpleInstanceBuilder() { - @Override - protected Object buildImpl(List children) { - byte[] vals = new byte[children.size()]; - for (int i = 0; i < children.size(); i++) { - vals[i] = ((Number) children.get(i)).byteValue(); - } - BytesValue value = BytesValue.wrap(vals); - - return NodeRecord.fromBytes(value); - } - }; - } - } + /** + * All optional fields, set varies by identity scheme. `id`, identity scheme is not optional. Most + * common field names are in constants: {@link #FIELD_IP_V4} etc. (FIELD_*) + */ + Map getFields(); } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordInfo.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordInfo.java index 3147e0690..eb789b03f 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordInfo.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordInfo.java @@ -1,20 +1,24 @@ package org.ethereum.beacon.discovery.enr; import com.google.common.base.Objects; -import org.ethereum.beacon.ssz.annotation.SSZ; -import org.ethereum.beacon.ssz.annotation.SSZSerializable; +import org.web3j.rlp.RlpDecoder; +import org.web3j.rlp.RlpEncoder; +import org.web3j.rlp.RlpList; +import org.web3j.rlp.RlpString; +import org.web3j.rlp.RlpType; +import tech.pegasys.artemis.util.bytes.BytesValue; + +import java.util.ArrayList; +import java.util.List; /** * Container for {@link NodeRecord}. Also saves all necessary data about presence of this node and * last test of its availability */ -@SSZSerializable public class NodeRecordInfo { - @SSZ private final NodeRecord node; - @SSZ private final Long lastRetry; - @SSZ private final NodeStatus status; - - @SSZ(type = "uint8") + private final NodeRecord node; + private final Long lastRetry; + private final NodeStatus status; private final Integer retry; public NodeRecordInfo(NodeRecord node, Long lastRetry, NodeStatus status, Integer retry) { @@ -28,6 +32,26 @@ public static NodeRecordInfo createDefault(NodeRecord nodeRecord) { return new NodeRecordInfo(nodeRecord, -1L, NodeStatus.ACTIVE, 0); } + public static NodeRecordInfo fromRlpBytes(BytesValue bytes) { + RlpList internalList = (RlpList) RlpDecoder.decode(bytes.extractArray()).getValues().get(0); + return new NodeRecordInfo( + NodeRecord.fromBytes(((RlpString)internalList.getValues().get(0)).getBytes()), + ((RlpString) internalList.getValues().get(1)).asPositiveBigInteger().longValue(), + NodeStatus.fromNumber(((RlpString) internalList.getValues().get(2)).getBytes()[0]), + ((RlpString) internalList.getValues().get(1)).asPositiveBigInteger().intValue() + ); + } + + public BytesValue toRlpBytes() { + List values = new ArrayList<>(); + values.add(RlpString.create(getNode().serialize().extractArray())); + values.add(RlpString.create(getLastRetry())); + values.add(RlpString.create(getStatus().byteCode())); + values.add(RlpString.create(getRetry())); + byte[] bytes = RlpEncoder.encode(new RlpList(values)); + return BytesValue.wrap(bytes); + } + public NodeRecordV5 getNode() { return (NodeRecordV5) node; } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV4.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV4.java index c4f795d8d..e09e129e7 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV4.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV4.java @@ -15,6 +15,7 @@ import tech.pegasys.artemis.util.bytes.BytesValue; import tech.pegasys.artemis.util.uint.UInt64; +import java.math.BigInteger; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -42,25 +43,11 @@ public class NodeRecordV4 implements NodeRecord { return builder.build(); }; - // secp256k1 compressed secp256k1 public key, 33 bytes - private BytesValue publicKey; - // ip IPv4 address, 4 bytes - private Bytes4 ipV4address; - // tcp TCP port, big endian integer - private Integer tcpPort; - // udp UDP port, big endian integer - private Integer udpPort; - // ip6 IPv6 address, 16 bytes - private Bytes16 ipV6address; - // tcp6 IPv6-specific TCP port, big endian integer - private Integer tcpV6Port; - // udp6 IPv6-specific UDP port, big endian integer - private Integer udpV6Port; - // seq The sequence number, a 64-bit unsigned integer. Nodes should increase the number whenever - // the record changes and republish the record. private UInt64 seq; // Signature private BytesValue signature; + // optional fields + private Map fields = new HashMap<>(); private NodeRecordV4( BytesValue publicKey, @@ -72,13 +59,13 @@ private NodeRecordV4( Integer udpV6Port, UInt64 seq, BytesValue signature) { - this.publicKey = publicKey; - this.ipV4address = ipV4address; - this.tcpPort = tcpPort; - this.udpPort = udpPort; - this.ipV6address = ipV6address; - this.tcpV6Port = tcpV6Port; - this.udpV6Port = udpV6Port; + fields.put(FIELD_PKEY_SECP256K1, publicKey); + fields.put(FIELD_IP_V4, ipV4address); + fields.put(FIELD_TCP_V4, tcpPort); + fields.put(FIELD_UDP_V4, udpPort); + fields.put(FIELD_IP_V6, ipV6address); + fields.put(FIELD_TCP_V6, tcpV6Port); + fields.put(FIELD_UDP_V6, udpV6Port); this.seq = seq; this.signature = signature; } @@ -116,59 +103,61 @@ public IdentityScheme getIdentityScheme() { } public BytesValue getPublicKey() { - return publicKey; + return fields.containsKey(FIELD_PKEY_SECP256K1) + ? (BytesValue) fields.get(FIELD_PKEY_SECP256K1) + : null; } public void setPublicKey(BytesValue publicKey) { - this.publicKey = publicKey; + fields.put(FIELD_PKEY_SECP256K1, publicKey); } public Bytes4 getIpV4address() { - return ipV4address; + return fields.containsKey(FIELD_IP_V4) ? (Bytes4) fields.get(FIELD_IP_V4) : null; } public void setIpV4address(Bytes4 ipV4address) { - this.ipV4address = ipV4address; + fields.put(FIELD_IP_V4, ipV4address); } public Integer getTcpPort() { - return tcpPort; + return fields.containsKey(FIELD_TCP_V4) ? (Integer) fields.get(FIELD_TCP_V4) : null; } public void setTcpPort(Integer tcpPort) { - this.tcpPort = tcpPort; + fields.put(FIELD_TCP_V4, tcpPort); } public Integer getUdpPort() { - return udpPort; + return fields.containsKey(FIELD_UDP_V4) ? (Integer) fields.get(FIELD_UDP_V4) : null; } public void setUdpPort(Integer udpPort) { - this.udpPort = udpPort; + fields.put(FIELD_UDP_V4, udpPort); } public Bytes16 getIpV6address() { - return ipV6address; + return fields.containsKey(FIELD_IP_V6) ? (Bytes16) fields.get(FIELD_IP_V6) : null; } public void setIpV6address(Bytes16 ipV6address) { - this.ipV6address = ipV6address; + fields.put(FIELD_IP_V6, ipV6address); } public Integer getTcpV6Port() { - return tcpV6Port; + return fields.containsKey(FIELD_TCP_V6) ? (Integer) fields.get(FIELD_TCP_V6) : null; } public void setTcpV6Port(Integer tcpV6Port) { - this.tcpV6Port = tcpV6Port; + fields.put(FIELD_TCP_V6, tcpV6Port); } public Integer getUdpV6Port() { - return udpV6Port; + return fields.containsKey(FIELD_UDP_V6) ? (Integer) fields.get(FIELD_UDP_V6) : null; } public void setUdpV6Port(Integer udpV6Port) { - this.udpV6Port = udpV6Port; + fields.put(FIELD_UDP_V6, udpV6Port); } public UInt64 getSeq() { @@ -188,34 +177,24 @@ public void setSignature(BytesValue signature) { this.signature = signature; } + @Override + public Map getFields() { + return fields; + } + @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; NodeRecordV4 that = (NodeRecordV4) o; - return Objects.equal(publicKey, that.publicKey) - && Objects.equal(ipV4address, that.ipV4address) - && Objects.equal(tcpPort, that.tcpPort) - && Objects.equal(udpPort, that.udpPort) - && Objects.equal(ipV6address, that.ipV6address) - && Objects.equal(tcpV6Port, that.tcpV6Port) - && Objects.equal(udpV6Port, that.udpV6Port) - && Objects.equal(seq, that.seq) - && Objects.equal(signature, that.signature); + return Objects.equal(seq, that.seq) + && Objects.equal(signature, that.signature) + && Objects.equal(fields, that.fields); } @Override public int hashCode() { - return Objects.hashCode( - publicKey, - ipV4address, - tcpPort, - udpPort, - ipV6address, - tcpV6Port, - udpV6Port, - seq, - signature); + return Objects.hashCode(seq, signature, fields); } @Override @@ -231,40 +210,46 @@ public BytesValue serialize() { values.add(RlpString.create(getSeq().toBI())); values.add(RlpString.create("id")); values.add(RlpString.create(getIdentityScheme().stringName())); - if (getPublicKey() != null) { - values.add(RlpString.create("secp256k1")); - values.add(RlpString.create(getPublicKey().extractArray())); - } - if (getIpV4address() != null) { - values.add(RlpString.create("ip")); - values.add(RlpString.create(getIpV4address().extractArray())); - } - if (getTcpPort() != null) { - values.add(RlpString.create("tcp")); - values.add(RlpString.create(getTcpPort())); - } - if (getUdpPort() != null) { - values.add(RlpString.create("udp")); - values.add(RlpString.create(getUdpPort())); - } - if (getIpV6address() != null) { - values.add(RlpString.create("ip6")); - values.add(RlpString.create(getIpV6address().extractArray())); - } - if (getTcpV6Port() != null) { - values.add(RlpString.create("tcp6")); - values.add(RlpString.create(getTcpV6Port())); - } - if (getUdpV6Port() != null) { - values.add(RlpString.create("udp6")); - values.add(RlpString.create(getUdpV6Port())); + for (Map.Entry keyPair : fields.entrySet()) { + if (keyPair.getValue() == null) { + continue; + } + values.add(RlpString.create(keyPair.getKey())); + if (keyPair.getValue() instanceof BytesValue) { + values.add(fromBytes((BytesValue) keyPair.getValue())); + } else if (keyPair.getValue() instanceof Number) { + values.add(fromNumber((Number) keyPair.getValue())); + } else if (keyPair.getValue() == null) { + values.add(RlpString.create(new byte[0])); + } else { + throw new RuntimeException( + String.format( + "Couldn't serialize node record field %s with value %s: no serializer found.", + keyPair.getKey(), keyPair.getValue())); + } } - byte[] bytes = RlpEncoder.encode(new RlpList(values)); assert bytes.length <= 300; return BytesValue.wrap(bytes); } + private RlpString fromNumber(Number number) { + if (number instanceof BigInteger) { + return RlpString.create((BigInteger) number); + } else if (number instanceof Long) { + return RlpString.create((Long) number); + } else if (number instanceof Integer) { + return RlpString.create((Integer) number); + } else { + throw new RuntimeException( + String.format("Couldn't serialize number %s : no serializer found.", number)); + } + } + + private RlpString fromBytes(BytesValue bytes) { + return RlpString.create(bytes.extractArray()); + } + @Override public Bytes32 getNodeId() { return Hashes.sha256(getPublicKey()); @@ -274,11 +259,11 @@ public Bytes32 getNodeId() { public String toString() { return "NodeRecordV4{" + "publicKey=" - + publicKey + + fields.get(FIELD_PKEY_SECP256K1) + ", ipV4address=" - + ipV4address + + fields.get(FIELD_IP_V4) + ", udpPort=" - + udpPort + + fields.get(FIELD_UDP_V4) + '}'; } @@ -288,24 +273,31 @@ public static class Builder { static { fieldFillersV4.put( - "ip", + FIELD_IP_V4, objects -> objects .getValue0() .withIpV4Address(BytesValue.wrap(((RlpString) objects.getValue1()).getBytes()))); fieldFillersV4.put( - "secp256k1", + FIELD_PKEY_SECP256K1, objects -> objects .getValue0() .withSecp256k1(BytesValue.wrap(((RlpString) objects.getValue1()).getBytes()))); fieldFillersV4.put( - "udp", + FIELD_UDP_V4, objects -> objects .getValue0() .withUdpPort( ((RlpString) objects.getValue1()).asPositiveBigInteger().intValue())); + fieldFillersV4.put( + FIELD_TCP_V4, + objects -> + objects + .getValue0() + .withTcpPort( + ((RlpString) objects.getValue1()).asPositiveBigInteger().intValue())); } private Bytes4 ipV4Address; diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV5.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV5.java index 7b853a39f..2799bc944 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV5.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV5.java @@ -16,6 +16,7 @@ import tech.pegasys.artemis.util.bytes.BytesValue; import tech.pegasys.artemis.util.uint.UInt64; +import java.math.BigInteger; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -45,25 +46,11 @@ public class NodeRecordV5 implements NodeRecord { }; public static Function NODE_ID_FUNCTION = nodeRecordV5 -> Hashes.sha256(nodeRecordV5.getPublicKey()); - // secp256k1 compressed secp256k1 public key, 33 bytes - private BytesValue publicKey; - // ip IPv4 address, 4 bytes - private Bytes4 ipV4address; - // tcp TCP port, big endian integer - private Integer tcpPort; - // udp UDP port, big endian integer - private Integer udpPort; - // ip6 IPv6 address, 16 bytes - private Bytes16 ipV6address; - // tcp6 IPv6-specific TCP port, big endian integer - private Integer tcpV6Port; - // udp6 IPv6-specific UDP port, big endian integer - private Integer udpV6Port; - // seq The sequence number, a 64-bit unsigned integer. Nodes should increase the number whenever - // the record changes and republish the record. private UInt64 seq; // Signature private BytesValue signature; + // optional fields + private Map fields = new HashMap<>(); private NodeRecordV5( BytesValue publicKey, @@ -75,13 +62,13 @@ private NodeRecordV5( Integer udpV6Port, UInt64 seq, BytesValue signature) { - this.publicKey = publicKey; - this.ipV4address = ipV4address; - this.tcpPort = tcpPort; - this.udpPort = udpPort; - this.ipV6address = ipV6address; - this.tcpV6Port = tcpV6Port; - this.udpV6Port = udpV6Port; + fields.put(FIELD_PKEY_SECP256K1, publicKey); + fields.put(FIELD_IP_V4, ipV4address); + fields.put(FIELD_TCP_V4, tcpPort); + fields.put(FIELD_UDP_V4, udpPort); + fields.put(FIELD_IP_V6, ipV6address); + fields.put(FIELD_TCP_V6, tcpV6Port); + fields.put(FIELD_UDP_V6, udpV6Port); this.seq = seq; this.signature = signature; } @@ -123,62 +110,63 @@ public IdentityScheme getIdentityScheme() { } public BytesValue getPublicKey() { - return publicKey; + return fields.containsKey(FIELD_PKEY_SECP256K1) + ? (BytesValue) fields.get(FIELD_PKEY_SECP256K1) + : null; } public void setPublicKey(BytesValue publicKey) { - this.publicKey = publicKey; + fields.put(FIELD_PKEY_SECP256K1, publicKey); } public Bytes4 getIpV4address() { - return ipV4address; + return fields.containsKey(FIELD_IP_V4) ? (Bytes4) fields.get(FIELD_IP_V4) : null; } public void setIpV4address(Bytes4 ipV4address) { - this.ipV4address = ipV4address; + fields.put(FIELD_IP_V4, ipV4address); } public Integer getTcpPort() { - return tcpPort; + return fields.containsKey(FIELD_TCP_V4) ? (Integer) fields.get(FIELD_TCP_V4) : null; } public void setTcpPort(Integer tcpPort) { - this.tcpPort = tcpPort; + fields.put(FIELD_TCP_V4, tcpPort); } public Integer getUdpPort() { - return udpPort; + return fields.containsKey(FIELD_UDP_V4) ? (Integer) fields.get(FIELD_UDP_V4) : null; } public void setUdpPort(Integer udpPort) { - this.udpPort = udpPort; + fields.put(FIELD_UDP_V4, udpPort); } public Bytes16 getIpV6address() { - return ipV6address; + return fields.containsKey(FIELD_IP_V6) ? (Bytes16) fields.get(FIELD_IP_V6) : null; } public void setIpV6address(Bytes16 ipV6address) { - this.ipV6address = ipV6address; + fields.put(FIELD_IP_V6, ipV6address); } public Integer getTcpV6Port() { - return tcpV6Port; + return fields.containsKey(FIELD_TCP_V6) ? (Integer) fields.get(FIELD_TCP_V6) : null; } public void setTcpV6Port(Integer tcpV6Port) { - this.tcpV6Port = tcpV6Port; + fields.put(FIELD_TCP_V6, tcpV6Port); } public Integer getUdpV6Port() { - return udpV6Port; + return fields.containsKey(FIELD_UDP_V6) ? (Integer) fields.get(FIELD_UDP_V6) : null; } public void setUdpV6Port(Integer udpV6Port) { - this.udpV6Port = udpV6Port; + fields.put(FIELD_UDP_V6, udpV6Port); } - @Override public UInt64 getSeq() { return seq; } @@ -196,6 +184,11 @@ public void setSignature(BytesValue signature) { this.signature = signature; } + @Override + public Map getFields() { + return fields; + } + /** Uses empty 96 bytes signature when no signature is provided\ */ @Override public BytesValue serialize() { @@ -211,33 +204,23 @@ public BytesValue serialize() { values.add(RlpString.create(getSeq().toBI())); values.add(RlpString.create("id")); values.add(RlpString.create(getIdentityScheme().stringName())); - if (getPublicKey() != null) { - values.add(RlpString.create("secp256k1")); - values.add(RlpString.create(getPublicKey().extractArray())); - } - if (getIpV4address() != null) { - values.add(RlpString.create("ip")); - values.add(RlpString.create(getIpV4address().extractArray())); - } - if (getTcpPort() != null) { - values.add(RlpString.create("tcp")); - values.add(RlpString.create(getTcpPort())); - } - if (getUdpPort() != null) { - values.add(RlpString.create("udp")); - values.add(RlpString.create(getUdpPort())); - } - if (getIpV6address() != null) { - values.add(RlpString.create("ip6")); - values.add(RlpString.create(getIpV6address().extractArray())); - } - if (getTcpV6Port() != null) { - values.add(RlpString.create("tcp6")); - values.add(RlpString.create(getTcpV6Port())); - } - if (getUdpV6Port() != null) { - values.add(RlpString.create("udp6")); - values.add(RlpString.create(getUdpV6Port())); + for (Map.Entry keyPair : fields.entrySet()) { + if (keyPair.getValue() == null) { + continue; + } + values.add(RlpString.create(keyPair.getKey())); + if (keyPair.getValue() instanceof BytesValue) { + values.add(fromBytes((BytesValue) keyPair.getValue())); + } else if (keyPair.getValue() instanceof Number) { + values.add(fromNumber((Number) keyPair.getValue())); + } else if (keyPair.getValue() == null) { + values.add(RlpString.create(new byte[0])); + } else { + throw new RuntimeException( + String.format( + "Couldn't serialize node record field %s with value %s: no serializer found.", + keyPair.getKey(), keyPair.getValue())); + } } byte[] bytes = RlpEncoder.encode(new RlpList(values)); @@ -245,34 +228,36 @@ public BytesValue serialize() { return BytesValue.wrap(bytes); } + private RlpString fromNumber(Number number) { + if (number instanceof BigInteger) { + return RlpString.create((BigInteger) number); + } else if (number instanceof Long) { + return RlpString.create((Long) number); + } else if (number instanceof Integer) { + return RlpString.create((Integer) number); + } else { + throw new RuntimeException( + String.format("Couldn't serialize number %s : no serializer found.", number)); + } + } + + private RlpString fromBytes(BytesValue bytes) { + return RlpString.create(bytes.extractArray()); + } + @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; NodeRecordV5 that = (NodeRecordV5) o; - return Objects.equal(publicKey, that.publicKey) - && Objects.equal(ipV4address, that.ipV4address) - && Objects.equal(tcpPort, that.tcpPort) - && Objects.equal(udpPort, that.udpPort) - && Objects.equal(ipV6address, that.ipV6address) - && Objects.equal(tcpV6Port, that.tcpV6Port) - && Objects.equal(udpV6Port, that.udpV6Port) - && Objects.equal(seq, that.seq) - && Objects.equal(signature, that.signature); + return Objects.equal(seq, that.seq) + && Objects.equal(signature, that.signature) + && Objects.equal(fields, that.fields); } @Override public int hashCode() { - return Objects.hashCode( - publicKey, - ipV4address, - tcpPort, - udpPort, - ipV6address, - tcpV6Port, - udpV6Port, - seq, - signature); + return Objects.hashCode(seq, signature, fields); } @Override @@ -284,11 +269,11 @@ public Bytes32 getNodeId() { public String toString() { return "NodeRecordV5{" + "publicKey=" - + publicKey + + fields.get(FIELD_PKEY_SECP256K1) + ", ipV4address=" - + ipV4address + + fields.get(FIELD_IP_V4) + ", udpPort=" - + udpPort + + fields.get(FIELD_UDP_V4) + '}'; } @@ -298,24 +283,31 @@ public static class Builder { static { fieldFillersV5.put( - "ip", + FIELD_IP_V4, objects -> objects .getValue0() .withIpV4Address(BytesValue.wrap(((RlpString) objects.getValue1()).getBytes()))); fieldFillersV5.put( - "secp256k1", + FIELD_PKEY_SECP256K1, objects -> objects .getValue0() .withSecp256k1(BytesValue.wrap(((RlpString) objects.getValue1()).getBytes()))); fieldFillersV5.put( - "udp", + FIELD_UDP_V4, objects -> objects .getValue0() .withUdpPort( ((RlpString) objects.getValue1()).asPositiveBigInteger().intValue())); + fieldFillersV5.put( + FIELD_TCP_V4, + objects -> + objects + .getValue0() + .withTcpPort( + ((RlpString) objects.getValue1()).asPositiveBigInteger().intValue())); } private Bytes4 ipV4Address; @@ -365,7 +357,7 @@ public Builder withKeyField(String key, RlpString value) { Function, NodeRecordV5.Builder> fieldFiller = fieldFillersV5.get(key); if (fieldFiller == null) { - throw new RuntimeException(String.format("Couldn't find filler for V4 field '%s'", key)); + throw new RuntimeException(String.format("Couldn't find filler for V5 field '%s'", key)); } return fieldFiller.apply(Pair.with(this, value)); } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeStatus.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeStatus.java index a1064bd9d..fb87a4266 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeStatus.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeStatus.java @@ -1,17 +1,8 @@ package org.ethereum.beacon.discovery.enr; -import org.ethereum.beacon.ssz.access.SSZField; -import org.ethereum.beacon.ssz.access.basic.UIntPrimitive; -import org.ethereum.beacon.ssz.annotation.SSZSerializable; -import org.ethereum.beacon.ssz.visitor.SSZReader; - -import java.io.OutputStream; import java.util.HashMap; -import java.util.HashSet; import java.util.Map; -import java.util.Set; -@SSZSerializable(basicAccessor = NodeStatus.NodeStatusAccessor.class) public enum NodeStatus { ACTIVE(0x01), SLEEP(0x02), @@ -38,48 +29,4 @@ public static NodeStatus fromNumber(int i) { public byte byteCode() { return (byte) code; } - - public static class NodeStatusAccessor extends UIntPrimitive { - @Override - public Set getSupportedClasses() { - return new HashSet() { - { - add(NodeStatus.class); - } - }; - } - - @Override - public int getSize(SSZField field) { - return 1; - } - - @Override - public void encode(Object value, SSZField field, OutputStream result) { - NodeStatus nodeStatus = (NodeStatus) value; - SSZField overrided = - new SSZField( - byte.class, - field.getFieldAnnotation(), - "uint", - 8, - field.getName(), - field.getGetter()); - super.encode(nodeStatus.byteCode(), overrided, result); - } - - @Override - public Object decode(SSZField field, SSZReader reader) { - SSZField overrided = - new SSZField( - byte.class, - field.getFieldAnnotation(), - "uint", - 8, - field.getName(), - field.getGetter()); - int code = (int) super.decode(overrided, reader); - return NodeStatus.fromNumber(code); - } - } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeIndex.java b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeIndex.java index 28d08dc14..ff3a39776 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeIndex.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeIndex.java @@ -1,21 +1,36 @@ package org.ethereum.beacon.discovery.storage; -import org.ethereum.beacon.ssz.annotation.SSZ; -import org.ethereum.beacon.ssz.annotation.SSZSerializable; +import org.web3j.rlp.RlpDecoder; +import org.web3j.rlp.RlpEncoder; +import org.web3j.rlp.RlpList; +import org.web3j.rlp.RlpString; +import org.web3j.rlp.RlpType; import tech.pegasys.artemis.ethereum.core.Hash32; +import tech.pegasys.artemis.util.bytes.Bytes32; +import tech.pegasys.artemis.util.bytes.BytesValue; import java.util.ArrayList; import java.util.List; /** Node Index. Stores several node keys. */ -@SSZSerializable public class NodeIndex { - @SSZ private List entries; + private List entries; public NodeIndex() { this.entries = new ArrayList<>(); } + public static NodeIndex fromRlpBytes(BytesValue bytes) { + RlpList internalList = (RlpList) RlpDecoder.decode(bytes.extractArray()).getValues().get(0); + List entries = new ArrayList<>(); + for (RlpType entry : internalList.getValues()) { + entries.add(Hash32.wrap(Bytes32.wrap(((RlpString) entry).getBytes()))); + } + NodeIndex res = new NodeIndex(); + res.setEntries(entries); + return res; + } + public List getEntries() { return entries; } @@ -23,4 +38,13 @@ public List getEntries() { public void setEntries(List entries) { this.entries = entries; } + + public BytesValue toRlpBytes() { + List values = new ArrayList<>(); + for (Hash32 hash32 : getEntries()) { + values.add(RlpString.create(hash32.extractArray())); + } + byte[] bytes = RlpEncoder.encode(new RlpList(values)); + return BytesValue.wrap(bytes); + } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeSerializerFactory.java b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeSerializerFactory.java new file mode 100644 index 000000000..4bfdc72ba --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeSerializerFactory.java @@ -0,0 +1,31 @@ +package org.ethereum.beacon.discovery.storage; + +import org.ethereum.beacon.chain.storage.impl.SerializerFactory; +import org.ethereum.beacon.discovery.enr.NodeRecordInfo; +import tech.pegasys.artemis.util.bytes.BytesValue; + +import java.util.function.Function; + +public class NodeSerializerFactory implements SerializerFactory { + @Override + public Function getDeserializer(Class objectClass) { + if ((!objectClass.equals(NodeRecordInfo.class)) && (!objectClass.equals(NodeIndex.class))) { + throw new RuntimeException(String.format("Type %s is not supported", objectClass)); + } + return bytes -> + objectClass.equals(NodeRecordInfo.class) + ? (T) NodeRecordInfo.fromRlpBytes(bytes) + : (T) NodeIndex.fromRlpBytes(bytes); + } + + @Override + public Function getSerializer(Class objectClass) { + if ((!objectClass.equals(NodeRecordInfo.class)) && (!objectClass.equals(NodeIndex.class))) { + throw new RuntimeException(String.format("Type %s is not supported", objectClass)); + } + return value -> + objectClass.equals(NodeRecordInfo.class) + ? ((NodeRecordInfo) value).toRlpBytes() + : ((NodeIndex) value).toRlpBytes(); + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorage.java b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorage.java index 2a636d0da..eee40e0e1 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorage.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorage.java @@ -1,10 +1,13 @@ package org.ethereum.beacon.discovery.storage; +import org.ethereum.beacon.chain.storage.impl.SerializerFactory; import org.ethereum.beacon.db.source.SingleValueSource; import org.ethereum.beacon.discovery.enr.NodeRecordInfo; /** Stores {@link NodeTable} and home node info */ public interface NodeTableStorage { + SerializerFactory DEFAULT_SERIALIZER = new NodeSerializerFactory(); + NodeTable get(); SingleValueSource getHomeNodeSource(); diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java index 028501ef8..ce3f2b63a 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java @@ -1,7 +1,5 @@ package org.ethereum.beacon.discovery; -import org.ethereum.beacon.chain.storage.impl.SerializerFactory; -import org.ethereum.beacon.consensus.BeaconChainSpec; import org.ethereum.beacon.db.Database; import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.enr.NodeRecordV5; @@ -28,6 +26,7 @@ import java.util.concurrent.TimeUnit; import static org.ethereum.beacon.discovery.NodeContext.DEFAULT_DISTANCE; +import static org.ethereum.beacon.discovery.storage.NodeTableStorage.DEFAULT_SERIALIZER; /** Same as {@link DiscoveryNoNetworkTest} but using real network */ public class DiscoveryNetworkTest { @@ -55,12 +54,10 @@ public void test() throws Exception { NodeTableStorageFactoryImpl nodeTableStorageFactory = new NodeTableStorageFactoryImpl(); Database database1 = Database.inMemoryDB(); Database database2 = Database.inMemoryDB(); - SerializerFactory serializerFactory = - SerializerFactory.createSSZ(BeaconChainSpec.DEFAULT_CONSTANTS); NodeTableStorage nodeTableStorage1 = nodeTableStorageFactory.create( database1, - serializerFactory, + DEFAULT_SERIALIZER, () -> nodeRecord1, () -> new ArrayList() { @@ -71,7 +68,7 @@ public void test() throws Exception { NodeTableStorage nodeTableStorage2 = nodeTableStorageFactory.create( database2, - serializerFactory, + DEFAULT_SERIALIZER, () -> nodeRecord2, () -> new ArrayList() { diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java index 132c09458..1dcf3da99 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java @@ -1,7 +1,5 @@ package org.ethereum.beacon.discovery; -import org.ethereum.beacon.chain.storage.impl.SerializerFactory; -import org.ethereum.beacon.consensus.BeaconChainSpec; import org.ethereum.beacon.db.Database; import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.enr.NodeRecordV5; @@ -30,6 +28,7 @@ import java.util.concurrent.TimeUnit; import static org.ethereum.beacon.discovery.NodeContext.DEFAULT_DISTANCE; +import static org.ethereum.beacon.discovery.storage.NodeTableStorage.DEFAULT_SERIALIZER; /** * Discovery test without real network, instead outgoing stream of each peer is connected with @@ -60,12 +59,10 @@ public void test() throws Exception { NodeTableStorageFactoryImpl nodeTableStorageFactory = new NodeTableStorageFactoryImpl(); Database database1 = Database.inMemoryDB(); Database database2 = Database.inMemoryDB(); - SerializerFactory serializerFactory = - SerializerFactory.createSSZ(BeaconChainSpec.DEFAULT_CONSTANTS); NodeTableStorage nodeTableStorage1 = nodeTableStorageFactory.create( database1, - serializerFactory, + DEFAULT_SERIALIZER, () -> nodeRecord1, () -> new ArrayList() { @@ -76,7 +73,7 @@ public void test() throws Exception { NodeTableStorage nodeTableStorage2 = nodeTableStorageFactory.create( database2, - serializerFactory, + DEFAULT_SERIALIZER, () -> nodeRecord2, () -> new ArrayList() { diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeTableTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeTableTest.java index be078837e..5abc9bd5a 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeTableTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeTableTest.java @@ -1,7 +1,5 @@ package org.ethereum.beacon.discovery.storage; -import org.ethereum.beacon.chain.storage.impl.SerializerFactory; -import org.ethereum.beacon.consensus.BeaconChainSpec; import org.ethereum.beacon.db.Database; import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.enr.NodeRecordInfo; @@ -20,6 +18,7 @@ import java.util.function.Supplier; import static org.ethereum.beacon.discovery.enr.NodeRecordV5.NODE_ID_FUNCTION; +import static org.ethereum.beacon.discovery.storage.NodeTableStorage.DEFAULT_SERIALIZER; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -48,12 +47,10 @@ public void testCreate() throws Exception { NodeRecordV5 nodeRecordV5 = (NodeRecordV5) NodeRecord.fromBase64(localhostEnr); NodeTableStorageFactoryImpl nodeTableStorageFactory = new NodeTableStorageFactoryImpl(); Database database = Database.inMemoryDB(); - SerializerFactory serializerFactory = - SerializerFactory.createSSZ(BeaconChainSpec.DEFAULT_CONSTANTS); NodeTableStorage nodeTableStorage = nodeTableStorageFactory.create( database, - serializerFactory, + DEFAULT_SERIALIZER, homeNodeSupplier, () -> { List nodes = new ArrayList<>(); @@ -77,12 +74,10 @@ public void testFind() throws Exception { NodeRecordV5 localHostNode = (NodeRecordV5) NodeRecord.fromBase64(localhostEnr); NodeTableStorageFactoryImpl nodeTableStorageFactory = new NodeTableStorageFactoryImpl(); Database database = Database.inMemoryDB(); - SerializerFactory serializerFactory = - SerializerFactory.createSSZ(BeaconChainSpec.DEFAULT_CONSTANTS); NodeTableStorage nodeTableStorage = nodeTableStorageFactory.create( database, - serializerFactory, + DEFAULT_SERIALIZER, homeNodeSupplier, () -> { List nodes = new ArrayList<>(); @@ -122,8 +117,8 @@ public void testFind() throws Exception { List closestNodes = nodeTableStorage.get().findClosestNodes(NODE_ID_FUNCTION.apply(closestNode), 252); assertEquals(2, closestNodes.size()); - assertEquals(closestNodes.get(0).getNode().getPublicKey(), localHostNode.getPublicKey()); - assertEquals(closestNodes.get(1).getNode().getPublicKey(), closestNode.getPublicKey()); + assertEquals(closestNodes.get(1).getNode().getPublicKey(), localHostNode.getPublicKey()); + assertEquals(closestNodes.get(0).getNode().getPublicKey(), closestNode.getPublicKey()); List farNodes = nodeTableStorage.get().findClosestNodes(NODE_ID_FUNCTION.apply(farNode), 1); assertEquals(1, farNodes.size()); From e6f734ec420913d5ff54cba52b4b532df8a3fcf4 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Sun, 29 Sep 2019 11:58:41 +0300 Subject: [PATCH 20/77] discovery: fix logDistance to use bit index instead of logarithm --- .../ethereum/beacon/discovery/Functions.java | 28 +++++++++--------- .../beacon/discovery/FunctionsTest.java | 29 ++++++++++++------- .../artemis/util/bytes/BytesValue.java | 6 ++++ 3 files changed, 37 insertions(+), 26 deletions(-) diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java b/discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java index 24f33c5ec..29386a938 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java @@ -152,22 +152,20 @@ public static Random getRandom() { * *

distance(n₁, n₂) = n₁ XOR n₂ * - *

In many situations, the logarithmic distance (i.e. length of common prefix in bits) is used - * in place of the actual distance. - * - *

logdistance(n₁, n₂) = log2(distance(n₁, n₂)) + *

LogDistance is reverse of length of common prefix in bits (length - number of leftmost zeros + * in XOR) */ public static int logDistance(Bytes32 nodeId1, Bytes32 nodeId2) { - BigInteger distance = new BigInteger(1, Bytes32s.xor(nodeId1, nodeId2).extractArray()); - return log2(distance.doubleValue()); - } - - /** - * Logarithm with base 2. See https://stackoverflow.com/a/3305400 for - * implementation details. - */ - private static int log2(double x) { - return (int) Math.floor((Math.log(x) / Math.log(2.0) + 1e-10)); + BytesValue distance = Bytes32s.xor(nodeId1, nodeId2); + int logDistance = Byte.SIZE * distance.size(); // 256 + final int maxLogDistance = logDistance; + for (int i = 0; i < maxLogDistance; ++i) { + if (distance.getHighBit(i)) { + break; + } else { + logDistance--; + } + } + return logDistance; } } diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/FunctionsTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/FunctionsTest.java index fd2eac170..f6b4259d4 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/FunctionsTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/FunctionsTest.java @@ -8,20 +8,27 @@ public class FunctionsTest { @Test public void testLogDistance() { - Bytes32 nodeId0 = Bytes32.fromHexString("0000000000000000000000000000000000000000000000000000000000000000"); - Bytes32 nodeId1a = Bytes32.fromHexString("0000000000000000000000000000000000000000000000000000000000000001"); - Bytes32 nodeId1b = Bytes32.fromHexString("1000000000000000000000000000000000000000000000000000000000000000"); - Bytes32 nodeId1s = Bytes32.fromHexString("1111111111111111111111111111111111111111111111111111111111111111"); - Bytes32 nodeId9s = Bytes32.fromHexString("9999999999999999999999999999999999999999999999999999999999999999"); - Bytes32 nodeIdfs = Bytes32.fromHexString("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"); - assertEquals(0, Functions.logDistance(nodeId0, nodeId1a)); + Bytes32 nodeId0 = + Bytes32.fromHexString("0000000000000000000000000000000000000000000000000000000000000000"); + Bytes32 nodeId1a = + Bytes32.fromHexString("0000000000000000000000000000000000000000000000000000000000000001"); + Bytes32 nodeId1b = + Bytes32.fromHexString("1000000000000000000000000000000000000000000000000000000000000000"); + Bytes32 nodeId1s = + Bytes32.fromHexString("1111111111111111111111111111111111111111111111111111111111111111"); + Bytes32 nodeId9s = + Bytes32.fromHexString("9999999999999999999999999999999999999999999999999999999999999999"); + Bytes32 nodeIdfs = + Bytes32.fromHexString("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"); + assertEquals(0, Functions.logDistance(nodeId1a, nodeId1a)); + assertEquals(1, Functions.logDistance(nodeId0, nodeId1a)); // So it's big endian - assertEquals(252, Functions.logDistance(nodeId0, nodeId1b)); - assertEquals(252, Functions.logDistance(nodeId0, nodeId1s)); - assertEquals(255, Functions.logDistance(nodeId0, nodeId9s)); + assertEquals(253, Functions.logDistance(nodeId0, nodeId1b)); + assertEquals(253, Functions.logDistance(nodeId0, nodeId1s)); + assertEquals(256, Functions.logDistance(nodeId0, nodeId9s)); // maximum distance assertEquals(256, Functions.logDistance(nodeId0, nodeIdfs)); // logDistance is not an additive function - assertEquals(255, Functions.logDistance(nodeId1s, nodeIdfs)); + assertEquals(255, Functions.logDistance(nodeId9s, nodeIdfs)); } } diff --git a/types/src/main/java/tech/pegasys/artemis/util/bytes/BytesValue.java b/types/src/main/java/tech/pegasys/artemis/util/bytes/BytesValue.java index b9a774322..4c7f0f773 100644 --- a/types/src/main/java/tech/pegasys/artemis/util/bytes/BytesValue.java +++ b/types/src/main/java/tech/pegasys/artemis/util/bytes/BytesValue.java @@ -293,10 +293,16 @@ static BytesValue fromHexString(String str, int destinationSize) { */ byte get(int i); + /** @return bit at `bitIndex`, with 0 index and bitIndex(0) of 0x01 == 1 */ default boolean getBit(int bitIndex) { return ((get(bitIndex / 8) >> (bitIndex % 8)) & 1) == 1; } + /** @return bit at `bitIndex`, with 0 index and bitIndex(7) of 0x01 == 1 */ + default boolean getHighBit(int bitIndex) { + return ((get(bitIndex / 8) >> (7 - (bitIndex % 8))) & 1) == 1; + } + /** * Retrieves the 4 bytes starting at the provided index in this value as an integer. * From 159457e21b240152519e47a49f9cc6e9cd773101 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Tue, 1 Oct 2019 20:53:12 +0300 Subject: [PATCH 21/77] discovery: make NODES answer use Buckets --- .../discovery/DiscoveryManagerImpl.java | 23 ++-- .../discovery/DiscoveryTaskManager.java | 26 +++- .../DiscoveryV5MessageProcessor.java | 3 +- .../beacon/discovery/NodeContext.java | 9 ++ .../message/handler/FindNodeHandler.java | 55 ++++----- .../beacon/discovery/storage/NodeBucket.java | 102 ++++++++++++++++ .../discovery/storage/NodeBucketStorage.java | 14 +++ .../storage/NodeBucketStorageImpl.java | 64 ++++++++++ .../storage/NodeSerializerFactory.java | 28 +++-- .../storage/NodeTableStorageFactory.java | 6 +- .../storage/NodeTableStorageFactoryImpl.java | 9 +- .../discovery/DiscoveryNetworkTest.java | 13 +- .../discovery/DiscoveryNoNetworkTest.java | 17 ++- .../mock/DiscoveryManagerNoNetwork.java | 13 +- .../discovery/storage/NodeBucketTest.java | 115 ++++++++++++++++++ .../discovery/storage/NodeTableTest.java | 8 +- 16 files changed, 437 insertions(+), 68 deletions(-) create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeBucket.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeBucketStorage.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeBucketStorageImpl.java create mode 100644 discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeBucketTest.java diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java index 90a129564..fdfaa6c11 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java @@ -14,6 +14,7 @@ import org.ethereum.beacon.discovery.network.NetworkParcelV5; import org.ethereum.beacon.discovery.packet.UnknownPacket; import org.ethereum.beacon.discovery.storage.AuthTagRepository; +import org.ethereum.beacon.discovery.storage.NodeBucketStorage; import org.ethereum.beacon.discovery.storage.NodeTable; import org.ethereum.beacon.schedulers.Scheduler; import org.ethereum.beacon.util.ExpirationScheduler; @@ -38,19 +39,24 @@ public class DiscoveryManagerImpl implements DiscoveryManager { private final Bytes32 homeNodeId; private final NodeRecordV5 homeNodeRecord; private final NodeTable nodeTable; + private final NodeBucketStorage nodeBucketStorage; private final Map recentContexts = new ConcurrentHashMap<>(); // nodeId -> context - private ExpirationScheduler contextExpirationScheduler = - new ExpirationScheduler<>(CLEANUP_DELAY_SECONDS, TimeUnit.SECONDS); private final AuthTagRepository authTagRepo; private final DiscoveryServer discoveryServer; private final CompletableFuture discoveryClientAssigned; private final Scheduler scheduler; + private ExpirationScheduler contextExpirationScheduler = + new ExpirationScheduler<>(CLEANUP_DELAY_SECONDS, TimeUnit.SECONDS); private DiscoveryClient discoveryClient; public DiscoveryManagerImpl( - NodeTable nodeTable, NodeRecordV5 homeNode, Scheduler serverScheduler) { + NodeTable nodeTable, + NodeBucketStorage nodeBucketStorage, + NodeRecordV5 homeNode, + Scheduler serverScheduler) { this.nodeTable = nodeTable; + this.nodeBucketStorage = nodeBucketStorage; this.homeNodeId = homeNode.getNodeId(); this.homeNodeRecord = (NodeRecordV5) nodeTable.getHomeNode(); this.authTagRepo = new AuthTagRepository(); @@ -117,6 +123,7 @@ private Optional getContext(Bytes32 nodeId) { nodeRecord, homeNodeRecord, nodeTable, + nodeBucketStorage, authTagRepo, packet -> outgoingSink.next(new NetworkParcelV5(packet, nodeRecord)), random); @@ -124,10 +131,12 @@ private Optional getContext(Bytes32 nodeId) { } final NodeContext contextBackup = context; - contextExpirationScheduler.put(context.getNodeRecord().getNodeId(), () -> { - recentContexts.remove(contextBackup.getNodeRecord().getNodeId()); - contextBackup.cleanup(); - }); + contextExpirationScheduler.put( + context.getNodeRecord().getNodeId(), + () -> { + recentContexts.remove(contextBackup.getNodeRecord().getNodeId()); + contextBackup.cleanup(); + }); return Optional.of(context); } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryTaskManager.java b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryTaskManager.java index e87085956..45bd515dd 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryTaskManager.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryTaskManager.java @@ -3,6 +3,7 @@ import org.ethereum.beacon.discovery.enr.NodeRecordInfo; import org.ethereum.beacon.discovery.enr.NodeRecordV5; import org.ethereum.beacon.discovery.enr.NodeStatus; +import org.ethereum.beacon.discovery.storage.NodeBucketStorage; import org.ethereum.beacon.discovery.storage.NodeTable; import org.ethereum.beacon.schedulers.Scheduler; import tech.pegasys.artemis.util.bytes.Bytes32; @@ -25,7 +26,8 @@ public class DiscoveryTaskManager { private final Scheduler scheduler; private final Bytes32 homeNodeId; private final NodeConnectTasks nodeConnectTasks; - private NodeTable nodeTable; + private final NodeTable nodeTable; + private final NodeBucketStorage nodeBucketStorage; /** * Checks whether {@link org.ethereum.beacon.discovery.enr.NodeRecord} is ready for alive status * check. Plus, marks records as DEAD if there were a lot of unsuccessful retries to get reply @@ -49,7 +51,7 @@ public class DiscoveryTaskManager { return false; // no need to rediscover } if (nodeRecord.getRetry() >= MAX_RETRIES) { - nodeTable.save( + updateNode( new NodeRecordInfo( nodeRecord.getNode(), nodeRecord.getLastRetry(), NodeStatus.DEAD, 0)); return false; @@ -61,11 +63,14 @@ public class DiscoveryTaskManager { return true; }; + private boolean resetDead = false; /** * @param discoveryManager Discovery manager - * @param nodeTable Ethereum node records storage + * @param nodeTable Ethereum node records storage, stores all found nodes + * @param nodeBucketStorage Node bucket storage. stores only closest nodes in ready-to-answer + * format * @param homeNode Home node * @param scheduler scheduler to run recurrent tasks on * @param resetDead Whether to reset dead status of the nodes. If set to true, resets its status @@ -74,13 +79,17 @@ public class DiscoveryTaskManager { public DiscoveryTaskManager( DiscoveryManager discoveryManager, NodeTable nodeTable, + NodeBucketStorage nodeBucketStorage, NodeRecordV5 homeNode, Scheduler scheduler, boolean resetDead) { this.scheduler = scheduler; this.nodeTable = nodeTable; + this.nodeBucketStorage = nodeBucketStorage; this.homeNodeId = NODE_ID_FUNCTION.apply(homeNode); - this.nodeConnectTasks = new NodeConnectTasks(discoveryManager, scheduler, Duration.ofSeconds(RETRY_TIMEOUT_SECONDS)); + this.nodeConnectTasks = + new NodeConnectTasks( + discoveryManager, scheduler, Duration.ofSeconds(RETRY_TIMEOUT_SECONDS)); this.resetDead = resetDead; } @@ -110,18 +119,23 @@ private void recurrentTask() { nodeConnectTasks.add( nodeRecord, () -> - nodeTable.save( + updateNode( new NodeRecordInfo( nodeRecord.getNode(), System.currentTimeMillis() / MS_IN_SECOND, NodeStatus.ACTIVE, 0)), () -> - nodeTable.save( + updateNode( new NodeRecordInfo( nodeRecord.getNode(), System.currentTimeMillis() / MS_IN_SECOND, NodeStatus.SLEEP, (nodeRecord.getRetry() + 1))))); } + + private void updateNode(NodeRecordInfo nodeRecordInfo) { + nodeTable.save(nodeRecordInfo); + nodeBucketStorage.put(nodeRecordInfo); + } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryV5MessageProcessor.java b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryV5MessageProcessor.java index 5225725bd..26c2dc282 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryV5MessageProcessor.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryV5MessageProcessor.java @@ -12,13 +12,12 @@ import java.util.Map; public class DiscoveryV5MessageProcessor implements DiscoveryMessageProcessor { - private static final int MAX_NODES_IN_MSG = 24; private final Map messageHandlers = new HashMap<>(); public DiscoveryV5MessageProcessor() { messageHandlers.put(MessageCode.PING, new PingHandler()); messageHandlers.put(MessageCode.PONG, new PongHandler()); - messageHandlers.put(MessageCode.FINDNODE, new FindNodeHandler(MAX_NODES_IN_MSG)); + messageHandlers.put(MessageCode.FINDNODE, new FindNodeHandler()); messageHandlers.put(MessageCode.NODES, new NodesHandler()); } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeContext.java b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeContext.java index 6fa7d1a4d..8f4390b0d 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeContext.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeContext.java @@ -18,6 +18,8 @@ import org.ethereum.beacon.discovery.packet.handler.UnknownPacketHandler; import org.ethereum.beacon.discovery.packet.handler.WhoAreYouPacketHandler; import org.ethereum.beacon.discovery.storage.AuthTagRepository; +import org.ethereum.beacon.discovery.storage.NodeBucket; +import org.ethereum.beacon.discovery.storage.NodeBucketStorage; import org.ethereum.beacon.discovery.storage.NodeTable; import tech.pegasys.artemis.util.bytes.Bytes32; import tech.pegasys.artemis.util.bytes.BytesValue; @@ -39,6 +41,7 @@ public class NodeContext { private final Bytes32 homeNodeId; private final AuthTagRepository authTagRepo; private final NodeTable nodeTable; + private final NodeBucketStorage nodeBucketStorage; private final Consumer outgoing; private final Random rnd; private final Map packetHandlers = new HashMap<>(); @@ -53,6 +56,7 @@ public NodeContext( NodeRecordV5 nodeRecord, NodeRecordV5 homeNodeRecord, NodeTable nodeTable, + NodeBucketStorage nodeBucketStorage, AuthTagRepository authTagRepo, Consumer outgoing, Random rnd) { @@ -60,6 +64,7 @@ public NodeContext( this.outgoing = outgoing; this.authTagRepo = authTagRepo; this.nodeTable = nodeTable; + this.nodeBucketStorage = nodeBucketStorage; this.homeNodeRecord = homeNodeRecord; this.homeNodeId = homeNodeRecord.getNodeId(); this.rnd = rnd; @@ -238,6 +243,10 @@ public NodeTable getNodeTable() { return nodeTable; } + public Optional getBucket(int index) { + return nodeBucketStorage.get(index); + } + public synchronized Bytes32 getIdNonce() { return idNonce; } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/FindNodeHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/FindNodeHandler.java index 9e37d24aa..380e6cb68 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/FindNodeHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/FindNodeHandler.java @@ -6,42 +6,41 @@ import org.ethereum.beacon.discovery.message.FindNodeMessage; import org.ethereum.beacon.discovery.message.NodesMessage; import org.ethereum.beacon.discovery.packet.MessagePacket; +import org.ethereum.beacon.discovery.storage.NodeBucket; import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; +import java.util.Optional; import java.util.stream.Collectors; - -import static org.ethereum.beacon.discovery.NodeContext.DEFAULT_DISTANCE; +import java.util.stream.IntStream; public class FindNodeHandler implements MessageHandler { - private final int maxNodesInMsg; - public FindNodeHandler(int maxNodesInMsg) { - this.maxNodesInMsg = maxNodesInMsg; - } + public FindNodeHandler() {} @Override public void handle(FindNodeMessage message, NodeContext context) { - List nodes = - context.getNodeTable().findClosestNodes(context.getNodeRecord().getNodeId(), DEFAULT_DISTANCE); - final AtomicInteger counter = new AtomicInteger(); - nodes.stream() - .map(NodeRecordInfo::getNode) - .collect(Collectors.groupingBy(it -> counter.getAndIncrement() / maxNodesInMsg)) - .values() - .forEach( - c -> - context.addOutgoingEvent( - MessagePacket.create( - context.getHomeNodeId(), - context.getNodeRecord().getNodeId(), - context.getAuthTag().get(), - context.getInitiatorKey(), - DiscoveryV5Message.from( - new NodesMessage( - message.getRequestId(), - nodes.size() / maxNodesInMsg, - () -> c, - nodes.size()))))); + List nodeBuckets = + IntStream.range(0, message.getDistance()) + .mapToObj(context::getBucket) + .filter(Optional::isPresent) + .map(Optional::get) + .collect(Collectors.toList()); + nodeBuckets.forEach( + bucket -> + context.addOutgoingEvent( + MessagePacket.create( + context.getHomeNodeId(), + context.getNodeRecord().getNodeId(), + context.getAuthTag().get(), + context.getInitiatorKey(), + DiscoveryV5Message.from( + new NodesMessage( + message.getRequestId(), + nodeBuckets.size(), + () -> + bucket.getNodeRecords().stream() + .map(NodeRecordInfo::getNode) + .collect(Collectors.toList()), + bucket.size()))))); } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeBucket.java b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeBucket.java new file mode 100644 index 000000000..9dfd6c7dd --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeBucket.java @@ -0,0 +1,102 @@ +package org.ethereum.beacon.discovery.storage; + +import org.ethereum.beacon.discovery.enr.NodeRecordInfo; +import org.ethereum.beacon.discovery.enr.NodeStatus; +import org.web3j.rlp.RlpDecoder; +import org.web3j.rlp.RlpEncoder; +import org.web3j.rlp.RlpList; +import org.web3j.rlp.RlpString; +import tech.pegasys.artemis.util.bytes.BytesValue; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Comparator; +import java.util.List; +import java.util.TreeSet; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +/** + * Storage for nodes, K-Bucket. Holds only {@link #K} nodes, replacing nodes with the same nodeId + * and nodes with old lastRetry. Also throws out DEAD nodes without taking any notice on other + * fields. + */ +public class NodeBucket { + /** Bucket size, number of nodes */ + public static final int K = 16; + + private static final Predicate FILTER = + nodeRecord -> nodeRecord.getStatus().equals(NodeStatus.ACTIVE); + private static final Comparator COMPARATOR = + (o1, o2) -> { + if (o1.getNode().getNodeId().equals(o2.getNode().getNodeId())) { + return 0; + } else { + return Long.signum(o1.getLastRetry() - o2.getLastRetry()); + } + }; + private final TreeSet bucket = new TreeSet<>(COMPARATOR); + + public static NodeBucket fromRlpBytes(BytesValue bytes) { + NodeBucket nodeBucket = new NodeBucket(); + ((RlpList) RlpDecoder.decode(bytes.extractArray()).getValues().get(0)) + .getValues().stream() + .map(rt -> (RlpString) rt) + .map(RlpString::getBytes) + .map(BytesValue::wrap) + .map(NodeRecordInfo::fromRlpBytes) + .forEach(nodeBucket::put); + return nodeBucket; + } + + public synchronized boolean put(NodeRecordInfo nodeRecord) { + if (FILTER.test(nodeRecord)) { + if (!bucket.contains(nodeRecord)) { + boolean modified = bucket.add(nodeRecord); + if (bucket.size() > K) { + bucket.pollFirst(); + } + return modified; + } else { + NodeRecordInfo bucketNode = bucket.subSet(nodeRecord, true, nodeRecord, true).first(); + if (nodeRecord.getLastRetry() > bucketNode.getLastRetry()) { + bucket.remove(bucketNode); + bucket.add(nodeRecord); + return true; + } + } + } else { + return bucket.remove(nodeRecord); + } + + return false; + } + + public boolean contains(NodeRecordInfo nodeRecordInfo) { + return bucket.contains(nodeRecordInfo); + } + + public void putAll(Collection nodeRecords) { + nodeRecords.forEach(this::put); + } + + public synchronized BytesValue toRlpBytes() { + byte[] res = + RlpEncoder.encode( + new RlpList( + bucket.stream() + .map(NodeRecordInfo::toRlpBytes) + .map(BytesValue::extractArray) + .map(RlpString::create) + .collect(Collectors.toList()))); + return BytesValue.wrap(res); + } + + public int size() { + return bucket.size(); + } + + public List getNodeRecords() { + return new ArrayList<>(bucket); + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeBucketStorage.java b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeBucketStorage.java new file mode 100644 index 000000000..91cb8c70a --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeBucketStorage.java @@ -0,0 +1,14 @@ +package org.ethereum.beacon.discovery.storage; + +import org.ethereum.beacon.discovery.enr.NodeRecordInfo; + +import java.util.Optional; + +/** Stores {@link NodeRecordInfo}'s in {@link NodeBucket}'s */ +public interface NodeBucketStorage { + Optional get(int index); + + void put(NodeRecordInfo nodeRecordInfo); + + void commit(); +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeBucketStorageImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeBucketStorageImpl.java new file mode 100644 index 000000000..b29531de8 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeBucketStorageImpl.java @@ -0,0 +1,64 @@ +package org.ethereum.beacon.discovery.storage; + +import org.ethereum.beacon.chain.storage.impl.SerializerFactory; +import org.ethereum.beacon.db.Database; +import org.ethereum.beacon.db.source.DataSource; +import org.ethereum.beacon.db.source.HoleyList; +import org.ethereum.beacon.db.source.impl.DataSourceList; +import org.ethereum.beacon.discovery.Functions; +import org.ethereum.beacon.discovery.enr.NodeRecordInfo; +import tech.pegasys.artemis.util.bytes.Bytes32; +import tech.pegasys.artemis.util.bytes.BytesValue; + +import java.util.Optional; + +/** + * Stores {@link NodeRecordInfo}'s in {@link NodeBucket}'s calculating index number of bucket as + * {@link Functions#logDistance(Bytes32, Bytes32)} from homeNodeId and ignoring index above {@link + * #MAXIMUM_BUCKET} + */ +public class NodeBucketStorageImpl implements NodeBucketStorage { + public static final String NODE_BUCKET_STORAGE_NAME = "node-bucket-table"; + public static final int MAXIMUM_BUCKET = 255; + private final HoleyList nodeBucketsTable; + private final Bytes32 homeNodeId; + + public NodeBucketStorageImpl( + Database database, SerializerFactory serializerFactory, Bytes32 homeNodeId) { + DataSource nodeBucketsSource = + database.createStorage(NODE_BUCKET_STORAGE_NAME); + this.nodeBucketsTable = + new DataSourceList<>( + nodeBucketsSource, + serializerFactory.getSerializer(NodeBucket.class), + serializerFactory.getDeserializer(NodeBucket.class)); + this.homeNodeId = homeNodeId; + } + + @Override + public Optional get(int index) { + return nodeBucketsTable.get(index); + } + + @Override + public void put(NodeRecordInfo nodeRecordInfo) { + int logDistance = Functions.logDistance(homeNodeId, nodeRecordInfo.getNode().getNodeId()); + if (logDistance <= MAXIMUM_BUCKET) { + Optional nodeBucketOpt = nodeBucketsTable.get(logDistance); + if (nodeBucketOpt.isPresent()) { + NodeBucket nodeBucket = nodeBucketOpt.get(); + boolean updated = nodeBucket.put(nodeRecordInfo); + if (updated) { + nodeBucketsTable.put(logDistance, nodeBucket); + } + } else { + NodeBucket nodeBucket = new NodeBucket(); + nodeBucket.put(nodeRecordInfo); + nodeBucketsTable.put(logDistance, nodeBucket); + } + } + } + + @Override + public void commit() {} +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeSerializerFactory.java b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeSerializerFactory.java index 4bfdc72ba..90fd28713 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeSerializerFactory.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeSerializerFactory.java @@ -4,28 +4,36 @@ import org.ethereum.beacon.discovery.enr.NodeRecordInfo; import tech.pegasys.artemis.util.bytes.BytesValue; +import java.util.HashMap; +import java.util.Map; import java.util.function.Function; public class NodeSerializerFactory implements SerializerFactory { + private final Map> deserializerMap = new HashMap<>(); + private final Map> serializerMap = new HashMap<>(); + + public NodeSerializerFactory() { + deserializerMap.put(NodeRecordInfo.class, NodeRecordInfo::fromRlpBytes); + serializerMap.put(NodeRecordInfo.class, o -> ((NodeRecordInfo) o).toRlpBytes()); + deserializerMap.put(NodeIndex.class, NodeIndex::fromRlpBytes); + serializerMap.put(NodeIndex.class, o -> ((NodeIndex) o).toRlpBytes()); + deserializerMap.put(NodeBucket.class, NodeBucket::fromRlpBytes); + serializerMap.put(NodeBucket.class, o -> ((NodeBucket) o).toRlpBytes()); + } + @Override public Function getDeserializer(Class objectClass) { - if ((!objectClass.equals(NodeRecordInfo.class)) && (!objectClass.equals(NodeIndex.class))) { + if (!deserializerMap.containsKey(objectClass)) { throw new RuntimeException(String.format("Type %s is not supported", objectClass)); } - return bytes -> - objectClass.equals(NodeRecordInfo.class) - ? (T) NodeRecordInfo.fromRlpBytes(bytes) - : (T) NodeIndex.fromRlpBytes(bytes); + return bytes -> (T) deserializerMap.get(objectClass).apply(bytes); } @Override public Function getSerializer(Class objectClass) { - if ((!objectClass.equals(NodeRecordInfo.class)) && (!objectClass.equals(NodeIndex.class))) { + if (!serializerMap.containsKey(objectClass)) { throw new RuntimeException(String.format("Type %s is not supported", objectClass)); } - return value -> - objectClass.equals(NodeRecordInfo.class) - ? ((NodeRecordInfo) value).toRlpBytes() - : ((NodeIndex) value).toRlpBytes(); + return value -> serializerMap.get(objectClass).apply(value); } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactory.java b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactory.java index 7ff9252e6..559f98acf 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactory.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactory.java @@ -4,14 +4,18 @@ import org.ethereum.beacon.db.Database; import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.enr.NodeRecordV5; +import tech.pegasys.artemis.util.bytes.Bytes32; import java.util.List; import java.util.function.Supplier; public interface NodeTableStorageFactory { - NodeTableStorage create( + NodeTableStorage createTable( Database database, SerializerFactory serializerFactory, Supplier homeNodeSupplier, Supplier> bootNodes); + + NodeBucketStorage createBuckets( + Database database, SerializerFactory serializerFactory, Bytes32 homeNodeId); } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactoryImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactoryImpl.java index 449ddc2ff..7ce3428c9 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactoryImpl.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactoryImpl.java @@ -5,6 +5,7 @@ import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.enr.NodeRecordInfo; import org.ethereum.beacon.discovery.enr.NodeRecordV5; +import tech.pegasys.artemis.util.bytes.Bytes32; import java.util.List; import java.util.function.Supplier; @@ -21,7 +22,7 @@ private boolean isStorageEmpty(NodeTableStorage nodeTableStorage) { * Uses `serializerFactory` for node records serialization. */ @Override - public NodeTableStorage create( + public NodeTableStorage createTable( Database database, SerializerFactory serializerFactory, Supplier homeNodeSupplier, @@ -48,4 +49,10 @@ public NodeTableStorage create( return nodeTableStorage; } + + @Override + public NodeBucketStorage createBuckets( + Database database, SerializerFactory serializerFactory, Bytes32 homeNodeId) { + return new NodeBucketStorageImpl(database, serializerFactory, homeNodeId); + } } diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java index ce3f2b63a..81d45c593 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java @@ -12,6 +12,7 @@ import org.ethereum.beacon.discovery.packet.RandomPacket; import org.ethereum.beacon.discovery.packet.UnknownPacket; import org.ethereum.beacon.discovery.packet.WhoAreYouPacket; +import org.ethereum.beacon.discovery.storage.NodeBucketStorage; import org.ethereum.beacon.discovery.storage.NodeTableStorage; import org.ethereum.beacon.discovery.storage.NodeTableStorageFactoryImpl; import org.ethereum.beacon.schedulers.Schedulers; @@ -55,7 +56,7 @@ public void test() throws Exception { Database database1 = Database.inMemoryDB(); Database database2 = Database.inMemoryDB(); NodeTableStorage nodeTableStorage1 = - nodeTableStorageFactory.create( + nodeTableStorageFactory.createTable( database1, DEFAULT_SERIALIZER, () -> nodeRecord1, @@ -65,8 +66,11 @@ public void test() throws Exception { add(nodeRecord2); } }); + NodeBucketStorage nodeBucketStorage1 = + nodeTableStorageFactory.createBuckets( + database1, DEFAULT_SERIALIZER, nodeRecord1.getNodeId()); NodeTableStorage nodeTableStorage2 = - nodeTableStorageFactory.create( + nodeTableStorageFactory.createTable( database2, DEFAULT_SERIALIZER, () -> nodeRecord2, @@ -76,14 +80,19 @@ public void test() throws Exception { add(nodeRecord1); } }); + NodeBucketStorage nodeBucketStorage2 = + nodeTableStorageFactory.createBuckets( + database2, DEFAULT_SERIALIZER, nodeRecord2.getNodeId()); DiscoveryManagerImpl discoveryManager1 = new DiscoveryManagerImpl( nodeTableStorage1.get(), + nodeBucketStorage1, nodeRecord1, Schedulers.createDefault().newSingleThreadDaemon("server-1")); DiscoveryManagerImpl discoveryManager2 = new DiscoveryManagerImpl( nodeTableStorage2.get(), + nodeBucketStorage2, nodeRecord2, Schedulers.createDefault().newSingleThreadDaemon("server-2")); diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java index 1dcf3da99..42f93e632 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java @@ -13,6 +13,7 @@ import org.ethereum.beacon.discovery.packet.RandomPacket; import org.ethereum.beacon.discovery.packet.UnknownPacket; import org.ethereum.beacon.discovery.packet.WhoAreYouPacket; +import org.ethereum.beacon.discovery.storage.NodeBucketStorage; import org.ethereum.beacon.discovery.storage.NodeTableStorage; import org.ethereum.beacon.discovery.storage.NodeTableStorageFactoryImpl; import org.ethereum.beacon.schedulers.Schedulers; @@ -60,7 +61,7 @@ public void test() throws Exception { Database database1 = Database.inMemoryDB(); Database database2 = Database.inMemoryDB(); NodeTableStorage nodeTableStorage1 = - nodeTableStorageFactory.create( + nodeTableStorageFactory.createTable( database1, DEFAULT_SERIALIZER, () -> nodeRecord1, @@ -70,8 +71,11 @@ public void test() throws Exception { add(nodeRecord2); } }); + NodeBucketStorage nodeBucketStorage1 = + nodeTableStorageFactory.createBuckets( + database1, DEFAULT_SERIALIZER, nodeRecord1.getNodeId()); NodeTableStorage nodeTableStorage2 = - nodeTableStorageFactory.create( + nodeTableStorageFactory.createTable( database2, DEFAULT_SERIALIZER, () -> nodeRecord2, @@ -81,6 +85,9 @@ public void test() throws Exception { add(nodeRecord1); } }); + NodeBucketStorage nodeBucketStorage2 = + nodeTableStorageFactory.createBuckets( + database2, DEFAULT_SERIALIZER, nodeRecord2.getNodeId()); SimpleProcessor from1to2 = new SimpleProcessor<>( Schedulers.createDefault().newSingleThreadDaemon("from1to2-thread"), "from1to2"); @@ -88,9 +95,11 @@ public void test() throws Exception { new SimpleProcessor<>( Schedulers.createDefault().newSingleThreadDaemon("from2to1-thread"), "from2to1"); DiscoveryManagerNoNetwork discoveryManager1 = - new DiscoveryManagerNoNetwork(nodeTableStorage1.get(), nodeRecord1, from2to1); + new DiscoveryManagerNoNetwork( + nodeTableStorage1.get(), nodeBucketStorage1, nodeRecord1, from2to1); DiscoveryManagerNoNetwork discoveryManager2 = - new DiscoveryManagerNoNetwork(nodeTableStorage2.get(), nodeRecord2, from1to2); + new DiscoveryManagerNoNetwork( + nodeTableStorage2.get(), nodeBucketStorage2, nodeRecord2, from1to2); discoveryManager1.start(); discoveryManager2.start(); diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java b/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java index 72fb8e534..821c0d975 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java @@ -4,7 +4,6 @@ import org.apache.logging.log4j.Logger; import org.ethereum.beacon.discovery.DiscoveryManager; import org.ethereum.beacon.discovery.NodeContext; -import org.ethereum.beacon.discovery.storage.NodeTable; import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.enr.NodeRecordInfo; import org.ethereum.beacon.discovery.enr.NodeRecordV5; @@ -12,6 +11,8 @@ import org.ethereum.beacon.discovery.network.NetworkParcelV5; import org.ethereum.beacon.discovery.packet.UnknownPacket; import org.ethereum.beacon.discovery.storage.AuthTagRepository; +import org.ethereum.beacon.discovery.storage.NodeBucketStorage; +import org.ethereum.beacon.discovery.storage.NodeTable; import org.reactivestreams.Publisher; import reactor.core.publisher.Flux; import reactor.core.publisher.FluxSink; @@ -36,14 +37,19 @@ public class DiscoveryManagerNoNetwork implements DiscoveryManager { private final FluxSink outgoingSink = outgoingMessages.sink(); private final Bytes32 homeNodeId; private final NodeRecordV5 homeNodeRecord; - private NodeTable nodeTable; + private final NodeTable nodeTable; + private final NodeBucketStorage nodeBucketStorage; private Publisher incomingPackets; private Map recentContexts = new ConcurrentHashMap<>(); // nodeId -> context private AuthTagRepository authTagRepo; public DiscoveryManagerNoNetwork( - NodeTable nodeTable, NodeRecordV5 homeNode, Publisher incomingPackets) { + NodeTable nodeTable, + NodeBucketStorage nodeBucketStorage, + NodeRecordV5 homeNode, + Publisher incomingPackets) { this.nodeTable = nodeTable; + this.nodeBucketStorage = nodeBucketStorage; this.incomingPackets = incomingPackets; this.homeNodeId = homeNode.getNodeId(); this.homeNodeRecord = (NodeRecordV5) nodeTable.getHomeNode(); @@ -97,6 +103,7 @@ private Optional getContext(Bytes32 nodeId) { nodeRecord, homeNodeRecord, nodeTable, + nodeBucketStorage, authTagRepo, packet -> outgoingSink.next(new NetworkParcelV5(packet, nodeRecord)), random); diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeBucketTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeBucketTest.java new file mode 100644 index 000000000..8293832b2 --- /dev/null +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeBucketTest.java @@ -0,0 +1,115 @@ +package org.ethereum.beacon.discovery.storage; + +import org.ethereum.beacon.db.Database; +import org.ethereum.beacon.discovery.Functions; +import org.ethereum.beacon.discovery.enr.NodeRecordInfo; +import org.ethereum.beacon.discovery.enr.NodeRecordV5; +import org.ethereum.beacon.discovery.enr.NodeStatus; +import org.junit.Test; +import tech.pegasys.artemis.util.bytes.BytesValue; +import tech.pegasys.artemis.util.uint.UInt64; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Random; +import java.util.stream.IntStream; + +import static org.ethereum.beacon.discovery.storage.NodeTableStorage.DEFAULT_SERIALIZER; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class NodeBucketTest { + private final Random rnd = new Random(); + + private NodeRecordInfo generateUniqueRecord() { + try { + byte[] pkey = new byte[33]; + rnd.nextBytes(pkey); + NodeRecordV5 nodeRecordV5 = + NodeRecordV5.Builder.empty() + .withIpV4Address(BytesValue.wrap(InetAddress.getByName("127.0.0.1").getAddress())) + .withSeq(UInt64.valueOf(1)) + .withUdpPort(30303) + .withSecp256k1(BytesValue.wrap(pkey)) + .build(); + return new NodeRecordInfo(nodeRecordV5, (long) rnd.nextInt(1000), NodeStatus.ACTIVE, 0); + } catch (UnknownHostException e) { + throw new RuntimeException(e); + } + } + + @Test + public void testBucket() { + NodeBucket nodeBucket = new NodeBucket(); + IntStream.range(0, 20).forEach(value -> nodeBucket.put(generateUniqueRecord())); + assertEquals(NodeBucket.K, nodeBucket.size()); + assertEquals(NodeBucket.K, nodeBucket.getNodeRecords().size()); + + long lastRetrySaved = -1L; + for (NodeRecordInfo nodeRecordInfo : nodeBucket.getNodeRecords()) { + assert nodeRecordInfo.getLastRetry() + >= lastRetrySaved; // Assert sorted by last retry, latest retry in the end + lastRetrySaved = nodeRecordInfo.getLastRetry(); + } + NodeRecordInfo willNotInsertNode = + new NodeRecordInfo(generateUniqueRecord().getNode(), -2L, NodeStatus.ACTIVE, 0); + nodeBucket.put(willNotInsertNode); + assertFalse(nodeBucket.contains(willNotInsertNode)); + NodeRecordInfo willInsertNode = + new NodeRecordInfo(generateUniqueRecord().getNode(), 1001L, NodeStatus.ACTIVE, 0); + NodeRecordInfo top = + nodeBucket.getNodeRecords().get(NodeBucket.K - 1); // latest retry should be kept + NodeRecordInfo bottom = nodeBucket.getNodeRecords().get(0); + nodeBucket.put(willInsertNode); + assertTrue(nodeBucket.contains(willInsertNode)); + assertTrue(nodeBucket.contains(top)); + assertFalse(nodeBucket.contains(bottom)); + NodeRecordInfo willInsertNode2 = + new NodeRecordInfo(willInsertNode.getNode(), 1002L, NodeStatus.ACTIVE, 0); + nodeBucket.put(willInsertNode2); // replaces willInsertNode with better last retry + assertEquals(willInsertNode2, nodeBucket.getNodeRecords().get(NodeBucket.K - 1)); + NodeRecordInfo willNotInsertNode3 = + new NodeRecordInfo(willInsertNode.getNode(), 999L, NodeStatus.ACTIVE, 0); + nodeBucket.put(willNotInsertNode3); // does not replace willInsertNode with worse last retry + assertEquals(willInsertNode2, nodeBucket.getNodeRecords().get(NodeBucket.K - 1)); // still 2nd + assertEquals(top, nodeBucket.getNodeRecords().get(NodeBucket.K - 2)); + + NodeRecordInfo willInsertNodeDead = + new NodeRecordInfo(willInsertNode.getNode(), 1001L, NodeStatus.DEAD, 0); + nodeBucket.put(willInsertNodeDead); // removes willInsertNode + assertEquals(NodeBucket.K - 1, nodeBucket.size()); + assertFalse(nodeBucket.contains(willInsertNode)); + } + + @Test + public void testStorage() { + NodeRecordInfo initial = generateUniqueRecord(); + Database database = Database.inMemoryDB(); + NodeTableStorageFactoryImpl nodeTableStorageFactory = new NodeTableStorageFactoryImpl(); + NodeBucketStorage nodeBucketStorage = + nodeTableStorageFactory.createBuckets( + database, DEFAULT_SERIALIZER, initial.getNode().getNodeId()); + + for (int i = 0; i < 20; ) { + NodeRecordInfo nodeRecordInfo = generateUniqueRecord(); + if (Functions.logDistance(initial.getNode().getNodeId(), nodeRecordInfo.getNode().getNodeId()) + == 255) { + nodeBucketStorage.put(nodeRecordInfo); + ++i; + } + } + for (int i = 0; i < 3; ) { + NodeRecordInfo nodeRecordInfo = generateUniqueRecord(); + if (Functions.logDistance(initial.getNode().getNodeId(), nodeRecordInfo.getNode().getNodeId()) + == 254) { + nodeBucketStorage.put(nodeRecordInfo); + ++i; + } + } + assertEquals(16, nodeBucketStorage.get(255).get().size()); + assertEquals(3, nodeBucketStorage.get(254).get().size()); + assertFalse(nodeBucketStorage.get(253).isPresent()); + assertFalse(nodeBucketStorage.get(256).isPresent()); + } +} diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeTableTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeTableTest.java index 5abc9bd5a..39fe5c301 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeTableTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeTableTest.java @@ -48,7 +48,7 @@ public void testCreate() throws Exception { NodeTableStorageFactoryImpl nodeTableStorageFactory = new NodeTableStorageFactoryImpl(); Database database = Database.inMemoryDB(); NodeTableStorage nodeTableStorage = - nodeTableStorageFactory.create( + nodeTableStorageFactory.createTable( database, DEFAULT_SERIALIZER, homeNodeSupplier, @@ -75,7 +75,7 @@ public void testFind() throws Exception { NodeTableStorageFactoryImpl nodeTableStorageFactory = new NodeTableStorageFactoryImpl(); Database database = Database.inMemoryDB(); NodeTableStorage nodeTableStorage = - nodeTableStorageFactory.create( + nodeTableStorageFactory.createTable( database, DEFAULT_SERIALIZER, homeNodeSupplier, @@ -117,8 +117,8 @@ public void testFind() throws Exception { List closestNodes = nodeTableStorage.get().findClosestNodes(NODE_ID_FUNCTION.apply(closestNode), 252); assertEquals(2, closestNodes.size()); - assertEquals(closestNodes.get(1).getNode().getPublicKey(), localHostNode.getPublicKey()); - assertEquals(closestNodes.get(0).getNode().getPublicKey(), closestNode.getPublicKey()); + assertEquals(closestNodes.get(0).getNode().getPublicKey(), localHostNode.getPublicKey()); + assertEquals(closestNodes.get(1).getNode().getPublicKey(), closestNode.getPublicKey()); List farNodes = nodeTableStorage.get().findClosestNodes(NODE_ID_FUNCTION.apply(farNode), 1); assertEquals(1, farNodes.size()); From 7e90aefc7bec3f47cb5858085745f1bfbee94001 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Tue, 1 Oct 2019 21:36:37 +0300 Subject: [PATCH 22/77] discover: fix ECDSA signature sign and recover --- .../ethereum/beacon/discovery/Functions.java | 47 +++++++++++++------ 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java b/discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java index 29386a938..402a05f8f 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java @@ -13,9 +13,9 @@ import org.bouncycastle.math.ec.ECCurve; import org.ethereum.beacon.crypto.Hashes; import org.javatuples.Triplet; -import org.web3j.crypto.ECDSASignature; import org.web3j.crypto.ECKeyPair; import org.web3j.crypto.Sign; +import tech.pegasys.artemis.util.bytes.Bytes1; import tech.pegasys.artemis.util.bytes.Bytes32; import tech.pegasys.artemis.util.bytes.Bytes32s; import tech.pegasys.artemis.util.bytes.BytesValue; @@ -25,6 +25,7 @@ import javax.crypto.spec.SecretKeySpec; import java.math.BigInteger; import java.security.SecureRandom; +import java.security.SignatureException; import java.util.Random; public class Functions { @@ -36,22 +37,40 @@ public static Bytes32 hash(BytesValue value) { return Hashes.sha256(value); } - /** Creates a signature of x using the given key */ + /** + * Creates a signature of message `x` using the given key + * + * @param key private key + * @param x message + * @return ECDSA signature with properties merged together: v || r || s + */ public static BytesValue sign(BytesValue key, BytesValue x) { - ECDSASignature signature = ECKeyPair.create(key.extractArray()).sign(x.extractArray()); - Bytes32 r = Bytes32.wrap(signature.r.toByteArray()); - Bytes32 s = Bytes32.wrap(signature.s.toByteArray()); - return r.concat(s); + Sign.SignatureData signatureData = + Sign.signMessage(x.extractArray(), ECKeyPair.create(key.extractArray())); + Bytes1 v = Bytes1.wrap(new byte[] {signatureData.getV()}); + Bytes32 r = Bytes32.wrap(signatureData.getR()); + Bytes32 s = Bytes32.wrap(signatureData.getS()); + return v.concat(r).concat(s); } - public static BytesValue recoverFromSignature(BytesValue signature, BytesValue x) { - assert 64 == signature.size(); - BigInteger r = new BigInteger(signature.slice(0, 32).extractArray()); - BigInteger s = new BigInteger(signature.slice(32).extractArray()); - ECDSASignature ecdsaSignature = new ECDSASignature(r, s); - // FIXME: recId, which number should it use? - BigInteger pubKey = Sign.recoverFromSignature(0, ecdsaSignature, x.extractArray()); - return BytesValue.wrap(pubKey.toByteArray()); + /** + * Recovers public key from message and signature + * + * @param signature Signature, ECDSA + * @param x message + * @return public key + * @throws SignatureException when recovery is not possible + */ + public static BytesValue recoverFromSignature(BytesValue signature, BytesValue x) + throws SignatureException { + BigInteger publicKey = + Sign.signedMessageToKey( + x.extractArray(), + new Sign.SignatureData( + signature.get(0), + signature.slice(1, 33).extractArray(), + signature.slice(33).extractArray())); + return BytesValue.wrap(publicKey.toByteArray()); } /** From f593c22aed928cbc2dd7c153b6ac8702f9b764ae Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Tue, 1 Oct 2019 22:14:23 +0300 Subject: [PATCH 23/77] discovery: remove NodeRecordV5 as V4 is the only available scheme --- .../discovery/DiscoveryManagerImpl.java | 10 +- .../discovery/DiscoveryTaskManager.java | 7 +- .../beacon/discovery/NodeConnectTasks.java | 12 +- .../beacon/discovery/NodeContext.java | 12 +- .../beacon/discovery/enr/NodeRecord.java | 47 ++- .../beacon/discovery/enr/NodeRecordInfo.java | 4 +- .../beacon/discovery/enr/NodeRecordV4.java | 32 +- .../beacon/discovery/enr/NodeRecordV5.java | 379 ------------------ .../discovery/message/DiscoveryV5Message.java | 4 +- .../discovery/message/NodesMessage.java | 10 +- .../discovery/network/DiscoveryClient.java | 8 +- .../discovery/network/NetworkParcelV5.java | 6 +- .../packet/AuthHeaderMessagePacket.java | 8 +- .../storage/NodeTableStorageFactory.java | 4 +- .../storage/NodeTableStorageFactoryImpl.java | 8 +- .../discovery/DiscoveryNetworkTest.java | 13 +- .../discovery/DiscoveryNoNetworkTest.java | 13 +- .../beacon/discovery/NodeRecordTest.java | 47 +-- .../mock/DiscoveryManagerNoNetwork.java | 10 +- .../discovery/storage/NodeBucketTest.java | 8 +- .../discovery/storage/NodeTableTest.java | 47 ++- 21 files changed, 157 insertions(+), 532 deletions(-) delete mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV5.java diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java index fdfaa6c11..2d5f02d15 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java @@ -5,7 +5,7 @@ import org.apache.logging.log4j.Logger; import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.enr.NodeRecordInfo; -import org.ethereum.beacon.discovery.enr.NodeRecordV5; +import org.ethereum.beacon.discovery.enr.NodeRecordV4; import org.ethereum.beacon.discovery.network.DiscoveryClient; import org.ethereum.beacon.discovery.network.DiscoveryServer; import org.ethereum.beacon.discovery.network.DiscoveryServerImpl; @@ -37,7 +37,7 @@ public class DiscoveryManagerImpl implements DiscoveryManager { private final ReplayProcessor outgoingMessages = ReplayProcessor.cacheLast(); private final FluxSink outgoingSink = outgoingMessages.sink(); private final Bytes32 homeNodeId; - private final NodeRecordV5 homeNodeRecord; + private final NodeRecordV4 homeNodeRecord; private final NodeTable nodeTable; private final NodeBucketStorage nodeBucketStorage; private final Map recentContexts = @@ -53,12 +53,12 @@ public class DiscoveryManagerImpl implements DiscoveryManager { public DiscoveryManagerImpl( NodeTable nodeTable, NodeBucketStorage nodeBucketStorage, - NodeRecordV5 homeNode, + NodeRecordV4 homeNode, Scheduler serverScheduler) { this.nodeTable = nodeTable; this.nodeBucketStorage = nodeBucketStorage; this.homeNodeId = homeNode.getNodeId(); - this.homeNodeRecord = (NodeRecordV5) nodeTable.getHomeNode(); + this.homeNodeRecord = (NodeRecordV4) nodeTable.getHomeNode(); this.authTagRepo = new AuthTagRepository(); this.scheduler = serverScheduler; this.discoveryServer = @@ -116,7 +116,7 @@ private Optional getContext(Bytes32 nodeId) { () -> String.format("Couldn't find node record for nodeId %s, ignoring", nodeId)); return Optional.empty(); } - NodeRecordV5 nodeRecord = nodeOptional.get().getNode(); + NodeRecordV4 nodeRecord = nodeOptional.get().getNode(); SecureRandom random = new SecureRandom(); context = new NodeContext( diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryTaskManager.java b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryTaskManager.java index 45bd515dd..18604e689 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryTaskManager.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryTaskManager.java @@ -1,7 +1,7 @@ package org.ethereum.beacon.discovery; import org.ethereum.beacon.discovery.enr.NodeRecordInfo; -import org.ethereum.beacon.discovery.enr.NodeRecordV5; +import org.ethereum.beacon.discovery.enr.NodeRecordV4; import org.ethereum.beacon.discovery.enr.NodeStatus; import org.ethereum.beacon.discovery.storage.NodeBucketStorage; import org.ethereum.beacon.discovery.storage.NodeTable; @@ -14,7 +14,6 @@ import java.util.stream.Stream; import static org.ethereum.beacon.discovery.NodeContext.DEFAULT_DISTANCE; -import static org.ethereum.beacon.discovery.enr.NodeRecordV5.NODE_ID_FUNCTION; /** Manages recurrent node check task(s) */ public class DiscoveryTaskManager { @@ -80,13 +79,13 @@ public DiscoveryTaskManager( DiscoveryManager discoveryManager, NodeTable nodeTable, NodeBucketStorage nodeBucketStorage, - NodeRecordV5 homeNode, + NodeRecordV4 homeNode, Scheduler scheduler, boolean resetDead) { this.scheduler = scheduler; this.nodeTable = nodeTable; this.nodeBucketStorage = nodeBucketStorage; - this.homeNodeId = NODE_ID_FUNCTION.apply(homeNode); + this.homeNodeId = homeNode.getNodeId(); this.nodeConnectTasks = new NodeConnectTasks( discoveryManager, scheduler, Duration.ofSeconds(RETRY_TIMEOUT_SECONDS)); diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeConnectTasks.java b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeConnectTasks.java index 0c2d5025e..f86af9685 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeConnectTasks.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeConnectTasks.java @@ -13,8 +13,6 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; -import static org.ethereum.beacon.discovery.enr.NodeRecordV5.NODE_ID_FUNCTION; - /** * Executes tasks {@link DiscoveryManager#connect(NodeRecord)} for all NodeRecords added via {@link * #add(NodeRecordInfo, Runnable, Runnable)}. Tasks is called failed if timeout is reached and reply @@ -36,27 +34,27 @@ public NodeConnectTasks( public void add(NodeRecordInfo nodeRecordInfo, Runnable successCallback, Runnable failCallback) { synchronized (this) { - if (currentTasks.contains(NODE_ID_FUNCTION.apply(nodeRecordInfo.getNode()))) { + if (currentTasks.contains(nodeRecordInfo.getNode().getNodeId())) { return; } - currentTasks.add(NODE_ID_FUNCTION.apply(nodeRecordInfo.getNode())); + currentTasks.add(nodeRecordInfo.getNode().getNodeId()); } scheduler.execute( () -> { CompletableFuture retry = discoveryManager.connect(nodeRecordInfo.getNode()); taskTimeouts.put( - NODE_ID_FUNCTION.apply(nodeRecordInfo.getNode()), + nodeRecordInfo.getNode().getNodeId(), () -> retry.completeExceptionally(new RuntimeException("Timeout for node check task"))); retry.whenComplete( (aVoid, throwable) -> { if (throwable != null) { failCallback.run(); - currentTasks.remove(NODE_ID_FUNCTION.apply(nodeRecordInfo.getNode())); + currentTasks.remove(nodeRecordInfo.getNode().getNodeId()); } else { successCallback.run(); - currentTasks.remove(NODE_ID_FUNCTION.apply(nodeRecordInfo.getNode())); + currentTasks.remove(nodeRecordInfo.getNode().getNodeId()); } }); }); diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeContext.java b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeContext.java index 8f4390b0d..a297882f2 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeContext.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeContext.java @@ -3,7 +3,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.ethereum.beacon.discovery.enr.NodeRecord; -import org.ethereum.beacon.discovery.enr.NodeRecordV5; +import org.ethereum.beacon.discovery.enr.NodeRecordV4; import org.ethereum.beacon.discovery.message.DiscoveryMessage; import org.ethereum.beacon.discovery.message.MessageCode; import org.ethereum.beacon.discovery.packet.AuthHeaderMessagePacket; @@ -36,8 +36,8 @@ public class NodeContext { public static final int DEFAULT_DISTANCE = 10; // FIXME: I shouldn't be here private static final Logger logger = LogManager.getLogger(NodeContext.class); - private final NodeRecordV5 nodeRecord; - private final NodeRecordV5 homeNodeRecord; + private final NodeRecordV4 nodeRecord; + private final NodeRecordV4 homeNodeRecord; private final Bytes32 homeNodeId; private final AuthTagRepository authTagRepo; private final NodeTable nodeTable; @@ -53,8 +53,8 @@ public class NodeContext { private CompletableFuture connectFuture = null; public NodeContext( - NodeRecordV5 nodeRecord, - NodeRecordV5 homeNodeRecord, + NodeRecordV4 nodeRecord, + NodeRecordV4 homeNodeRecord, NodeTable nodeTable, NodeBucketStorage nodeBucketStorage, AuthTagRepository authTagRepo, @@ -75,7 +75,7 @@ public NodeContext( packetHandlers.put(MessagePacket.class, new MessagePacketHandler(this, logger)); } - public NodeRecordV5 getNodeRecord() { + public NodeRecordV4 getNodeRecord() { return nodeRecord; } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecord.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecord.java index d63598bca..96569e3a9 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecord.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecord.java @@ -1,6 +1,5 @@ package org.ethereum.beacon.discovery.enr; -import org.ethereum.beacon.discovery.IdentityScheme; import org.web3j.rlp.RlpDecoder; import org.web3j.rlp.RlpList; import org.web3j.rlp.RlpString; @@ -10,8 +9,10 @@ import tech.pegasys.artemis.util.uint.UInt64; import java.util.Base64; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; /** * Ethereum Node Record @@ -56,7 +57,7 @@ static NodeRecord fromBytes(byte[] bytes) { } RlpString idVersion = (RlpString) values.get(3); - IdentityScheme nodeIdentity = IdentityScheme.fromString(new String(idVersion.getBytes())); + EnrScheme nodeIdentity = EnrScheme.fromString(new String(idVersion.getBytes())); if (nodeIdentity == null) { throw new RuntimeException( String.format( @@ -68,10 +69,6 @@ static NodeRecord fromBytes(byte[] bytes) { { return NodeRecordV4.fromRlpList(values); } - case V5: - { - return NodeRecordV5.fromRlpList(values); - } default: { throw new RuntimeException( @@ -84,8 +81,8 @@ static NodeRecord fromBytes(BytesValue bytes) { return fromBytes(bytes.extractArray()); } - /** Every {@link IdentityScheme} links with its own implementation */ - IdentityScheme getIdentityScheme(); + /** Every {@link EnrScheme} links with its own implementation */ + EnrScheme getIdentityScheme(); BytesValue getSignature(); @@ -107,5 +104,37 @@ default String asBase64() { * All optional fields, set varies by identity scheme. `id`, identity scheme is not optional. Most * common field names are in constants: {@link #FIELD_IP_V4} etc. (FIELD_*) */ - Map getFields(); + Set getKeys(); + + /** + * @return key value or null, if no field associated with that key. Get keys using {@link + * #getKeys()} + */ + Object getKey(String key); + + enum EnrScheme { + V4("v4"); + + private static final Map nameMap = new HashMap<>(); + + static { + for (EnrScheme scheme : EnrScheme.values()) { + nameMap.put(scheme.name, scheme); + } + } + + private String name; + + private EnrScheme(String name) { + this.name = name; + } + + public static EnrScheme fromString(String name) { + return nameMap.get(name); + } + + public String stringName() { + return name; + } + } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordInfo.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordInfo.java index eb789b03f..72cb86d95 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordInfo.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordInfo.java @@ -52,8 +52,8 @@ public BytesValue toRlpBytes() { return BytesValue.wrap(bytes); } - public NodeRecordV5 getNode() { - return (NodeRecordV5) node; + public NodeRecordV4 getNode() { + return (NodeRecordV4) node; } public Long getLastRetry() { diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV4.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV4.java index e09e129e7..ebb7bb58a 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV4.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV4.java @@ -2,7 +2,6 @@ import com.google.common.base.Objects; import org.ethereum.beacon.crypto.Hashes; -import org.ethereum.beacon.discovery.IdentityScheme; import org.javatuples.Pair; import org.web3j.rlp.RlpEncoder; import org.web3j.rlp.RlpList; @@ -18,14 +17,16 @@ import java.math.BigInteger; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Function; -/** Node record for V4 protocol */ +/** Node record for V4 scheme. Uses secp256k1 as signature */ public class NodeRecordV4 implements NodeRecord { // id name of identity scheme, e.g. “v4” - private static final IdentityScheme identityScheme = IdentityScheme.V4; + private static final EnrScheme identityScheme = EnrScheme.V4; private static final Function, NodeRecordV4> nodeRecordV4Creator = fields -> { NodeRecordV4.Builder builder = @@ -34,11 +35,17 @@ public class NodeRecordV4 implements NodeRecord { .withSeq( UInt64.fromBytesBigEndian( Bytes8.leftPad(BytesValue.wrap(((RlpString) fields.get(1)).getBytes())))); + boolean secp256k1Found = false; for (int i = 4; i < fields.size(); i += 2) { - builder = - builder.withKeyField( - new String(((RlpString) fields.get(i)).getBytes()), - (RlpString) fields.get(i + 1)); + String key = new String(((RlpString) fields.get(i)).getBytes()); + if (key.equals(FIELD_PKEY_SECP256K1)) { + secp256k1Found = true; + } + builder = builder.withKeyField(key, (RlpString) fields.get(i + 1)); + } + if (!secp256k1Found) { + throw new RuntimeException( + String.format("NodeRecord V4 requires %s field", FIELD_PKEY_SECP256K1)); } return builder.build(); @@ -98,7 +105,7 @@ public static NodeRecordV4 fromRlpList(List values) { return nodeRecordV4Creator.apply(values); } - public IdentityScheme getIdentityScheme() { + public EnrScheme getIdentityScheme() { return identityScheme; } @@ -178,8 +185,13 @@ public void setSignature(BytesValue signature) { } @Override - public Map getFields() { - return fields; + public Set getKeys() { + return new HashSet<>(fields.keySet()); + } + + @Override + public Object getKey(String key) { + return fields.get(key); } @Override diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV5.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV5.java deleted file mode 100644 index 2799bc944..000000000 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV5.java +++ /dev/null @@ -1,379 +0,0 @@ -package org.ethereum.beacon.discovery.enr; - -import com.google.common.base.Objects; -import org.ethereum.beacon.crypto.Hashes; -import org.ethereum.beacon.discovery.IdentityScheme; -import org.javatuples.Pair; -import org.web3j.rlp.RlpEncoder; -import org.web3j.rlp.RlpList; -import org.web3j.rlp.RlpString; -import org.web3j.rlp.RlpType; -import tech.pegasys.artemis.util.bytes.Bytes16; -import tech.pegasys.artemis.util.bytes.Bytes32; -import tech.pegasys.artemis.util.bytes.Bytes4; -import tech.pegasys.artemis.util.bytes.Bytes8; -import tech.pegasys.artemis.util.bytes.Bytes96; -import tech.pegasys.artemis.util.bytes.BytesValue; -import tech.pegasys.artemis.util.uint.UInt64; - -import java.math.BigInteger; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.function.Function; - -/** Node record for V5 protocol */ -public class NodeRecordV5 implements NodeRecord { - // id name of identity scheme, e.g. “v4” - private static final IdentityScheme identityScheme = IdentityScheme.V5; - private static final Function, NodeRecordV5> nodeRecordV5Creator = - fields -> { - NodeRecordV5.Builder builder = - NodeRecordV5.Builder.empty() - .withSignature(BytesValue.wrap(((RlpString) fields.get(0)).getBytes())) - .withSeq( - UInt64.fromBytesBigEndian( - Bytes8.leftPad(BytesValue.wrap(((RlpString) fields.get(1)).getBytes())))); - for (int i = 4; i < fields.size(); i += 2) { - builder = - builder.withKeyField( - new String(((RlpString) fields.get(i)).getBytes()), - (RlpString) fields.get(i + 1)); - } - - return builder.build(); - }; - public static Function NODE_ID_FUNCTION = - nodeRecordV5 -> Hashes.sha256(nodeRecordV5.getPublicKey()); - private UInt64 seq; - // Signature - private BytesValue signature; - // optional fields - private Map fields = new HashMap<>(); - - private NodeRecordV5( - BytesValue publicKey, - Bytes4 ipV4address, - Integer tcpPort, - Integer udpPort, - Bytes16 ipV6address, - Integer tcpV6Port, - Integer udpV6Port, - UInt64 seq, - BytesValue signature) { - fields.put(FIELD_PKEY_SECP256K1, publicKey); - fields.put(FIELD_IP_V4, ipV4address); - fields.put(FIELD_TCP_V4, tcpPort); - fields.put(FIELD_UDP_V4, udpPort); - fields.put(FIELD_IP_V6, ipV6address); - fields.put(FIELD_TCP_V6, tcpV6Port); - fields.put(FIELD_UDP_V6, udpV6Port); - this.seq = seq; - this.signature = signature; - } - - private NodeRecordV5() {} - - public NodeRecordV5(BytesValue bytes) { - NodeRecord.fromBytes(bytes); - } - - public static NodeRecordV5 fromValues( - BytesValue publicKey, - Bytes4 ipV4address, - Integer tcpPort, - Integer udpPort, - Bytes16 ipV6address, - Integer tcpV6Port, - Integer udpV6Port, - UInt64 seq, - BytesValue signature) { - return new NodeRecordV5( - publicKey, - ipV4address, - tcpPort, - udpPort, - ipV6address, - tcpV6Port, - udpV6Port, - seq, - signature); - } - - public static NodeRecordV5 fromRlpList(List values) { - return nodeRecordV5Creator.apply(values); - } - - public IdentityScheme getIdentityScheme() { - return identityScheme; - } - - public BytesValue getPublicKey() { - return fields.containsKey(FIELD_PKEY_SECP256K1) - ? (BytesValue) fields.get(FIELD_PKEY_SECP256K1) - : null; - } - - public void setPublicKey(BytesValue publicKey) { - fields.put(FIELD_PKEY_SECP256K1, publicKey); - } - - public Bytes4 getIpV4address() { - return fields.containsKey(FIELD_IP_V4) ? (Bytes4) fields.get(FIELD_IP_V4) : null; - } - - public void setIpV4address(Bytes4 ipV4address) { - fields.put(FIELD_IP_V4, ipV4address); - } - - public Integer getTcpPort() { - return fields.containsKey(FIELD_TCP_V4) ? (Integer) fields.get(FIELD_TCP_V4) : null; - } - - public void setTcpPort(Integer tcpPort) { - fields.put(FIELD_TCP_V4, tcpPort); - } - - public Integer getUdpPort() { - return fields.containsKey(FIELD_UDP_V4) ? (Integer) fields.get(FIELD_UDP_V4) : null; - } - - public void setUdpPort(Integer udpPort) { - fields.put(FIELD_UDP_V4, udpPort); - } - - public Bytes16 getIpV6address() { - return fields.containsKey(FIELD_IP_V6) ? (Bytes16) fields.get(FIELD_IP_V6) : null; - } - - public void setIpV6address(Bytes16 ipV6address) { - fields.put(FIELD_IP_V6, ipV6address); - } - - public Integer getTcpV6Port() { - return fields.containsKey(FIELD_TCP_V6) ? (Integer) fields.get(FIELD_TCP_V6) : null; - } - - public void setTcpV6Port(Integer tcpV6Port) { - fields.put(FIELD_TCP_V6, tcpV6Port); - } - - public Integer getUdpV6Port() { - return fields.containsKey(FIELD_UDP_V6) ? (Integer) fields.get(FIELD_UDP_V6) : null; - } - - public void setUdpV6Port(Integer udpV6Port) { - fields.put(FIELD_UDP_V6, udpV6Port); - } - - public UInt64 getSeq() { - return seq; - } - - public void setSeq(UInt64 seq) { - this.seq = seq; - } - - @Override - public BytesValue getSignature() { - return signature; - } - - public void setSignature(BytesValue signature) { - this.signature = signature; - } - - @Override - public Map getFields() { - return fields; - } - - /** Uses empty 96 bytes signature when no signature is provided\ */ - @Override - public BytesValue serialize() { - // content = [seq, k, v, ...] - // signature = sign(content) - // record = [signature, seq, k, v, ...] - List values = new ArrayList<>(); - if (getSignature() != null) { - values.add(RlpString.create(getSignature().extractArray())); - } else { - values.add(RlpString.create(Bytes96.ZERO.extractArray())); // FIXME: is it ok? - } - values.add(RlpString.create(getSeq().toBI())); - values.add(RlpString.create("id")); - values.add(RlpString.create(getIdentityScheme().stringName())); - for (Map.Entry keyPair : fields.entrySet()) { - if (keyPair.getValue() == null) { - continue; - } - values.add(RlpString.create(keyPair.getKey())); - if (keyPair.getValue() instanceof BytesValue) { - values.add(fromBytes((BytesValue) keyPair.getValue())); - } else if (keyPair.getValue() instanceof Number) { - values.add(fromNumber((Number) keyPair.getValue())); - } else if (keyPair.getValue() == null) { - values.add(RlpString.create(new byte[0])); - } else { - throw new RuntimeException( - String.format( - "Couldn't serialize node record field %s with value %s: no serializer found.", - keyPair.getKey(), keyPair.getValue())); - } - } - - byte[] bytes = RlpEncoder.encode(new RlpList(values)); - assert bytes.length <= 300; - return BytesValue.wrap(bytes); - } - - private RlpString fromNumber(Number number) { - if (number instanceof BigInteger) { - return RlpString.create((BigInteger) number); - } else if (number instanceof Long) { - return RlpString.create((Long) number); - } else if (number instanceof Integer) { - return RlpString.create((Integer) number); - } else { - throw new RuntimeException( - String.format("Couldn't serialize number %s : no serializer found.", number)); - } - } - - private RlpString fromBytes(BytesValue bytes) { - return RlpString.create(bytes.extractArray()); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - NodeRecordV5 that = (NodeRecordV5) o; - return Objects.equal(seq, that.seq) - && Objects.equal(signature, that.signature) - && Objects.equal(fields, that.fields); - } - - @Override - public int hashCode() { - return Objects.hashCode(seq, signature, fields); - } - - @Override - public Bytes32 getNodeId() { - return NODE_ID_FUNCTION.apply(this); - } - - @Override - public String toString() { - return "NodeRecordV5{" - + "publicKey=" - + fields.get(FIELD_PKEY_SECP256K1) - + ", ipV4address=" - + fields.get(FIELD_IP_V4) - + ", udpPort=" - + fields.get(FIELD_UDP_V4) - + '}'; - } - - public static class Builder { - private static final Map, Builder>> fieldFillersV5 = - new HashMap<>(); - - static { - fieldFillersV5.put( - FIELD_IP_V4, - objects -> - objects - .getValue0() - .withIpV4Address(BytesValue.wrap(((RlpString) objects.getValue1()).getBytes()))); - fieldFillersV5.put( - FIELD_PKEY_SECP256K1, - objects -> - objects - .getValue0() - .withSecp256k1(BytesValue.wrap(((RlpString) objects.getValue1()).getBytes()))); - fieldFillersV5.put( - FIELD_UDP_V4, - objects -> - objects - .getValue0() - .withUdpPort( - ((RlpString) objects.getValue1()).asPositiveBigInteger().intValue())); - fieldFillersV5.put( - FIELD_TCP_V4, - objects -> - objects - .getValue0() - .withTcpPort( - ((RlpString) objects.getValue1()).asPositiveBigInteger().intValue())); - } - - private Bytes4 ipV4Address; - private BytesValue secp256k1; - private Integer tcpPort; - private Integer udpPort; - private UInt64 seqNumber; - private BytesValue signature; - - private Builder() {} - - public static Builder empty() { - return new Builder(); - } - - public Builder withIpV4Address(BytesValue ipV4Address) { - this.ipV4Address = Bytes4.wrap(ipV4Address, 0); - return this; - } - - public Builder withTcpPort(Integer port) { - this.tcpPort = port; - return this; - } - - public Builder withUdpPort(Integer port) { - this.udpPort = port; - return this; - } - - public Builder withSecp256k1(BytesValue bytes) { - this.secp256k1 = bytes; - return this; - } - - public Builder withSeq(UInt64 seq) { - this.seqNumber = seq; - return this; - } - - public Builder withSignature(BytesValue signature) { - this.signature = signature; - return this; - } - - public Builder withKeyField(String key, RlpString value) { - Function, NodeRecordV5.Builder> fieldFiller = - fieldFillersV5.get(key); - if (fieldFiller == null) { - throw new RuntimeException(String.format("Couldn't find filler for V5 field '%s'", key)); - } - return fieldFiller.apply(Pair.with(this, value)); - } - - public NodeRecordV5 build() { - assert seqNumber != null; - assert secp256k1 != null; - - NodeRecordV5 nodeRecord = new NodeRecordV5(); - nodeRecord.setIpV4address(ipV4Address); - nodeRecord.setUdpPort(udpPort); - nodeRecord.setTcpPort(tcpPort); - nodeRecord.setSeq(seqNumber); - nodeRecord.setSignature(signature); - nodeRecord.setPublicKey(secp256k1); - return nodeRecord; - } - } -} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/DiscoveryV5Message.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/DiscoveryV5Message.java index 840482683..521eb98a5 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/message/DiscoveryV5Message.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/DiscoveryV5Message.java @@ -2,7 +2,7 @@ import org.ethereum.beacon.discovery.IdentityScheme; import org.ethereum.beacon.discovery.enr.NodeRecord; -import org.ethereum.beacon.discovery.enr.NodeRecordV5; +import org.ethereum.beacon.discovery.enr.NodeRecordV4; import org.web3j.rlp.RlpDecoder; import org.web3j.rlp.RlpList; import org.web3j.rlp.RlpString; @@ -87,7 +87,7 @@ public V5Message create() { ((RlpString) payload.get(1)).asPositiveBigInteger().intValueExact(), () -> nodeRecords.getValues().stream() - .map(rs -> (NodeRecordV5) (NodeRecord.fromBytes(((RlpString) rs).getBytes()))) + .map(rs -> (NodeRecordV4) (NodeRecord.fromBytes(((RlpString) rs).getBytes()))) .collect(Collectors.toList()), nodeRecords.getValues().size()); } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/NodesMessage.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/NodesMessage.java index bed61fea7..ae31b7676 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/message/NodesMessage.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/NodesMessage.java @@ -1,7 +1,7 @@ package org.ethereum.beacon.discovery.message; import com.google.common.base.Objects; -import org.ethereum.beacon.discovery.enr.NodeRecordV5; +import org.ethereum.beacon.discovery.enr.NodeRecordV4; import org.web3j.rlp.RlpEncoder; import org.web3j.rlp.RlpList; import org.web3j.rlp.RlpString; @@ -22,15 +22,15 @@ public class NodesMessage implements V5Message { // Total number of responses to the request private final Integer total; // List of nodes upon request - private final Supplier> nodeRecordsSupplier; + private final Supplier> nodeRecordsSupplier; // Size of nodes in current response private final Integer nodeRecordsSize; - private List nodeRecords = null; + private List nodeRecords = null; public NodesMessage( BytesValue requestId, Integer total, - Supplier> nodeRecordsSupplier, + Supplier> nodeRecordsSupplier, Integer nodeRecordsSize) { this.requestId = requestId; this.total = total; @@ -47,7 +47,7 @@ public Integer getTotal() { return total; } - public synchronized List getNodeRecords() { + public synchronized List getNodeRecords() { if (nodeRecords == null) { this.nodeRecords = nodeRecordsSupplier.get(); } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/network/DiscoveryClient.java b/discovery/src/main/java/org/ethereum/beacon/discovery/network/DiscoveryClient.java index 67e1abd24..8cb0be78c 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/network/DiscoveryClient.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/network/DiscoveryClient.java @@ -6,7 +6,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.ethereum.beacon.discovery.enr.NodeRecord; -import org.ethereum.beacon.discovery.enr.NodeRecordV5; +import org.ethereum.beacon.discovery.enr.NodeRecordV4; import org.reactivestreams.Publisher; import reactor.core.publisher.Flux; import tech.pegasys.artemis.util.bytes.BytesValue; @@ -35,7 +35,7 @@ public DiscoveryClient(NioDatagramChannel channel, Publisher outg } private void send(BytesValue data, NodeRecord recipient) { - if (!(recipient instanceof NodeRecordV5)) { + if (!(recipient instanceof NodeRecordV4)) { String error = String.format( "Accepts only V5 versions of recipient's node records. Got %s instead", recipient); @@ -46,8 +46,8 @@ private void send(BytesValue data, NodeRecord recipient) { try { address = new InetSocketAddress( - InetAddress.getByAddress(((NodeRecordV5) recipient).getIpV4address().extractArray()), - ((NodeRecordV5) recipient).getUdpPort()); + InetAddress.getByAddress(((NodeRecordV4) recipient).getIpV4address().extractArray()), + ((NodeRecordV4) recipient).getUdpPort()); } catch (UnknownHostException e) { String error = String.format("Failed to resolve host for node record: %s", recipient); logger.error(error); diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/network/NetworkParcelV5.java b/discovery/src/main/java/org/ethereum/beacon/discovery/network/NetworkParcelV5.java index 8d36e8178..997033154 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/network/NetworkParcelV5.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/network/NetworkParcelV5.java @@ -1,14 +1,14 @@ package org.ethereum.beacon.discovery.network; import org.ethereum.beacon.discovery.enr.NodeRecord; -import org.ethereum.beacon.discovery.enr.NodeRecordV5; +import org.ethereum.beacon.discovery.enr.NodeRecordV4; import org.ethereum.beacon.discovery.packet.Packet; public class NetworkParcelV5 implements NetworkParcel { private final Packet packet; - private final NodeRecordV5 nodeRecord; + private final NodeRecordV4 nodeRecord; - public NetworkParcelV5(Packet packet, NodeRecordV5 nodeRecord) { + public NetworkParcelV5(Packet packet, NodeRecordV4 nodeRecord) { this.packet = packet; this.nodeRecord = nodeRecord; } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/AuthHeaderMessagePacket.java b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/AuthHeaderMessagePacket.java index 5bca3ec0c..6f98544bd 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/AuthHeaderMessagePacket.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/AuthHeaderMessagePacket.java @@ -2,7 +2,7 @@ import org.ethereum.beacon.discovery.Functions; import org.ethereum.beacon.discovery.enr.NodeRecord; -import org.ethereum.beacon.discovery.enr.NodeRecordV5; +import org.ethereum.beacon.discovery.enr.NodeRecordV4; import org.ethereum.beacon.discovery.message.DiscoveryMessage; import org.ethereum.beacon.discovery.message.DiscoveryV5Message; import org.web3j.rlp.RlpDecoder; @@ -122,7 +122,7 @@ public BytesValue getIdNonceSig() { return decoded.idNonceSig; } - public NodeRecordV5 getNodeRecord() { + public NodeRecordV4 getNodeRecord() { verifyDecode(); return decoded.nodeRecord; } @@ -165,7 +165,7 @@ public void decode(BytesValue initiatorKey, BytesValue authResponseKey) { blank.idNonceSig = BytesValue.wrap(((RlpString) authResponsePtParts.getValues().get(1)).getBytes()); blank.nodeRecord = - (NodeRecordV5) + (NodeRecordV4) NodeRecord.fromBytes(((RlpString) authResponsePtParts.getValues().get(2)).getBytes()); BytesValue messageAad = blank.tag.concat(getBytes().slice(32)); blank.message = @@ -205,7 +205,7 @@ private static class MessagePacketDecoded { private BytesValue idNonce; private BytesValue ephemeralPubkey; private BytesValue idNonceSig; - private NodeRecordV5 nodeRecord; + private NodeRecordV4 nodeRecord; private DiscoveryMessage message; } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactory.java b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactory.java index 559f98acf..28856366f 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactory.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactory.java @@ -3,7 +3,7 @@ import org.ethereum.beacon.chain.storage.impl.SerializerFactory; import org.ethereum.beacon.db.Database; import org.ethereum.beacon.discovery.enr.NodeRecord; -import org.ethereum.beacon.discovery.enr.NodeRecordV5; +import org.ethereum.beacon.discovery.enr.NodeRecordV4; import tech.pegasys.artemis.util.bytes.Bytes32; import java.util.List; @@ -13,7 +13,7 @@ public interface NodeTableStorageFactory { NodeTableStorage createTable( Database database, SerializerFactory serializerFactory, - Supplier homeNodeSupplier, + Supplier homeNodeSupplier, Supplier> bootNodes); NodeBucketStorage createBuckets( diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactoryImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactoryImpl.java index 7ce3428c9..17b159741 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactoryImpl.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactoryImpl.java @@ -4,7 +4,7 @@ import org.ethereum.beacon.db.Database; import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.enr.NodeRecordInfo; -import org.ethereum.beacon.discovery.enr.NodeRecordV5; +import org.ethereum.beacon.discovery.enr.NodeRecordV4; import tech.pegasys.artemis.util.bytes.Bytes32; import java.util.List; @@ -25,7 +25,7 @@ private boolean isStorageEmpty(NodeTableStorage nodeTableStorage) { public NodeTableStorage createTable( Database database, SerializerFactory serializerFactory, - Supplier homeNodeSupplier, + Supplier homeNodeSupplier, Supplier> bootNodesSupplier) { NodeTableStorage nodeTableStorage = new NodeTableStorageImpl(database, serializerFactory); @@ -38,8 +38,8 @@ public NodeTableStorage createTable( .get() .forEach( nodeRecord -> { - if (!(nodeRecord instanceof NodeRecordV5)) { - throw new RuntimeException("Only V5 node records are supported as boot nodes"); + if (!(nodeRecord instanceof NodeRecordV4)) { + throw new RuntimeException("Only V4 node records are supported as boot nodes"); } NodeRecordInfo nodeRecordInfo = NodeRecordInfo.createDefault(nodeRecord); nodeTableStorage.get().save(nodeRecordInfo); diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java index 81d45c593..884d8bbc6 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java @@ -2,7 +2,7 @@ import org.ethereum.beacon.db.Database; import org.ethereum.beacon.discovery.enr.NodeRecord; -import org.ethereum.beacon.discovery.enr.NodeRecordV5; +import org.ethereum.beacon.discovery.enr.NodeRecordV4; import org.ethereum.beacon.discovery.message.DiscoveryMessage; import org.ethereum.beacon.discovery.message.DiscoveryV5Message; import org.ethereum.beacon.discovery.message.FindNodeMessage; @@ -18,6 +18,7 @@ import org.ethereum.beacon.schedulers.Schedulers; import org.junit.Test; import reactor.core.publisher.Flux; +import tech.pegasys.artemis.util.bytes.Bytes96; import tech.pegasys.artemis.util.bytes.BytesValue; import tech.pegasys.artemis.util.uint.UInt64; @@ -34,23 +35,25 @@ public class DiscoveryNetworkTest { @Test public void test() throws Exception { // 1) start 2 nodes - NodeRecordV5 nodeRecord1 = - NodeRecordV5.Builder.empty() + NodeRecordV4 nodeRecord1 = + NodeRecordV4.Builder.empty() .withIpV4Address(BytesValue.wrap(InetAddress.getByName("127.0.0.1").getAddress())) .withSeq(UInt64.valueOf(1)) .withUdpPort(30303) .withSecp256k1( BytesValue.fromHexString( "0bfb48004b1698f05872cf18b1f278998ad8f7d4c135aa41f83744e7b850ab6b98")) + .withSignature(Bytes96.EMPTY) .build(); - NodeRecordV5 nodeRecord2 = - NodeRecordV5.Builder.empty() + NodeRecordV4 nodeRecord2 = + NodeRecordV4.Builder.empty() .withIpV4Address(BytesValue.wrap(InetAddress.getByName("127.0.0.1").getAddress())) .withSeq(UInt64.valueOf(1)) .withUdpPort(30304) .withSecp256k1( BytesValue.fromHexString( "7ef3502240a42891771de732f5ee6bee3eb881939edf3e6008c0d07b502756e426")) + .withSignature(Bytes96.EMPTY) .build(); NodeTableStorageFactoryImpl nodeTableStorageFactory = new NodeTableStorageFactoryImpl(); Database database1 = Database.inMemoryDB(); diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java index 42f93e632..a0804934d 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java @@ -2,7 +2,7 @@ import org.ethereum.beacon.db.Database; import org.ethereum.beacon.discovery.enr.NodeRecord; -import org.ethereum.beacon.discovery.enr.NodeRecordV5; +import org.ethereum.beacon.discovery.enr.NodeRecordV4; import org.ethereum.beacon.discovery.message.DiscoveryMessage; import org.ethereum.beacon.discovery.message.DiscoveryV5Message; import org.ethereum.beacon.discovery.message.FindNodeMessage; @@ -20,6 +20,7 @@ import org.ethereum.beacon.stream.SimpleProcessor; import org.junit.Test; import reactor.core.publisher.Flux; +import tech.pegasys.artemis.util.bytes.Bytes96; import tech.pegasys.artemis.util.bytes.BytesValue; import tech.pegasys.artemis.util.uint.UInt64; @@ -39,23 +40,25 @@ public class DiscoveryNoNetworkTest { @Test public void test() throws Exception { // 1) start 2 nodes - NodeRecordV5 nodeRecord1 = - NodeRecordV5.Builder.empty() + NodeRecordV4 nodeRecord1 = + NodeRecordV4.Builder.empty() .withIpV4Address(BytesValue.wrap(InetAddress.getByName("127.0.0.1").getAddress())) .withSeq(UInt64.valueOf(1)) .withUdpPort(30303) .withSecp256k1( BytesValue.fromHexString( "0bfb48004b1698f05872cf18b1f278998ad8f7d4c135aa41f83744e7b850ab6b98")) + .withSignature(Bytes96.EMPTY) .build(); - NodeRecordV5 nodeRecord2 = - NodeRecordV5.Builder.empty() + NodeRecordV4 nodeRecord2 = + NodeRecordV4.Builder.empty() .withIpV4Address(BytesValue.wrap(InetAddress.getByName("192.168.0.1").getAddress())) .withSeq(UInt64.valueOf(1)) .withUdpPort(30303) .withSecp256k1( BytesValue.fromHexString( "7ef3502240a42891771de732f5ee6bee3eb881939edf3e6008c0d07b502756e426")) + .withSignature(Bytes96.EMPTY) .build(); NodeTableStorageFactoryImpl nodeTableStorageFactory = new NodeTableStorageFactoryImpl(); Database database1 = Database.inMemoryDB(); diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/NodeRecordTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/NodeRecordTest.java index fd9f3c697..2d34bc8d2 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/NodeRecordTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/NodeRecordTest.java @@ -2,10 +2,7 @@ import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.enr.NodeRecordV4; -import org.ethereum.beacon.discovery.enr.NodeRecordV5; import org.junit.Test; -import tech.pegasys.artemis.util.bytes.Bytes4; -import tech.pegasys.artemis.util.bytes.Bytes96; import tech.pegasys.artemis.util.bytes.BytesValue; import tech.pegasys.artemis.util.uint.UInt64; @@ -39,7 +36,7 @@ public void testLocalhostV4() throws Exception { "-IS4QHCYrYZbAKWCBRlAy5zzaDZXJBGkcnh4MHcBFZntXNFrdvJjX04jRzjzCBOonrkTfj499SZuOh8R33Ls8RRcy5wBgmlkgnY0gmlwhH8AAAGJc2VjcDI1NmsxoQPKY0yuDUmstAHYpMa2_oxVtw0RW_QAdpzBQA8yWM0xOIN1ZHCCdl8"; NodeRecord nodeRecord = NodeRecord.fromBase64(localhostEnr); - assertEquals(IdentityScheme.V4, nodeRecord.getIdentityScheme()); + assertEquals(NodeRecord.EnrScheme.V4, nodeRecord.getIdentityScheme()); NodeRecordV4 nodeRecordV4 = (NodeRecordV4) nodeRecord; assertArrayEquals( InetAddress.getByName(expectedHost).getAddress(), @@ -54,7 +51,7 @@ public void testLocalhostV4() throws Exception { // The order of fields is not strict so we don't compare strings NodeRecord nodeRecordRestored = NodeRecord.fromBase64(localhostEnrRestored); - assertEquals(IdentityScheme.V4, nodeRecordRestored.getIdentityScheme()); + assertEquals(NodeRecord.EnrScheme.V4, nodeRecordRestored.getIdentityScheme()); NodeRecordV4 nodeRecordV4Restored = (NodeRecordV4) nodeRecordRestored; assertArrayEquals( InetAddress.getByName(expectedHost).getAddress(), @@ -65,44 +62,4 @@ public void testLocalhostV4() throws Exception { assertEquals(expectedPublicKey, nodeRecordV4Restored.getPublicKey()); assertEquals(expectedSignature, nodeRecordV4Restored.getSignature()); } - - @Test - public void testLocalhostV5() throws Exception { - final String expectedHost = "127.0.0.1"; - final Integer expectedUdpPort = 30303; - final Integer expectedTcpPort = null; - final UInt64 expectedSeqNumber = UInt64.valueOf(1); - final BytesValue expectedPublicKey = - BytesValue.fromHexString( - "03ca634cae0d49acb401d8a4c6b6fe8c55b70d115bf400769cc1400f3258cd3138"); - final BytesValue expectedSignature = - BytesValue.fromHexString( - "7098ad865b00a582051940cb9cf36836572411a47278783077011599ed5cd16b76f2635f4e234738f30813a89eb9137e3e3df5266e3a1f11df72ecf1145ccb9c"); - - NodeRecordV5 nodeRecordV5 = - NodeRecordV5.fromValues( - expectedPublicKey, - Bytes4.wrap(InetAddress.getByName(expectedHost).getAddress()), - expectedTcpPort, - expectedUdpPort, - null, - null, - null, - UInt64.valueOf(1), - Bytes96.ZERO); - nodeRecordV5.setSeq(expectedSeqNumber); - nodeRecordV5.setSignature(expectedSignature); - - String enrV5 = nodeRecordV5.asBase64(); - NodeRecordV5 nodeRecordV5Restored = (NodeRecordV5) NodeRecord.fromBase64(enrV5); - assertEquals(IdentityScheme.V5, nodeRecordV5Restored.getIdentityScheme()); - assertArrayEquals( - InetAddress.getByName(expectedHost).getAddress(), - nodeRecordV5Restored.getIpV4address().extractArray()); - assertEquals(expectedUdpPort, nodeRecordV5Restored.getUdpPort()); - assertEquals(expectedTcpPort, nodeRecordV5Restored.getTcpPort()); - assertEquals(expectedSeqNumber, nodeRecordV5Restored.getSeq()); - assertEquals(expectedPublicKey, nodeRecordV5Restored.getPublicKey()); - assertEquals(expectedSignature, nodeRecordV5Restored.getSignature()); - } } diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java b/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java index 821c0d975..cd635a92c 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java @@ -6,7 +6,7 @@ import org.ethereum.beacon.discovery.NodeContext; import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.enr.NodeRecordInfo; -import org.ethereum.beacon.discovery.enr.NodeRecordV5; +import org.ethereum.beacon.discovery.enr.NodeRecordV4; import org.ethereum.beacon.discovery.network.NetworkParcel; import org.ethereum.beacon.discovery.network.NetworkParcelV5; import org.ethereum.beacon.discovery.packet.UnknownPacket; @@ -36,7 +36,7 @@ public class DiscoveryManagerNoNetwork implements DiscoveryManager { private final ReplayProcessor outgoingMessages = ReplayProcessor.cacheLast(); private final FluxSink outgoingSink = outgoingMessages.sink(); private final Bytes32 homeNodeId; - private final NodeRecordV5 homeNodeRecord; + private final NodeRecordV4 homeNodeRecord; private final NodeTable nodeTable; private final NodeBucketStorage nodeBucketStorage; private Publisher incomingPackets; @@ -46,13 +46,13 @@ public class DiscoveryManagerNoNetwork implements DiscoveryManager { public DiscoveryManagerNoNetwork( NodeTable nodeTable, NodeBucketStorage nodeBucketStorage, - NodeRecordV5 homeNode, + NodeRecordV4 homeNode, Publisher incomingPackets) { this.nodeTable = nodeTable; this.nodeBucketStorage = nodeBucketStorage; this.incomingPackets = incomingPackets; this.homeNodeId = homeNode.getNodeId(); - this.homeNodeRecord = (NodeRecordV5) nodeTable.getHomeNode(); + this.homeNodeRecord = (NodeRecordV4) nodeTable.getHomeNode(); this.authTagRepo = new AuthTagRepository(); } @@ -96,7 +96,7 @@ private Optional getContext(Bytes32 nodeId) { () -> String.format("Couldn't find node record for nodeId %s, ignoring", nodeId)); return Optional.empty(); } - NodeRecordV5 nodeRecord = nodeOptional.get().getNode(); + NodeRecordV4 nodeRecord = nodeOptional.get().getNode(); SecureRandom random = new SecureRandom(); context = new NodeContext( diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeBucketTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeBucketTest.java index 8293832b2..df76d1366 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeBucketTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeBucketTest.java @@ -3,7 +3,7 @@ import org.ethereum.beacon.db.Database; import org.ethereum.beacon.discovery.Functions; import org.ethereum.beacon.discovery.enr.NodeRecordInfo; -import org.ethereum.beacon.discovery.enr.NodeRecordV5; +import org.ethereum.beacon.discovery.enr.NodeRecordV4; import org.ethereum.beacon.discovery.enr.NodeStatus; import org.junit.Test; import tech.pegasys.artemis.util.bytes.BytesValue; @@ -26,14 +26,14 @@ private NodeRecordInfo generateUniqueRecord() { try { byte[] pkey = new byte[33]; rnd.nextBytes(pkey); - NodeRecordV5 nodeRecordV5 = - NodeRecordV5.Builder.empty() + NodeRecordV4 nodeRecord = + NodeRecordV4.Builder.empty() .withIpV4Address(BytesValue.wrap(InetAddress.getByName("127.0.0.1").getAddress())) .withSeq(UInt64.valueOf(1)) .withUdpPort(30303) .withSecp256k1(BytesValue.wrap(pkey)) .build(); - return new NodeRecordInfo(nodeRecordV5, (long) rnd.nextInt(1000), NodeStatus.ACTIVE, 0); + return new NodeRecordInfo(nodeRecord, (long) rnd.nextInt(1000), NodeStatus.ACTIVE, 0); } catch (UnknownHostException e) { throw new RuntimeException(e); } diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeTableTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeTableTest.java index 39fe5c301..d46c0ce93 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeTableTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeTableTest.java @@ -3,10 +3,11 @@ import org.ethereum.beacon.db.Database; import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.enr.NodeRecordInfo; -import org.ethereum.beacon.discovery.enr.NodeRecordV5; +import org.ethereum.beacon.discovery.enr.NodeRecordV4; import org.ethereum.beacon.discovery.enr.NodeStatus; import org.junit.Test; import tech.pegasys.artemis.util.bytes.Bytes32; +import tech.pegasys.artemis.util.bytes.Bytes96; import tech.pegasys.artemis.util.bytes.BytesValue; import tech.pegasys.artemis.util.uint.UInt64; @@ -17,23 +18,23 @@ import java.util.Optional; import java.util.function.Supplier; -import static org.ethereum.beacon.discovery.enr.NodeRecordV5.NODE_ID_FUNCTION; import static org.ethereum.beacon.discovery.storage.NodeTableStorage.DEFAULT_SERIALIZER; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; public class NodeTableTest { - private Supplier homeNodeSupplier = + private Supplier homeNodeSupplier = () -> { try { - return NodeRecordV5.Builder.empty() + return NodeRecordV4.Builder.empty() .withIpV4Address(BytesValue.wrap(InetAddress.getByName("127.0.0.1").getAddress())) .withSeq(UInt64.valueOf(1)) .withUdpPort(30303) .withSecp256k1( BytesValue.fromHexString( "0bfb48004b1698f05872cf18b1f278998ad8f7d4c135aa41f83744e7b850ab6b98")) + .withSignature(Bytes96.EMPTY) .build(); } catch (UnknownHostException e) { throw new RuntimeException(e); @@ -43,8 +44,8 @@ public class NodeTableTest { @Test public void testCreate() throws Exception { final String localhostEnr = - "-IS4QHCYrYZbAKWCBRlAy5zzaDZXJBGkcnh4MHcBFZntXNFrdvJjX04jRzjzCBOonrkTfj499SZuOh8R33Ls8RRcy5wBgmlkgnY1iXNlY3AyNTZrMaEDymNMrg1JrLQB2KTGtv6MVbcNEVv0AHacwUAPMljNMTiCaXCEfwAAAYN1ZHCCdl8="; - NodeRecordV5 nodeRecordV5 = (NodeRecordV5) NodeRecord.fromBase64(localhostEnr); + "-IS4QHCYrYZbAKWCBRlAy5zzaDZXJBGkcnh4MHcBFZntXNFrdvJjX04jRzjzCBOonrkTfj499SZuOh8R33Ls8RRcy5wBgmlkgnY0gmlwhH8AAAGJc2VjcDI1NmsxoQPKY0yuDUmstAHYpMa2_oxVtw0RW_QAdpzBQA8yWM0xOIN1ZHCCdl8"; + NodeRecordV4 nodeRecord = (NodeRecordV4) NodeRecord.fromBase64(localhostEnr); NodeTableStorageFactoryImpl nodeTableStorageFactory = new NodeTableStorageFactoryImpl(); Database database = Database.inMemoryDB(); NodeTableStorage nodeTableStorage = @@ -54,24 +55,24 @@ public void testCreate() throws Exception { homeNodeSupplier, () -> { List nodes = new ArrayList<>(); - nodes.add(nodeRecordV5); + nodes.add(nodeRecord); return nodes; }); Optional extendedEnr = - nodeTableStorage.get().getNode(NODE_ID_FUNCTION.apply(nodeRecordV5)); + nodeTableStorage.get().getNode(nodeRecord.getNodeId()); assertTrue(extendedEnr.isPresent()); - NodeRecordInfo nodeRecord = extendedEnr.get(); - assertEquals(nodeRecordV5.getPublicKey(), nodeRecord.getNode().getPublicKey()); + NodeRecordInfo nodeRecord2 = extendedEnr.get(); + assertEquals(nodeRecord.getPublicKey(), nodeRecord2.getNode().getPublicKey()); assertEquals( nodeTableStorage.get().getHomeNode().getNodeId(), - NODE_ID_FUNCTION.apply(homeNodeSupplier.get())); + homeNodeSupplier.get().getNodeId()); } @Test public void testFind() throws Exception { final String localhostEnr = - "-IS4QHCYrYZbAKWCBRlAy5zzaDZXJBGkcnh4MHcBFZntXNFrdvJjX04jRzjzCBOonrkTfj499SZuOh8R33Ls8RRcy5wBgmlkgnY1iXNlY3AyNTZrMaEDymNMrg1JrLQB2KTGtv6MVbcNEVv0AHacwUAPMljNMTiCaXCEfwAAAYN1ZHCCdl8="; - NodeRecordV5 localHostNode = (NodeRecordV5) NodeRecord.fromBase64(localhostEnr); + "-IS4QHCYrYZbAKWCBRlAy5zzaDZXJBGkcnh4MHcBFZntXNFrdvJjX04jRzjzCBOonrkTfj499SZuOh8R33Ls8RRcy5wBgmlkgnY0gmlwhH8AAAGJc2VjcDI1NmsxoQPKY0yuDUmstAHYpMa2_oxVtw0RW_QAdpzBQA8yWM0xOIN1ZHCCdl8"; + NodeRecordV4 localHostNode = (NodeRecordV4) NodeRecord.fromBase64(localhostEnr); NodeTableStorageFactoryImpl nodeTableStorageFactory = new NodeTableStorageFactoryImpl(); Database database = Database.inMemoryDB(); NodeTableStorage nodeTableStorage = @@ -86,41 +87,43 @@ public void testFind() throws Exception { }); // node is adjusted to be close to localhostEnr - NodeRecordV5 closestNode = - NodeRecordV5.Builder.empty() + NodeRecordV4 closestNode = + NodeRecordV4.Builder.empty() .withIpV4Address(BytesValue.wrap(InetAddress.getByName("127.0.0.2").getAddress())) .withSeq(UInt64.valueOf(1)) .withUdpPort(30303) .withSecp256k1( BytesValue.fromHexString( "aafb48004b1698f05872cf18b1f278998ad8f7d4c135aa41f83744e7b850ab6b98")) + .withSignature(Bytes96.EMPTY) .build(); nodeTableStorage.get().save(new NodeRecordInfo(closestNode, -1L, NodeStatus.ACTIVE, 0)); assertEquals( nodeTableStorage .get() - .getNode(NODE_ID_FUNCTION.apply(closestNode)) + .getNode(closestNode.getNodeId()) .get() .getNode() .getPublicKey(), closestNode.getPublicKey()); - NodeRecordV5 farNode = - NodeRecordV5.Builder.empty() + NodeRecordV4 farNode = + NodeRecordV4.Builder.empty() .withIpV4Address(BytesValue.wrap(InetAddress.getByName("127.0.0.3").getAddress())) .withSeq(UInt64.valueOf(1)) .withUdpPort(30303) .withSecp256k1( BytesValue.fromHexString( "bafb48004b1698f05872cf18b1f278998ad8f7d4c135aa41f83744e7b850ab6b98")) + .withSignature(Bytes96.EMPTY) .build(); nodeTableStorage.get().save(new NodeRecordInfo(farNode, -1L, NodeStatus.ACTIVE, 0)); List closestNodes = - nodeTableStorage.get().findClosestNodes(NODE_ID_FUNCTION.apply(closestNode), 252); + nodeTableStorage.get().findClosestNodes(closestNode.getNodeId(), 252); assertEquals(2, closestNodes.size()); - assertEquals(closestNodes.get(0).getNode().getPublicKey(), localHostNode.getPublicKey()); - assertEquals(closestNodes.get(1).getNode().getPublicKey(), closestNode.getPublicKey()); + assertEquals(closestNodes.get(1).getNode().getPublicKey(), localHostNode.getPublicKey()); + assertEquals(closestNodes.get(0).getNode().getPublicKey(), closestNode.getPublicKey()); List farNodes = - nodeTableStorage.get().findClosestNodes(NODE_ID_FUNCTION.apply(farNode), 1); + nodeTableStorage.get().findClosestNodes(farNode.getNodeId(), 1); assertEquals(1, farNodes.size()); assertEquals(farNodes.get(0).getNode().getPublicKey(), farNode.getPublicKey()); } From 25f222fc44b34ab4ad5e8c998e672881e3b7b3d9 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Tue, 1 Oct 2019 22:19:39 +0300 Subject: [PATCH 24/77] discovery: refactor Ethereum Node Record to further detach it from other entities --- .../discovery/DiscoveryManagerImpl.java | 1 - .../discovery/DiscoveryTaskManager.java | 2 -- .../beacon/discovery/NodeConnectTasks.java | 1 - .../discovery/{enr => }/NodeRecordInfo.java | 4 ++- .../discovery/{enr => }/NodeStatus.java | 2 +- .../beacon/discovery/enr/EnrScheme.java | 31 +++++++++++++++++++ .../beacon/discovery/enr/NodeRecord.java | 28 ----------------- .../message/handler/FindNodeHandler.java | 2 +- .../message/handler/NodesHandler.java | 5 +-- .../beacon/discovery/storage/NodeBucket.java | 4 +-- .../discovery/storage/NodeBucketStorage.java | 2 +- .../storage/NodeBucketStorageImpl.java | 2 +- .../storage/NodeSerializerFactory.java | 2 +- .../beacon/discovery/storage/NodeTable.java | 2 +- .../discovery/storage/NodeTableImpl.java | 2 +- .../discovery/storage/NodeTableStorage.java | 2 +- .../storage/NodeTableStorageFactoryImpl.java | 2 +- .../storage/NodeTableStorageImpl.java | 2 +- .../beacon/discovery/NodeRecordTest.java | 5 +-- .../mock/DiscoveryManagerNoNetwork.java | 2 +- .../discovery/storage/NodeBucketTest.java | 4 +-- .../discovery/storage/NodeTableTest.java | 4 +-- 22 files changed, 55 insertions(+), 56 deletions(-) rename discovery/src/main/java/org/ethereum/beacon/discovery/{enr => }/NodeRecordInfo.java (95%) rename discovery/src/main/java/org/ethereum/beacon/discovery/{enr => }/NodeStatus.java (92%) create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrScheme.java diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java index 2d5f02d15..50cbd8fe8 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java @@ -4,7 +4,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.ethereum.beacon.discovery.enr.NodeRecord; -import org.ethereum.beacon.discovery.enr.NodeRecordInfo; import org.ethereum.beacon.discovery.enr.NodeRecordV4; import org.ethereum.beacon.discovery.network.DiscoveryClient; import org.ethereum.beacon.discovery.network.DiscoveryServer; diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryTaskManager.java b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryTaskManager.java index 18604e689..c25a197b2 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryTaskManager.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryTaskManager.java @@ -1,8 +1,6 @@ package org.ethereum.beacon.discovery; -import org.ethereum.beacon.discovery.enr.NodeRecordInfo; import org.ethereum.beacon.discovery.enr.NodeRecordV4; -import org.ethereum.beacon.discovery.enr.NodeStatus; import org.ethereum.beacon.discovery.storage.NodeBucketStorage; import org.ethereum.beacon.discovery.storage.NodeTable; import org.ethereum.beacon.schedulers.Scheduler; diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeConnectTasks.java b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeConnectTasks.java index f86af9685..84f292e10 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeConnectTasks.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeConnectTasks.java @@ -2,7 +2,6 @@ import com.google.common.collect.Sets; import org.ethereum.beacon.discovery.enr.NodeRecord; -import org.ethereum.beacon.discovery.enr.NodeRecordInfo; import org.ethereum.beacon.schedulers.Scheduler; import org.ethereum.beacon.util.ExpirationScheduler; import tech.pegasys.artemis.util.bytes.Bytes32; diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordInfo.java b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeRecordInfo.java similarity index 95% rename from discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordInfo.java rename to discovery/src/main/java/org/ethereum/beacon/discovery/NodeRecordInfo.java index 72cb86d95..0c1679de4 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordInfo.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeRecordInfo.java @@ -1,6 +1,8 @@ -package org.ethereum.beacon.discovery.enr; +package org.ethereum.beacon.discovery; import com.google.common.base.Objects; +import org.ethereum.beacon.discovery.enr.NodeRecord; +import org.ethereum.beacon.discovery.enr.NodeRecordV4; import org.web3j.rlp.RlpDecoder; import org.web3j.rlp.RlpEncoder; import org.web3j.rlp.RlpList; diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeStatus.java b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeStatus.java similarity index 92% rename from discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeStatus.java rename to discovery/src/main/java/org/ethereum/beacon/discovery/NodeStatus.java index fb87a4266..553214c85 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeStatus.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeStatus.java @@ -1,4 +1,4 @@ -package org.ethereum.beacon.discovery.enr; +package org.ethereum.beacon.discovery; import java.util.HashMap; import java.util.Map; diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrScheme.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrScheme.java new file mode 100644 index 000000000..9e8545543 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrScheme.java @@ -0,0 +1,31 @@ +package org.ethereum.beacon.discovery.enr; + +import java.util.HashMap; +import java.util.Map; + +/** Available identity schemes of {@link NodeRecord} */ +public enum EnrScheme { + V4("v4"); + + private static final Map nameMap = new HashMap<>(); + + static { + for (EnrScheme scheme : EnrScheme.values()) { + nameMap.put(scheme.name, scheme); + } + } + + private String name; + + private EnrScheme(String name) { + this.name = name; + } + + public static EnrScheme fromString(String name) { + return nameMap.get(name); + } + + public String stringName() { + return name; + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecord.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecord.java index 96569e3a9..1c26dab42 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecord.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecord.java @@ -9,9 +9,7 @@ import tech.pegasys.artemis.util.uint.UInt64; import java.util.Base64; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Set; /** @@ -111,30 +109,4 @@ default String asBase64() { * #getKeys()} */ Object getKey(String key); - - enum EnrScheme { - V4("v4"); - - private static final Map nameMap = new HashMap<>(); - - static { - for (EnrScheme scheme : EnrScheme.values()) { - nameMap.put(scheme.name, scheme); - } - } - - private String name; - - private EnrScheme(String name) { - this.name = name; - } - - public static EnrScheme fromString(String name) { - return nameMap.get(name); - } - - public String stringName() { - return name; - } - } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/FindNodeHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/FindNodeHandler.java index 380e6cb68..496562afb 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/FindNodeHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/FindNodeHandler.java @@ -1,7 +1,7 @@ package org.ethereum.beacon.discovery.message.handler; import org.ethereum.beacon.discovery.NodeContext; -import org.ethereum.beacon.discovery.enr.NodeRecordInfo; +import org.ethereum.beacon.discovery.NodeRecordInfo; import org.ethereum.beacon.discovery.message.DiscoveryV5Message; import org.ethereum.beacon.discovery.message.FindNodeMessage; import org.ethereum.beacon.discovery.message.NodesMessage; diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/NodesHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/NodesHandler.java index dc1b1e66f..b575aa11d 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/NodesHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/NodesHandler.java @@ -1,12 +1,9 @@ package org.ethereum.beacon.discovery.message.handler; import org.ethereum.beacon.discovery.NodeContext; -import org.ethereum.beacon.discovery.enr.NodeRecordInfo; -import org.ethereum.beacon.discovery.message.DiscoveryV5Message; -import org.ethereum.beacon.discovery.message.FindNodeMessage; +import org.ethereum.beacon.discovery.NodeRecordInfo; import org.ethereum.beacon.discovery.message.MessageCode; import org.ethereum.beacon.discovery.message.NodesMessage; -import org.ethereum.beacon.discovery.message.PongMessage; import org.ethereum.beacon.util.ExpirationScheduler; import tech.pegasys.artemis.util.bytes.BytesValue; diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeBucket.java b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeBucket.java index 9dfd6c7dd..5eb431076 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeBucket.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeBucket.java @@ -1,7 +1,7 @@ package org.ethereum.beacon.discovery.storage; -import org.ethereum.beacon.discovery.enr.NodeRecordInfo; -import org.ethereum.beacon.discovery.enr.NodeStatus; +import org.ethereum.beacon.discovery.NodeRecordInfo; +import org.ethereum.beacon.discovery.NodeStatus; import org.web3j.rlp.RlpDecoder; import org.web3j.rlp.RlpEncoder; import org.web3j.rlp.RlpList; diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeBucketStorage.java b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeBucketStorage.java index 91cb8c70a..5f58850fb 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeBucketStorage.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeBucketStorage.java @@ -1,6 +1,6 @@ package org.ethereum.beacon.discovery.storage; -import org.ethereum.beacon.discovery.enr.NodeRecordInfo; +import org.ethereum.beacon.discovery.NodeRecordInfo; import java.util.Optional; diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeBucketStorageImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeBucketStorageImpl.java index b29531de8..57b0618d6 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeBucketStorageImpl.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeBucketStorageImpl.java @@ -6,7 +6,7 @@ import org.ethereum.beacon.db.source.HoleyList; import org.ethereum.beacon.db.source.impl.DataSourceList; import org.ethereum.beacon.discovery.Functions; -import org.ethereum.beacon.discovery.enr.NodeRecordInfo; +import org.ethereum.beacon.discovery.NodeRecordInfo; import tech.pegasys.artemis.util.bytes.Bytes32; import tech.pegasys.artemis.util.bytes.BytesValue; diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeSerializerFactory.java b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeSerializerFactory.java index 90fd28713..a78be3671 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeSerializerFactory.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeSerializerFactory.java @@ -1,7 +1,7 @@ package org.ethereum.beacon.discovery.storage; import org.ethereum.beacon.chain.storage.impl.SerializerFactory; -import org.ethereum.beacon.discovery.enr.NodeRecordInfo; +import org.ethereum.beacon.discovery.NodeRecordInfo; import tech.pegasys.artemis.util.bytes.BytesValue; import java.util.HashMap; diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTable.java b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTable.java index 9a3a6f2b9..66d854c9d 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTable.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTable.java @@ -1,7 +1,7 @@ package org.ethereum.beacon.discovery.storage; import org.ethereum.beacon.discovery.enr.NodeRecord; -import org.ethereum.beacon.discovery.enr.NodeRecordInfo; +import org.ethereum.beacon.discovery.NodeRecordInfo; import tech.pegasys.artemis.util.bytes.Bytes32; import java.util.List; diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableImpl.java index c754b277f..328ffbadb 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableImpl.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableImpl.java @@ -8,7 +8,7 @@ import org.ethereum.beacon.db.source.SingleValueSource; import org.ethereum.beacon.discovery.Functions; import org.ethereum.beacon.discovery.enr.NodeRecord; -import org.ethereum.beacon.discovery.enr.NodeRecordInfo; +import org.ethereum.beacon.discovery.NodeRecordInfo; import tech.pegasys.artemis.ethereum.core.Hash32; import tech.pegasys.artemis.util.bytes.Bytes32; import tech.pegasys.artemis.util.bytes.BytesValue; diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorage.java b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorage.java index eee40e0e1..4cee79dc7 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorage.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorage.java @@ -2,7 +2,7 @@ import org.ethereum.beacon.chain.storage.impl.SerializerFactory; import org.ethereum.beacon.db.source.SingleValueSource; -import org.ethereum.beacon.discovery.enr.NodeRecordInfo; +import org.ethereum.beacon.discovery.NodeRecordInfo; /** Stores {@link NodeTable} and home node info */ public interface NodeTableStorage { diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactoryImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactoryImpl.java index 17b159741..aaf8ce4ea 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactoryImpl.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactoryImpl.java @@ -3,7 +3,7 @@ import org.ethereum.beacon.chain.storage.impl.SerializerFactory; import org.ethereum.beacon.db.Database; import org.ethereum.beacon.discovery.enr.NodeRecord; -import org.ethereum.beacon.discovery.enr.NodeRecordInfo; +import org.ethereum.beacon.discovery.NodeRecordInfo; import org.ethereum.beacon.discovery.enr.NodeRecordV4; import tech.pegasys.artemis.util.bytes.Bytes32; diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageImpl.java index b41f61c3d..c872b8926 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageImpl.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageImpl.java @@ -8,7 +8,7 @@ import org.ethereum.beacon.db.source.HoleyList; import org.ethereum.beacon.db.source.SingleValueSource; import org.ethereum.beacon.db.source.impl.DataSourceList; -import org.ethereum.beacon.discovery.enr.NodeRecordInfo; +import org.ethereum.beacon.discovery.NodeRecordInfo; import tech.pegasys.artemis.ethereum.core.Hash32; import tech.pegasys.artemis.util.bytes.BytesValue; diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/NodeRecordTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/NodeRecordTest.java index 2d34bc8d2..373680279 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/NodeRecordTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/NodeRecordTest.java @@ -1,5 +1,6 @@ package org.ethereum.beacon.discovery; +import org.ethereum.beacon.discovery.enr.EnrScheme; import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.enr.NodeRecordV4; import org.junit.Test; @@ -36,7 +37,7 @@ public void testLocalhostV4() throws Exception { "-IS4QHCYrYZbAKWCBRlAy5zzaDZXJBGkcnh4MHcBFZntXNFrdvJjX04jRzjzCBOonrkTfj499SZuOh8R33Ls8RRcy5wBgmlkgnY0gmlwhH8AAAGJc2VjcDI1NmsxoQPKY0yuDUmstAHYpMa2_oxVtw0RW_QAdpzBQA8yWM0xOIN1ZHCCdl8"; NodeRecord nodeRecord = NodeRecord.fromBase64(localhostEnr); - assertEquals(NodeRecord.EnrScheme.V4, nodeRecord.getIdentityScheme()); + assertEquals(EnrScheme.V4, nodeRecord.getIdentityScheme()); NodeRecordV4 nodeRecordV4 = (NodeRecordV4) nodeRecord; assertArrayEquals( InetAddress.getByName(expectedHost).getAddress(), @@ -51,7 +52,7 @@ public void testLocalhostV4() throws Exception { // The order of fields is not strict so we don't compare strings NodeRecord nodeRecordRestored = NodeRecord.fromBase64(localhostEnrRestored); - assertEquals(NodeRecord.EnrScheme.V4, nodeRecordRestored.getIdentityScheme()); + assertEquals(EnrScheme.V4, nodeRecordRestored.getIdentityScheme()); NodeRecordV4 nodeRecordV4Restored = (NodeRecordV4) nodeRecordRestored; assertArrayEquals( InetAddress.getByName(expectedHost).getAddress(), diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java b/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java index cd635a92c..afbcfeb67 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java @@ -5,7 +5,7 @@ import org.ethereum.beacon.discovery.DiscoveryManager; import org.ethereum.beacon.discovery.NodeContext; import org.ethereum.beacon.discovery.enr.NodeRecord; -import org.ethereum.beacon.discovery.enr.NodeRecordInfo; +import org.ethereum.beacon.discovery.NodeRecordInfo; import org.ethereum.beacon.discovery.enr.NodeRecordV4; import org.ethereum.beacon.discovery.network.NetworkParcel; import org.ethereum.beacon.discovery.network.NetworkParcelV5; diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeBucketTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeBucketTest.java index df76d1366..4089ced1c 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeBucketTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeBucketTest.java @@ -2,9 +2,9 @@ import org.ethereum.beacon.db.Database; import org.ethereum.beacon.discovery.Functions; -import org.ethereum.beacon.discovery.enr.NodeRecordInfo; +import org.ethereum.beacon.discovery.NodeRecordInfo; import org.ethereum.beacon.discovery.enr.NodeRecordV4; -import org.ethereum.beacon.discovery.enr.NodeStatus; +import org.ethereum.beacon.discovery.NodeStatus; import org.junit.Test; import tech.pegasys.artemis.util.bytes.BytesValue; import tech.pegasys.artemis.util.uint.UInt64; diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeTableTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeTableTest.java index d46c0ce93..4d8cf6a48 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeTableTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeTableTest.java @@ -2,9 +2,9 @@ import org.ethereum.beacon.db.Database; import org.ethereum.beacon.discovery.enr.NodeRecord; -import org.ethereum.beacon.discovery.enr.NodeRecordInfo; +import org.ethereum.beacon.discovery.NodeRecordInfo; import org.ethereum.beacon.discovery.enr.NodeRecordV4; -import org.ethereum.beacon.discovery.enr.NodeStatus; +import org.ethereum.beacon.discovery.NodeStatus; import org.junit.Test; import tech.pegasys.artemis.util.bytes.Bytes32; import tech.pegasys.artemis.util.bytes.Bytes96; From bfd48b1a6eea4e86026318c9c948c63690c748a5 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Wed, 2 Oct 2019 13:12:49 +0300 Subject: [PATCH 25/77] discovery: make udp client independent of server --- .../discovery/DiscoveryManagerImpl.java | 13 +- .../discovery/network/DiscoveryClient.java | 58 +------- .../network/DiscoveryClientImpl.java | 133 ++++++++++++++++++ .../network/DiscoveryServerImpl.java | 44 +----- .../network/NettyDiscoveryServer.java | 20 --- .../discovery/DiscoveryNetworkTest.java | 6 +- 6 files changed, 147 insertions(+), 127 deletions(-) create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/network/DiscoveryClientImpl.java delete mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/network/NettyDiscoveryServer.java diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java index 50cbd8fe8..deee7bfa6 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java @@ -6,9 +6,9 @@ import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.enr.NodeRecordV4; import org.ethereum.beacon.discovery.network.DiscoveryClient; +import org.ethereum.beacon.discovery.network.DiscoveryClientImpl; import org.ethereum.beacon.discovery.network.DiscoveryServer; import org.ethereum.beacon.discovery.network.DiscoveryServerImpl; -import org.ethereum.beacon.discovery.network.NettyDiscoveryServer; import org.ethereum.beacon.discovery.network.NetworkParcel; import org.ethereum.beacon.discovery.network.NetworkParcelV5; import org.ethereum.beacon.discovery.packet.UnknownPacket; @@ -43,7 +43,6 @@ public class DiscoveryManagerImpl implements DiscoveryManager { new ConcurrentHashMap<>(); // nodeId -> context private final AuthTagRepository authTagRepo; private final DiscoveryServer discoveryServer; - private final CompletableFuture discoveryClientAssigned; private final Scheduler scheduler; private ExpirationScheduler contextExpirationScheduler = new ExpirationScheduler<>(CLEANUP_DELAY_SECONDS, TimeUnit.SECONDS); @@ -53,7 +52,8 @@ public DiscoveryManagerImpl( NodeTable nodeTable, NodeBucketStorage nodeBucketStorage, NodeRecordV4 homeNode, - Scheduler serverScheduler) { + Scheduler serverScheduler, + Scheduler clientScheduler) { this.nodeTable = nodeTable; this.nodeBucketStorage = nodeBucketStorage; this.homeNodeId = homeNode.getNodeId(); @@ -62,11 +62,7 @@ public DiscoveryManagerImpl( this.scheduler = serverScheduler; this.discoveryServer = new DiscoveryServerImpl(homeNodeRecord.getIpV4address(), homeNodeRecord.getUdpPort()); - this.discoveryClientAssigned = - ((NettyDiscoveryServer) discoveryServer) - .useDatagramChannel( - nioDatagramChannel -> - discoveryClient = new DiscoveryClient(nioDatagramChannel, outgoingMessages)); + this.discoveryClient = new DiscoveryClientImpl(outgoingMessages, clientScheduler); } @Override @@ -90,7 +86,6 @@ public void start() { } }); discoveryServer.start(scheduler); - discoveryClientAssigned.join(); } @Override diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/network/DiscoveryClient.java b/discovery/src/main/java/org/ethereum/beacon/discovery/network/DiscoveryClient.java index 8cb0be78c..fab7112d7 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/network/DiscoveryClient.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/network/DiscoveryClient.java @@ -1,61 +1,11 @@ package org.ethereum.beacon.discovery.network; -import io.netty.buffer.Unpooled; -import io.netty.channel.socket.DatagramPacket; -import io.netty.channel.socket.nio.NioDatagramChannel; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.ethereum.beacon.discovery.enr.NodeRecord; -import org.ethereum.beacon.discovery.enr.NodeRecordV4; -import org.reactivestreams.Publisher; -import reactor.core.publisher.Flux; import tech.pegasys.artemis.util.bytes.BytesValue; -import java.net.InetAddress; -import java.net.InetSocketAddress; -import java.net.UnknownHostException; +/** Discovery client sends outgoing messages */ +public interface DiscoveryClient { + void stop(); -/** Discovery UDP client */ -public class DiscoveryClient { - private static final Logger logger = LogManager.getLogger(DiscoveryClient.class); - private final NioDatagramChannel channel; - - /** - * Constructs UDP client using - * - * @param channel Netty UDP datagram channel - * @param outgoingStream Stream of outgoing packets, client will forward them to the channel - */ - public DiscoveryClient(NioDatagramChannel channel, Publisher outgoingStream) { - this.channel = channel; - Flux.from(outgoingStream) - .subscribe( - networkPacket -> - send(networkPacket.getPacket().getBytes(), networkPacket.getNodeRecord())); - } - - private void send(BytesValue data, NodeRecord recipient) { - if (!(recipient instanceof NodeRecordV4)) { - String error = - String.format( - "Accepts only V5 versions of recipient's node records. Got %s instead", recipient); - logger.error(error); - throw new RuntimeException(error); - } - InetSocketAddress address; - try { - address = - new InetSocketAddress( - InetAddress.getByAddress(((NodeRecordV4) recipient).getIpV4address().extractArray()), - ((NodeRecordV4) recipient).getUdpPort()); - } catch (UnknownHostException e) { - String error = String.format("Failed to resolve host for node record: %s", recipient); - logger.error(error); - throw new RuntimeException(error); - } - DatagramPacket packet = new DatagramPacket(Unpooled.copiedBuffer(data.extractArray()), address); - logger.trace(() -> String.format("Sending packet %s", packet)); - channel.write(packet); - channel.flush(); - } + void send(BytesValue data, NodeRecord recipient); } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/network/DiscoveryClientImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/network/DiscoveryClientImpl.java new file mode 100644 index 000000000..c66058f47 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/network/DiscoveryClientImpl.java @@ -0,0 +1,133 @@ +package org.ethereum.beacon.discovery.network; + +import io.netty.bootstrap.Bootstrap; +import io.netty.buffer.Unpooled; +import io.netty.channel.Channel; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.DatagramPacket; +import io.netty.channel.socket.nio.NioDatagramChannel; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.ethereum.beacon.discovery.enr.NodeRecord; +import org.ethereum.beacon.discovery.enr.NodeRecordV4; +import org.ethereum.beacon.schedulers.Scheduler; +import org.reactivestreams.Publisher; +import reactor.core.publisher.Flux; +import tech.pegasys.artemis.util.bytes.BytesValue; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.UnknownHostException; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicBoolean; + +/** Discovery UDP client */ +public class DiscoveryClientImpl implements DiscoveryClient { + private static final int RECREATION_TIMEOUT = 5000; + private static final int STOPPING_TIMEOUT = 10000; + private static final Logger logger = LogManager.getLogger(DiscoveryClientImpl.class); + private AtomicBoolean listen = new AtomicBoolean(true); + private CountDownLatch starting = new CountDownLatch(1); + private Channel channel; + + /** + * Constructs UDP client using + * + * @param outgoingStream Stream of outgoing packets, client will forward them to the channel + * @param scheduler Scheduler to run client loop on + */ + public DiscoveryClientImpl(Publisher outgoingStream, Scheduler scheduler) { + Flux.from(outgoingStream) + .subscribe( + networkPacket -> + send(networkPacket.getPacket().getBytes(), networkPacket.getNodeRecord())); + logger.info("Starting UDP discovery client"); + scheduler.execute(this::clientLoop); + try { + starting.await(); + logger.info("UDP discovery client started"); + } catch (InterruptedException e) { + throw new RuntimeException("Initialization of discovery client broke by interruption", e); + } + } + + private void clientLoop() { + NioEventLoopGroup group = new NioEventLoopGroup(1); + try { + while (listen.get()) { + Bootstrap b = new Bootstrap(); + b.group(group) + .channel(NioDatagramChannel.class) + .handler( + new ChannelInitializer() { + @Override + protected void initChannel(NioDatagramChannel ch) throws Exception { + starting.countDown(); + } + }); + + channel = b.bind(0).sync().channel(); + channel.closeFuture().sync(); + + if (!listen.get()) { + logger.info("Shutting down discovery client"); + break; + } + logger.error("Discovery client closed. Trying to restore after %s seconds delay"); + Thread.sleep(RECREATION_TIMEOUT); + } + } catch (Exception e) { + logger.error("Can't start discovery client", e); + } finally { + try { + group.shutdownGracefully().sync(); + } catch (Exception ex) { + logger.error("Failed to shutdown discovery client thread group", ex); + } + } + } + + @Override + public void stop() { + if (listen.get()) { + logger.info("Stopping discovery client"); + listen.set(false); + if (channel != null) { + try { + channel.close().await(STOPPING_TIMEOUT); + } catch (InterruptedException ex) { + logger.error("Failed to stop discovery client", ex); + } + } + } else { + logger.warn("An attempt to stop already stopping/stopped discovery client"); + } + } + + @Override + public void send(BytesValue data, NodeRecord recipient) { + if (!(recipient instanceof NodeRecordV4)) { + String error = + String.format( + "Accepts only V4 versions of recipient's node records. Got %s instead", recipient); + logger.error(error); + throw new RuntimeException(error); + } + InetSocketAddress address; + try { + address = + new InetSocketAddress( + InetAddress.getByAddress(((NodeRecordV4) recipient).getIpV4address().extractArray()), + ((NodeRecordV4) recipient).getUdpPort()); + } catch (UnknownHostException e) { + String error = String.format("Failed to resolve host for node record: %s", recipient); + logger.error(error); + throw new RuntimeException(error); + } + DatagramPacket packet = new DatagramPacket(Unpooled.copiedBuffer(data.extractArray()), address); + logger.trace(() -> String.format("Sending packet %s", packet)); + channel.write(packet); + channel.flush(); + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/network/DiscoveryServerImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/network/DiscoveryServerImpl.java index 1ef045031..415f933f9 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/network/DiscoveryServerImpl.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/network/DiscoveryServerImpl.java @@ -7,7 +7,6 @@ import io.netty.channel.socket.nio.NioDatagramChannel; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.ethereum.beacon.schedulers.RunnableEx; import org.ethereum.beacon.schedulers.Scheduler; import org.reactivestreams.Publisher; import reactor.core.publisher.FluxSink; @@ -17,13 +16,9 @@ import java.net.InetAddress; import java.net.UnknownHostException; -import java.util.HashSet; -import java.util.Set; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.Consumer; -public class DiscoveryServerImpl implements NettyDiscoveryServer { +public class DiscoveryServerImpl implements DiscoveryServer { private static final int RECREATION_TIMEOUT = 5000; private static final int STOPPING_TIMEOUT = 10000; private static final Logger logger = LogManager.getLogger(DiscoveryServerImpl.class); @@ -33,13 +28,6 @@ public class DiscoveryServerImpl implements NettyDiscoveryServer { private final String udpListenHost; private AtomicBoolean listen = new AtomicBoolean(true); private Channel channel; - private NioDatagramChannel datagramChannel; - private Set> datagramChannelUsageQueue = new HashSet<>(); - - public DiscoveryServerImpl(Scheduler scheduler, String udpListenHost, Integer udpListenPort) { - this.udpListenHost = udpListenHost; - this.udpListenPort = udpListenPort; - } public DiscoveryServerImpl(Bytes4 udpListenHost, Integer udpListenPort) { try { @@ -53,13 +41,7 @@ public DiscoveryServerImpl(Bytes4 udpListenHost, Integer udpListenPort) { @Override public void start(Scheduler scheduler) { logger.info(String.format("Starting discovery server on UDP port %s", udpListenPort)); - scheduler.execute( - new RunnableEx() { - @Override - public void run() throws Exception { - serverLoop(); - } - }); + scheduler.execute(this::serverLoop); } private void serverLoop() { @@ -76,11 +58,6 @@ public void initChannel(NioDatagramChannel ch) throws Exception { ch.pipeline() .addLast(new DatagramToBytesValue()) .addLast(new IncomingMessageSink(incomingSink)); - synchronized (DiscoveryServerImpl.class) { - datagramChannel = ch; - datagramChannelUsageQueue.forEach( - nioDatagramChannelConsumer -> nioDatagramChannelConsumer.accept(ch)); - } } }); @@ -110,23 +87,6 @@ public Publisher getIncomingPackets() { return incomingPackets; } - public synchronized CompletableFuture useDatagramChannel( - Consumer consumer) { - CompletableFuture usage = new CompletableFuture<>(); - if (datagramChannel != null) { - consumer.accept(datagramChannel); - usage.complete(null); - } else { - datagramChannelUsageQueue.add( - nioDatagramChannel -> { - consumer.accept(nioDatagramChannel); - usage.complete(null); - }); - } - - return usage; - } - @Override public void stop() { if (listen.get()) { diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/network/NettyDiscoveryServer.java b/discovery/src/main/java/org/ethereum/beacon/discovery/network/NettyDiscoveryServer.java deleted file mode 100644 index f3a77e1b8..000000000 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/network/NettyDiscoveryServer.java +++ /dev/null @@ -1,20 +0,0 @@ -package org.ethereum.beacon.discovery.network; - -import io.netty.channel.socket.nio.NioDatagramChannel; - -import java.util.concurrent.CompletableFuture; -import java.util.function.Consumer; - -public interface NettyDiscoveryServer extends DiscoveryServer { - - /** - * Provides safe way to use internal datagram channel for other services. Channel is usually not - * available on request, instead it will be available at some time in the future. After server - * starts and some events happened. Return future is fired when it finally happens. - * - * @param consumer Consumer which needs Netty datagram channel. For example, UDP client - * application could share the channel with the server - * @return Future which is completed when channel is provided to consumer - */ - CompletableFuture useDatagramChannel(Consumer consumer); -} diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java index 884d8bbc6..e330cf47c 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java @@ -91,13 +91,15 @@ public void test() throws Exception { nodeTableStorage1.get(), nodeBucketStorage1, nodeRecord1, - Schedulers.createDefault().newSingleThreadDaemon("server-1")); + Schedulers.createDefault().newSingleThreadDaemon("server-1"), + Schedulers.createDefault().newSingleThreadDaemon("client-1")); DiscoveryManagerImpl discoveryManager2 = new DiscoveryManagerImpl( nodeTableStorage2.get(), nodeBucketStorage2, nodeRecord2, - Schedulers.createDefault().newSingleThreadDaemon("server-2")); + Schedulers.createDefault().newSingleThreadDaemon("server-2"), + Schedulers.createDefault().newSingleThreadDaemon("client-2")); discoveryManager1.start(); discoveryManager2.start(); From 84dabf53cc411f164d8b9b7f301730499adbb4cf Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Wed, 2 Oct 2019 17:44:31 +0300 Subject: [PATCH 26/77] discovery: use enr scheme interpreter instead of full enr abstractions --- .../discovery/DiscoveryManagerImpl.java | 14 +- .../discovery/DiscoveryTaskManager.java | 4 +- .../beacon/discovery/NodeContext.java | 11 +- .../beacon/discovery/NodeRecordInfo.java | 28 +- .../discovery/enr/EnrSchemeInterpreter.java | 21 + .../discovery/enr/EnrSchemeV4Interpreter.java | 97 +++++ .../beacon/discovery/enr/NodeRecord.java | 201 +++++---- .../discovery/enr/NodeRecordFactory.java | 91 +++++ .../beacon/discovery/enr/NodeRecordV4.java | 381 ------------------ .../discovery/message/DiscoveryV5Message.java | 6 +- .../discovery/message/NodesMessage.java | 10 +- .../message/handler/PingHandler.java | 6 +- .../network/DiscoveryClientImpl.java | 12 +- .../discovery/network/NetworkParcelV5.java | 5 +- .../packet/AuthHeaderMessagePacket.java | 11 +- .../AuthHeaderMessagePacketHandler.java | 3 +- .../handler/WhoAreYouPacketHandler.java | 3 +- .../storage/NodeTableStorageFactory.java | 3 +- .../storage/NodeTableStorageFactoryImpl.java | 5 +- .../discovery/DiscoveryNetworkTest.java | 65 ++- .../discovery/DiscoveryNoNetworkTest.java | 65 ++- .../beacon/discovery/NodeRecordTest.java | 32 +- .../mock/DiscoveryManagerNoNetwork.java | 9 +- .../discovery/storage/NodeBucketTest.java | 31 +- .../discovery/storage/NodeTableTest.java | 128 +++--- 25 files changed, 619 insertions(+), 623 deletions(-) create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrSchemeInterpreter.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrSchemeV4Interpreter.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordFactory.java delete mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV4.java diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java index deee7bfa6..ec2c6365b 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java @@ -4,7 +4,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.ethereum.beacon.discovery.enr.NodeRecord; -import org.ethereum.beacon.discovery.enr.NodeRecordV4; import org.ethereum.beacon.discovery.network.DiscoveryClient; import org.ethereum.beacon.discovery.network.DiscoveryClientImpl; import org.ethereum.beacon.discovery.network.DiscoveryServer; @@ -22,6 +21,7 @@ import reactor.core.publisher.FluxSink; import reactor.core.publisher.ReplayProcessor; import tech.pegasys.artemis.util.bytes.Bytes32; +import tech.pegasys.artemis.util.bytes.Bytes4; import java.security.SecureRandom; import java.util.Map; @@ -36,7 +36,7 @@ public class DiscoveryManagerImpl implements DiscoveryManager { private final ReplayProcessor outgoingMessages = ReplayProcessor.cacheLast(); private final FluxSink outgoingSink = outgoingMessages.sink(); private final Bytes32 homeNodeId; - private final NodeRecordV4 homeNodeRecord; + private final NodeRecord homeNodeRecord; private final NodeTable nodeTable; private final NodeBucketStorage nodeBucketStorage; private final Map recentContexts = @@ -51,17 +51,19 @@ public class DiscoveryManagerImpl implements DiscoveryManager { public DiscoveryManagerImpl( NodeTable nodeTable, NodeBucketStorage nodeBucketStorage, - NodeRecordV4 homeNode, + NodeRecord homeNode, Scheduler serverScheduler, Scheduler clientScheduler) { this.nodeTable = nodeTable; this.nodeBucketStorage = nodeBucketStorage; this.homeNodeId = homeNode.getNodeId(); - this.homeNodeRecord = (NodeRecordV4) nodeTable.getHomeNode(); + this.homeNodeRecord = (NodeRecord) nodeTable.getHomeNode(); this.authTagRepo = new AuthTagRepository(); this.scheduler = serverScheduler; this.discoveryServer = - new DiscoveryServerImpl(homeNodeRecord.getIpV4address(), homeNodeRecord.getUdpPort()); + new DiscoveryServerImpl( + ((Bytes4) homeNodeRecord.get(NodeRecord.FIELD_IP_V4)), + (int) homeNodeRecord.get(NodeRecord.FIELD_UDP_V4)); this.discoveryClient = new DiscoveryClientImpl(outgoingMessages, clientScheduler); } @@ -110,7 +112,7 @@ private Optional getContext(Bytes32 nodeId) { () -> String.format("Couldn't find node record for nodeId %s, ignoring", nodeId)); return Optional.empty(); } - NodeRecordV4 nodeRecord = nodeOptional.get().getNode(); + NodeRecord nodeRecord = nodeOptional.get().getNode(); SecureRandom random = new SecureRandom(); context = new NodeContext( diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryTaskManager.java b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryTaskManager.java index c25a197b2..304f3cdce 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryTaskManager.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryTaskManager.java @@ -1,6 +1,6 @@ package org.ethereum.beacon.discovery; -import org.ethereum.beacon.discovery.enr.NodeRecordV4; +import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.storage.NodeBucketStorage; import org.ethereum.beacon.discovery.storage.NodeTable; import org.ethereum.beacon.schedulers.Scheduler; @@ -77,7 +77,7 @@ public DiscoveryTaskManager( DiscoveryManager discoveryManager, NodeTable nodeTable, NodeBucketStorage nodeBucketStorage, - NodeRecordV4 homeNode, + NodeRecord homeNode, Scheduler scheduler, boolean resetDead) { this.scheduler = scheduler; diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeContext.java b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeContext.java index a297882f2..ecec8916f 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeContext.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeContext.java @@ -3,7 +3,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.ethereum.beacon.discovery.enr.NodeRecord; -import org.ethereum.beacon.discovery.enr.NodeRecordV4; import org.ethereum.beacon.discovery.message.DiscoveryMessage; import org.ethereum.beacon.discovery.message.MessageCode; import org.ethereum.beacon.discovery.packet.AuthHeaderMessagePacket; @@ -36,8 +35,8 @@ public class NodeContext { public static final int DEFAULT_DISTANCE = 10; // FIXME: I shouldn't be here private static final Logger logger = LogManager.getLogger(NodeContext.class); - private final NodeRecordV4 nodeRecord; - private final NodeRecordV4 homeNodeRecord; + private final NodeRecord nodeRecord; + private final NodeRecord homeNodeRecord; private final Bytes32 homeNodeId; private final AuthTagRepository authTagRepo; private final NodeTable nodeTable; @@ -53,8 +52,8 @@ public class NodeContext { private CompletableFuture connectFuture = null; public NodeContext( - NodeRecordV4 nodeRecord, - NodeRecordV4 homeNodeRecord, + NodeRecord nodeRecord, + NodeRecord homeNodeRecord, NodeTable nodeTable, NodeBucketStorage nodeBucketStorage, AuthTagRepository authTagRepo, @@ -75,7 +74,7 @@ public NodeContext( packetHandlers.put(MessagePacket.class, new MessagePacketHandler(this, logger)); } - public NodeRecordV4 getNodeRecord() { + public NodeRecord getNodeRecord() { return nodeRecord; } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeRecordInfo.java b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeRecordInfo.java index 0c1679de4..9df57717a 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeRecordInfo.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeRecordInfo.java @@ -2,7 +2,7 @@ import com.google.common.base.Objects; import org.ethereum.beacon.discovery.enr.NodeRecord; -import org.ethereum.beacon.discovery.enr.NodeRecordV4; +import org.ethereum.beacon.discovery.enr.NodeRecordFactory; import org.web3j.rlp.RlpDecoder; import org.web3j.rlp.RlpEncoder; import org.web3j.rlp.RlpList; @@ -18,6 +18,7 @@ * last test of its availability */ public class NodeRecordInfo { + private static final NodeRecordFactory nodeRecordFactory = NodeRecordFactory.DEFAULT; private final NodeRecord node; private final Long lastRetry; private final NodeStatus status; @@ -37,11 +38,10 @@ public static NodeRecordInfo createDefault(NodeRecord nodeRecord) { public static NodeRecordInfo fromRlpBytes(BytesValue bytes) { RlpList internalList = (RlpList) RlpDecoder.decode(bytes.extractArray()).getValues().get(0); return new NodeRecordInfo( - NodeRecord.fromBytes(((RlpString)internalList.getValues().get(0)).getBytes()), + nodeRecordFactory.fromBytes(((RlpString) internalList.getValues().get(0)).getBytes()), ((RlpString) internalList.getValues().get(1)).asPositiveBigInteger().longValue(), NodeStatus.fromNumber(((RlpString) internalList.getValues().get(2)).getBytes()[0]), - ((RlpString) internalList.getValues().get(1)).asPositiveBigInteger().intValue() - ); + ((RlpString) internalList.getValues().get(1)).asPositiveBigInteger().intValue()); } public BytesValue toRlpBytes() { @@ -54,8 +54,8 @@ public BytesValue toRlpBytes() { return BytesValue.wrap(bytes); } - public NodeRecordV4 getNode() { - return (NodeRecordV4) node; + public NodeRecord getNode() { + return (NodeRecord) node; } public Long getLastRetry() { @@ -88,11 +88,15 @@ public int hashCode() { @Override public String toString() { - return "NodeRecordInfo{" + - "node=" + node + - ", lastRetry=" + lastRetry + - ", status=" + status + - ", retry=" + retry + - '}'; + return "NodeRecordInfo{" + + "node=" + + node + + ", lastRetry=" + + lastRetry + + ", status=" + + status + + ", retry=" + + retry + + '}'; } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrSchemeInterpreter.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrSchemeInterpreter.java new file mode 100644 index 000000000..860ffb73f --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrSchemeInterpreter.java @@ -0,0 +1,21 @@ +package org.ethereum.beacon.discovery.enr; + +import org.web3j.rlp.RlpString; +import tech.pegasys.artemis.util.bytes.Bytes32; + +public interface EnrSchemeInterpreter { + /** Returns supported scheme */ + EnrScheme getScheme(); + + /** Verifies that `nodeRecord` is of scheme implementation */ + default boolean verify(NodeRecord nodeRecord) { + return nodeRecord.getIdentityScheme().equals(getScheme()); + } + + /** Delivers nodeId according to identity scheme scheme */ + Bytes32 getNodeId(NodeRecord nodeRecord); + + Object decode(String key, RlpString rlpString); + + RlpString encode(String key, Object object); +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrSchemeV4Interpreter.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrSchemeV4Interpreter.java new file mode 100644 index 000000000..91be46811 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrSchemeV4Interpreter.java @@ -0,0 +1,97 @@ +package org.ethereum.beacon.discovery.enr; + +import org.ethereum.beacon.crypto.Hashes; +import org.web3j.rlp.RlpString; +import tech.pegasys.artemis.util.bytes.Bytes16; +import tech.pegasys.artemis.util.bytes.Bytes32; +import tech.pegasys.artemis.util.bytes.Bytes4; +import tech.pegasys.artemis.util.bytes.BytesValue; + +import java.math.BigInteger; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Function; + +import static org.ethereum.beacon.discovery.enr.NodeRecord.FIELD_IP_V4; +import static org.ethereum.beacon.discovery.enr.NodeRecord.FIELD_IP_V6; +import static org.ethereum.beacon.discovery.enr.NodeRecord.FIELD_PKEY_SECP256K1; +import static org.ethereum.beacon.discovery.enr.NodeRecord.FIELD_TCP_V4; +import static org.ethereum.beacon.discovery.enr.NodeRecord.FIELD_TCP_V6; +import static org.ethereum.beacon.discovery.enr.NodeRecord.FIELD_UDP_V4; +import static org.ethereum.beacon.discovery.enr.NodeRecord.FIELD_UDP_V6; + +public class EnrSchemeV4Interpreter implements EnrSchemeInterpreter { + + private Map> fieldDecoders = new HashMap<>(); + + public EnrSchemeV4Interpreter() { + fieldDecoders.put(FIELD_PKEY_SECP256K1, rlpString -> BytesValue.wrap(rlpString.getBytes())); + fieldDecoders.put( + FIELD_IP_V4, rlpString -> Bytes4.wrap(BytesValue.wrap(rlpString.getBytes()), 0)); + fieldDecoders.put(FIELD_TCP_V4, rlpString -> rlpString.asPositiveBigInteger().intValue()); + fieldDecoders.put(FIELD_UDP_V4, rlpString -> rlpString.asPositiveBigInteger().intValue()); + fieldDecoders.put( + FIELD_IP_V6, rlpString -> Bytes16.wrap(BytesValue.wrap(rlpString.getBytes()), 0)); + fieldDecoders.put(FIELD_TCP_V6, rlpString -> rlpString.asPositiveBigInteger().intValue()); + fieldDecoders.put(FIELD_UDP_V6, rlpString -> rlpString.asPositiveBigInteger().intValue()); + } + + @Override + public boolean verify(NodeRecord nodeRecord) { + return EnrSchemeInterpreter.super.verify(nodeRecord) + && nodeRecord.get(FIELD_PKEY_SECP256K1) != null; + } + + @Override + public EnrScheme getScheme() { + return EnrScheme.V4; + } + + @Override + public Bytes32 getNodeId(NodeRecord nodeRecord) { + verify(nodeRecord); + return Hashes.sha256((BytesValue) nodeRecord.getKey(FIELD_PKEY_SECP256K1)); + } + + @Override + public Object decode(String key, RlpString rlpString) { + Function fieldDecoder = fieldDecoders.get(key); + if (fieldDecoder == null) { + throw new RuntimeException(String.format("No decoder found for field `%s`", key)); + } + return fieldDecoder.apply(rlpString); + } + + @Override + public RlpString encode(String key, Object object) { + if (object instanceof BytesValue) { + return fromBytesValue((BytesValue) object); + } else if (object instanceof Number) { + return fromNumber((Number) object); + } else if (object == null) { + return RlpString.create(new byte[0]); + } else { + throw new RuntimeException( + String.format( + "Couldn't serialize node record field %s with value %s: no serializer found.", + key, object)); + } + } + + private RlpString fromNumber(Number number) { + if (number instanceof BigInteger) { + return RlpString.create((BigInteger) number); + } else if (number instanceof Long) { + return RlpString.create((Long) number); + } else if (number instanceof Integer) { + return RlpString.create((Integer) number); + } else { + throw new RuntimeException( + String.format("Couldn't serialize number %s : no serializer found.", number)); + } + } + + private RlpString fromBytesValue(BytesValue bytes) { + return RlpString.create(bytes.extractArray()); + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecord.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecord.java index 1c26dab42..658d6cc90 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecord.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecord.java @@ -1,6 +1,8 @@ package org.ethereum.beacon.discovery.enr; -import org.web3j.rlp.RlpDecoder; +import com.google.common.base.Objects; +import org.javatuples.Pair; +import org.web3j.rlp.RlpEncoder; import org.web3j.rlp.RlpList; import org.web3j.rlp.RlpString; import org.web3j.rlp.RlpType; @@ -8,8 +10,12 @@ import tech.pegasys.artemis.util.bytes.BytesValue; import tech.pegasys.artemis.util.uint.UInt64; +import java.util.ArrayList; import java.util.Base64; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; /** @@ -17,96 +23,153 @@ * *

Node record as described in EIP-778 */ -public interface NodeRecord { +public class NodeRecord { // Compressed secp256k1 public key, 33 bytes - String FIELD_PKEY_SECP256K1 = "secp256k1"; + public static String FIELD_PKEY_SECP256K1 = "secp256k1"; // IPv4 address - String FIELD_IP_V4 = "ip"; + public static String FIELD_IP_V4 = "ip"; // TCP port, integer - String FIELD_TCP_V4 = "tcp"; + public static String FIELD_TCP_V4 = "tcp"; // UDP port, integer - String FIELD_UDP_V4 = "udp"; + public static String FIELD_UDP_V4 = "udp"; // IPv6 address - String FIELD_IP_V6 = "ip6"; + public static String FIELD_IP_V6 = "ip6"; // IPv6-specific TCP port - String FIELD_TCP_V6 = "tcp6"; + public static String FIELD_TCP_V6 = "tcp6"; // IPv6-specific UDP port - String FIELD_UDP_V6 = "udp6"; + public static String FIELD_UDP_V6 = "udp6"; - static NodeRecord fromBase64(String enrBase64) { - return fromBytes(Base64.getUrlDecoder().decode(enrBase64)); + private UInt64 seq; + // Signature + private BytesValue signature; + // optional fields + private Map fields = new HashMap<>(); + + private EnrSchemeInterpreter enrSchemeInterpreter; + + private NodeRecord(EnrSchemeInterpreter enrSchemeInterpreter, UInt64 seq, BytesValue signature) { + this.seq = seq; + this.signature = signature; + this.enrSchemeInterpreter = enrSchemeInterpreter; } - static NodeRecord fromBytes(byte[] bytes) { - // record = [signature, seq, k, v, ...] - RlpList rlpList = (RlpList) RlpDecoder.decode(bytes).getValues().get(0); - List values = rlpList.getValues(); - if (values.size() < 4) { - throw new RuntimeException( - String.format( - "Unable to deserialize ENR with less than 4 fields, [%s]", BytesValue.wrap(bytes))); - } - RlpString id = (RlpString) values.get(2); - if (!"id".equals(new String(id.getBytes()))) { - throw new RuntimeException( - String.format( - "Unable to deserialize ENR with no id field at 2-3 records, [%s]", - BytesValue.wrap(bytes))); - } + private NodeRecord() {} - RlpString idVersion = (RlpString) values.get(3); - EnrScheme nodeIdentity = EnrScheme.fromString(new String(idVersion.getBytes())); - if (nodeIdentity == null) { - throw new RuntimeException( - String.format( - "Unknown node identity scheme '%s', couldn't create node record.", - idVersion.asString())); - } - switch (nodeIdentity) { - case V4: - { - return NodeRecordV4.fromRlpList(values); - } - default: - { - throw new RuntimeException( - String.format("Builder for identity %s not found", nodeIdentity)); - } + public static NodeRecord fromValues( + EnrSchemeInterpreter enrSchemeInterpreter, + UInt64 seq, + BytesValue signature, + List> fieldKeyPairs) { + NodeRecord nodeRecord = new NodeRecord(enrSchemeInterpreter, seq, signature); + fieldKeyPairs.forEach(objects -> nodeRecord.set(objects.getValue0(), objects.getValue1())); + return nodeRecord; + } + + public static NodeRecord fromRawFields( + EnrSchemeInterpreter enrSchemeInterpreter, + UInt64 seq, + BytesValue signature, + List rawFields) { + NodeRecord nodeRecord = new NodeRecord(enrSchemeInterpreter, seq, signature); + for (int i = 0; i < rawFields.size(); i += 2) { + String key = new String(((RlpString) rawFields.get(i)).getBytes()); + nodeRecord.set(key, enrSchemeInterpreter.decode(key, (RlpString) rawFields.get(i + 1))); } + return nodeRecord; } - static NodeRecord fromBytes(BytesValue bytes) { - return fromBytes(bytes.extractArray()); + public String asBase64() { + return new String(Base64.getUrlEncoder().encode(serialize().extractArray())); + } + + public EnrScheme getIdentityScheme() { + return enrSchemeInterpreter.getScheme(); } - /** Every {@link EnrScheme} links with its own implementation */ - EnrScheme getIdentityScheme(); + public void set(String key, Object value) { + fields.put(key, value); + } - BytesValue getSignature(); + public Object get(String key) { + return fields.get(key); + } - /** - * @return The sequence number, a 64-bit unsigned integer. Nodes should increase the number - * whenever the record changes and republish the record. - */ - UInt64 getSeq(); + public UInt64 getSeq() { + return seq; + } - BytesValue serialize(); + public void setSeq(UInt64 seq) { + this.seq = seq; + } - default String asBase64() { - return new String(Base64.getUrlEncoder().encode(serialize().extractArray())); + public BytesValue getSignature() { + return signature; } - Bytes32 getNodeId(); + public void setSignature(BytesValue signature) { + this.signature = signature; + } + + public Set getKeys() { + return new HashSet<>(fields.keySet()); + } + + public Object getKey(String key) { + return fields.get(key); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + NodeRecord that = (NodeRecord) o; + return Objects.equal(seq, that.seq) + && Objects.equal(signature, that.signature) + && Objects.equal(fields, that.fields); + } + + @Override + public int hashCode() { + return Objects.hashCode(seq, signature, fields); + } + + public BytesValue serialize() { + assert getSignature() != null; + assert getSeq() != null; + + // content = [seq, k, v, ...] + // signature = sign(content) + // record = [signature, seq, k, v, ...] + List values = new ArrayList<>(); + values.add(RlpString.create(getSignature().extractArray())); + values.add(RlpString.create(getSeq().toBI())); + values.add(RlpString.create("id")); + values.add(RlpString.create(getIdentityScheme().stringName())); + for (Map.Entry keyPair : fields.entrySet()) { + if (keyPair.getValue() == null) { + continue; + } + values.add(RlpString.create(keyPair.getKey())); + values.add(enrSchemeInterpreter.encode(keyPair.getKey(), keyPair.getValue())); + } + byte[] bytes = RlpEncoder.encode(new RlpList(values)); + assert bytes.length <= 300; + return BytesValue.wrap(bytes); + } - /** - * All optional fields, set varies by identity scheme. `id`, identity scheme is not optional. Most - * common field names are in constants: {@link #FIELD_IP_V4} etc. (FIELD_*) - */ - Set getKeys(); + public Bytes32 getNodeId() { + return enrSchemeInterpreter.getNodeId(this); + } - /** - * @return key value or null, if no field associated with that key. Get keys using {@link - * #getKeys()} - */ - Object getKey(String key); + @Override + public String toString() { + return "NodeRecordV4{" + + "publicKey=" + + fields.get(FIELD_PKEY_SECP256K1) + + ", ipV4address=" + + fields.get(FIELD_IP_V4) + + ", udpPort=" + + fields.get(FIELD_UDP_V4) + + '}'; + } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordFactory.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordFactory.java new file mode 100644 index 000000000..4471e673b --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordFactory.java @@ -0,0 +1,91 @@ +package org.ethereum.beacon.discovery.enr; + +import org.javatuples.Pair; +import org.web3j.rlp.RlpDecoder; +import org.web3j.rlp.RlpList; +import org.web3j.rlp.RlpString; +import org.web3j.rlp.RlpType; +import tech.pegasys.artemis.util.bytes.Bytes8; +import tech.pegasys.artemis.util.bytes.BytesValue; +import tech.pegasys.artemis.util.uint.UInt64; + +import java.util.Base64; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class NodeRecordFactory { + public static final NodeRecordFactory DEFAULT = + new NodeRecordFactory(new EnrSchemeV4Interpreter()); + Map interpreters = new HashMap<>(); + + public NodeRecordFactory(EnrSchemeInterpreter... enrSchemeInterpreters) { + for (EnrSchemeInterpreter enrSchemeInterpreter : enrSchemeInterpreters) { + interpreters.put(enrSchemeInterpreter.getScheme(), enrSchemeInterpreter); + } + } + + public NodeRecord createFromValues( + EnrScheme enrScheme, + UInt64 seq, + BytesValue signature, + List> fieldKeyPairs) { + + EnrSchemeInterpreter enrSchemeInterpreter = interpreters.get(enrScheme); + if (enrSchemeInterpreter == null) { + throw new RuntimeException( + String.format("No ethereum record interpreter found for identity scheme %s", enrScheme)); + } + + return NodeRecord.fromValues(enrSchemeInterpreter, seq, signature, fieldKeyPairs); + } + + public NodeRecord fromBase64(String enrBase64) { + return fromBytes(Base64.getUrlDecoder().decode(enrBase64)); + } + + public NodeRecord fromBytes(BytesValue bytes) { + return fromBytes(bytes.extractArray()); + } + + public NodeRecord fromBytes(byte[] bytes) { + // record = [signature, seq, k, v, ...] + RlpList rlpList = (RlpList) RlpDecoder.decode(bytes).getValues().get(0); + List values = rlpList.getValues(); + if (values.size() < 4) { + throw new RuntimeException( + String.format( + "Unable to deserialize ENR with less than 4 fields, [%s]", BytesValue.wrap(bytes))); + } + RlpString id = (RlpString) values.get(2); + if (!"id".equals(new String(id.getBytes()))) { + throw new RuntimeException( + String.format( + "Unable to deserialize ENR with no id field at 2-3 records, [%s]", + BytesValue.wrap(bytes))); + } + + RlpString idVersion = (RlpString) values.get(3); + EnrScheme nodeIdentity = EnrScheme.fromString(new String(idVersion.getBytes())); + if (nodeIdentity == null) { + throw new RuntimeException( + String.format( + "Unknown node identity scheme '%s', couldn't create node record.", + idVersion.asString())); + } + + EnrSchemeInterpreter enrSchemeInterpreter = interpreters.get(nodeIdentity); + if (enrSchemeInterpreter == null) { + throw new RuntimeException( + String.format( + "No ethereum record interpreter found for identity scheme %s", nodeIdentity)); + } + + return NodeRecord.fromRawFields( + enrSchemeInterpreter, + UInt64.fromBytesBigEndian( + Bytes8.leftPad(BytesValue.wrap(((RlpString) values.get(1)).getBytes()))), + BytesValue.wrap(((RlpString) values.get(0)).getBytes()), + values.subList(4, values.size())); + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV4.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV4.java deleted file mode 100644 index ebb7bb58a..000000000 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordV4.java +++ /dev/null @@ -1,381 +0,0 @@ -package org.ethereum.beacon.discovery.enr; - -import com.google.common.base.Objects; -import org.ethereum.beacon.crypto.Hashes; -import org.javatuples.Pair; -import org.web3j.rlp.RlpEncoder; -import org.web3j.rlp.RlpList; -import org.web3j.rlp.RlpString; -import org.web3j.rlp.RlpType; -import tech.pegasys.artemis.util.bytes.Bytes16; -import tech.pegasys.artemis.util.bytes.Bytes32; -import tech.pegasys.artemis.util.bytes.Bytes4; -import tech.pegasys.artemis.util.bytes.Bytes8; -import tech.pegasys.artemis.util.bytes.BytesValue; -import tech.pegasys.artemis.util.uint.UInt64; - -import java.math.BigInteger; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.function.Function; - -/** Node record for V4 scheme. Uses secp256k1 as signature */ -public class NodeRecordV4 implements NodeRecord { - // id name of identity scheme, e.g. “v4” - private static final EnrScheme identityScheme = EnrScheme.V4; - private static final Function, NodeRecordV4> nodeRecordV4Creator = - fields -> { - NodeRecordV4.Builder builder = - NodeRecordV4.Builder.empty() - .withSignature(BytesValue.wrap(((RlpString) fields.get(0)).getBytes())) - .withSeq( - UInt64.fromBytesBigEndian( - Bytes8.leftPad(BytesValue.wrap(((RlpString) fields.get(1)).getBytes())))); - boolean secp256k1Found = false; - for (int i = 4; i < fields.size(); i += 2) { - String key = new String(((RlpString) fields.get(i)).getBytes()); - if (key.equals(FIELD_PKEY_SECP256K1)) { - secp256k1Found = true; - } - builder = builder.withKeyField(key, (RlpString) fields.get(i + 1)); - } - if (!secp256k1Found) { - throw new RuntimeException( - String.format("NodeRecord V4 requires %s field", FIELD_PKEY_SECP256K1)); - } - - return builder.build(); - }; - private UInt64 seq; - // Signature - private BytesValue signature; - // optional fields - private Map fields = new HashMap<>(); - - private NodeRecordV4( - BytesValue publicKey, - Bytes4 ipV4address, - Integer tcpPort, - Integer udpPort, - Bytes16 ipV6address, - Integer tcpV6Port, - Integer udpV6Port, - UInt64 seq, - BytesValue signature) { - fields.put(FIELD_PKEY_SECP256K1, publicKey); - fields.put(FIELD_IP_V4, ipV4address); - fields.put(FIELD_TCP_V4, tcpPort); - fields.put(FIELD_UDP_V4, udpPort); - fields.put(FIELD_IP_V6, ipV6address); - fields.put(FIELD_TCP_V6, tcpV6Port); - fields.put(FIELD_UDP_V6, udpV6Port); - this.seq = seq; - this.signature = signature; - } - - private NodeRecordV4() {} - - public static NodeRecordV4 fromValues( - BytesValue publicKey, - Bytes4 ipV4address, - Integer tcpPort, - Integer udpPort, - Bytes16 ipV6address, - Integer tcpV6Port, - Integer udpV6Port, - UInt64 seq, - BytesValue signature) { - return new NodeRecordV4( - publicKey, - ipV4address, - tcpPort, - udpPort, - ipV6address, - tcpV6Port, - udpV6Port, - seq, - signature); - } - - public static NodeRecordV4 fromRlpList(List values) { - return nodeRecordV4Creator.apply(values); - } - - public EnrScheme getIdentityScheme() { - return identityScheme; - } - - public BytesValue getPublicKey() { - return fields.containsKey(FIELD_PKEY_SECP256K1) - ? (BytesValue) fields.get(FIELD_PKEY_SECP256K1) - : null; - } - - public void setPublicKey(BytesValue publicKey) { - fields.put(FIELD_PKEY_SECP256K1, publicKey); - } - - public Bytes4 getIpV4address() { - return fields.containsKey(FIELD_IP_V4) ? (Bytes4) fields.get(FIELD_IP_V4) : null; - } - - public void setIpV4address(Bytes4 ipV4address) { - fields.put(FIELD_IP_V4, ipV4address); - } - - public Integer getTcpPort() { - return fields.containsKey(FIELD_TCP_V4) ? (Integer) fields.get(FIELD_TCP_V4) : null; - } - - public void setTcpPort(Integer tcpPort) { - fields.put(FIELD_TCP_V4, tcpPort); - } - - public Integer getUdpPort() { - return fields.containsKey(FIELD_UDP_V4) ? (Integer) fields.get(FIELD_UDP_V4) : null; - } - - public void setUdpPort(Integer udpPort) { - fields.put(FIELD_UDP_V4, udpPort); - } - - public Bytes16 getIpV6address() { - return fields.containsKey(FIELD_IP_V6) ? (Bytes16) fields.get(FIELD_IP_V6) : null; - } - - public void setIpV6address(Bytes16 ipV6address) { - fields.put(FIELD_IP_V6, ipV6address); - } - - public Integer getTcpV6Port() { - return fields.containsKey(FIELD_TCP_V6) ? (Integer) fields.get(FIELD_TCP_V6) : null; - } - - public void setTcpV6Port(Integer tcpV6Port) { - fields.put(FIELD_TCP_V6, tcpV6Port); - } - - public Integer getUdpV6Port() { - return fields.containsKey(FIELD_UDP_V6) ? (Integer) fields.get(FIELD_UDP_V6) : null; - } - - public void setUdpV6Port(Integer udpV6Port) { - fields.put(FIELD_UDP_V6, udpV6Port); - } - - public UInt64 getSeq() { - return seq; - } - - public void setSeq(UInt64 seq) { - this.seq = seq; - } - - @Override - public BytesValue getSignature() { - return signature; - } - - public void setSignature(BytesValue signature) { - this.signature = signature; - } - - @Override - public Set getKeys() { - return new HashSet<>(fields.keySet()); - } - - @Override - public Object getKey(String key) { - return fields.get(key); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - NodeRecordV4 that = (NodeRecordV4) o; - return Objects.equal(seq, that.seq) - && Objects.equal(signature, that.signature) - && Objects.equal(fields, that.fields); - } - - @Override - public int hashCode() { - return Objects.hashCode(seq, signature, fields); - } - - @Override - public BytesValue serialize() { - assert getSignature() != null; - assert getSeq() != null; - - // content = [seq, k, v, ...] - // signature = sign(content) - // record = [signature, seq, k, v, ...] - List values = new ArrayList<>(); - values.add(RlpString.create(getSignature().extractArray())); - values.add(RlpString.create(getSeq().toBI())); - values.add(RlpString.create("id")); - values.add(RlpString.create(getIdentityScheme().stringName())); - for (Map.Entry keyPair : fields.entrySet()) { - if (keyPair.getValue() == null) { - continue; - } - values.add(RlpString.create(keyPair.getKey())); - if (keyPair.getValue() instanceof BytesValue) { - values.add(fromBytes((BytesValue) keyPair.getValue())); - } else if (keyPair.getValue() instanceof Number) { - values.add(fromNumber((Number) keyPair.getValue())); - } else if (keyPair.getValue() == null) { - values.add(RlpString.create(new byte[0])); - } else { - throw new RuntimeException( - String.format( - "Couldn't serialize node record field %s with value %s: no serializer found.", - keyPair.getKey(), keyPair.getValue())); - } - } - byte[] bytes = RlpEncoder.encode(new RlpList(values)); - assert bytes.length <= 300; - return BytesValue.wrap(bytes); - } - - private RlpString fromNumber(Number number) { - if (number instanceof BigInteger) { - return RlpString.create((BigInteger) number); - } else if (number instanceof Long) { - return RlpString.create((Long) number); - } else if (number instanceof Integer) { - return RlpString.create((Integer) number); - } else { - throw new RuntimeException( - String.format("Couldn't serialize number %s : no serializer found.", number)); - } - } - - private RlpString fromBytes(BytesValue bytes) { - return RlpString.create(bytes.extractArray()); - } - - @Override - public Bytes32 getNodeId() { - return Hashes.sha256(getPublicKey()); - } - - @Override - public String toString() { - return "NodeRecordV4{" - + "publicKey=" - + fields.get(FIELD_PKEY_SECP256K1) - + ", ipV4address=" - + fields.get(FIELD_IP_V4) - + ", udpPort=" - + fields.get(FIELD_UDP_V4) - + '}'; - } - - public static class Builder { - private static final Map, Builder>> fieldFillersV4 = - new HashMap<>(); - - static { - fieldFillersV4.put( - FIELD_IP_V4, - objects -> - objects - .getValue0() - .withIpV4Address(BytesValue.wrap(((RlpString) objects.getValue1()).getBytes()))); - fieldFillersV4.put( - FIELD_PKEY_SECP256K1, - objects -> - objects - .getValue0() - .withSecp256k1(BytesValue.wrap(((RlpString) objects.getValue1()).getBytes()))); - fieldFillersV4.put( - FIELD_UDP_V4, - objects -> - objects - .getValue0() - .withUdpPort( - ((RlpString) objects.getValue1()).asPositiveBigInteger().intValue())); - fieldFillersV4.put( - FIELD_TCP_V4, - objects -> - objects - .getValue0() - .withTcpPort( - ((RlpString) objects.getValue1()).asPositiveBigInteger().intValue())); - } - - private Bytes4 ipV4Address; - private BytesValue secp256k1; - private Integer tcpPort; - private Integer udpPort; - private UInt64 seq; - private BytesValue signature; - - private Builder() {} - - public static Builder empty() { - return new Builder(); - } - - public Builder withIpV4Address(BytesValue ipV4Address) { - this.ipV4Address = Bytes4.wrap(ipV4Address, 0); - return this; - } - - public Builder withTcpPort(Integer port) { - this.tcpPort = port; - return this; - } - - public Builder withUdpPort(Integer port) { - this.udpPort = port; - return this; - } - - public Builder withSecp256k1(BytesValue bytes) { - this.secp256k1 = bytes; - return this; - } - - public Builder withSeq(UInt64 seq) { - this.seq = seq; - return this; - } - - public Builder withSignature(BytesValue signature) { - this.signature = signature; - return this; - } - - public Builder withKeyField(String key, RlpString value) { - Function, NodeRecordV4.Builder> fieldFiller = - fieldFillersV4.get(key); - if (fieldFiller == null) { - throw new RuntimeException(String.format("Couldn't find filler for V4 field '%s'", key)); - } - return fieldFiller.apply(Pair.with(this, value)); - } - - public NodeRecordV4 build() { - assert seq != null; - assert secp256k1 != null; - - NodeRecordV4 nodeRecord = new NodeRecordV4(); - nodeRecord.setIpV4address(ipV4Address); - nodeRecord.setUdpPort(udpPort); - nodeRecord.setTcpPort(tcpPort); - nodeRecord.setSeq(seq); - nodeRecord.setSignature(signature); - nodeRecord.setPublicKey(secp256k1); - return nodeRecord; - } - } -} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/DiscoveryV5Message.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/DiscoveryV5Message.java index 521eb98a5..88614701f 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/message/DiscoveryV5Message.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/DiscoveryV5Message.java @@ -1,8 +1,7 @@ package org.ethereum.beacon.discovery.message; import org.ethereum.beacon.discovery.IdentityScheme; -import org.ethereum.beacon.discovery.enr.NodeRecord; -import org.ethereum.beacon.discovery.enr.NodeRecordV4; +import org.ethereum.beacon.discovery.enr.NodeRecordFactory; import org.web3j.rlp.RlpDecoder; import org.web3j.rlp.RlpList; import org.web3j.rlp.RlpString; @@ -15,6 +14,7 @@ import java.util.stream.Collectors; public class DiscoveryV5Message implements DiscoveryMessage { + private static final NodeRecordFactory nodeRecordFactory = NodeRecordFactory.DEFAULT; private final BytesValue bytes; private List payload = null; @@ -87,7 +87,7 @@ public V5Message create() { ((RlpString) payload.get(1)).asPositiveBigInteger().intValueExact(), () -> nodeRecords.getValues().stream() - .map(rs -> (NodeRecordV4) (NodeRecord.fromBytes(((RlpString) rs).getBytes()))) + .map(rs -> nodeRecordFactory.fromBytes(((RlpString) rs).getBytes())) .collect(Collectors.toList()), nodeRecords.getValues().size()); } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/NodesMessage.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/NodesMessage.java index ae31b7676..384c17a8c 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/message/NodesMessage.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/NodesMessage.java @@ -1,7 +1,7 @@ package org.ethereum.beacon.discovery.message; import com.google.common.base.Objects; -import org.ethereum.beacon.discovery.enr.NodeRecordV4; +import org.ethereum.beacon.discovery.enr.NodeRecord; import org.web3j.rlp.RlpEncoder; import org.web3j.rlp.RlpList; import org.web3j.rlp.RlpString; @@ -22,15 +22,15 @@ public class NodesMessage implements V5Message { // Total number of responses to the request private final Integer total; // List of nodes upon request - private final Supplier> nodeRecordsSupplier; + private final Supplier> nodeRecordsSupplier; // Size of nodes in current response private final Integer nodeRecordsSize; - private List nodeRecords = null; + private List nodeRecords = null; public NodesMessage( BytesValue requestId, Integer total, - Supplier> nodeRecordsSupplier, + Supplier> nodeRecordsSupplier, Integer nodeRecordsSize) { this.requestId = requestId; this.total = total; @@ -47,7 +47,7 @@ public Integer getTotal() { return total; } - public synchronized List getNodeRecords() { + public synchronized List getNodeRecords() { if (nodeRecords == null) { this.nodeRecords = nodeRecordsSupplier.get(); } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/PingHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/PingHandler.java index 70260eb31..8f57878c7 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/PingHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/PingHandler.java @@ -1,10 +1,12 @@ package org.ethereum.beacon.discovery.message.handler; import org.ethereum.beacon.discovery.NodeContext; +import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.message.DiscoveryV5Message; import org.ethereum.beacon.discovery.message.PingMessage; import org.ethereum.beacon.discovery.message.PongMessage; import org.ethereum.beacon.discovery.packet.MessagePacket; +import tech.pegasys.artemis.util.bytes.Bytes4; public class PingHandler implements MessageHandler { @Override @@ -13,8 +15,8 @@ public void handle(PingMessage message, NodeContext context) { new PongMessage( message.getRequestId(), context.getNodeRecord().getSeq(), - context.getNodeRecord().getIpV4address(), - context.getNodeRecord().getUdpPort()); + ((Bytes4) context.getNodeRecord().get(NodeRecord.FIELD_IP_V4)), + (int) context.getNodeRecord().get(NodeRecord.FIELD_UDP_V4)); context.addOutgoingEvent( MessagePacket.create( context.getHomeNodeId(), diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/network/DiscoveryClientImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/network/DiscoveryClientImpl.java index c66058f47..52b4c8bba 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/network/DiscoveryClientImpl.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/network/DiscoveryClientImpl.java @@ -9,11 +9,12 @@ import io.netty.channel.socket.nio.NioDatagramChannel; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.ethereum.beacon.discovery.enr.EnrScheme; import org.ethereum.beacon.discovery.enr.NodeRecord; -import org.ethereum.beacon.discovery.enr.NodeRecordV4; import org.ethereum.beacon.schedulers.Scheduler; import org.reactivestreams.Publisher; import reactor.core.publisher.Flux; +import tech.pegasys.artemis.util.bytes.Bytes4; import tech.pegasys.artemis.util.bytes.BytesValue; import java.net.InetAddress; @@ -107,10 +108,10 @@ public void stop() { @Override public void send(BytesValue data, NodeRecord recipient) { - if (!(recipient instanceof NodeRecordV4)) { + if (!(recipient.getIdentityScheme().equals(EnrScheme.V4))) { String error = String.format( - "Accepts only V4 versions of recipient's node records. Got %s instead", recipient); + "Accepts only V4 version of recipient's node records. Got %s instead", recipient); logger.error(error); throw new RuntimeException(error); } @@ -118,8 +119,9 @@ public void send(BytesValue data, NodeRecord recipient) { try { address = new InetSocketAddress( - InetAddress.getByAddress(((NodeRecordV4) recipient).getIpV4address().extractArray()), - ((NodeRecordV4) recipient).getUdpPort()); + InetAddress.getByAddress( + ((Bytes4) recipient.get(NodeRecord.FIELD_IP_V4)).extractArray()), + (int) recipient.get(NodeRecord.FIELD_UDP_V4)); } catch (UnknownHostException e) { String error = String.format("Failed to resolve host for node record: %s", recipient); logger.error(error); diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/network/NetworkParcelV5.java b/discovery/src/main/java/org/ethereum/beacon/discovery/network/NetworkParcelV5.java index 997033154..2d5f58b47 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/network/NetworkParcelV5.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/network/NetworkParcelV5.java @@ -1,14 +1,13 @@ package org.ethereum.beacon.discovery.network; import org.ethereum.beacon.discovery.enr.NodeRecord; -import org.ethereum.beacon.discovery.enr.NodeRecordV4; import org.ethereum.beacon.discovery.packet.Packet; public class NetworkParcelV5 implements NetworkParcel { private final Packet packet; - private final NodeRecordV4 nodeRecord; + private final NodeRecord nodeRecord; - public NetworkParcelV5(Packet packet, NodeRecordV4 nodeRecord) { + public NetworkParcelV5(Packet packet, NodeRecord nodeRecord) { this.packet = packet; this.nodeRecord = nodeRecord; } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/AuthHeaderMessagePacket.java b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/AuthHeaderMessagePacket.java index 6f98544bd..99ed4c96e 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/AuthHeaderMessagePacket.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/AuthHeaderMessagePacket.java @@ -2,7 +2,7 @@ import org.ethereum.beacon.discovery.Functions; import org.ethereum.beacon.discovery.enr.NodeRecord; -import org.ethereum.beacon.discovery.enr.NodeRecordV4; +import org.ethereum.beacon.discovery.enr.NodeRecordFactory; import org.ethereum.beacon.discovery.message.DiscoveryMessage; import org.ethereum.beacon.discovery.message.DiscoveryV5Message; import org.web3j.rlp.RlpDecoder; @@ -40,6 +40,7 @@ */ public class AuthHeaderMessagePacket extends AbstractPacket { public static final String AUTH_SCHEME_NAME = "gcm"; + private static final NodeRecordFactory nodeRecordFactory = NodeRecordFactory.DEFAULT; private static final BytesValue DISCOVERY_ID_NONCE = BytesValue.wrap("discovery-id-nonce".getBytes()); private static final BytesValue ZERO_NONCE = BytesValue.wrap(new byte[12]); @@ -122,7 +123,7 @@ public BytesValue getIdNonceSig() { return decoded.idNonceSig; } - public NodeRecordV4 getNodeRecord() { + public NodeRecord getNodeRecord() { verifyDecode(); return decoded.nodeRecord; } @@ -165,8 +166,8 @@ public void decode(BytesValue initiatorKey, BytesValue authResponseKey) { blank.idNonceSig = BytesValue.wrap(((RlpString) authResponsePtParts.getValues().get(1)).getBytes()); blank.nodeRecord = - (NodeRecordV4) - NodeRecord.fromBytes(((RlpString) authResponsePtParts.getValues().get(2)).getBytes()); + nodeRecordFactory.fromBytes( + ((RlpString) authResponsePtParts.getValues().get(2)).getBytes()); BytesValue messageAad = blank.tag.concat(getBytes().slice(32)); blank.message = new DiscoveryV5Message( @@ -205,7 +206,7 @@ private static class MessagePacketDecoded { private BytesValue idNonce; private BytesValue ephemeralPubkey; private BytesValue idNonceSig; - private NodeRecordV4 nodeRecord; + private NodeRecord nodeRecord; private DiscoveryMessage message; } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/handler/AuthHeaderMessagePacketHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/handler/AuthHeaderMessagePacketHandler.java index e7dfc6607..0a3855309 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/handler/AuthHeaderMessagePacketHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/handler/AuthHeaderMessagePacketHandler.java @@ -3,6 +3,7 @@ import org.apache.logging.log4j.Logger; import org.ethereum.beacon.discovery.Functions; import org.ethereum.beacon.discovery.NodeContext; +import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.packet.AuthHeaderMessagePacket; import org.javatuples.Triplet; import org.web3j.crypto.ECKeyPair; @@ -29,7 +30,7 @@ public boolean handle(AuthHeaderMessagePacket packet) { context.getNodeRecord().getNodeId(), BytesValue.wrap(ephemeralKey.getPrivateKey().toByteArray()), context.getIdNonce(), - context.getNodeRecord().getPublicKey()); + (BytesValue) context.getNodeRecord().get(NodeRecord.FIELD_PKEY_SECP256K1)); BytesValue initiatorKey = hkdf.getValue0(); context.setInitiatorKey(initiatorKey); BytesValue authResponseKey = hkdf.getValue2(); diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/handler/WhoAreYouPacketHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/handler/WhoAreYouPacketHandler.java index a176002bc..44fe392a7 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/handler/WhoAreYouPacketHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/handler/WhoAreYouPacketHandler.java @@ -3,6 +3,7 @@ import org.apache.logging.log4j.Logger; import org.ethereum.beacon.discovery.Functions; import org.ethereum.beacon.discovery.NodeContext; +import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.message.DiscoveryV5Message; import org.ethereum.beacon.discovery.message.FindNodeMessage; import org.ethereum.beacon.discovery.message.MessageCode; @@ -38,7 +39,7 @@ public boolean handle(WhoAreYouPacket packet) { context.getNodeRecord().getNodeId(), BytesValue.wrap(ephemeralKey.getPrivateKey().toByteArray()), packet.getIdNonce(), - context.getNodeRecord().getPublicKey()); + (BytesValue) context.getNodeRecord().get(NodeRecord.FIELD_PKEY_SECP256K1)); BytesValue initiatorKey = hkdf.getValue0(); BytesValue staticNodeKey = hkdf.getValue1(); BytesValue authResponseKey = hkdf.getValue2(); diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactory.java b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactory.java index 28856366f..d291b68be 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactory.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactory.java @@ -3,7 +3,6 @@ import org.ethereum.beacon.chain.storage.impl.SerializerFactory; import org.ethereum.beacon.db.Database; import org.ethereum.beacon.discovery.enr.NodeRecord; -import org.ethereum.beacon.discovery.enr.NodeRecordV4; import tech.pegasys.artemis.util.bytes.Bytes32; import java.util.List; @@ -13,7 +12,7 @@ public interface NodeTableStorageFactory { NodeTableStorage createTable( Database database, SerializerFactory serializerFactory, - Supplier homeNodeSupplier, + Supplier homeNodeSupplier, Supplier> bootNodes); NodeBucketStorage createBuckets( diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactoryImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactoryImpl.java index aaf8ce4ea..db2bb5a7f 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactoryImpl.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactoryImpl.java @@ -4,7 +4,6 @@ import org.ethereum.beacon.db.Database; import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.NodeRecordInfo; -import org.ethereum.beacon.discovery.enr.NodeRecordV4; import tech.pegasys.artemis.util.bytes.Bytes32; import java.util.List; @@ -25,7 +24,7 @@ private boolean isStorageEmpty(NodeTableStorage nodeTableStorage) { public NodeTableStorage createTable( Database database, SerializerFactory serializerFactory, - Supplier homeNodeSupplier, + Supplier homeNodeSupplier, Supplier> bootNodesSupplier) { NodeTableStorage nodeTableStorage = new NodeTableStorageImpl(database, serializerFactory); @@ -38,7 +37,7 @@ public NodeTableStorage createTable( .get() .forEach( nodeRecord -> { - if (!(nodeRecord instanceof NodeRecordV4)) { + if (!(nodeRecord instanceof NodeRecord)) { throw new RuntimeException("Only V4 node records are supported as boot nodes"); } NodeRecordInfo nodeRecordInfo = NodeRecordInfo.createDefault(nodeRecord); diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java index e330cf47c..9abfb1878 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java @@ -1,8 +1,9 @@ package org.ethereum.beacon.discovery; import org.ethereum.beacon.db.Database; +import org.ethereum.beacon.discovery.enr.EnrScheme; import org.ethereum.beacon.discovery.enr.NodeRecord; -import org.ethereum.beacon.discovery.enr.NodeRecordV4; +import org.ethereum.beacon.discovery.enr.NodeRecordFactory; import org.ethereum.beacon.discovery.message.DiscoveryMessage; import org.ethereum.beacon.discovery.message.DiscoveryV5Message; import org.ethereum.beacon.discovery.message.FindNodeMessage; @@ -16,8 +17,10 @@ import org.ethereum.beacon.discovery.storage.NodeTableStorage; import org.ethereum.beacon.discovery.storage.NodeTableStorageFactoryImpl; import org.ethereum.beacon.schedulers.Schedulers; +import org.javatuples.Pair; import org.junit.Test; import reactor.core.publisher.Flux; +import tech.pegasys.artemis.util.bytes.Bytes4; import tech.pegasys.artemis.util.bytes.Bytes96; import tech.pegasys.artemis.util.bytes.BytesValue; import tech.pegasys.artemis.util.uint.UInt64; @@ -32,29 +35,49 @@ /** Same as {@link DiscoveryNoNetworkTest} but using real network */ public class DiscoveryNetworkTest { + private static final NodeRecordFactory NODE_RECORD_FACTORY = NodeRecordFactory.DEFAULT; + @Test public void test() throws Exception { // 1) start 2 nodes - NodeRecordV4 nodeRecord1 = - NodeRecordV4.Builder.empty() - .withIpV4Address(BytesValue.wrap(InetAddress.getByName("127.0.0.1").getAddress())) - .withSeq(UInt64.valueOf(1)) - .withUdpPort(30303) - .withSecp256k1( - BytesValue.fromHexString( - "0bfb48004b1698f05872cf18b1f278998ad8f7d4c135aa41f83744e7b850ab6b98")) - .withSignature(Bytes96.EMPTY) - .build(); - NodeRecordV4 nodeRecord2 = - NodeRecordV4.Builder.empty() - .withIpV4Address(BytesValue.wrap(InetAddress.getByName("127.0.0.1").getAddress())) - .withSeq(UInt64.valueOf(1)) - .withUdpPort(30304) - .withSecp256k1( - BytesValue.fromHexString( - "7ef3502240a42891771de732f5ee6bee3eb881939edf3e6008c0d07b502756e426")) - .withSignature(Bytes96.EMPTY) - .build(); + NodeRecord nodeRecord1 = + NODE_RECORD_FACTORY.createFromValues( + EnrScheme.V4, + UInt64.valueOf(1), + Bytes96.EMPTY, + new ArrayList>() { + { + add( + Pair.with( + NodeRecord.FIELD_IP_V4, + Bytes4.wrap(InetAddress.getByName("127.0.0.1").getAddress()))); + add(Pair.with(NodeRecord.FIELD_UDP_V4, 30303)); + add( + Pair.with( + NodeRecord.FIELD_PKEY_SECP256K1, + BytesValue.fromHexString( + "0bfb48004b1698f05872cf18b1f278998ad8f7d4c135aa41f83744e7b850ab6b98"))); + } + }); + NodeRecord nodeRecord2 = + NODE_RECORD_FACTORY.createFromValues( + EnrScheme.V4, + UInt64.valueOf(1), + Bytes96.EMPTY, + new ArrayList>() { + { + add( + Pair.with( + NodeRecord.FIELD_IP_V4, + Bytes4.wrap(InetAddress.getByName("127.0.0.1").getAddress()))); + add(Pair.with(NodeRecord.FIELD_UDP_V4, 30304)); + add( + Pair.with( + NodeRecord.FIELD_PKEY_SECP256K1, + BytesValue.fromHexString( + "7ef3502240a42891771de732f5ee6bee3eb881939edf3e6008c0d07b502756e426"))); + } + }); NodeTableStorageFactoryImpl nodeTableStorageFactory = new NodeTableStorageFactoryImpl(); Database database1 = Database.inMemoryDB(); Database database2 = Database.inMemoryDB(); diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java index a0804934d..930e18a58 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java @@ -1,8 +1,9 @@ package org.ethereum.beacon.discovery; import org.ethereum.beacon.db.Database; +import org.ethereum.beacon.discovery.enr.EnrScheme; import org.ethereum.beacon.discovery.enr.NodeRecord; -import org.ethereum.beacon.discovery.enr.NodeRecordV4; +import org.ethereum.beacon.discovery.enr.NodeRecordFactory; import org.ethereum.beacon.discovery.message.DiscoveryMessage; import org.ethereum.beacon.discovery.message.DiscoveryV5Message; import org.ethereum.beacon.discovery.message.FindNodeMessage; @@ -18,8 +19,10 @@ import org.ethereum.beacon.discovery.storage.NodeTableStorageFactoryImpl; import org.ethereum.beacon.schedulers.Schedulers; import org.ethereum.beacon.stream.SimpleProcessor; +import org.javatuples.Pair; import org.junit.Test; import reactor.core.publisher.Flux; +import tech.pegasys.artemis.util.bytes.Bytes4; import tech.pegasys.artemis.util.bytes.Bytes96; import tech.pegasys.artemis.util.bytes.BytesValue; import tech.pegasys.artemis.util.uint.UInt64; @@ -37,29 +40,49 @@ * incoming of another and vice versa */ public class DiscoveryNoNetworkTest { + private static final NodeRecordFactory NODE_RECORD_FACTORY = NodeRecordFactory.DEFAULT; + @Test public void test() throws Exception { // 1) start 2 nodes - NodeRecordV4 nodeRecord1 = - NodeRecordV4.Builder.empty() - .withIpV4Address(BytesValue.wrap(InetAddress.getByName("127.0.0.1").getAddress())) - .withSeq(UInt64.valueOf(1)) - .withUdpPort(30303) - .withSecp256k1( - BytesValue.fromHexString( - "0bfb48004b1698f05872cf18b1f278998ad8f7d4c135aa41f83744e7b850ab6b98")) - .withSignature(Bytes96.EMPTY) - .build(); - NodeRecordV4 nodeRecord2 = - NodeRecordV4.Builder.empty() - .withIpV4Address(BytesValue.wrap(InetAddress.getByName("192.168.0.1").getAddress())) - .withSeq(UInt64.valueOf(1)) - .withUdpPort(30303) - .withSecp256k1( - BytesValue.fromHexString( - "7ef3502240a42891771de732f5ee6bee3eb881939edf3e6008c0d07b502756e426")) - .withSignature(Bytes96.EMPTY) - .build(); + NodeRecord nodeRecord1 = + NODE_RECORD_FACTORY.createFromValues( + EnrScheme.V4, + UInt64.valueOf(1), + Bytes96.EMPTY, + new ArrayList>() { + { + add( + Pair.with( + NodeRecord.FIELD_IP_V4, + Bytes4.wrap(InetAddress.getByName("127.0.0.1").getAddress()))); + add(Pair.with(NodeRecord.FIELD_UDP_V4, 30303)); + add( + Pair.with( + NodeRecord.FIELD_PKEY_SECP256K1, + BytesValue.fromHexString( + "0bfb48004b1698f05872cf18b1f278998ad8f7d4c135aa41f83744e7b850ab6b98"))); + } + }); + NodeRecord nodeRecord2 = + NODE_RECORD_FACTORY.createFromValues( + EnrScheme.V4, + UInt64.valueOf(1), + Bytes96.EMPTY, + new ArrayList>() { + { + add( + Pair.with( + NodeRecord.FIELD_IP_V4, + Bytes4.wrap(InetAddress.getByName("192.168.0.1").getAddress()))); + add(Pair.with(NodeRecord.FIELD_UDP_V4, 30303)); + add( + Pair.with( + NodeRecord.FIELD_PKEY_SECP256K1, + BytesValue.fromHexString( + "7ef3502240a42891771de732f5ee6bee3eb881939edf3e6008c0d07b502756e426"))); + } + }); NodeTableStorageFactoryImpl nodeTableStorageFactory = new NodeTableStorageFactoryImpl(); Database database1 = Database.inMemoryDB(); Database database2 = Database.inMemoryDB(); diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/NodeRecordTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/NodeRecordTest.java index 373680279..540aa0cfd 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/NodeRecordTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/NodeRecordTest.java @@ -2,7 +2,7 @@ import org.ethereum.beacon.discovery.enr.EnrScheme; import org.ethereum.beacon.discovery.enr.NodeRecord; -import org.ethereum.beacon.discovery.enr.NodeRecordV4; +import org.ethereum.beacon.discovery.enr.NodeRecordFactory; import org.junit.Test; import tech.pegasys.artemis.util.bytes.BytesValue; import tech.pegasys.artemis.util.uint.UInt64; @@ -19,6 +19,7 @@ * href="https://eips.ethereum.org/EIPS/eip-778">https://eips.ethereum.org/EIPS/eip-778 */ public class NodeRecordTest { + private static final NodeRecordFactory NODE_RECORD_FACTORY = NodeRecordFactory.DEFAULT; @Test public void testLocalhostV4() throws Exception { @@ -35,32 +36,31 @@ public void testLocalhostV4() throws Exception { final String localhostEnr = "-IS4QHCYrYZbAKWCBRlAy5zzaDZXJBGkcnh4MHcBFZntXNFrdvJjX04jRzjzCBOonrkTfj499SZuOh8R33Ls8RRcy5wBgmlkgnY0gmlwhH8AAAGJc2VjcDI1NmsxoQPKY0yuDUmstAHYpMa2_oxVtw0RW_QAdpzBQA8yWM0xOIN1ZHCCdl8"; - NodeRecord nodeRecord = NodeRecord.fromBase64(localhostEnr); + NodeRecord nodeRecord = NODE_RECORD_FACTORY.fromBase64(localhostEnr); assertEquals(EnrScheme.V4, nodeRecord.getIdentityScheme()); - NodeRecordV4 nodeRecordV4 = (NodeRecordV4) nodeRecord; assertArrayEquals( InetAddress.getByName(expectedHost).getAddress(), - nodeRecordV4.getIpV4address().extractArray()); - assertEquals(expectedUdpPort, nodeRecordV4.getUdpPort()); - assertEquals(expectedTcpPort, nodeRecordV4.getTcpPort()); - assertEquals(expectedSeqNumber, nodeRecordV4.getSeq()); - assertEquals(expectedPublicKey, nodeRecordV4.getPublicKey()); - assertEquals(expectedSignature, nodeRecordV4.getSignature()); + ((BytesValue) nodeRecord.get(NodeRecord.FIELD_IP_V4)).extractArray()); + assertEquals(expectedUdpPort, nodeRecord.get(NodeRecord.FIELD_UDP_V4)); + assertEquals(expectedTcpPort, nodeRecord.get(NodeRecord.FIELD_TCP_V4)); + assertEquals(expectedSeqNumber, nodeRecord.getSeq()); + assertEquals(expectedPublicKey, nodeRecord.get(NodeRecord.FIELD_PKEY_SECP256K1)); + assertEquals(expectedSignature, nodeRecord.getSignature()); - String localhostEnrRestored = nodeRecordV4.asBase64(); + String localhostEnrRestored = nodeRecord.asBase64(); // The order of fields is not strict so we don't compare strings - NodeRecord nodeRecordRestored = NodeRecord.fromBase64(localhostEnrRestored); + NodeRecord nodeRecordRestored = NODE_RECORD_FACTORY.fromBase64(localhostEnrRestored); assertEquals(EnrScheme.V4, nodeRecordRestored.getIdentityScheme()); - NodeRecordV4 nodeRecordV4Restored = (NodeRecordV4) nodeRecordRestored; + NodeRecord nodeRecordV4Restored = (NodeRecord) nodeRecordRestored; assertArrayEquals( InetAddress.getByName(expectedHost).getAddress(), - nodeRecordV4Restored.getIpV4address().extractArray()); - assertEquals(expectedUdpPort, nodeRecordV4Restored.getUdpPort()); - assertEquals(expectedTcpPort, nodeRecordV4Restored.getTcpPort()); + ((BytesValue) nodeRecordV4Restored.get(NodeRecord.FIELD_IP_V4)).extractArray()); + assertEquals(expectedUdpPort, nodeRecordV4Restored.get(NodeRecord.FIELD_UDP_V4)); + assertEquals(expectedTcpPort, nodeRecordV4Restored.get(NodeRecord.FIELD_TCP_V4)); assertEquals(expectedSeqNumber, nodeRecordV4Restored.getSeq()); - assertEquals(expectedPublicKey, nodeRecordV4Restored.getPublicKey()); + assertEquals(expectedPublicKey, nodeRecordV4Restored.get(NodeRecord.FIELD_PKEY_SECP256K1)); assertEquals(expectedSignature, nodeRecordV4Restored.getSignature()); } } diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java b/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java index afbcfeb67..a35823cf8 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java @@ -6,7 +6,6 @@ import org.ethereum.beacon.discovery.NodeContext; import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.NodeRecordInfo; -import org.ethereum.beacon.discovery.enr.NodeRecordV4; import org.ethereum.beacon.discovery.network.NetworkParcel; import org.ethereum.beacon.discovery.network.NetworkParcelV5; import org.ethereum.beacon.discovery.packet.UnknownPacket; @@ -36,7 +35,7 @@ public class DiscoveryManagerNoNetwork implements DiscoveryManager { private final ReplayProcessor outgoingMessages = ReplayProcessor.cacheLast(); private final FluxSink outgoingSink = outgoingMessages.sink(); private final Bytes32 homeNodeId; - private final NodeRecordV4 homeNodeRecord; + private final NodeRecord homeNodeRecord; private final NodeTable nodeTable; private final NodeBucketStorage nodeBucketStorage; private Publisher incomingPackets; @@ -46,13 +45,13 @@ public class DiscoveryManagerNoNetwork implements DiscoveryManager { public DiscoveryManagerNoNetwork( NodeTable nodeTable, NodeBucketStorage nodeBucketStorage, - NodeRecordV4 homeNode, + NodeRecord homeNode, Publisher incomingPackets) { this.nodeTable = nodeTable; this.nodeBucketStorage = nodeBucketStorage; this.incomingPackets = incomingPackets; this.homeNodeId = homeNode.getNodeId(); - this.homeNodeRecord = (NodeRecordV4) nodeTable.getHomeNode(); + this.homeNodeRecord = (NodeRecord) nodeTable.getHomeNode(); this.authTagRepo = new AuthTagRepository(); } @@ -96,7 +95,7 @@ private Optional getContext(Bytes32 nodeId) { () -> String.format("Couldn't find node record for nodeId %s, ignoring", nodeId)); return Optional.empty(); } - NodeRecordV4 nodeRecord = nodeOptional.get().getNode(); + NodeRecord nodeRecord = nodeOptional.get().getNode(); SecureRandom random = new SecureRandom(); context = new NodeContext( diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeBucketTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeBucketTest.java index 4089ced1c..90e87a7ab 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeBucketTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeBucketTest.java @@ -3,14 +3,20 @@ import org.ethereum.beacon.db.Database; import org.ethereum.beacon.discovery.Functions; import org.ethereum.beacon.discovery.NodeRecordInfo; -import org.ethereum.beacon.discovery.enr.NodeRecordV4; import org.ethereum.beacon.discovery.NodeStatus; +import org.ethereum.beacon.discovery.enr.EnrScheme; +import org.ethereum.beacon.discovery.enr.NodeRecord; +import org.ethereum.beacon.discovery.enr.NodeRecordFactory; +import org.javatuples.Pair; import org.junit.Test; +import tech.pegasys.artemis.util.bytes.Bytes4; +import tech.pegasys.artemis.util.bytes.Bytes96; import tech.pegasys.artemis.util.bytes.BytesValue; import tech.pegasys.artemis.util.uint.UInt64; import java.net.InetAddress; import java.net.UnknownHostException; +import java.util.ArrayList; import java.util.Random; import java.util.stream.IntStream; @@ -20,19 +26,28 @@ import static org.junit.Assert.assertTrue; public class NodeBucketTest { + private static final NodeRecordFactory NODE_RECORD_FACTORY = NodeRecordFactory.DEFAULT; private final Random rnd = new Random(); private NodeRecordInfo generateUniqueRecord() { try { byte[] pkey = new byte[33]; rnd.nextBytes(pkey); - NodeRecordV4 nodeRecord = - NodeRecordV4.Builder.empty() - .withIpV4Address(BytesValue.wrap(InetAddress.getByName("127.0.0.1").getAddress())) - .withSeq(UInt64.valueOf(1)) - .withUdpPort(30303) - .withSecp256k1(BytesValue.wrap(pkey)) - .build(); + NodeRecord nodeRecord = + NODE_RECORD_FACTORY.createFromValues( + EnrScheme.V4, + UInt64.valueOf(1), + Bytes96.EMPTY, + new ArrayList>() { + { + add( + Pair.with( + NodeRecord.FIELD_IP_V4, + Bytes4.wrap(InetAddress.getByName("127.0.0.1").getAddress()))); + add(Pair.with(NodeRecord.FIELD_UDP_V4, 30303)); + add(Pair.with(NodeRecord.FIELD_PKEY_SECP256K1, BytesValue.wrap(pkey))); + } + }); return new NodeRecordInfo(nodeRecord, (long) rnd.nextInt(1000), NodeStatus.ACTIVE, 0); } catch (UnknownHostException e) { throw new RuntimeException(e); diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeTableTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeTableTest.java index 4d8cf6a48..f595e180d 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeTableTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeTableTest.java @@ -1,12 +1,15 @@ package org.ethereum.beacon.discovery.storage; import org.ethereum.beacon.db.Database; -import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.NodeRecordInfo; -import org.ethereum.beacon.discovery.enr.NodeRecordV4; import org.ethereum.beacon.discovery.NodeStatus; +import org.ethereum.beacon.discovery.enr.EnrScheme; +import org.ethereum.beacon.discovery.enr.NodeRecord; +import org.ethereum.beacon.discovery.enr.NodeRecordFactory; +import org.javatuples.Pair; import org.junit.Test; import tech.pegasys.artemis.util.bytes.Bytes32; +import tech.pegasys.artemis.util.bytes.Bytes4; import tech.pegasys.artemis.util.bytes.Bytes96; import tech.pegasys.artemis.util.bytes.BytesValue; import tech.pegasys.artemis.util.uint.UInt64; @@ -23,19 +26,29 @@ import static org.junit.Assert.assertTrue; public class NodeTableTest { + private static final NodeRecordFactory NODE_RECORD_FACTORY = NodeRecordFactory.DEFAULT; - private Supplier homeNodeSupplier = + private Supplier homeNodeSupplier = () -> { try { - return NodeRecordV4.Builder.empty() - .withIpV4Address(BytesValue.wrap(InetAddress.getByName("127.0.0.1").getAddress())) - .withSeq(UInt64.valueOf(1)) - .withUdpPort(30303) - .withSecp256k1( - BytesValue.fromHexString( - "0bfb48004b1698f05872cf18b1f278998ad8f7d4c135aa41f83744e7b850ab6b98")) - .withSignature(Bytes96.EMPTY) - .build(); + return NODE_RECORD_FACTORY.createFromValues( + EnrScheme.V4, + UInt64.valueOf(1), + Bytes96.EMPTY, + new ArrayList>() { + { + add( + Pair.with( + NodeRecord.FIELD_IP_V4, + Bytes4.wrap(InetAddress.getByName("127.0.0.1").getAddress()))); + add(Pair.with(NodeRecord.FIELD_UDP_V4, 30303)); + add( + Pair.with( + NodeRecord.FIELD_PKEY_SECP256K1, + BytesValue.fromHexString( + "0bfb48004b1698f05872cf18b1f278998ad8f7d4c135aa41f83744e7b850ab6b98"))); + } + }); } catch (UnknownHostException e) { throw new RuntimeException(e); } @@ -45,7 +58,7 @@ public class NodeTableTest { public void testCreate() throws Exception { final String localhostEnr = "-IS4QHCYrYZbAKWCBRlAy5zzaDZXJBGkcnh4MHcBFZntXNFrdvJjX04jRzjzCBOonrkTfj499SZuOh8R33Ls8RRcy5wBgmlkgnY0gmlwhH8AAAGJc2VjcDI1NmsxoQPKY0yuDUmstAHYpMa2_oxVtw0RW_QAdpzBQA8yWM0xOIN1ZHCCdl8"; - NodeRecordV4 nodeRecord = (NodeRecordV4) NodeRecord.fromBase64(localhostEnr); + NodeRecord nodeRecord = NODE_RECORD_FACTORY.fromBase64(localhostEnr); NodeTableStorageFactoryImpl nodeTableStorageFactory = new NodeTableStorageFactoryImpl(); Database database = Database.inMemoryDB(); NodeTableStorage nodeTableStorage = @@ -58,21 +71,21 @@ public void testCreate() throws Exception { nodes.add(nodeRecord); return nodes; }); - Optional extendedEnr = - nodeTableStorage.get().getNode(nodeRecord.getNodeId()); + Optional extendedEnr = nodeTableStorage.get().getNode(nodeRecord.getNodeId()); assertTrue(extendedEnr.isPresent()); NodeRecordInfo nodeRecord2 = extendedEnr.get(); - assertEquals(nodeRecord.getPublicKey(), nodeRecord2.getNode().getPublicKey()); assertEquals( - nodeTableStorage.get().getHomeNode().getNodeId(), - homeNodeSupplier.get().getNodeId()); + nodeRecord.get(NodeRecord.FIELD_PKEY_SECP256K1), + nodeRecord2.getNode().get(NodeRecord.FIELD_PKEY_SECP256K1)); + assertEquals( + nodeTableStorage.get().getHomeNode().getNodeId(), homeNodeSupplier.get().getNodeId()); } @Test public void testFind() throws Exception { final String localhostEnr = "-IS4QHCYrYZbAKWCBRlAy5zzaDZXJBGkcnh4MHcBFZntXNFrdvJjX04jRzjzCBOonrkTfj499SZuOh8R33Ls8RRcy5wBgmlkgnY0gmlwhH8AAAGJc2VjcDI1NmsxoQPKY0yuDUmstAHYpMa2_oxVtw0RW_QAdpzBQA8yWM0xOIN1ZHCCdl8"; - NodeRecordV4 localHostNode = (NodeRecordV4) NodeRecord.fromBase64(localhostEnr); + NodeRecord localHostNode = NODE_RECORD_FACTORY.fromBase64(localhostEnr); NodeTableStorageFactoryImpl nodeTableStorageFactory = new NodeTableStorageFactoryImpl(); Database database = Database.inMemoryDB(); NodeTableStorage nodeTableStorage = @@ -87,16 +100,25 @@ public void testFind() throws Exception { }); // node is adjusted to be close to localhostEnr - NodeRecordV4 closestNode = - NodeRecordV4.Builder.empty() - .withIpV4Address(BytesValue.wrap(InetAddress.getByName("127.0.0.2").getAddress())) - .withSeq(UInt64.valueOf(1)) - .withUdpPort(30303) - .withSecp256k1( - BytesValue.fromHexString( - "aafb48004b1698f05872cf18b1f278998ad8f7d4c135aa41f83744e7b850ab6b98")) - .withSignature(Bytes96.EMPTY) - .build(); + NodeRecord closestNode = + NODE_RECORD_FACTORY.createFromValues( + EnrScheme.V4, + UInt64.valueOf(1), + Bytes96.EMPTY, + new ArrayList>() { + { + add( + Pair.with( + NodeRecord.FIELD_IP_V4, + Bytes4.wrap(InetAddress.getByName("127.0.0.2").getAddress()))); + add(Pair.with(NodeRecord.FIELD_UDP_V4, 30303)); + add( + Pair.with( + NodeRecord.FIELD_PKEY_SECP256K1, + BytesValue.fromHexString( + "aafb48004b1698f05872cf18b1f278998ad8f7d4c135aa41f83744e7b850ab6b98"))); + } + }); nodeTableStorage.get().save(new NodeRecordInfo(closestNode, -1L, NodeStatus.ACTIVE, 0)); assertEquals( nodeTableStorage @@ -104,28 +126,42 @@ public void testFind() throws Exception { .getNode(closestNode.getNodeId()) .get() .getNode() - .getPublicKey(), - closestNode.getPublicKey()); - NodeRecordV4 farNode = - NodeRecordV4.Builder.empty() - .withIpV4Address(BytesValue.wrap(InetAddress.getByName("127.0.0.3").getAddress())) - .withSeq(UInt64.valueOf(1)) - .withUdpPort(30303) - .withSecp256k1( - BytesValue.fromHexString( - "bafb48004b1698f05872cf18b1f278998ad8f7d4c135aa41f83744e7b850ab6b98")) - .withSignature(Bytes96.EMPTY) - .build(); + .get(NodeRecord.FIELD_PKEY_SECP256K1), + closestNode.get(NodeRecord.FIELD_PKEY_SECP256K1)); + NodeRecord farNode = + NODE_RECORD_FACTORY.createFromValues( + EnrScheme.V4, + UInt64.valueOf(1), + Bytes96.EMPTY, + new ArrayList>() { + { + add( + Pair.with( + NodeRecord.FIELD_IP_V4, + Bytes4.wrap(InetAddress.getByName("127.0.0.3").getAddress()))); + add(Pair.with(NodeRecord.FIELD_UDP_V4, 30303)); + add( + Pair.with( + NodeRecord.FIELD_PKEY_SECP256K1, + BytesValue.fromHexString( + "bafb48004b1698f05872cf18b1f278998ad8f7d4c135aa41f83744e7b850ab6b98"))); + } + }); nodeTableStorage.get().save(new NodeRecordInfo(farNode, -1L, NodeStatus.ACTIVE, 0)); List closestNodes = nodeTableStorage.get().findClosestNodes(closestNode.getNodeId(), 252); assertEquals(2, closestNodes.size()); - assertEquals(closestNodes.get(1).getNode().getPublicKey(), localHostNode.getPublicKey()); - assertEquals(closestNodes.get(0).getNode().getPublicKey(), closestNode.getPublicKey()); - List farNodes = - nodeTableStorage.get().findClosestNodes(farNode.getNodeId(), 1); + assertEquals( + closestNodes.get(0).getNode().get(NodeRecord.FIELD_PKEY_SECP256K1), + localHostNode.get(NodeRecord.FIELD_PKEY_SECP256K1)); + assertEquals( + closestNodes.get(1).getNode().get(NodeRecord.FIELD_PKEY_SECP256K1), + closestNode.get(NodeRecord.FIELD_PKEY_SECP256K1)); + List farNodes = nodeTableStorage.get().findClosestNodes(farNode.getNodeId(), 1); assertEquals(1, farNodes.size()); - assertEquals(farNodes.get(0).getNode().getPublicKey(), farNode.getPublicKey()); + assertEquals( + farNodes.get(0).getNode().get(NodeRecord.FIELD_PKEY_SECP256K1), + farNode.get(NodeRecord.FIELD_PKEY_SECP256K1)); } /** From 69c33f5689853c615035a2313a67593dbc00c29e Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Thu, 3 Oct 2019 16:25:22 +0300 Subject: [PATCH 27/77] discovery: pipeline draft --- .../discovery/DiscoveryManagerImpl.java | 125 ++++++------------ .../beacon/discovery/pipeline/Envelope.java | 24 ++++ .../discovery/pipeline/EnvelopeHandler.java | 5 + .../beacon/discovery/pipeline/Pipeline.java | 13 ++ .../discovery/pipeline/PipelineImpl.java | 51 +++++++ .../pipeline/handler/IncomingDataHandler.java | 22 +++ .../handler/IncomingPacketContextHandler.java | 24 ++++ .../handler/NodeIdContextHandler.java | 100 ++++++++++++++ .../pipeline/handler/NodeToNodeIdHandler.java | 28 ++++ .../handler/OutgoingParcelHandler.java | 29 ++++ .../pipeline/handler/TaskHandler.java | 37 ++++++ .../handler/UnknownPacketContextHandler.java | 39 ++++++ .../handler/WhoAreYouContextHandler.java | 38 ++++++ .../pipeline/handler/WhoAreYouHandler.java | 33 +++++ 14 files changed, 486 insertions(+), 82 deletions(-) create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/Envelope.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/EnvelopeHandler.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/Pipeline.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/PipelineImpl.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/IncomingDataHandler.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/IncomingPacketContextHandler.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeIdContextHandler.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeToNodeIdHandler.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/OutgoingParcelHandler.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/TaskHandler.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/UnknownPacketContextHandler.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouContextHandler.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouHandler.java diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java index ec2c6365b..1cbc1eab9 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java @@ -9,43 +9,41 @@ import org.ethereum.beacon.discovery.network.DiscoveryServer; import org.ethereum.beacon.discovery.network.DiscoveryServerImpl; import org.ethereum.beacon.discovery.network.NetworkParcel; -import org.ethereum.beacon.discovery.network.NetworkParcelV5; -import org.ethereum.beacon.discovery.packet.UnknownPacket; +import org.ethereum.beacon.discovery.pipeline.Envelope; +import org.ethereum.beacon.discovery.pipeline.Pipeline; +import org.ethereum.beacon.discovery.pipeline.PipelineImpl; +import org.ethereum.beacon.discovery.pipeline.handler.IncomingDataHandler; +import org.ethereum.beacon.discovery.pipeline.handler.NodeIdContextHandler; +import org.ethereum.beacon.discovery.pipeline.handler.NodeToNodeIdHandler; +import org.ethereum.beacon.discovery.pipeline.handler.OutgoingParcelHandler; +import org.ethereum.beacon.discovery.pipeline.handler.UnknownPacketContextHandler; +import org.ethereum.beacon.discovery.pipeline.handler.WhoAreYouContextHandler; +import org.ethereum.beacon.discovery.pipeline.handler.WhoAreYouHandler; import org.ethereum.beacon.discovery.storage.AuthTagRepository; import org.ethereum.beacon.discovery.storage.NodeBucketStorage; import org.ethereum.beacon.discovery.storage.NodeTable; import org.ethereum.beacon.schedulers.Scheduler; -import org.ethereum.beacon.util.ExpirationScheduler; import org.reactivestreams.Publisher; import reactor.core.publisher.Flux; import reactor.core.publisher.FluxSink; import reactor.core.publisher.ReplayProcessor; -import tech.pegasys.artemis.util.bytes.Bytes32; import tech.pegasys.artemis.util.bytes.Bytes4; -import java.security.SecureRandom; -import java.util.Map; -import java.util.Optional; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; + +import static org.ethereum.beacon.discovery.pipeline.handler.NodeToNodeIdHandler.NODE; +import static org.ethereum.beacon.discovery.pipeline.handler.TaskHandler.FUTURE; +import static org.ethereum.beacon.discovery.pipeline.handler.TaskHandler.PING; +import static org.ethereum.beacon.discovery.pipeline.handler.TaskHandler.TASK; public class DiscoveryManagerImpl implements DiscoveryManager { - private static final int CLEANUP_DELAY_SECONDS = 180; private static final Logger logger = LogManager.getLogger(DiscoveryManagerImpl.class); private final ReplayProcessor outgoingMessages = ReplayProcessor.cacheLast(); private final FluxSink outgoingSink = outgoingMessages.sink(); - private final Bytes32 homeNodeId; - private final NodeRecord homeNodeRecord; - private final NodeTable nodeTable; - private final NodeBucketStorage nodeBucketStorage; - private final Map recentContexts = - new ConcurrentHashMap<>(); // nodeId -> context - private final AuthTagRepository authTagRepo; private final DiscoveryServer discoveryServer; private final Scheduler scheduler; - private ExpirationScheduler contextExpirationScheduler = - new ExpirationScheduler<>(CLEANUP_DELAY_SECONDS, TimeUnit.SECONDS); + private final Pipeline incomingPipeline = new PipelineImpl(); + private final Pipeline outgoingPipeline = new PipelineImpl(); private DiscoveryClient discoveryClient; public DiscoveryManagerImpl( @@ -54,39 +52,33 @@ public DiscoveryManagerImpl( NodeRecord homeNode, Scheduler serverScheduler, Scheduler clientScheduler) { - this.nodeTable = nodeTable; - this.nodeBucketStorage = nodeBucketStorage; - this.homeNodeId = homeNode.getNodeId(); - this.homeNodeRecord = (NodeRecord) nodeTable.getHomeNode(); - this.authTagRepo = new AuthTagRepository(); + AuthTagRepository authTagRepo = new AuthTagRepository(); this.scheduler = serverScheduler; this.discoveryServer = new DiscoveryServerImpl( - ((Bytes4) homeNodeRecord.get(NodeRecord.FIELD_IP_V4)), - (int) homeNodeRecord.get(NodeRecord.FIELD_UDP_V4)); + ((Bytes4) homeNode.get(NodeRecord.FIELD_IP_V4)), + (int) homeNode.get(NodeRecord.FIELD_UDP_V4)); this.discoveryClient = new DiscoveryClientImpl(outgoingMessages, clientScheduler); + NodeIdContextHandler nodeIdContextHandler = + new NodeIdContextHandler( + homeNode, nodeBucketStorage, authTagRepo, nodeTable, outgoingPipeline); + incomingPipeline + .addHandler(new IncomingDataHandler()) + .addHandler(new WhoAreYouHandler(homeNode.getNodeId())) + .addHandler(new WhoAreYouContextHandler(authTagRepo)) + .addHandler(new UnknownPacketContextHandler(homeNode)) + .addHandler(nodeIdContextHandler); + outgoingPipeline + .addHandler(new OutgoingParcelHandler(outgoingSink)) + .addHandler(new NodeToNodeIdHandler()) + .addHandler(nodeIdContextHandler); } @Override public void start() { - Flux.from(discoveryServer.getIncomingPackets()) - .map(UnknownPacket::new) - .subscribe( - unknownPacket -> { - if (unknownPacket.isWhoAreYouPacket(homeNodeId)) { - Optional nodeContextOptional = - authTagRepo.get(unknownPacket.getWhoAreYouPacket().getAuthTag()); - if (nodeContextOptional.isPresent()) { - nodeContextOptional.get().addIncomingEvent(unknownPacket); - } else { - // TODO: ban or whatever - } - } else { - Bytes32 fromNodeId = unknownPacket.getSourceNodeId(homeNodeId); - getContext(fromNodeId) - .ifPresent(context -> context.addIncomingEvent(unknownPacket)); - } - }); + incomingPipeline.build(); + outgoingPipeline.build(); + Flux.from(discoveryServer.getIncomingPackets()).subscribe(incomingPipeline::send); discoveryServer.start(scheduler); } @@ -96,44 +88,13 @@ public void stop() { } public CompletableFuture connect(NodeRecord nodeRecord) { - if (!nodeTable.getNode(nodeRecord.getNodeId()).isPresent()) { - nodeTable.save(NodeRecordInfo.createDefault(nodeRecord)); - } - NodeContext context = getContext(nodeRecord.getNodeId()).get(); - return context.initiate(); - } - - private Optional getContext(Bytes32 nodeId) { - NodeContext context = recentContexts.get(nodeId); - if (context == null) { - Optional nodeOptional = nodeTable.getNode(nodeId); - if (!nodeOptional.isPresent()) { - logger.trace( - () -> String.format("Couldn't find node record for nodeId %s, ignoring", nodeId)); - return Optional.empty(); - } - NodeRecord nodeRecord = nodeOptional.get().getNode(); - SecureRandom random = new SecureRandom(); - context = - new NodeContext( - nodeRecord, - homeNodeRecord, - nodeTable, - nodeBucketStorage, - authTagRepo, - packet -> outgoingSink.next(new NetworkParcelV5(packet, nodeRecord)), - random); - recentContexts.put(nodeId, context); - } - - final NodeContext contextBackup = context; - contextExpirationScheduler.put( - context.getNodeRecord().getNodeId(), - () -> { - recentContexts.remove(contextBackup.getNodeRecord().getNodeId()); - contextBackup.cleanup(); - }); - return Optional.of(context); + Envelope envelope = new Envelope(); + envelope.put(TASK, PING); + envelope.put(NODE, nodeRecord); + CompletableFuture completed = new CompletableFuture<>(); + envelope.put(FUTURE, completed); + outgoingPipeline.send(envelope); + return completed; } @VisibleForTesting diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/Envelope.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/Envelope.java new file mode 100644 index 000000000..494597311 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/Envelope.java @@ -0,0 +1,24 @@ +package org.ethereum.beacon.discovery.pipeline; + +import java.util.HashMap; +import java.util.Map; + +public class Envelope { + private Map data = new HashMap<>(); + + public synchronized void put(String key, Object value) { + data.put(key, value); + } + + public synchronized Object get(String key) { + return data.get(key); + } + + public synchronized boolean remove(String key) { + return data.remove(key) != null; + } + + public synchronized boolean contains(String key) { + return data.containsKey(key); + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/EnvelopeHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/EnvelopeHandler.java new file mode 100644 index 000000000..6f3fee096 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/EnvelopeHandler.java @@ -0,0 +1,5 @@ +package org.ethereum.beacon.discovery.pipeline; + +public interface EnvelopeHandler { + void handle(Envelope envelope); +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/Pipeline.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/Pipeline.java new file mode 100644 index 000000000..7795630b1 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/Pipeline.java @@ -0,0 +1,13 @@ +package org.ethereum.beacon.discovery.pipeline; + +import org.reactivestreams.Publisher; + +public interface Pipeline { + Pipeline build(); + + void send(Object object); + + Pipeline addHandler(EnvelopeHandler envelopeHandler); + + Publisher getOutgoingEnvelopes(); +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/PipelineImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/PipelineImpl.java new file mode 100644 index 000000000..3f2f356d7 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/PipelineImpl.java @@ -0,0 +1,51 @@ +package org.ethereum.beacon.discovery.pipeline; + +import org.reactivestreams.Publisher; +import reactor.core.publisher.Flux; +import reactor.core.publisher.FluxSink; +import reactor.core.publisher.ReplayProcessor; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; + +public class PipelineImpl implements Pipeline { + public static final String INCOMING = "raw"; + private final List envelopeHandlers = new ArrayList<>(); + private final AtomicBoolean started = new AtomicBoolean(false); + private Flux pipeline = ReplayProcessor.cacheLast(); + private final FluxSink pipelineSink = ((ReplayProcessor) pipeline).sink(); + + @Override + public synchronized Pipeline build() { + started.set(true); + for (EnvelopeHandler handler : envelopeHandlers) { + pipeline = pipeline.doOnNext(handler::handle); + } + return this; + } + + @Override + public void send(Object object) { + if (!started.get()) { + throw new RuntimeException("You should build pipeline first"); + } + Envelope envelope = new Envelope(); + envelope.put(INCOMING, object); + pipelineSink.next(envelope); + } + + @Override + public Pipeline addHandler(EnvelopeHandler envelopeHandler) { + if (started.get()) { + throw new RuntimeException("Pipeline already started, couldn't add any handlers"); + } + envelopeHandlers.add(envelopeHandler); + return this; + } + + @Override + public Publisher getOutgoingEnvelopes() { + return pipeline; + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/IncomingDataHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/IncomingDataHandler.java new file mode 100644 index 000000000..fabf12077 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/IncomingDataHandler.java @@ -0,0 +1,22 @@ +package org.ethereum.beacon.discovery.pipeline.handler; + +import org.ethereum.beacon.discovery.packet.UnknownPacket; +import org.ethereum.beacon.discovery.pipeline.Envelope; +import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; +import tech.pegasys.artemis.util.bytes.BytesValue; + +import static org.ethereum.beacon.discovery.pipeline.PipelineImpl.INCOMING; + +public class IncomingDataHandler implements EnvelopeHandler { + + public static final String UNKNOWN = "unknown"; + + @Override + public void handle(Envelope envelope) { + if (!envelope.contains(INCOMING)) { + return; + } + envelope.put(UNKNOWN, new UnknownPacket((BytesValue) envelope.get(INCOMING))); + envelope.remove(INCOMING); + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/IncomingPacketContextHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/IncomingPacketContextHandler.java new file mode 100644 index 000000000..8fcfac630 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/IncomingPacketContextHandler.java @@ -0,0 +1,24 @@ +package org.ethereum.beacon.discovery.pipeline.handler; + +import org.ethereum.beacon.discovery.NodeContext; +import org.ethereum.beacon.discovery.packet.UnknownPacket; +import org.ethereum.beacon.discovery.pipeline.Envelope; +import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; + +import static org.ethereum.beacon.discovery.pipeline.handler.IncomingDataHandler.UNKNOWN; +import static org.ethereum.beacon.discovery.pipeline.handler.WhoAreYouContextHandler.CONTEXT; + +public class IncomingPacketContextHandler implements EnvelopeHandler { + @Override + public void handle(Envelope envelope) { + if (!envelope.contains(CONTEXT)) { + return; + } + if (!envelope.contains(UNKNOWN)) { + return; + } + NodeContext context = (NodeContext) envelope.get(CONTEXT); + UnknownPacket packet = (UnknownPacket) envelope.get(UNKNOWN); + context.addIncomingEvent(packet); + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeIdContextHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeIdContextHandler.java new file mode 100644 index 000000000..4802f9bea --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeIdContextHandler.java @@ -0,0 +1,100 @@ +package org.ethereum.beacon.discovery.pipeline.handler; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.ethereum.beacon.discovery.NodeContext; +import org.ethereum.beacon.discovery.NodeRecordInfo; +import org.ethereum.beacon.discovery.enr.NodeRecord; +import org.ethereum.beacon.discovery.network.NetworkParcelV5; +import org.ethereum.beacon.discovery.packet.UnknownPacket; +import org.ethereum.beacon.discovery.pipeline.Envelope; +import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; +import org.ethereum.beacon.discovery.pipeline.Pipeline; +import org.ethereum.beacon.discovery.storage.AuthTagRepository; +import org.ethereum.beacon.discovery.storage.NodeBucketStorage; +import org.ethereum.beacon.discovery.storage.NodeTable; +import org.ethereum.beacon.util.ExpirationScheduler; +import org.javatuples.Pair; +import tech.pegasys.artemis.util.bytes.Bytes32; + +import java.security.SecureRandom; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; + +import static org.ethereum.beacon.discovery.pipeline.handler.IncomingDataHandler.UNKNOWN; +import static org.ethereum.beacon.discovery.pipeline.handler.WhoAreYouContextHandler.BAD_PACKET; +import static org.ethereum.beacon.discovery.pipeline.handler.WhoAreYouContextHandler.CONTEXT; + +public class NodeIdContextHandler implements EnvelopeHandler { + private static final int CLEANUP_DELAY_SECONDS = 180; + private static final Logger logger = LogManager.getLogger(NodeIdContextHandler.class); + public static final String NEED_CONTEXT = "NEED_CONTEXT"; + private final NodeRecord homeNodeRecord; + private final NodeBucketStorage nodeBucketStorage; + private final AuthTagRepository authTagRepo; + private final Map recentContexts = + new ConcurrentHashMap<>(); // nodeId -> context + private final NodeTable nodeTable; + private ExpirationScheduler contextExpirationScheduler = + new ExpirationScheduler<>(CLEANUP_DELAY_SECONDS, TimeUnit.SECONDS); + private final Pipeline outgoingPipeline; + + public NodeIdContextHandler(NodeRecord homeNodeRecord, NodeBucketStorage nodeBucketStorage, AuthTagRepository authTagRepo, NodeTable nodeTable, Pipeline outgoingPipeline) { + this.homeNodeRecord = homeNodeRecord; + this.nodeBucketStorage = nodeBucketStorage; + this.authTagRepo = authTagRepo; + this.nodeTable = nodeTable; + this.outgoingPipeline = outgoingPipeline; + } + + @Override + public void handle(Envelope envelope) { + if (!envelope.contains(NEED_CONTEXT)) { + return; + } + Pair contextRequest = (Pair) envelope.get(NEED_CONTEXT); + envelope.remove(NEED_CONTEXT); + Optional nodeContextOptional = getContext(contextRequest.getValue0()); + if (nodeContextOptional.isPresent()) { + envelope.put(CONTEXT, nodeContextOptional.get()); + } else { + contextRequest.getValue1().run(); + } + } + + + private Optional getContext(Bytes32 nodeId) { + NodeContext context = recentContexts.get(nodeId); + if (context == null) { + Optional nodeOptional = nodeTable.getNode(nodeId); + if (!nodeOptional.isPresent()) { + logger.trace( + () -> String.format("Couldn't find node record for nodeId %s, ignoring", nodeId)); + return Optional.empty(); + } + NodeRecord nodeRecord = nodeOptional.get().getNode(); + SecureRandom random = new SecureRandom(); + context = + new NodeContext( + nodeRecord, + homeNodeRecord, + nodeTable, + nodeBucketStorage, + authTagRepo, + packet -> outgoingPipeline.send(new NetworkParcelV5(packet, nodeRecord)), + random); + recentContexts.put(nodeId, context); + } + + final NodeContext contextBackup = context; + contextExpirationScheduler.put( + context.getNodeRecord().getNodeId(), + () -> { + recentContexts.remove(contextBackup.getNodeRecord().getNodeId()); + contextBackup.cleanup(); + }); + return Optional.of(context); + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeToNodeIdHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeToNodeIdHandler.java new file mode 100644 index 000000000..0d62fcd04 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeToNodeIdHandler.java @@ -0,0 +1,28 @@ +package org.ethereum.beacon.discovery.pipeline.handler; + +import org.ethereum.beacon.discovery.enr.NodeRecord; +import org.ethereum.beacon.discovery.pipeline.Envelope; +import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; +import org.javatuples.Pair; + +import static org.ethereum.beacon.discovery.pipeline.PipelineImpl.INCOMING; +import static org.ethereum.beacon.discovery.pipeline.handler.NodeIdContextHandler.NEED_CONTEXT; + +public class NodeToNodeIdHandler implements EnvelopeHandler { + + public static final String NODE = "node"; + + @Override + public void handle(Envelope envelope) { + if (!envelope.contains(NODE)) { + return; + } + envelope.put( + NEED_CONTEXT, + Pair.with( + ((NodeRecord) envelope.get(NODE)).getNodeId(), + (Runnable) + () -> { // TODO: failure + })); + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/OutgoingParcelHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/OutgoingParcelHandler.java new file mode 100644 index 000000000..168b68654 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/OutgoingParcelHandler.java @@ -0,0 +1,29 @@ +package org.ethereum.beacon.discovery.pipeline.handler; + +import org.ethereum.beacon.discovery.network.NetworkParcel; +import org.ethereum.beacon.discovery.network.NetworkParcelV5; +import org.ethereum.beacon.discovery.pipeline.Envelope; +import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; +import reactor.core.publisher.FluxSink; + +import static org.ethereum.beacon.discovery.pipeline.PipelineImpl.INCOMING; + +public class OutgoingParcelHandler implements EnvelopeHandler { + + private final FluxSink outgoingSink; + + public OutgoingParcelHandler(FluxSink outgoingSink) { + this.outgoingSink = outgoingSink; + } + + @Override + public void handle(Envelope envelope) { + if (!envelope.contains(INCOMING)) { + return; + } + if (envelope.get(INCOMING) instanceof NetworkParcel) { + outgoingSink.next((NetworkParcel) envelope.get(INCOMING)); + envelope.remove(INCOMING); + } + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/TaskHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/TaskHandler.java new file mode 100644 index 000000000..daf6ffe73 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/TaskHandler.java @@ -0,0 +1,37 @@ +package org.ethereum.beacon.discovery.pipeline.handler; + +import org.ethereum.beacon.discovery.NodeContext; +import org.ethereum.beacon.discovery.pipeline.Envelope; +import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; + +import java.util.concurrent.CompletableFuture; +import java.util.function.BiConsumer; + +import static org.ethereum.beacon.discovery.pipeline.handler.WhoAreYouContextHandler.CONTEXT; + +public class TaskHandler implements EnvelopeHandler { + + public static final String TASK = "task"; + public static final String PING = "PING"; + public static final String FUTURE = "future"; + + @Override + public void handle(Envelope envelope) { + if (!envelope.contains(TASK)) { + return; + } + if (!envelope.get(TASK).equals(PING)) { + return; // TODO: implement other tasks + } + NodeContext context = (NodeContext) envelope.get(CONTEXT); + CompletableFuture completableFuture = (CompletableFuture) envelope.get(FUTURE); + context.initiate().whenComplete((aVoid, throwable) -> { + if (throwable != null) { + completableFuture.completeExceptionally(throwable); + } else { + completableFuture.complete(aVoid); + } + }); + + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/UnknownPacketContextHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/UnknownPacketContextHandler.java new file mode 100644 index 000000000..965bbea1b --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/UnknownPacketContextHandler.java @@ -0,0 +1,39 @@ +package org.ethereum.beacon.discovery.pipeline.handler; + +import org.ethereum.beacon.discovery.enr.NodeRecord; +import org.ethereum.beacon.discovery.packet.UnknownPacket; +import org.ethereum.beacon.discovery.pipeline.Envelope; +import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; +import org.javatuples.Pair; +import tech.pegasys.artemis.util.bytes.Bytes32; + +import static org.ethereum.beacon.discovery.pipeline.handler.IncomingDataHandler.UNKNOWN; +import static org.ethereum.beacon.discovery.pipeline.handler.NodeIdContextHandler.NEED_CONTEXT; +import static org.ethereum.beacon.discovery.pipeline.handler.WhoAreYouContextHandler.BAD_PACKET; + +public class UnknownPacketContextHandler implements EnvelopeHandler { + private static final int CLEANUP_DELAY_SECONDS = 180; + private final Bytes32 homeNodeId; + + public UnknownPacketContextHandler(NodeRecord homeNodeRecord) { + this.homeNodeId = homeNodeRecord.getNodeId(); + } + + @Override + public void handle(Envelope envelope) { + if (!envelope.contains(UNKNOWN)) { + return; + } + UnknownPacket unknownPacket = (UnknownPacket) envelope.get(UNKNOWN); + Bytes32 fromNodeId = unknownPacket.getSourceNodeId(homeNodeId); + envelope.put( + NEED_CONTEXT, + Pair.with( + fromNodeId, + (Runnable) + () -> { + envelope.put(BAD_PACKET, envelope.get(UNKNOWN)); + envelope.remove(UNKNOWN); + })); + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouContextHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouContextHandler.java new file mode 100644 index 000000000..c370b6b46 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouContextHandler.java @@ -0,0 +1,38 @@ +package org.ethereum.beacon.discovery.pipeline.handler; + +import org.ethereum.beacon.discovery.NodeContext; +import org.ethereum.beacon.discovery.packet.WhoAreYouPacket; +import org.ethereum.beacon.discovery.pipeline.Envelope; +import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; +import org.ethereum.beacon.discovery.storage.AuthTagRepository; +import tech.pegasys.artemis.util.bytes.Bytes32; + +import java.util.Optional; + +import static org.ethereum.beacon.discovery.pipeline.handler.WhoAreYouHandler.WHOAREYOU; + +public class WhoAreYouContextHandler implements EnvelopeHandler { + public static final String CONTEXT = "context"; + public static final String BAD_PACKET = "BAD_PACKET"; + private final AuthTagRepository authTagRepo; + + public WhoAreYouContextHandler(AuthTagRepository authTagRepo) { + this.authTagRepo = authTagRepo; + } + + @Override + public void handle(Envelope envelope) { + if (!envelope.contains(WHOAREYOU)) { + return; + } + + WhoAreYouPacket whoAreYouPacket = (WhoAreYouPacket) envelope.get(WHOAREYOU); + Optional nodeContextOptional = authTagRepo.get(whoAreYouPacket.getAuthTag()); + if (nodeContextOptional.isPresent()) { + envelope.put(CONTEXT, nodeContextOptional.get()); + } else { + envelope.put(BAD_PACKET, envelope.get(WHOAREYOU)); + envelope.remove(WHOAREYOU); + } + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouHandler.java new file mode 100644 index 000000000..12222fb21 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouHandler.java @@ -0,0 +1,33 @@ +package org.ethereum.beacon.discovery.pipeline.handler; + +import org.ethereum.beacon.discovery.NodeContext; +import org.ethereum.beacon.discovery.packet.UnknownPacket; +import org.ethereum.beacon.discovery.pipeline.Envelope; +import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; +import org.ethereum.beacon.discovery.storage.AuthTagRepository; +import tech.pegasys.artemis.util.bytes.Bytes32; + +import java.util.Optional; + +import static org.ethereum.beacon.discovery.pipeline.handler.IncomingDataHandler.UNKNOWN; + +public class WhoAreYouHandler implements EnvelopeHandler { + public static final String WHOAREYOU = "WHOAREYOU"; + private final Bytes32 homeNodeId; + + public WhoAreYouHandler(Bytes32 homeNodeId) { + this.homeNodeId = homeNodeId; + } + + @Override + public void handle(Envelope envelope) { + if (!envelope.contains(UNKNOWN)) { + return; + } + if (!((UnknownPacket) envelope.get(UNKNOWN)).isWhoAreYouPacket(homeNodeId)) { + return; + } + UnknownPacket unknownPacket = (UnknownPacket) envelope.get(UNKNOWN); + envelope.put(WHOAREYOU, unknownPacket.getWhoAreYouPacket()); + } +} From 429930afb600d458855001761be8d3ef91f918bc Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Thu, 3 Oct 2019 17:21:24 +0300 Subject: [PATCH 28/77] discovery: fix pipeline draft --- .../beacon/discovery/DiscoveryManagerImpl.java | 14 +++++++++----- .../beacon/discovery/pipeline/PipelineImpl.java | 13 ++++++++++--- ...Handler.java => NodeContextRequestHandler.java} | 3 +-- .../pipeline/handler/WhoAreYouHandler.java | 4 ---- 4 files changed, 20 insertions(+), 14 deletions(-) rename discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/{NodeToNodeIdHandler.java => NodeContextRequestHandler.java} (83%) diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java index 1cbc1eab9..c3b217be6 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java @@ -13,9 +13,11 @@ import org.ethereum.beacon.discovery.pipeline.Pipeline; import org.ethereum.beacon.discovery.pipeline.PipelineImpl; import org.ethereum.beacon.discovery.pipeline.handler.IncomingDataHandler; +import org.ethereum.beacon.discovery.pipeline.handler.IncomingPacketContextHandler; +import org.ethereum.beacon.discovery.pipeline.handler.NodeContextRequestHandler; import org.ethereum.beacon.discovery.pipeline.handler.NodeIdContextHandler; -import org.ethereum.beacon.discovery.pipeline.handler.NodeToNodeIdHandler; import org.ethereum.beacon.discovery.pipeline.handler.OutgoingParcelHandler; +import org.ethereum.beacon.discovery.pipeline.handler.TaskHandler; import org.ethereum.beacon.discovery.pipeline.handler.UnknownPacketContextHandler; import org.ethereum.beacon.discovery.pipeline.handler.WhoAreYouContextHandler; import org.ethereum.beacon.discovery.pipeline.handler.WhoAreYouHandler; @@ -31,7 +33,7 @@ import java.util.concurrent.CompletableFuture; -import static org.ethereum.beacon.discovery.pipeline.handler.NodeToNodeIdHandler.NODE; +import static org.ethereum.beacon.discovery.pipeline.handler.NodeContextRequestHandler.NODE; import static org.ethereum.beacon.discovery.pipeline.handler.TaskHandler.FUTURE; import static org.ethereum.beacon.discovery.pipeline.handler.TaskHandler.PING; import static org.ethereum.beacon.discovery.pipeline.handler.TaskHandler.TASK; @@ -67,11 +69,13 @@ public DiscoveryManagerImpl( .addHandler(new WhoAreYouHandler(homeNode.getNodeId())) .addHandler(new WhoAreYouContextHandler(authTagRepo)) .addHandler(new UnknownPacketContextHandler(homeNode)) - .addHandler(nodeIdContextHandler); + .addHandler(nodeIdContextHandler) + .addHandler(new IncomingPacketContextHandler()); outgoingPipeline .addHandler(new OutgoingParcelHandler(outgoingSink)) - .addHandler(new NodeToNodeIdHandler()) - .addHandler(nodeIdContextHandler); + .addHandler(new NodeContextRequestHandler()) + .addHandler(nodeIdContextHandler) + .addHandler(new TaskHandler()); } @Override diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/PipelineImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/PipelineImpl.java index 3f2f356d7..039573c8e 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/PipelineImpl.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/PipelineImpl.java @@ -1,6 +1,7 @@ package org.ethereum.beacon.discovery.pipeline; import org.reactivestreams.Publisher; +import reactor.core.Disposable; import reactor.core.publisher.Flux; import reactor.core.publisher.FluxSink; import reactor.core.publisher.ReplayProcessor; @@ -15,6 +16,7 @@ public class PipelineImpl implements Pipeline { private final AtomicBoolean started = new AtomicBoolean(false); private Flux pipeline = ReplayProcessor.cacheLast(); private final FluxSink pipelineSink = ((ReplayProcessor) pipeline).sink(); + private Disposable subscription; @Override public synchronized Pipeline build() { @@ -22,6 +24,7 @@ public synchronized Pipeline build() { for (EnvelopeHandler handler : envelopeHandlers) { pipeline = pipeline.doOnNext(handler::handle); } + this.subscription = Flux.from(pipeline).subscribe(); return this; } @@ -30,9 +33,13 @@ public void send(Object object) { if (!started.get()) { throw new RuntimeException("You should build pipeline first"); } - Envelope envelope = new Envelope(); - envelope.put(INCOMING, object); - pipelineSink.next(envelope); + if (!(object instanceof Envelope)) { + Envelope envelope = new Envelope(); + envelope.put(INCOMING, object); + pipelineSink.next(envelope); + } else { + pipelineSink.next((Envelope) object); + } } @Override diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeToNodeIdHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeContextRequestHandler.java similarity index 83% rename from discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeToNodeIdHandler.java rename to discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeContextRequestHandler.java index 0d62fcd04..ad343fcb8 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeToNodeIdHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeContextRequestHandler.java @@ -5,10 +5,9 @@ import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; import org.javatuples.Pair; -import static org.ethereum.beacon.discovery.pipeline.PipelineImpl.INCOMING; import static org.ethereum.beacon.discovery.pipeline.handler.NodeIdContextHandler.NEED_CONTEXT; -public class NodeToNodeIdHandler implements EnvelopeHandler { +public class NodeContextRequestHandler implements EnvelopeHandler { public static final String NODE = "node"; diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouHandler.java index 12222fb21..3444fc0f0 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouHandler.java @@ -1,14 +1,10 @@ package org.ethereum.beacon.discovery.pipeline.handler; -import org.ethereum.beacon.discovery.NodeContext; import org.ethereum.beacon.discovery.packet.UnknownPacket; import org.ethereum.beacon.discovery.pipeline.Envelope; import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; -import org.ethereum.beacon.discovery.storage.AuthTagRepository; import tech.pegasys.artemis.util.bytes.Bytes32; -import java.util.Optional; - import static org.ethereum.beacon.discovery.pipeline.handler.IncomingDataHandler.UNKNOWN; public class WhoAreYouHandler implements EnvelopeHandler { From 9a8ff0e89f509322bccb6fd46ffaea94f7f31f44 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Sun, 6 Oct 2019 15:56:14 +0300 Subject: [PATCH 29/77] discovery: refactor packet flow to pipeline with actor-like packet handling --- .../beacon/discovery/DiscoveryManager.java | 6 +- .../discovery/DiscoveryManagerImpl.java | 63 ++++---- .../beacon/discovery/NodeContext.java | 148 +++--------------- .../discovery/message/DiscoveryV5Message.java | 2 +- .../beacon/discovery/message/PingMessage.java | 9 +- .../message/handler/FindNodeHandler.java | 2 +- .../message/handler/PingHandler.java | 2 +- .../AuthHeaderMessagePacketHandler.java | 55 ------- .../packet/handler/MessagePacketHandler.java | 37 ----- .../packet/handler/PacketHandler.java | 7 - .../beacon/discovery/pipeline/Envelope.java | 21 ++- .../beacon/discovery/pipeline/Field.java | 17 ++ .../beacon/discovery/pipeline/Pipeline.java | 2 +- .../discovery/pipeline/PipelineImpl.java | 5 +- .../AuthHeaderMessagePacketHandler.java | 92 +++++++++++ .../pipeline/handler/IncomingDataHandler.java | 22 --- .../pipeline/handler/IncomingDataPacker.java | 26 +++ .../handler/IncomingPacketContextHandler.java | 24 --- .../pipeline/handler/MessageHandler.java | 27 ++++ .../handler/MessagePacketHandler.java | 71 +++++++++ .../handler/NodeContextRequestHandler.java | 15 +- ...ntextHandler.java => NodeIdToContext.java} | 41 +++-- .../NotExpectedIncomingPacketHandler.java} | 44 ++++-- .../handler/OutgoingParcelHandler.java | 17 +- .../pipeline/handler/TaskHandler.java | 57 +++++-- .../handler/UnknownPacketContextHandler.java | 39 ----- .../handler/UnknownPacketTagToSender.java | 40 +++++ .../handler/UnknownPacketTypeByStatus.java | 67 ++++++++ .../pipeline/handler/WhoAreYouAttempt.java | 32 ++++ .../handler/WhoAreYouContextHandler.java | 38 ----- .../handler/WhoAreYouContextResolver.java | 52 ++++++ .../pipeline/handler/WhoAreYouHandler.java | 29 ---- .../handler/WhoAreYouPacketHandler.java | 60 ++++--- .../{ => task}/DiscoveryTaskManager.java | 76 +++++++-- .../LiveCheckTasks.java} | 18 ++- .../discovery/task/RecursiveLookupTasks.java | 63 ++++++++ .../discovery/task/TaskMessageFactory.java | 41 +++++ .../beacon/discovery/task/TaskType.java | 6 + .../discovery/DiscoveryNetworkTest.java | 5 +- .../discovery/DiscoveryNoNetworkTest.java | 15 +- .../mock/DiscoveryManagerNoNetwork.java | 128 +++++++-------- 41 files changed, 923 insertions(+), 598 deletions(-) delete mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/packet/handler/AuthHeaderMessagePacketHandler.java delete mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/packet/handler/MessagePacketHandler.java delete mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/packet/handler/PacketHandler.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/Field.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/AuthHeaderMessagePacketHandler.java delete mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/IncomingDataHandler.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/IncomingDataPacker.java delete mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/IncomingPacketContextHandler.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/MessageHandler.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/MessagePacketHandler.java rename discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/{NodeIdContextHandler.java => NodeIdToContext.java} (75%) rename discovery/src/main/java/org/ethereum/beacon/discovery/{packet/handler/UnknownPacketHandler.java => pipeline/handler/NotExpectedIncomingPacketHandler.java} (52%) delete mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/UnknownPacketContextHandler.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/UnknownPacketTagToSender.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/UnknownPacketTypeByStatus.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouAttempt.java delete mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouContextHandler.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouContextResolver.java delete mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouHandler.java rename discovery/src/main/java/org/ethereum/beacon/discovery/{packet => pipeline}/handler/WhoAreYouPacketHandler.java (54%) rename discovery/src/main/java/org/ethereum/beacon/discovery/{ => task}/DiscoveryTaskManager.java (63%) rename discovery/src/main/java/org/ethereum/beacon/discovery/{NodeConnectTasks.java => task/LiveCheckTasks.java} (77%) create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/task/RecursiveLookupTasks.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/task/TaskMessageFactory.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/task/TaskType.java diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManager.java b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManager.java index ac87849b1..b1529aaee 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManager.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManager.java @@ -1,6 +1,7 @@ package org.ethereum.beacon.discovery; import org.ethereum.beacon.discovery.enr.NodeRecord; +import org.ethereum.beacon.discovery.task.TaskType; import java.util.concurrent.CompletableFuture; @@ -14,11 +15,12 @@ public interface DiscoveryManager { void stop(); /** - * Initiates auth handshake with node, sending some message and receiving reply. + * Initiates task of task type with node `nodeRecord` * * @param nodeRecord Ethereum Node record + * @param taskType Task type * @return Future which is fired when reply is received or fails in timeout/not successful * handshake/bad message exchange. */ - CompletableFuture connect(NodeRecord nodeRecord); + CompletableFuture executeTask(NodeRecord nodeRecord, TaskType taskType); } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java index c3b217be6..2d5027558 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java @@ -10,17 +10,24 @@ import org.ethereum.beacon.discovery.network.DiscoveryServerImpl; import org.ethereum.beacon.discovery.network.NetworkParcel; import org.ethereum.beacon.discovery.pipeline.Envelope; +import org.ethereum.beacon.discovery.pipeline.Field; import org.ethereum.beacon.discovery.pipeline.Pipeline; import org.ethereum.beacon.discovery.pipeline.PipelineImpl; -import org.ethereum.beacon.discovery.pipeline.handler.IncomingDataHandler; -import org.ethereum.beacon.discovery.pipeline.handler.IncomingPacketContextHandler; +import org.ethereum.beacon.discovery.pipeline.handler.AuthHeaderMessagePacketHandler; +import org.ethereum.beacon.discovery.pipeline.handler.IncomingDataPacker; +import org.ethereum.beacon.discovery.pipeline.handler.MessageHandler; +import org.ethereum.beacon.discovery.pipeline.handler.MessagePacketHandler; import org.ethereum.beacon.discovery.pipeline.handler.NodeContextRequestHandler; -import org.ethereum.beacon.discovery.pipeline.handler.NodeIdContextHandler; +import org.ethereum.beacon.discovery.pipeline.handler.NodeIdToContext; +import org.ethereum.beacon.discovery.pipeline.handler.NotExpectedIncomingPacketHandler; import org.ethereum.beacon.discovery.pipeline.handler.OutgoingParcelHandler; import org.ethereum.beacon.discovery.pipeline.handler.TaskHandler; -import org.ethereum.beacon.discovery.pipeline.handler.UnknownPacketContextHandler; -import org.ethereum.beacon.discovery.pipeline.handler.WhoAreYouContextHandler; -import org.ethereum.beacon.discovery.pipeline.handler.WhoAreYouHandler; +import org.ethereum.beacon.discovery.pipeline.handler.UnknownPacketTagToSender; +import org.ethereum.beacon.discovery.pipeline.handler.UnknownPacketTypeByStatus; +import org.ethereum.beacon.discovery.pipeline.handler.WhoAreYouAttempt; +import org.ethereum.beacon.discovery.pipeline.handler.WhoAreYouContextResolver; +import org.ethereum.beacon.discovery.pipeline.handler.WhoAreYouPacketHandler; +import org.ethereum.beacon.discovery.task.TaskType; import org.ethereum.beacon.discovery.storage.AuthTagRepository; import org.ethereum.beacon.discovery.storage.NodeBucketStorage; import org.ethereum.beacon.discovery.storage.NodeTable; @@ -31,13 +38,9 @@ import reactor.core.publisher.ReplayProcessor; import tech.pegasys.artemis.util.bytes.Bytes4; +import java.security.SecureRandom; import java.util.concurrent.CompletableFuture; -import static org.ethereum.beacon.discovery.pipeline.handler.NodeContextRequestHandler.NODE; -import static org.ethereum.beacon.discovery.pipeline.handler.TaskHandler.FUTURE; -import static org.ethereum.beacon.discovery.pipeline.handler.TaskHandler.PING; -import static org.ethereum.beacon.discovery.pipeline.handler.TaskHandler.TASK; - public class DiscoveryManagerImpl implements DiscoveryManager { private static final Logger logger = LogManager.getLogger(DiscoveryManagerImpl.class); private final ReplayProcessor outgoingMessages = ReplayProcessor.cacheLast(); @@ -61,28 +64,32 @@ public DiscoveryManagerImpl( ((Bytes4) homeNode.get(NodeRecord.FIELD_IP_V4)), (int) homeNode.get(NodeRecord.FIELD_UDP_V4)); this.discoveryClient = new DiscoveryClientImpl(outgoingMessages, clientScheduler); - NodeIdContextHandler nodeIdContextHandler = - new NodeIdContextHandler( - homeNode, nodeBucketStorage, authTagRepo, nodeTable, outgoingPipeline); + NodeIdToContext nodeIdToContext = + new NodeIdToContext(homeNode, nodeBucketStorage, authTagRepo, nodeTable, outgoingPipeline); incomingPipeline - .addHandler(new IncomingDataHandler()) - .addHandler(new WhoAreYouHandler(homeNode.getNodeId())) - .addHandler(new WhoAreYouContextHandler(authTagRepo)) - .addHandler(new UnknownPacketContextHandler(homeNode)) - .addHandler(nodeIdContextHandler) - .addHandler(new IncomingPacketContextHandler()); + .addHandler(new IncomingDataPacker()) + .addHandler(new WhoAreYouAttempt(homeNode.getNodeId())) + .addHandler(new WhoAreYouContextResolver(authTagRepo)) + .addHandler(new UnknownPacketTagToSender(homeNode)) + .addHandler(nodeIdToContext) + .addHandler(new UnknownPacketTypeByStatus()) + .addHandler(new NotExpectedIncomingPacketHandler()) + .addHandler(new WhoAreYouPacketHandler()) + .addHandler(new AuthHeaderMessagePacketHandler()) + .addHandler(new MessagePacketHandler()) + .addHandler(new MessageHandler()); outgoingPipeline .addHandler(new OutgoingParcelHandler(outgoingSink)) .addHandler(new NodeContextRequestHandler()) - .addHandler(nodeIdContextHandler) - .addHandler(new TaskHandler()); + .addHandler(nodeIdToContext) + .addHandler(new TaskHandler(new SecureRandom())); } @Override public void start() { incomingPipeline.build(); outgoingPipeline.build(); - Flux.from(discoveryServer.getIncomingPackets()).subscribe(incomingPipeline::send); + Flux.from(discoveryServer.getIncomingPackets()).subscribe(incomingPipeline::push); discoveryServer.start(scheduler); } @@ -91,13 +98,13 @@ public void stop() { discoveryServer.stop(); } - public CompletableFuture connect(NodeRecord nodeRecord) { + public CompletableFuture executeTask(NodeRecord nodeRecord, TaskType taskType) { Envelope envelope = new Envelope(); - envelope.put(TASK, PING); - envelope.put(NODE, nodeRecord); + envelope.put(Field.TASK, taskType); + envelope.put(Field.NODE, nodeRecord); CompletableFuture completed = new CompletableFuture<>(); - envelope.put(FUTURE, completed); - outgoingPipeline.send(envelope); + envelope.put(Field.FUTURE, completed); + outgoingPipeline.push(envelope); return completed; } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeContext.java b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeContext.java index ecec8916f..f89963127 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeContext.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeContext.java @@ -3,19 +3,9 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.ethereum.beacon.discovery.enr.NodeRecord; -import org.ethereum.beacon.discovery.message.DiscoveryMessage; import org.ethereum.beacon.discovery.message.MessageCode; -import org.ethereum.beacon.discovery.packet.AuthHeaderMessagePacket; -import org.ethereum.beacon.discovery.packet.MessagePacket; import org.ethereum.beacon.discovery.packet.Packet; -import org.ethereum.beacon.discovery.packet.RandomPacket; -import org.ethereum.beacon.discovery.packet.UnknownPacket; -import org.ethereum.beacon.discovery.packet.WhoAreYouPacket; -import org.ethereum.beacon.discovery.packet.handler.AuthHeaderMessagePacketHandler; -import org.ethereum.beacon.discovery.packet.handler.MessagePacketHandler; -import org.ethereum.beacon.discovery.packet.handler.PacketHandler; -import org.ethereum.beacon.discovery.packet.handler.UnknownPacketHandler; -import org.ethereum.beacon.discovery.packet.handler.WhoAreYouPacketHandler; +import org.ethereum.beacon.discovery.task.TaskType; import org.ethereum.beacon.discovery.storage.AuthTagRepository; import org.ethereum.beacon.discovery.storage.NodeBucket; import org.ethereum.beacon.discovery.storage.NodeBucketStorage; @@ -23,8 +13,7 @@ import tech.pegasys.artemis.util.bytes.Bytes32; import tech.pegasys.artemis.util.bytes.BytesValue; -import java.security.SecureRandom; -import java.util.HashMap; +import javax.annotation.Nullable; import java.util.Map; import java.util.Optional; import java.util.Random; @@ -33,7 +22,6 @@ import java.util.function.Consumer; public class NodeContext { - public static final int DEFAULT_DISTANCE = 10; // FIXME: I shouldn't be here private static final Logger logger = LogManager.getLogger(NodeContext.class); private final NodeRecord nodeRecord; private final NodeRecord homeNodeRecord; @@ -43,13 +31,12 @@ public class NodeContext { private final NodeBucketStorage nodeBucketStorage; private final Consumer outgoing; private final Random rnd; - private final Map packetHandlers = new HashMap<>(); private SessionStatus status = SessionStatus.INITIAL; private Bytes32 idNonce; private BytesValue initiatorKey; - private MessageProcessor messageProcessor = null; private Map requestIdReservations = new ConcurrentHashMap<>(); - private CompletableFuture connectFuture = null; + private CompletableFuture completableFuture = null; + private TaskType task = null; public NodeContext( NodeRecord nodeRecord, @@ -67,119 +54,20 @@ public NodeContext( this.homeNodeRecord = homeNodeRecord; this.homeNodeId = homeNodeRecord.getNodeId(); this.rnd = rnd; - packetHandlers.put(UnknownPacket.class, new UnknownPacketHandler(this, logger)); - packetHandlers.put(WhoAreYouPacket.class, new WhoAreYouPacketHandler(this, logger)); - packetHandlers.put( - AuthHeaderMessagePacket.class, new AuthHeaderMessagePacketHandler(this, logger)); - packetHandlers.put(MessagePacket.class, new MessagePacketHandler(this, logger)); } public NodeRecord getNodeRecord() { return nodeRecord; } - public void addIncomingEvent(UnknownPacket packet) { - logger.trace(() -> String.format("Incoming packet in context %s", this)); - switch (status) { - case INITIAL: - { - if (packetHandlers.get(UnknownPacket.class).handle(packet)) { - setStatus(SessionStatus.WHOAREYOU_SENT); - } else { - failConnectFuture(); - } - break; - } - case RANDOM_PACKET_SENT: - { - WhoAreYouPacket whoAreYouPacket = packet.getWhoAreYouPacket(); - if (packetHandlers.get(WhoAreYouPacket.class).handle(whoAreYouPacket)) { - setStatus(SessionStatus.AUTHENTICATED); - } else { - failConnectFuture(); - } - break; - } - case WHOAREYOU_SENT: - { - AuthHeaderMessagePacket authHeaderMessagePacket = packet.getAuthHeaderMessagePacket(); - if (packetHandlers.get(AuthHeaderMessagePacket.class).handle(authHeaderMessagePacket)) { - setStatus(SessionStatus.AUTHENTICATED); - completeConnectFuture(); - } else { - failConnectFuture(); - } - break; - } - case AUTHENTICATED: - { - MessagePacket messagePacket = packet.getMessagePacket(); - if (packetHandlers.get(MessagePacket.class).handle(messagePacket)) { - completeConnectFuture(); - } else { - failConnectFuture(); - } - break; - } - default: - { - String error = String.format("Not expected status:%s from node: %s", status, nodeRecord); - logger.error(error); - throw new RuntimeException(error); - } - } - } - - public void handleMessage(DiscoveryMessage message) { - synchronized (this) { - if (messageProcessor == null) { - this.messageProcessor = new MessageProcessor(new DiscoveryV5MessageProcessor()); - } - } - messageProcessor.handleIncoming(message, this); - } - private void completeConnectFuture() { - if (connectFuture != null) { - connectFuture.complete(null); - connectFuture = null; - } - } - - private void failConnectFuture() { - if (connectFuture != null) { - connectFuture.completeExceptionally( - new RuntimeException( - String.format("Peer message initiation failed for %s", this.getNodeRecord()))); - connectFuture = null; - } - } - - /** Sends random packet to start initiation of session with node */ - public CompletableFuture initiate() { - CompletableFuture connectFuture = new CompletableFuture<>(); - if (this.connectFuture != null) { - connectFuture.completeExceptionally( - new RuntimeException( - String.format( - "Only one simultaneous handshake initiation allowed per peer. Got second for %s", - this.getNodeRecord()))); - } - if (status == SessionStatus.AUTHENTICATED) { - completeConnectFuture(); + if (completableFuture != null) { + completableFuture.complete(null); + completableFuture = null; } - byte[] authTagBytes = new byte[12]; - rnd.nextBytes(authTagBytes); - BytesValue authTag = BytesValue.wrap(authTagBytes); - RandomPacket randomPacket = - RandomPacket.create(homeNodeId, nodeRecord.getNodeId(), authTag, new SecureRandom()); - authTagRepo.put(authTag, this); - this.addOutgoingEvent(randomPacket); - this.status = SessionStatus.RANDOM_PACKET_SENT; - return connectFuture; } - public synchronized void addOutgoingEvent(Packet packet) { + public synchronized void sendOutgoing(Packet packet) { outgoing.accept(packet); } @@ -274,7 +162,7 @@ public synchronized SessionStatus getStatus() { return status; } - private synchronized void setStatus(SessionStatus newStatus) { + public synchronized void setStatus(SessionStatus newStatus) { logger.debug( () -> String.format( @@ -282,7 +170,23 @@ private synchronized void setStatus(SessionStatus newStatus) { this.status = newStatus; } - enum SessionStatus { + public void saveFuture(@Nullable CompletableFuture completableFuture) { + this.completableFuture = completableFuture; + } + + public CompletableFuture loadFuture() { + return completableFuture; + } + + public void saveTask(@Nullable TaskType task) { + this.task = task; + } + + public TaskType loadTask() { + return task; + } + + public enum SessionStatus { INITIAL, // other side is trying to connect, or we are initiating (before random packet is sent WHOAREYOU_SENT, // other side is initiator, we've sent whoareyou in response RANDOM_PACKET_SENT, // our node is initiator, we've sent random packet diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/DiscoveryV5Message.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/DiscoveryV5Message.java index 88614701f..144f2146e 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/message/DiscoveryV5Message.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/DiscoveryV5Message.java @@ -62,7 +62,7 @@ public V5Message create() { { return new PingMessage( BytesValue.wrap(((RlpString) payload.get(0)).getBytes()), - ((RlpString) payload.get(1)).asPositiveBigInteger().longValueExact()); + UInt64.fromBytesBigEndian(Bytes8.wrap(((RlpString) payload.get(1)).getBytes()))); } case PONG: { diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/PingMessage.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/PingMessage.java index 2033602db..601e9582d 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/message/PingMessage.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/PingMessage.java @@ -6,6 +6,7 @@ import org.web3j.rlp.RlpString; import tech.pegasys.artemis.util.bytes.Bytes1; import tech.pegasys.artemis.util.bytes.BytesValue; +import tech.pegasys.artemis.util.uint.UInt64; /** * PING checks whether the recipient is alive and informs it about the sender's ENR sequence number. @@ -14,9 +15,9 @@ public class PingMessage implements V5Message { // Unique request id private final BytesValue requestId; // Local ENR sequence number of sender - private final Long enrSeq; + private final UInt64 enrSeq; - public PingMessage(BytesValue requestId, Long enrSeq) { + public PingMessage(BytesValue requestId, UInt64 enrSeq) { this.requestId = requestId; this.enrSeq = enrSeq; } @@ -26,7 +27,7 @@ public BytesValue getRequestId() { return requestId; } - public Long getEnrSeq() { + public UInt64 getEnrSeq() { return enrSeq; } @@ -37,7 +38,7 @@ public BytesValue getBytes() { BytesValue.wrap( RlpEncoder.encode( new RlpList( - RlpString.create(requestId.extractArray()), RlpString.create(enrSeq))))); + RlpString.create(requestId.extractArray()), RlpString.create(enrSeq.toBI()))))); } @Override diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/FindNodeHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/FindNodeHandler.java index 496562afb..7fb47ef84 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/FindNodeHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/FindNodeHandler.java @@ -27,7 +27,7 @@ public void handle(FindNodeMessage message, NodeContext context) { .collect(Collectors.toList()); nodeBuckets.forEach( bucket -> - context.addOutgoingEvent( + context.sendOutgoing( MessagePacket.create( context.getHomeNodeId(), context.getNodeRecord().getNodeId(), diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/PingHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/PingHandler.java index 8f57878c7..9c73ac00f 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/PingHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/PingHandler.java @@ -17,7 +17,7 @@ public void handle(PingMessage message, NodeContext context) { context.getNodeRecord().getSeq(), ((Bytes4) context.getNodeRecord().get(NodeRecord.FIELD_IP_V4)), (int) context.getNodeRecord().get(NodeRecord.FIELD_UDP_V4)); - context.addOutgoingEvent( + context.sendOutgoing( MessagePacket.create( context.getHomeNodeId(), context.getNodeRecord().getNodeId(), diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/handler/AuthHeaderMessagePacketHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/handler/AuthHeaderMessagePacketHandler.java deleted file mode 100644 index 0a3855309..000000000 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/handler/AuthHeaderMessagePacketHandler.java +++ /dev/null @@ -1,55 +0,0 @@ -package org.ethereum.beacon.discovery.packet.handler; - -import org.apache.logging.log4j.Logger; -import org.ethereum.beacon.discovery.Functions; -import org.ethereum.beacon.discovery.NodeContext; -import org.ethereum.beacon.discovery.enr.NodeRecord; -import org.ethereum.beacon.discovery.packet.AuthHeaderMessagePacket; -import org.javatuples.Triplet; -import org.web3j.crypto.ECKeyPair; -import tech.pegasys.artemis.util.bytes.BytesValue; - -public class AuthHeaderMessagePacketHandler implements PacketHandler { - private final NodeContext context; - private final Logger logger; - - public AuthHeaderMessagePacketHandler(NodeContext context, Logger logger) { - this.context = context; - this.logger = logger; - } - - @Override - public boolean handle(AuthHeaderMessagePacket packet) { - try { - byte[] ephemeralKeyBytes = new byte[32]; - Functions.getRandom().nextBytes(ephemeralKeyBytes); - ECKeyPair ephemeralKey = ECKeyPair.create(ephemeralKeyBytes); - Triplet hkdf = - Functions.hkdf_expand( - context.getHomeNodeId(), - context.getNodeRecord().getNodeId(), - BytesValue.wrap(ephemeralKey.getPrivateKey().toByteArray()), - context.getIdNonce(), - (BytesValue) context.getNodeRecord().get(NodeRecord.FIELD_PKEY_SECP256K1)); - BytesValue initiatorKey = hkdf.getValue0(); - context.setInitiatorKey(initiatorKey); - BytesValue authResponseKey = hkdf.getValue2(); - packet.decode(initiatorKey, authResponseKey); - packet.verify(context.getAuthTag().get(), context.getIdNonce()); - context.handleMessage(packet.getMessage()); - } catch (AssertionError ex) { - logger.info( - String.format( - "Verification not passed for message [%s] from node %s in status %s", - packet, context.getNodeRecord(), context.getStatus())); - } catch (Exception ex) { - String error = - String.format( - "Failed to read message [%s] from node %s in status %s", - packet, context.getNodeRecord(), context.getStatus()); - logger.error(error, ex); - return false; - } - return true; - } -} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/handler/MessagePacketHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/handler/MessagePacketHandler.java deleted file mode 100644 index 23cc49003..000000000 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/handler/MessagePacketHandler.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.ethereum.beacon.discovery.packet.handler; - -import org.apache.logging.log4j.Logger; -import org.ethereum.beacon.discovery.NodeContext; -import org.ethereum.beacon.discovery.packet.MessagePacket; - -public class MessagePacketHandler implements PacketHandler { - private final NodeContext context; - private final Logger logger; - - public MessagePacketHandler(NodeContext context, Logger logger) { - this.context = context; - this.logger = logger; - } - - @Override - public boolean handle(MessagePacket packet) { - try { - packet.decode(context.getInitiatorKey()); - packet.verify(context.getAuthTag().get()); - context.handleMessage(packet.getMessage()); - } catch (AssertionError ex) { - logger.info( - String.format( - "Verification not passed for message [%s] from node %s in status %s", - packet, context.getNodeRecord(), context.getStatus())); - } catch (Exception ex) { - String error = - String.format( - "Failed to read message [%s] from node %s in status %s", - packet, context.getNodeRecord(), context.getStatus()); - logger.error(error, ex); - return false; - } - return true; - } -} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/handler/PacketHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/handler/PacketHandler.java deleted file mode 100644 index b2b1e4a5b..000000000 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/handler/PacketHandler.java +++ /dev/null @@ -1,7 +0,0 @@ -package org.ethereum.beacon.discovery.packet.handler; - -import org.ethereum.beacon.discovery.packet.Packet; - -public interface PacketHandler

{ - boolean handle(P packet); -} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/Envelope.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/Envelope.java index 494597311..7280605ab 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/Envelope.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/Envelope.java @@ -2,23 +2,34 @@ import java.util.HashMap; import java.util.Map; +import java.util.UUID; public class Envelope { - private Map data = new HashMap<>(); + private UUID id; - public synchronized void put(String key, Object value) { + public Envelope() { + this.id = UUID.randomUUID(); + } + + private Map data = new HashMap<>(); + + public synchronized void put(Field key, Object value) { data.put(key, value); } - public synchronized Object get(String key) { + public synchronized Object get(Field key) { return data.get(key); } - public synchronized boolean remove(String key) { + public synchronized boolean remove(Field key) { return data.remove(key) != null; } - public synchronized boolean contains(String key) { + public synchronized boolean contains(Field key) { return data.containsKey(key); } + + public UUID getId() { + return id; + } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/Field.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/Field.java new file mode 100644 index 000000000..365a40ab1 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/Field.java @@ -0,0 +1,17 @@ +package org.ethereum.beacon.discovery.pipeline; + +public enum Field { + NEED_CONTEXT, // Node id, requests context resolving + CONTEXT, // Node context + INCOMING, // Raw incoming data + PACKET_UNKNOWN, // Unknown packet + PACKET_WHOAREYOU, // WhoAreYou packet + PACKET_AUTH_HEADER_MESSAGE, // Auth header message packet + PACKET_MESSAGE, // Standard message packet + MESSAGE, // Message extracted from the packet + NODE, // Sender/recipient node + BAD_PACKET, // Bad, rejected packet + TASK, // Task to perform + FUTURE, // Completable future + +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/Pipeline.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/Pipeline.java index 7795630b1..da58903f3 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/Pipeline.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/Pipeline.java @@ -5,7 +5,7 @@ public interface Pipeline { Pipeline build(); - void send(Object object); + void push(Object object); Pipeline addHandler(EnvelopeHandler envelopeHandler); diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/PipelineImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/PipelineImpl.java index 039573c8e..baafbde25 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/PipelineImpl.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/PipelineImpl.java @@ -10,8 +10,9 @@ import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; +import static org.ethereum.beacon.discovery.pipeline.Field.INCOMING; + public class PipelineImpl implements Pipeline { - public static final String INCOMING = "raw"; private final List envelopeHandlers = new ArrayList<>(); private final AtomicBoolean started = new AtomicBoolean(false); private Flux pipeline = ReplayProcessor.cacheLast(); @@ -29,7 +30,7 @@ public synchronized Pipeline build() { } @Override - public void send(Object object) { + public void push(Object object) { if (!started.get()) { throw new RuntimeException("You should build pipeline first"); } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/AuthHeaderMessagePacketHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/AuthHeaderMessagePacketHandler.java new file mode 100644 index 000000000..e22c09c97 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/AuthHeaderMessagePacketHandler.java @@ -0,0 +1,92 @@ +package org.ethereum.beacon.discovery.pipeline.handler; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.ethereum.beacon.discovery.Functions; +import org.ethereum.beacon.discovery.NodeContext; +import org.ethereum.beacon.discovery.enr.NodeRecord; +import org.ethereum.beacon.discovery.message.DiscoveryV5Message; +import org.ethereum.beacon.discovery.message.MessageCode; +import org.ethereum.beacon.discovery.packet.AuthHeaderMessagePacket; +import org.ethereum.beacon.discovery.pipeline.Envelope; +import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; +import org.ethereum.beacon.discovery.pipeline.Field; +import org.ethereum.beacon.discovery.task.TaskType; +import org.javatuples.Triplet; +import org.web3j.crypto.ECKeyPair; +import tech.pegasys.artemis.util.bytes.BytesValue; + +import java.util.concurrent.CompletableFuture; + +import static org.ethereum.beacon.discovery.NodeContext.SessionStatus.AUTHENTICATED; + +/** Handles {@link AuthHeaderMessagePacket} in {@link Field#PACKET_AUTH_HEADER_MESSAGE} field */ +public class AuthHeaderMessagePacketHandler implements EnvelopeHandler { + private static final Logger logger = LogManager.getLogger(AuthHeaderMessagePacketHandler.class); + + @Override + public void handle(Envelope envelope) { + if (!envelope.contains(Field.PACKET_AUTH_HEADER_MESSAGE)) { + return; + } + if (!envelope.contains(Field.CONTEXT)) { + return; + } + AuthHeaderMessagePacket packet = + (AuthHeaderMessagePacket) envelope.get(Field.PACKET_AUTH_HEADER_MESSAGE); + NodeContext context = (NodeContext) envelope.get(Field.CONTEXT); + + try { + byte[] ephemeralKeyBytes = new byte[32]; + Functions.getRandom().nextBytes(ephemeralKeyBytes); + ECKeyPair ephemeralKey = ECKeyPair.create(ephemeralKeyBytes); + Triplet hkdf = + Functions.hkdf_expand( + context.getHomeNodeId(), + context.getNodeRecord().getNodeId(), + BytesValue.wrap(ephemeralKey.getPrivateKey().toByteArray()), + context.getIdNonce(), + (BytesValue) context.getNodeRecord().get(NodeRecord.FIELD_PKEY_SECP256K1)); + BytesValue initiatorKey = hkdf.getValue0(); + context.setInitiatorKey(initiatorKey); + BytesValue authResponseKey = hkdf.getValue2(); + packet.decode(initiatorKey, authResponseKey); + packet.verify(context.getAuthTag().get(), context.getIdNonce()); + envelope.put(Field.MESSAGE, packet.getMessage()); + } catch (AssertionError ex) { + logger.info( + String.format( + "Verification not passed for message [%s] from node %s in status %s", + packet, context.getNodeRecord(), context.getStatus())); + } catch (Exception ex) { + String error = + String.format( + "Failed to read message [%s] from node %s in status %s", + packet, context.getNodeRecord(), context.getStatus()); + logger.error(error, ex); + envelope.remove(Field.PACKET_AUTH_HEADER_MESSAGE); + if (context.loadFuture() != null) { + CompletableFuture future = context.loadFuture(); + context.saveFuture(null); + future.completeExceptionally(ex); + } + return; + } + context.setStatus(AUTHENTICATED); + envelope.remove(Field.PACKET_AUTH_HEADER_MESSAGE); + if (context.loadFuture() != null) { + boolean taskCompleted = false; + if (envelope.get(Field.TASK).equals(TaskType.PING) && packet.getMessage() instanceof DiscoveryV5Message && ((DiscoveryV5Message) packet.getMessage()).getCode() == MessageCode.PONG) { + taskCompleted = true; + } + if (envelope.get(Field.TASK).equals(TaskType.FINDNODE) && packet.getMessage() instanceof DiscoveryV5Message && ((DiscoveryV5Message) packet.getMessage()).getCode() == MessageCode.NODES) { + taskCompleted = true; + } + if (taskCompleted) { + CompletableFuture future = context.loadFuture(); + future.complete(null); + context.saveFuture(null); + } + } + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/IncomingDataHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/IncomingDataHandler.java deleted file mode 100644 index fabf12077..000000000 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/IncomingDataHandler.java +++ /dev/null @@ -1,22 +0,0 @@ -package org.ethereum.beacon.discovery.pipeline.handler; - -import org.ethereum.beacon.discovery.packet.UnknownPacket; -import org.ethereum.beacon.discovery.pipeline.Envelope; -import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; -import tech.pegasys.artemis.util.bytes.BytesValue; - -import static org.ethereum.beacon.discovery.pipeline.PipelineImpl.INCOMING; - -public class IncomingDataHandler implements EnvelopeHandler { - - public static final String UNKNOWN = "unknown"; - - @Override - public void handle(Envelope envelope) { - if (!envelope.contains(INCOMING)) { - return; - } - envelope.put(UNKNOWN, new UnknownPacket((BytesValue) envelope.get(INCOMING))); - envelope.remove(INCOMING); - } -} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/IncomingDataPacker.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/IncomingDataPacker.java new file mode 100644 index 000000000..91fc6585e --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/IncomingDataPacker.java @@ -0,0 +1,26 @@ +package org.ethereum.beacon.discovery.pipeline.handler; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.ethereum.beacon.discovery.packet.UnknownPacket; +import org.ethereum.beacon.discovery.pipeline.Envelope; +import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; +import org.ethereum.beacon.discovery.pipeline.Field; +import tech.pegasys.artemis.util.bytes.BytesValue; + +/** Handles raw BytesValue incoming data in {@link Field#INCOMING} */ +public class IncomingDataPacker implements EnvelopeHandler { + private static final Logger logger = LogManager.getLogger(IncomingDataPacker.class); + + @Override + public void handle(Envelope envelope) { + if (!envelope.contains(Field.INCOMING)) { + return; + } + UnknownPacket unknownPacket = new UnknownPacket((BytesValue) envelope.get(Field.INCOMING)); + envelope.put(Field.PACKET_UNKNOWN, unknownPacket); + logger.trace( + () -> String.format("Incoming packet %s in envelope #%s", unknownPacket, envelope.getId())); + envelope.remove(Field.INCOMING); + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/IncomingPacketContextHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/IncomingPacketContextHandler.java deleted file mode 100644 index 8fcfac630..000000000 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/IncomingPacketContextHandler.java +++ /dev/null @@ -1,24 +0,0 @@ -package org.ethereum.beacon.discovery.pipeline.handler; - -import org.ethereum.beacon.discovery.NodeContext; -import org.ethereum.beacon.discovery.packet.UnknownPacket; -import org.ethereum.beacon.discovery.pipeline.Envelope; -import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; - -import static org.ethereum.beacon.discovery.pipeline.handler.IncomingDataHandler.UNKNOWN; -import static org.ethereum.beacon.discovery.pipeline.handler.WhoAreYouContextHandler.CONTEXT; - -public class IncomingPacketContextHandler implements EnvelopeHandler { - @Override - public void handle(Envelope envelope) { - if (!envelope.contains(CONTEXT)) { - return; - } - if (!envelope.contains(UNKNOWN)) { - return; - } - NodeContext context = (NodeContext) envelope.get(CONTEXT); - UnknownPacket packet = (UnknownPacket) envelope.get(UNKNOWN); - context.addIncomingEvent(packet); - } -} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/MessageHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/MessageHandler.java new file mode 100644 index 000000000..06855c58e --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/MessageHandler.java @@ -0,0 +1,27 @@ +package org.ethereum.beacon.discovery.pipeline.handler; + +import org.ethereum.beacon.discovery.DiscoveryV5MessageProcessor; +import org.ethereum.beacon.discovery.MessageProcessor; +import org.ethereum.beacon.discovery.NodeContext; +import org.ethereum.beacon.discovery.message.DiscoveryMessage; +import org.ethereum.beacon.discovery.pipeline.Envelope; +import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; +import org.ethereum.beacon.discovery.pipeline.Field; + +public class MessageHandler implements EnvelopeHandler { + @Override + public void handle(Envelope envelope) { + if (!envelope.contains(Field.MESSAGE)) { + return; + } + if (!envelope.contains(Field.CONTEXT)) { + return; + } + + NodeContext context = (NodeContext) envelope.get(Field.CONTEXT); + DiscoveryMessage message = (DiscoveryMessage) envelope.get(Field.MESSAGE); + // TODO: optimize + MessageProcessor messageProcessor = new MessageProcessor(new DiscoveryV5MessageProcessor()); + messageProcessor.handleIncoming(message, context); + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/MessagePacketHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/MessagePacketHandler.java new file mode 100644 index 000000000..1cf9d14e2 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/MessagePacketHandler.java @@ -0,0 +1,71 @@ +package org.ethereum.beacon.discovery.pipeline.handler; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.ethereum.beacon.discovery.NodeContext; +import org.ethereum.beacon.discovery.message.DiscoveryV5Message; +import org.ethereum.beacon.discovery.message.MessageCode; +import org.ethereum.beacon.discovery.packet.MessagePacket; +import org.ethereum.beacon.discovery.pipeline.Envelope; +import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; +import org.ethereum.beacon.discovery.pipeline.Field; +import org.ethereum.beacon.discovery.task.TaskType; + +import java.util.concurrent.CompletableFuture; + +/** Handles {@link MessagePacket} in {@link Field#PACKET_MESSAGE} field */ +public class MessagePacketHandler implements EnvelopeHandler { + private static final Logger logger = LogManager.getLogger(MessagePacketHandler.class); + + @Override + public void handle(Envelope envelope) { + if (!envelope.contains(Field.PACKET_MESSAGE)) { + return; + } + if (!envelope.contains(Field.CONTEXT)) { + return; + } + MessagePacket packet = (MessagePacket) envelope.get(Field.PACKET_MESSAGE); + NodeContext context = (NodeContext) envelope.get(Field.CONTEXT); + + try { + packet.decode(context.getInitiatorKey()); + packet.verify(context.getAuthTag().get()); + envelope.put(Field.MESSAGE, packet.getMessage()); + } catch (AssertionError ex) { + logger.info( + String.format( + "Verification not passed for message [%s] from node %s in status %s", + packet, context.getNodeRecord(), context.getStatus())); + } catch (Exception ex) { + String error = + String.format( + "Failed to read message [%s] from node %s in status %s", + packet, context.getNodeRecord(), context.getStatus()); + logger.error(error, ex); + envelope.remove(Field.PACKET_MESSAGE); + if (envelope.contains(Field.FUTURE)) { + CompletableFuture future = (CompletableFuture) envelope.get(Field.FUTURE); + future.completeExceptionally(ex); + } + return; + } + envelope.remove(Field.PACKET_MESSAGE); + CompletableFuture future = context.loadFuture(); + TaskType taskType = context.loadTask(); + if (future != null) { + boolean taskCompleted = false; + if (TaskType.PING.equals(taskType) && packet.getMessage() instanceof DiscoveryV5Message && ((DiscoveryV5Message) packet.getMessage()).getCode() == MessageCode.PONG) { + taskCompleted = true; + } + if (TaskType.FINDNODE.equals(taskType) && packet.getMessage() instanceof DiscoveryV5Message && ((DiscoveryV5Message) packet.getMessage()).getCode() == MessageCode.NODES) { + taskCompleted = true; + } + if (taskCompleted) { + future.complete(null); + context.saveFuture(null); + context.saveTask(null); + } + } + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeContextRequestHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeContextRequestHandler.java index ad343fcb8..8a8fa704e 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeContextRequestHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeContextRequestHandler.java @@ -3,23 +3,24 @@ import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.pipeline.Envelope; import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; +import org.ethereum.beacon.discovery.pipeline.Field; import org.javatuples.Pair; -import static org.ethereum.beacon.discovery.pipeline.handler.NodeIdContextHandler.NEED_CONTEXT; - +/** + * Searches for node in {@link Field#NODE} and requests context resolving using {@link + * Field#NEED_CONTEXT} + */ public class NodeContextRequestHandler implements EnvelopeHandler { - public static final String NODE = "node"; - @Override public void handle(Envelope envelope) { - if (!envelope.contains(NODE)) { + if (!envelope.contains(Field.NODE)) { return; } envelope.put( - NEED_CONTEXT, + Field.NEED_CONTEXT, Pair.with( - ((NodeRecord) envelope.get(NODE)).getNodeId(), + ((NodeRecord) envelope.get(Field.NODE)).getNodeId(), (Runnable) () -> { // TODO: failure })); diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeIdContextHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeIdToContext.java similarity index 75% rename from discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeIdContextHandler.java rename to discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeIdToContext.java index 4802f9bea..3bd091983 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeIdContextHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeIdToContext.java @@ -6,9 +6,9 @@ import org.ethereum.beacon.discovery.NodeRecordInfo; import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.network.NetworkParcelV5; -import org.ethereum.beacon.discovery.packet.UnknownPacket; import org.ethereum.beacon.discovery.pipeline.Envelope; import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; +import org.ethereum.beacon.discovery.pipeline.Field; import org.ethereum.beacon.discovery.pipeline.Pipeline; import org.ethereum.beacon.discovery.storage.AuthTagRepository; import org.ethereum.beacon.discovery.storage.NodeBucketStorage; @@ -23,25 +23,29 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; -import static org.ethereum.beacon.discovery.pipeline.handler.IncomingDataHandler.UNKNOWN; -import static org.ethereum.beacon.discovery.pipeline.handler.WhoAreYouContextHandler.BAD_PACKET; -import static org.ethereum.beacon.discovery.pipeline.handler.WhoAreYouContextHandler.CONTEXT; - -public class NodeIdContextHandler implements EnvelopeHandler { +/** + * Performs {@link Field#NEED_CONTEXT} request. Looks up for Node context based on NodeId, which + * should be in request field and stores it in {@link Field#CONTEXT} field. + */ +public class NodeIdToContext implements EnvelopeHandler { private static final int CLEANUP_DELAY_SECONDS = 180; - private static final Logger logger = LogManager.getLogger(NodeIdContextHandler.class); - public static final String NEED_CONTEXT = "NEED_CONTEXT"; + private static final Logger logger = LogManager.getLogger(NodeIdToContext.class); private final NodeRecord homeNodeRecord; private final NodeBucketStorage nodeBucketStorage; private final AuthTagRepository authTagRepo; private final Map recentContexts = new ConcurrentHashMap<>(); // nodeId -> context private final NodeTable nodeTable; + private final Pipeline outgoingPipeline; private ExpirationScheduler contextExpirationScheduler = new ExpirationScheduler<>(CLEANUP_DELAY_SECONDS, TimeUnit.SECONDS); - private final Pipeline outgoingPipeline; - public NodeIdContextHandler(NodeRecord homeNodeRecord, NodeBucketStorage nodeBucketStorage, AuthTagRepository authTagRepo, NodeTable nodeTable, Pipeline outgoingPipeline) { + public NodeIdToContext( + NodeRecord homeNodeRecord, + NodeBucketStorage nodeBucketStorage, + AuthTagRepository authTagRepo, + NodeTable nodeTable, + Pipeline outgoingPipeline) { this.homeNodeRecord = homeNodeRecord; this.nodeBucketStorage = nodeBucketStorage; this.authTagRepo = authTagRepo; @@ -51,20 +55,25 @@ public NodeIdContextHandler(NodeRecord homeNodeRecord, NodeBucketStorage nodeBuc @Override public void handle(Envelope envelope) { - if (!envelope.contains(NEED_CONTEXT)) { + if (!envelope.contains(Field.NEED_CONTEXT)) { return; } - Pair contextRequest = (Pair) envelope.get(NEED_CONTEXT); - envelope.remove(NEED_CONTEXT); + Pair contextRequest = + (Pair) envelope.get(Field.NEED_CONTEXT); + envelope.remove(Field.NEED_CONTEXT); Optional nodeContextOptional = getContext(contextRequest.getValue0()); if (nodeContextOptional.isPresent()) { - envelope.put(CONTEXT, nodeContextOptional.get()); + envelope.put(Field.CONTEXT, nodeContextOptional.get()); + logger.trace( + () -> + String.format( + "Context resolved: %s in envelope #%s", + nodeContextOptional.get(), envelope.getId())); } else { contextRequest.getValue1().run(); } } - private Optional getContext(Bytes32 nodeId) { NodeContext context = recentContexts.get(nodeId); if (context == null) { @@ -83,7 +92,7 @@ private Optional getContext(Bytes32 nodeId) { nodeTable, nodeBucketStorage, authTagRepo, - packet -> outgoingPipeline.send(new NetworkParcelV5(packet, nodeRecord)), + packet -> outgoingPipeline.push(new NetworkParcelV5(packet, nodeRecord)), random); recentContexts.put(nodeId, context); } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/handler/UnknownPacketHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NotExpectedIncomingPacketHandler.java similarity index 52% rename from discovery/src/main/java/org/ethereum/beacon/discovery/packet/handler/UnknownPacketHandler.java rename to discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NotExpectedIncomingPacketHandler.java index a84955e39..e1d221221 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/handler/UnknownPacketHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NotExpectedIncomingPacketHandler.java @@ -1,5 +1,6 @@ -package org.ethereum.beacon.discovery.packet.handler; +package org.ethereum.beacon.discovery.pipeline.handler; +import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.ethereum.beacon.discovery.Functions; import org.ethereum.beacon.discovery.NodeContext; @@ -7,32 +8,38 @@ import org.ethereum.beacon.discovery.packet.RandomPacket; import org.ethereum.beacon.discovery.packet.UnknownPacket; import org.ethereum.beacon.discovery.packet.WhoAreYouPacket; +import org.ethereum.beacon.discovery.pipeline.Envelope; +import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; +import org.ethereum.beacon.discovery.pipeline.Field; import tech.pegasys.artemis.util.bytes.Bytes32; import tech.pegasys.artemis.util.bytes.BytesValue; -public class UnknownPacketHandler implements PacketHandler { - private final NodeContext context; - private final Logger logger; - - public UnknownPacketHandler(NodeContext context, Logger logger) { - this.context = context; - this.logger = logger; - } +/** Handles {@link UnknownPacket} from node, which is not on any stage of the handshake with us */ +public class NotExpectedIncomingPacketHandler implements EnvelopeHandler { + private static final Logger logger = LogManager.getLogger(NotExpectedIncomingPacketHandler.class); @Override - public boolean handle(UnknownPacket packet) { + public void handle(Envelope envelope) { + if (!envelope.contains(Field.PACKET_UNKNOWN)) { + return; + } + if (!envelope.contains(Field.CONTEXT)) { + return; + } + UnknownPacket unknownPacket = (UnknownPacket) envelope.get(Field.PACKET_UNKNOWN); + NodeContext context = (NodeContext) envelope.get(Field.CONTEXT); try { // packet it either random or message packet if session is expired BytesValue authTag = null; try { - RandomPacket randomPacket = packet.getRandomPacket(); + RandomPacket randomPacket = unknownPacket.getRandomPacket(); authTag = randomPacket.getAuthTag(); } catch (Exception ex) { // Not fatal, 1st attempt } // 2nd attempt if (authTag == null) { - MessagePacket messagePacket = packet.getMessagePacket(); + MessagePacket messagePacket = unknownPacket.getMessagePacket(); authTag = messagePacket.getAuthTag(); } context.setAuthTag(authTag); @@ -46,20 +53,23 @@ public boolean handle(UnknownPacket packet) { authTag, idNonce, context.getNodeRecord().getSeq()); - context.addOutgoingEvent(whoAreYouPacket); + context.sendOutgoing(whoAreYouPacket); } catch (AssertionError ex) { logger.info( String.format( "Verification not passed for message [%s] from node %s in status %s", - packet, context.getNodeRecord(), context.getStatus())); + unknownPacket, context.getNodeRecord(), context.getStatus())); } catch (Exception ex) { String error = String.format( "Failed to read message [%s] from node %s in status %s", - packet, context.getNodeRecord(), context.getStatus()); + unknownPacket, context.getNodeRecord(), context.getStatus()); logger.error(error, ex); - return false; + envelope.put(Field.BAD_PACKET, envelope.get(Field.PACKET_UNKNOWN)); + envelope.remove(Field.PACKET_UNKNOWN); + return; } - return true; + context.setStatus(NodeContext.SessionStatus.WHOAREYOU_SENT); + envelope.remove(Field.PACKET_UNKNOWN); } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/OutgoingParcelHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/OutgoingParcelHandler.java index 168b68654..ae1444071 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/OutgoingParcelHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/OutgoingParcelHandler.java @@ -1,13 +1,16 @@ package org.ethereum.beacon.discovery.pipeline.handler; import org.ethereum.beacon.discovery.network.NetworkParcel; -import org.ethereum.beacon.discovery.network.NetworkParcelV5; import org.ethereum.beacon.discovery.pipeline.Envelope; import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; +import org.ethereum.beacon.discovery.pipeline.Field; import reactor.core.publisher.FluxSink; -import static org.ethereum.beacon.discovery.pipeline.PipelineImpl.INCOMING; - +/** + * Looks up for {@link NetworkParcel} in {@link Field#INCOMING} field. If it's found, it shows that + * we have outgoing parcel at the very first stage. Handler pushes it to `outgoingSink` stream which + * is linked with discovery client. + */ public class OutgoingParcelHandler implements EnvelopeHandler { private final FluxSink outgoingSink; @@ -18,12 +21,12 @@ public OutgoingParcelHandler(FluxSink outgoingSink) { @Override public void handle(Envelope envelope) { - if (!envelope.contains(INCOMING)) { + if (!envelope.contains(Field.INCOMING)) { return; } - if (envelope.get(INCOMING) instanceof NetworkParcel) { - outgoingSink.next((NetworkParcel) envelope.get(INCOMING)); - envelope.remove(INCOMING); + if (envelope.get(Field.INCOMING) instanceof NetworkParcel) { + outgoingSink.next((NetworkParcel) envelope.get(Field.INCOMING)); + envelope.remove(Field.INCOMING); } } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/TaskHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/TaskHandler.java index daf6ffe73..905fda4b2 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/TaskHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/TaskHandler.java @@ -1,37 +1,62 @@ package org.ethereum.beacon.discovery.pipeline.handler; import org.ethereum.beacon.discovery.NodeContext; +import org.ethereum.beacon.discovery.packet.RandomPacket; import org.ethereum.beacon.discovery.pipeline.Envelope; import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; +import org.ethereum.beacon.discovery.pipeline.Field; +import org.ethereum.beacon.discovery.task.TaskMessageFactory; +import org.ethereum.beacon.discovery.task.TaskType; +import tech.pegasys.artemis.util.bytes.BytesValue; +import java.security.SecureRandom; +import java.util.Random; import java.util.concurrent.CompletableFuture; -import java.util.function.BiConsumer; - -import static org.ethereum.beacon.discovery.pipeline.handler.WhoAreYouContextHandler.CONTEXT; +/** Performs task execution for any task found in {@link Field#TASK} */ public class TaskHandler implements EnvelopeHandler { + private final Random rnd; - public static final String TASK = "task"; - public static final String PING = "PING"; - public static final String FUTURE = "future"; + public TaskHandler(Random rnd) { + this.rnd = rnd; + } @Override public void handle(Envelope envelope) { - if (!envelope.contains(TASK)) { + if (!envelope.contains(Field.TASK)) { return; } - if (!envelope.get(TASK).equals(PING)) { + if (!envelope.get(Field.TASK).equals(TaskType.PING)) { return; // TODO: implement other tasks } - NodeContext context = (NodeContext) envelope.get(CONTEXT); - CompletableFuture completableFuture = (CompletableFuture) envelope.get(FUTURE); - context.initiate().whenComplete((aVoid, throwable) -> { - if (throwable != null) { - completableFuture.completeExceptionally(throwable); + NodeContext context = (NodeContext) envelope.get(Field.CONTEXT); + CompletableFuture completableFuture = + (CompletableFuture) envelope.get(Field.FUTURE); + if (context.getStatus().equals(NodeContext.SessionStatus.INITIAL)) { + byte[] authTagBytes = new byte[12]; + rnd.nextBytes(authTagBytes); + BytesValue authTag = BytesValue.wrap(authTagBytes); + RandomPacket randomPacket = + RandomPacket.create(context.getHomeNodeId(), context.getNodeRecord().getNodeId(), authTag, new SecureRandom()); + context.setAuthTag(authTag); + context.sendOutgoing(randomPacket); + context.setStatus(NodeContext.SessionStatus.RANDOM_PACKET_SENT); + context.saveFuture(completableFuture); + } else if (context.getStatus().equals(NodeContext.SessionStatus.AUTHENTICATED)) { + if (envelope.get(Field.TASK).equals(TaskType.PING)) { + context.sendOutgoing(TaskMessageFactory.createPingPacket(context)); + context.saveTask((TaskType) envelope.get(Field.TASK)); + context.saveFuture(completableFuture); + } else if (envelope.get(Field.TASK).equals(TaskType.FINDNODE)) { + context.sendOutgoing(TaskMessageFactory.createFindNodePacket(context)); + context.saveTask((TaskType) envelope.get(Field.TASK)); + context.saveFuture(completableFuture); } else { - completableFuture.complete(aVoid); + throw new RuntimeException(String.format("Task type %s handler not found in envelope %s", envelope.get(Field.TASK), envelope.getId())); } - }); - + } else { + completableFuture.completeExceptionally(new RuntimeException("Already initiating")); + // FIXME: or should we queue? + } } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/UnknownPacketContextHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/UnknownPacketContextHandler.java deleted file mode 100644 index 965bbea1b..000000000 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/UnknownPacketContextHandler.java +++ /dev/null @@ -1,39 +0,0 @@ -package org.ethereum.beacon.discovery.pipeline.handler; - -import org.ethereum.beacon.discovery.enr.NodeRecord; -import org.ethereum.beacon.discovery.packet.UnknownPacket; -import org.ethereum.beacon.discovery.pipeline.Envelope; -import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; -import org.javatuples.Pair; -import tech.pegasys.artemis.util.bytes.Bytes32; - -import static org.ethereum.beacon.discovery.pipeline.handler.IncomingDataHandler.UNKNOWN; -import static org.ethereum.beacon.discovery.pipeline.handler.NodeIdContextHandler.NEED_CONTEXT; -import static org.ethereum.beacon.discovery.pipeline.handler.WhoAreYouContextHandler.BAD_PACKET; - -public class UnknownPacketContextHandler implements EnvelopeHandler { - private static final int CLEANUP_DELAY_SECONDS = 180; - private final Bytes32 homeNodeId; - - public UnknownPacketContextHandler(NodeRecord homeNodeRecord) { - this.homeNodeId = homeNodeRecord.getNodeId(); - } - - @Override - public void handle(Envelope envelope) { - if (!envelope.contains(UNKNOWN)) { - return; - } - UnknownPacket unknownPacket = (UnknownPacket) envelope.get(UNKNOWN); - Bytes32 fromNodeId = unknownPacket.getSourceNodeId(homeNodeId); - envelope.put( - NEED_CONTEXT, - Pair.with( - fromNodeId, - (Runnable) - () -> { - envelope.put(BAD_PACKET, envelope.get(UNKNOWN)); - envelope.remove(UNKNOWN); - })); - } -} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/UnknownPacketTagToSender.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/UnknownPacketTagToSender.java new file mode 100644 index 000000000..bc1385938 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/UnknownPacketTagToSender.java @@ -0,0 +1,40 @@ +package org.ethereum.beacon.discovery.pipeline.handler; + +import org.ethereum.beacon.discovery.enr.NodeRecord; +import org.ethereum.beacon.discovery.packet.UnknownPacket; +import org.ethereum.beacon.discovery.pipeline.Envelope; +import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; +import org.ethereum.beacon.discovery.pipeline.Field; +import org.javatuples.Pair; +import tech.pegasys.artemis.util.bytes.Bytes32; + +/** + * Assuming we have some unknown packet in {@link Field#PACKET_UNKNOWN}, resolves sender node id + * using `tag` field of the packet. Next, puts it to the {@link Field#NEED_CONTEXT} so sender + * context could be resolved by another handler. + */ +public class UnknownPacketTagToSender implements EnvelopeHandler { + private final Bytes32 homeNodeId; + + public UnknownPacketTagToSender(NodeRecord homeNodeRecord) { + this.homeNodeId = homeNodeRecord.getNodeId(); + } + + @Override + public void handle(Envelope envelope) { + if (!envelope.contains(Field.PACKET_UNKNOWN)) { + return; + } + UnknownPacket unknownPacket = (UnknownPacket) envelope.get(Field.PACKET_UNKNOWN); + Bytes32 fromNodeId = unknownPacket.getSourceNodeId(homeNodeId); + envelope.put( + Field.NEED_CONTEXT, + Pair.with( + fromNodeId, + (Runnable) + () -> { + envelope.put(Field.BAD_PACKET, envelope.get(Field.PACKET_UNKNOWN)); + envelope.remove(Field.PACKET_UNKNOWN); + })); + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/UnknownPacketTypeByStatus.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/UnknownPacketTypeByStatus.java new file mode 100644 index 000000000..488e8e215 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/UnknownPacketTypeByStatus.java @@ -0,0 +1,67 @@ +package org.ethereum.beacon.discovery.pipeline.handler; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.ethereum.beacon.discovery.NodeContext; +import org.ethereum.beacon.discovery.packet.AuthHeaderMessagePacket; +import org.ethereum.beacon.discovery.packet.MessagePacket; +import org.ethereum.beacon.discovery.packet.UnknownPacket; +import org.ethereum.beacon.discovery.pipeline.Envelope; +import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; +import org.ethereum.beacon.discovery.pipeline.Field; + +/** + * Resolves incoming packet type based on context states and places packet into the corresponding + * field. Doesn't recognize WhoAreYou packet, last should be resolved by {@link WhoAreYouAttempt} + */ +public class UnknownPacketTypeByStatus implements EnvelopeHandler { + private static final Logger logger = LogManager.getLogger(UnknownPacketTypeByStatus.class); + + @Override + public void handle(Envelope envelope) { + if (!envelope.contains(Field.CONTEXT)) { + return; + } + if (!envelope.contains(Field.PACKET_UNKNOWN)) { + return; + } + UnknownPacket unknownPacket = (UnknownPacket) envelope.get(Field.PACKET_UNKNOWN); + NodeContext context = (NodeContext) envelope.get(Field.CONTEXT); + switch (context.getStatus()) { + case INITIAL: + { + // We still don't know what's the type of the packet + break; + } + case RANDOM_PACKET_SENT: + { + // Should receive WHOAREYOU in answer, not our case + break; + } + case WHOAREYOU_SENT: + { + AuthHeaderMessagePacket authHeaderMessagePacket = + unknownPacket.getAuthHeaderMessagePacket(); + envelope.put(Field.PACKET_AUTH_HEADER_MESSAGE, authHeaderMessagePacket); + envelope.remove(Field.PACKET_UNKNOWN); + break; + } + case AUTHENTICATED: + { + MessagePacket messagePacket = unknownPacket.getMessagePacket(); + envelope.put(Field.PACKET_MESSAGE, messagePacket); + envelope.remove(Field.PACKET_UNKNOWN); + break; + } + default: + { + String error = + String.format( + "Not expected status:%s from node: %s", + context.getStatus(), context.getNodeRecord()); + logger.error(error); + throw new RuntimeException(error); + } + } + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouAttempt.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouAttempt.java new file mode 100644 index 000000000..1077b0745 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouAttempt.java @@ -0,0 +1,32 @@ +package org.ethereum.beacon.discovery.pipeline.handler; + +import org.ethereum.beacon.discovery.packet.UnknownPacket; +import org.ethereum.beacon.discovery.pipeline.Envelope; +import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; +import org.ethereum.beacon.discovery.pipeline.Field; +import tech.pegasys.artemis.util.bytes.Bytes32; + +/** + * Tries to get WHOAREYOU packet from unknown incoming packet in {@link Field#PACKET_UNKNOWN}. If it + * was successful, places the result in {@link Field#PACKET_WHOAREYOU} + */ +public class WhoAreYouAttempt implements EnvelopeHandler { + private final Bytes32 homeNodeId; + + public WhoAreYouAttempt(Bytes32 homeNodeId) { + this.homeNodeId = homeNodeId; + } + + @Override + public void handle(Envelope envelope) { + if (!envelope.contains(Field.PACKET_UNKNOWN)) { + return; + } + if (!((UnknownPacket) envelope.get(Field.PACKET_UNKNOWN)).isWhoAreYouPacket(homeNodeId)) { + return; + } + UnknownPacket unknownPacket = (UnknownPacket) envelope.get(Field.PACKET_UNKNOWN); + envelope.put(Field.PACKET_WHOAREYOU, unknownPacket.getWhoAreYouPacket()); + envelope.remove(Field.PACKET_UNKNOWN); + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouContextHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouContextHandler.java deleted file mode 100644 index c370b6b46..000000000 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouContextHandler.java +++ /dev/null @@ -1,38 +0,0 @@ -package org.ethereum.beacon.discovery.pipeline.handler; - -import org.ethereum.beacon.discovery.NodeContext; -import org.ethereum.beacon.discovery.packet.WhoAreYouPacket; -import org.ethereum.beacon.discovery.pipeline.Envelope; -import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; -import org.ethereum.beacon.discovery.storage.AuthTagRepository; -import tech.pegasys.artemis.util.bytes.Bytes32; - -import java.util.Optional; - -import static org.ethereum.beacon.discovery.pipeline.handler.WhoAreYouHandler.WHOAREYOU; - -public class WhoAreYouContextHandler implements EnvelopeHandler { - public static final String CONTEXT = "context"; - public static final String BAD_PACKET = "BAD_PACKET"; - private final AuthTagRepository authTagRepo; - - public WhoAreYouContextHandler(AuthTagRepository authTagRepo) { - this.authTagRepo = authTagRepo; - } - - @Override - public void handle(Envelope envelope) { - if (!envelope.contains(WHOAREYOU)) { - return; - } - - WhoAreYouPacket whoAreYouPacket = (WhoAreYouPacket) envelope.get(WHOAREYOU); - Optional nodeContextOptional = authTagRepo.get(whoAreYouPacket.getAuthTag()); - if (nodeContextOptional.isPresent()) { - envelope.put(CONTEXT, nodeContextOptional.get()); - } else { - envelope.put(BAD_PACKET, envelope.get(WHOAREYOU)); - envelope.remove(WHOAREYOU); - } - } -} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouContextResolver.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouContextResolver.java new file mode 100644 index 000000000..b91d4be30 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouContextResolver.java @@ -0,0 +1,52 @@ +package org.ethereum.beacon.discovery.pipeline.handler; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.ethereum.beacon.discovery.NodeContext; +import org.ethereum.beacon.discovery.packet.WhoAreYouPacket; +import org.ethereum.beacon.discovery.pipeline.Envelope; +import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; +import org.ethereum.beacon.discovery.pipeline.Field; +import org.ethereum.beacon.discovery.storage.AuthTagRepository; + +import java.util.Optional; + +/** + * Resolves context using `authTagRepo` for `WHOAREYOU` packets which should be placed in {@link + * Field#PACKET_WHOAREYOU} + */ +public class WhoAreYouContextResolver implements EnvelopeHandler { + private static final Logger logger = LogManager.getLogger(WhoAreYouContextResolver.class); + private final AuthTagRepository authTagRepo; + + public WhoAreYouContextResolver(AuthTagRepository authTagRepo) { + this.authTagRepo = authTagRepo; + } + + @Override + public void handle(Envelope envelope) { + if (!envelope.contains(Field.PACKET_WHOAREYOU)) { + return; + } + + WhoAreYouPacket whoAreYouPacket = (WhoAreYouPacket) envelope.get(Field.PACKET_WHOAREYOU); + Optional nodeContextOptional = authTagRepo.get(whoAreYouPacket.getAuthTag()); + if (nodeContextOptional.isPresent() + && nodeContextOptional + .get() + .getStatus() + .equals( + NodeContext.SessionStatus + .RANDOM_PACKET_SENT)) { // FIXME: doesn't handle session expiration + envelope.put(Field.CONTEXT, nodeContextOptional.get()); + logger.trace( + () -> + String.format( + "Context resolved: %s in envelope #%s", + nodeContextOptional.get(), envelope.getId())); + } else { + envelope.put(Field.BAD_PACKET, envelope.get(Field.PACKET_WHOAREYOU)); + envelope.remove(Field.PACKET_WHOAREYOU); + } + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouHandler.java deleted file mode 100644 index 3444fc0f0..000000000 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouHandler.java +++ /dev/null @@ -1,29 +0,0 @@ -package org.ethereum.beacon.discovery.pipeline.handler; - -import org.ethereum.beacon.discovery.packet.UnknownPacket; -import org.ethereum.beacon.discovery.pipeline.Envelope; -import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; -import tech.pegasys.artemis.util.bytes.Bytes32; - -import static org.ethereum.beacon.discovery.pipeline.handler.IncomingDataHandler.UNKNOWN; - -public class WhoAreYouHandler implements EnvelopeHandler { - public static final String WHOAREYOU = "WHOAREYOU"; - private final Bytes32 homeNodeId; - - public WhoAreYouHandler(Bytes32 homeNodeId) { - this.homeNodeId = homeNodeId; - } - - @Override - public void handle(Envelope envelope) { - if (!envelope.contains(UNKNOWN)) { - return; - } - if (!((UnknownPacket) envelope.get(UNKNOWN)).isWhoAreYouPacket(homeNodeId)) { - return; - } - UnknownPacket unknownPacket = (UnknownPacket) envelope.get(UNKNOWN); - envelope.put(WHOAREYOU, unknownPacket.getWhoAreYouPacket()); - } -} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/handler/WhoAreYouPacketHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouPacketHandler.java similarity index 54% rename from discovery/src/main/java/org/ethereum/beacon/discovery/packet/handler/WhoAreYouPacketHandler.java rename to discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouPacketHandler.java index 44fe392a7..4db2c48e1 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/handler/WhoAreYouPacketHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouPacketHandler.java @@ -1,31 +1,39 @@ -package org.ethereum.beacon.discovery.packet.handler; +package org.ethereum.beacon.discovery.pipeline.handler; +import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.ethereum.beacon.discovery.Functions; import org.ethereum.beacon.discovery.NodeContext; import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.message.DiscoveryV5Message; -import org.ethereum.beacon.discovery.message.FindNodeMessage; -import org.ethereum.beacon.discovery.message.MessageCode; +import org.ethereum.beacon.discovery.message.V5Message; import org.ethereum.beacon.discovery.packet.AuthHeaderMessagePacket; import org.ethereum.beacon.discovery.packet.WhoAreYouPacket; +import org.ethereum.beacon.discovery.pipeline.Envelope; +import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; +import org.ethereum.beacon.discovery.pipeline.Field; +import org.ethereum.beacon.discovery.task.TaskMessageFactory; +import org.ethereum.beacon.discovery.task.TaskType; import org.javatuples.Triplet; import org.web3j.crypto.ECKeyPair; import tech.pegasys.artemis.util.bytes.BytesValue; -import static org.ethereum.beacon.discovery.NodeContext.DEFAULT_DISTANCE; +import java.util.concurrent.CompletableFuture; -public class WhoAreYouPacketHandler implements PacketHandler { - private final NodeContext context; - private final Logger logger; - - public WhoAreYouPacketHandler(NodeContext context, Logger logger) { - this.context = context; - this.logger = logger; - } +/** Handles {@link WhoAreYouPacket} in {@link Field#PACKET_WHOAREYOU} field */ +public class WhoAreYouPacketHandler implements EnvelopeHandler { + private static final Logger logger = LogManager.getLogger(WhoAreYouPacketHandler.class); @Override - public boolean handle(WhoAreYouPacket packet) { + public void handle(Envelope envelope) { + if (!envelope.contains(Field.PACKET_WHOAREYOU)) { + return; + } + if (!envelope.contains(Field.CONTEXT)) { + return; + } + WhoAreYouPacket packet = (WhoAreYouPacket) envelope.get(Field.PACKET_WHOAREYOU); + NodeContext context = (NodeContext) envelope.get(Field.CONTEXT); try { BytesValue authTag = context.getAuthTag().get(); packet.verify(context.getHomeNodeId(), authTag); @@ -43,6 +51,16 @@ public boolean handle(WhoAreYouPacket packet) { BytesValue initiatorKey = hkdf.getValue0(); BytesValue staticNodeKey = hkdf.getValue1(); BytesValue authResponseKey = hkdf.getValue2(); + V5Message taskMessage = null; + if (context.loadTask() == TaskType.PING) { + taskMessage = TaskMessageFactory.createPing(context); + } else if (context.loadTask() == TaskType.FINDNODE) { + taskMessage = TaskMessageFactory.createFindNode(context); + } else { + throw new RuntimeException( + String.format( + "Type %s in envelope #%s is not known", context.loadTask(), envelope.getId())); + } AuthHeaderMessagePacket response = AuthHeaderMessagePacket.create( @@ -55,10 +73,8 @@ public boolean handle(WhoAreYouPacket packet) { BytesValue.wrap(ephemeralKey.getPublicKey().toByteArray()), authTag, initiatorKey, - DiscoveryV5Message.from( - new FindNodeMessage( - context.getNextRequestId(MessageCode.FINDNODE), DEFAULT_DISTANCE))); - context.addOutgoingEvent(response); + DiscoveryV5Message.from(taskMessage)); + context.sendOutgoing(response); } catch (AssertionError ex) { logger.info( String.format( @@ -70,8 +86,14 @@ public boolean handle(WhoAreYouPacket packet) { "Failed to read message [%s] from node %s in status %s", packet, context.getNodeRecord(), context.getStatus()); logger.error(error, ex); - return false; + envelope.remove(Field.PACKET_WHOAREYOU); + if (envelope.contains(Field.FUTURE)) { + CompletableFuture future = (CompletableFuture) envelope.get(Field.FUTURE); + future.completeExceptionally(ex); + } + return; } - return true; + context.setStatus(NodeContext.SessionStatus.AUTHENTICATED); + envelope.remove(Field.PACKET_WHOAREYOU); } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryTaskManager.java b/discovery/src/main/java/org/ethereum/beacon/discovery/task/DiscoveryTaskManager.java similarity index 63% rename from discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryTaskManager.java rename to discovery/src/main/java/org/ethereum/beacon/discovery/task/DiscoveryTaskManager.java index 304f3cdce..66e065476 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryTaskManager.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/task/DiscoveryTaskManager.java @@ -1,5 +1,8 @@ -package org.ethereum.beacon.discovery; +package org.ethereum.beacon.discovery.task; +import org.ethereum.beacon.discovery.DiscoveryManager; +import org.ethereum.beacon.discovery.NodeRecordInfo; +import org.ethereum.beacon.discovery.NodeStatus; import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.storage.NodeBucketStorage; import org.ethereum.beacon.discovery.storage.NodeTable; @@ -11,18 +14,22 @@ import java.util.function.Predicate; import java.util.stream.Stream; -import static org.ethereum.beacon.discovery.NodeContext.DEFAULT_DISTANCE; +import static org.ethereum.beacon.discovery.task.TaskMessageFactory.DEFAULT_DISTANCE; /** Manages recurrent node check task(s) */ public class DiscoveryTaskManager { + private static final int LIVE_CHECK_DISTANCE = DEFAULT_DISTANCE; + private static final int RECURSIVE_LOOKUP_DISTANCE = DEFAULT_DISTANCE; private static final int MS_IN_SECOND = 1000; private static final int STATUS_EXPIRATION_SECONDS = 600; - private static final int TASK_INTERVAL_SECONDS = 10; + private static final int LIVE_CHECK_INTERVAL_SECONDS = 1; + private static final int RECURSIVE_LOOKUP_INTERVAL_SECONDS = 10; private static final int RETRY_TIMEOUT_SECONDS = 60; private static final int MAX_RETRIES = 10; private final Scheduler scheduler; private final Bytes32 homeNodeId; - private final NodeConnectTasks nodeConnectTasks; + private final LiveCheckTasks liveCheckTasks; + private final RecursiveLookupTasks recursiveLookupTasks; private final NodeTable nodeTable; private final NodeBucketStorage nodeBucketStorage; /** @@ -40,7 +47,7 @@ public class DiscoveryTaskManager { * *

In all other cases method returns true, meaning node is ready for ping check */ - private final Predicate READY_FOR_PING = + private final Predicate LIVE_CHECK_NODE_RULE = nodeRecord -> { long currentTime = System.currentTimeMillis() / MS_IN_SECOND; if (nodeRecord.getStatus() == NodeStatus.ACTIVE @@ -61,6 +68,27 @@ public class DiscoveryTaskManager { return true; }; + /** + * Checks whether {@link org.ethereum.beacon.discovery.enr.NodeRecord} is ready for FINDNODE query + * which expands the list of all known nodes. + * + *

Node is eligible if + * + *

    + *
  • Node is ACTIVE and last connection retry was not too much time ago + *
+ */ + private final Predicate RECURSIVE_LOOKUP_NODE_RULE = + nodeRecord -> { + long currentTime = System.currentTimeMillis() / MS_IN_SECOND; + if (nodeRecord.getStatus() == NodeStatus.ACTIVE + && nodeRecord.getLastRetry() > currentTime - STATUS_EXPIRATION_SECONDS) { + return true; + } + + return false; + }; + private boolean resetDead = false; /** @@ -84,19 +112,25 @@ public DiscoveryTaskManager( this.nodeTable = nodeTable; this.nodeBucketStorage = nodeBucketStorage; this.homeNodeId = homeNode.getNodeId(); - this.nodeConnectTasks = - new NodeConnectTasks( + this.liveCheckTasks = + new LiveCheckTasks(discoveryManager, scheduler, Duration.ofSeconds(RETRY_TIMEOUT_SECONDS)); + this.recursiveLookupTasks = + new RecursiveLookupTasks( discoveryManager, scheduler, Duration.ofSeconds(RETRY_TIMEOUT_SECONDS)); this.resetDead = resetDead; } public void start() { scheduler.executeAtFixedRate( - Duration.ZERO, Duration.ofSeconds(TASK_INTERVAL_SECONDS), this::recurrentTask); + Duration.ZERO, Duration.ofSeconds(LIVE_CHECK_INTERVAL_SECONDS), this::liveCheckTask); + scheduler.executeAtFixedRate( + Duration.ZERO, + Duration.ofSeconds(RECURSIVE_LOOKUP_INTERVAL_SECONDS), + this::recursiveLookupTask); } - private void recurrentTask() { - List nodes = nodeTable.findClosestNodes(homeNodeId, DEFAULT_DISTANCE); + private void liveCheckTask() { + List nodes = nodeTable.findClosestNodes(homeNodeId, LIVE_CHECK_DISTANCE); Stream closestNodes = nodes.stream(); if (resetDead) { closestNodes = @@ -110,10 +144,10 @@ private void recurrentTask() { resetDead = false; } closestNodes - .filter(READY_FOR_PING) + .filter(LIVE_CHECK_NODE_RULE) .forEach( nodeRecord -> - nodeConnectTasks.add( + liveCheckTasks.add( nodeRecord, () -> updateNode( @@ -131,6 +165,24 @@ private void recurrentTask() { (nodeRecord.getRetry() + 1))))); } + private void recursiveLookupTask() { + List nodes = nodeTable.findClosestNodes(homeNodeId, RECURSIVE_LOOKUP_DISTANCE); + nodes.stream() + .filter(RECURSIVE_LOOKUP_NODE_RULE) + .forEach( + nodeRecord -> + liveCheckTasks.add( + nodeRecord, + () -> {}, + () -> + updateNode( + new NodeRecordInfo( + nodeRecord.getNode(), + System.currentTimeMillis() / MS_IN_SECOND, + NodeStatus.SLEEP, + (nodeRecord.getRetry() + 1))))); + } + private void updateNode(NodeRecordInfo nodeRecordInfo) { nodeTable.save(nodeRecordInfo); nodeBucketStorage.put(nodeRecordInfo); diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeConnectTasks.java b/discovery/src/main/java/org/ethereum/beacon/discovery/task/LiveCheckTasks.java similarity index 77% rename from discovery/src/main/java/org/ethereum/beacon/discovery/NodeConnectTasks.java rename to discovery/src/main/java/org/ethereum/beacon/discovery/task/LiveCheckTasks.java index 84f292e10..f73a49aec 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeConnectTasks.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/task/LiveCheckTasks.java @@ -1,7 +1,8 @@ -package org.ethereum.beacon.discovery; +package org.ethereum.beacon.discovery.task; import com.google.common.collect.Sets; -import org.ethereum.beacon.discovery.enr.NodeRecord; +import org.ethereum.beacon.discovery.DiscoveryManager; +import org.ethereum.beacon.discovery.NodeRecordInfo; import org.ethereum.beacon.schedulers.Scheduler; import org.ethereum.beacon.util.ExpirationScheduler; import tech.pegasys.artemis.util.bytes.Bytes32; @@ -13,17 +14,17 @@ import java.util.concurrent.TimeUnit; /** - * Executes tasks {@link DiscoveryManager#connect(NodeRecord)} for all NodeRecords added via {@link - * #add(NodeRecordInfo, Runnable, Runnable)}. Tasks is called failed if timeout is reached and reply - * from node is not received. + * Sends {@link TaskType#PING} to closest NodeRecords added via {@link #add(NodeRecordInfo, + * Runnable, Runnable)}. Tasks is called failed if timeout is reached and reply from node is not + * received. */ -public class NodeConnectTasks { +public class LiveCheckTasks { private final Scheduler scheduler; private final DiscoveryManager discoveryManager; private final Set currentTasks = Sets.newConcurrentHashSet(); private final ExpirationScheduler taskTimeouts; - public NodeConnectTasks( + public LiveCheckTasks( DiscoveryManager discoveryManager, Scheduler scheduler, Duration timeout) { this.discoveryManager = discoveryManager; this.scheduler = scheduler; @@ -41,7 +42,8 @@ public void add(NodeRecordInfo nodeRecordInfo, Runnable successCallback, Runnabl scheduler.execute( () -> { - CompletableFuture retry = discoveryManager.connect(nodeRecordInfo.getNode()); + CompletableFuture retry = + discoveryManager.executeTask(nodeRecordInfo.getNode(), TaskType.PING); taskTimeouts.put( nodeRecordInfo.getNode().getNodeId(), () -> diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/task/RecursiveLookupTasks.java b/discovery/src/main/java/org/ethereum/beacon/discovery/task/RecursiveLookupTasks.java new file mode 100644 index 000000000..5da805449 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/task/RecursiveLookupTasks.java @@ -0,0 +1,63 @@ +package org.ethereum.beacon.discovery.task; + +import com.google.common.collect.Sets; +import org.ethereum.beacon.discovery.DiscoveryManager; +import org.ethereum.beacon.discovery.NodeRecordInfo; +import org.ethereum.beacon.schedulers.Scheduler; +import org.ethereum.beacon.util.ExpirationScheduler; +import tech.pegasys.artemis.util.bytes.Bytes32; + +import java.time.Duration; +import java.time.temporal.ChronoUnit; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; + +/** + * Sends {@link TaskType#FINDNODE} to closest NodeRecords added via {@link #add(NodeRecordInfo, + * Runnable, Runnable)}. Tasks is called failed if timeout is reached and reply from node is not + * received. + */ +public class RecursiveLookupTasks { + private final Scheduler scheduler; + private final DiscoveryManager discoveryManager; + private final Set currentTasks = Sets.newConcurrentHashSet(); + private final ExpirationScheduler taskTimeouts; + + public RecursiveLookupTasks( + DiscoveryManager discoveryManager, Scheduler scheduler, Duration timeout) { + this.discoveryManager = discoveryManager; + this.scheduler = scheduler; + this.taskTimeouts = + new ExpirationScheduler<>(timeout.get(ChronoUnit.MILLIS), TimeUnit.MILLISECONDS); + } + + public void add(NodeRecordInfo nodeRecordInfo, Runnable successCallback, Runnable failCallback) { + synchronized (this) { + if (currentTasks.contains(nodeRecordInfo.getNode().getNodeId())) { + return; + } + currentTasks.add(nodeRecordInfo.getNode().getNodeId()); + } + + scheduler.execute( + () -> { + CompletableFuture retry = + discoveryManager.executeTask(nodeRecordInfo.getNode(), TaskType.FINDNODE); + taskTimeouts.put( + nodeRecordInfo.getNode().getNodeId(), + () -> + retry.completeExceptionally(new RuntimeException("Timeout for node recursive lookup task"))); + retry.whenComplete( + (aVoid, throwable) -> { + if (throwable != null) { + failCallback.run(); + currentTasks.remove(nodeRecordInfo.getNode().getNodeId()); + } else { + successCallback.run(); + currentTasks.remove(nodeRecordInfo.getNode().getNodeId()); + } + }); + }); + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/task/TaskMessageFactory.java b/discovery/src/main/java/org/ethereum/beacon/discovery/task/TaskMessageFactory.java new file mode 100644 index 000000000..50b7839fc --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/task/TaskMessageFactory.java @@ -0,0 +1,41 @@ +package org.ethereum.beacon.discovery.task; + +import org.ethereum.beacon.discovery.NodeContext; +import org.ethereum.beacon.discovery.message.DiscoveryV5Message; +import org.ethereum.beacon.discovery.message.FindNodeMessage; +import org.ethereum.beacon.discovery.message.MessageCode; +import org.ethereum.beacon.discovery.message.PingMessage; +import org.ethereum.beacon.discovery.packet.MessagePacket; + +public class TaskMessageFactory { + public static final int DEFAULT_DISTANCE = 10; + + public static MessagePacket createPingPacket(NodeContext context) { + + return MessagePacket.create( + context.getHomeNodeId(), + context.getNodeRecord().getNodeId(), + context.getAuthTag().get(), + context.getInitiatorKey(), + DiscoveryV5Message.from(createPing(context))); + } + + public static PingMessage createPing(NodeContext context) { + return new PingMessage( + context.getNextRequestId(MessageCode.PING), context.getNodeRecord().getSeq()); + } + + public static MessagePacket createFindNodePacket(NodeContext context) { + FindNodeMessage findNodeMessage = createFindNode(context); + return MessagePacket.create( + context.getHomeNodeId(), + context.getNodeRecord().getNodeId(), + context.getAuthTag().get(), + context.getInitiatorKey(), + DiscoveryV5Message.from(findNodeMessage)); + } + + public static FindNodeMessage createFindNode(NodeContext context) { + return new FindNodeMessage(context.getNextRequestId(MessageCode.FINDNODE), DEFAULT_DISTANCE); + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/task/TaskType.java b/discovery/src/main/java/org/ethereum/beacon/discovery/task/TaskType.java new file mode 100644 index 000000000..efdad457d --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/task/TaskType.java @@ -0,0 +1,6 @@ +package org.ethereum.beacon.discovery.task; + +public enum TaskType { + PING, + FINDNODE; +} diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java index 9abfb1878..4f087b7b4 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java @@ -13,6 +13,7 @@ import org.ethereum.beacon.discovery.packet.RandomPacket; import org.ethereum.beacon.discovery.packet.UnknownPacket; import org.ethereum.beacon.discovery.packet.WhoAreYouPacket; +import org.ethereum.beacon.discovery.task.TaskType; import org.ethereum.beacon.discovery.storage.NodeBucketStorage; import org.ethereum.beacon.discovery.storage.NodeTableStorage; import org.ethereum.beacon.discovery.storage.NodeTableStorageFactoryImpl; @@ -30,7 +31,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import static org.ethereum.beacon.discovery.NodeContext.DEFAULT_DISTANCE; +import static org.ethereum.beacon.discovery.task.TaskMessageFactory.DEFAULT_DISTANCE; import static org.ethereum.beacon.discovery.storage.NodeTableStorage.DEFAULT_SERIALIZER; /** Same as {@link DiscoveryNoNetworkTest} but using real network */ @@ -185,7 +186,7 @@ public void test() throws Exception { }); // 4) fire 1 to 2 dialog - discoveryManager1.connect(nodeRecord2); + discoveryManager1.executeTask(nodeRecord2, TaskType.FINDNODE); assert randomSent1to2.await(1, TimeUnit.SECONDS); assert whoareyouSent2to1.await(1, TimeUnit.SECONDS); diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java index 930e18a58..92b140c98 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java @@ -17,6 +17,7 @@ import org.ethereum.beacon.discovery.storage.NodeBucketStorage; import org.ethereum.beacon.discovery.storage.NodeTableStorage; import org.ethereum.beacon.discovery.storage.NodeTableStorageFactoryImpl; +import org.ethereum.beacon.discovery.task.TaskType; import org.ethereum.beacon.schedulers.Schedulers; import org.ethereum.beacon.stream.SimpleProcessor; import org.javatuples.Pair; @@ -32,8 +33,8 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import static org.ethereum.beacon.discovery.NodeContext.DEFAULT_DISTANCE; import static org.ethereum.beacon.discovery.storage.NodeTableStorage.DEFAULT_SERIALIZER; +import static org.ethereum.beacon.discovery.task.TaskMessageFactory.DEFAULT_DISTANCE; /** * Discovery test without real network, instead outgoing stream of each peer is connected with @@ -114,10 +115,10 @@ public void test() throws Exception { NodeBucketStorage nodeBucketStorage2 = nodeTableStorageFactory.createBuckets( database2, DEFAULT_SERIALIZER, nodeRecord2.getNodeId()); - SimpleProcessor from1to2 = + SimpleProcessor from1to2 = new SimpleProcessor<>( Schedulers.createDefault().newSingleThreadDaemon("from1to2-thread"), "from1to2"); - SimpleProcessor from2to1 = + SimpleProcessor from2to1 = new SimpleProcessor<>( Schedulers.createDefault().newSingleThreadDaemon("from2to1-thread"), "from2to1"); DiscoveryManagerNoNetwork discoveryManager1 = @@ -131,9 +132,9 @@ public void test() throws Exception { discoveryManager2.start(); // 2) Link outgoing of each one with incoming of another Flux.from(discoveryManager1.getOutgoingMessages()) - .subscribe(t -> from1to2.onNext(new UnknownPacket(t.getPacket().getBytes()))); + .subscribe(t -> from1to2.onNext(t.getPacket().getBytes())); Flux.from(discoveryManager2.getOutgoingMessages()) - .subscribe(t -> from2to1.onNext(new UnknownPacket(t.getPacket().getBytes()))); + .subscribe(t -> from2to1.onNext(t.getPacket().getBytes())); // 3) Expect standard 1 => 2 dialog CountDownLatch randomSent1to2 = new CountDownLatch(1); @@ -142,6 +143,7 @@ public void test() throws Exception { CountDownLatch findNodeSent2to1 = new CountDownLatch(1); CountDownLatch nodesSent1to2 = new CountDownLatch(1); Flux.from(from1to2) + .map(UnknownPacket::new) .subscribe( networkPacket -> { // 1 -> 2 random @@ -170,6 +172,7 @@ public void test() throws Exception { } }); Flux.from(from2to1) + .map(UnknownPacket::new) .subscribe( networkPacket -> { // 2 -> 1 whoareyou @@ -191,7 +194,7 @@ public void test() throws Exception { }); // 4) fire 1 to 2 dialog - discoveryManager1.connect(nodeRecord2); + discoveryManager1.executeTask(nodeRecord2, TaskType.FINDNODE); assert randomSent1to2.await(1, TimeUnit.SECONDS); assert whoareyouSent2to1.await(1, TimeUnit.SECONDS); diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java b/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java index a35823cf8..8dd945fa8 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java @@ -3,26 +3,39 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.ethereum.beacon.discovery.DiscoveryManager; -import org.ethereum.beacon.discovery.NodeContext; +import org.ethereum.beacon.discovery.DiscoveryManagerImpl; import org.ethereum.beacon.discovery.enr.NodeRecord; -import org.ethereum.beacon.discovery.NodeRecordInfo; import org.ethereum.beacon.discovery.network.NetworkParcel; -import org.ethereum.beacon.discovery.network.NetworkParcelV5; -import org.ethereum.beacon.discovery.packet.UnknownPacket; +import org.ethereum.beacon.discovery.pipeline.Envelope; +import org.ethereum.beacon.discovery.pipeline.Field; +import org.ethereum.beacon.discovery.pipeline.Pipeline; +import org.ethereum.beacon.discovery.pipeline.PipelineImpl; +import org.ethereum.beacon.discovery.pipeline.handler.AuthHeaderMessagePacketHandler; +import org.ethereum.beacon.discovery.pipeline.handler.IncomingDataPacker; +import org.ethereum.beacon.discovery.pipeline.handler.MessageHandler; +import org.ethereum.beacon.discovery.pipeline.handler.MessagePacketHandler; +import org.ethereum.beacon.discovery.pipeline.handler.NodeContextRequestHandler; +import org.ethereum.beacon.discovery.pipeline.handler.NodeIdToContext; +import org.ethereum.beacon.discovery.pipeline.handler.NotExpectedIncomingPacketHandler; +import org.ethereum.beacon.discovery.pipeline.handler.OutgoingParcelHandler; +import org.ethereum.beacon.discovery.pipeline.handler.TaskHandler; +import org.ethereum.beacon.discovery.pipeline.handler.UnknownPacketTagToSender; +import org.ethereum.beacon.discovery.pipeline.handler.UnknownPacketTypeByStatus; +import org.ethereum.beacon.discovery.pipeline.handler.WhoAreYouAttempt; +import org.ethereum.beacon.discovery.pipeline.handler.WhoAreYouContextResolver; +import org.ethereum.beacon.discovery.pipeline.handler.WhoAreYouPacketHandler; import org.ethereum.beacon.discovery.storage.AuthTagRepository; import org.ethereum.beacon.discovery.storage.NodeBucketStorage; import org.ethereum.beacon.discovery.storage.NodeTable; +import org.ethereum.beacon.discovery.task.TaskType; import org.reactivestreams.Publisher; import reactor.core.publisher.Flux; import reactor.core.publisher.FluxSink; import reactor.core.publisher.ReplayProcessor; -import tech.pegasys.artemis.util.bytes.Bytes32; +import tech.pegasys.artemis.util.bytes.BytesValue; import java.security.SecureRandom; -import java.util.Map; -import java.util.Optional; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; /** * Implementation of {@link DiscoveryManager} without network as an opposite to Netty network @@ -31,84 +44,59 @@ * packets could be provided through the constructor parameter `incomingPackets` */ public class DiscoveryManagerNoNetwork implements DiscoveryManager { - private static final Logger logger = LogManager.getLogger(DiscoveryManager.class); + private static final Logger logger = LogManager.getLogger(DiscoveryManagerImpl.class); private final ReplayProcessor outgoingMessages = ReplayProcessor.cacheLast(); private final FluxSink outgoingSink = outgoingMessages.sink(); - private final Bytes32 homeNodeId; - private final NodeRecord homeNodeRecord; - private final NodeTable nodeTable; - private final NodeBucketStorage nodeBucketStorage; - private Publisher incomingPackets; - private Map recentContexts = new ConcurrentHashMap<>(); // nodeId -> context - private AuthTagRepository authTagRepo; + private final Publisher incomingPackets; + private final Pipeline incomingPipeline = new PipelineImpl(); + private final Pipeline outgoingPipeline = new PipelineImpl(); public DiscoveryManagerNoNetwork( NodeTable nodeTable, NodeBucketStorage nodeBucketStorage, NodeRecord homeNode, - Publisher incomingPackets) { - this.nodeTable = nodeTable; - this.nodeBucketStorage = nodeBucketStorage; + Publisher incomingPackets) { + AuthTagRepository authTagRepo = new AuthTagRepository(); this.incomingPackets = incomingPackets; - this.homeNodeId = homeNode.getNodeId(); - this.homeNodeRecord = (NodeRecord) nodeTable.getHomeNode(); - this.authTagRepo = new AuthTagRepository(); + NodeIdToContext nodeIdToContext = + new NodeIdToContext(homeNode, nodeBucketStorage, authTagRepo, nodeTable, outgoingPipeline); + incomingPipeline + .addHandler(new IncomingDataPacker()) + .addHandler(new WhoAreYouAttempt(homeNode.getNodeId())) + .addHandler(new WhoAreYouContextResolver(authTagRepo)) + .addHandler(new UnknownPacketTagToSender(homeNode)) + .addHandler(nodeIdToContext) + .addHandler(new UnknownPacketTypeByStatus()) + .addHandler(new NotExpectedIncomingPacketHandler()) + .addHandler(new WhoAreYouPacketHandler()) + .addHandler(new AuthHeaderMessagePacketHandler()) + .addHandler(new MessagePacketHandler()) + .addHandler(new MessageHandler()); + outgoingPipeline + .addHandler(new OutgoingParcelHandler(outgoingSink)) + .addHandler(new NodeContextRequestHandler()) + .addHandler(nodeIdToContext) + .addHandler(new TaskHandler(new SecureRandom())); } + @Override public void start() { - Flux.from(incomingPackets) - .subscribe( - unknownPacket -> { - if (unknownPacket.isWhoAreYouPacket(homeNodeId)) { - Optional nodeContextOptional = - authTagRepo.get(unknownPacket.getWhoAreYouPacket().getAuthTag()); - if (nodeContextOptional.isPresent()) { - nodeContextOptional.get().addIncomingEvent(unknownPacket); - } else { - // TODO: ban or whatever - } - } else { - Bytes32 fromNodeId = unknownPacket.getSourceNodeId(homeNodeId); - getContext(fromNodeId) - .ifPresent(context -> context.addIncomingEvent(unknownPacket)); - } - }); + incomingPipeline.build(); + outgoingPipeline.build(); + Flux.from(incomingPackets).subscribe(incomingPipeline::push); } @Override public void stop() {} - public CompletableFuture connect(NodeRecord nodeRecord) { - if (!nodeTable.getNode(nodeRecord.getNodeId()).isPresent()) { - nodeTable.save(NodeRecordInfo.createDefault(nodeRecord)); - } - NodeContext context = getContext(nodeRecord.getNodeId()).get(); - return context.initiate(); - } - - private Optional getContext(Bytes32 nodeId) { - NodeContext context = recentContexts.get(nodeId); - if (context == null) { - Optional nodeOptional = nodeTable.getNode(nodeId); - if (!nodeOptional.isPresent()) { - logger.trace( - () -> String.format("Couldn't find node record for nodeId %s, ignoring", nodeId)); - return Optional.empty(); - } - NodeRecord nodeRecord = nodeOptional.get().getNode(); - SecureRandom random = new SecureRandom(); - context = - new NodeContext( - nodeRecord, - homeNodeRecord, - nodeTable, - nodeBucketStorage, - authTagRepo, - packet -> outgoingSink.next(new NetworkParcelV5(packet, nodeRecord)), - random); - } - - return Optional.of(context); + public CompletableFuture executeTask(NodeRecord nodeRecord, TaskType taskType) { + Envelope envelope = new Envelope(); + envelope.put(Field.TASK, taskType); + envelope.put(Field.NODE, nodeRecord); + CompletableFuture completed = new CompletableFuture<>(); + envelope.put(Field.FUTURE, completed); + outgoingPipeline.push(envelope); + return completed; } public Publisher getOutgoingMessages() { From 382db53a3997fb3bdff0dae11ccaaba809802929 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Sun, 6 Oct 2019 16:01:46 +0300 Subject: [PATCH 30/77] discovery: fix node availability statuses --- .../ethereum/beacon/discovery/task/DiscoveryTaskManager.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/task/DiscoveryTaskManager.java b/discovery/src/main/java/org/ethereum/beacon/discovery/task/DiscoveryTaskManager.java index 66e065476..c5bcb337f 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/task/DiscoveryTaskManager.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/task/DiscoveryTaskManager.java @@ -57,7 +57,10 @@ public class DiscoveryTaskManager { if (nodeRecord.getRetry() >= MAX_RETRIES) { updateNode( new NodeRecordInfo( - nodeRecord.getNode(), nodeRecord.getLastRetry(), NodeStatus.DEAD, 0)); + nodeRecord.getNode(), + System.currentTimeMillis() / MS_IN_SECOND, + NodeStatus.SLEEP, + 1)); return false; } if ((currentTime - nodeRecord.getLastRetry()) From 34df11a2f273d1a08fd406fe26cb20fe2ecd80b6 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Sun, 6 Oct 2019 16:06:33 +0300 Subject: [PATCH 31/77] discovery: remove obsolete condition --- .../discovery/pipeline/handler/TaskHandler.java | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/TaskHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/TaskHandler.java index 905fda4b2..c609c5d49 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/TaskHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/TaskHandler.java @@ -26,9 +26,6 @@ public void handle(Envelope envelope) { if (!envelope.contains(Field.TASK)) { return; } - if (!envelope.get(Field.TASK).equals(TaskType.PING)) { - return; // TODO: implement other tasks - } NodeContext context = (NodeContext) envelope.get(Field.CONTEXT); CompletableFuture completableFuture = (CompletableFuture) envelope.get(Field.FUTURE); @@ -37,12 +34,16 @@ public void handle(Envelope envelope) { rnd.nextBytes(authTagBytes); BytesValue authTag = BytesValue.wrap(authTagBytes); RandomPacket randomPacket = - RandomPacket.create(context.getHomeNodeId(), context.getNodeRecord().getNodeId(), authTag, new SecureRandom()); + RandomPacket.create( + context.getHomeNodeId(), + context.getNodeRecord().getNodeId(), + authTag, + new SecureRandom()); context.setAuthTag(authTag); context.sendOutgoing(randomPacket); context.setStatus(NodeContext.SessionStatus.RANDOM_PACKET_SENT); context.saveFuture(completableFuture); - } else if (context.getStatus().equals(NodeContext.SessionStatus.AUTHENTICATED)) { + } else if (context.getStatus().equals(NodeContext.SessionStatus.AUTHENTICATED)) { if (envelope.get(Field.TASK).equals(TaskType.PING)) { context.sendOutgoing(TaskMessageFactory.createPingPacket(context)); context.saveTask((TaskType) envelope.get(Field.TASK)); @@ -52,7 +53,10 @@ public void handle(Envelope envelope) { context.saveTask((TaskType) envelope.get(Field.TASK)); context.saveFuture(completableFuture); } else { - throw new RuntimeException(String.format("Task type %s handler not found in envelope %s", envelope.get(Field.TASK), envelope.getId())); + throw new RuntimeException( + String.format( + "Task type %s handler not found in envelope %s", + envelope.get(Field.TASK), envelope.getId())); } } else { completableFuture.completeExceptionally(new RuntimeException("Already initiating")); From cd32bc383454fe8b9362d2c26fdcee94fafa5e88 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Sun, 6 Oct 2019 16:16:42 +0300 Subject: [PATCH 32/77] discovery: add bad packet logging --- .../discovery/DiscoveryManagerImpl.java | 6 +++-- .../beacon/discovery/pipeline/Field.java | 1 + .../pipeline/handler/BadPacketLogger.java | 24 +++++++++++++++++++ .../NotExpectedIncomingPacketHandler.java | 1 + .../handler/UnknownPacketTagToSender.java | 4 ++++ .../handler/WhoAreYouContextResolver.java | 2 ++ .../mock/DiscoveryManagerNoNetwork.java | 4 +++- 7 files changed, 39 insertions(+), 3 deletions(-) create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/BadPacketLogger.java diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java index 2d5027558..ed8c7a853 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java @@ -14,6 +14,7 @@ import org.ethereum.beacon.discovery.pipeline.Pipeline; import org.ethereum.beacon.discovery.pipeline.PipelineImpl; import org.ethereum.beacon.discovery.pipeline.handler.AuthHeaderMessagePacketHandler; +import org.ethereum.beacon.discovery.pipeline.handler.BadPacketLogger; import org.ethereum.beacon.discovery.pipeline.handler.IncomingDataPacker; import org.ethereum.beacon.discovery.pipeline.handler.MessageHandler; import org.ethereum.beacon.discovery.pipeline.handler.MessagePacketHandler; @@ -27,10 +28,10 @@ import org.ethereum.beacon.discovery.pipeline.handler.WhoAreYouAttempt; import org.ethereum.beacon.discovery.pipeline.handler.WhoAreYouContextResolver; import org.ethereum.beacon.discovery.pipeline.handler.WhoAreYouPacketHandler; -import org.ethereum.beacon.discovery.task.TaskType; import org.ethereum.beacon.discovery.storage.AuthTagRepository; import org.ethereum.beacon.discovery.storage.NodeBucketStorage; import org.ethereum.beacon.discovery.storage.NodeTable; +import org.ethereum.beacon.discovery.task.TaskType; import org.ethereum.beacon.schedulers.Scheduler; import org.reactivestreams.Publisher; import reactor.core.publisher.Flux; @@ -77,7 +78,8 @@ public DiscoveryManagerImpl( .addHandler(new WhoAreYouPacketHandler()) .addHandler(new AuthHeaderMessagePacketHandler()) .addHandler(new MessagePacketHandler()) - .addHandler(new MessageHandler()); + .addHandler(new MessageHandler()) + .addHandler(new BadPacketLogger()); outgoingPipeline .addHandler(new OutgoingParcelHandler(outgoingSink)) .addHandler(new NodeContextRequestHandler()) diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/Field.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/Field.java index 365a40ab1..46ceaeda2 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/Field.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/Field.java @@ -11,6 +11,7 @@ public enum Field { MESSAGE, // Message extracted from the packet NODE, // Sender/recipient node BAD_PACKET, // Bad, rejected packet + BAD_PACKET_EXCEPTION, // Stores exception for bad packet TASK, // Task to perform FUTURE, // Completable future diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/BadPacketLogger.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/BadPacketLogger.java new file mode 100644 index 000000000..7a3582c08 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/BadPacketLogger.java @@ -0,0 +1,24 @@ +package org.ethereum.beacon.discovery.pipeline.handler; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.ethereum.beacon.discovery.pipeline.Envelope; +import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; +import org.ethereum.beacon.discovery.pipeline.Field; + +/** Logs all packets which are stored in {@link Field#BAD_PACKET} */ +public class BadPacketLogger implements EnvelopeHandler { + private static final Logger logger = LogManager.getLogger(BadPacketLogger.class); + + @Override + public void handle(Envelope envelope) { + if (!envelope.contains(Field.BAD_PACKET)) { + return; + } + logger.debug( + () -> + String.format( + "Bad packet: %s in envelope #%s", envelope.get(Field.BAD_PACKET), envelope.getId()), + (Exception) envelope.get(Field.BAD_PACKET_EXCEPTION)); + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NotExpectedIncomingPacketHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NotExpectedIncomingPacketHandler.java index e1d221221..289df5200 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NotExpectedIncomingPacketHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NotExpectedIncomingPacketHandler.java @@ -66,6 +66,7 @@ public void handle(Envelope envelope) { unknownPacket, context.getNodeRecord(), context.getStatus()); logger.error(error, ex); envelope.put(Field.BAD_PACKET, envelope.get(Field.PACKET_UNKNOWN)); + envelope.put(Field.BAD_PACKET_EXCEPTION, ex); envelope.remove(Field.PACKET_UNKNOWN); return; } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/UnknownPacketTagToSender.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/UnknownPacketTagToSender.java index bc1385938..209bf0b3b 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/UnknownPacketTagToSender.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/UnknownPacketTagToSender.java @@ -34,6 +34,10 @@ public void handle(Envelope envelope) { (Runnable) () -> { envelope.put(Field.BAD_PACKET, envelope.get(Field.PACKET_UNKNOWN)); + envelope.put( + Field.BAD_PACKET_EXCEPTION, + new RuntimeException( + String.format("Context not found for nodeId %s", fromNodeId))); envelope.remove(Field.PACKET_UNKNOWN); })); } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouContextResolver.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouContextResolver.java index b91d4be30..61a7df5c8 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouContextResolver.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouContextResolver.java @@ -47,6 +47,8 @@ public void handle(Envelope envelope) { } else { envelope.put(Field.BAD_PACKET, envelope.get(Field.PACKET_WHOAREYOU)); envelope.remove(Field.PACKET_WHOAREYOU); + envelope.put( + Field.BAD_PACKET_EXCEPTION, new RuntimeException("Not expected WHOAREYOU packet")); } } } diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java b/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java index 8dd945fa8..eed4cacfc 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java @@ -11,6 +11,7 @@ import org.ethereum.beacon.discovery.pipeline.Pipeline; import org.ethereum.beacon.discovery.pipeline.PipelineImpl; import org.ethereum.beacon.discovery.pipeline.handler.AuthHeaderMessagePacketHandler; +import org.ethereum.beacon.discovery.pipeline.handler.BadPacketLogger; import org.ethereum.beacon.discovery.pipeline.handler.IncomingDataPacker; import org.ethereum.beacon.discovery.pipeline.handler.MessageHandler; import org.ethereum.beacon.discovery.pipeline.handler.MessagePacketHandler; @@ -71,7 +72,8 @@ public DiscoveryManagerNoNetwork( .addHandler(new WhoAreYouPacketHandler()) .addHandler(new AuthHeaderMessagePacketHandler()) .addHandler(new MessagePacketHandler()) - .addHandler(new MessageHandler()); + .addHandler(new MessageHandler()) + .addHandler(new BadPacketLogger()); outgoingPipeline .addHandler(new OutgoingParcelHandler(outgoingSink)) .addHandler(new NodeContextRequestHandler()) From 14ce2d4763779d4812ff9e73bc7b07183aee3641 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Sun, 6 Oct 2019 16:29:17 +0300 Subject: [PATCH 33/77] discovery: context renamed to session to be more intuitive --- .../discovery/DiscoveryManagerImpl.java | 18 +++---- .../discovery/DiscoveryMessageProcessor.java | 2 +- .../DiscoveryV5MessageProcessor.java | 4 +- .../beacon/discovery/MessageProcessor.java | 8 +-- .../{NodeContext.java => NodeSession.java} | 8 +-- .../message/handler/FindNodeHandler.java | 16 +++--- .../message/handler/MessageHandler.java | 4 +- .../message/handler/NodesHandler.java | 24 ++++----- .../message/handler/PingHandler.java | 20 +++---- .../message/handler/PongHandler.java | 9 ++-- .../network/IncomingMessageSink.java | 2 +- .../discovery/network/NetworkParcel.java | 2 +- .../beacon/discovery/pipeline/Field.java | 4 +- .../AuthHeaderMessagePacketHandler.java | 38 +++++++------- .../pipeline/handler/MessageHandler.java | 8 +-- .../handler/MessagePacketHandler.java | 22 ++++---- ...eIdToContext.java => NodeIdToSession.java} | 52 +++++++++---------- ...er.java => NodeSessionRequestHandler.java} | 8 +-- .../NotExpectedIncomingPacketHandler.java | 22 ++++---- .../pipeline/handler/TaskHandler.java | 32 ++++++------ .../handler/UnknownPacketTagToSender.java | 8 +-- .../handler/UnknownPacketTypeByStatus.java | 12 ++--- .../handler/WhoAreYouPacketHandler.java | 40 +++++++------- ...ver.java => WhoAreYouSessionResolver.java} | 24 ++++----- .../discovery/storage/AuthTagRepository.java | 44 ++++++++-------- .../discovery/task/TaskMessageFactory.java | 34 ++++++------ .../mock/DiscoveryManagerNoNetwork.java | 18 +++---- 27 files changed, 240 insertions(+), 243 deletions(-) rename discovery/src/main/java/org/ethereum/beacon/discovery/{NodeContext.java => NodeSession.java} (98%) rename discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/{NodeIdToContext.java => NodeIdToSession.java} (66%) rename discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/{NodeContextRequestHandler.java => NodeSessionRequestHandler.java} (78%) rename discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/{WhoAreYouContextResolver.java => WhoAreYouSessionResolver.java} (69%) diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java index ed8c7a853..cbe5ad6b3 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java @@ -18,15 +18,15 @@ import org.ethereum.beacon.discovery.pipeline.handler.IncomingDataPacker; import org.ethereum.beacon.discovery.pipeline.handler.MessageHandler; import org.ethereum.beacon.discovery.pipeline.handler.MessagePacketHandler; -import org.ethereum.beacon.discovery.pipeline.handler.NodeContextRequestHandler; -import org.ethereum.beacon.discovery.pipeline.handler.NodeIdToContext; +import org.ethereum.beacon.discovery.pipeline.handler.NodeSessionRequestHandler; +import org.ethereum.beacon.discovery.pipeline.handler.NodeIdToSession; import org.ethereum.beacon.discovery.pipeline.handler.NotExpectedIncomingPacketHandler; import org.ethereum.beacon.discovery.pipeline.handler.OutgoingParcelHandler; import org.ethereum.beacon.discovery.pipeline.handler.TaskHandler; import org.ethereum.beacon.discovery.pipeline.handler.UnknownPacketTagToSender; import org.ethereum.beacon.discovery.pipeline.handler.UnknownPacketTypeByStatus; import org.ethereum.beacon.discovery.pipeline.handler.WhoAreYouAttempt; -import org.ethereum.beacon.discovery.pipeline.handler.WhoAreYouContextResolver; +import org.ethereum.beacon.discovery.pipeline.handler.WhoAreYouSessionResolver; import org.ethereum.beacon.discovery.pipeline.handler.WhoAreYouPacketHandler; import org.ethereum.beacon.discovery.storage.AuthTagRepository; import org.ethereum.beacon.discovery.storage.NodeBucketStorage; @@ -65,14 +65,14 @@ public DiscoveryManagerImpl( ((Bytes4) homeNode.get(NodeRecord.FIELD_IP_V4)), (int) homeNode.get(NodeRecord.FIELD_UDP_V4)); this.discoveryClient = new DiscoveryClientImpl(outgoingMessages, clientScheduler); - NodeIdToContext nodeIdToContext = - new NodeIdToContext(homeNode, nodeBucketStorage, authTagRepo, nodeTable, outgoingPipeline); + NodeIdToSession nodeIdToSession = + new NodeIdToSession(homeNode, nodeBucketStorage, authTagRepo, nodeTable, outgoingPipeline); incomingPipeline .addHandler(new IncomingDataPacker()) .addHandler(new WhoAreYouAttempt(homeNode.getNodeId())) - .addHandler(new WhoAreYouContextResolver(authTagRepo)) + .addHandler(new WhoAreYouSessionResolver(authTagRepo)) .addHandler(new UnknownPacketTagToSender(homeNode)) - .addHandler(nodeIdToContext) + .addHandler(nodeIdToSession) .addHandler(new UnknownPacketTypeByStatus()) .addHandler(new NotExpectedIncomingPacketHandler()) .addHandler(new WhoAreYouPacketHandler()) @@ -82,8 +82,8 @@ public DiscoveryManagerImpl( .addHandler(new BadPacketLogger()); outgoingPipeline .addHandler(new OutgoingParcelHandler(outgoingSink)) - .addHandler(new NodeContextRequestHandler()) - .addHandler(nodeIdToContext) + .addHandler(new NodeSessionRequestHandler()) + .addHandler(nodeIdToSession) .addHandler(new TaskHandler(new SecureRandom())); } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryMessageProcessor.java b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryMessageProcessor.java index 9d26de022..0d51ec9fa 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryMessageProcessor.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryMessageProcessor.java @@ -5,5 +5,5 @@ public interface DiscoveryMessageProcessor { IdentityScheme getSupportedIdentity(); - void handleMessage(M message, NodeContext context); + void handleMessage(M message, NodeSession session); } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryV5MessageProcessor.java b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryV5MessageProcessor.java index 26c2dc282..947d1fd31 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryV5MessageProcessor.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryV5MessageProcessor.java @@ -27,12 +27,12 @@ public IdentityScheme getSupportedIdentity() { } @Override - public void handleMessage(DiscoveryV5Message message, NodeContext context) { + public void handleMessage(DiscoveryV5Message message, NodeSession session) { MessageCode code = message.getCode(); MessageHandler messageHandler = messageHandlers.get(code); if (messageHandler == null) { throw new RuntimeException("Not implemented yet"); } - messageHandler.handle(message.create(), context); + messageHandler.handle(message.create(), session); } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/MessageProcessor.java b/discovery/src/main/java/org/ethereum/beacon/discovery/MessageProcessor.java index 37efef666..945ab7caa 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/MessageProcessor.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/MessageProcessor.java @@ -17,17 +17,17 @@ public MessageProcessor(DiscoveryMessageProcessor... messageHandlers) { } } - public void handleIncoming(DiscoveryMessage message, NodeContext context) { + public void handleIncoming(DiscoveryMessage message, NodeSession session) { IdentityScheme identityScheme = message.getIdentityScheme(); DiscoveryMessageProcessor messageHandler = messageHandlers.get(identityScheme); if (messageHandler == null) { String error = String.format( - "Message %s with identity %s received in context %s is not supported", - message, identityScheme, context); + "Message %s with identity %s received in session %s is not supported", + message, identityScheme, session); logger.error(error); throw new RuntimeException(error); } - messageHandler.handleMessage(message, context); + messageHandler.handleMessage(message, session); } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeContext.java b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeSession.java similarity index 98% rename from discovery/src/main/java/org/ethereum/beacon/discovery/NodeContext.java rename to discovery/src/main/java/org/ethereum/beacon/discovery/NodeSession.java index f89963127..c744787b3 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeContext.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeSession.java @@ -21,8 +21,8 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.function.Consumer; -public class NodeContext { - private static final Logger logger = LogManager.getLogger(NodeContext.class); +public class NodeSession { + private static final Logger logger = LogManager.getLogger(NodeSession.class); private final NodeRecord nodeRecord; private final NodeRecord homeNodeRecord; private final Bytes32 homeNodeId; @@ -38,7 +38,7 @@ public class NodeContext { private CompletableFuture completableFuture = null; private TaskType task = null; - public NodeContext( + public NodeSession( NodeRecord nodeRecord, NodeRecord homeNodeRecord, NodeTable nodeTable, @@ -148,7 +148,7 @@ public NodeRecord getHomeNodeRecord() { @Override public String toString() { - return "NodeContext{" + return "NodeSession{" + "nodeRecord=" + nodeRecord + ", homeNodeId=" diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/FindNodeHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/FindNodeHandler.java index 7fb47ef84..fc1fc5618 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/FindNodeHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/FindNodeHandler.java @@ -1,6 +1,6 @@ package org.ethereum.beacon.discovery.message.handler; -import org.ethereum.beacon.discovery.NodeContext; +import org.ethereum.beacon.discovery.NodeSession; import org.ethereum.beacon.discovery.NodeRecordInfo; import org.ethereum.beacon.discovery.message.DiscoveryV5Message; import org.ethereum.beacon.discovery.message.FindNodeMessage; @@ -18,21 +18,21 @@ public class FindNodeHandler implements MessageHandler { public FindNodeHandler() {} @Override - public void handle(FindNodeMessage message, NodeContext context) { + public void handle(FindNodeMessage message, NodeSession session) { List nodeBuckets = IntStream.range(0, message.getDistance()) - .mapToObj(context::getBucket) + .mapToObj(session::getBucket) .filter(Optional::isPresent) .map(Optional::get) .collect(Collectors.toList()); nodeBuckets.forEach( bucket -> - context.sendOutgoing( + session.sendOutgoing( MessagePacket.create( - context.getHomeNodeId(), - context.getNodeRecord().getNodeId(), - context.getAuthTag().get(), - context.getInitiatorKey(), + session.getHomeNodeId(), + session.getNodeRecord().getNodeId(), + session.getAuthTag().get(), + session.getInitiatorKey(), DiscoveryV5Message.from( new NodesMessage( message.getRequestId(), diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/MessageHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/MessageHandler.java index 011acd4ad..dc25df62d 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/MessageHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/MessageHandler.java @@ -1,7 +1,7 @@ package org.ethereum.beacon.discovery.message.handler; -import org.ethereum.beacon.discovery.NodeContext; +import org.ethereum.beacon.discovery.NodeSession; public interface MessageHandler { - void handle(Message message, NodeContext context); + void handle(Message message, NodeSession session); } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/NodesHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/NodesHandler.java index b575aa11d..fbdc32e60 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/NodesHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/NodesHandler.java @@ -1,6 +1,6 @@ package org.ethereum.beacon.discovery.message.handler; -import org.ethereum.beacon.discovery.NodeContext; +import org.ethereum.beacon.discovery.NodeSession; import org.ethereum.beacon.discovery.NodeRecordInfo; import org.ethereum.beacon.discovery.message.MessageCode; import org.ethereum.beacon.discovery.message.NodesMessage; @@ -18,23 +18,23 @@ public class NodesHandler implements MessageHandler { private Map nodesCounters = new ConcurrentHashMap<>(); @Override - public void handle(NodesMessage message, NodeContext context) { + public void handle(NodesMessage message, NodeSession session) { // NODES total count handling + cleanup schedule if (nodesCounters.containsKey(message.getRequestId())) { synchronized (this) { int counter = nodesCounters.get(message.getRequestId()) - 1; if (counter == 0) { - cleanUp(message.getRequestId(), context); + cleanUp(message.getRequestId(), session); } else { nodesCounters.put(message.getRequestId(), counter); - updateExpiration(message.getRequestId(), context); + updateExpiration(message.getRequestId(), session); } } } else if (message.getTotal() > 1) { nodesCounters.put(message.getRequestId(), message.getTotal() - 1); - updateExpiration(message.getRequestId(), context); + updateExpiration(message.getRequestId(), session); } else { - context.clearRequestId(message.getRequestId(), MessageCode.FINDNODE); + session.clearRequestId(message.getRequestId(), MessageCode.FINDNODE); } // Parse node records @@ -42,24 +42,24 @@ public void handle(NodesMessage message, NodeContext context) { .getNodeRecords() .forEach( nodeRecordV5 -> { - if (!context.getNodeTable().getNode(nodeRecordV5.getNodeId()).isPresent()) { + if (!session.getNodeTable().getNode(nodeRecordV5.getNodeId()).isPresent()) { // TODO: should we update-merge? - context.getNodeTable().save(NodeRecordInfo.createDefault(nodeRecordV5)); + session.getNodeTable().save(NodeRecordInfo.createDefault(nodeRecordV5)); } }); } - private synchronized void updateExpiration(BytesValue requestId, NodeContext context) { + private synchronized void updateExpiration(BytesValue requestId, NodeSession session) { nodeExpirationScheduler.put( requestId, () -> { - cleanUp(requestId, context); + cleanUp(requestId, session); }); } - private synchronized void cleanUp(BytesValue requestId, NodeContext context) { + private synchronized void cleanUp(BytesValue requestId, NodeSession session) { nodesCounters.remove(requestId); - context.clearRequestId(requestId, MessageCode.FINDNODE); + session.clearRequestId(requestId, MessageCode.FINDNODE); } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/PingHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/PingHandler.java index 9c73ac00f..b2da520fa 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/PingHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/PingHandler.java @@ -1,6 +1,6 @@ package org.ethereum.beacon.discovery.message.handler; -import org.ethereum.beacon.discovery.NodeContext; +import org.ethereum.beacon.discovery.NodeSession; import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.message.DiscoveryV5Message; import org.ethereum.beacon.discovery.message.PingMessage; @@ -10,19 +10,19 @@ public class PingHandler implements MessageHandler { @Override - public void handle(PingMessage message, NodeContext context) { + public void handle(PingMessage message, NodeSession session) { PongMessage responseMessage = new PongMessage( message.getRequestId(), - context.getNodeRecord().getSeq(), - ((Bytes4) context.getNodeRecord().get(NodeRecord.FIELD_IP_V4)), - (int) context.getNodeRecord().get(NodeRecord.FIELD_UDP_V4)); - context.sendOutgoing( + session.getNodeRecord().getSeq(), + ((Bytes4) session.getNodeRecord().get(NodeRecord.FIELD_IP_V4)), + (int) session.getNodeRecord().get(NodeRecord.FIELD_UDP_V4)); + session.sendOutgoing( MessagePacket.create( - context.getHomeNodeId(), - context.getNodeRecord().getNodeId(), - context.getAuthTag().get(), - context.getInitiatorKey(), + session.getHomeNodeId(), + session.getNodeRecord().getNodeId(), + session.getAuthTag().get(), + session.getInitiatorKey(), DiscoveryV5Message.from(responseMessage))); } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/PongHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/PongHandler.java index c568f90aa..f30cbeb39 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/PongHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/PongHandler.java @@ -1,15 +1,12 @@ package org.ethereum.beacon.discovery.message.handler; -import org.ethereum.beacon.discovery.NodeContext; -import org.ethereum.beacon.discovery.message.DiscoveryV5Message; +import org.ethereum.beacon.discovery.NodeSession; import org.ethereum.beacon.discovery.message.MessageCode; -import org.ethereum.beacon.discovery.message.PingMessage; import org.ethereum.beacon.discovery.message.PongMessage; -import org.ethereum.beacon.discovery.packet.MessagePacket; public class PongHandler implements MessageHandler { @Override - public void handle(PongMessage message, NodeContext context) { - context.clearRequestId(message.getRequestId(), MessageCode.PING); + public void handle(PongMessage message, NodeSession session) { + session.clearRequestId(message.getRequestId(), MessageCode.PING); } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/network/IncomingMessageSink.java b/discovery/src/main/java/org/ethereum/beacon/discovery/network/IncomingMessageSink.java index 35775b6ea..f6c1226d0 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/network/IncomingMessageSink.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/network/IncomingMessageSink.java @@ -22,7 +22,7 @@ public IncomingMessageSink(FluxSink bytesValueSink) { @Override protected void channelRead0(ChannelHandlerContext ctx, BytesValue msg) throws Exception { - logger.trace(() -> String.format("Incoming packet %s in context %s", msg, ctx)); + logger.trace(() -> String.format("Incoming packet %s in session %s", msg, ctx)); bytesValueSink.next(msg); } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/network/NetworkParcel.java b/discovery/src/main/java/org/ethereum/beacon/discovery/network/NetworkParcel.java index 339958dfc..97b19cec0 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/network/NetworkParcel.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/network/NetworkParcel.java @@ -7,7 +7,7 @@ * Abstraction on the top of the {@link Packet}. * *

Stores `packet` and associated node record. Record could be a sender or recipient, depends on - * context. + * session. */ public interface NetworkParcel { Packet getPacket(); diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/Field.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/Field.java index 46ceaeda2..d5ae55da5 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/Field.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/Field.java @@ -1,8 +1,8 @@ package org.ethereum.beacon.discovery.pipeline; public enum Field { - NEED_CONTEXT, // Node id, requests context resolving - CONTEXT, // Node context + SESSION_LOOKUP, // Node id, requests session lookup + SESSION, // Node session INCOMING, // Raw incoming data PACKET_UNKNOWN, // Unknown packet PACKET_WHOAREYOU, // WhoAreYou packet diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/AuthHeaderMessagePacketHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/AuthHeaderMessagePacketHandler.java index e22c09c97..bca330a72 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/AuthHeaderMessagePacketHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/AuthHeaderMessagePacketHandler.java @@ -3,7 +3,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.ethereum.beacon.discovery.Functions; -import org.ethereum.beacon.discovery.NodeContext; +import org.ethereum.beacon.discovery.NodeSession; import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.message.DiscoveryV5Message; import org.ethereum.beacon.discovery.message.MessageCode; @@ -18,7 +18,7 @@ import java.util.concurrent.CompletableFuture; -import static org.ethereum.beacon.discovery.NodeContext.SessionStatus.AUTHENTICATED; +import static org.ethereum.beacon.discovery.NodeSession.SessionStatus.AUTHENTICATED; /** Handles {@link AuthHeaderMessagePacket} in {@link Field#PACKET_AUTH_HEADER_MESSAGE} field */ public class AuthHeaderMessagePacketHandler implements EnvelopeHandler { @@ -29,12 +29,12 @@ public void handle(Envelope envelope) { if (!envelope.contains(Field.PACKET_AUTH_HEADER_MESSAGE)) { return; } - if (!envelope.contains(Field.CONTEXT)) { + if (!envelope.contains(Field.SESSION)) { return; } AuthHeaderMessagePacket packet = (AuthHeaderMessagePacket) envelope.get(Field.PACKET_AUTH_HEADER_MESSAGE); - NodeContext context = (NodeContext) envelope.get(Field.CONTEXT); + NodeSession session = (NodeSession) envelope.get(Field.SESSION); try { byte[] ephemeralKeyBytes = new byte[32]; @@ -42,39 +42,39 @@ public void handle(Envelope envelope) { ECKeyPair ephemeralKey = ECKeyPair.create(ephemeralKeyBytes); Triplet hkdf = Functions.hkdf_expand( - context.getHomeNodeId(), - context.getNodeRecord().getNodeId(), + session.getHomeNodeId(), + session.getNodeRecord().getNodeId(), BytesValue.wrap(ephemeralKey.getPrivateKey().toByteArray()), - context.getIdNonce(), - (BytesValue) context.getNodeRecord().get(NodeRecord.FIELD_PKEY_SECP256K1)); + session.getIdNonce(), + (BytesValue) session.getNodeRecord().get(NodeRecord.FIELD_PKEY_SECP256K1)); BytesValue initiatorKey = hkdf.getValue0(); - context.setInitiatorKey(initiatorKey); + session.setInitiatorKey(initiatorKey); BytesValue authResponseKey = hkdf.getValue2(); packet.decode(initiatorKey, authResponseKey); - packet.verify(context.getAuthTag().get(), context.getIdNonce()); + packet.verify(session.getAuthTag().get(), session.getIdNonce()); envelope.put(Field.MESSAGE, packet.getMessage()); } catch (AssertionError ex) { logger.info( String.format( "Verification not passed for message [%s] from node %s in status %s", - packet, context.getNodeRecord(), context.getStatus())); + packet, session.getNodeRecord(), session.getStatus())); } catch (Exception ex) { String error = String.format( "Failed to read message [%s] from node %s in status %s", - packet, context.getNodeRecord(), context.getStatus()); + packet, session.getNodeRecord(), session.getStatus()); logger.error(error, ex); envelope.remove(Field.PACKET_AUTH_HEADER_MESSAGE); - if (context.loadFuture() != null) { - CompletableFuture future = context.loadFuture(); - context.saveFuture(null); + if (session.loadFuture() != null) { + CompletableFuture future = session.loadFuture(); + session.saveFuture(null); future.completeExceptionally(ex); } return; } - context.setStatus(AUTHENTICATED); + session.setStatus(AUTHENTICATED); envelope.remove(Field.PACKET_AUTH_HEADER_MESSAGE); - if (context.loadFuture() != null) { + if (session.loadFuture() != null) { boolean taskCompleted = false; if (envelope.get(Field.TASK).equals(TaskType.PING) && packet.getMessage() instanceof DiscoveryV5Message && ((DiscoveryV5Message) packet.getMessage()).getCode() == MessageCode.PONG) { taskCompleted = true; @@ -83,9 +83,9 @@ public void handle(Envelope envelope) { taskCompleted = true; } if (taskCompleted) { - CompletableFuture future = context.loadFuture(); + CompletableFuture future = session.loadFuture(); future.complete(null); - context.saveFuture(null); + session.saveFuture(null); } } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/MessageHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/MessageHandler.java index 06855c58e..51f011689 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/MessageHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/MessageHandler.java @@ -2,7 +2,7 @@ import org.ethereum.beacon.discovery.DiscoveryV5MessageProcessor; import org.ethereum.beacon.discovery.MessageProcessor; -import org.ethereum.beacon.discovery.NodeContext; +import org.ethereum.beacon.discovery.NodeSession; import org.ethereum.beacon.discovery.message.DiscoveryMessage; import org.ethereum.beacon.discovery.pipeline.Envelope; import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; @@ -14,14 +14,14 @@ public void handle(Envelope envelope) { if (!envelope.contains(Field.MESSAGE)) { return; } - if (!envelope.contains(Field.CONTEXT)) { + if (!envelope.contains(Field.SESSION)) { return; } - NodeContext context = (NodeContext) envelope.get(Field.CONTEXT); + NodeSession session = (NodeSession) envelope.get(Field.SESSION); DiscoveryMessage message = (DiscoveryMessage) envelope.get(Field.MESSAGE); // TODO: optimize MessageProcessor messageProcessor = new MessageProcessor(new DiscoveryV5MessageProcessor()); - messageProcessor.handleIncoming(message, context); + messageProcessor.handleIncoming(message, session); } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/MessagePacketHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/MessagePacketHandler.java index 1cf9d14e2..b7ca44124 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/MessagePacketHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/MessagePacketHandler.java @@ -2,7 +2,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.ethereum.beacon.discovery.NodeContext; +import org.ethereum.beacon.discovery.NodeSession; import org.ethereum.beacon.discovery.message.DiscoveryV5Message; import org.ethereum.beacon.discovery.message.MessageCode; import org.ethereum.beacon.discovery.packet.MessagePacket; @@ -22,26 +22,26 @@ public void handle(Envelope envelope) { if (!envelope.contains(Field.PACKET_MESSAGE)) { return; } - if (!envelope.contains(Field.CONTEXT)) { + if (!envelope.contains(Field.SESSION)) { return; } MessagePacket packet = (MessagePacket) envelope.get(Field.PACKET_MESSAGE); - NodeContext context = (NodeContext) envelope.get(Field.CONTEXT); + NodeSession session = (NodeSession) envelope.get(Field.SESSION); try { - packet.decode(context.getInitiatorKey()); - packet.verify(context.getAuthTag().get()); + packet.decode(session.getInitiatorKey()); + packet.verify(session.getAuthTag().get()); envelope.put(Field.MESSAGE, packet.getMessage()); } catch (AssertionError ex) { logger.info( String.format( "Verification not passed for message [%s] from node %s in status %s", - packet, context.getNodeRecord(), context.getStatus())); + packet, session.getNodeRecord(), session.getStatus())); } catch (Exception ex) { String error = String.format( "Failed to read message [%s] from node %s in status %s", - packet, context.getNodeRecord(), context.getStatus()); + packet, session.getNodeRecord(), session.getStatus()); logger.error(error, ex); envelope.remove(Field.PACKET_MESSAGE); if (envelope.contains(Field.FUTURE)) { @@ -51,8 +51,8 @@ public void handle(Envelope envelope) { return; } envelope.remove(Field.PACKET_MESSAGE); - CompletableFuture future = context.loadFuture(); - TaskType taskType = context.loadTask(); + CompletableFuture future = session.loadFuture(); + TaskType taskType = session.loadTask(); if (future != null) { boolean taskCompleted = false; if (TaskType.PING.equals(taskType) && packet.getMessage() instanceof DiscoveryV5Message && ((DiscoveryV5Message) packet.getMessage()).getCode() == MessageCode.PONG) { @@ -63,8 +63,8 @@ public void handle(Envelope envelope) { } if (taskCompleted) { future.complete(null); - context.saveFuture(null); - context.saveTask(null); + session.saveFuture(null); + session.saveTask(null); } } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeIdToContext.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeIdToSession.java similarity index 66% rename from discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeIdToContext.java rename to discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeIdToSession.java index 3bd091983..340c34fab 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeIdToContext.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeIdToSession.java @@ -2,7 +2,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.ethereum.beacon.discovery.NodeContext; +import org.ethereum.beacon.discovery.NodeSession; import org.ethereum.beacon.discovery.NodeRecordInfo; import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.network.NetworkParcelV5; @@ -24,23 +24,23 @@ import java.util.concurrent.TimeUnit; /** - * Performs {@link Field#NEED_CONTEXT} request. Looks up for Node context based on NodeId, which - * should be in request field and stores it in {@link Field#CONTEXT} field. + * Performs {@link Field#SESSION_LOOKUP} request. Looks up for Node session based on NodeId, which + * should be in request field and stores it in {@link Field#SESSION} field. */ -public class NodeIdToContext implements EnvelopeHandler { +public class NodeIdToSession implements EnvelopeHandler { private static final int CLEANUP_DELAY_SECONDS = 180; - private static final Logger logger = LogManager.getLogger(NodeIdToContext.class); + private static final Logger logger = LogManager.getLogger(NodeIdToSession.class); private final NodeRecord homeNodeRecord; private final NodeBucketStorage nodeBucketStorage; private final AuthTagRepository authTagRepo; - private final Map recentContexts = - new ConcurrentHashMap<>(); // nodeId -> context + private final Map recentSessions = + new ConcurrentHashMap<>(); // nodeId -> session private final NodeTable nodeTable; private final Pipeline outgoingPipeline; - private ExpirationScheduler contextExpirationScheduler = + private ExpirationScheduler sessionExpirationScheduler = new ExpirationScheduler<>(CLEANUP_DELAY_SECONDS, TimeUnit.SECONDS); - public NodeIdToContext( + public NodeIdToSession( NodeRecord homeNodeRecord, NodeBucketStorage nodeBucketStorage, AuthTagRepository authTagRepo, @@ -55,27 +55,27 @@ public NodeIdToContext( @Override public void handle(Envelope envelope) { - if (!envelope.contains(Field.NEED_CONTEXT)) { + if (!envelope.contains(Field.SESSION_LOOKUP)) { return; } - Pair contextRequest = - (Pair) envelope.get(Field.NEED_CONTEXT); - envelope.remove(Field.NEED_CONTEXT); - Optional nodeContextOptional = getContext(contextRequest.getValue0()); - if (nodeContextOptional.isPresent()) { - envelope.put(Field.CONTEXT, nodeContextOptional.get()); + Pair sessionRequest = + (Pair) envelope.get(Field.SESSION_LOOKUP); + envelope.remove(Field.SESSION_LOOKUP); + Optional nodeSessionOptional = getSession(sessionRequest.getValue0()); + if (nodeSessionOptional.isPresent()) { + envelope.put(Field.SESSION, nodeSessionOptional.get()); logger.trace( () -> String.format( - "Context resolved: %s in envelope #%s", - nodeContextOptional.get(), envelope.getId())); + "Session resolved: %s in envelope #%s", + nodeSessionOptional.get(), envelope.getId())); } else { - contextRequest.getValue1().run(); + sessionRequest.getValue1().run(); } } - private Optional getContext(Bytes32 nodeId) { - NodeContext context = recentContexts.get(nodeId); + private Optional getSession(Bytes32 nodeId) { + NodeSession context = recentSessions.get(nodeId); if (context == null) { Optional nodeOptional = nodeTable.getNode(nodeId); if (!nodeOptional.isPresent()) { @@ -86,7 +86,7 @@ private Optional getContext(Bytes32 nodeId) { NodeRecord nodeRecord = nodeOptional.get().getNode(); SecureRandom random = new SecureRandom(); context = - new NodeContext( + new NodeSession( nodeRecord, homeNodeRecord, nodeTable, @@ -94,14 +94,14 @@ private Optional getContext(Bytes32 nodeId) { authTagRepo, packet -> outgoingPipeline.push(new NetworkParcelV5(packet, nodeRecord)), random); - recentContexts.put(nodeId, context); + recentSessions.put(nodeId, context); } - final NodeContext contextBackup = context; - contextExpirationScheduler.put( + final NodeSession contextBackup = context; + sessionExpirationScheduler.put( context.getNodeRecord().getNodeId(), () -> { - recentContexts.remove(contextBackup.getNodeRecord().getNodeId()); + recentSessions.remove(contextBackup.getNodeRecord().getNodeId()); contextBackup.cleanup(); }); return Optional.of(context); diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeContextRequestHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeSessionRequestHandler.java similarity index 78% rename from discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeContextRequestHandler.java rename to discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeSessionRequestHandler.java index 8a8fa704e..42a8da9bc 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeContextRequestHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeSessionRequestHandler.java @@ -7,10 +7,10 @@ import org.javatuples.Pair; /** - * Searches for node in {@link Field#NODE} and requests context resolving using {@link - * Field#NEED_CONTEXT} + * Searches for node in {@link Field#NODE} and requests session resolving using {@link + * Field#SESSION_LOOKUP} */ -public class NodeContextRequestHandler implements EnvelopeHandler { +public class NodeSessionRequestHandler implements EnvelopeHandler { @Override public void handle(Envelope envelope) { @@ -18,7 +18,7 @@ public void handle(Envelope envelope) { return; } envelope.put( - Field.NEED_CONTEXT, + Field.SESSION_LOOKUP, Pair.with( ((NodeRecord) envelope.get(Field.NODE)).getNodeId(), (Runnable) diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NotExpectedIncomingPacketHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NotExpectedIncomingPacketHandler.java index 289df5200..543f1c239 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NotExpectedIncomingPacketHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NotExpectedIncomingPacketHandler.java @@ -3,7 +3,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.ethereum.beacon.discovery.Functions; -import org.ethereum.beacon.discovery.NodeContext; +import org.ethereum.beacon.discovery.NodeSession; import org.ethereum.beacon.discovery.packet.MessagePacket; import org.ethereum.beacon.discovery.packet.RandomPacket; import org.ethereum.beacon.discovery.packet.UnknownPacket; @@ -23,11 +23,11 @@ public void handle(Envelope envelope) { if (!envelope.contains(Field.PACKET_UNKNOWN)) { return; } - if (!envelope.contains(Field.CONTEXT)) { + if (!envelope.contains(Field.SESSION)) { return; } UnknownPacket unknownPacket = (UnknownPacket) envelope.get(Field.PACKET_UNKNOWN); - NodeContext context = (NodeContext) envelope.get(Field.CONTEXT); + NodeSession session = (NodeSession) envelope.get(Field.SESSION); try { // packet it either random or message packet if session is expired BytesValue authTag = null; @@ -42,35 +42,35 @@ public void handle(Envelope envelope) { MessagePacket messagePacket = unknownPacket.getMessagePacket(); authTag = messagePacket.getAuthTag(); } - context.setAuthTag(authTag); + session.setAuthTag(authTag); byte[] idNonceBytes = new byte[32]; Functions.getRandom().nextBytes(idNonceBytes); Bytes32 idNonce = Bytes32.wrap(idNonceBytes); - context.setIdNonce(idNonce); + session.setIdNonce(idNonce); WhoAreYouPacket whoAreYouPacket = WhoAreYouPacket.create( - context.getNodeRecord().getNodeId(), + session.getNodeRecord().getNodeId(), authTag, idNonce, - context.getNodeRecord().getSeq()); - context.sendOutgoing(whoAreYouPacket); + session.getNodeRecord().getSeq()); + session.sendOutgoing(whoAreYouPacket); } catch (AssertionError ex) { logger.info( String.format( "Verification not passed for message [%s] from node %s in status %s", - unknownPacket, context.getNodeRecord(), context.getStatus())); + unknownPacket, session.getNodeRecord(), session.getStatus())); } catch (Exception ex) { String error = String.format( "Failed to read message [%s] from node %s in status %s", - unknownPacket, context.getNodeRecord(), context.getStatus()); + unknownPacket, session.getNodeRecord(), session.getStatus()); logger.error(error, ex); envelope.put(Field.BAD_PACKET, envelope.get(Field.PACKET_UNKNOWN)); envelope.put(Field.BAD_PACKET_EXCEPTION, ex); envelope.remove(Field.PACKET_UNKNOWN); return; } - context.setStatus(NodeContext.SessionStatus.WHOAREYOU_SENT); + session.setStatus(NodeSession.SessionStatus.WHOAREYOU_SENT); envelope.remove(Field.PACKET_UNKNOWN); } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/TaskHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/TaskHandler.java index c609c5d49..6941b6e02 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/TaskHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/TaskHandler.java @@ -1,6 +1,6 @@ package org.ethereum.beacon.discovery.pipeline.handler; -import org.ethereum.beacon.discovery.NodeContext; +import org.ethereum.beacon.discovery.NodeSession; import org.ethereum.beacon.discovery.packet.RandomPacket; import org.ethereum.beacon.discovery.pipeline.Envelope; import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; @@ -26,32 +26,32 @@ public void handle(Envelope envelope) { if (!envelope.contains(Field.TASK)) { return; } - NodeContext context = (NodeContext) envelope.get(Field.CONTEXT); + NodeSession session = (NodeSession) envelope.get(Field.SESSION); CompletableFuture completableFuture = (CompletableFuture) envelope.get(Field.FUTURE); - if (context.getStatus().equals(NodeContext.SessionStatus.INITIAL)) { + if (session.getStatus().equals(NodeSession.SessionStatus.INITIAL)) { byte[] authTagBytes = new byte[12]; rnd.nextBytes(authTagBytes); BytesValue authTag = BytesValue.wrap(authTagBytes); RandomPacket randomPacket = RandomPacket.create( - context.getHomeNodeId(), - context.getNodeRecord().getNodeId(), + session.getHomeNodeId(), + session.getNodeRecord().getNodeId(), authTag, new SecureRandom()); - context.setAuthTag(authTag); - context.sendOutgoing(randomPacket); - context.setStatus(NodeContext.SessionStatus.RANDOM_PACKET_SENT); - context.saveFuture(completableFuture); - } else if (context.getStatus().equals(NodeContext.SessionStatus.AUTHENTICATED)) { + session.setAuthTag(authTag); + session.sendOutgoing(randomPacket); + session.setStatus(NodeSession.SessionStatus.RANDOM_PACKET_SENT); + session.saveFuture(completableFuture); + } else if (session.getStatus().equals(NodeSession.SessionStatus.AUTHENTICATED)) { if (envelope.get(Field.TASK).equals(TaskType.PING)) { - context.sendOutgoing(TaskMessageFactory.createPingPacket(context)); - context.saveTask((TaskType) envelope.get(Field.TASK)); - context.saveFuture(completableFuture); + session.sendOutgoing(TaskMessageFactory.createPingPacket(session)); + session.saveTask((TaskType) envelope.get(Field.TASK)); + session.saveFuture(completableFuture); } else if (envelope.get(Field.TASK).equals(TaskType.FINDNODE)) { - context.sendOutgoing(TaskMessageFactory.createFindNodePacket(context)); - context.saveTask((TaskType) envelope.get(Field.TASK)); - context.saveFuture(completableFuture); + session.sendOutgoing(TaskMessageFactory.createFindNodePacket(session)); + session.saveTask((TaskType) envelope.get(Field.TASK)); + session.saveFuture(completableFuture); } else { throw new RuntimeException( String.format( diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/UnknownPacketTagToSender.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/UnknownPacketTagToSender.java index 209bf0b3b..9ec4f777d 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/UnknownPacketTagToSender.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/UnknownPacketTagToSender.java @@ -10,8 +10,8 @@ /** * Assuming we have some unknown packet in {@link Field#PACKET_UNKNOWN}, resolves sender node id - * using `tag` field of the packet. Next, puts it to the {@link Field#NEED_CONTEXT} so sender - * context could be resolved by another handler. + * using `tag` field of the packet. Next, puts it to the {@link Field#SESSION_LOOKUP} so sender + * session could be resolved by another handler. */ public class UnknownPacketTagToSender implements EnvelopeHandler { private final Bytes32 homeNodeId; @@ -28,7 +28,7 @@ public void handle(Envelope envelope) { UnknownPacket unknownPacket = (UnknownPacket) envelope.get(Field.PACKET_UNKNOWN); Bytes32 fromNodeId = unknownPacket.getSourceNodeId(homeNodeId); envelope.put( - Field.NEED_CONTEXT, + Field.SESSION_LOOKUP, Pair.with( fromNodeId, (Runnable) @@ -37,7 +37,7 @@ public void handle(Envelope envelope) { envelope.put( Field.BAD_PACKET_EXCEPTION, new RuntimeException( - String.format("Context not found for nodeId %s", fromNodeId))); + String.format("Session couldn't be created for nodeId %s", fromNodeId))); envelope.remove(Field.PACKET_UNKNOWN); })); } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/UnknownPacketTypeByStatus.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/UnknownPacketTypeByStatus.java index 488e8e215..b212e3b1e 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/UnknownPacketTypeByStatus.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/UnknownPacketTypeByStatus.java @@ -2,7 +2,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.ethereum.beacon.discovery.NodeContext; +import org.ethereum.beacon.discovery.NodeSession; import org.ethereum.beacon.discovery.packet.AuthHeaderMessagePacket; import org.ethereum.beacon.discovery.packet.MessagePacket; import org.ethereum.beacon.discovery.packet.UnknownPacket; @@ -11,7 +11,7 @@ import org.ethereum.beacon.discovery.pipeline.Field; /** - * Resolves incoming packet type based on context states and places packet into the corresponding + * Resolves incoming packet type based on session states and places packet into the corresponding * field. Doesn't recognize WhoAreYou packet, last should be resolved by {@link WhoAreYouAttempt} */ public class UnknownPacketTypeByStatus implements EnvelopeHandler { @@ -19,15 +19,15 @@ public class UnknownPacketTypeByStatus implements EnvelopeHandler { @Override public void handle(Envelope envelope) { - if (!envelope.contains(Field.CONTEXT)) { + if (!envelope.contains(Field.SESSION)) { return; } if (!envelope.contains(Field.PACKET_UNKNOWN)) { return; } UnknownPacket unknownPacket = (UnknownPacket) envelope.get(Field.PACKET_UNKNOWN); - NodeContext context = (NodeContext) envelope.get(Field.CONTEXT); - switch (context.getStatus()) { + NodeSession session = (NodeSession) envelope.get(Field.SESSION); + switch (session.getStatus()) { case INITIAL: { // We still don't know what's the type of the packet @@ -58,7 +58,7 @@ public void handle(Envelope envelope) { String error = String.format( "Not expected status:%s from node: %s", - context.getStatus(), context.getNodeRecord()); + session.getStatus(), session.getNodeRecord()); logger.error(error); throw new RuntimeException(error); } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouPacketHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouPacketHandler.java index 4db2c48e1..621723067 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouPacketHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouPacketHandler.java @@ -3,7 +3,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.ethereum.beacon.discovery.Functions; -import org.ethereum.beacon.discovery.NodeContext; +import org.ethereum.beacon.discovery.NodeSession; import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.message.DiscoveryV5Message; import org.ethereum.beacon.discovery.message.V5Message; @@ -29,62 +29,62 @@ public void handle(Envelope envelope) { if (!envelope.contains(Field.PACKET_WHOAREYOU)) { return; } - if (!envelope.contains(Field.CONTEXT)) { + if (!envelope.contains(Field.SESSION)) { return; } WhoAreYouPacket packet = (WhoAreYouPacket) envelope.get(Field.PACKET_WHOAREYOU); - NodeContext context = (NodeContext) envelope.get(Field.CONTEXT); + NodeSession session = (NodeSession) envelope.get(Field.SESSION); try { - BytesValue authTag = context.getAuthTag().get(); - packet.verify(context.getHomeNodeId(), authTag); + BytesValue authTag = session.getAuthTag().get(); + packet.verify(session.getHomeNodeId(), authTag); packet.getEnrSeq(); // FIXME: Their side enr seq. Do we need it? byte[] ephemeralKeyBytes = new byte[32]; Functions.getRandom().nextBytes(ephemeralKeyBytes); ECKeyPair ephemeralKey = ECKeyPair.create(ephemeralKeyBytes); // TODO: generate Triplet hkdf = Functions.hkdf_expand( - context.getHomeNodeId(), - context.getNodeRecord().getNodeId(), + session.getHomeNodeId(), + session.getNodeRecord().getNodeId(), BytesValue.wrap(ephemeralKey.getPrivateKey().toByteArray()), packet.getIdNonce(), - (BytesValue) context.getNodeRecord().get(NodeRecord.FIELD_PKEY_SECP256K1)); + (BytesValue) session.getNodeRecord().get(NodeRecord.FIELD_PKEY_SECP256K1)); BytesValue initiatorKey = hkdf.getValue0(); BytesValue staticNodeKey = hkdf.getValue1(); BytesValue authResponseKey = hkdf.getValue2(); V5Message taskMessage = null; - if (context.loadTask() == TaskType.PING) { - taskMessage = TaskMessageFactory.createPing(context); - } else if (context.loadTask() == TaskType.FINDNODE) { - taskMessage = TaskMessageFactory.createFindNode(context); + if (session.loadTask() == TaskType.PING) { + taskMessage = TaskMessageFactory.createPing(session); + } else if (session.loadTask() == TaskType.FINDNODE) { + taskMessage = TaskMessageFactory.createFindNode(session); } else { throw new RuntimeException( String.format( - "Type %s in envelope #%s is not known", context.loadTask(), envelope.getId())); + "Type %s in envelope #%s is not known", session.loadTask(), envelope.getId())); } AuthHeaderMessagePacket response = AuthHeaderMessagePacket.create( - context.getHomeNodeId(), - context.getNodeRecord().getNodeId(), + session.getHomeNodeId(), + session.getNodeRecord().getNodeId(), authResponseKey, packet.getIdNonce(), staticNodeKey, - context.getHomeNodeRecord(), + session.getHomeNodeRecord(), BytesValue.wrap(ephemeralKey.getPublicKey().toByteArray()), authTag, initiatorKey, DiscoveryV5Message.from(taskMessage)); - context.sendOutgoing(response); + session.sendOutgoing(response); } catch (AssertionError ex) { logger.info( String.format( "Verification not passed for message [%s] from node %s in status %s", - packet, context.getNodeRecord(), context.getStatus())); + packet, session.getNodeRecord(), session.getStatus())); } catch (Exception ex) { String error = String.format( "Failed to read message [%s] from node %s in status %s", - packet, context.getNodeRecord(), context.getStatus()); + packet, session.getNodeRecord(), session.getStatus()); logger.error(error, ex); envelope.remove(Field.PACKET_WHOAREYOU); if (envelope.contains(Field.FUTURE)) { @@ -93,7 +93,7 @@ public void handle(Envelope envelope) { } return; } - context.setStatus(NodeContext.SessionStatus.AUTHENTICATED); + session.setStatus(NodeSession.SessionStatus.AUTHENTICATED); envelope.remove(Field.PACKET_WHOAREYOU); } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouContextResolver.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouSessionResolver.java similarity index 69% rename from discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouContextResolver.java rename to discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouSessionResolver.java index 61a7df5c8..23c2477c4 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouContextResolver.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouSessionResolver.java @@ -2,7 +2,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.ethereum.beacon.discovery.NodeContext; +import org.ethereum.beacon.discovery.NodeSession; import org.ethereum.beacon.discovery.packet.WhoAreYouPacket; import org.ethereum.beacon.discovery.pipeline.Envelope; import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; @@ -12,14 +12,14 @@ import java.util.Optional; /** - * Resolves context using `authTagRepo` for `WHOAREYOU` packets which should be placed in {@link + * Resolves session using `authTagRepo` for `WHOAREYOU` packets which should be placed in {@link * Field#PACKET_WHOAREYOU} */ -public class WhoAreYouContextResolver implements EnvelopeHandler { - private static final Logger logger = LogManager.getLogger(WhoAreYouContextResolver.class); +public class WhoAreYouSessionResolver implements EnvelopeHandler { + private static final Logger logger = LogManager.getLogger(WhoAreYouSessionResolver.class); private final AuthTagRepository authTagRepo; - public WhoAreYouContextResolver(AuthTagRepository authTagRepo) { + public WhoAreYouSessionResolver(AuthTagRepository authTagRepo) { this.authTagRepo = authTagRepo; } @@ -30,20 +30,20 @@ public void handle(Envelope envelope) { } WhoAreYouPacket whoAreYouPacket = (WhoAreYouPacket) envelope.get(Field.PACKET_WHOAREYOU); - Optional nodeContextOptional = authTagRepo.get(whoAreYouPacket.getAuthTag()); - if (nodeContextOptional.isPresent() - && nodeContextOptional + Optional nodeSessionOptional = authTagRepo.get(whoAreYouPacket.getAuthTag()); + if (nodeSessionOptional.isPresent() + && nodeSessionOptional .get() .getStatus() .equals( - NodeContext.SessionStatus + NodeSession.SessionStatus .RANDOM_PACKET_SENT)) { // FIXME: doesn't handle session expiration - envelope.put(Field.CONTEXT, nodeContextOptional.get()); + envelope.put(Field.SESSION, nodeSessionOptional.get()); logger.trace( () -> String.format( - "Context resolved: %s in envelope #%s", - nodeContextOptional.get(), envelope.getId())); + "Session resolved: %s in envelope #%s", + nodeSessionOptional.get(), envelope.getId())); } else { envelope.put(Field.BAD_PACKET, envelope.get(Field.PACKET_WHOAREYOU)); envelope.remove(Field.PACKET_WHOAREYOU); diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/AuthTagRepository.java b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/AuthTagRepository.java index 6d88dfcbe..83f938853 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/AuthTagRepository.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/AuthTagRepository.java @@ -2,7 +2,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.ethereum.beacon.discovery.NodeContext; +import org.ethereum.beacon.discovery.NodeSession; import tech.pegasys.artemis.util.bytes.BytesValue; import java.util.Map; @@ -10,46 +10,46 @@ import java.util.concurrent.ConcurrentHashMap; /** - * In memory repository with authTags, corresponding contexts {@link NodeContext} and 2-way getters: - * {@link #get(BytesValue)} and {@link #getTag(NodeContext)} + * In memory repository with authTags, corresponding sessions {@link NodeSession} and 2-way getters: + * {@link #get(BytesValue)} and {@link #getTag(NodeSession)} * - *

Expired authTags should be manually removed with {@link #expire(NodeContext)} + *

Expired authTags should be manually removed with {@link #expire(NodeSession)} */ public class AuthTagRepository { private static final Logger logger = LogManager.getLogger(AuthTagRepository.class); - private Map authTags = new ConcurrentHashMap<>(); - private Map contexts = new ConcurrentHashMap<>(); + private Map authTags = new ConcurrentHashMap<>(); + private Map sessions = new ConcurrentHashMap<>(); - public synchronized void put(BytesValue authTag, NodeContext context) { + public synchronized void put(BytesValue authTag, NodeSession session) { logger.trace( () -> String.format( - "PUT: authTag[%s] => nodeContext[%s]", - authTag, context.getNodeRecord().getNodeId())); - authTags.put(authTag, context); - contexts.put(context, authTag); + "PUT: authTag[%s] => nodeSession[%s]", + authTag, session.getNodeRecord().getNodeId())); + authTags.put(authTag, session); + sessions.put(session, authTag); } - public Optional get(BytesValue authTag) { + public Optional get(BytesValue authTag) { logger.trace(() -> String.format("GET: authTag[%s]", authTag)); - NodeContext context = authTags.get(authTag); - return context == null ? Optional.empty() : Optional.of(context); + NodeSession session = authTags.get(authTag); + return session == null ? Optional.empty() : Optional.of(session); } - public Optional getTag(NodeContext context) { - logger.trace(() -> String.format("GET: context %s", context)); - BytesValue authTag = contexts.get(context); + public Optional getTag(NodeSession session) { + logger.trace(() -> String.format("GET: session %s", session)); + BytesValue authTag = sessions.get(session); return authTag == null ? Optional.empty() : Optional.of(authTag); } - public synchronized void expire(NodeContext context) { - logger.trace(() -> String.format("REMOVE: context %s", context)); - BytesValue authTag = contexts.remove(context); + public synchronized void expire(NodeSession session) { + logger.trace(() -> String.format("REMOVE: session %s", session)); + BytesValue authTag = sessions.remove(session); logger.trace( () -> authTag == null - ? "Context %s not found, was not removed" - : String.format("Context %s removed with authTag[%s]", context, authTag)); + ? "Session %s not found, was not removed" + : String.format("Session %s removed with authTag[%s]", session, authTag)); if (authTag != null) { authTags.remove(authTag); } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/task/TaskMessageFactory.java b/discovery/src/main/java/org/ethereum/beacon/discovery/task/TaskMessageFactory.java index 50b7839fc..102c5d153 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/task/TaskMessageFactory.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/task/TaskMessageFactory.java @@ -1,6 +1,6 @@ package org.ethereum.beacon.discovery.task; -import org.ethereum.beacon.discovery.NodeContext; +import org.ethereum.beacon.discovery.NodeSession; import org.ethereum.beacon.discovery.message.DiscoveryV5Message; import org.ethereum.beacon.discovery.message.FindNodeMessage; import org.ethereum.beacon.discovery.message.MessageCode; @@ -10,32 +10,32 @@ public class TaskMessageFactory { public static final int DEFAULT_DISTANCE = 10; - public static MessagePacket createPingPacket(NodeContext context) { + public static MessagePacket createPingPacket(NodeSession session) { return MessagePacket.create( - context.getHomeNodeId(), - context.getNodeRecord().getNodeId(), - context.getAuthTag().get(), - context.getInitiatorKey(), - DiscoveryV5Message.from(createPing(context))); + session.getHomeNodeId(), + session.getNodeRecord().getNodeId(), + session.getAuthTag().get(), + session.getInitiatorKey(), + DiscoveryV5Message.from(createPing(session))); } - public static PingMessage createPing(NodeContext context) { + public static PingMessage createPing(NodeSession session) { return new PingMessage( - context.getNextRequestId(MessageCode.PING), context.getNodeRecord().getSeq()); + session.getNextRequestId(MessageCode.PING), session.getNodeRecord().getSeq()); } - public static MessagePacket createFindNodePacket(NodeContext context) { - FindNodeMessage findNodeMessage = createFindNode(context); + public static MessagePacket createFindNodePacket(NodeSession session) { + FindNodeMessage findNodeMessage = createFindNode(session); return MessagePacket.create( - context.getHomeNodeId(), - context.getNodeRecord().getNodeId(), - context.getAuthTag().get(), - context.getInitiatorKey(), + session.getHomeNodeId(), + session.getNodeRecord().getNodeId(), + session.getAuthTag().get(), + session.getInitiatorKey(), DiscoveryV5Message.from(findNodeMessage)); } - public static FindNodeMessage createFindNode(NodeContext context) { - return new FindNodeMessage(context.getNextRequestId(MessageCode.FINDNODE), DEFAULT_DISTANCE); + public static FindNodeMessage createFindNode(NodeSession session) { + return new FindNodeMessage(session.getNextRequestId(MessageCode.FINDNODE), DEFAULT_DISTANCE); } } diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java b/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java index eed4cacfc..8548884fe 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java @@ -15,15 +15,15 @@ import org.ethereum.beacon.discovery.pipeline.handler.IncomingDataPacker; import org.ethereum.beacon.discovery.pipeline.handler.MessageHandler; import org.ethereum.beacon.discovery.pipeline.handler.MessagePacketHandler; -import org.ethereum.beacon.discovery.pipeline.handler.NodeContextRequestHandler; -import org.ethereum.beacon.discovery.pipeline.handler.NodeIdToContext; +import org.ethereum.beacon.discovery.pipeline.handler.NodeSessionRequestHandler; +import org.ethereum.beacon.discovery.pipeline.handler.NodeIdToSession; import org.ethereum.beacon.discovery.pipeline.handler.NotExpectedIncomingPacketHandler; import org.ethereum.beacon.discovery.pipeline.handler.OutgoingParcelHandler; import org.ethereum.beacon.discovery.pipeline.handler.TaskHandler; import org.ethereum.beacon.discovery.pipeline.handler.UnknownPacketTagToSender; import org.ethereum.beacon.discovery.pipeline.handler.UnknownPacketTypeByStatus; import org.ethereum.beacon.discovery.pipeline.handler.WhoAreYouAttempt; -import org.ethereum.beacon.discovery.pipeline.handler.WhoAreYouContextResolver; +import org.ethereum.beacon.discovery.pipeline.handler.WhoAreYouSessionResolver; import org.ethereum.beacon.discovery.pipeline.handler.WhoAreYouPacketHandler; import org.ethereum.beacon.discovery.storage.AuthTagRepository; import org.ethereum.beacon.discovery.storage.NodeBucketStorage; @@ -59,14 +59,14 @@ public DiscoveryManagerNoNetwork( Publisher incomingPackets) { AuthTagRepository authTagRepo = new AuthTagRepository(); this.incomingPackets = incomingPackets; - NodeIdToContext nodeIdToContext = - new NodeIdToContext(homeNode, nodeBucketStorage, authTagRepo, nodeTable, outgoingPipeline); + NodeIdToSession nodeIdToSession = + new NodeIdToSession(homeNode, nodeBucketStorage, authTagRepo, nodeTable, outgoingPipeline); incomingPipeline .addHandler(new IncomingDataPacker()) .addHandler(new WhoAreYouAttempt(homeNode.getNodeId())) - .addHandler(new WhoAreYouContextResolver(authTagRepo)) + .addHandler(new WhoAreYouSessionResolver(authTagRepo)) .addHandler(new UnknownPacketTagToSender(homeNode)) - .addHandler(nodeIdToContext) + .addHandler(nodeIdToSession) .addHandler(new UnknownPacketTypeByStatus()) .addHandler(new NotExpectedIncomingPacketHandler()) .addHandler(new WhoAreYouPacketHandler()) @@ -76,8 +76,8 @@ public DiscoveryManagerNoNetwork( .addHandler(new BadPacketLogger()); outgoingPipeline .addHandler(new OutgoingParcelHandler(outgoingSink)) - .addHandler(new NodeContextRequestHandler()) - .addHandler(nodeIdToContext) + .addHandler(new NodeSessionRequestHandler()) + .addHandler(nodeIdToSession) .addHandler(new TaskHandler(new SecureRandom())); } From ec36b97acba0ddf999a5254cc6a4815f65ecd9d7 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Mon, 14 Oct 2019 05:44:02 +0300 Subject: [PATCH 34/77] discovery: fix bucket max distance --- .../beacon/discovery/storage/NodeBucketStorageImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeBucketStorageImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeBucketStorageImpl.java index 57b0618d6..e0cb1920e 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeBucketStorageImpl.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeBucketStorageImpl.java @@ -19,7 +19,7 @@ */ public class NodeBucketStorageImpl implements NodeBucketStorage { public static final String NODE_BUCKET_STORAGE_NAME = "node-bucket-table"; - public static final int MAXIMUM_BUCKET = 255; + public static final int MAXIMUM_BUCKET = 256; private final HoleyList nodeBucketsTable; private final Bytes32 homeNodeId; From 3dac85e94bb38d665c0bb4e37f2b907d9d75bd46 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Tue, 15 Oct 2019 17:15:21 +0300 Subject: [PATCH 35/77] discovery: fix DH hkdf_expand function --- .../ethereum/beacon/discovery/Functions.java | 48 +++++++++---------- .../AuthHeaderMessagePacketHandler.java | 12 +++-- .../handler/WhoAreYouPacketHandler.java | 4 +- .../beacon/discovery/FunctionsTest.java | 35 ++++++++++++++ 4 files changed, 69 insertions(+), 30 deletions(-) diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java b/discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java index 402a05f8f..6cd3fbafc 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java @@ -1,16 +1,11 @@ package org.ethereum.beacon.discovery; -import org.bouncycastle.crypto.BasicAgreement; import org.bouncycastle.crypto.Digest; -import org.bouncycastle.crypto.agreement.ECDHBasicAgreement; import org.bouncycastle.crypto.digests.SHA256Digest; import org.bouncycastle.crypto.generators.HKDFBytesGenerator; import org.bouncycastle.crypto.params.ECDomainParameters; -import org.bouncycastle.crypto.params.ECPrivateKeyParameters; -import org.bouncycastle.crypto.params.ECPublicKeyParameters; import org.bouncycastle.crypto.params.HKDFParameters; -import org.bouncycastle.crypto.util.PrivateKeyFactory; -import org.bouncycastle.math.ec.ECCurve; +import org.bouncycastle.math.ec.ECPoint; import org.ethereum.beacon.crypto.Hashes; import org.javatuples.Triplet; import org.web3j.crypto.ECKeyPair; @@ -28,10 +23,12 @@ import java.security.SignatureException; import java.util.Random; +import static org.web3j.crypto.Sign.CURVE_PARAMS; + public class Functions { - private static final int RECIPIENT_KEY_LENGTH = 32; // FIXME - private static final int INITIATOR_KEY_LENGTH = 32; // FIXME - private static final int AUTH_RESP_KEY_LENGTH = 32; // FIXME + private static final int RECIPIENT_KEY_LENGTH = 16; + private static final int INITIATOR_KEY_LENGTH = 16; + private static final int AUTH_RESP_KEY_LENGTH = 16; public static Bytes32 hash(BytesValue value) { return Hashes.sha256(value); @@ -123,21 +120,23 @@ public static BytesValue aesgcm_decrypt( public static Triplet hkdf_expand( BytesValue srcNodeId, BytesValue destNodeId, - BytesValue ephemeralKey, - BytesValue idNonce, - BytesValue destPubKey) { + BytesValue srcPrivKey, + BytesValue destPubKey, + BytesValue idNonce) { try { - Digest digest = new SHA256Digest(); // FIXME: or whatever - ECPrivateKeyParameters ecdhPrivateKeyParameters = - (ECPrivateKeyParameters) (PrivateKeyFactory.createKey(ephemeralKey.extractArray())); - ECDomainParameters ecDomainParameters = ecdhPrivateKeyParameters.getParameters(); - ECCurve ecCurve = ecDomainParameters.getCurve(); - ECPublicKeyParameters ecPublicKeyParameters = - new ECPublicKeyParameters( - ecCurve.decodePoint(destPubKey.extractArray()), ecDomainParameters); - BasicAgreement agree = new ECDHBasicAgreement(); - agree.init(ecdhPrivateKeyParameters); - byte[] keyAgreement = agree.calculateAgreement(ecPublicKeyParameters).toByteArray(); + ECDomainParameters CURVE = + new ECDomainParameters( + CURVE_PARAMS.getCurve(), + CURVE_PARAMS.getG(), + CURVE_PARAMS.getN(), + CURVE_PARAMS.getH()); + + byte[] destPubPointBytes = new byte[destPubKey.size() + 1]; + destPubPointBytes[0] = 0x04; // default prefix + System.arraycopy(destPubKey.extractArray(), 0, destPubPointBytes, 1, destPubKey.size()); + ECPoint pudDestPoint = CURVE.getCurve().decodePoint(destPubPointBytes); + ECPoint mult = pudDestPoint.multiply(new BigInteger(1, srcPrivKey.extractArray())); + byte[] keyAgreement = mult.getEncoded(true); BytesValue info = BytesValue.wrap("discovery v5 key agreement".getBytes()) @@ -145,9 +144,10 @@ public static Triplet hkdf_expand( .concat(destNodeId); HKDFParameters hkdfParameters = new HKDFParameters(keyAgreement, idNonce.extractArray(), info.extractArray()); + Digest digest = new SHA256Digest(); HKDFBytesGenerator hkdfBytesGenerator = new HKDFBytesGenerator(digest); hkdfBytesGenerator.init(hkdfParameters); - // initiator-key, recipient-key, auth-resp-key + // initiator-key || recipient-key || auth-resp-key byte[] hkdfOutputBytes = new byte[INITIATOR_KEY_LENGTH + RECIPIENT_KEY_LENGTH + AUTH_RESP_KEY_LENGTH]; hkdfBytesGenerator.generateBytes( diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/AuthHeaderMessagePacketHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/AuthHeaderMessagePacketHandler.java index bca330a72..cb4fb276b 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/AuthHeaderMessagePacketHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/AuthHeaderMessagePacketHandler.java @@ -45,8 +45,8 @@ public void handle(Envelope envelope) { session.getHomeNodeId(), session.getNodeRecord().getNodeId(), BytesValue.wrap(ephemeralKey.getPrivateKey().toByteArray()), - session.getIdNonce(), - (BytesValue) session.getNodeRecord().get(NodeRecord.FIELD_PKEY_SECP256K1)); + (BytesValue) session.getNodeRecord().get(NodeRecord.FIELD_PKEY_SECP256K1), + session.getIdNonce()); BytesValue initiatorKey = hkdf.getValue0(); session.setInitiatorKey(initiatorKey); BytesValue authResponseKey = hkdf.getValue2(); @@ -76,10 +76,14 @@ public void handle(Envelope envelope) { envelope.remove(Field.PACKET_AUTH_HEADER_MESSAGE); if (session.loadFuture() != null) { boolean taskCompleted = false; - if (envelope.get(Field.TASK).equals(TaskType.PING) && packet.getMessage() instanceof DiscoveryV5Message && ((DiscoveryV5Message) packet.getMessage()).getCode() == MessageCode.PONG) { + if (envelope.get(Field.TASK).equals(TaskType.PING) + && packet.getMessage() instanceof DiscoveryV5Message + && ((DiscoveryV5Message) packet.getMessage()).getCode() == MessageCode.PONG) { taskCompleted = true; } - if (envelope.get(Field.TASK).equals(TaskType.FINDNODE) && packet.getMessage() instanceof DiscoveryV5Message && ((DiscoveryV5Message) packet.getMessage()).getCode() == MessageCode.NODES) { + if (envelope.get(Field.TASK).equals(TaskType.FINDNODE) + && packet.getMessage() instanceof DiscoveryV5Message + && ((DiscoveryV5Message) packet.getMessage()).getCode() == MessageCode.NODES) { taskCompleted = true; } if (taskCompleted) { diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouPacketHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouPacketHandler.java index 621723067..c184e13ae 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouPacketHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouPacketHandler.java @@ -46,8 +46,8 @@ public void handle(Envelope envelope) { session.getHomeNodeId(), session.getNodeRecord().getNodeId(), BytesValue.wrap(ephemeralKey.getPrivateKey().toByteArray()), - packet.getIdNonce(), - (BytesValue) session.getNodeRecord().get(NodeRecord.FIELD_PKEY_SECP256K1)); + (BytesValue) session.getNodeRecord().get(NodeRecord.FIELD_PKEY_SECP256K1), + packet.getIdNonce()); BytesValue initiatorKey = hkdf.getValue0(); BytesValue staticNodeKey = hkdf.getValue1(); BytesValue authResponseKey = hkdf.getValue2(); diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/FunctionsTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/FunctionsTest.java index f6b4259d4..6d78cebc0 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/FunctionsTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/FunctionsTest.java @@ -1,7 +1,10 @@ package org.ethereum.beacon.discovery; +import org.javatuples.Triplet; import org.junit.Test; +import org.web3j.crypto.ECKeyPair; import tech.pegasys.artemis.util.bytes.Bytes32; +import tech.pegasys.artemis.util.bytes.BytesValue; import static org.junit.Assert.assertEquals; @@ -31,4 +34,36 @@ public void testLogDistance() { // logDistance is not an additive function assertEquals(255, Functions.logDistance(nodeId9s, nodeIdfs)); } + + @Test + public void hkdfExpandTest() { + BytesValue testKeyA = + BytesValue.fromHexString( + "eef77acb6c6a6eebc5b363a475ac583ec7eccdb42b6481424c60f59aa326547f"); + BytesValue testKeyB = + BytesValue.fromHexString( + "66fb62bfbd66b9177a138c1e5cddbe4f7c30c343e94e68df8769459cb1cde628"); + BytesValue idNonce = Bytes32.ZERO; + byte[] homeNodeIdBytes = new byte[32]; + homeNodeIdBytes[0] = 0x01; + byte[] destNodeIdBytes = new byte[32]; + destNodeIdBytes[0] = 0x02; + BytesValue homeNodeId = BytesValue.wrap(homeNodeIdBytes); + BytesValue destNodeId = BytesValue.wrap(destNodeIdBytes); + Triplet sec1 = + Functions.hkdf_expand( + homeNodeId, + destNodeId, + testKeyA, + BytesValue.wrap(ECKeyPair.create(testKeyB.extractArray()).getPublicKey().toByteArray()), + idNonce); + Triplet sec2 = + Functions.hkdf_expand( + homeNodeId, + destNodeId, + testKeyB, + BytesValue.wrap(ECKeyPair.create(testKeyA.extractArray()).getPublicKey().toByteArray()), + idNonce); + assertEquals(sec1, sec2); + } } From 077717aba713c0a920fa155d8dbab416b3f8e4dd Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Thu, 17 Oct 2019 19:24:52 +0300 Subject: [PATCH 36/77] discovery: fixes of AuthHeaderMessage handshake part + tests --- .../discovery/DiscoveryManagerImpl.java | 14 +- .../ethereum/beacon/discovery/Functions.java | 10 +- .../beacon/discovery/NodeSession.java | 25 +- .../ethereum/beacon/discovery/RlpUtil.java | 52 ++++ .../discovery/enr/NodeRecordFactory.java | 10 + .../packet/AuthHeaderMessagePacket.java | 39 +-- .../beacon/discovery/packet/RandomPacket.java | 14 +- .../discovery/packet/WhoAreYouPacket.java | 10 +- .../AuthHeaderMessagePacketHandler.java | 33 +-- .../pipeline/handler/NodeIdToSession.java | 7 +- .../pipeline/handler/TaskHandler.java | 9 +- .../handler/WhoAreYouPacketHandler.java | 33 ++- .../discovery/task/TaskMessageFactory.java | 9 +- .../discovery/DiscoveryNetworkTest.java | 10 +- .../discovery/DiscoveryNoNetworkTest.java | 8 +- .../beacon/discovery/FunctionsTest.java | 61 ++-- .../discovery/HandshakeHandlersTest.java | 278 ++++++++++++++++++ .../mock/DiscoveryManagerNoNetwork.java | 13 +- 18 files changed, 528 insertions(+), 107 deletions(-) create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/RlpUtil.java create mode 100644 discovery/src/test/java/org/ethereum/beacon/discovery/HandshakeHandlersTest.java diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java index cbe5ad6b3..99e2d8dc6 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java @@ -18,16 +18,16 @@ import org.ethereum.beacon.discovery.pipeline.handler.IncomingDataPacker; import org.ethereum.beacon.discovery.pipeline.handler.MessageHandler; import org.ethereum.beacon.discovery.pipeline.handler.MessagePacketHandler; -import org.ethereum.beacon.discovery.pipeline.handler.NodeSessionRequestHandler; import org.ethereum.beacon.discovery.pipeline.handler.NodeIdToSession; +import org.ethereum.beacon.discovery.pipeline.handler.NodeSessionRequestHandler; import org.ethereum.beacon.discovery.pipeline.handler.NotExpectedIncomingPacketHandler; import org.ethereum.beacon.discovery.pipeline.handler.OutgoingParcelHandler; import org.ethereum.beacon.discovery.pipeline.handler.TaskHandler; import org.ethereum.beacon.discovery.pipeline.handler.UnknownPacketTagToSender; import org.ethereum.beacon.discovery.pipeline.handler.UnknownPacketTypeByStatus; import org.ethereum.beacon.discovery.pipeline.handler.WhoAreYouAttempt; -import org.ethereum.beacon.discovery.pipeline.handler.WhoAreYouSessionResolver; import org.ethereum.beacon.discovery.pipeline.handler.WhoAreYouPacketHandler; +import org.ethereum.beacon.discovery.pipeline.handler.WhoAreYouSessionResolver; import org.ethereum.beacon.discovery.storage.AuthTagRepository; import org.ethereum.beacon.discovery.storage.NodeBucketStorage; import org.ethereum.beacon.discovery.storage.NodeTable; @@ -38,6 +38,7 @@ import reactor.core.publisher.FluxSink; import reactor.core.publisher.ReplayProcessor; import tech.pegasys.artemis.util.bytes.Bytes4; +import tech.pegasys.artemis.util.bytes.BytesValue; import java.security.SecureRandom; import java.util.concurrent.CompletableFuture; @@ -56,6 +57,7 @@ public DiscoveryManagerImpl( NodeTable nodeTable, NodeBucketStorage nodeBucketStorage, NodeRecord homeNode, + BytesValue homeNodePrivateKey, Scheduler serverScheduler, Scheduler clientScheduler) { AuthTagRepository authTagRepo = new AuthTagRepository(); @@ -66,7 +68,13 @@ public DiscoveryManagerImpl( (int) homeNode.get(NodeRecord.FIELD_UDP_V4)); this.discoveryClient = new DiscoveryClientImpl(outgoingMessages, clientScheduler); NodeIdToSession nodeIdToSession = - new NodeIdToSession(homeNode, nodeBucketStorage, authTagRepo, nodeTable, outgoingPipeline); + new NodeIdToSession( + homeNode, + homeNodePrivateKey, + nodeBucketStorage, + authTagRepo, + nodeTable, + outgoingPipeline); incomingPipeline .addHandler(new IncomingDataPacker()) .addHandler(new WhoAreYouAttempt(homeNode.getNodeId())) diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java b/discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java index 6cd3fbafc..bcbf25d8e 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java @@ -16,7 +16,7 @@ import tech.pegasys.artemis.util.bytes.BytesValue; import javax.crypto.Cipher; -import javax.crypto.spec.IvParameterSpec; +import javax.crypto.spec.GCMParameterSpec; import javax.crypto.spec.SecretKeySpec; import java.math.BigInteger; import java.security.SecureRandom; @@ -77,11 +77,11 @@ public static BytesValue recoverFromSignature(BytesValue signature, BytesValue x public static BytesValue aesgcm_encrypt( BytesValue privateKey, BytesValue nonce, BytesValue message, BytesValue aad) { try { - Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding", "BC"); + Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding"); cipher.init( Cipher.ENCRYPT_MODE, new SecretKeySpec(privateKey.extractArray(), "AES"), - new IvParameterSpec(nonce.extractArray())); + new GCMParameterSpec(128, nonce.extractArray())); cipher.updateAAD(aad.extractArray()); return BytesValue.wrap(cipher.doFinal(message.extractArray())); } catch (Exception e) { @@ -92,11 +92,11 @@ public static BytesValue aesgcm_encrypt( public static BytesValue aesgcm_decrypt( BytesValue privateKey, BytesValue nonce, BytesValue encoded, BytesValue aad) { try { - Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding", "BC"); + Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding"); cipher.init( Cipher.DECRYPT_MODE, new SecretKeySpec(privateKey.extractArray(), "AES"), - new IvParameterSpec(nonce.extractArray())); + new GCMParameterSpec(128, nonce.extractArray())); cipher.updateAAD(aad.extractArray()); return BytesValue.wrap(cipher.doFinal(encoded.extractArray())); } catch (Exception e) { diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeSession.java b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeSession.java index c744787b3..575bb3ca4 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeSession.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeSession.java @@ -5,11 +5,11 @@ import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.message.MessageCode; import org.ethereum.beacon.discovery.packet.Packet; -import org.ethereum.beacon.discovery.task.TaskType; import org.ethereum.beacon.discovery.storage.AuthTagRepository; import org.ethereum.beacon.discovery.storage.NodeBucket; import org.ethereum.beacon.discovery.storage.NodeBucketStorage; import org.ethereum.beacon.discovery.storage.NodeTable; +import org.ethereum.beacon.discovery.task.TaskType; import tech.pegasys.artemis.util.bytes.Bytes32; import tech.pegasys.artemis.util.bytes.BytesValue; @@ -22,6 +22,7 @@ import java.util.function.Consumer; public class NodeSession { + public static final int NONCE_SIZE = 12; private static final Logger logger = LogManager.getLogger(NodeSession.class); private final NodeRecord nodeRecord; private final NodeRecord homeNodeRecord; @@ -34,13 +35,16 @@ public class NodeSession { private SessionStatus status = SessionStatus.INITIAL; private Bytes32 idNonce; private BytesValue initiatorKey; + private BytesValue recipientKey; private Map requestIdReservations = new ConcurrentHashMap<>(); private CompletableFuture completableFuture = null; private TaskType task = null; + private BytesValue staticNodeKey; public NodeSession( NodeRecord nodeRecord, NodeRecord homeNodeRecord, + BytesValue staticNodeKey, NodeTable nodeTable, NodeBucketStorage nodeBucketStorage, AuthTagRepository authTagRepo, @@ -52,6 +56,7 @@ public NodeSession( this.nodeTable = nodeTable; this.nodeBucketStorage = nodeBucketStorage; this.homeNodeRecord = homeNodeRecord; + this.staticNodeKey = staticNodeKey; this.homeNodeId = homeNodeRecord.getNodeId(); this.rnd = rnd; } @@ -89,6 +94,12 @@ public synchronized BytesValue getNextRequestId(MessageCode messageCode) { return wrapped; } + public synchronized BytesValue generateNonce() { + byte[] nonce = new byte[NONCE_SIZE]; + rnd.nextBytes(nonce); + return BytesValue.wrap(nonce); + } + public synchronized boolean isAuthenticated() { return SessionStatus.AUTHENTICATED.equals(status); } @@ -117,6 +128,14 @@ public void setInitiatorKey(BytesValue initiatorKey) { this.initiatorKey = initiatorKey; } + public BytesValue getRecipientKey() { + return recipientKey; + } + + public void setRecipientKey(BytesValue recipientKey) { + this.recipientKey = recipientKey; + } + public synchronized void clearRequestId(BytesValue requestId, MessageCode messageCode) { assert requestIdReservations.remove(requestId).equals(messageCode); } @@ -186,6 +205,10 @@ public TaskType loadTask() { return task; } + public BytesValue getStaticNodeKey() { + return staticNodeKey; + } + public enum SessionStatus { INITIAL, // other side is trying to connect, or we are initiating (before random packet is sent WHOAREYOU_SENT, // other side is initiator, we've sent whoareyou in response diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/RlpUtil.java b/discovery/src/main/java/org/ethereum/beacon/discovery/RlpUtil.java new file mode 100644 index 000000000..91eebb0e6 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/RlpUtil.java @@ -0,0 +1,52 @@ +package org.ethereum.beacon.discovery; + +import org.javatuples.Pair; +import org.web3j.rlp.RlpDecoder; +import org.web3j.rlp.RlpList; +import tech.pegasys.artemis.util.bytes.Bytes8; +import tech.pegasys.artemis.util.bytes.BytesValue; +import tech.pegasys.artemis.util.uint.UInt64; + +import static org.web3j.rlp.RlpDecoder.OFFSET_LONG_LIST; +import static org.web3j.rlp.RlpDecoder.OFFSET_SHORT_LIST; + +public class RlpUtil { + public static int calcListLen(BytesValue data) { + int prefix = data.get(0) & 0xFF; + int prefixAddon = 1; + if (prefix >= OFFSET_SHORT_LIST && prefix <= OFFSET_LONG_LIST) { + + // 4. the data is a list if the range of the + // first byte is [0xc0, 0xf7], and the concatenation of + // the RLP encodings of all items of the list which the + // total payload is equal to the first byte minus 0xc0 follows the first byte; + + byte listLen = (byte) (prefix - OFFSET_SHORT_LIST); + return listLen & 0xFF + prefixAddon; + } else if (prefix > OFFSET_LONG_LIST) { + + // 5. the data is a list if the range of the + // first byte is [0xf8, 0xff], and the total payload of the + // list which length is equal to the + // first byte minus 0xf7 follows the first byte, + // and the concatenation of the RLP encodings of all items of + // the list follows the total payload of the list; + + int lenOfListLen = (prefix - OFFSET_LONG_LIST) & 0xFF; + prefixAddon += lenOfListLen; + return UInt64.fromBytesBigEndian(Bytes8.leftPad(data.slice(1, lenOfListLen & 0xFF))) + .intValue() + + prefixAddon; + } else { + throw new RuntimeException("Not a start of RLP list!!"); + } + } + + /** + * @return first rlp list in provided data, plus remaining data starting from the end of this list + */ + public static Pair decodeFirstList(BytesValue data) { + int len = RlpUtil.calcListLen(data); + return Pair.with(RlpDecoder.decode(data.slice(0, len).extractArray()), data.slice(len)); + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordFactory.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordFactory.java index 4471e673b..491f03507 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordFactory.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordFactory.java @@ -9,6 +9,7 @@ import tech.pegasys.artemis.util.bytes.BytesValue; import tech.pegasys.artemis.util.uint.UInt64; +import java.util.Arrays; import java.util.Base64; import java.util.HashMap; import java.util.List; @@ -25,6 +26,15 @@ public NodeRecordFactory(EnrSchemeInterpreter... enrSchemeInterpreters) { } } + @SafeVarargs + public final NodeRecord createFromValues( + EnrScheme enrScheme, + UInt64 seq, + BytesValue signature, + Pair... fieldKeyPairs) { + return createFromValues(enrScheme, seq, signature, Arrays.asList(fieldKeyPairs)); + } + public NodeRecord createFromValues( EnrScheme enrScheme, UInt64 seq, diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/AuthHeaderMessagePacket.java b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/AuthHeaderMessagePacket.java index 99ed4c96e..6aa1f0ad3 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/AuthHeaderMessagePacket.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/AuthHeaderMessagePacket.java @@ -1,10 +1,13 @@ package org.ethereum.beacon.discovery.packet; import org.ethereum.beacon.discovery.Functions; +import org.ethereum.beacon.discovery.RlpUtil; import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.enr.NodeRecordFactory; import org.ethereum.beacon.discovery.message.DiscoveryMessage; import org.ethereum.beacon.discovery.message.DiscoveryV5Message; +import org.javatuples.Pair; +import org.javatuples.Triplet; import org.web3j.rlp.RlpDecoder; import org.web3j.rlp.RlpEncoder; import org.web3j.rlp.RlpList; @@ -14,6 +17,7 @@ import tech.pegasys.artemis.util.bytes.BytesValue; import java.math.BigInteger; +import java.util.function.Function; /** * Used as first encrypted message sent in response to WHOAREYOU {@link WhoAreYouPacket}. Contains @@ -63,18 +67,17 @@ public static AuthHeaderMessagePacket create( DiscoveryMessage message) { BytesValue tag = Packet.createTag(homeNodeId, destNodeId); BytesValue idNonceSig = - Functions.sign(staticNodeKey, Functions.hash(DISCOVERY_ID_NONCE.concat(idNonce))); + Functions.sign( + staticNodeKey, + Functions.hash(DISCOVERY_ID_NONCE.concat(idNonce).concat(ephemeralPubkey))); + idNonceSig = idNonceSig.slice(1); // Remove recovery id byte[] authResponsePt = RlpEncoder.encode( new RlpList( RlpString.create(5), RlpString.create(idNonceSig.extractArray()), RlpString.create( - nodeRecord - .serialize() - .extractArray()) // FIXME: record of sender OR [] if enr-seq in WHOAREYOU != - // current seq - )); + nodeRecord == null ? null : nodeRecord.serialize().extractArray()))); BytesValue authResponse = Functions.aesgcm_encrypt( authResponseKey, ZERO_NONCE, BytesValue.wrap(authResponsePt), BytesValue.EMPTY); @@ -91,11 +94,9 @@ public static AuthHeaderMessagePacket create( return new AuthHeaderMessagePacket(tag.concat(authHeader).concat(encryptedData)); } - public void verify(BytesValue expectedAuthTag, BytesValue expectedIdNonce) { + public void verify(BytesValue expectedIdNonce) { verifyDecode(); - assert expectedAuthTag.equals(getAuthTag()); assert expectedIdNonce.equals(getIdNonce()); - // TODO: verify signature } public Bytes32 getHomeNodeId(Bytes32 destNodeId) { @@ -139,15 +140,15 @@ private void verifyDecode() { } } - public void decode(BytesValue initiatorKey, BytesValue authResponseKey) { + public void decode(Function> ephemeralPubToKeysFunction) { if (decoded != null) { return; } MessagePacketDecoded blank = new MessagePacketDecoded(); blank.tag = Bytes32.wrap(getBytes().slice(0, 32), 0); - RlpList authHeaderParts = - (RlpList) RlpDecoder.decode(getBytes().slice(32).extractArray()).getValues().get(0); - int rlpLength = RlpEncoder.encode(authHeaderParts).length; // FIXME: bad hack + Pair decodeRes = RlpUtil.decodeFirstList(getBytes().slice(32)); + int authHeaderLength = getBytes().size() - 32 - decodeRes.getValue1().size(); + RlpList authHeaderParts = (RlpList) decodeRes.getValue0().getValues().get(0); // [auth-tag, id-nonce, auth-scheme-name, ephemeral-pubkey, auth-response] blank.authTag = BytesValue.wrap(((RlpString) authHeaderParts.getValues().get(0)).getBytes()); blank.idNonce = BytesValue.wrap(((RlpString) authHeaderParts.getValues().get(1)).getBytes()); @@ -155,24 +156,26 @@ public void decode(BytesValue initiatorKey, BytesValue authResponseKey) { new String(((RlpString) authHeaderParts.getValues().get(2)).getBytes())); blank.ephemeralPubkey = BytesValue.wrap(((RlpString) authHeaderParts.getValues().get(3)).getBytes()); + Triplet keys = ephemeralPubToKeysFunction.apply(blank.ephemeralPubkey); BytesValue authResponse = BytesValue.wrap(((RlpString) authHeaderParts.getValues().get(4)).getBytes()); + BytesValue authResponsePt = - Functions.aesgcm_decrypt(authResponseKey, ZERO_NONCE, authResponse, BytesValue.EMPTY); + Functions.aesgcm_decrypt(keys.getValue2(), ZERO_NONCE, authResponse, BytesValue.EMPTY); RlpList authResponsePtParts = (RlpList) RlpDecoder.decode(authResponsePt.extractArray()).getValues().get(0); assert BigInteger.valueOf(5) .equals(((RlpString) authResponsePtParts.getValues().get(0)).asPositiveBigInteger()); blank.idNonceSig = BytesValue.wrap(((RlpString) authResponsePtParts.getValues().get(1)).getBytes()); + byte[] nodeRecordBytes = ((RlpString) authResponsePtParts.getValues().get(2)).getBytes(); blank.nodeRecord = - nodeRecordFactory.fromBytes( - ((RlpString) authResponsePtParts.getValues().get(2)).getBytes()); - BytesValue messageAad = blank.tag.concat(getBytes().slice(32)); + nodeRecordBytes.length == 0 ? null : nodeRecordFactory.fromBytes(nodeRecordBytes); + BytesValue messageAad = blank.tag.concat(getBytes().slice(32, authHeaderLength)); blank.message = new DiscoveryV5Message( Functions.aesgcm_decrypt( - initiatorKey, blank.authTag, getBytes().slice(32 + rlpLength), messageAad)); + keys.getValue0(), blank.authTag, decodeRes.getValue1(), messageAad)); this.decoded = blank; } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/RandomPacket.java b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/RandomPacket.java index c0c5f483b..4b1be696d 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/RandomPacket.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/RandomPacket.java @@ -28,8 +28,8 @@ public RandomPacket(BytesValue bytes) { public static RandomPacket create( Bytes32 homeNodeId, Bytes32 destNodeId, BytesValue authTag, Random rnd) { BytesValue tag = Packet.createTag(homeNodeId, destNodeId); - byte[] authTagBytesRlp = RlpEncoder.encode(RlpString.create(authTag.extractArray())); - BytesValue authTagEncoded = BytesValue.wrap(authTagBytesRlp); + byte[] authTagRlp = RlpEncoder.encode(RlpString.create(authTag.extractArray())); + BytesValue authTagEncoded = BytesValue.wrap(authTagRlp); byte[] randomBytes = new byte[44]; rnd.nextBytes(randomBytes); // at least 44 bytes of random data return new RandomPacket(tag.concat(authTagEncoded).concat(BytesValue.wrap(randomBytes))); @@ -51,17 +51,17 @@ private synchronized void decode() { } RandomPacketDecoded blank = new RandomPacketDecoded(); blank.tag = Bytes32.wrap(getBytes().slice(0, 32), 0); - blank.authTag = BytesValue.wrap(((RlpString) RlpDecoder.decode(getBytes().slice(32).extractArray()).getValues().get(0)).getBytes()); + blank.authTag = + BytesValue.wrap( + ((RlpString) RlpDecoder.decode(getBytes().slice(32).extractArray()).getValues().get(0)) + .getBytes()); this.decoded = blank; } @Override public String toString() { if (decoded != null) { - return "RandomPacket{" + - "tag=" + decoded.tag + - ", authTag=" + decoded.authTag + - '}'; + return "RandomPacket{" + "tag=" + decoded.tag + ", authTag=" + decoded.authTag + '}'; } else { return "RandomPacket{" + getBytes() + '}'; } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/WhoAreYouPacket.java b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/WhoAreYouPacket.java index d0476ac6e..fcba30e31 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/WhoAreYouPacket.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/WhoAreYouPacket.java @@ -56,7 +56,7 @@ public Bytes32 getIdNonce() { return decoded.idNonce; } - public Long getEnrSeq() { + public UInt64 getEnrSeq() { decode(); return decoded.enrSeq; } @@ -78,10 +78,8 @@ private synchronized void decode() { blank.authTag = BytesValue.wrap(((RlpString) payload.getValues().get(0)).getBytes()); blank.idNonce = Bytes32.wrap(((RlpString) payload.getValues().get(1)).getBytes()); blank.enrSeq = - UInt64.fromBytesLittleEndian( - Bytes8.rightPad( - BytesValue.wrap(((RlpString) payload.getValues().get(2)).getBytes()))) - .getValue(); + UInt64.fromBytesBigEndian( + Bytes8.leftPad(BytesValue.wrap(((RlpString) payload.getValues().get(2)).getBytes()))); this.decoded = blank; } @@ -107,6 +105,6 @@ private static class WhoAreYouDecoded { private Bytes32 magic; private BytesValue authTag; private Bytes32 idNonce; - private Long enrSeq; + private UInt64 enrSeq; } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/AuthHeaderMessagePacketHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/AuthHeaderMessagePacketHandler.java index cb4fb276b..42e87f6ff 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/AuthHeaderMessagePacketHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/AuthHeaderMessagePacketHandler.java @@ -4,7 +4,6 @@ import org.apache.logging.log4j.Logger; import org.ethereum.beacon.discovery.Functions; import org.ethereum.beacon.discovery.NodeSession; -import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.message.DiscoveryV5Message; import org.ethereum.beacon.discovery.message.MessageCode; import org.ethereum.beacon.discovery.packet.AuthHeaderMessagePacket; @@ -13,7 +12,6 @@ import org.ethereum.beacon.discovery.pipeline.Field; import org.ethereum.beacon.discovery.task.TaskType; import org.javatuples.Triplet; -import org.web3j.crypto.ECKeyPair; import tech.pegasys.artemis.util.bytes.BytesValue; import java.util.concurrent.CompletableFuture; @@ -35,23 +33,22 @@ public void handle(Envelope envelope) { AuthHeaderMessagePacket packet = (AuthHeaderMessagePacket) envelope.get(Field.PACKET_AUTH_HEADER_MESSAGE); NodeSession session = (NodeSession) envelope.get(Field.SESSION); - try { - byte[] ephemeralKeyBytes = new byte[32]; - Functions.getRandom().nextBytes(ephemeralKeyBytes); - ECKeyPair ephemeralKey = ECKeyPair.create(ephemeralKeyBytes); - Triplet hkdf = - Functions.hkdf_expand( - session.getHomeNodeId(), - session.getNodeRecord().getNodeId(), - BytesValue.wrap(ephemeralKey.getPrivateKey().toByteArray()), - (BytesValue) session.getNodeRecord().get(NodeRecord.FIELD_PKEY_SECP256K1), - session.getIdNonce()); - BytesValue initiatorKey = hkdf.getValue0(); - session.setInitiatorKey(initiatorKey); - BytesValue authResponseKey = hkdf.getValue2(); - packet.decode(initiatorKey, authResponseKey); - packet.verify(session.getAuthTag().get(), session.getIdNonce()); + // FIXME: make this logic without side-effect + packet.decode( + ephemeralPubKey -> { + Triplet hkdf = + Functions.hkdf_expand( + session.getNodeRecord().getNodeId(), + session.getHomeNodeId(), + session.getStaticNodeKey(), + ephemeralPubKey, + session.getIdNonce()); + session.setInitiatorKey(hkdf.getValue0()); + session.setRecipientKey(hkdf.getValue1()); + return hkdf; + }); + packet.verify(session.getIdNonce()); envelope.put(Field.MESSAGE, packet.getMessage()); } catch (AssertionError ex) { logger.info( diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeIdToSession.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeIdToSession.java index 340c34fab..5d7dcfa03 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeIdToSession.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeIdToSession.java @@ -2,8 +2,8 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.ethereum.beacon.discovery.NodeSession; import org.ethereum.beacon.discovery.NodeRecordInfo; +import org.ethereum.beacon.discovery.NodeSession; import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.network.NetworkParcelV5; import org.ethereum.beacon.discovery.pipeline.Envelope; @@ -16,6 +16,7 @@ import org.ethereum.beacon.util.ExpirationScheduler; import org.javatuples.Pair; import tech.pegasys.artemis.util.bytes.Bytes32; +import tech.pegasys.artemis.util.bytes.BytesValue; import java.security.SecureRandom; import java.util.Map; @@ -31,6 +32,7 @@ public class NodeIdToSession implements EnvelopeHandler { private static final int CLEANUP_DELAY_SECONDS = 180; private static final Logger logger = LogManager.getLogger(NodeIdToSession.class); private final NodeRecord homeNodeRecord; + private final BytesValue staticNodeKey; private final NodeBucketStorage nodeBucketStorage; private final AuthTagRepository authTagRepo; private final Map recentSessions = @@ -42,11 +44,13 @@ public class NodeIdToSession implements EnvelopeHandler { public NodeIdToSession( NodeRecord homeNodeRecord, + BytesValue staticNodeKey, NodeBucketStorage nodeBucketStorage, AuthTagRepository authTagRepo, NodeTable nodeTable, Pipeline outgoingPipeline) { this.homeNodeRecord = homeNodeRecord; + this.staticNodeKey = staticNodeKey; this.nodeBucketStorage = nodeBucketStorage; this.authTagRepo = authTagRepo; this.nodeTable = nodeTable; @@ -89,6 +93,7 @@ private Optional getSession(Bytes32 nodeId) { new NodeSession( nodeRecord, homeNodeRecord, + staticNodeKey, nodeTable, nodeBucketStorage, authTagRepo, diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/TaskHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/TaskHandler.java index 6941b6e02..5ce28c3b6 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/TaskHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/TaskHandler.java @@ -29,10 +29,9 @@ public void handle(Envelope envelope) { NodeSession session = (NodeSession) envelope.get(Field.SESSION); CompletableFuture completableFuture = (CompletableFuture) envelope.get(Field.FUTURE); + // FIXME: this logic shouldbn't be here!!!!11 + BytesValue authTag = session.generateNonce(); if (session.getStatus().equals(NodeSession.SessionStatus.INITIAL)) { - byte[] authTagBytes = new byte[12]; - rnd.nextBytes(authTagBytes); - BytesValue authTag = BytesValue.wrap(authTagBytes); RandomPacket randomPacket = RandomPacket.create( session.getHomeNodeId(), @@ -45,11 +44,11 @@ public void handle(Envelope envelope) { session.saveFuture(completableFuture); } else if (session.getStatus().equals(NodeSession.SessionStatus.AUTHENTICATED)) { if (envelope.get(Field.TASK).equals(TaskType.PING)) { - session.sendOutgoing(TaskMessageFactory.createPingPacket(session)); + session.sendOutgoing(TaskMessageFactory.createPingPacket(authTag, session)); session.saveTask((TaskType) envelope.get(Field.TASK)); session.saveFuture(completableFuture); } else if (envelope.get(Field.TASK).equals(TaskType.FINDNODE)) { - session.sendOutgoing(TaskMessageFactory.createFindNodePacket(session)); + session.sendOutgoing(TaskMessageFactory.createFindNodePacket(authTag, session)); session.saveTask((TaskType) envelope.get(Field.TASK)); session.saveFuture(completableFuture); } else { diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouPacketHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouPacketHandler.java index c184e13ae..491168372 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouPacketHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouPacketHandler.java @@ -20,6 +20,8 @@ import java.util.concurrent.CompletableFuture; +import static org.ethereum.beacon.discovery.enr.NodeRecord.FIELD_PKEY_SECP256K1; + /** Handles {@link WhoAreYouPacket} in {@link Field#PACKET_WHOAREYOU} field */ public class WhoAreYouPacketHandler implements EnvelopeHandler { private static final Logger logger = LogManager.getLogger(WhoAreYouPacketHandler.class); @@ -35,21 +37,26 @@ public void handle(Envelope envelope) { WhoAreYouPacket packet = (WhoAreYouPacket) envelope.get(Field.PACKET_WHOAREYOU); NodeSession session = (NodeSession) envelope.get(Field.SESSION); try { - BytesValue authTag = session.getAuthTag().get(); - packet.verify(session.getHomeNodeId(), authTag); - packet.getEnrSeq(); // FIXME: Their side enr seq. Do we need it? + NodeRecord respRecord = null; + if (packet.getEnrSeq().compareTo(session.getHomeNodeRecord().getSeq()) < 0) { + respRecord = session.getHomeNodeRecord(); + } + BytesValue remotePubKey = (BytesValue) session.getNodeRecord().getKey(FIELD_PKEY_SECP256K1); byte[] ephemeralKeyBytes = new byte[32]; Functions.getRandom().nextBytes(ephemeralKeyBytes); - ECKeyPair ephemeralKey = ECKeyPair.create(ephemeralKeyBytes); // TODO: generate + ECKeyPair ephemeralKey = ECKeyPair.create(ephemeralKeyBytes); + Triplet hkdf = Functions.hkdf_expand( session.getHomeNodeId(), session.getNodeRecord().getNodeId(), - BytesValue.wrap(ephemeralKey.getPrivateKey().toByteArray()), - (BytesValue) session.getNodeRecord().get(NodeRecord.FIELD_PKEY_SECP256K1), + BytesValue.wrap(ephemeralKeyBytes), + remotePubKey, packet.getIdNonce()); BytesValue initiatorKey = hkdf.getValue0(); - BytesValue staticNodeKey = hkdf.getValue1(); + BytesValue recipientKey = hkdf.getValue1(); + session.setInitiatorKey(initiatorKey); + session.setRecipientKey(recipientKey); BytesValue authResponseKey = hkdf.getValue2(); V5Message taskMessage = null; if (session.loadTask() == TaskType.PING) { @@ -62,16 +69,20 @@ public void handle(Envelope envelope) { "Type %s in envelope #%s is not known", session.loadTask(), envelope.getId())); } + BytesValue ephemeralPubKey = BytesValue.wrap(ephemeralKey.getPublicKey().toByteArray()); + if (ephemeralPubKey.size() == 65) { + ephemeralPubKey = ephemeralPubKey.slice(1); // slice leading 00 + } AuthHeaderMessagePacket response = AuthHeaderMessagePacket.create( session.getHomeNodeId(), session.getNodeRecord().getNodeId(), authResponseKey, packet.getIdNonce(), - staticNodeKey, - session.getHomeNodeRecord(), - BytesValue.wrap(ephemeralKey.getPublicKey().toByteArray()), - authTag, + session.getStaticNodeKey(), + respRecord, + ephemeralPubKey, + session.generateNonce(), initiatorKey, DiscoveryV5Message.from(taskMessage)); session.sendOutgoing(response); diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/task/TaskMessageFactory.java b/discovery/src/main/java/org/ethereum/beacon/discovery/task/TaskMessageFactory.java index 102c5d153..977b24431 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/task/TaskMessageFactory.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/task/TaskMessageFactory.java @@ -6,16 +6,17 @@ import org.ethereum.beacon.discovery.message.MessageCode; import org.ethereum.beacon.discovery.message.PingMessage; import org.ethereum.beacon.discovery.packet.MessagePacket; +import tech.pegasys.artemis.util.bytes.BytesValue; public class TaskMessageFactory { public static final int DEFAULT_DISTANCE = 10; - public static MessagePacket createPingPacket(NodeSession session) { + public static MessagePacket createPingPacket(BytesValue authTag, NodeSession session) { return MessagePacket.create( session.getHomeNodeId(), session.getNodeRecord().getNodeId(), - session.getAuthTag().get(), + authTag, session.getInitiatorKey(), DiscoveryV5Message.from(createPing(session))); } @@ -25,12 +26,12 @@ public static PingMessage createPing(NodeSession session) { session.getNextRequestId(MessageCode.PING), session.getNodeRecord().getSeq()); } - public static MessagePacket createFindNodePacket(NodeSession session) { + public static MessagePacket createFindNodePacket(BytesValue authTag, NodeSession session) { FindNodeMessage findNodeMessage = createFindNode(session); return MessagePacket.create( session.getHomeNodeId(), session.getNodeRecord().getNodeId(), - session.getAuthTag().get(), + authTag, session.getInitiatorKey(), DiscoveryV5Message.from(findNodeMessage)); } diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java index 4f087b7b4..7e2d166be 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java @@ -13,10 +13,10 @@ import org.ethereum.beacon.discovery.packet.RandomPacket; import org.ethereum.beacon.discovery.packet.UnknownPacket; import org.ethereum.beacon.discovery.packet.WhoAreYouPacket; -import org.ethereum.beacon.discovery.task.TaskType; import org.ethereum.beacon.discovery.storage.NodeBucketStorage; import org.ethereum.beacon.discovery.storage.NodeTableStorage; import org.ethereum.beacon.discovery.storage.NodeTableStorageFactoryImpl; +import org.ethereum.beacon.discovery.task.TaskType; import org.ethereum.beacon.schedulers.Schedulers; import org.javatuples.Pair; import org.junit.Test; @@ -31,12 +31,16 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import static org.ethereum.beacon.discovery.task.TaskMessageFactory.DEFAULT_DISTANCE; import static org.ethereum.beacon.discovery.storage.NodeTableStorage.DEFAULT_SERIALIZER; +import static org.ethereum.beacon.discovery.task.TaskMessageFactory.DEFAULT_DISTANCE; /** Same as {@link DiscoveryNoNetworkTest} but using real network */ public class DiscoveryNetworkTest { private static final NodeRecordFactory NODE_RECORD_FACTORY = NodeRecordFactory.DEFAULT; + private final BytesValue testKey1 = + BytesValue.fromHexString("eef77acb6c6a6eebc5b363a475ac583ec7eccdb42b6481424c60f59aa326547f"); + private final BytesValue testKey2 = + BytesValue.fromHexString("66fb62bfbd66b9177a138c1e5cddbe4f7c30c343e94e68df8769459cb1cde628"); @Test public void test() throws Exception { @@ -115,6 +119,7 @@ public void test() throws Exception { nodeTableStorage1.get(), nodeBucketStorage1, nodeRecord1, + testKey1, Schedulers.createDefault().newSingleThreadDaemon("server-1"), Schedulers.createDefault().newSingleThreadDaemon("client-1")); DiscoveryManagerImpl discoveryManager2 = @@ -122,6 +127,7 @@ public void test() throws Exception { nodeTableStorage2.get(), nodeBucketStorage2, nodeRecord2, + testKey2, Schedulers.createDefault().newSingleThreadDaemon("server-2"), Schedulers.createDefault().newSingleThreadDaemon("client-2")); diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java index 92b140c98..9a9b8fb95 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java @@ -42,6 +42,10 @@ */ public class DiscoveryNoNetworkTest { private static final NodeRecordFactory NODE_RECORD_FACTORY = NodeRecordFactory.DEFAULT; + private final BytesValue testKey1 = + BytesValue.fromHexString("eef77acb6c6a6eebc5b363a475ac583ec7eccdb42b6481424c60f59aa326547f"); + private final BytesValue testKey2 = + BytesValue.fromHexString("66fb62bfbd66b9177a138c1e5cddbe4f7c30c343e94e68df8769459cb1cde628"); @Test public void test() throws Exception { @@ -123,10 +127,10 @@ public void test() throws Exception { Schedulers.createDefault().newSingleThreadDaemon("from2to1-thread"), "from2to1"); DiscoveryManagerNoNetwork discoveryManager1 = new DiscoveryManagerNoNetwork( - nodeTableStorage1.get(), nodeBucketStorage1, nodeRecord1, from2to1); + nodeTableStorage1.get(), nodeBucketStorage1, nodeRecord1, testKey1, from2to1); DiscoveryManagerNoNetwork discoveryManager2 = new DiscoveryManagerNoNetwork( - nodeTableStorage2.get(), nodeBucketStorage2, nodeRecord2, from1to2); + nodeTableStorage2.get(), nodeBucketStorage2, nodeRecord2, testKey2, from1to2); discoveryManager1.start(); discoveryManager2.start(); diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/FunctionsTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/FunctionsTest.java index 6d78cebc0..f502af4d1 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/FunctionsTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/FunctionsTest.java @@ -9,6 +9,22 @@ import static org.junit.Assert.assertEquals; public class FunctionsTest { + private final BytesValue testKey1 = + BytesValue.fromHexString("3332ca2b7003810449b6e596c3d284e914a1a51c9f76e4d9d7d43ef84adf6ed6"); + private final BytesValue testKey2 = + BytesValue.fromHexString("66fb62bfbd66b9177a138c1e5cddbe4f7c30c343e94e68df8769459cb1cde628"); + private Bytes32 nodeId1; + private Bytes32 nodeId2; + + public FunctionsTest() { + byte[] homeNodeIdBytes = new byte[32]; + homeNodeIdBytes[0] = 0x01; + byte[] destNodeIdBytes = new byte[32]; + destNodeIdBytes[0] = 0x02; + this.nodeId1 = Bytes32.wrap(homeNodeIdBytes); + this.nodeId2 = Bytes32.wrap(destNodeIdBytes); + } + @Test public void testLogDistance() { Bytes32 nodeId0 = @@ -37,33 +53,36 @@ public void testLogDistance() { @Test public void hkdfExpandTest() { - BytesValue testKeyA = - BytesValue.fromHexString( - "eef77acb6c6a6eebc5b363a475ac583ec7eccdb42b6481424c60f59aa326547f"); - BytesValue testKeyB = - BytesValue.fromHexString( - "66fb62bfbd66b9177a138c1e5cddbe4f7c30c343e94e68df8769459cb1cde628"); - BytesValue idNonce = Bytes32.ZERO; - byte[] homeNodeIdBytes = new byte[32]; - homeNodeIdBytes[0] = 0x01; - byte[] destNodeIdBytes = new byte[32]; - destNodeIdBytes[0] = 0x02; - BytesValue homeNodeId = BytesValue.wrap(homeNodeIdBytes); - BytesValue destNodeId = BytesValue.wrap(destNodeIdBytes); + BytesValue idNonce = + Bytes32.fromHexString("68b02a985ecb99cc2d10cf188879d93ae7684c4f4707770017b078c6497c5a5d"); Triplet sec1 = Functions.hkdf_expand( - homeNodeId, - destNodeId, - testKeyA, - BytesValue.wrap(ECKeyPair.create(testKeyB.extractArray()).getPublicKey().toByteArray()), + nodeId1, + nodeId2, + testKey1, + BytesValue.wrap(ECKeyPair.create(testKey2.extractArray()).getPublicKey().toByteArray()), idNonce); Triplet sec2 = Functions.hkdf_expand( - homeNodeId, - destNodeId, - testKeyB, - BytesValue.wrap(ECKeyPair.create(testKeyA.extractArray()).getPublicKey().toByteArray()), + nodeId1, + nodeId2, + testKey2, + BytesValue.wrap(ECKeyPair.create(testKey1.extractArray()).getPublicKey().toByteArray()), idNonce); assertEquals(sec1, sec2); } + + @Test + public void testGcmSimple() { + BytesValue authResponseKey = BytesValue.fromHexString("0x60bfc5c924a8d640f47df8b781f5a0e5"); + BytesValue authResponsePt = + BytesValue.fromHexString( + "0xf8aa05b8404f5fa8309cab170dbeb049de504b519288777aae0c4b25686f82310206a4a1e264dc6e8bfaca9187e8b3dbb56f49c7aa3d22bff3a279bf38fb00cb158b7b8ca7b865f86380018269648276348375647082765f826970847f00000189736563703235366b31b84013d14211e0287b2361a1615890a9b5212080546d0a257ae4cff96cf534992cb97e6adeb003652e807c7f2fe843e0c48d02d4feb0272e2e01f6e27915a431e773"); + BytesValue zeroNonce = BytesValue.wrap(new byte[12]); + BytesValue authResponse = + Functions.aesgcm_encrypt(authResponseKey, zeroNonce, authResponsePt, BytesValue.EMPTY); + BytesValue authResponsePtDecrypted = + Functions.aesgcm_decrypt(authResponseKey, zeroNonce, authResponse, BytesValue.EMPTY); + assertEquals(authResponsePt, authResponsePtDecrypted); + } } diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/HandshakeHandlersTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/HandshakeHandlersTest.java new file mode 100644 index 000000000..6ed66110d --- /dev/null +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/HandshakeHandlersTest.java @@ -0,0 +1,278 @@ +package org.ethereum.beacon.discovery; + +import org.ethereum.beacon.db.Database; +import org.ethereum.beacon.discovery.enr.EnrScheme; +import org.ethereum.beacon.discovery.enr.NodeRecord; +import org.ethereum.beacon.discovery.enr.NodeRecordFactory; +import org.ethereum.beacon.discovery.message.FindNodeMessage; +import org.ethereum.beacon.discovery.packet.Packet; +import org.ethereum.beacon.discovery.packet.WhoAreYouPacket; +import org.ethereum.beacon.discovery.pipeline.Envelope; +import org.ethereum.beacon.discovery.pipeline.Field; +import org.ethereum.beacon.discovery.pipeline.handler.AuthHeaderMessagePacketHandler; +import org.ethereum.beacon.discovery.pipeline.handler.WhoAreYouPacketHandler; +import org.ethereum.beacon.discovery.storage.AuthTagRepository; +import org.ethereum.beacon.discovery.storage.NodeBucketStorage; +import org.ethereum.beacon.discovery.storage.NodeTableStorage; +import org.ethereum.beacon.discovery.storage.NodeTableStorageFactoryImpl; +import org.ethereum.beacon.discovery.task.TaskType; +import org.javatuples.Pair; +import org.junit.Test; +import org.web3j.crypto.ECKeyPair; +import tech.pegasys.artemis.util.bytes.Bytes32; +import tech.pegasys.artemis.util.bytes.Bytes4; +import tech.pegasys.artemis.util.bytes.Bytes96; +import tech.pegasys.artemis.util.bytes.BytesValue; +import tech.pegasys.artemis.util.uint.UInt64; + +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.Random; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +import static org.ethereum.beacon.discovery.pipeline.Field.PACKET_AUTH_HEADER_MESSAGE; +import static org.ethereum.beacon.discovery.pipeline.Field.SESSION; +import static org.ethereum.beacon.discovery.storage.NodeTableStorage.DEFAULT_SERIALIZER; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class HandshakeHandlersTest { + private final BytesValue testKey1 = + BytesValue.fromHexString("eef77acb6c6a6eebc5b363a475ac583ec7eccdb42b6481424c60f59aa326547f"); + private final BytesValue testKey2 = + BytesValue.fromHexString("66fb62bfbd66b9177a138c1e5cddbe4f7c30c343e94e68df8769459cb1cde628"); + private Bytes32 nodeId1; + private Bytes32 nodeId2; + + public HandshakeHandlersTest() { + byte[] homeNodeIdBytes = new byte[32]; + homeNodeIdBytes[0] = 0x01; + byte[] destNodeIdBytes = new byte[32]; + destNodeIdBytes[0] = 0x02; + this.nodeId1 = Bytes32.wrap(homeNodeIdBytes); + this.nodeId2 = Bytes32.wrap(destNodeIdBytes); + } + + @Test + public void authHeaderHandlerTest() throws Exception { + // Node1 + NodeRecord nodeRecord1 = + NodeRecordFactory.DEFAULT.createFromValues( + EnrScheme.V4, + UInt64.valueOf(1), + Bytes96.EMPTY, + Pair.with( + NodeRecord.FIELD_IP_V4, + Bytes4.wrap(InetAddress.getByName("127.0.0.1").getAddress())), + Pair.with(NodeRecord.FIELD_UDP_V4, 30303), + Pair.with(NodeRecord.FIELD_PKEY_SECP256K1, BytesValue.wrap(ECKeyPair.create(testKey1.extractArray()).getPublicKey().toByteArray()))); + // Node2 + NodeRecord nodeRecord2 = + NodeRecordFactory.DEFAULT.createFromValues( + EnrScheme.V4, + UInt64.valueOf(1), + Bytes96.EMPTY, + Pair.with( + NodeRecord.FIELD_IP_V4, + Bytes4.wrap(InetAddress.getByName("127.0.0.1").getAddress())), + Pair.with(NodeRecord.FIELD_UDP_V4, 30304), + Pair.with(NodeRecord.FIELD_PKEY_SECP256K1, BytesValue.wrap(ECKeyPair.create(testKey2.extractArray()).getPublicKey().toByteArray()))); + + Random rnd = new Random(); + NodeTableStorageFactoryImpl nodeTableStorageFactory = new NodeTableStorageFactoryImpl(); + Database database1 = Database.inMemoryDB(); + Database database2 = Database.inMemoryDB(); + NodeTableStorage nodeTableStorage1 = + nodeTableStorageFactory.createTable( + database1, + DEFAULT_SERIALIZER, + () -> nodeRecord1, + () -> + new ArrayList() { + { + add(nodeRecord2); + } + }); + NodeBucketStorage nodeBucketStorage1 = + nodeTableStorageFactory.createBuckets( + database1, DEFAULT_SERIALIZER, nodeRecord1.getNodeId()); + NodeTableStorage nodeTableStorage2 = + nodeTableStorageFactory.createTable( + database2, + DEFAULT_SERIALIZER, + () -> nodeRecord2, + () -> + new ArrayList() { + { + add(nodeRecord1); + } + }); + NodeBucketStorage nodeBucketStorage2 = + nodeTableStorageFactory.createBuckets( + database2, DEFAULT_SERIALIZER, nodeRecord2.getNodeId()); + + // Node1 create AuthHeaderPacket + final Packet[] authHeaderPacket = new Packet[1]; + final CountDownLatch authHeaderPacketLatch = new CountDownLatch(1); + final Consumer outgoingMessages1to2 = + packet -> { + authHeaderPacket[0] = packet; + authHeaderPacketLatch.countDown(); + }; + AuthTagRepository authTagRepository1 = new AuthTagRepository(); + NodeSession nodeSessionAt1For2 = + new NodeSession( + nodeRecord2, + nodeRecord1, + testKey1, + nodeTableStorage1.get(), + nodeBucketStorage1, + authTagRepository1, + outgoingMessages1to2, + rnd); + final Consumer outgoingMessages2to1 = + packet -> { + // do nothing, we don't need to test it here + }; + NodeSession nodeSessionAt2For1 = + new NodeSession( + nodeRecord1, + nodeRecord2, + testKey2, + nodeTableStorage2.get(), + nodeBucketStorage2, + new AuthTagRepository(), + outgoingMessages2to1, + rnd); + WhoAreYouPacketHandler whoAreYouPacketHandlerNode1 = new WhoAreYouPacketHandler(); + Envelope envelopeAt1From2 = new Envelope(); + byte[] idNonceBytes = new byte[32]; + Functions.getRandom().nextBytes(idNonceBytes); + Bytes32 idNonce = Bytes32.wrap(idNonceBytes); + nodeSessionAt2For1.setIdNonce(idNonce); + BytesValue authTag = nodeSessionAt2For1.generateNonce(); + authTagRepository1.put(authTag, nodeSessionAt1For2); + envelopeAt1From2.put( + Field.PACKET_WHOAREYOU, + WhoAreYouPacket.create(nodeId1, authTag, idNonce, UInt64.ZERO)); + envelopeAt1From2.put(Field.SESSION, nodeSessionAt1For2); + nodeSessionAt1For2.saveTask(TaskType.FINDNODE); + whoAreYouPacketHandlerNode1.handle(envelopeAt1From2); + authHeaderPacketLatch.await(1, TimeUnit.SECONDS); + + // Node2 handle AuthHeaderPacket and finish handshake + AuthHeaderMessagePacketHandler authHeaderMessagePacketHandlerNode2 = + new AuthHeaderMessagePacketHandler(); + Envelope envelopeAt2From1 = new Envelope(); + envelopeAt2From1.put(PACKET_AUTH_HEADER_MESSAGE, authHeaderPacket[0]); + envelopeAt2From1.put(SESSION, nodeSessionAt2For1); + assertFalse(nodeSessionAt2For1.isAuthenticated()); + authHeaderMessagePacketHandlerNode2.handle(envelopeAt2From1); + assertTrue(nodeSessionAt2For1.isAuthenticated()); + } + // net := newHandshakeTest() + // defer net.close() + // + // var ( + // idA = net.nodeA.id() + // addrA = net.nodeA.addr() + // challenge = &whoareyouV5{AuthTag: []byte("authresp"), RecordSeq: 0, node: net.nodeB.n()} + // nonce = make([]byte, gcmNonceSize) + // ) + // header, _, _ := net.nodeA.c.makeAuthHeader(nonce, challenge) + // challenge.node = nil // force ENR signature verification in decoder + // b.ResetTimer() + // + // for i := 0; i < b.N; i++ { + // _, _, err := net.nodeB.c.decodeAuthResp(idA, addrA, header, challenge) + // if err != nil { + // b.Fatal(err) + // } + // } + + @Test + public void testHandshake() { + // A -> B RANDOM PACKET + FindNodeMessage findNodeMessage = new FindNodeMessage(BytesValue.fromHexString("01"), 10); + + // func TestHandshakeV5(t *testing.T) { + // t.Parallel() + // net := newHandshakeTest() + // defer net.close() + // + // // A -> B RANDOM PACKET + // packet, _ := net.nodeA.encode(t, net.nodeB, &findnodeV5{}) + // resp := net.nodeB.expectDecode(t, p_unknownV5, packet) + // + // // A <- B WHOAREYOU + // challenge := &whoareyouV5{ + // AuthTag: resp.(*unknownV5).AuthTag, + // IDNonce: testIDnonce, + // RecordSeq: 0, + // } + // whoareyou, _ := net.nodeB.encode(t, net.nodeA, challenge) + // net.nodeA.expectDecode(t, p_whoareyouV5, whoareyou) + // + // // A -> B FINDNODE + // findnode, _ := net.nodeA.encodeWithChallenge(t, net.nodeB, challenge, &findnodeV5{}) + // net.nodeB.expectDecode(t, p_findnodeV5, findnode) + // if len(net.nodeB.c.handshakes) > 0 { + // t.Fatalf("node B didn't remove handshake from challenge map") + // } + // + // // A <- B NODES + // nodes, _ := net.nodeB.encode(t, net.nodeA, &nodesV5{Total: 1}) + // net.nodeA.expectDecode(t, p_nodesV5, nodes) + // + } + + // private BytesValue encodeWithChallenge(V5Message v5Message, WhoAreYouPacket challenge) { + // Copy challenge and add destination node. This avoids sharing challenge among the two codecs. + // var challenge *whoareyouV5 + // if c != nil { + // challengeCopy := *c + // challenge = &challengeCopy + // challenge.node = to.n() + // } + // // Encode to destination. + // enc, authTag, err := n.c.encode(to.id(), to.addr(), p, challenge) + // if err != nil { + // t.Fatal(fmt.Errorf("(%s) %v", n.ln.ID().TerminalString(), err)) + // } + // t.Logf("(%s) -> (%s) %s\n%s", n.ln.ID().TerminalString(), to.id().TerminalString(), + // p.name(), hex.Dump(enc)) + // return enc, authTag + // } + + // private BytesValue encode(V5Message v5Message) { + // // Encode to destination. + // enc, authTag, err := n.c.encode(to.id(), to.addr(), p, challenge) + // return enc, authTag + + // // encode encodes a packet to a node. 'id' and 'addr' specify the destination node. The + //// 'token' parameter should be set to the token in the most recently received WHOAREYOU + //// packet. + // func (c *wireCodec) encode(id enode.ID, addr *net.UDPAddr, packet packetV5, challenge + // *whoareyouV5) ([]byte, []byte, error) { + // if packet.kind() == p_whoareyouV5 { + // p := packet.(*whoareyouV5) + // enc, err := c.encodeWhoareyou(id, p) + // if err == nil { + // c.storeSentHandshake(id, addr, p) + // } + // return enc, nil, err + // } + // // Ensure calling code sets node if needed. + // if challenge != nil && challenge.node == nil { + // panic("BUG: missing challenge.node in encode") + // } + // _, writeKey := c.loadKeys(id, addr) + // if writeKey != nil || challenge != nil { + // return c.encodeEncrypted(id, addr, packet, writeKey, challenge) + // } + // return c.encodeRandom(id) + // } + // } +} diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java b/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java index 8548884fe..c64bd507a 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java @@ -15,16 +15,16 @@ import org.ethereum.beacon.discovery.pipeline.handler.IncomingDataPacker; import org.ethereum.beacon.discovery.pipeline.handler.MessageHandler; import org.ethereum.beacon.discovery.pipeline.handler.MessagePacketHandler; -import org.ethereum.beacon.discovery.pipeline.handler.NodeSessionRequestHandler; import org.ethereum.beacon.discovery.pipeline.handler.NodeIdToSession; +import org.ethereum.beacon.discovery.pipeline.handler.NodeSessionRequestHandler; import org.ethereum.beacon.discovery.pipeline.handler.NotExpectedIncomingPacketHandler; import org.ethereum.beacon.discovery.pipeline.handler.OutgoingParcelHandler; import org.ethereum.beacon.discovery.pipeline.handler.TaskHandler; import org.ethereum.beacon.discovery.pipeline.handler.UnknownPacketTagToSender; import org.ethereum.beacon.discovery.pipeline.handler.UnknownPacketTypeByStatus; import org.ethereum.beacon.discovery.pipeline.handler.WhoAreYouAttempt; -import org.ethereum.beacon.discovery.pipeline.handler.WhoAreYouSessionResolver; import org.ethereum.beacon.discovery.pipeline.handler.WhoAreYouPacketHandler; +import org.ethereum.beacon.discovery.pipeline.handler.WhoAreYouSessionResolver; import org.ethereum.beacon.discovery.storage.AuthTagRepository; import org.ethereum.beacon.discovery.storage.NodeBucketStorage; import org.ethereum.beacon.discovery.storage.NodeTable; @@ -56,11 +56,18 @@ public DiscoveryManagerNoNetwork( NodeTable nodeTable, NodeBucketStorage nodeBucketStorage, NodeRecord homeNode, + BytesValue homeNodePrivateKey, Publisher incomingPackets) { AuthTagRepository authTagRepo = new AuthTagRepository(); this.incomingPackets = incomingPackets; NodeIdToSession nodeIdToSession = - new NodeIdToSession(homeNode, nodeBucketStorage, authTagRepo, nodeTable, outgoingPipeline); + new NodeIdToSession( + homeNode, + homeNodePrivateKey, + nodeBucketStorage, + authTagRepo, + nodeTable, + outgoingPipeline); incomingPipeline .addHandler(new IncomingDataPacker()) .addHandler(new WhoAreYouAttempt(homeNode.getNodeId())) From eb50643cdfa8469f2156331f68009ce6f2fefd4d Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Fri, 18 Oct 2019 07:49:57 +0300 Subject: [PATCH 37/77] discovery: handler logging improved --- .../discovery/pipeline/HandlerUtil.java | 30 +++++++++++++++++++ .../AuthHeaderMessagePacketHandler.java | 16 ++++++++-- .../pipeline/handler/BadPacketLogger.java | 13 +++++++- .../pipeline/handler/IncomingDataPacker.java | 14 ++++++++- .../pipeline/handler/MessageHandler.java | 18 +++++++++-- .../handler/MessagePacketHandler.java | 24 ++++++++++++--- .../pipeline/handler/NodeIdToSession.java | 13 +++++++- .../handler/NodeSessionRequestHandler.java | 17 ++++++++++- .../NotExpectedIncomingPacketHandler.java | 16 ++++++++-- .../handler/OutgoingParcelHandler.java | 17 ++++++++++- .../pipeline/handler/TaskHandler.java | 29 ++++++++++++++---- .../handler/UnknownPacketTagToSender.java | 17 +++++++++++ .../handler/UnknownPacketTypeByStatus.java | 15 ++++++++-- .../pipeline/handler/WhoAreYouAttempt.java | 21 +++++++++++-- .../handler/WhoAreYouPacketHandler.java | 16 ++++++++-- .../handler/WhoAreYouSessionResolver.java | 13 +++++++- 16 files changed, 261 insertions(+), 28 deletions(-) create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/HandlerUtil.java diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/HandlerUtil.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/HandlerUtil.java new file mode 100644 index 000000000..4916918e2 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/HandlerUtil.java @@ -0,0 +1,30 @@ +package org.ethereum.beacon.discovery.pipeline; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.function.Function; + +public class HandlerUtil { + private static final Logger logger = LogManager.getLogger(HandlerUtil.class); + + public static boolean requireField(Field field, Envelope envelope) { + if (envelope.contains(field)) { + return true; + } else { + logger.trace(() -> String.format("Requirement not satisfied: field %s not exists in envelope %s", + field, envelope.getId())); + return false; + } + } + + public static boolean requireCondition(Function conditionFunction, Envelope envelope) { + if (conditionFunction.apply(envelope)) { + return true; + } else { + logger.trace(() -> String.format("Requirement not satisfied: condition %s not met for envelope %s", + conditionFunction, envelope.getId())); + return false; + } + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/AuthHeaderMessagePacketHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/AuthHeaderMessagePacketHandler.java index 42e87f6ff..3d7472990 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/AuthHeaderMessagePacketHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/AuthHeaderMessagePacketHandler.java @@ -10,6 +10,7 @@ import org.ethereum.beacon.discovery.pipeline.Envelope; import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; import org.ethereum.beacon.discovery.pipeline.Field; +import org.ethereum.beacon.discovery.pipeline.HandlerUtil; import org.ethereum.beacon.discovery.task.TaskType; import org.javatuples.Triplet; import tech.pegasys.artemis.util.bytes.BytesValue; @@ -24,12 +25,23 @@ public class AuthHeaderMessagePacketHandler implements EnvelopeHandler { @Override public void handle(Envelope envelope) { - if (!envelope.contains(Field.PACKET_AUTH_HEADER_MESSAGE)) { + logger.trace( + () -> + String.format( + "Envelope %s in AuthHeaderMessagePacketHandler, checking requirements satisfaction", + envelope.getId())); + if (!HandlerUtil.requireField(Field.PACKET_AUTH_HEADER_MESSAGE, envelope)) { return; } - if (!envelope.contains(Field.SESSION)) { + if (!HandlerUtil.requireField(Field.SESSION, envelope)) { return; } + logger.trace( + () -> + String.format( + "Envelope %s in AuthHeaderMessagePacketHandler, requirements are satisfied!", + envelope.getId())); + AuthHeaderMessagePacket packet = (AuthHeaderMessagePacket) envelope.get(Field.PACKET_AUTH_HEADER_MESSAGE); NodeSession session = (NodeSession) envelope.get(Field.SESSION); diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/BadPacketLogger.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/BadPacketLogger.java index 7a3582c08..e599ded64 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/BadPacketLogger.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/BadPacketLogger.java @@ -5,6 +5,7 @@ import org.ethereum.beacon.discovery.pipeline.Envelope; import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; import org.ethereum.beacon.discovery.pipeline.Field; +import org.ethereum.beacon.discovery.pipeline.HandlerUtil; /** Logs all packets which are stored in {@link Field#BAD_PACKET} */ public class BadPacketLogger implements EnvelopeHandler { @@ -12,9 +13,19 @@ public class BadPacketLogger implements EnvelopeHandler { @Override public void handle(Envelope envelope) { - if (!envelope.contains(Field.BAD_PACKET)) { + logger.trace( + () -> + String.format( + "Envelope %s in BadPacketLogger, checking requirements satisfaction", + envelope.getId())); + if (!HandlerUtil.requireField(Field.BAD_PACKET, envelope)) { return; } + logger.trace( + () -> + String.format( + "Envelope %s in BadPacketLogger, requirements are satisfied!", envelope.getId())); + logger.debug( () -> String.format( diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/IncomingDataPacker.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/IncomingDataPacker.java index 91fc6585e..5cfb7df9e 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/IncomingDataPacker.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/IncomingDataPacker.java @@ -6,6 +6,7 @@ import org.ethereum.beacon.discovery.pipeline.Envelope; import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; import org.ethereum.beacon.discovery.pipeline.Field; +import org.ethereum.beacon.discovery.pipeline.HandlerUtil; import tech.pegasys.artemis.util.bytes.BytesValue; /** Handles raw BytesValue incoming data in {@link Field#INCOMING} */ @@ -14,9 +15,20 @@ public class IncomingDataPacker implements EnvelopeHandler { @Override public void handle(Envelope envelope) { - if (!envelope.contains(Field.INCOMING)) { + logger.trace( + () -> + String.format( + "Envelope %s in IncomingDataPacker, checking requirements satisfaction", + envelope.getId())); + if (!HandlerUtil.requireField(Field.INCOMING, envelope)) { return; } + logger.trace( + () -> + String.format( + "Envelope %s in IncomingDataPacker, requirements are satisfied!", + envelope.getId())); + UnknownPacket unknownPacket = new UnknownPacket((BytesValue) envelope.get(Field.INCOMING)); envelope.put(Field.PACKET_UNKNOWN, unknownPacket); logger.trace( diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/MessageHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/MessageHandler.java index 51f011689..a287deefc 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/MessageHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/MessageHandler.java @@ -1,5 +1,7 @@ package org.ethereum.beacon.discovery.pipeline.handler; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.ethereum.beacon.discovery.DiscoveryV5MessageProcessor; import org.ethereum.beacon.discovery.MessageProcessor; import org.ethereum.beacon.discovery.NodeSession; @@ -7,16 +9,28 @@ import org.ethereum.beacon.discovery.pipeline.Envelope; import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; import org.ethereum.beacon.discovery.pipeline.Field; +import org.ethereum.beacon.discovery.pipeline.HandlerUtil; public class MessageHandler implements EnvelopeHandler { + private static final Logger logger = LogManager.getLogger(MessageHandler.class); + @Override public void handle(Envelope envelope) { - if (!envelope.contains(Field.MESSAGE)) { + logger.trace( + () -> + String.format( + "Envelope %s in MessageHandler, checking requirements satisfaction", + envelope.getId())); + if (!HandlerUtil.requireField(Field.MESSAGE, envelope)) { return; } - if (!envelope.contains(Field.SESSION)) { + if (!HandlerUtil.requireField(Field.SESSION, envelope)) { return; } + logger.trace( + () -> + String.format( + "Envelope %s in MessageHandler, requirements are satisfied!", envelope.getId())); NodeSession session = (NodeSession) envelope.get(Field.SESSION); DiscoveryMessage message = (DiscoveryMessage) envelope.get(Field.MESSAGE); diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/MessagePacketHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/MessagePacketHandler.java index b7ca44124..2b283f0f0 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/MessagePacketHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/MessagePacketHandler.java @@ -9,6 +9,7 @@ import org.ethereum.beacon.discovery.pipeline.Envelope; import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; import org.ethereum.beacon.discovery.pipeline.Field; +import org.ethereum.beacon.discovery.pipeline.HandlerUtil; import org.ethereum.beacon.discovery.task.TaskType; import java.util.concurrent.CompletableFuture; @@ -19,12 +20,23 @@ public class MessagePacketHandler implements EnvelopeHandler { @Override public void handle(Envelope envelope) { - if (!envelope.contains(Field.PACKET_MESSAGE)) { + logger.trace( + () -> + String.format( + "Envelope %s in MessagePacketHandler, checking requirements satisfaction", + envelope.getId())); + if (!HandlerUtil.requireField(Field.PACKET_MESSAGE, envelope)) { return; } - if (!envelope.contains(Field.SESSION)) { + if (!HandlerUtil.requireField(Field.SESSION, envelope)) { return; } + logger.trace( + () -> + String.format( + "Envelope %s in MessagePacketHandler, requirements are satisfied!", + envelope.getId())); + MessagePacket packet = (MessagePacket) envelope.get(Field.PACKET_MESSAGE); NodeSession session = (NodeSession) envelope.get(Field.SESSION); @@ -55,10 +67,14 @@ public void handle(Envelope envelope) { TaskType taskType = session.loadTask(); if (future != null) { boolean taskCompleted = false; - if (TaskType.PING.equals(taskType) && packet.getMessage() instanceof DiscoveryV5Message && ((DiscoveryV5Message) packet.getMessage()).getCode() == MessageCode.PONG) { + if (TaskType.PING.equals(taskType) + && packet.getMessage() instanceof DiscoveryV5Message + && ((DiscoveryV5Message) packet.getMessage()).getCode() == MessageCode.PONG) { taskCompleted = true; } - if (TaskType.FINDNODE.equals(taskType) && packet.getMessage() instanceof DiscoveryV5Message && ((DiscoveryV5Message) packet.getMessage()).getCode() == MessageCode.NODES) { + if (TaskType.FINDNODE.equals(taskType) + && packet.getMessage() instanceof DiscoveryV5Message + && ((DiscoveryV5Message) packet.getMessage()).getCode() == MessageCode.NODES) { taskCompleted = true; } if (taskCompleted) { diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeIdToSession.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeIdToSession.java index 5d7dcfa03..e4f719f39 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeIdToSession.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeIdToSession.java @@ -9,6 +9,7 @@ import org.ethereum.beacon.discovery.pipeline.Envelope; import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; import org.ethereum.beacon.discovery.pipeline.Field; +import org.ethereum.beacon.discovery.pipeline.HandlerUtil; import org.ethereum.beacon.discovery.pipeline.Pipeline; import org.ethereum.beacon.discovery.storage.AuthTagRepository; import org.ethereum.beacon.discovery.storage.NodeBucketStorage; @@ -59,9 +60,19 @@ public NodeIdToSession( @Override public void handle(Envelope envelope) { - if (!envelope.contains(Field.SESSION_LOOKUP)) { + logger.trace( + () -> + String.format( + "Envelope %s in NodeIdToSession, checking requirements satisfaction", + envelope.getId())); + if (!HandlerUtil.requireField(Field.SESSION_LOOKUP, envelope)) { return; } + logger.trace( + () -> + String.format( + "Envelope %s in NodeIdToSession, requirements are satisfied!", envelope.getId())); + Pair sessionRequest = (Pair) envelope.get(Field.SESSION_LOOKUP); envelope.remove(Field.SESSION_LOOKUP); diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeSessionRequestHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeSessionRequestHandler.java index 42a8da9bc..4bfcddbfd 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeSessionRequestHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeSessionRequestHandler.java @@ -1,9 +1,12 @@ package org.ethereum.beacon.discovery.pipeline.handler; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.pipeline.Envelope; import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; import org.ethereum.beacon.discovery.pipeline.Field; +import org.ethereum.beacon.discovery.pipeline.HandlerUtil; import org.javatuples.Pair; /** @@ -11,12 +14,24 @@ * Field#SESSION_LOOKUP} */ public class NodeSessionRequestHandler implements EnvelopeHandler { + private static final Logger logger = LogManager.getLogger(NodeSessionRequestHandler.class); @Override public void handle(Envelope envelope) { - if (!envelope.contains(Field.NODE)) { + logger.trace( + () -> + String.format( + "Envelope %s in NodeSessionRequestHandler, checking requirements satisfaction", + envelope.getId())); + if (!HandlerUtil.requireField(Field.NODE, envelope)) { return; } + logger.trace( + () -> + String.format( + "Envelope %s in NodeSessionRequestHandler, requirements are satisfied!", + envelope.getId())); + envelope.put( Field.SESSION_LOOKUP, Pair.with( diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NotExpectedIncomingPacketHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NotExpectedIncomingPacketHandler.java index 543f1c239..3a67a4b3d 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NotExpectedIncomingPacketHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NotExpectedIncomingPacketHandler.java @@ -11,6 +11,7 @@ import org.ethereum.beacon.discovery.pipeline.Envelope; import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; import org.ethereum.beacon.discovery.pipeline.Field; +import org.ethereum.beacon.discovery.pipeline.HandlerUtil; import tech.pegasys.artemis.util.bytes.Bytes32; import tech.pegasys.artemis.util.bytes.BytesValue; @@ -20,12 +21,23 @@ public class NotExpectedIncomingPacketHandler implements EnvelopeHandler { @Override public void handle(Envelope envelope) { - if (!envelope.contains(Field.PACKET_UNKNOWN)) { + logger.trace( + () -> + String.format( + "Envelope %s in NotExpectedIncomingPacketHandler, checking requirements satisfaction", + envelope.getId())); + if (!HandlerUtil.requireField(Field.PACKET_UNKNOWN, envelope)) { return; } - if (!envelope.contains(Field.SESSION)) { + if (!HandlerUtil.requireField(Field.SESSION, envelope)) { return; } + logger.trace( + () -> + String.format( + "Envelope %s in NotExpectedIncomingPacketHandler, requirements are satisfied!", + envelope.getId())); + UnknownPacket unknownPacket = (UnknownPacket) envelope.get(Field.PACKET_UNKNOWN); NodeSession session = (NodeSession) envelope.get(Field.SESSION); try { diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/OutgoingParcelHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/OutgoingParcelHandler.java index ae1444071..29b936cda 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/OutgoingParcelHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/OutgoingParcelHandler.java @@ -1,9 +1,12 @@ package org.ethereum.beacon.discovery.pipeline.handler; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.ethereum.beacon.discovery.network.NetworkParcel; import org.ethereum.beacon.discovery.pipeline.Envelope; import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; import org.ethereum.beacon.discovery.pipeline.Field; +import org.ethereum.beacon.discovery.pipeline.HandlerUtil; import reactor.core.publisher.FluxSink; /** @@ -12,6 +15,7 @@ * is linked with discovery client. */ public class OutgoingParcelHandler implements EnvelopeHandler { + private static final Logger logger = LogManager.getLogger(OutgoingParcelHandler.class); private final FluxSink outgoingSink; @@ -21,9 +25,20 @@ public OutgoingParcelHandler(FluxSink outgoingSink) { @Override public void handle(Envelope envelope) { - if (!envelope.contains(Field.INCOMING)) { + logger.trace( + () -> + String.format( + "Envelope %s in OutgoingParcelHandler, checking requirements satisfaction", + envelope.getId())); + if (!HandlerUtil.requireField(Field.INCOMING, envelope)) { return; } + logger.trace( + () -> + String.format( + "Envelope %s in OutgoingParcelHandler, requirements are satisfied!", + envelope.getId())); + if (envelope.get(Field.INCOMING) instanceof NetworkParcel) { outgoingSink.next((NetworkParcel) envelope.get(Field.INCOMING)); envelope.remove(Field.INCOMING); diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/TaskHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/TaskHandler.java index 5ce28c3b6..b665cd140 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/TaskHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/TaskHandler.java @@ -1,10 +1,13 @@ package org.ethereum.beacon.discovery.pipeline.handler; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.ethereum.beacon.discovery.NodeSession; import org.ethereum.beacon.discovery.packet.RandomPacket; import org.ethereum.beacon.discovery.pipeline.Envelope; import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; import org.ethereum.beacon.discovery.pipeline.Field; +import org.ethereum.beacon.discovery.pipeline.HandlerUtil; import org.ethereum.beacon.discovery.task.TaskMessageFactory; import org.ethereum.beacon.discovery.task.TaskType; import tech.pegasys.artemis.util.bytes.BytesValue; @@ -15,6 +18,7 @@ /** Performs task execution for any task found in {@link Field#TASK} */ public class TaskHandler implements EnvelopeHandler { + private static final Logger logger = LogManager.getLogger(TaskHandler.class); private final Random rnd; public TaskHandler(Random rnd) { @@ -23,9 +27,23 @@ public TaskHandler(Random rnd) { @Override public void handle(Envelope envelope) { - if (!envelope.contains(Field.TASK)) { + logger.trace( + () -> + String.format( + "Envelope %s in TaskHandler, checking requirements satisfaction", + envelope.getId())); + if (!HandlerUtil.requireField(Field.TASK, envelope)) { + return; + } + if (!HandlerUtil.requireField(Field.SESSION, envelope)) { return; } + logger.trace( + () -> + String.format( + "Envelope %s in TaskHandler, requirements are satisfied!", envelope.getId())); + + TaskType task = (TaskType) envelope.get(Field.TASK); NodeSession session = (NodeSession) envelope.get(Field.SESSION); CompletableFuture completableFuture = (CompletableFuture) envelope.get(Field.FUTURE); @@ -39,17 +57,16 @@ public void handle(Envelope envelope) { authTag, new SecureRandom()); session.setAuthTag(authTag); + session.saveFuture(completableFuture); + session.saveTask(task); session.sendOutgoing(randomPacket); session.setStatus(NodeSession.SessionStatus.RANDOM_PACKET_SENT); - session.saveFuture(completableFuture); } else if (session.getStatus().equals(NodeSession.SessionStatus.AUTHENTICATED)) { - if (envelope.get(Field.TASK).equals(TaskType.PING)) { + if (TaskType.PING.equals(task)) { session.sendOutgoing(TaskMessageFactory.createPingPacket(authTag, session)); - session.saveTask((TaskType) envelope.get(Field.TASK)); session.saveFuture(completableFuture); - } else if (envelope.get(Field.TASK).equals(TaskType.FINDNODE)) { + } else if (TaskType.FINDNODE.equals(task)) { session.sendOutgoing(TaskMessageFactory.createFindNodePacket(authTag, session)); - session.saveTask((TaskType) envelope.get(Field.TASK)); session.saveFuture(completableFuture); } else { throw new RuntimeException( diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/UnknownPacketTagToSender.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/UnknownPacketTagToSender.java index 9ec4f777d..ade32fff3 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/UnknownPacketTagToSender.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/UnknownPacketTagToSender.java @@ -1,10 +1,13 @@ package org.ethereum.beacon.discovery.pipeline.handler; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.packet.UnknownPacket; import org.ethereum.beacon.discovery.pipeline.Envelope; import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; import org.ethereum.beacon.discovery.pipeline.Field; +import org.ethereum.beacon.discovery.pipeline.HandlerUtil; import org.javatuples.Pair; import tech.pegasys.artemis.util.bytes.Bytes32; @@ -14,6 +17,7 @@ * session could be resolved by another handler. */ public class UnknownPacketTagToSender implements EnvelopeHandler { + private static final Logger logger = LogManager.getLogger(UnknownPacketTagToSender.class); private final Bytes32 homeNodeId; public UnknownPacketTagToSender(NodeRecord homeNodeRecord) { @@ -22,6 +26,19 @@ public UnknownPacketTagToSender(NodeRecord homeNodeRecord) { @Override public void handle(Envelope envelope) { + logger.trace( + () -> + String.format( + "Envelope %s in UnknownPacketTagToSender, checking requirements satisfaction", + envelope.getId())); + if (!HandlerUtil.requireField(Field.PACKET_UNKNOWN, envelope)) { + return; + } + logger.trace( + () -> + String.format( + "Envelope %s in UnknownPacketTagToSender, requirements are satisfied!", envelope.getId())); + if (!envelope.contains(Field.PACKET_UNKNOWN)) { return; } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/UnknownPacketTypeByStatus.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/UnknownPacketTypeByStatus.java index b212e3b1e..749861a80 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/UnknownPacketTypeByStatus.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/UnknownPacketTypeByStatus.java @@ -9,6 +9,7 @@ import org.ethereum.beacon.discovery.pipeline.Envelope; import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; import org.ethereum.beacon.discovery.pipeline.Field; +import org.ethereum.beacon.discovery.pipeline.HandlerUtil; /** * Resolves incoming packet type based on session states and places packet into the corresponding @@ -19,12 +20,22 @@ public class UnknownPacketTypeByStatus implements EnvelopeHandler { @Override public void handle(Envelope envelope) { - if (!envelope.contains(Field.SESSION)) { + logger.trace( + () -> + String.format( + "Envelope %s in UnknownPacketTypeByStatus, checking requirements satisfaction", + envelope.getId())); + if (!HandlerUtil.requireField(Field.SESSION, envelope)) { return; } - if (!envelope.contains(Field.PACKET_UNKNOWN)) { + if (!HandlerUtil.requireField(Field.PACKET_UNKNOWN, envelope)) { return; } + logger.trace( + () -> + String.format( + "Envelope %s in UnknownPacketTypeByStatus, requirements are satisfied!", envelope.getId())); + UnknownPacket unknownPacket = (UnknownPacket) envelope.get(Field.PACKET_UNKNOWN); NodeSession session = (NodeSession) envelope.get(Field.SESSION); switch (session.getStatus()) { diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouAttempt.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouAttempt.java index 1077b0745..93583f3c5 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouAttempt.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouAttempt.java @@ -1,9 +1,12 @@ package org.ethereum.beacon.discovery.pipeline.handler; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.ethereum.beacon.discovery.packet.UnknownPacket; import org.ethereum.beacon.discovery.pipeline.Envelope; import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; import org.ethereum.beacon.discovery.pipeline.Field; +import org.ethereum.beacon.discovery.pipeline.HandlerUtil; import tech.pegasys.artemis.util.bytes.Bytes32; /** @@ -11,6 +14,7 @@ * was successful, places the result in {@link Field#PACKET_WHOAREYOU} */ public class WhoAreYouAttempt implements EnvelopeHandler { + private static final Logger logger = LogManager.getLogger(WhoAreYouAttempt.class); private final Bytes32 homeNodeId; public WhoAreYouAttempt(Bytes32 homeNodeId) { @@ -19,12 +23,25 @@ public WhoAreYouAttempt(Bytes32 homeNodeId) { @Override public void handle(Envelope envelope) { - if (!envelope.contains(Field.PACKET_UNKNOWN)) { + logger.trace( + () -> + String.format( + "Envelope %s in WhoAreYouAttempt, checking requirements satisfaction", + envelope.getId())); + if (!HandlerUtil.requireField(Field.PACKET_UNKNOWN, envelope)) { return; } - if (!((UnknownPacket) envelope.get(Field.PACKET_UNKNOWN)).isWhoAreYouPacket(homeNodeId)) { + if (!(HandlerUtil.requireCondition( + envelope1 -> + ((UnknownPacket) envelope1.get(Field.PACKET_UNKNOWN)).isWhoAreYouPacket(homeNodeId), + envelope))) { return; } + logger.trace( + () -> + String.format( + "Envelope %s in WhoAreYouAttempt, requirements are satisfied!", envelope.getId())); + UnknownPacket unknownPacket = (UnknownPacket) envelope.get(Field.PACKET_UNKNOWN); envelope.put(Field.PACKET_WHOAREYOU, unknownPacket.getWhoAreYouPacket()); envelope.remove(Field.PACKET_UNKNOWN); diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouPacketHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouPacketHandler.java index 491168372..af421a4f2 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouPacketHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouPacketHandler.java @@ -12,6 +12,7 @@ import org.ethereum.beacon.discovery.pipeline.Envelope; import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; import org.ethereum.beacon.discovery.pipeline.Field; +import org.ethereum.beacon.discovery.pipeline.HandlerUtil; import org.ethereum.beacon.discovery.task.TaskMessageFactory; import org.ethereum.beacon.discovery.task.TaskType; import org.javatuples.Triplet; @@ -28,12 +29,23 @@ public class WhoAreYouPacketHandler implements EnvelopeHandler { @Override public void handle(Envelope envelope) { - if (!envelope.contains(Field.PACKET_WHOAREYOU)) { + logger.trace( + () -> + String.format( + "Envelope %s in WhoAreYouPacketHandler, checking requirements satisfaction", + envelope.getId())); + if (!HandlerUtil.requireField(Field.SESSION, envelope)) { return; } - if (!envelope.contains(Field.SESSION)) { + if (!HandlerUtil.requireField(Field.PACKET_WHOAREYOU, envelope)) { return; } + logger.trace( + () -> + String.format( + "Envelope %s in WhoAreYouPacketHandler, requirements are satisfied!", + envelope.getId())); + WhoAreYouPacket packet = (WhoAreYouPacket) envelope.get(Field.PACKET_WHOAREYOU); NodeSession session = (NodeSession) envelope.get(Field.SESSION); try { diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouSessionResolver.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouSessionResolver.java index 23c2477c4..8f3c2a18c 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouSessionResolver.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouSessionResolver.java @@ -7,6 +7,7 @@ import org.ethereum.beacon.discovery.pipeline.Envelope; import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; import org.ethereum.beacon.discovery.pipeline.Field; +import org.ethereum.beacon.discovery.pipeline.HandlerUtil; import org.ethereum.beacon.discovery.storage.AuthTagRepository; import java.util.Optional; @@ -25,9 +26,19 @@ public WhoAreYouSessionResolver(AuthTagRepository authTagRepo) { @Override public void handle(Envelope envelope) { - if (!envelope.contains(Field.PACKET_WHOAREYOU)) { + logger.trace( + () -> + String.format( + "Envelope %s in WhoAreYouSessionResolver, checking requirements satisfaction", + envelope.getId())); + if (!HandlerUtil.requireField(Field.PACKET_WHOAREYOU, envelope)) { return; } + logger.trace( + () -> + String.format( + "Envelope %s in WhoAreYouSessionResolver, requirements are satisfied!", + envelope.getId())); WhoAreYouPacket whoAreYouPacket = (WhoAreYouPacket) envelope.get(Field.PACKET_WHOAREYOU); Optional nodeSessionOptional = authTagRepo.get(whoAreYouPacket.getAuthTag()); From 15a841f2d0f514e66ce57f41fb455d34797df8d3 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Fri, 18 Oct 2019 14:57:39 +0300 Subject: [PATCH 38/77] discovery: fix message request-response in discovery --- .../DiscoveryV5MessageProcessor.java | 4 + .../beacon/discovery/NodeSession.java | 5 + .../discovery/message/DiscoveryV5Message.java | 5 + .../discovery/message/NodesMessage.java | 2 +- .../message/handler/FindNodeHandler.java | 10 +- .../message/handler/NodesHandler.java | 15 ++- .../packet/AuthHeaderMessagePacket.java | 2 +- .../storage/NodeBucketStorageImpl.java | 11 +- .../storage/NodeTableStorageFactory.java | 5 +- .../storage/NodeTableStorageFactoryImpl.java | 6 +- .../discovery/DiscoveryNetworkTest.java | 112 +++++------------ .../discovery/DiscoveryNoNetworkTest.java | 119 ++++++------------ .../discovery/HandshakeHandlersTest.java | 58 ++------- .../ethereum/beacon/discovery/TestUtil.java | 62 +++++++++ .../discovery/storage/NodeBucketTest.java | 3 +- 15 files changed, 187 insertions(+), 232 deletions(-) create mode 100644 discovery/src/test/java/org/ethereum/beacon/discovery/TestUtil.java diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryV5MessageProcessor.java b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryV5MessageProcessor.java index 947d1fd31..e2ba69348 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryV5MessageProcessor.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryV5MessageProcessor.java @@ -1,5 +1,7 @@ package org.ethereum.beacon.discovery; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.ethereum.beacon.discovery.message.DiscoveryV5Message; import org.ethereum.beacon.discovery.message.MessageCode; import org.ethereum.beacon.discovery.message.handler.FindNodeHandler; @@ -12,6 +14,7 @@ import java.util.Map; public class DiscoveryV5MessageProcessor implements DiscoveryMessageProcessor { + private static final Logger logger = LogManager.getLogger(DiscoveryV5MessageProcessor.class); private final Map messageHandlers = new HashMap<>(); public DiscoveryV5MessageProcessor() { @@ -30,6 +33,7 @@ public IdentityScheme getSupportedIdentity() { public void handleMessage(DiscoveryV5Message message, NodeSession session) { MessageCode code = message.getCode(); MessageHandler messageHandler = messageHandlers.get(code); + logger.trace(() -> String.format("Handling message %s in session %s", message, session)); if (messageHandler == null) { throw new RuntimeException("Not implemented yet"); } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeSession.java b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeSession.java index 575bb3ca4..77de1d40a 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeSession.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeSession.java @@ -73,6 +73,7 @@ private void completeConnectFuture() { } public synchronized void sendOutgoing(Packet packet) { + logger.trace(() -> String.format("Sending outgoing packet %s in session %s", packet, this)); outgoing.accept(packet); } @@ -149,6 +150,10 @@ public NodeTable getNodeTable() { return nodeTable; } + public void putRecordInBucket(NodeRecordInfo nodeRecordInfo) { + nodeBucketStorage.put(nodeRecordInfo); + } + public Optional getBucket(int index) { return nodeBucketStorage.get(index); } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/DiscoveryV5Message.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/DiscoveryV5Message.java index 144f2146e..e71d50d0e 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/message/DiscoveryV5Message.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/DiscoveryV5Message.java @@ -99,4 +99,9 @@ public V5Message create() { } } } + + @Override + public String toString() { + return "DiscoveryV5Message{" + "code=" + getCode() + ", bytes=" + getBytes() + '}'; + } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/NodesMessage.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/NodesMessage.java index 384c17a8c..953039daa 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/message/NodesMessage.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/NodesMessage.java @@ -60,7 +60,7 @@ public Integer getNodeRecordsSize() { @Override public BytesValue getBytes() { - return Bytes1.intToBytes1(MessageCode.PONG.byteCode()) + return Bytes1.intToBytes1(MessageCode.NODES.byteCode()) .concat( BytesValue.wrap( RlpEncoder.encode( diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/FindNodeHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/FindNodeHandler.java index fc1fc5618..dabaa2dee 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/FindNodeHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/FindNodeHandler.java @@ -1,7 +1,9 @@ package org.ethereum.beacon.discovery.message.handler; -import org.ethereum.beacon.discovery.NodeSession; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.ethereum.beacon.discovery.NodeRecordInfo; +import org.ethereum.beacon.discovery.NodeSession; import org.ethereum.beacon.discovery.message.DiscoveryV5Message; import org.ethereum.beacon.discovery.message.FindNodeMessage; import org.ethereum.beacon.discovery.message.NodesMessage; @@ -14,6 +16,7 @@ import java.util.stream.IntStream; public class FindNodeHandler implements MessageHandler { + private static final Logger logger = LogManager.getLogger(FindNodeHandler.class); public FindNodeHandler() {} @@ -25,6 +28,11 @@ public void handle(FindNodeMessage message, NodeSession session) { .filter(Optional::isPresent) .map(Optional::get) .collect(Collectors.toList()); + logger.trace( + () -> + String.format( + "Sending %s nodeBuckets in reply to request in session %s", + nodeBuckets.size(), session)); nodeBuckets.forEach( bucket -> session.sendOutgoing( diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/NodesHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/NodesHandler.java index fbdc32e60..fb59e30ac 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/NodesHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/NodesHandler.java @@ -1,7 +1,9 @@ package org.ethereum.beacon.discovery.message.handler; -import org.ethereum.beacon.discovery.NodeSession; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.ethereum.beacon.discovery.NodeRecordInfo; +import org.ethereum.beacon.discovery.NodeSession; import org.ethereum.beacon.discovery.message.MessageCode; import org.ethereum.beacon.discovery.message.NodesMessage; import org.ethereum.beacon.util.ExpirationScheduler; @@ -12,6 +14,7 @@ import java.util.concurrent.TimeUnit; public class NodesHandler implements MessageHandler { + private static final Logger logger = LogManager.getLogger(FindNodeHandler.class); private static final int CLEANUP_DELAY_SECONDS = 60; private ExpirationScheduler nodeExpirationScheduler = new ExpirationScheduler<>(CLEANUP_DELAY_SECONDS, TimeUnit.SECONDS); @@ -38,18 +41,24 @@ public void handle(NodesMessage message, NodeSession session) { } // Parse node records + logger.trace( + () -> + String.format( + "Received %s node records in session %s. Total buckets expected: %s", + message.getNodeRecordsSize(), session, message.getTotal())); message .getNodeRecords() .forEach( nodeRecordV5 -> { + NodeRecordInfo nodeRecordInfo = NodeRecordInfo.createDefault(nodeRecordV5); if (!session.getNodeTable().getNode(nodeRecordV5.getNodeId()).isPresent()) { + session.getNodeTable().save(nodeRecordInfo); // TODO: should we update-merge? - session.getNodeTable().save(NodeRecordInfo.createDefault(nodeRecordV5)); } + session.putRecordInBucket(nodeRecordInfo); }); } - private synchronized void updateExpiration(BytesValue requestId, NodeSession session) { nodeExpirationScheduler.put( requestId, diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/AuthHeaderMessagePacket.java b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/AuthHeaderMessagePacket.java index 6aa1f0ad3..503ec7f4a 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/AuthHeaderMessagePacket.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/AuthHeaderMessagePacket.java @@ -77,7 +77,7 @@ public static AuthHeaderMessagePacket create( RlpString.create(5), RlpString.create(idNonceSig.extractArray()), RlpString.create( - nodeRecord == null ? null : nodeRecord.serialize().extractArray()))); + nodeRecord == null ? new byte[0] : nodeRecord.serialize().extractArray()))); BytesValue authResponse = Functions.aesgcm_encrypt( authResponseKey, ZERO_NONCE, BytesValue.wrap(authResponsePt), BytesValue.EMPTY); diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeBucketStorageImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeBucketStorageImpl.java index e0cb1920e..92beb306b 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeBucketStorageImpl.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeBucketStorageImpl.java @@ -7,6 +7,7 @@ import org.ethereum.beacon.db.source.impl.DataSourceList; import org.ethereum.beacon.discovery.Functions; import org.ethereum.beacon.discovery.NodeRecordInfo; +import org.ethereum.beacon.discovery.enr.NodeRecord; import tech.pegasys.artemis.util.bytes.Bytes32; import tech.pegasys.artemis.util.bytes.BytesValue; @@ -24,7 +25,7 @@ public class NodeBucketStorageImpl implements NodeBucketStorage { private final Bytes32 homeNodeId; public NodeBucketStorageImpl( - Database database, SerializerFactory serializerFactory, Bytes32 homeNodeId) { + Database database, SerializerFactory serializerFactory, NodeRecord homeNode) { DataSource nodeBucketsSource = database.createStorage(NODE_BUCKET_STORAGE_NAME); this.nodeBucketsTable = @@ -32,7 +33,13 @@ public NodeBucketStorageImpl( nodeBucketsSource, serializerFactory.getSerializer(NodeBucket.class), serializerFactory.getDeserializer(NodeBucket.class)); - this.homeNodeId = homeNodeId; + this.homeNodeId = homeNode.getNodeId(); + // Empty storage, saving home node + if (!nodeBucketsTable.get(0).isPresent()) { + NodeBucket zero = new NodeBucket(); + zero.put(NodeRecordInfo.createDefault(homeNode)); + nodeBucketsTable.put(0, zero); + } } @Override diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactory.java b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactory.java index d291b68be..6134b2a33 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactory.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactory.java @@ -3,7 +3,6 @@ import org.ethereum.beacon.chain.storage.impl.SerializerFactory; import org.ethereum.beacon.db.Database; import org.ethereum.beacon.discovery.enr.NodeRecord; -import tech.pegasys.artemis.util.bytes.Bytes32; import java.util.List; import java.util.function.Supplier; @@ -15,6 +14,6 @@ NodeTableStorage createTable( Supplier homeNodeSupplier, Supplier> bootNodes); - NodeBucketStorage createBuckets( - Database database, SerializerFactory serializerFactory, Bytes32 homeNodeId); + NodeBucketStorage createBucketStorage( + Database database, SerializerFactory serializerFactory, NodeRecord homeNode); } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactoryImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactoryImpl.java index db2bb5a7f..7af567065 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactoryImpl.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactoryImpl.java @@ -50,8 +50,8 @@ public NodeTableStorage createTable( } @Override - public NodeBucketStorage createBuckets( - Database database, SerializerFactory serializerFactory, Bytes32 homeNodeId) { - return new NodeBucketStorageImpl(database, serializerFactory, homeNodeId); + public NodeBucketStorage createBucketStorage( + Database database, SerializerFactory serializerFactory, NodeRecord homeNode) { + return new NodeBucketStorageImpl(database, serializerFactory, homeNode); } } diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java index 7e2d166be..d7480b41a 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java @@ -1,18 +1,13 @@ package org.ethereum.beacon.discovery; import org.ethereum.beacon.db.Database; -import org.ethereum.beacon.discovery.enr.EnrScheme; import org.ethereum.beacon.discovery.enr.NodeRecord; -import org.ethereum.beacon.discovery.enr.NodeRecordFactory; -import org.ethereum.beacon.discovery.message.DiscoveryMessage; -import org.ethereum.beacon.discovery.message.DiscoveryV5Message; -import org.ethereum.beacon.discovery.message.FindNodeMessage; -import org.ethereum.beacon.discovery.message.NodesMessage; import org.ethereum.beacon.discovery.packet.AuthHeaderMessagePacket; import org.ethereum.beacon.discovery.packet.MessagePacket; import org.ethereum.beacon.discovery.packet.RandomPacket; import org.ethereum.beacon.discovery.packet.UnknownPacket; import org.ethereum.beacon.discovery.packet.WhoAreYouPacket; +import org.ethereum.beacon.discovery.storage.NodeBucket; import org.ethereum.beacon.discovery.storage.NodeBucketStorage; import org.ethereum.beacon.discovery.storage.NodeTableStorage; import org.ethereum.beacon.discovery.storage.NodeTableStorageFactoryImpl; @@ -21,68 +16,26 @@ import org.javatuples.Pair; import org.junit.Test; import reactor.core.publisher.Flux; -import tech.pegasys.artemis.util.bytes.Bytes4; -import tech.pegasys.artemis.util.bytes.Bytes96; import tech.pegasys.artemis.util.bytes.BytesValue; -import tech.pegasys.artemis.util.uint.UInt64; -import java.net.InetAddress; import java.util.ArrayList; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import static org.ethereum.beacon.discovery.storage.NodeTableStorage.DEFAULT_SERIALIZER; -import static org.ethereum.beacon.discovery.task.TaskMessageFactory.DEFAULT_DISTANCE; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; /** Same as {@link DiscoveryNoNetworkTest} but using real network */ public class DiscoveryNetworkTest { - private static final NodeRecordFactory NODE_RECORD_FACTORY = NodeRecordFactory.DEFAULT; - private final BytesValue testKey1 = - BytesValue.fromHexString("eef77acb6c6a6eebc5b363a475ac583ec7eccdb42b6481424c60f59aa326547f"); - private final BytesValue testKey2 = - BytesValue.fromHexString("66fb62bfbd66b9177a138c1e5cddbe4f7c30c343e94e68df8769459cb1cde628"); - @Test public void test() throws Exception { // 1) start 2 nodes - NodeRecord nodeRecord1 = - NODE_RECORD_FACTORY.createFromValues( - EnrScheme.V4, - UInt64.valueOf(1), - Bytes96.EMPTY, - new ArrayList>() { - { - add( - Pair.with( - NodeRecord.FIELD_IP_V4, - Bytes4.wrap(InetAddress.getByName("127.0.0.1").getAddress()))); - add(Pair.with(NodeRecord.FIELD_UDP_V4, 30303)); - add( - Pair.with( - NodeRecord.FIELD_PKEY_SECP256K1, - BytesValue.fromHexString( - "0bfb48004b1698f05872cf18b1f278998ad8f7d4c135aa41f83744e7b850ab6b98"))); - } - }); - NodeRecord nodeRecord2 = - NODE_RECORD_FACTORY.createFromValues( - EnrScheme.V4, - UInt64.valueOf(1), - Bytes96.EMPTY, - new ArrayList>() { - { - add( - Pair.with( - NodeRecord.FIELD_IP_V4, - Bytes4.wrap(InetAddress.getByName("127.0.0.1").getAddress()))); - add(Pair.with(NodeRecord.FIELD_UDP_V4, 30304)); - add( - Pair.with( - NodeRecord.FIELD_PKEY_SECP256K1, - BytesValue.fromHexString( - "7ef3502240a42891771de732f5ee6bee3eb881939edf3e6008c0d07b502756e426"))); - } - }); + Pair nodePair1 = TestUtil.generateNode(30303); + Pair nodePair2 = TestUtil.generateNode(30304); + Pair nodePair3 = TestUtil.generateNode(40412); + NodeRecord nodeRecord1 = nodePair1.getValue1(); + NodeRecord nodeRecord2 = nodePair2.getValue1(); NodeTableStorageFactoryImpl nodeTableStorageFactory = new NodeTableStorageFactoryImpl(); Database database1 = Database.inMemoryDB(); Database database2 = Database.inMemoryDB(); @@ -98,8 +51,7 @@ public void test() throws Exception { } }); NodeBucketStorage nodeBucketStorage1 = - nodeTableStorageFactory.createBuckets( - database1, DEFAULT_SERIALIZER, nodeRecord1.getNodeId()); + nodeTableStorageFactory.createBucketStorage(database1, DEFAULT_SERIALIZER, nodeRecord1); NodeTableStorage nodeTableStorage2 = nodeTableStorageFactory.createTable( database2, @@ -112,14 +64,13 @@ public void test() throws Exception { } }); NodeBucketStorage nodeBucketStorage2 = - nodeTableStorageFactory.createBuckets( - database2, DEFAULT_SERIALIZER, nodeRecord2.getNodeId()); + nodeTableStorageFactory.createBucketStorage(database2, DEFAULT_SERIALIZER, nodeRecord2); DiscoveryManagerImpl discoveryManager1 = new DiscoveryManagerImpl( nodeTableStorage1.get(), nodeBucketStorage1, nodeRecord1, - testKey1, + nodePair1.getValue0(), Schedulers.createDefault().newSingleThreadDaemon("server-1"), Schedulers.createDefault().newSingleThreadDaemon("client-1")); DiscoveryManagerImpl discoveryManager2 = @@ -127,7 +78,7 @@ public void test() throws Exception { nodeTableStorage2.get(), nodeBucketStorage2, nodeRecord2, - testKey2, + nodePair2.getValue0(), Schedulers.createDefault().newSingleThreadDaemon("server-2"), Schedulers.createDefault().newSingleThreadDaemon("client-2")); @@ -136,10 +87,10 @@ public void test() throws Exception { // 3) Expect standard 1 => 2 dialog CountDownLatch randomSent1to2 = new CountDownLatch(1); - CountDownLatch authPacketSent1to2 = new CountDownLatch(1); CountDownLatch whoareyouSent2to1 = new CountDownLatch(1); - CountDownLatch findNodeSent2to1 = new CountDownLatch(1); - CountDownLatch nodesSent1to2 = new CountDownLatch(1); + CountDownLatch authPacketSent1to2 = new CountDownLatch(1); + CountDownLatch nodesSent2to1 = new CountDownLatch(1); + Flux.from(discoveryManager1.getOutgoingMessages()) .map(p -> new UnknownPacket(p.getPacket().getBytes())) .subscribe( @@ -156,17 +107,7 @@ public void test() throws Exception { System.out.println("1 => 2: " + authHeaderMessagePacket); authPacketSent1to2.countDown(); } else { - // 1 -> 2 NODES packet - MessagePacket messagePacket = networkPacket.getMessagePacket(); - System.out.println("1 => 2: " + messagePacket); - DiscoveryMessage discoveryMessage = messagePacket.getMessage(); - assert IdentityScheme.V5.equals(discoveryMessage.getIdentityScheme()); - DiscoveryV5Message discoveryV5Message = (DiscoveryV5Message) discoveryMessage; - NodesMessage nodesMessage = (NodesMessage) discoveryV5Message.create(); - assert 1 == nodesMessage.getTotal(); - assert 1 == nodesMessage.getNodeRecordsSize(); - assert 1 == nodesMessage.getNodeRecords().size(); - nodesSent1to2.countDown(); + throw new RuntimeException("Not expected!"); } }); Flux.from(discoveryManager2.getOutgoingMessages()) @@ -179,15 +120,10 @@ public void test() throws Exception { System.out.println("2 => 1: " + whoAreYouPacket); whoareyouSent2to1.countDown(); } else { - // 2 -> 1 findNode + // 2 -> 1 nodes MessagePacket messagePacket = networkPacket.getMessagePacket(); System.out.println("2 => 1: " + messagePacket); - DiscoveryMessage discoveryMessage = messagePacket.getMessage(); - assert IdentityScheme.V5.equals(discoveryMessage.getIdentityScheme()); - DiscoveryV5Message discoveryV5Message = (DiscoveryV5Message) discoveryMessage; - FindNodeMessage findNodeMessage = (FindNodeMessage) discoveryV5Message.create(); - assert DEFAULT_DISTANCE == findNodeMessage.getDistance(); - findNodeSent2to1.countDown(); + nodesSent2to1.countDown(); } }); @@ -196,9 +132,17 @@ public void test() throws Exception { assert randomSent1to2.await(1, TimeUnit.SECONDS); assert whoareyouSent2to1.await(1, TimeUnit.SECONDS); + int distance1To2 = Functions.logDistance(nodeRecord1.getNodeId(), nodeRecord2.getNodeId()); + assertFalse(nodeBucketStorage1.get(distance1To2).isPresent()); assert authPacketSent1to2.await(1, TimeUnit.SECONDS); - assert findNodeSent2to1.await(1, TimeUnit.SECONDS); - assert nodesSent1to2.await(1, TimeUnit.SECONDS); + assert nodesSent2to1.await(1, TimeUnit.SECONDS); + Thread.sleep(50); + // 1 sent findnodes to 2, received only (2) in answer, because 3 is not checked + // 1 added 2 to its nodeBuckets, because its now checked, but not before + NodeBucket bucketAt1With2 = nodeBucketStorage1.get(distance1To2).get(); + assertEquals(1, bucketAt1With2.size()); + assertEquals( + nodeRecord2.getNodeId(), bucketAt1With2.getNodeRecords().get(0).getNode().getNodeId()); } // TODO: discovery tasks are emitted from time to time as they should diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java index 9a9b8fb95..d00625225 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java @@ -1,19 +1,14 @@ package org.ethereum.beacon.discovery; import org.ethereum.beacon.db.Database; -import org.ethereum.beacon.discovery.enr.EnrScheme; import org.ethereum.beacon.discovery.enr.NodeRecord; -import org.ethereum.beacon.discovery.enr.NodeRecordFactory; -import org.ethereum.beacon.discovery.message.DiscoveryMessage; -import org.ethereum.beacon.discovery.message.DiscoveryV5Message; -import org.ethereum.beacon.discovery.message.FindNodeMessage; -import org.ethereum.beacon.discovery.message.NodesMessage; import org.ethereum.beacon.discovery.mock.DiscoveryManagerNoNetwork; import org.ethereum.beacon.discovery.packet.AuthHeaderMessagePacket; import org.ethereum.beacon.discovery.packet.MessagePacket; import org.ethereum.beacon.discovery.packet.RandomPacket; import org.ethereum.beacon.discovery.packet.UnknownPacket; import org.ethereum.beacon.discovery.packet.WhoAreYouPacket; +import org.ethereum.beacon.discovery.storage.NodeBucket; import org.ethereum.beacon.discovery.storage.NodeBucketStorage; import org.ethereum.beacon.discovery.storage.NodeTableStorage; import org.ethereum.beacon.discovery.storage.NodeTableStorageFactoryImpl; @@ -23,71 +18,30 @@ import org.javatuples.Pair; import org.junit.Test; import reactor.core.publisher.Flux; -import tech.pegasys.artemis.util.bytes.Bytes4; -import tech.pegasys.artemis.util.bytes.Bytes96; import tech.pegasys.artemis.util.bytes.BytesValue; -import tech.pegasys.artemis.util.uint.UInt64; -import java.net.InetAddress; import java.util.ArrayList; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import static org.ethereum.beacon.discovery.storage.NodeTableStorage.DEFAULT_SERIALIZER; -import static org.ethereum.beacon.discovery.task.TaskMessageFactory.DEFAULT_DISTANCE; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; /** * Discovery test without real network, instead outgoing stream of each peer is connected with * incoming of another and vice versa */ public class DiscoveryNoNetworkTest { - private static final NodeRecordFactory NODE_RECORD_FACTORY = NodeRecordFactory.DEFAULT; - private final BytesValue testKey1 = - BytesValue.fromHexString("eef77acb6c6a6eebc5b363a475ac583ec7eccdb42b6481424c60f59aa326547f"); - private final BytesValue testKey2 = - BytesValue.fromHexString("66fb62bfbd66b9177a138c1e5cddbe4f7c30c343e94e68df8769459cb1cde628"); @Test public void test() throws Exception { // 1) start 2 nodes - NodeRecord nodeRecord1 = - NODE_RECORD_FACTORY.createFromValues( - EnrScheme.V4, - UInt64.valueOf(1), - Bytes96.EMPTY, - new ArrayList>() { - { - add( - Pair.with( - NodeRecord.FIELD_IP_V4, - Bytes4.wrap(InetAddress.getByName("127.0.0.1").getAddress()))); - add(Pair.with(NodeRecord.FIELD_UDP_V4, 30303)); - add( - Pair.with( - NodeRecord.FIELD_PKEY_SECP256K1, - BytesValue.fromHexString( - "0bfb48004b1698f05872cf18b1f278998ad8f7d4c135aa41f83744e7b850ab6b98"))); - } - }); - NodeRecord nodeRecord2 = - NODE_RECORD_FACTORY.createFromValues( - EnrScheme.V4, - UInt64.valueOf(1), - Bytes96.EMPTY, - new ArrayList>() { - { - add( - Pair.with( - NodeRecord.FIELD_IP_V4, - Bytes4.wrap(InetAddress.getByName("192.168.0.1").getAddress()))); - add(Pair.with(NodeRecord.FIELD_UDP_V4, 30303)); - add( - Pair.with( - NodeRecord.FIELD_PKEY_SECP256K1, - BytesValue.fromHexString( - "7ef3502240a42891771de732f5ee6bee3eb881939edf3e6008c0d07b502756e426"))); - } - }); + Pair nodePair1 = TestUtil.generateNode(30303); + Pair nodePair2 = TestUtil.generateNode(30304); + Pair nodePair3 = TestUtil.generateNode(40412); + NodeRecord nodeRecord1 = nodePair1.getValue1(); + NodeRecord nodeRecord2 = nodePair2.getValue1(); NodeTableStorageFactoryImpl nodeTableStorageFactory = new NodeTableStorageFactoryImpl(); Database database1 = Database.inMemoryDB(); Database database2 = Database.inMemoryDB(); @@ -103,8 +57,7 @@ public void test() throws Exception { } }); NodeBucketStorage nodeBucketStorage1 = - nodeTableStorageFactory.createBuckets( - database1, DEFAULT_SERIALIZER, nodeRecord1.getNodeId()); + nodeTableStorageFactory.createBucketStorage(database1, DEFAULT_SERIALIZER, nodeRecord1); NodeTableStorage nodeTableStorage2 = nodeTableStorageFactory.createTable( database2, @@ -114,11 +67,11 @@ public void test() throws Exception { new ArrayList() { { add(nodeRecord1); + add(nodePair3.getValue1()); } }); NodeBucketStorage nodeBucketStorage2 = - nodeTableStorageFactory.createBuckets( - database2, DEFAULT_SERIALIZER, nodeRecord2.getNodeId()); + nodeTableStorageFactory.createBucketStorage(database2, DEFAULT_SERIALIZER, nodeRecord2); SimpleProcessor from1to2 = new SimpleProcessor<>( Schedulers.createDefault().newSingleThreadDaemon("from1to2-thread"), "from1to2"); @@ -127,10 +80,18 @@ public void test() throws Exception { Schedulers.createDefault().newSingleThreadDaemon("from2to1-thread"), "from2to1"); DiscoveryManagerNoNetwork discoveryManager1 = new DiscoveryManagerNoNetwork( - nodeTableStorage1.get(), nodeBucketStorage1, nodeRecord1, testKey1, from2to1); + nodeTableStorage1.get(), + nodeBucketStorage1, + nodeRecord1, + nodePair1.getValue0(), + from2to1); DiscoveryManagerNoNetwork discoveryManager2 = new DiscoveryManagerNoNetwork( - nodeTableStorage2.get(), nodeBucketStorage2, nodeRecord2, testKey2, from1to2); + nodeTableStorage2.get(), + nodeBucketStorage2, + nodeRecord2, + nodePair2.getValue0(), + from1to2); discoveryManager1.start(); discoveryManager2.start(); @@ -142,10 +103,9 @@ public void test() throws Exception { // 3) Expect standard 1 => 2 dialog CountDownLatch randomSent1to2 = new CountDownLatch(1); - CountDownLatch authPacketSent1to2 = new CountDownLatch(1); CountDownLatch whoareyouSent2to1 = new CountDownLatch(1); - CountDownLatch findNodeSent2to1 = new CountDownLatch(1); - CountDownLatch nodesSent1to2 = new CountDownLatch(1); + CountDownLatch authPacketSent1to2 = new CountDownLatch(1); + CountDownLatch nodesSent2to1 = new CountDownLatch(1); Flux.from(from1to2) .map(UnknownPacket::new) .subscribe( @@ -162,17 +122,7 @@ public void test() throws Exception { System.out.println("1 => 2: " + authHeaderMessagePacket); authPacketSent1to2.countDown(); } else { - // 1 -> 2 NODES packet - MessagePacket messagePacket = networkPacket.getMessagePacket(); - System.out.println("1 => 2: " + messagePacket); - DiscoveryMessage discoveryMessage = messagePacket.getMessage(); - assert IdentityScheme.V5.equals(discoveryMessage.getIdentityScheme()); - DiscoveryV5Message discoveryV5Message = (DiscoveryV5Message) discoveryMessage; - NodesMessage nodesMessage = (NodesMessage) discoveryV5Message.create(); - assert 1 == nodesMessage.getTotal(); - assert 1 == nodesMessage.getNodeRecordsSize(); - assert 1 == nodesMessage.getNodeRecords().size(); - nodesSent1to2.countDown(); + throw new RuntimeException("Not expected!"); } }); Flux.from(from2to1) @@ -185,15 +135,10 @@ public void test() throws Exception { System.out.println("2 => 1: " + whoAreYouPacket); whoareyouSent2to1.countDown(); } else { - // 2 -> 1 findNode + // 2 -> 1 nodes MessagePacket messagePacket = networkPacket.getMessagePacket(); System.out.println("2 => 1: " + messagePacket); - DiscoveryMessage discoveryMessage = messagePacket.getMessage(); - assert IdentityScheme.V5.equals(discoveryMessage.getIdentityScheme()); - DiscoveryV5Message discoveryV5Message = (DiscoveryV5Message) discoveryMessage; - FindNodeMessage findNodeMessage = (FindNodeMessage) discoveryV5Message.create(); - assert DEFAULT_DISTANCE == findNodeMessage.getDistance(); - findNodeSent2to1.countDown(); + nodesSent2to1.countDown(); } }); @@ -202,9 +147,17 @@ public void test() throws Exception { assert randomSent1to2.await(1, TimeUnit.SECONDS); assert whoareyouSent2to1.await(1, TimeUnit.SECONDS); + int distance1To2 = Functions.logDistance(nodeRecord1.getNodeId(), nodeRecord2.getNodeId()); + assertFalse(nodeBucketStorage1.get(distance1To2).isPresent()); assert authPacketSent1to2.await(1, TimeUnit.SECONDS); - assert findNodeSent2to1.await(1, TimeUnit.SECONDS); - assert nodesSent1to2.await(1, TimeUnit.SECONDS); + assert nodesSent2to1.await(1, TimeUnit.SECONDS); + Thread.sleep(50); + // 1 sent findnodes to 2, received only (2) in answer, because 3 is not checked + // 1 added 2 to its nodeBuckets, because its now checked, but not before + NodeBucket bucketAt1With2 = nodeBucketStorage1.get(distance1To2).get(); + assertEquals(1, bucketAt1With2.size()); + assertEquals( + nodeRecord2.getNodeId(), bucketAt1With2.getNodeRecords().get(0).getNode().getNodeId()); } // TODO: discovery tasks are emitted from time to time as they should diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/HandshakeHandlersTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/HandshakeHandlersTest.java index 6ed66110d..beab5e45a 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/HandshakeHandlersTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/HandshakeHandlersTest.java @@ -1,9 +1,7 @@ package org.ethereum.beacon.discovery; import org.ethereum.beacon.db.Database; -import org.ethereum.beacon.discovery.enr.EnrScheme; import org.ethereum.beacon.discovery.enr.NodeRecord; -import org.ethereum.beacon.discovery.enr.NodeRecordFactory; import org.ethereum.beacon.discovery.message.FindNodeMessage; import org.ethereum.beacon.discovery.packet.Packet; import org.ethereum.beacon.discovery.packet.WhoAreYouPacket; @@ -18,14 +16,10 @@ import org.ethereum.beacon.discovery.task.TaskType; import org.javatuples.Pair; import org.junit.Test; -import org.web3j.crypto.ECKeyPair; import tech.pegasys.artemis.util.bytes.Bytes32; -import tech.pegasys.artemis.util.bytes.Bytes4; -import tech.pegasys.artemis.util.bytes.Bytes96; import tech.pegasys.artemis.util.bytes.BytesValue; import tech.pegasys.artemis.util.uint.UInt64; -import java.net.InetAddress; import java.util.ArrayList; import java.util.Random; import java.util.concurrent.CountDownLatch; @@ -39,47 +33,15 @@ import static org.junit.Assert.assertTrue; public class HandshakeHandlersTest { - private final BytesValue testKey1 = - BytesValue.fromHexString("eef77acb6c6a6eebc5b363a475ac583ec7eccdb42b6481424c60f59aa326547f"); - private final BytesValue testKey2 = - BytesValue.fromHexString("66fb62bfbd66b9177a138c1e5cddbe4f7c30c343e94e68df8769459cb1cde628"); - private Bytes32 nodeId1; - private Bytes32 nodeId2; - - public HandshakeHandlersTest() { - byte[] homeNodeIdBytes = new byte[32]; - homeNodeIdBytes[0] = 0x01; - byte[] destNodeIdBytes = new byte[32]; - destNodeIdBytes[0] = 0x02; - this.nodeId1 = Bytes32.wrap(homeNodeIdBytes); - this.nodeId2 = Bytes32.wrap(destNodeIdBytes); - } @Test public void authHeaderHandlerTest() throws Exception { // Node1 - NodeRecord nodeRecord1 = - NodeRecordFactory.DEFAULT.createFromValues( - EnrScheme.V4, - UInt64.valueOf(1), - Bytes96.EMPTY, - Pair.with( - NodeRecord.FIELD_IP_V4, - Bytes4.wrap(InetAddress.getByName("127.0.0.1").getAddress())), - Pair.with(NodeRecord.FIELD_UDP_V4, 30303), - Pair.with(NodeRecord.FIELD_PKEY_SECP256K1, BytesValue.wrap(ECKeyPair.create(testKey1.extractArray()).getPublicKey().toByteArray()))); + Pair nodePair1 = TestUtil.generateNode(30303); + NodeRecord nodeRecord1 = nodePair1.getValue1(); // Node2 - NodeRecord nodeRecord2 = - NodeRecordFactory.DEFAULT.createFromValues( - EnrScheme.V4, - UInt64.valueOf(1), - Bytes96.EMPTY, - Pair.with( - NodeRecord.FIELD_IP_V4, - Bytes4.wrap(InetAddress.getByName("127.0.0.1").getAddress())), - Pair.with(NodeRecord.FIELD_UDP_V4, 30304), - Pair.with(NodeRecord.FIELD_PKEY_SECP256K1, BytesValue.wrap(ECKeyPair.create(testKey2.extractArray()).getPublicKey().toByteArray()))); - + Pair nodePair2 = TestUtil.generateNode(30304); + NodeRecord nodeRecord2 = nodePair2.getValue1(); Random rnd = new Random(); NodeTableStorageFactoryImpl nodeTableStorageFactory = new NodeTableStorageFactoryImpl(); Database database1 = Database.inMemoryDB(); @@ -96,8 +58,7 @@ public void authHeaderHandlerTest() throws Exception { } }); NodeBucketStorage nodeBucketStorage1 = - nodeTableStorageFactory.createBuckets( - database1, DEFAULT_SERIALIZER, nodeRecord1.getNodeId()); + nodeTableStorageFactory.createBucketStorage(database1, DEFAULT_SERIALIZER, nodeRecord1); NodeTableStorage nodeTableStorage2 = nodeTableStorageFactory.createTable( database2, @@ -110,8 +71,7 @@ public void authHeaderHandlerTest() throws Exception { } }); NodeBucketStorage nodeBucketStorage2 = - nodeTableStorageFactory.createBuckets( - database2, DEFAULT_SERIALIZER, nodeRecord2.getNodeId()); + nodeTableStorageFactory.createBucketStorage(database2, DEFAULT_SERIALIZER, nodeRecord2); // Node1 create AuthHeaderPacket final Packet[] authHeaderPacket = new Packet[1]; @@ -126,7 +86,7 @@ public void authHeaderHandlerTest() throws Exception { new NodeSession( nodeRecord2, nodeRecord1, - testKey1, + nodePair1.getValue0(), nodeTableStorage1.get(), nodeBucketStorage1, authTagRepository1, @@ -140,7 +100,7 @@ public void authHeaderHandlerTest() throws Exception { new NodeSession( nodeRecord1, nodeRecord2, - testKey2, + nodePair2.getValue0(), nodeTableStorage2.get(), nodeBucketStorage2, new AuthTagRepository(), @@ -156,7 +116,7 @@ public void authHeaderHandlerTest() throws Exception { authTagRepository1.put(authTag, nodeSessionAt1For2); envelopeAt1From2.put( Field.PACKET_WHOAREYOU, - WhoAreYouPacket.create(nodeId1, authTag, idNonce, UInt64.ZERO)); + WhoAreYouPacket.create(nodePair1.getValue1().getNodeId(), authTag, idNonce, UInt64.ZERO)); envelopeAt1From2.put(Field.SESSION, nodeSessionAt1For2); nodeSessionAt1For2.saveTask(TaskType.FINDNODE); whoAreYouPacketHandlerNode1.handle(envelopeAt1From2); diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/TestUtil.java b/discovery/src/test/java/org/ethereum/beacon/discovery/TestUtil.java new file mode 100644 index 000000000..0801c44bd --- /dev/null +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/TestUtil.java @@ -0,0 +1,62 @@ +package org.ethereum.beacon.discovery; + +import org.ethereum.beacon.discovery.enr.EnrScheme; +import org.ethereum.beacon.discovery.enr.NodeRecord; +import org.ethereum.beacon.discovery.enr.NodeRecordFactory; +import org.javatuples.Pair; +import org.web3j.crypto.ECKeyPair; +import tech.pegasys.artemis.util.bytes.Bytes4; +import tech.pegasys.artemis.util.bytes.Bytes96; +import tech.pegasys.artemis.util.bytes.BytesValue; +import tech.pegasys.artemis.util.uint.UInt64; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.Random; + +public class TestUtil { + private static final NodeRecordFactory NODE_RECORD_FACTORY = NodeRecordFactory.DEFAULT; + private static final int SEED = 123456789; + + /** + * Generates node on 127.0.0.1 with provided port. Node key is random, but always the same for the + * same port + * + * @return + */ + public static Pair generateNode(int port) { + final Random rnd = new Random(SEED); + Bytes4 localIp = null; + try { + localIp = Bytes4.wrap(InetAddress.getByName("127.0.0.1").getAddress()); + } catch (UnknownHostException e) { + throw new RuntimeException(e); + } + final Bytes4 finalLocalIp = localIp; + for (int i = 0; i < port; ++i) { + rnd.nextBoolean(); // skip according to input + } + byte[] privateKey = new byte[32]; + rnd.nextBytes(privateKey); + ECKeyPair ecKeyPair = ECKeyPair.create(privateKey); + BytesValue publicKey = BytesValue.wrap(ecKeyPair.getPublicKey().toByteArray()); + if (publicKey.size() == 65) { + publicKey = publicKey.slice(1); // slice leading zero + } + final BytesValue finalPublicKey = publicKey; + NodeRecord nodeRecord = + NODE_RECORD_FACTORY.createFromValues( + EnrScheme.V4, + UInt64.valueOf(1), + Bytes96.EMPTY, + new ArrayList>() { + { + add(Pair.with(NodeRecord.FIELD_IP_V4, finalLocalIp)); + add(Pair.with(NodeRecord.FIELD_UDP_V4, port)); + add(Pair.with(NodeRecord.FIELD_PKEY_SECP256K1, finalPublicKey)); + } + }); + return Pair.with(BytesValue.wrap(privateKey), nodeRecord); + } +} diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeBucketTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeBucketTest.java index 90e87a7ab..5b9211f1d 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeBucketTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeBucketTest.java @@ -103,8 +103,7 @@ public void testStorage() { Database database = Database.inMemoryDB(); NodeTableStorageFactoryImpl nodeTableStorageFactory = new NodeTableStorageFactoryImpl(); NodeBucketStorage nodeBucketStorage = - nodeTableStorageFactory.createBuckets( - database, DEFAULT_SERIALIZER, initial.getNode().getNodeId()); + nodeTableStorageFactory.createBucketStorage(database, DEFAULT_SERIALIZER, initial.getNode()); for (int i = 0; i < 20; ) { NodeRecordInfo nodeRecordInfo = generateUniqueRecord(); From 85209d10c6c0c0043f56b5ec8c33e26d28d8258b Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Mon, 21 Oct 2019 06:09:07 +0300 Subject: [PATCH 39/77] discovery: remove duplicated commented out test --- .../discovery/HandshakeHandlersTest.java | 104 ------------------ 1 file changed, 104 deletions(-) diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/HandshakeHandlersTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/HandshakeHandlersTest.java index beab5e45a..87e060cdb 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/HandshakeHandlersTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/HandshakeHandlersTest.java @@ -2,7 +2,6 @@ import org.ethereum.beacon.db.Database; import org.ethereum.beacon.discovery.enr.NodeRecord; -import org.ethereum.beacon.discovery.message.FindNodeMessage; import org.ethereum.beacon.discovery.packet.Packet; import org.ethereum.beacon.discovery.packet.WhoAreYouPacket; import org.ethereum.beacon.discovery.pipeline.Envelope; @@ -132,107 +131,4 @@ public void authHeaderHandlerTest() throws Exception { authHeaderMessagePacketHandlerNode2.handle(envelopeAt2From1); assertTrue(nodeSessionAt2For1.isAuthenticated()); } - // net := newHandshakeTest() - // defer net.close() - // - // var ( - // idA = net.nodeA.id() - // addrA = net.nodeA.addr() - // challenge = &whoareyouV5{AuthTag: []byte("authresp"), RecordSeq: 0, node: net.nodeB.n()} - // nonce = make([]byte, gcmNonceSize) - // ) - // header, _, _ := net.nodeA.c.makeAuthHeader(nonce, challenge) - // challenge.node = nil // force ENR signature verification in decoder - // b.ResetTimer() - // - // for i := 0; i < b.N; i++ { - // _, _, err := net.nodeB.c.decodeAuthResp(idA, addrA, header, challenge) - // if err != nil { - // b.Fatal(err) - // } - // } - - @Test - public void testHandshake() { - // A -> B RANDOM PACKET - FindNodeMessage findNodeMessage = new FindNodeMessage(BytesValue.fromHexString("01"), 10); - - // func TestHandshakeV5(t *testing.T) { - // t.Parallel() - // net := newHandshakeTest() - // defer net.close() - // - // // A -> B RANDOM PACKET - // packet, _ := net.nodeA.encode(t, net.nodeB, &findnodeV5{}) - // resp := net.nodeB.expectDecode(t, p_unknownV5, packet) - // - // // A <- B WHOAREYOU - // challenge := &whoareyouV5{ - // AuthTag: resp.(*unknownV5).AuthTag, - // IDNonce: testIDnonce, - // RecordSeq: 0, - // } - // whoareyou, _ := net.nodeB.encode(t, net.nodeA, challenge) - // net.nodeA.expectDecode(t, p_whoareyouV5, whoareyou) - // - // // A -> B FINDNODE - // findnode, _ := net.nodeA.encodeWithChallenge(t, net.nodeB, challenge, &findnodeV5{}) - // net.nodeB.expectDecode(t, p_findnodeV5, findnode) - // if len(net.nodeB.c.handshakes) > 0 { - // t.Fatalf("node B didn't remove handshake from challenge map") - // } - // - // // A <- B NODES - // nodes, _ := net.nodeB.encode(t, net.nodeA, &nodesV5{Total: 1}) - // net.nodeA.expectDecode(t, p_nodesV5, nodes) - // - } - - // private BytesValue encodeWithChallenge(V5Message v5Message, WhoAreYouPacket challenge) { - // Copy challenge and add destination node. This avoids sharing challenge among the two codecs. - // var challenge *whoareyouV5 - // if c != nil { - // challengeCopy := *c - // challenge = &challengeCopy - // challenge.node = to.n() - // } - // // Encode to destination. - // enc, authTag, err := n.c.encode(to.id(), to.addr(), p, challenge) - // if err != nil { - // t.Fatal(fmt.Errorf("(%s) %v", n.ln.ID().TerminalString(), err)) - // } - // t.Logf("(%s) -> (%s) %s\n%s", n.ln.ID().TerminalString(), to.id().TerminalString(), - // p.name(), hex.Dump(enc)) - // return enc, authTag - // } - - // private BytesValue encode(V5Message v5Message) { - // // Encode to destination. - // enc, authTag, err := n.c.encode(to.id(), to.addr(), p, challenge) - // return enc, authTag - - // // encode encodes a packet to a node. 'id' and 'addr' specify the destination node. The - //// 'token' parameter should be set to the token in the most recently received WHOAREYOU - //// packet. - // func (c *wireCodec) encode(id enode.ID, addr *net.UDPAddr, packet packetV5, challenge - // *whoareyouV5) ([]byte, []byte, error) { - // if packet.kind() == p_whoareyouV5 { - // p := packet.(*whoareyouV5) - // enc, err := c.encodeWhoareyou(id, p) - // if err == nil { - // c.storeSentHandshake(id, addr, p) - // } - // return enc, nil, err - // } - // // Ensure calling code sets node if needed. - // if challenge != nil && challenge.node == nil { - // panic("BUG: missing challenge.node in encode") - // } - // _, writeKey := c.loadKeys(id, addr) - // if writeKey != nil || challenge != nil { - // return c.encodeEncrypted(id, addr, packet, writeKey, challenge) - // } - // return c.encodeRandom(id) - // } - // } } From 89d054e2fc0ea311cefca542a77619636d2be411 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Mon, 21 Oct 2019 07:08:56 +0300 Subject: [PATCH 40/77] discovery: Clarifying 65-bytes public key issue --- .../handler/WhoAreYouPacketHandler.java | 7 ++-- .../beacon/discovery/RlpExtraTest.java | 35 ------------------- .../ethereum/beacon/discovery/SubTests.java | 30 ++++++++++++++++ .../ethereum/beacon/discovery/TestUtil.java | 10 +++--- .../java/org/ethereum/beacon/util/Utils.java | 35 +++++++++++++++---- 5 files changed, 66 insertions(+), 51 deletions(-) delete mode 100644 discovery/src/test/java/org/ethereum/beacon/discovery/RlpExtraTest.java create mode 100644 discovery/src/test/java/org/ethereum/beacon/discovery/SubTests.java diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouPacketHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouPacketHandler.java index af421a4f2..8ac4b7bb6 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouPacketHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouPacketHandler.java @@ -15,6 +15,7 @@ import org.ethereum.beacon.discovery.pipeline.HandlerUtil; import org.ethereum.beacon.discovery.task.TaskMessageFactory; import org.ethereum.beacon.discovery.task.TaskType; +import org.ethereum.beacon.util.Utils; import org.javatuples.Triplet; import org.web3j.crypto.ECKeyPair; import tech.pegasys.artemis.util.bytes.BytesValue; @@ -81,10 +82,8 @@ public void handle(Envelope envelope) { "Type %s in envelope #%s is not known", session.loadTask(), envelope.getId())); } - BytesValue ephemeralPubKey = BytesValue.wrap(ephemeralKey.getPublicKey().toByteArray()); - if (ephemeralPubKey.size() == 65) { - ephemeralPubKey = ephemeralPubKey.slice(1); // slice leading 00 - } + BytesValue ephemeralPubKey = + BytesValue.wrap(Utils.extractBytesFromUnsignedBigInt(ephemeralKey.getPublicKey())); AuthHeaderMessagePacket response = AuthHeaderMessagePacket.create( session.getHomeNodeId(), diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/RlpExtraTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/RlpExtraTest.java deleted file mode 100644 index 5671007da..000000000 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/RlpExtraTest.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.ethereum.beacon.discovery; - -import org.junit.Test; -import org.web3j.rlp.RlpDecoder; -import org.web3j.rlp.RlpEncoder; -import org.web3j.rlp.RlpList; -import org.web3j.rlp.RlpString; - -import static org.junit.Assert.assertEquals; - -public class RlpExtraTest { - private static final String ABC = "abc"; - private static final int INT = 12345; - - /** - * Testing what could we do when rlp is appended with some extra data, how could we split rlp and - * that data - */ - @Test - public void testMakeRlpExtra() { - RlpList rlpList = new RlpList(RlpString.create(ABC), RlpString.create(INT)); - byte[] encoded = RlpEncoder.encode(rlpList); - byte[] encodedPlus = new byte[encoded.length + 2]; - System.arraycopy(encoded, 0, encodedPlus, 0, encoded.length); - encodedPlus[encodedPlus.length - 2] = 0x12; - encodedPlus[encodedPlus.length - 1] = 0x34; - RlpList rlpList1 = RlpDecoder.decode(encodedPlus); - RlpList rlpList2 = ((RlpList) rlpList1.getValues().get(0)); - assertEquals(ABC, new String(((RlpString) rlpList2.getValues().get(0)).getBytes())); - assertEquals(INT, ((RlpString) rlpList2.getValues().get(1)).asPositiveBigInteger().intValue()); - // but what else could we do? - int length = RlpEncoder.encode(rlpList2).length; - assertEquals(length + 2, encodedPlus.length); - } -} diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/SubTests.java b/discovery/src/test/java/org/ethereum/beacon/discovery/SubTests.java new file mode 100644 index 000000000..191b21bb2 --- /dev/null +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/SubTests.java @@ -0,0 +1,30 @@ +package org.ethereum.beacon.discovery; + +import org.ethereum.beacon.util.Utils; +import org.junit.Test; +import org.web3j.crypto.ECKeyPair; +import tech.pegasys.artemis.util.bytes.BytesValue; + +import java.math.BigInteger; + +import static org.junit.Assert.assertEquals; + +/** + * Secondary tests not directly related to discovery but clarifying functions used somewhere in + * discovery routines + */ +public class SubTests { + /** + * Tests BigInteger to byte[]. Take a look at {@link + * Utils#extractBytesFromUnsignedBigInt(BigInteger)} for understanding the issue. + */ + @Test + public void testPubKeyBadPrefix() { + BytesValue privKey = + BytesValue.fromHexString( + "0xade78b68f25611ea57532f86bf01da909cc463465ed9efce9395403ff7fc99b5"); + ECKeyPair badKey = ECKeyPair.create(privKey.extractArray()); + byte[] pubKey = Utils.extractBytesFromUnsignedBigInt(badKey.getPublicKey()); + assertEquals(64, pubKey.length); + } +} diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/TestUtil.java b/discovery/src/test/java/org/ethereum/beacon/discovery/TestUtil.java index 0801c44bd..7f2236610 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/TestUtil.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/TestUtil.java @@ -3,6 +3,7 @@ import org.ethereum.beacon.discovery.enr.EnrScheme; import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.enr.NodeRecordFactory; +import org.ethereum.beacon.util.Utils; import org.javatuples.Pair; import org.web3j.crypto.ECKeyPair; import tech.pegasys.artemis.util.bytes.Bytes4; @@ -40,11 +41,8 @@ public static Pair generateNode(int port) { byte[] privateKey = new byte[32]; rnd.nextBytes(privateKey); ECKeyPair ecKeyPair = ECKeyPair.create(privateKey); - BytesValue publicKey = BytesValue.wrap(ecKeyPair.getPublicKey().toByteArray()); - if (publicKey.size() == 65) { - publicKey = publicKey.slice(1); // slice leading zero - } - final BytesValue finalPublicKey = publicKey; + final BytesValue pubKey = + BytesValue.wrap(Utils.extractBytesFromUnsignedBigInt(ecKeyPair.getPublicKey())); NodeRecord nodeRecord = NODE_RECORD_FACTORY.createFromValues( EnrScheme.V4, @@ -54,7 +52,7 @@ public static Pair generateNode(int port) { { add(Pair.with(NodeRecord.FIELD_IP_V4, finalLocalIp)); add(Pair.with(NodeRecord.FIELD_UDP_V4, port)); - add(Pair.with(NodeRecord.FIELD_PKEY_SECP256K1, finalPublicKey)); + add(Pair.with(NodeRecord.FIELD_PKEY_SECP256K1, pubKey)); } }); return Pair.with(BytesValue.wrap(privateKey), nodeRecord); diff --git a/util/src/main/java/org/ethereum/beacon/util/Utils.java b/util/src/main/java/org/ethereum/beacon/util/Utils.java index 668498c23..e2c565e41 100644 --- a/util/src/main/java/org/ethereum/beacon/util/Utils.java +++ b/util/src/main/java/org/ethereum/beacon/util/Utils.java @@ -1,5 +1,6 @@ package org.ethereum.beacon.util; +import java.math.BigInteger; import java.util.Collections; import java.util.LinkedHashMap; import java.util.Map; @@ -19,7 +20,8 @@ public static Function> nullableFlatMap(Function func) return n -> n != null ? Stream.of(func.apply(n)) : Stream.empty(); } - public static void futureForward(CompletableFuture result, CompletableFuture forwardToFuture) { + public static void futureForward( + CompletableFuture result, CompletableFuture forwardToFuture) { result.whenComplete( (res, t) -> { if (t != null) { @@ -31,10 +33,31 @@ public static void futureForward(CompletableFuture result, CompletableFut } public static Set newLRUSet(int size) { - return Collections.newSetFromMap(new LinkedHashMap() { - protected boolean removeEldestEntry(Map.Entry eldest) { - return size() > size; - } - }); + return Collections.newSetFromMap( + new LinkedHashMap() { + protected boolean removeEldestEntry(Map.Entry eldest) { + return size() > size; + } + }); + } + + /** + * @return byte array representation of BigInteger for unsigned numeric + *

{@link BigInteger#toByteArray()} adds a bit for the sign. If you work with unsigned + * numerics it's always a 0. But if an integer uses exactly 8-some bits, sign bit will add an + * extra 0 byte to the result, which could broke some things. This method removes this + * redundant prefix byte when extracting byte array from BigInteger + */ + public static byte[] extractBytesFromUnsignedBigInt(BigInteger bigInteger) { + byte[] bigIntBytes = bigInteger.toByteArray(); + byte[] res; + if (bigIntBytes[0] == 0) { + res = new byte[bigIntBytes.length - 1]; + System.arraycopy(bigIntBytes, 1, res, 0, res.length); + } else { + res = bigIntBytes; + } + + return res; } } From 5eb8c2129f7585902d69517ed893b66af32b26e8 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Mon, 21 Oct 2019 07:43:42 +0300 Subject: [PATCH 41/77] discovery: remove TODO on merge NodeRecord, because we should prefer any new info against old one completely --- .../ethereum/beacon/discovery/message/handler/NodesHandler.java | 1 - 1 file changed, 1 deletion(-) diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/NodesHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/NodesHandler.java index fb59e30ac..c02826bda 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/NodesHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/NodesHandler.java @@ -53,7 +53,6 @@ public void handle(NodesMessage message, NodeSession session) { NodeRecordInfo nodeRecordInfo = NodeRecordInfo.createDefault(nodeRecordV5); if (!session.getNodeTable().getNode(nodeRecordV5.getNodeId()).isPresent()) { session.getNodeTable().save(nodeRecordInfo); - // TODO: should we update-merge? } session.putRecordInBucket(nodeRecordInfo); }); From 68b1cba7a2fa5b5dbc7e97e8e6b4ea97c3331065 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Mon, 21 Oct 2019 07:51:50 +0300 Subject: [PATCH 42/77] discovery: fix whoareyou received in case of session expiration was handled as bad packet --- .../handler/WhoAreYouSessionResolver.java | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouSessionResolver.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouSessionResolver.java index 8f3c2a18c..fffa493c8 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouSessionResolver.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouSessionResolver.java @@ -43,12 +43,17 @@ public void handle(Envelope envelope) { WhoAreYouPacket whoAreYouPacket = (WhoAreYouPacket) envelope.get(Field.PACKET_WHOAREYOU); Optional nodeSessionOptional = authTagRepo.get(whoAreYouPacket.getAuthTag()); if (nodeSessionOptional.isPresent() - && nodeSessionOptional - .get() - .getStatus() - .equals( - NodeSession.SessionStatus - .RANDOM_PACKET_SENT)) { // FIXME: doesn't handle session expiration + && (nodeSessionOptional + .get() + .getStatus() + .equals( + NodeSession.SessionStatus.RANDOM_PACKET_SENT) // We've started handshake before + || nodeSessionOptional + .get() + .getStatus() + .equals( + NodeSession.SessionStatus + .AUTHENTICATED))) { // We had authenticated session but it's expired envelope.put(Field.SESSION, nodeSessionOptional.get()); logger.trace( () -> From 9e01ed3ccbed838a507715692f902ea400a87b6b Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Mon, 21 Oct 2019 07:56:17 +0300 Subject: [PATCH 43/77] discovery: fix requestId size --- .../main/java/org/ethereum/beacon/discovery/NodeSession.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeSession.java b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeSession.java index 77de1d40a..f32b88dec 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeSession.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeSession.java @@ -23,6 +23,7 @@ public class NodeSession { public static final int NONCE_SIZE = 12; + public static final int REQUEST_ID_SIZE = 8; private static final Logger logger = LogManager.getLogger(NodeSession.class); private final NodeRecord nodeRecord; private final NodeRecord homeNodeRecord; @@ -77,7 +78,6 @@ public synchronized void sendOutgoing(Packet packet) { outgoing.accept(packet); } - // FIXME: size, algo /** * The value selected as request ID must allow for concurrent conversations. Using a timestamp can * result in parallel conversations with the same id, so this should be avoided. Request IDs also @@ -88,7 +88,7 @@ public synchronized void sendOutgoing(Packet packet) { *

Request ID is reserved for `messageCode` */ public synchronized BytesValue getNextRequestId(MessageCode messageCode) { - byte[] requestId = new byte[12]; + byte[] requestId = new byte[REQUEST_ID_SIZE]; rnd.nextBytes(requestId); BytesValue wrapped = BytesValue.wrap(requestId); requestIdReservations.put(wrapped, messageCode); From 4af79dd39fab44ca683a46a405b28f4f83386db9 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Mon, 21 Oct 2019 07:58:42 +0300 Subject: [PATCH 44/77] discovery: clarify NodeSession class goal in Javadoc --- .../main/java/org/ethereum/beacon/discovery/NodeSession.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeSession.java b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeSession.java index f32b88dec..3f0e83c9a 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeSession.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeSession.java @@ -21,6 +21,9 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.function.Consumer; +/** + * Stores session status and all keys for discovery session between us (homeNode) and the other node + */ public class NodeSession { public static final int NONCE_SIZE = 12; public static final int REQUEST_ID_SIZE = 8; From 4cced86285e9446cf27160eec638938b14bf2c3b Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Mon, 21 Oct 2019 11:11:09 +0300 Subject: [PATCH 45/77] discovery: fix message handlers had state and were created for each case --- .../beacon/discovery/NodeSession.java | 77 +++++++++++++++++-- .../message/handler/NodesHandler.java | 69 +++++++++-------- .../beacon/discovery/pipeline/Field.java | 4 +- .../pipeline/handler/BadPacketLogger.java | 2 +- .../pipeline/handler/MessageHandler.java | 16 +++- .../NotExpectedIncomingPacketHandler.java | 2 +- .../handler/UnknownPacketTagToSender.java | 5 +- .../handler/WhoAreYouSessionResolver.java | 3 +- .../discovery/storage/NodeTableTest.java | 13 ++-- .../beacon/util/ExpirationScheduler.java | 15 ++-- 10 files changed, 144 insertions(+), 62 deletions(-) diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeSession.java b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeSession.java index 3f0e83c9a..3605133be 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeSession.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeSession.java @@ -10,6 +10,7 @@ import org.ethereum.beacon.discovery.storage.NodeBucketStorage; import org.ethereum.beacon.discovery.storage.NodeTable; import org.ethereum.beacon.discovery.task.TaskType; +import org.ethereum.beacon.util.ExpirationScheduler; import tech.pegasys.artemis.util.bytes.Bytes32; import tech.pegasys.artemis.util.bytes.BytesValue; @@ -19,6 +20,7 @@ import java.util.Random; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; import java.util.function.Consumer; /** @@ -28,6 +30,7 @@ public class NodeSession { public static final int NONCE_SIZE = 12; public static final int REQUEST_ID_SIZE = 8; private static final Logger logger = LogManager.getLogger(NodeSession.class); + private static final int CLEANUP_DELAY_SECONDS = 60; private final NodeRecord nodeRecord; private final NodeRecord homeNodeRecord; private final Bytes32 homeNodeId; @@ -40,7 +43,9 @@ public class NodeSession { private Bytes32 idNonce; private BytesValue initiatorKey; private BytesValue recipientKey; - private Map requestIdReservations = new ConcurrentHashMap<>(); + private Map requestIdStatuses = new ConcurrentHashMap<>(); + private ExpirationScheduler requestExpirationScheduler = + new ExpirationScheduler<>(CLEANUP_DELAY_SECONDS, TimeUnit.SECONDS); private CompletableFuture completableFuture = null; private TaskType task = null; private BytesValue staticNodeKey; @@ -94,10 +99,49 @@ public synchronized BytesValue getNextRequestId(MessageCode messageCode) { byte[] requestId = new byte[REQUEST_ID_SIZE]; rnd.nextBytes(requestId); BytesValue wrapped = BytesValue.wrap(requestId); - requestIdReservations.put(wrapped, messageCode); + RequestInfo requestInfo = new GeneralRequestInfo(messageCode); + requestIdStatuses.put(wrapped, requestInfo); + requestExpirationScheduler.put( + wrapped, + new Runnable() { + @Override + public void run() { + logger.debug( + () -> + String.format( + "Request %s expired for id %s in session %s: no reply", + requestInfo, wrapped, this)); + requestIdStatuses.remove(wrapped); + } + }); return wrapped; } + public synchronized void updateRequestInfo(BytesValue requestId, RequestInfo newRequestInfo) { + RequestInfo oldRequestInfo = requestIdStatuses.remove(requestId); + if (oldRequestInfo == null) { + logger.debug( + () -> + String.format( + "An attempt to update requestId %s in session %s which does not exist", + requestId, this)); + return; + } + requestIdStatuses.put(requestId, newRequestInfo); + requestExpirationScheduler.put( + requestId, + new Runnable() { + @Override + public void run() { + logger.debug( + String.format( + "Request %s expired for id %s in session %s: no reply", + newRequestInfo, requestId, this)); + requestIdStatuses.remove(requestId); + } + }); + } + public synchronized BytesValue generateNonce() { byte[] nonce = new byte[NONCE_SIZE]; rnd.nextBytes(nonce); @@ -140,13 +184,15 @@ public void setRecipientKey(BytesValue recipientKey) { this.recipientKey = recipientKey; } - public synchronized void clearRequestId(BytesValue requestId, MessageCode messageCode) { - assert requestIdReservations.remove(requestId).equals(messageCode); + public synchronized void clearRequestId(BytesValue requestId, MessageCode expectedMessageCode) { + RequestInfo requestInfo = requestIdStatuses.remove(requestId); + assert expectedMessageCode.equals(requestInfo.getMessageCode()); + requestExpirationScheduler.cancel(requestId); } - public synchronized Optional getRequestId(BytesValue requestId) { - MessageCode messageCode = requestIdReservations.get(requestId); - return messageCode == null ? Optional.empty() : Optional.of(messageCode); + public synchronized Optional getRequestId(BytesValue requestId) { + RequestInfo requestInfo = requestIdStatuses.get(requestId); + return requestId == null ? Optional.empty() : Optional.of(requestInfo); } public NodeTable getNodeTable() { @@ -223,4 +269,21 @@ public enum SessionStatus { RANDOM_PACKET_SENT, // our node is initiator, we've sent random packet AUTHENTICATED } + + public static interface RequestInfo { + public MessageCode getMessageCode(); + } + + public static class GeneralRequestInfo implements RequestInfo { + private final MessageCode messageCode; + + public GeneralRequestInfo(MessageCode messageCode) { + this.messageCode = messageCode; + } + + @Override + public MessageCode getMessageCode() { + return messageCode; + } + } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/NodesHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/NodesHandler.java index c02826bda..0237a833a 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/NodesHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/NodesHandler.java @@ -6,38 +6,37 @@ import org.ethereum.beacon.discovery.NodeSession; import org.ethereum.beacon.discovery.message.MessageCode; import org.ethereum.beacon.discovery.message.NodesMessage; -import org.ethereum.beacon.util.ExpirationScheduler; -import tech.pegasys.artemis.util.bytes.BytesValue; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; +import java.util.Optional; public class NodesHandler implements MessageHandler { private static final Logger logger = LogManager.getLogger(FindNodeHandler.class); - private static final int CLEANUP_DELAY_SECONDS = 60; - private ExpirationScheduler nodeExpirationScheduler = - new ExpirationScheduler<>(CLEANUP_DELAY_SECONDS, TimeUnit.SECONDS); - private Map nodesCounters = new ConcurrentHashMap<>(); @Override public void handle(NodesMessage message, NodeSession session) { - // NODES total count handling + cleanup schedule - if (nodesCounters.containsKey(message.getRequestId())) { - synchronized (this) { - int counter = nodesCounters.get(message.getRequestId()) - 1; - if (counter == 0) { - cleanUp(message.getRequestId(), session); - } else { - nodesCounters.put(message.getRequestId(), counter); - updateExpiration(message.getRequestId(), session); - } + // NODES total count handling + Optional requestInfoOpt = session.getRequestId(message.getRequestId()); + if (!requestInfoOpt.isPresent()) { + throw new RuntimeException( + String.format( + "Request #%s not found in session %s when handling message %s", + message.getRequestId(), session, message)); + } + NodeSession.RequestInfo requestInfo = requestInfoOpt.get(); + if (requestInfo instanceof FindNodeRequestInfo) { + int newNodesCount = ((FindNodeRequestInfo) requestInfo).getRemainingNodes() - 1; + if (newNodesCount == 0) { + session.clearRequestId(message.getRequestId(), MessageCode.FINDNODE); + } else { + session.updateRequestInfo(message.getRequestId(), new FindNodeRequestInfo(newNodesCount)); } - } else if (message.getTotal() > 1) { - nodesCounters.put(message.getRequestId(), message.getTotal() - 1); - updateExpiration(message.getRequestId(), session); } else { - session.clearRequestId(message.getRequestId(), MessageCode.FINDNODE); + if (message.getTotal() > 1) { + session.updateRequestInfo( + message.getRequestId(), new FindNodeRequestInfo(message.getTotal() - 1)); + } else { + session.clearRequestId(message.getRequestId(), MessageCode.FINDNODE); + } } // Parse node records @@ -58,16 +57,20 @@ public void handle(NodesMessage message, NodeSession session) { }); } - private synchronized void updateExpiration(BytesValue requestId, NodeSession session) { - nodeExpirationScheduler.put( - requestId, - () -> { - cleanUp(requestId, session); - }); - } + public static class FindNodeRequestInfo implements NodeSession.RequestInfo { + private final int remainingNodes; + + public FindNodeRequestInfo(int remainingNodes) { + this.remainingNodes = remainingNodes; + } + + @Override + public MessageCode getMessageCode() { + return MessageCode.FINDNODE; + } - private synchronized void cleanUp(BytesValue requestId, NodeSession session) { - nodesCounters.remove(requestId); - session.clearRequestId(requestId, MessageCode.FINDNODE); + public int getRemainingNodes() { + return remainingNodes; + } } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/Field.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/Field.java index d5ae55da5..e30276e28 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/Field.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/Field.java @@ -11,8 +11,8 @@ public enum Field { MESSAGE, // Message extracted from the packet NODE, // Sender/recipient node BAD_PACKET, // Bad, rejected packet - BAD_PACKET_EXCEPTION, // Stores exception for bad packet + BAD_MESSAGE, // Bad, rejected message + BAD_EXCEPTION, // Stores exception for bad packet or message TASK, // Task to perform FUTURE, // Completable future - } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/BadPacketLogger.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/BadPacketLogger.java index e599ded64..e165affe3 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/BadPacketLogger.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/BadPacketLogger.java @@ -30,6 +30,6 @@ public void handle(Envelope envelope) { () -> String.format( "Bad packet: %s in envelope #%s", envelope.get(Field.BAD_PACKET), envelope.getId()), - (Exception) envelope.get(Field.BAD_PACKET_EXCEPTION)); + (Exception) envelope.get(Field.BAD_EXCEPTION)); } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/MessageHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/MessageHandler.java index a287deefc..11b7afa6f 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/MessageHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/MessageHandler.java @@ -13,6 +13,8 @@ public class MessageHandler implements EnvelopeHandler { private static final Logger logger = LogManager.getLogger(MessageHandler.class); + private static final MessageProcessor messageProcessor = + new MessageProcessor(new DiscoveryV5MessageProcessor());; @Override public void handle(Envelope envelope) { @@ -34,8 +36,16 @@ public void handle(Envelope envelope) { NodeSession session = (NodeSession) envelope.get(Field.SESSION); DiscoveryMessage message = (DiscoveryMessage) envelope.get(Field.MESSAGE); - // TODO: optimize - MessageProcessor messageProcessor = new MessageProcessor(new DiscoveryV5MessageProcessor()); - messageProcessor.handleIncoming(message, session); + try { + messageProcessor.handleIncoming(message, session); + } catch (Exception ex) { + logger.trace( + () -> + String.format( + "Failed to handle message %s in envelope #%s", message, envelope.getId())); + envelope.put(Field.BAD_MESSAGE, message); + envelope.put(Field.BAD_EXCEPTION, ex); + envelope.remove(Field.MESSAGE); + } } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NotExpectedIncomingPacketHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NotExpectedIncomingPacketHandler.java index 3a67a4b3d..d1d9ce9e2 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NotExpectedIncomingPacketHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NotExpectedIncomingPacketHandler.java @@ -78,7 +78,7 @@ public void handle(Envelope envelope) { unknownPacket, session.getNodeRecord(), session.getStatus()); logger.error(error, ex); envelope.put(Field.BAD_PACKET, envelope.get(Field.PACKET_UNKNOWN)); - envelope.put(Field.BAD_PACKET_EXCEPTION, ex); + envelope.put(Field.BAD_EXCEPTION, ex); envelope.remove(Field.PACKET_UNKNOWN); return; } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/UnknownPacketTagToSender.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/UnknownPacketTagToSender.java index ade32fff3..9d66044ea 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/UnknownPacketTagToSender.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/UnknownPacketTagToSender.java @@ -37,7 +37,8 @@ public void handle(Envelope envelope) { logger.trace( () -> String.format( - "Envelope %s in UnknownPacketTagToSender, requirements are satisfied!", envelope.getId())); + "Envelope %s in UnknownPacketTagToSender, requirements are satisfied!", + envelope.getId())); if (!envelope.contains(Field.PACKET_UNKNOWN)) { return; @@ -52,7 +53,7 @@ public void handle(Envelope envelope) { () -> { envelope.put(Field.BAD_PACKET, envelope.get(Field.PACKET_UNKNOWN)); envelope.put( - Field.BAD_PACKET_EXCEPTION, + Field.BAD_EXCEPTION, new RuntimeException( String.format("Session couldn't be created for nodeId %s", fromNodeId))); envelope.remove(Field.PACKET_UNKNOWN); diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouSessionResolver.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouSessionResolver.java index fffa493c8..4a3e9fc31 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouSessionResolver.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouSessionResolver.java @@ -63,8 +63,7 @@ public void handle(Envelope envelope) { } else { envelope.put(Field.BAD_PACKET, envelope.get(Field.PACKET_WHOAREYOU)); envelope.remove(Field.PACKET_WHOAREYOU); - envelope.put( - Field.BAD_PACKET_EXCEPTION, new RuntimeException("Not expected WHOAREYOU packet")); + envelope.put(Field.BAD_EXCEPTION, new RuntimeException("Not expected WHOAREYOU packet")); } } } diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeTableTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeTableTest.java index f595e180d..a2046f54c 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeTableTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeTableTest.java @@ -17,8 +17,10 @@ import java.net.InetAddress; import java.net.UnknownHostException; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.function.Supplier; import static org.ethereum.beacon.discovery.storage.NodeTableStorage.DEFAULT_SERIALIZER; @@ -151,12 +153,11 @@ public void testFind() throws Exception { List closestNodes = nodeTableStorage.get().findClosestNodes(closestNode.getNodeId(), 252); assertEquals(2, closestNodes.size()); - assertEquals( - closestNodes.get(0).getNode().get(NodeRecord.FIELD_PKEY_SECP256K1), - localHostNode.get(NodeRecord.FIELD_PKEY_SECP256K1)); - assertEquals( - closestNodes.get(1).getNode().get(NodeRecord.FIELD_PKEY_SECP256K1), - closestNode.get(NodeRecord.FIELD_PKEY_SECP256K1)); + Set publicKeys = new HashSet<>(); + closestNodes.forEach( + n -> publicKeys.add((BytesValue) n.getNode().get(NodeRecord.FIELD_PKEY_SECP256K1))); + assertTrue(publicKeys.contains(localHostNode.get(NodeRecord.FIELD_PKEY_SECP256K1))); + assertTrue(publicKeys.contains(closestNode.get(NodeRecord.FIELD_PKEY_SECP256K1))); List farNodes = nodeTableStorage.get().findClosestNodes(farNode.getNodeId(), 1); assertEquals(1, farNodes.size()); assertEquals( diff --git a/util/src/main/java/org/ethereum/beacon/util/ExpirationScheduler.java b/util/src/main/java/org/ethereum/beacon/util/ExpirationScheduler.java index 28dc81aa0..4dfcfa178 100644 --- a/util/src/main/java/org/ethereum/beacon/util/ExpirationScheduler.java +++ b/util/src/main/java/org/ethereum/beacon/util/ExpirationScheduler.java @@ -29,11 +29,7 @@ public ExpirationScheduler(long delay, TimeUnit timeUnit) { * @param runnable Task */ public void put(Key key, Runnable runnable) { - synchronized (this) { - if (expirationTasks.containsKey(key)) { - expirationTasks.remove(key).cancel(true); - } - } + cancel(key); ScheduledFuture future = scheduler.schedule( () -> { @@ -44,4 +40,13 @@ public void put(Key key, Runnable runnable) { timeUnit); expirationTasks.put(key, future); } + + /** Cancels task for key and removes it from storage */ + public void cancel(Key key) { + synchronized (this) { + if (expirationTasks.containsKey(key)) { + expirationTasks.remove(key).cancel(true); + } + } + } } From c28a9b7a4e4bd78446f9e176c7f2ed1ad2fe90da Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Mon, 21 Oct 2019 11:21:56 +0300 Subject: [PATCH 46/77] discovery: add toString to requestInfo --- .../java/org/ethereum/beacon/discovery/NodeSession.java | 7 +++++++ .../beacon/discovery/message/handler/NodesHandler.java | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeSession.java b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeSession.java index 3605133be..694fe5ceb 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeSession.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeSession.java @@ -285,5 +285,12 @@ public GeneralRequestInfo(MessageCode messageCode) { public MessageCode getMessageCode() { return messageCode; } + + @Override + public String toString() { + return "GeneralRequestInfo{" + + "messageCode=" + messageCode + + '}'; + } } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/NodesHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/NodesHandler.java index 0237a833a..1ba6860b9 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/NodesHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/NodesHandler.java @@ -72,5 +72,12 @@ public MessageCode getMessageCode() { public int getRemainingNodes() { return remainingNodes; } + + @Override + public String toString() { + return "FindNodeRequestInfo{" + + "remainingNodes=" + remainingNodes + + '}'; + } } } From b90e80a76ed618b85abe2100fe1fbee9a7f9052b Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Mon, 21 Oct 2019 15:29:32 +0300 Subject: [PATCH 47/77] discovery: refactor and remove side task from AuthHeaderMessagePacket --- .../ethereum/beacon/discovery/Functions.java | 45 ++++++- .../packet/AuthHeaderMessagePacket.java | 113 +++++++++++------- .../AuthHeaderMessagePacketHandler.java | 27 ++--- .../handler/WhoAreYouPacketHandler.java | 13 +- .../discovery/DiscoveryNetworkTest.java | 5 +- .../discovery/DiscoveryNoNetworkTest.java | 4 +- .../beacon/discovery/FunctionsTest.java | 7 +- 7 files changed, 137 insertions(+), 77 deletions(-) diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java b/discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java index bcbf25d8e..2402fd52b 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java @@ -1,5 +1,6 @@ package org.ethereum.beacon.discovery; +import com.google.common.base.Objects; import org.bouncycastle.crypto.Digest; import org.bouncycastle.crypto.digests.SHA256Digest; import org.bouncycastle.crypto.generators.HKDFBytesGenerator; @@ -7,7 +8,6 @@ import org.bouncycastle.crypto.params.HKDFParameters; import org.bouncycastle.math.ec.ECPoint; import org.ethereum.beacon.crypto.Hashes; -import org.javatuples.Triplet; import org.web3j.crypto.ECKeyPair; import org.web3j.crypto.Sign; import tech.pegasys.artemis.util.bytes.Bytes1; @@ -117,7 +117,7 @@ public static BytesValue aesgcm_decrypt( * prk = HKDF-Extract(secret, id-nonce) * initiator-key, recipient-key, auth-resp-key = HKDF-Expand(prk, info) */ - public static Triplet hkdf_expand( + public static HKDFKeys hkdf_expand( BytesValue srcNodeId, BytesValue destNodeId, BytesValue srcPrivKey, @@ -156,7 +156,7 @@ public static Triplet hkdf_expand( BytesValue initiatorKey = hkdfOutput.slice(0, INITIATOR_KEY_LENGTH); BytesValue recipientKey = hkdfOutput.slice(INITIATOR_KEY_LENGTH, RECIPIENT_KEY_LENGTH); BytesValue authRespKey = hkdfOutput.slice(INITIATOR_KEY_LENGTH + RECIPIENT_KEY_LENGTH); - return Triplet.with(initiatorKey, recipientKey, authRespKey); + return new HKDFKeys(initiatorKey, recipientKey, authRespKey); } catch (Exception ex) { throw new RuntimeException(ex); } @@ -187,4 +187,43 @@ public static int logDistance(Bytes32 nodeId1, Bytes32 nodeId2) { } return logDistance; } + + public static class HKDFKeys { + private final BytesValue initiatorKey; + private final BytesValue recipientKey; + private final BytesValue authResponseKey; + + public HKDFKeys(BytesValue initiatorKey, BytesValue recipientKey, BytesValue authResponseKey) { + this.initiatorKey = initiatorKey; + this.recipientKey = recipientKey; + this.authResponseKey = authResponseKey; + } + + public BytesValue getInitiatorKey() { + return initiatorKey; + } + + public BytesValue getRecipientKey() { + return recipientKey; + } + + public BytesValue getAuthResponseKey() { + return authResponseKey; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + HKDFKeys hkdfKeys = (HKDFKeys) o; + return Objects.equal(initiatorKey, hkdfKeys.initiatorKey) + && Objects.equal(recipientKey, hkdfKeys.recipientKey) + && Objects.equal(authResponseKey, hkdfKeys.authResponseKey); + } + + @Override + public int hashCode() { + return Objects.hashCode(initiatorKey, recipientKey, authResponseKey); + } + } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/AuthHeaderMessagePacket.java b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/AuthHeaderMessagePacket.java index 503ec7f4a..0e31e73e4 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/AuthHeaderMessagePacket.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/AuthHeaderMessagePacket.java @@ -7,7 +7,6 @@ import org.ethereum.beacon.discovery.message.DiscoveryMessage; import org.ethereum.beacon.discovery.message.DiscoveryV5Message; import org.javatuples.Pair; -import org.javatuples.Triplet; import org.web3j.rlp.RlpDecoder; import org.web3j.rlp.RlpEncoder; import org.web3j.rlp.RlpList; @@ -17,7 +16,6 @@ import tech.pegasys.artemis.util.bytes.BytesValue; import java.math.BigInteger; -import java.util.function.Function; /** * Used as first encrypted message sent in response to WHOAREYOU {@link WhoAreYouPacket}. Contains @@ -48,7 +46,8 @@ public class AuthHeaderMessagePacket extends AbstractPacket { private static final BytesValue DISCOVERY_ID_NONCE = BytesValue.wrap("discovery-id-nonce".getBytes()); private static final BytesValue ZERO_NONCE = BytesValue.wrap(new byte[12]); - private MessagePacketDecoded decoded = null; + private EphemeralPubKeyDecoded decodedEphemeralPubKeyPt = null; + private MessagePtDecoded decodedMessagePt = null; public AuthHeaderMessagePacket(BytesValue bytes) { super(bytes); @@ -101,52 +100,59 @@ public void verify(BytesValue expectedIdNonce) { public Bytes32 getHomeNodeId(Bytes32 destNodeId) { verifyDecode(); - return Bytes32s.xor(Functions.hash(destNodeId), decoded.tag); + return Bytes32s.xor(Functions.hash(destNodeId), decodedEphemeralPubKeyPt.tag); } public BytesValue getAuthTag() { verifyDecode(); - return decoded.authTag; + return decodedEphemeralPubKeyPt.authTag; } public BytesValue getIdNonce() { verifyDecode(); - return decoded.idNonce; + return decodedEphemeralPubKeyPt.idNonce; } public BytesValue getEphemeralPubkey() { - verifyDecode(); - return decoded.ephemeralPubkey; + verifyEphemeralPubKeyDecode(); + return decodedEphemeralPubKeyPt.ephemeralPubkey; } public BytesValue getIdNonceSig() { verifyDecode(); - return decoded.idNonceSig; + return decodedMessagePt.idNonceSig; } public NodeRecord getNodeRecord() { verifyDecode(); - return decoded.nodeRecord; + return decodedMessagePt.nodeRecord; } public DiscoveryMessage getMessage() { verifyDecode(); - return decoded.message; + return decodedMessagePt.message; + } + + private void verifyEphemeralPubKeyDecode() { + if (decodedEphemeralPubKeyPt == null) { + throw new RuntimeException("You should run decodeEphemeralPubKey before!"); + } } private void verifyDecode() { - if (decoded == null) { - throw new RuntimeException("You should decode packet at first!"); + if (decodedEphemeralPubKeyPt == null || decodedMessagePt == null) { + throw new RuntimeException("You should run decodeEphemeralPubKey and decodeMessage before!"); } } - public void decode(Function> ephemeralPubToKeysFunction) { - if (decoded != null) { + public void decodeEphemeralPubKey() { + if (decodedEphemeralPubKeyPt != null) { return; } - MessagePacketDecoded blank = new MessagePacketDecoded(); + EphemeralPubKeyDecoded blank = new EphemeralPubKeyDecoded(); blank.tag = Bytes32.wrap(getBytes().slice(0, 32), 0); Pair decodeRes = RlpUtil.decodeFirstList(getBytes().slice(32)); + blank.messageEncrypted = decodeRes.getValue1(); int authHeaderLength = getBytes().size() - 32 - decodeRes.getValue1().size(); RlpList authHeaderParts = (RlpList) decodeRes.getValue0().getValues().get(0); // [auth-tag, id-nonce, auth-scheme-name, ephemeral-pubkey, auth-response] @@ -156,12 +162,24 @@ public void decode(Function keys = ephemeralPubToKeysFunction.apply(blank.ephemeralPubkey); - BytesValue authResponse = + blank.authResponse = BytesValue.wrap(((RlpString) authHeaderParts.getValues().get(4)).getBytes()); + blank.authHeaderRaw = getBytes().slice(32, authHeaderLength); + this.decodedEphemeralPubKeyPt = blank; + } + /** Run {@link AuthHeaderMessagePacket#decodeEphemeralPubKey()} before second part */ + public void decodeMessage(BytesValue initiatorKey, BytesValue authResponseKey) { + if (decodedEphemeralPubKeyPt == null) { + throw new RuntimeException("Run decodeEphemeralPubKey() before"); + } + if (decodedMessagePt != null) { + return; + } + MessagePtDecoded blank = new MessagePtDecoded(); BytesValue authResponsePt = - Functions.aesgcm_decrypt(keys.getValue2(), ZERO_NONCE, authResponse, BytesValue.EMPTY); + Functions.aesgcm_decrypt( + authResponseKey, ZERO_NONCE, decodedEphemeralPubKeyPt.authResponse, BytesValue.EMPTY); RlpList authResponsePtParts = (RlpList) RlpDecoder.decode(authResponsePt.extractArray()).getValues().get(0); assert BigInteger.valueOf(5) @@ -171,43 +189,54 @@ public void decode(Function 2 dialog CountDownLatch randomSent1to2 = new CountDownLatch(1); CountDownLatch whoareyouSent2to1 = new CountDownLatch(1); @@ -128,6 +125,8 @@ public void test() throws Exception { }); // 4) fire 1 to 2 dialog + discoveryManager1.start(); + discoveryManager2.start(); discoveryManager1.executeTask(nodeRecord2, TaskType.FINDNODE); assert randomSent1to2.await(1, TimeUnit.SECONDS); diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java index d00625225..2059db520 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java @@ -93,8 +93,6 @@ public void test() throws Exception { nodePair2.getValue0(), from1to2); - discoveryManager1.start(); - discoveryManager2.start(); // 2) Link outgoing of each one with incoming of another Flux.from(discoveryManager1.getOutgoingMessages()) .subscribe(t -> from1to2.onNext(t.getPacket().getBytes())); @@ -143,6 +141,8 @@ public void test() throws Exception { }); // 4) fire 1 to 2 dialog + discoveryManager1.start(); + discoveryManager2.start(); discoveryManager1.executeTask(nodeRecord2, TaskType.FINDNODE); assert randomSent1to2.await(1, TimeUnit.SECONDS); diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/FunctionsTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/FunctionsTest.java index f502af4d1..2ab709730 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/FunctionsTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/FunctionsTest.java @@ -1,6 +1,5 @@ package org.ethereum.beacon.discovery; -import org.javatuples.Triplet; import org.junit.Test; import org.web3j.crypto.ECKeyPair; import tech.pegasys.artemis.util.bytes.Bytes32; @@ -55,21 +54,21 @@ public void testLogDistance() { public void hkdfExpandTest() { BytesValue idNonce = Bytes32.fromHexString("68b02a985ecb99cc2d10cf188879d93ae7684c4f4707770017b078c6497c5a5d"); - Triplet sec1 = + Functions.HKDFKeys keys1 = Functions.hkdf_expand( nodeId1, nodeId2, testKey1, BytesValue.wrap(ECKeyPair.create(testKey2.extractArray()).getPublicKey().toByteArray()), idNonce); - Triplet sec2 = + Functions.HKDFKeys keys2 = Functions.hkdf_expand( nodeId1, nodeId2, testKey2, BytesValue.wrap(ECKeyPair.create(testKey1.extractArray()).getPublicKey().toByteArray()), idNonce); - assertEquals(sec1, sec2); + assertEquals(keys1, keys2); } @Test From 80c6641f3bc74ae5deed6e50ff96b4338c91b8e5 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Mon, 21 Oct 2019 15:52:01 +0300 Subject: [PATCH 48/77] discovery: replace TODO in NodeSessionRequestHandler with appropriate logging --- .../discovery/pipeline/handler/NodeIdToSession.java | 10 ++++++++++ .../pipeline/handler/NodeSessionRequestHandler.java | 6 +----- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeIdToSession.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeIdToSession.java index e4f719f39..0b301dd43 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeIdToSession.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeIdToSession.java @@ -76,6 +76,11 @@ public void handle(Envelope envelope) { Pair sessionRequest = (Pair) envelope.get(Field.SESSION_LOOKUP); envelope.remove(Field.SESSION_LOOKUP); + logger.trace( + () -> + String.format( + "Envelope %s: Session lookup requested for nodeId %s", + envelope.getId(), sessionRequest.getValue0())); Optional nodeSessionOptional = getSession(sessionRequest.getValue0()); if (nodeSessionOptional.isPresent()) { envelope.put(Field.SESSION, nodeSessionOptional.get()); @@ -85,6 +90,11 @@ public void handle(Envelope envelope) { "Session resolved: %s in envelope #%s", nodeSessionOptional.get(), envelope.getId())); } else { + logger.debug( + () -> + String.format( + "Envelope %s: Session not resolved for nodeId %s", + envelope.getId(), sessionRequest.getValue0())); sessionRequest.getValue1().run(); } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeSessionRequestHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeSessionRequestHandler.java index 4bfcddbfd..47362c042 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeSessionRequestHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NodeSessionRequestHandler.java @@ -34,10 +34,6 @@ public void handle(Envelope envelope) { envelope.put( Field.SESSION_LOOKUP, - Pair.with( - ((NodeRecord) envelope.get(Field.NODE)).getNodeId(), - (Runnable) - () -> { // TODO: failure - })); + Pair.with(((NodeRecord) envelope.get(Field.NODE)).getNodeId(), (Runnable) () -> {})); } } From 2acb0d195ade4befb52445125d33df4e8632cab7 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Mon, 21 Oct 2019 16:17:31 +0300 Subject: [PATCH 49/77] discovery: this logic should be here --- .../ethereum/beacon/discovery/pipeline/handler/TaskHandler.java | 1 - 1 file changed, 1 deletion(-) diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/TaskHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/TaskHandler.java index b665cd140..b08575b24 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/TaskHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/TaskHandler.java @@ -47,7 +47,6 @@ public void handle(Envelope envelope) { NodeSession session = (NodeSession) envelope.get(Field.SESSION); CompletableFuture completableFuture = (CompletableFuture) envelope.get(Field.FUTURE); - // FIXME: this logic shouldbn't be here!!!!11 BytesValue authTag = session.generateNonce(); if (session.getStatus().equals(NodeSession.SessionStatus.INITIAL)) { RandomPacket randomPacket = From b293d39223dbbddf62e78ac50bea85a99f1553f2 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Mon, 21 Oct 2019 16:33:41 +0300 Subject: [PATCH 50/77] discovery: fix task manager didn't use recursive tasks --- .../ethereum/beacon/discovery/task/DiscoveryTaskManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/task/DiscoveryTaskManager.java b/discovery/src/main/java/org/ethereum/beacon/discovery/task/DiscoveryTaskManager.java index c5bcb337f..4bd92dba6 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/task/DiscoveryTaskManager.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/task/DiscoveryTaskManager.java @@ -174,7 +174,7 @@ private void recursiveLookupTask() { .filter(RECURSIVE_LOOKUP_NODE_RULE) .forEach( nodeRecord -> - liveCheckTasks.add( + recursiveLookupTasks.add( nodeRecord, () -> {}, () -> From 8d54fe07e8ac30ed3a5c5eb882c2e9cf2b351214 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Wed, 23 Oct 2019 06:23:45 +0300 Subject: [PATCH 51/77] discovery: queue tasks, several simultaneous tasks in session capability --- .../discovery/DiscoveryManagerImpl.java | 22 +-- .../beacon/discovery/NodeSession.java | 147 ++++++++++++------ .../message/handler/NodesHandler.java | 42 +++-- .../message/handler/PongHandler.java | 4 +- .../AuthHeaderMessagePacketHandler.java | 39 ++--- .../handler/MessagePacketHandler.java | 35 +---- .../pipeline/handler/NewTaskHandler.java | 47 ++++++ .../pipeline/handler/NextTaskHandler.java | 97 ++++++++++++ .../pipeline/handler/TaskHandler.java | 81 ---------- .../handler/WhoAreYouPacketHandler.java | 52 ++++--- .../discovery/task/TaskMessageFactory.java | 59 +++++-- .../beacon/discovery/task/TaskStatus.java | 8 + .../discovery/DiscoveryNetworkTest.java | 6 +- .../discovery/DiscoveryNoNetworkTest.java | 6 +- .../discovery/HandshakeHandlersTest.java | 16 +- .../mock/DiscoveryManagerNoNetwork.java | 23 +-- 16 files changed, 426 insertions(+), 258 deletions(-) create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NewTaskHandler.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NextTaskHandler.java delete mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/TaskHandler.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/task/TaskStatus.java diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java index 99e2d8dc6..967bacab0 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java @@ -18,11 +18,12 @@ import org.ethereum.beacon.discovery.pipeline.handler.IncomingDataPacker; import org.ethereum.beacon.discovery.pipeline.handler.MessageHandler; import org.ethereum.beacon.discovery.pipeline.handler.MessagePacketHandler; +import org.ethereum.beacon.discovery.pipeline.handler.NewTaskHandler; +import org.ethereum.beacon.discovery.pipeline.handler.NextTaskHandler; import org.ethereum.beacon.discovery.pipeline.handler.NodeIdToSession; import org.ethereum.beacon.discovery.pipeline.handler.NodeSessionRequestHandler; import org.ethereum.beacon.discovery.pipeline.handler.NotExpectedIncomingPacketHandler; import org.ethereum.beacon.discovery.pipeline.handler.OutgoingParcelHandler; -import org.ethereum.beacon.discovery.pipeline.handler.TaskHandler; import org.ethereum.beacon.discovery.pipeline.handler.UnknownPacketTagToSender; import org.ethereum.beacon.discovery.pipeline.handler.UnknownPacketTypeByStatus; import org.ethereum.beacon.discovery.pipeline.handler.WhoAreYouAttempt; @@ -40,7 +41,6 @@ import tech.pegasys.artemis.util.bytes.Bytes4; import tech.pegasys.artemis.util.bytes.BytesValue; -import java.security.SecureRandom; import java.util.concurrent.CompletableFuture; public class DiscoveryManagerImpl implements DiscoveryManager { @@ -59,7 +59,8 @@ public DiscoveryManagerImpl( NodeRecord homeNode, BytesValue homeNodePrivateKey, Scheduler serverScheduler, - Scheduler clientScheduler) { + Scheduler clientScheduler, + Scheduler taskScheduler) { AuthTagRepository authTagRepo = new AuthTagRepository(); this.scheduler = serverScheduler; this.discoveryServer = @@ -83,8 +84,8 @@ public DiscoveryManagerImpl( .addHandler(nodeIdToSession) .addHandler(new UnknownPacketTypeByStatus()) .addHandler(new NotExpectedIncomingPacketHandler()) - .addHandler(new WhoAreYouPacketHandler()) - .addHandler(new AuthHeaderMessagePacketHandler()) + .addHandler(new WhoAreYouPacketHandler(outgoingPipeline, taskScheduler)) + .addHandler(new AuthHeaderMessagePacketHandler(outgoingPipeline, taskScheduler)) .addHandler(new MessagePacketHandler()) .addHandler(new MessageHandler()) .addHandler(new BadPacketLogger()); @@ -92,7 +93,8 @@ public DiscoveryManagerImpl( .addHandler(new OutgoingParcelHandler(outgoingSink)) .addHandler(new NodeSessionRequestHandler()) .addHandler(nodeIdToSession) - .addHandler(new TaskHandler(new SecureRandom())); + .addHandler(new NewTaskHandler()) + .addHandler(new NextTaskHandler(outgoingPipeline, taskScheduler)); } @Override @@ -110,12 +112,12 @@ public void stop() { public CompletableFuture executeTask(NodeRecord nodeRecord, TaskType taskType) { Envelope envelope = new Envelope(); - envelope.put(Field.TASK, taskType); envelope.put(Field.NODE, nodeRecord); - CompletableFuture completed = new CompletableFuture<>(); - envelope.put(Field.FUTURE, completed); + CompletableFuture future = new CompletableFuture<>(); + envelope.put(Field.TASK, taskType); + envelope.put(Field.FUTURE, future); outgoingPipeline.push(envelope); - return completed; + return future; } @VisibleForTesting diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeSession.java b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeSession.java index 694fe5ceb..33127e189 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeSession.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeSession.java @@ -3,26 +3,29 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.ethereum.beacon.discovery.enr.NodeRecord; -import org.ethereum.beacon.discovery.message.MessageCode; import org.ethereum.beacon.discovery.packet.Packet; import org.ethereum.beacon.discovery.storage.AuthTagRepository; import org.ethereum.beacon.discovery.storage.NodeBucket; import org.ethereum.beacon.discovery.storage.NodeBucketStorage; import org.ethereum.beacon.discovery.storage.NodeTable; +import org.ethereum.beacon.discovery.task.TaskStatus; import org.ethereum.beacon.discovery.task.TaskType; import org.ethereum.beacon.util.ExpirationScheduler; import tech.pegasys.artemis.util.bytes.Bytes32; import tech.pegasys.artemis.util.bytes.BytesValue; -import javax.annotation.Nullable; +import java.util.HashSet; import java.util.Map; import java.util.Optional; import java.util.Random; +import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; +import static org.ethereum.beacon.discovery.task.TaskStatus.AWAIT; + /** * Stores session status and all keys for discovery session between us (homeNode) and the other node */ @@ -47,7 +50,6 @@ public class NodeSession { private ExpirationScheduler requestExpirationScheduler = new ExpirationScheduler<>(CLEANUP_DELAY_SECONDS, TimeUnit.SECONDS); private CompletableFuture completableFuture = null; - private TaskType task = null; private BytesValue staticNodeKey; public NodeSession( @@ -87,22 +89,30 @@ public synchronized void sendOutgoing(Packet packet) { } /** - * The value selected as request ID must allow for concurrent conversations. Using a timestamp can - * result in parallel conversations with the same id, so this should be avoided. Request IDs also - * prevent replay of responses. Using a simple counter would be fine if the implementation could - * ensure that restarts or even re-installs would increment the counter based on previously saved - * state in all circumstances. The easiest to implement is a random number. + * Creates object with request information: requestId etc, RequestInfo, designed to maintain + * request status and its changes. Also stores info in session repository to track related + * messages. + * + *

The value selected as request ID must allow for concurrent conversations. Using a timestamp + * can result in parallel conversations with the same id, so this should be avoided. Request IDs + * also prevent replay of responses. Using a simple counter would be fine if the implementation + * could ensure that restarts or even re-installs would increment the counter based on previously + * saved state in all circumstances. The easiest to implement is a random number. * - *

Request ID is reserved for `messageCode` + * @param taskType Type of task, clarifies starting and reply message types + * @param future Future to be fired when task is succcessfully completed or exceptionally break + * when its failed + * @return info bundle. */ - public synchronized BytesValue getNextRequestId(MessageCode messageCode) { + public synchronized RequestInfo createNextRequest( + TaskType taskType, CompletableFuture future) { byte[] requestId = new byte[REQUEST_ID_SIZE]; rnd.nextBytes(requestId); - BytesValue wrapped = BytesValue.wrap(requestId); - RequestInfo requestInfo = new GeneralRequestInfo(messageCode); - requestIdStatuses.put(wrapped, requestInfo); + BytesValue wrappedId = BytesValue.wrap(requestId); + RequestInfo requestInfo = new GeneralRequestInfo(taskType, AWAIT, wrappedId, future); + requestIdStatuses.put(wrappedId, requestInfo); requestExpirationScheduler.put( - wrapped, + wrappedId, new Runnable() { @Override public void run() { @@ -110,11 +120,11 @@ public void run() { () -> String.format( "Request %s expired for id %s in session %s: no reply", - requestInfo, wrapped, this)); - requestIdStatuses.remove(wrapped); + requestInfo, wrappedId, this)); + requestIdStatuses.remove(wrappedId); } }); - return wrapped; + return requestInfo; } public synchronized void updateRequestInfo(BytesValue requestId, RequestInfo newRequestInfo) { @@ -142,6 +152,21 @@ public void run() { }); } + public synchronized void cancelAllRequests(String message) { + logger.debug(() -> String.format("Cancelling all requests in session %s", this)); + Set requestIdsCopy = new HashSet<>(requestIdStatuses.keySet()); + requestIdsCopy.forEach( + requestId -> { + RequestInfo requestInfo = clearRequestId(requestId); + requestInfo + .getFuture() + .completeExceptionally( + new RuntimeException( + String.format( + "Request %s cancelled due to reason: %s", requestInfo, message))); + }); + } + public synchronized BytesValue generateNonce() { byte[] nonce = new byte[NONCE_SIZE]; rnd.nextBytes(nonce); @@ -184,10 +209,16 @@ public void setRecipientKey(BytesValue recipientKey) { this.recipientKey = recipientKey; } - public synchronized void clearRequestId(BytesValue requestId, MessageCode expectedMessageCode) { + public synchronized void clearRequestId(BytesValue requestId, TaskType taskType) { + RequestInfo requestInfo = clearRequestId(requestId); + requestInfo.getFuture().complete(null); + assert taskType.equals(requestInfo.getTaskType()); + } + + private synchronized RequestInfo clearRequestId(BytesValue requestId) { RequestInfo requestInfo = requestIdStatuses.remove(requestId); - assert expectedMessageCode.equals(requestInfo.getMessageCode()); requestExpirationScheduler.cancel(requestId); + return requestInfo; } public synchronized Optional getRequestId(BytesValue requestId) { @@ -195,6 +226,12 @@ public synchronized Optional getRequestId(BytesValue requestId) { return requestId == null ? Optional.empty() : Optional.of(requestInfo); } + public synchronized Optional getFirstAwaitRequestInfo() { + return requestIdStatuses.values().stream() + .filter(requestInfo -> AWAIT.equals(requestInfo.getTaskStatus())) + .findFirst(); + } + public NodeTable getNodeTable() { return nodeTable; } @@ -243,22 +280,6 @@ public synchronized void setStatus(SessionStatus newStatus) { this.status = newStatus; } - public void saveFuture(@Nullable CompletableFuture completableFuture) { - this.completableFuture = completableFuture; - } - - public CompletableFuture loadFuture() { - return completableFuture; - } - - public void saveTask(@Nullable TaskType task) { - this.task = task; - } - - public TaskType loadTask() { - return task; - } - public BytesValue getStaticNodeKey() { return staticNodeKey; } @@ -270,27 +291,63 @@ public enum SessionStatus { AUTHENTICATED } - public static interface RequestInfo { - public MessageCode getMessageCode(); + public interface RequestInfo { + TaskType getTaskType(); + + TaskStatus getTaskStatus(); + + BytesValue getRequestId(); + + CompletableFuture getFuture(); } public static class GeneralRequestInfo implements RequestInfo { - private final MessageCode messageCode; + private final TaskType taskType; + private final TaskStatus taskStatus; + private final BytesValue requestId; + private final CompletableFuture future; + + public GeneralRequestInfo( + TaskType taskType, + TaskStatus taskStatus, + BytesValue requestId, + CompletableFuture future) { + this.taskType = taskType; + this.taskStatus = taskStatus; + this.requestId = requestId; + this.future = future; + } - public GeneralRequestInfo(MessageCode messageCode) { - this.messageCode = messageCode; + @Override + public TaskType getTaskType() { + return taskType; + } + + @Override + public TaskStatus getTaskStatus() { + return taskStatus; + } + + @Override + public BytesValue getRequestId() { + return requestId; } @Override - public MessageCode getMessageCode() { - return messageCode; + public CompletableFuture getFuture() { + return future; } @Override public String toString() { - return "GeneralRequestInfo{" + - "messageCode=" + messageCode + - '}'; + return "GeneralRequestInfo{" + + "taskType=" + + taskType + + ", taskStatus=" + + taskStatus + + ", requestId=" + + requestId + + '}'; } } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/NodesHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/NodesHandler.java index 1ba6860b9..bd7bca3e5 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/NodesHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/NodesHandler.java @@ -4,10 +4,13 @@ import org.apache.logging.log4j.Logger; import org.ethereum.beacon.discovery.NodeRecordInfo; import org.ethereum.beacon.discovery.NodeSession; -import org.ethereum.beacon.discovery.message.MessageCode; import org.ethereum.beacon.discovery.message.NodesMessage; +import org.ethereum.beacon.discovery.task.TaskStatus; +import org.ethereum.beacon.discovery.task.TaskType; +import tech.pegasys.artemis.util.bytes.BytesValue; import java.util.Optional; +import java.util.concurrent.CompletableFuture; public class NodesHandler implements MessageHandler { private static final Logger logger = LogManager.getLogger(FindNodeHandler.class); @@ -26,16 +29,27 @@ public void handle(NodesMessage message, NodeSession session) { if (requestInfo instanceof FindNodeRequestInfo) { int newNodesCount = ((FindNodeRequestInfo) requestInfo).getRemainingNodes() - 1; if (newNodesCount == 0) { - session.clearRequestId(message.getRequestId(), MessageCode.FINDNODE); + session.clearRequestId(message.getRequestId(), TaskType.FINDNODE); } else { - session.updateRequestInfo(message.getRequestId(), new FindNodeRequestInfo(newNodesCount)); + session.updateRequestInfo( + message.getRequestId(), + new FindNodeRequestInfo( + TaskStatus.IN_PROCESS, + message.getRequestId(), + requestInfo.getFuture(), + newNodesCount)); } } else { if (message.getTotal() > 1) { session.updateRequestInfo( - message.getRequestId(), new FindNodeRequestInfo(message.getTotal() - 1)); + message.getRequestId(), + new FindNodeRequestInfo( + TaskStatus.IN_PROCESS, + message.getRequestId(), + requestInfo.getFuture(), + message.getTotal() - 1)); } else { - session.clearRequestId(message.getRequestId(), MessageCode.FINDNODE); + session.clearRequestId(message.getRequestId(), TaskType.FINDNODE); } } @@ -57,27 +71,25 @@ public void handle(NodesMessage message, NodeSession session) { }); } - public static class FindNodeRequestInfo implements NodeSession.RequestInfo { + public static class FindNodeRequestInfo extends NodeSession.GeneralRequestInfo { private final int remainingNodes; - public FindNodeRequestInfo(int remainingNodes) { + public FindNodeRequestInfo( + TaskStatus taskStatus, + BytesValue requestId, + CompletableFuture future, + int remainingNodes) { + super(TaskType.FINDNODE, taskStatus, requestId, future); this.remainingNodes = remainingNodes; } - @Override - public MessageCode getMessageCode() { - return MessageCode.FINDNODE; - } - public int getRemainingNodes() { return remainingNodes; } @Override public String toString() { - return "FindNodeRequestInfo{" + - "remainingNodes=" + remainingNodes + - '}'; + return "FindNodeRequestInfo{" + "remainingNodes=" + remainingNodes + '}'; } } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/PongHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/PongHandler.java index f30cbeb39..a212d39c6 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/PongHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/PongHandler.java @@ -1,12 +1,12 @@ package org.ethereum.beacon.discovery.message.handler; import org.ethereum.beacon.discovery.NodeSession; -import org.ethereum.beacon.discovery.message.MessageCode; import org.ethereum.beacon.discovery.message.PongMessage; +import org.ethereum.beacon.discovery.task.TaskType; public class PongHandler implements MessageHandler { @Override public void handle(PongMessage message, NodeSession session) { - session.clearRequestId(message.getRequestId(), MessageCode.PING); + session.clearRequestId(message.getRequestId(), TaskType.PING); } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/AuthHeaderMessagePacketHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/AuthHeaderMessagePacketHandler.java index 8d21e6e9a..9868aae59 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/AuthHeaderMessagePacketHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/AuthHeaderMessagePacketHandler.java @@ -4,23 +4,27 @@ import org.apache.logging.log4j.Logger; import org.ethereum.beacon.discovery.Functions; import org.ethereum.beacon.discovery.NodeSession; -import org.ethereum.beacon.discovery.message.DiscoveryV5Message; -import org.ethereum.beacon.discovery.message.MessageCode; import org.ethereum.beacon.discovery.packet.AuthHeaderMessagePacket; import org.ethereum.beacon.discovery.pipeline.Envelope; import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; import org.ethereum.beacon.discovery.pipeline.Field; import org.ethereum.beacon.discovery.pipeline.HandlerUtil; -import org.ethereum.beacon.discovery.task.TaskType; +import org.ethereum.beacon.discovery.pipeline.Pipeline; +import org.ethereum.beacon.schedulers.Scheduler; import tech.pegasys.artemis.util.bytes.BytesValue; -import java.util.concurrent.CompletableFuture; - import static org.ethereum.beacon.discovery.NodeSession.SessionStatus.AUTHENTICATED; /** Handles {@link AuthHeaderMessagePacket} in {@link Field#PACKET_AUTH_HEADER_MESSAGE} field */ public class AuthHeaderMessagePacketHandler implements EnvelopeHandler { private static final Logger logger = LogManager.getLogger(AuthHeaderMessagePacketHandler.class); + private final Pipeline outgoingPipeline; + private final Scheduler scheduler; + + public AuthHeaderMessagePacketHandler(Pipeline outgoingPipeline, Scheduler scheduler) { + this.outgoingPipeline = outgoingPipeline; + this.scheduler = scheduler; + } @Override public void handle(Envelope envelope) { @@ -71,32 +75,11 @@ public void handle(Envelope envelope) { packet, session.getNodeRecord(), session.getStatus()); logger.error(error, ex); envelope.remove(Field.PACKET_AUTH_HEADER_MESSAGE); - if (session.loadFuture() != null) { - CompletableFuture future = session.loadFuture(); - session.saveFuture(null); - future.completeExceptionally(ex); - } + session.cancelAllRequests("Failed to handshake"); return; } session.setStatus(AUTHENTICATED); envelope.remove(Field.PACKET_AUTH_HEADER_MESSAGE); - if (session.loadFuture() != null) { - boolean taskCompleted = false; - if (envelope.get(Field.TASK).equals(TaskType.PING) - && packet.getMessage() instanceof DiscoveryV5Message - && ((DiscoveryV5Message) packet.getMessage()).getCode() == MessageCode.PONG) { - taskCompleted = true; - } - if (envelope.get(Field.TASK).equals(TaskType.FINDNODE) - && packet.getMessage() instanceof DiscoveryV5Message - && ((DiscoveryV5Message) packet.getMessage()).getCode() == MessageCode.NODES) { - taskCompleted = true; - } - if (taskCompleted) { - CompletableFuture future = session.loadFuture(); - future.complete(null); - session.saveFuture(null); - } - } + NextTaskHandler.tryToSendAwaitTaskIfAny(session, outgoingPipeline, scheduler); } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/MessagePacketHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/MessagePacketHandler.java index 2b283f0f0..88b236f85 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/MessagePacketHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/MessagePacketHandler.java @@ -3,16 +3,11 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.ethereum.beacon.discovery.NodeSession; -import org.ethereum.beacon.discovery.message.DiscoveryV5Message; -import org.ethereum.beacon.discovery.message.MessageCode; import org.ethereum.beacon.discovery.packet.MessagePacket; import org.ethereum.beacon.discovery.pipeline.Envelope; import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; import org.ethereum.beacon.discovery.pipeline.Field; import org.ethereum.beacon.discovery.pipeline.HandlerUtil; -import org.ethereum.beacon.discovery.task.TaskType; - -import java.util.concurrent.CompletableFuture; /** Handles {@link MessagePacket} in {@link Field#PACKET_MESSAGE} field */ public class MessagePacketHandler implements EnvelopeHandler { @@ -45,10 +40,13 @@ public void handle(Envelope envelope) { packet.verify(session.getAuthTag().get()); envelope.put(Field.MESSAGE, packet.getMessage()); } catch (AssertionError ex) { - logger.info( + logger.error( String.format( "Verification not passed for message [%s] from node %s in status %s", packet, session.getNodeRecord(), session.getStatus())); + envelope.remove(Field.PACKET_MESSAGE); + envelope.put(Field.BAD_PACKET, packet); + return; } catch (Exception ex) { String error = String.format( @@ -56,32 +54,9 @@ public void handle(Envelope envelope) { packet, session.getNodeRecord(), session.getStatus()); logger.error(error, ex); envelope.remove(Field.PACKET_MESSAGE); - if (envelope.contains(Field.FUTURE)) { - CompletableFuture future = (CompletableFuture) envelope.get(Field.FUTURE); - future.completeExceptionally(ex); - } + envelope.put(Field.BAD_PACKET, packet); return; } envelope.remove(Field.PACKET_MESSAGE); - CompletableFuture future = session.loadFuture(); - TaskType taskType = session.loadTask(); - if (future != null) { - boolean taskCompleted = false; - if (TaskType.PING.equals(taskType) - && packet.getMessage() instanceof DiscoveryV5Message - && ((DiscoveryV5Message) packet.getMessage()).getCode() == MessageCode.PONG) { - taskCompleted = true; - } - if (TaskType.FINDNODE.equals(taskType) - && packet.getMessage() instanceof DiscoveryV5Message - && ((DiscoveryV5Message) packet.getMessage()).getCode() == MessageCode.NODES) { - taskCompleted = true; - } - if (taskCompleted) { - future.complete(null); - session.saveFuture(null); - session.saveTask(null); - } - } } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NewTaskHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NewTaskHandler.java new file mode 100644 index 000000000..9b87eac99 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NewTaskHandler.java @@ -0,0 +1,47 @@ +package org.ethereum.beacon.discovery.pipeline.handler; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.ethereum.beacon.discovery.NodeSession; +import org.ethereum.beacon.discovery.pipeline.Envelope; +import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; +import org.ethereum.beacon.discovery.pipeline.Field; +import org.ethereum.beacon.discovery.pipeline.HandlerUtil; +import org.ethereum.beacon.discovery.task.TaskType; + +import java.util.concurrent.CompletableFuture; + +/** Enqueues task in session for any task found in {@link Field#TASK} */ +public class NewTaskHandler implements EnvelopeHandler { + private static final Logger logger = LogManager.getLogger(NewTaskHandler.class); + + @Override + public void handle(Envelope envelope) { + logger.trace( + () -> + String.format( + "Envelope %s in NewTaskHandler, checking requirements satisfaction", + envelope.getId())); + if (!HandlerUtil.requireField(Field.TASK, envelope)) { + return; + } + if (!HandlerUtil.requireField(Field.SESSION, envelope)) { + return; + } + if (!HandlerUtil.requireField(Field.FUTURE, envelope)) { + return; + } + logger.trace( + () -> + String.format( + "Envelope %s in NewTaskHandler, requirements are satisfied!", envelope.getId())); + + TaskType task = (TaskType) envelope.get(Field.TASK); + NodeSession session = (NodeSession) envelope.get(Field.SESSION); + CompletableFuture completableFuture = + (CompletableFuture) envelope.get(Field.FUTURE); + session.createNextRequest(task, completableFuture); + envelope.remove(Field.TASK); + envelope.remove(Field.FUTURE); + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NextTaskHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NextTaskHandler.java new file mode 100644 index 000000000..90c4d9312 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NextTaskHandler.java @@ -0,0 +1,97 @@ +package org.ethereum.beacon.discovery.pipeline.handler; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.ethereum.beacon.discovery.NodeSession; +import org.ethereum.beacon.discovery.packet.MessagePacket; +import org.ethereum.beacon.discovery.packet.RandomPacket; +import org.ethereum.beacon.discovery.pipeline.Envelope; +import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; +import org.ethereum.beacon.discovery.pipeline.Field; +import org.ethereum.beacon.discovery.pipeline.HandlerUtil; +import org.ethereum.beacon.discovery.pipeline.Pipeline; +import org.ethereum.beacon.discovery.task.TaskMessageFactory; +import org.ethereum.beacon.discovery.task.TaskStatus; +import org.ethereum.beacon.schedulers.Scheduler; +import tech.pegasys.artemis.util.bytes.BytesValue; + +import java.security.SecureRandom; +import java.time.Duration; +import java.util.Optional; + +/** Gets next request task in session and processes it */ +public class NextTaskHandler implements EnvelopeHandler { + private static final Logger logger = LogManager.getLogger(NextTaskHandler.class); + private static final int DEFAULT_DELAY_MS = 1000; + private final Pipeline outgoingPipeline; + private final Scheduler scheduler; + + public NextTaskHandler(Pipeline outgoingPipeline, Scheduler scheduler) { + this.outgoingPipeline = outgoingPipeline; + this.scheduler = scheduler; + } + + public static void tryToSendAwaitTaskIfAny( + NodeSession session, Pipeline outgoingPipeline, Scheduler scheduler) { + if (session.getFirstAwaitRequestInfo().isPresent()) { + Envelope dummy = new Envelope(); + dummy.put(Field.SESSION, session); + scheduler.executeWithDelay( + Duration.ofMillis(DEFAULT_DELAY_MS), () -> outgoingPipeline.push(dummy)); + } + } + + @Override + public void handle(Envelope envelope) { + logger.trace( + () -> + String.format( + "Envelope %s in NextTaskHandler, checking requirements satisfaction", + envelope.getId())); + if (!HandlerUtil.requireField(Field.SESSION, envelope)) { + return; + } + logger.trace( + () -> + String.format( + "Envelope %s in NextTaskHandler, requirements are satisfied!", envelope.getId())); + + NodeSession session = (NodeSession) envelope.get(Field.SESSION); + Optional requestInfoOpt = session.getFirstAwaitRequestInfo(); + if (!requestInfoOpt.isPresent()) { + logger.trace(() -> String.format("Envelope %s: no awaiting requests", envelope.getId())); + return; + } + + NodeSession.RequestInfo requestInfo = requestInfoOpt.get(); + logger.trace( + () -> + String.format( + "Envelope %s: processing awaiting request %s", envelope.getId(), requestInfo)); + BytesValue authTag = session.generateNonce(); + BytesValue requestId = requestInfo.getRequestId(); + if (session.getStatus().equals(NodeSession.SessionStatus.INITIAL)) { + RandomPacket randomPacket = + RandomPacket.create( + session.getHomeNodeId(), + session.getNodeRecord().getNodeId(), + authTag, + new SecureRandom()); + session.setAuthTag(authTag); + session.sendOutgoing(randomPacket); + session.setStatus(NodeSession.SessionStatus.RANDOM_PACKET_SENT); + } else if (session.getStatus().equals(NodeSession.SessionStatus.AUTHENTICATED)) { + MessagePacket messagePacket = + TaskMessageFactory.createPacketFromRequest(requestInfo, authTag, session); + session.sendOutgoing(messagePacket); + NodeSession.RequestInfo sentRequestInfo = + new NodeSession.GeneralRequestInfo( + requestInfo.getTaskType(), + TaskStatus.SENT, + requestInfo.getRequestId(), + requestInfo.getFuture()); + session.updateRequestInfo(requestId, sentRequestInfo); + tryToSendAwaitTaskIfAny(session, outgoingPipeline, scheduler); + } + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/TaskHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/TaskHandler.java deleted file mode 100644 index b08575b24..000000000 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/TaskHandler.java +++ /dev/null @@ -1,81 +0,0 @@ -package org.ethereum.beacon.discovery.pipeline.handler; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.ethereum.beacon.discovery.NodeSession; -import org.ethereum.beacon.discovery.packet.RandomPacket; -import org.ethereum.beacon.discovery.pipeline.Envelope; -import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; -import org.ethereum.beacon.discovery.pipeline.Field; -import org.ethereum.beacon.discovery.pipeline.HandlerUtil; -import org.ethereum.beacon.discovery.task.TaskMessageFactory; -import org.ethereum.beacon.discovery.task.TaskType; -import tech.pegasys.artemis.util.bytes.BytesValue; - -import java.security.SecureRandom; -import java.util.Random; -import java.util.concurrent.CompletableFuture; - -/** Performs task execution for any task found in {@link Field#TASK} */ -public class TaskHandler implements EnvelopeHandler { - private static final Logger logger = LogManager.getLogger(TaskHandler.class); - private final Random rnd; - - public TaskHandler(Random rnd) { - this.rnd = rnd; - } - - @Override - public void handle(Envelope envelope) { - logger.trace( - () -> - String.format( - "Envelope %s in TaskHandler, checking requirements satisfaction", - envelope.getId())); - if (!HandlerUtil.requireField(Field.TASK, envelope)) { - return; - } - if (!HandlerUtil.requireField(Field.SESSION, envelope)) { - return; - } - logger.trace( - () -> - String.format( - "Envelope %s in TaskHandler, requirements are satisfied!", envelope.getId())); - - TaskType task = (TaskType) envelope.get(Field.TASK); - NodeSession session = (NodeSession) envelope.get(Field.SESSION); - CompletableFuture completableFuture = - (CompletableFuture) envelope.get(Field.FUTURE); - BytesValue authTag = session.generateNonce(); - if (session.getStatus().equals(NodeSession.SessionStatus.INITIAL)) { - RandomPacket randomPacket = - RandomPacket.create( - session.getHomeNodeId(), - session.getNodeRecord().getNodeId(), - authTag, - new SecureRandom()); - session.setAuthTag(authTag); - session.saveFuture(completableFuture); - session.saveTask(task); - session.sendOutgoing(randomPacket); - session.setStatus(NodeSession.SessionStatus.RANDOM_PACKET_SENT); - } else if (session.getStatus().equals(NodeSession.SessionStatus.AUTHENTICATED)) { - if (TaskType.PING.equals(task)) { - session.sendOutgoing(TaskMessageFactory.createPingPacket(authTag, session)); - session.saveFuture(completableFuture); - } else if (TaskType.FINDNODE.equals(task)) { - session.sendOutgoing(TaskMessageFactory.createFindNodePacket(authTag, session)); - session.saveFuture(completableFuture); - } else { - throw new RuntimeException( - String.format( - "Task type %s handler not found in envelope %s", - envelope.get(Field.TASK), envelope.getId())); - } - } else { - completableFuture.completeExceptionally(new RuntimeException("Already initiating")); - // FIXME: or should we queue? - } - } -} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouPacketHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouPacketHandler.java index 3d8ed92b1..fb521541f 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouPacketHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouPacketHandler.java @@ -13,19 +13,28 @@ import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; import org.ethereum.beacon.discovery.pipeline.Field; import org.ethereum.beacon.discovery.pipeline.HandlerUtil; +import org.ethereum.beacon.discovery.pipeline.Pipeline; import org.ethereum.beacon.discovery.task.TaskMessageFactory; -import org.ethereum.beacon.discovery.task.TaskType; +import org.ethereum.beacon.schedulers.Scheduler; import org.ethereum.beacon.util.Utils; import org.web3j.crypto.ECKeyPair; import tech.pegasys.artemis.util.bytes.BytesValue; -import java.util.concurrent.CompletableFuture; +import java.util.Optional; +import java.util.function.Supplier; import static org.ethereum.beacon.discovery.enr.NodeRecord.FIELD_PKEY_SECP256K1; /** Handles {@link WhoAreYouPacket} in {@link Field#PACKET_WHOAREYOU} field */ public class WhoAreYouPacketHandler implements EnvelopeHandler { private static final Logger logger = LogManager.getLogger(WhoAreYouPacketHandler.class); + private final Pipeline outgoingPipeline; + private final Scheduler scheduler; + + public WhoAreYouPacketHandler(Pipeline outgoingPipeline, Scheduler scheduler) { + this.outgoingPipeline = outgoingPipeline; + this.scheduler = scheduler; + } @Override public void handle(Envelope envelope) { @@ -68,16 +77,17 @@ public void handle(Envelope envelope) { session.setInitiatorKey(hkdfKeys.getInitiatorKey()); session.setRecipientKey(hkdfKeys.getRecipientKey()); BytesValue authResponseKey = hkdfKeys.getAuthResponseKey(); - V5Message taskMessage = null; - if (session.loadTask() == TaskType.PING) { - taskMessage = TaskMessageFactory.createPing(session); - } else if (session.loadTask() == TaskType.FINDNODE) { - taskMessage = TaskMessageFactory.createFindNode(session); - } else { - throw new RuntimeException( - String.format( - "Type %s in envelope #%s is not known", session.loadTask(), envelope.getId())); - } + Optional requestInfoOpt = session.getFirstAwaitRequestInfo(); + final V5Message message = + requestInfoOpt + .map(requestInfo -> TaskMessageFactory.createMessageFromRequest(requestInfo, session)) + .orElseThrow( + (Supplier) + () -> + new RuntimeException( + String.format( + "Received WHOAREYOU in envelope #%s but no requests await in %s session", + envelope.getId(), session))); BytesValue ephemeralPubKey = BytesValue.wrap(Utils.extractBytesFromUnsignedBigInt(ephemeralKey.getPublicKey())); @@ -92,27 +102,29 @@ public void handle(Envelope envelope) { ephemeralPubKey, session.generateNonce(), hkdfKeys.getInitiatorKey(), - DiscoveryV5Message.from(taskMessage)); + DiscoveryV5Message.from(message)); session.sendOutgoing(response); } catch (AssertionError ex) { - logger.info( + String error = String.format( "Verification not passed for message [%s] from node %s in status %s", - packet, session.getNodeRecord(), session.getStatus())); - } catch (Exception ex) { + packet, session.getNodeRecord(), session.getStatus()); + logger.error(error, ex); + envelope.remove(Field.PACKET_WHOAREYOU); + session.cancelAllRequests("Bad WHOAREYOU received from node"); + return; + } catch (Throwable ex) { String error = String.format( "Failed to read message [%s] from node %s in status %s", packet, session.getNodeRecord(), session.getStatus()); logger.error(error, ex); envelope.remove(Field.PACKET_WHOAREYOU); - if (envelope.contains(Field.FUTURE)) { - CompletableFuture future = (CompletableFuture) envelope.get(Field.FUTURE); - future.completeExceptionally(ex); - } + session.cancelAllRequests("Bad WHOAREYOU received from node"); return; } session.setStatus(NodeSession.SessionStatus.AUTHENTICATED); envelope.remove(Field.PACKET_WHOAREYOU); + NextTaskHandler.tryToSendAwaitTaskIfAny(session, outgoingPipeline, scheduler); } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/task/TaskMessageFactory.java b/discovery/src/main/java/org/ethereum/beacon/discovery/task/TaskMessageFactory.java index 977b24431..f9658198e 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/task/TaskMessageFactory.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/task/TaskMessageFactory.java @@ -3,31 +3,70 @@ import org.ethereum.beacon.discovery.NodeSession; import org.ethereum.beacon.discovery.message.DiscoveryV5Message; import org.ethereum.beacon.discovery.message.FindNodeMessage; -import org.ethereum.beacon.discovery.message.MessageCode; import org.ethereum.beacon.discovery.message.PingMessage; +import org.ethereum.beacon.discovery.message.V5Message; import org.ethereum.beacon.discovery.packet.MessagePacket; import tech.pegasys.artemis.util.bytes.BytesValue; public class TaskMessageFactory { public static final int DEFAULT_DISTANCE = 10; - public static MessagePacket createPingPacket(BytesValue authTag, NodeSession session) { + public static MessagePacket createPacketFromRequest( + NodeSession.RequestInfo requestInfo, BytesValue authTag, NodeSession session) { + switch (requestInfo.getTaskType()) { + case PING: + { + return createPingPacket(authTag, session, requestInfo.getRequestId()); + } + case FINDNODE: + { + return createFindNodePacket(authTag, session, requestInfo.getRequestId()); + } + default: + { + throw new RuntimeException( + String.format("Type %s is not supported!", requestInfo.getTaskType())); + } + } + } + + public static V5Message createMessageFromRequest( + NodeSession.RequestInfo requestInfo, NodeSession session) { + switch (requestInfo.getTaskType()) { + case PING: + { + return createPing(session, requestInfo.getRequestId()); + } + case FINDNODE: + { + return createFindNode(requestInfo.getRequestId()); + } + default: + { + throw new RuntimeException( + String.format("Type %s is not supported!", requestInfo.getTaskType())); + } + } + } + + public static MessagePacket createPingPacket( + BytesValue authTag, NodeSession session, BytesValue requestId) { return MessagePacket.create( session.getHomeNodeId(), session.getNodeRecord().getNodeId(), authTag, session.getInitiatorKey(), - DiscoveryV5Message.from(createPing(session))); + DiscoveryV5Message.from(createPing(session, requestId))); } - public static PingMessage createPing(NodeSession session) { - return new PingMessage( - session.getNextRequestId(MessageCode.PING), session.getNodeRecord().getSeq()); + public static PingMessage createPing(NodeSession session, BytesValue requestId) { + return new PingMessage(requestId, session.getNodeRecord().getSeq()); } - public static MessagePacket createFindNodePacket(BytesValue authTag, NodeSession session) { - FindNodeMessage findNodeMessage = createFindNode(session); + public static MessagePacket createFindNodePacket( + BytesValue authTag, NodeSession session, BytesValue requestId) { + FindNodeMessage findNodeMessage = createFindNode(requestId); return MessagePacket.create( session.getHomeNodeId(), session.getNodeRecord().getNodeId(), @@ -36,7 +75,7 @@ public static MessagePacket createFindNodePacket(BytesValue authTag, NodeSession DiscoveryV5Message.from(findNodeMessage)); } - public static FindNodeMessage createFindNode(NodeSession session) { - return new FindNodeMessage(session.getNextRequestId(MessageCode.FINDNODE), DEFAULT_DISTANCE); + public static FindNodeMessage createFindNode(BytesValue requestId) { + return new FindNodeMessage(requestId, DEFAULT_DISTANCE); } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/task/TaskStatus.java b/discovery/src/main/java/org/ethereum/beacon/discovery/task/TaskStatus.java new file mode 100644 index 000000000..ea259b400 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/task/TaskStatus.java @@ -0,0 +1,8 @@ +package org.ethereum.beacon.discovery.task; + +public enum TaskStatus { + AWAIT, // waiting for handshake or whatever + SENT, // request sent + IN_PROCESS, // reply should contain several messages, at least one received but not all + // XXX: completed task is not stored, so no status for completed +} diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java index 32ba2aaa8..26f1050a7 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java @@ -72,7 +72,8 @@ public void test() throws Exception { nodeRecord1, nodePair1.getValue0(), Schedulers.createDefault().newSingleThreadDaemon("server-1"), - Schedulers.createDefault().newSingleThreadDaemon("client-1")); + Schedulers.createDefault().newSingleThreadDaemon("client-1"), + Schedulers.createDefault().newSingleThreadDaemon("tasks-1")); DiscoveryManagerImpl discoveryManager2 = new DiscoveryManagerImpl( nodeTableStorage2.get(), @@ -80,7 +81,8 @@ public void test() throws Exception { nodeRecord2, nodePair2.getValue0(), Schedulers.createDefault().newSingleThreadDaemon("server-2"), - Schedulers.createDefault().newSingleThreadDaemon("client-2")); + Schedulers.createDefault().newSingleThreadDaemon("client-2"), + Schedulers.createDefault().newSingleThreadDaemon("tasks-2")); // 3) Expect standard 1 => 2 dialog CountDownLatch randomSent1to2 = new CountDownLatch(1); diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java index 2059db520..60b218cae 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java @@ -84,14 +84,16 @@ public void test() throws Exception { nodeBucketStorage1, nodeRecord1, nodePair1.getValue0(), - from2to1); + from2to1, + Schedulers.createDefault().newSingleThreadDaemon("tasks-1")); DiscoveryManagerNoNetwork discoveryManager2 = new DiscoveryManagerNoNetwork( nodeTableStorage2.get(), nodeBucketStorage2, nodeRecord2, nodePair2.getValue0(), - from1to2); + from1to2, + Schedulers.createDefault().newSingleThreadDaemon("tasks-2")); // 2) Link outgoing of each one with incoming of another Flux.from(discoveryManager1.getOutgoingMessages()) diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/HandshakeHandlersTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/HandshakeHandlersTest.java index 87e060cdb..e57f4d58e 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/HandshakeHandlersTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/HandshakeHandlersTest.java @@ -6,6 +6,8 @@ import org.ethereum.beacon.discovery.packet.WhoAreYouPacket; import org.ethereum.beacon.discovery.pipeline.Envelope; import org.ethereum.beacon.discovery.pipeline.Field; +import org.ethereum.beacon.discovery.pipeline.Pipeline; +import org.ethereum.beacon.discovery.pipeline.PipelineImpl; import org.ethereum.beacon.discovery.pipeline.handler.AuthHeaderMessagePacketHandler; import org.ethereum.beacon.discovery.pipeline.handler.WhoAreYouPacketHandler; import org.ethereum.beacon.discovery.storage.AuthTagRepository; @@ -13,6 +15,8 @@ import org.ethereum.beacon.discovery.storage.NodeTableStorage; import org.ethereum.beacon.discovery.storage.NodeTableStorageFactoryImpl; import org.ethereum.beacon.discovery.task.TaskType; +import org.ethereum.beacon.schedulers.Scheduler; +import org.ethereum.beacon.schedulers.Schedulers; import org.javatuples.Pair; import org.junit.Test; import tech.pegasys.artemis.util.bytes.Bytes32; @@ -21,6 +25,7 @@ import java.util.ArrayList; import java.util.Random; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; @@ -105,7 +110,11 @@ public void authHeaderHandlerTest() throws Exception { new AuthTagRepository(), outgoingMessages2to1, rnd); - WhoAreYouPacketHandler whoAreYouPacketHandlerNode1 = new WhoAreYouPacketHandler(); + + Scheduler taskScheduler = Schedulers.createDefault().events(); + Pipeline outgoingPipeline = new PipelineImpl(); + WhoAreYouPacketHandler whoAreYouPacketHandlerNode1 = + new WhoAreYouPacketHandler(outgoingPipeline, taskScheduler); Envelope envelopeAt1From2 = new Envelope(); byte[] idNonceBytes = new byte[32]; Functions.getRandom().nextBytes(idNonceBytes); @@ -117,13 +126,14 @@ public void authHeaderHandlerTest() throws Exception { Field.PACKET_WHOAREYOU, WhoAreYouPacket.create(nodePair1.getValue1().getNodeId(), authTag, idNonce, UInt64.ZERO)); envelopeAt1From2.put(Field.SESSION, nodeSessionAt1For2); - nodeSessionAt1For2.saveTask(TaskType.FINDNODE); + CompletableFuture future = new CompletableFuture<>(); + nodeSessionAt1For2.createNextRequest(TaskType.FINDNODE, future); whoAreYouPacketHandlerNode1.handle(envelopeAt1From2); authHeaderPacketLatch.await(1, TimeUnit.SECONDS); // Node2 handle AuthHeaderPacket and finish handshake AuthHeaderMessagePacketHandler authHeaderMessagePacketHandlerNode2 = - new AuthHeaderMessagePacketHandler(); + new AuthHeaderMessagePacketHandler(outgoingPipeline, taskScheduler); Envelope envelopeAt2From1 = new Envelope(); envelopeAt2From1.put(PACKET_AUTH_HEADER_MESSAGE, authHeaderPacket[0]); envelopeAt2From1.put(SESSION, nodeSessionAt2For1); diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java b/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java index c64bd507a..a6b5a5226 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java @@ -15,11 +15,12 @@ import org.ethereum.beacon.discovery.pipeline.handler.IncomingDataPacker; import org.ethereum.beacon.discovery.pipeline.handler.MessageHandler; import org.ethereum.beacon.discovery.pipeline.handler.MessagePacketHandler; +import org.ethereum.beacon.discovery.pipeline.handler.NewTaskHandler; +import org.ethereum.beacon.discovery.pipeline.handler.NextTaskHandler; import org.ethereum.beacon.discovery.pipeline.handler.NodeIdToSession; import org.ethereum.beacon.discovery.pipeline.handler.NodeSessionRequestHandler; import org.ethereum.beacon.discovery.pipeline.handler.NotExpectedIncomingPacketHandler; import org.ethereum.beacon.discovery.pipeline.handler.OutgoingParcelHandler; -import org.ethereum.beacon.discovery.pipeline.handler.TaskHandler; import org.ethereum.beacon.discovery.pipeline.handler.UnknownPacketTagToSender; import org.ethereum.beacon.discovery.pipeline.handler.UnknownPacketTypeByStatus; import org.ethereum.beacon.discovery.pipeline.handler.WhoAreYouAttempt; @@ -29,13 +30,13 @@ import org.ethereum.beacon.discovery.storage.NodeBucketStorage; import org.ethereum.beacon.discovery.storage.NodeTable; import org.ethereum.beacon.discovery.task.TaskType; +import org.ethereum.beacon.schedulers.Scheduler; import org.reactivestreams.Publisher; import reactor.core.publisher.Flux; import reactor.core.publisher.FluxSink; import reactor.core.publisher.ReplayProcessor; import tech.pegasys.artemis.util.bytes.BytesValue; -import java.security.SecureRandom; import java.util.concurrent.CompletableFuture; /** @@ -57,7 +58,8 @@ public DiscoveryManagerNoNetwork( NodeBucketStorage nodeBucketStorage, NodeRecord homeNode, BytesValue homeNodePrivateKey, - Publisher incomingPackets) { + Publisher incomingPackets, + Scheduler taskScheduler) { AuthTagRepository authTagRepo = new AuthTagRepository(); this.incomingPackets = incomingPackets; NodeIdToSession nodeIdToSession = @@ -76,8 +78,8 @@ public DiscoveryManagerNoNetwork( .addHandler(nodeIdToSession) .addHandler(new UnknownPacketTypeByStatus()) .addHandler(new NotExpectedIncomingPacketHandler()) - .addHandler(new WhoAreYouPacketHandler()) - .addHandler(new AuthHeaderMessagePacketHandler()) + .addHandler(new WhoAreYouPacketHandler(outgoingPipeline, taskScheduler)) + .addHandler(new AuthHeaderMessagePacketHandler(outgoingPipeline, taskScheduler)) .addHandler(new MessagePacketHandler()) .addHandler(new MessageHandler()) .addHandler(new BadPacketLogger()); @@ -85,7 +87,8 @@ public DiscoveryManagerNoNetwork( .addHandler(new OutgoingParcelHandler(outgoingSink)) .addHandler(new NodeSessionRequestHandler()) .addHandler(nodeIdToSession) - .addHandler(new TaskHandler(new SecureRandom())); + .addHandler(new NewTaskHandler()) + .addHandler(new NextTaskHandler(outgoingPipeline, taskScheduler)); } @Override @@ -100,12 +103,12 @@ public void stop() {} public CompletableFuture executeTask(NodeRecord nodeRecord, TaskType taskType) { Envelope envelope = new Envelope(); - envelope.put(Field.TASK, taskType); envelope.put(Field.NODE, nodeRecord); - CompletableFuture completed = new CompletableFuture<>(); - envelope.put(Field.FUTURE, completed); + CompletableFuture future = new CompletableFuture<>(); + envelope.put(Field.TASK, taskType); + envelope.put(Field.FUTURE, future); outgoingPipeline.push(envelope); - return completed; + return future; } public Publisher getOutgoingMessages() { From 35935d803d4ae7240a59a951f61331e895300b1f Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Thu, 24 Oct 2019 04:38:32 +0300 Subject: [PATCH 52/77] discovery: fix that dead nodes were not removed --- .../beacon/discovery/storage/NodeTable.java | 4 +- .../discovery/storage/NodeTableImpl.java | 14 ++++- .../discovery/task/DiscoveryTaskManager.java | 58 +++++++++++++------ 3 files changed, 55 insertions(+), 21 deletions(-) diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTable.java b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTable.java index 66d854c9d..89ccfbbb8 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTable.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTable.java @@ -1,7 +1,7 @@ package org.ethereum.beacon.discovery.storage; -import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.NodeRecordInfo; +import org.ethereum.beacon.discovery.enr.NodeRecord; import tech.pegasys.artemis.util.bytes.Bytes32; import java.util.List; @@ -14,6 +14,8 @@ public interface NodeTable { void save(NodeRecordInfo node); + void remove(NodeRecordInfo node); + Optional getNode(Bytes32 nodeId); /** Returns list of nodes including `nodeId` (if it's found) in logLimit distance from it. */ diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableImpl.java index 328ffbadb..5a85c22b3 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableImpl.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableImpl.java @@ -7,8 +7,8 @@ import org.ethereum.beacon.db.source.HoleyList; import org.ethereum.beacon.db.source.SingleValueSource; import org.ethereum.beacon.discovery.Functions; -import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.NodeRecordInfo; +import org.ethereum.beacon.discovery.enr.NodeRecord; import tech.pegasys.artemis.ethereum.core.Hash32; import tech.pegasys.artemis.util.bytes.Bytes32; import tech.pegasys.artemis.util.bytes.BytesValue; @@ -79,6 +79,18 @@ public void save(NodeRecordInfo node) { } } + @Override + public void remove(NodeRecordInfo node) { + Hash32 nodeKey = Hash32.wrap(node.getNode().getNodeId()); + nodeTable.remove(nodeKey); + NodeIndex activeIndex = indexTable.get(getNodeIndex(nodeKey)).orElseGet(NodeIndex::new); + List nodes = activeIndex.getEntries(); + if (nodes.contains(nodeKey)) { + nodes.remove(nodeKey); + indexTable.put(getNodeIndex(nodeKey), activeIndex); + } + } + @Override public Optional getNode(Bytes32 nodeId) { return nodeTable.get(Hash32.wrap(nodeId)); diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/task/DiscoveryTaskManager.java b/discovery/src/main/java/org/ethereum/beacon/discovery/task/DiscoveryTaskManager.java index 4bd92dba6..8ddc483eb 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/task/DiscoveryTaskManager.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/task/DiscoveryTaskManager.java @@ -14,6 +14,7 @@ import java.util.function.Predicate; import java.util.stream.Stream; +import static org.ethereum.beacon.discovery.NodeStatus.DEAD; import static org.ethereum.beacon.discovery.task.TaskMessageFactory.DEFAULT_DISTANCE; /** Manages recurrent node check task(s) */ @@ -41,7 +42,7 @@ public class DiscoveryTaskManager { * *

    *
  • Node is ACTIVE and last connection retry was not too much time ago - *
  • Number of unsuccessful retries exceeds settings + *
  • Node is marked as {@link NodeStatus#DEAD} *
  • Node is not ACTIVE but last connection retry was "seconds ago" *
* @@ -54,14 +55,8 @@ public class DiscoveryTaskManager { && nodeRecord.getLastRetry() > currentTime - STATUS_EXPIRATION_SECONDS) { return false; // no need to rediscover } - if (nodeRecord.getRetry() >= MAX_RETRIES) { - updateNode( - new NodeRecordInfo( - nodeRecord.getNode(), - System.currentTimeMillis() / MS_IN_SECOND, - NodeStatus.SLEEP, - 1)); - return false; + if (DEAD.equals(nodeRecord.getStatus())) { + return false; // node looks dead but we are keeping its records for some reason } if ((currentTime - nodeRecord.getLastRetry()) < (nodeRecord.getRetry() * nodeRecord.getRetry())) { @@ -92,7 +87,12 @@ public class DiscoveryTaskManager { return false; }; - private boolean resetDead = false; + /** Checks whether node is eligible to be considered as dead */ + private final Predicate DEAD_RULE = + nodeRecord -> nodeRecord.getRetry() >= MAX_RETRIES; + + private boolean resetDead; + private boolean removeDead; /** * @param discoveryManager Discovery manager @@ -101,8 +101,9 @@ public class DiscoveryTaskManager { * format * @param homeNode Home node * @param scheduler scheduler to run recurrent tasks on - * @param resetDead Whether to reset dead status of the nodes. If set to true, resets its status - * at startup and sets number of used retries to 0 + * @param resetDead Whether to reset dead status of the nodes on start. If set to true, resets its + * status at startup and sets number of used retries to 0 + * @param removeDead Whether to remove nodes that are found dead after several retries */ public DiscoveryTaskManager( DiscoveryManager discoveryManager, @@ -110,7 +111,8 @@ public DiscoveryTaskManager( NodeBucketStorage nodeBucketStorage, NodeRecord homeNode, Scheduler scheduler, - boolean resetDead) { + boolean resetDead, + boolean removeDead) { this.scheduler = scheduler; this.nodeTable = nodeTable; this.nodeBucketStorage = nodeBucketStorage; @@ -121,6 +123,7 @@ public DiscoveryTaskManager( new RecursiveLookupTasks( discoveryManager, scheduler, Duration.ofSeconds(RETRY_TIMEOUT_SECONDS)); this.resetDead = resetDead; + this.removeDead = removeDead; } public void start() { @@ -135,15 +138,32 @@ public void start() { private void liveCheckTask() { List nodes = nodeTable.findClosestNodes(homeNodeId, LIVE_CHECK_DISTANCE); Stream closestNodes = nodes.stream(); + closestNodes + .filter(DEAD_RULE) + .forEach( + deadMarkedNode -> { + if (removeDead) { + nodeTable.remove(deadMarkedNode); + } else { + nodeTable.save( + new NodeRecordInfo( + deadMarkedNode.getNode(), + deadMarkedNode.getLastRetry(), + DEAD, + deadMarkedNode.getRetry())); + } + }); if (resetDead) { closestNodes = closestNodes.map( - nodeRecordInfo -> - new NodeRecordInfo( - nodeRecordInfo.getNode(), - nodeRecordInfo.getLastRetry(), - NodeStatus.SLEEP, - 0)); + nodeRecordInfo -> { + if (DEAD.equals(nodeRecordInfo.getStatus())) { + return new NodeRecordInfo( + nodeRecordInfo.getNode(), nodeRecordInfo.getLastRetry(), NodeStatus.SLEEP, 0); + } else { + return nodeRecordInfo; + } + }); resetDead = false; } closestNodes From b5d3b56c34434c547e9e576949c288d05370cd91 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Thu, 24 Oct 2019 04:45:12 +0300 Subject: [PATCH 53/77] discovery: live check task cleanup and comments --- .../beacon/discovery/task/DiscoveryTaskManager.java | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/task/DiscoveryTaskManager.java b/discovery/src/main/java/org/ethereum/beacon/discovery/task/DiscoveryTaskManager.java index 8ddc483eb..a08a20f9b 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/task/DiscoveryTaskManager.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/task/DiscoveryTaskManager.java @@ -102,7 +102,8 @@ public class DiscoveryTaskManager { * @param homeNode Home node * @param scheduler scheduler to run recurrent tasks on * @param resetDead Whether to reset dead status of the nodes on start. If set to true, resets its - * status at startup and sets number of used retries to 0 + * status at startup and sets number of used retries to 0. Reset applies after remove, so if + * remove is on, reset will be applied to 0 nodes * @param removeDead Whether to remove nodes that are found dead after several retries */ public DiscoveryTaskManager( @@ -137,8 +138,9 @@ public void start() { private void liveCheckTask() { List nodes = nodeTable.findClosestNodes(homeNodeId, LIVE_CHECK_DISTANCE); - Stream closestNodes = nodes.stream(); - closestNodes + + // Dead nodes handling + nodes.stream() .filter(DEAD_RULE) .forEach( deadMarkedNode -> { @@ -153,6 +155,9 @@ private void liveCheckTask() { deadMarkedNode.getRetry())); } }); + + // resets dead records + Stream closestNodes = nodes.stream(); if (resetDead) { closestNodes = closestNodes.map( @@ -166,6 +171,8 @@ private void liveCheckTask() { }); resetDead = false; } + + // Live check task closestNodes .filter(LIVE_CHECK_NODE_RULE) .forEach( From 2b62afaf6193b4f194988fb35d339ac39501ad0a Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Thu, 24 Oct 2019 04:50:30 +0300 Subject: [PATCH 54/77] discovery: bad packet handler cleanup --- .../beacon/discovery/DiscoveryManagerImpl.java | 4 ++-- .../{BadPacketLogger.java => BadPacketHandler.java} | 11 +++++++---- .../discovery/mock/DiscoveryManagerNoNetwork.java | 4 ++-- 3 files changed, 11 insertions(+), 8 deletions(-) rename discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/{BadPacketLogger.java => BadPacketHandler.java} (76%) diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java index 967bacab0..996be045b 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java @@ -14,7 +14,7 @@ import org.ethereum.beacon.discovery.pipeline.Pipeline; import org.ethereum.beacon.discovery.pipeline.PipelineImpl; import org.ethereum.beacon.discovery.pipeline.handler.AuthHeaderMessagePacketHandler; -import org.ethereum.beacon.discovery.pipeline.handler.BadPacketLogger; +import org.ethereum.beacon.discovery.pipeline.handler.BadPacketHandler; import org.ethereum.beacon.discovery.pipeline.handler.IncomingDataPacker; import org.ethereum.beacon.discovery.pipeline.handler.MessageHandler; import org.ethereum.beacon.discovery.pipeline.handler.MessagePacketHandler; @@ -88,7 +88,7 @@ public DiscoveryManagerImpl( .addHandler(new AuthHeaderMessagePacketHandler(outgoingPipeline, taskScheduler)) .addHandler(new MessagePacketHandler()) .addHandler(new MessageHandler()) - .addHandler(new BadPacketLogger()); + .addHandler(new BadPacketHandler()); outgoingPipeline .addHandler(new OutgoingParcelHandler(outgoingSink)) .addHandler(new NodeSessionRequestHandler()) diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/BadPacketLogger.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/BadPacketHandler.java similarity index 76% rename from discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/BadPacketLogger.java rename to discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/BadPacketHandler.java index e165affe3..31fb91dd6 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/BadPacketLogger.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/BadPacketHandler.java @@ -7,9 +7,9 @@ import org.ethereum.beacon.discovery.pipeline.Field; import org.ethereum.beacon.discovery.pipeline.HandlerUtil; -/** Logs all packets which are stored in {@link Field#BAD_PACKET} */ -public class BadPacketLogger implements EnvelopeHandler { - private static final Logger logger = LogManager.getLogger(BadPacketLogger.class); +/** Handles packet from {@link Field#BAD_PACKET}. Currently just logs it. */ +public class BadPacketHandler implements EnvelopeHandler { + private static final Logger logger = LogManager.getLogger(BadPacketHandler.class); @Override public void handle(Envelope envelope) { @@ -30,6 +30,9 @@ public void handle(Envelope envelope) { () -> String.format( "Bad packet: %s in envelope #%s", envelope.get(Field.BAD_PACKET), envelope.getId()), - (Exception) envelope.get(Field.BAD_EXCEPTION)); + envelope.get(Field.BAD_EXCEPTION) == null + ? null + : (Exception) envelope.get(Field.BAD_EXCEPTION)); + // TODO: Reputation penalty etc } } diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java b/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java index a6b5a5226..6e0471575 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java @@ -11,7 +11,7 @@ import org.ethereum.beacon.discovery.pipeline.Pipeline; import org.ethereum.beacon.discovery.pipeline.PipelineImpl; import org.ethereum.beacon.discovery.pipeline.handler.AuthHeaderMessagePacketHandler; -import org.ethereum.beacon.discovery.pipeline.handler.BadPacketLogger; +import org.ethereum.beacon.discovery.pipeline.handler.BadPacketHandler; import org.ethereum.beacon.discovery.pipeline.handler.IncomingDataPacker; import org.ethereum.beacon.discovery.pipeline.handler.MessageHandler; import org.ethereum.beacon.discovery.pipeline.handler.MessagePacketHandler; @@ -82,7 +82,7 @@ public DiscoveryManagerNoNetwork( .addHandler(new AuthHeaderMessagePacketHandler(outgoingPipeline, taskScheduler)) .addHandler(new MessagePacketHandler()) .addHandler(new MessageHandler()) - .addHandler(new BadPacketLogger()); + .addHandler(new BadPacketHandler()); outgoingPipeline .addHandler(new OutgoingParcelHandler(outgoingSink)) .addHandler(new NodeSessionRequestHandler()) From 01a60ad1d8bf8cbfb3c6f1566bdc76a8123f0da8 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Thu, 24 Oct 2019 06:53:15 +0300 Subject: [PATCH 55/77] discovery: extend packet handlers test --- .../discovery/message/DiscoveryV5Message.java | 2 +- .../discovery/packet/MessagePacket.java | 4 +- .../pipeline/handler/MessageHandler.java | 3 +- .../handler/MessagePacketHandler.java | 1 - .../discovery/HandshakeHandlersTest.java | 62 ++++++++++++++++--- 5 files changed, 57 insertions(+), 15 deletions(-) diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/DiscoveryV5Message.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/DiscoveryV5Message.java index e71d50d0e..790b3ef25 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/message/DiscoveryV5Message.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/DiscoveryV5Message.java @@ -62,7 +62,7 @@ public V5Message create() { { return new PingMessage( BytesValue.wrap(((RlpString) payload.get(0)).getBytes()), - UInt64.fromBytesBigEndian(Bytes8.wrap(((RlpString) payload.get(1)).getBytes()))); + UInt64.fromBytesBigEndian(Bytes8.leftPad(BytesValue.wrap(((RlpString) payload.get(1)).getBytes())))); } case PONG: { diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/MessagePacket.java b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/MessagePacket.java index 8a006a38c..3377a9960 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/MessagePacket.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/MessagePacket.java @@ -40,9 +40,7 @@ public static MessagePacket create( return new MessagePacket(tag.concat(authTagEncoded).concat(encryptedData)); } - public void verify(BytesValue expectedAuthTag) { - assert expectedAuthTag.equals(getAuthTag()); - } + public void verify(BytesValue expectedAuthTag) {} public Bytes32 getHomeNodeId(Bytes32 destNodeId) { verifyDecode(); diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/MessageHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/MessageHandler.java index 11b7afa6f..7c2622f0a 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/MessageHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/MessageHandler.java @@ -42,7 +42,8 @@ public void handle(Envelope envelope) { logger.trace( () -> String.format( - "Failed to handle message %s in envelope #%s", message, envelope.getId())); + "Failed to handle message %s in envelope #%s", message, envelope.getId()), + ex); envelope.put(Field.BAD_MESSAGE, message); envelope.put(Field.BAD_EXCEPTION, ex); envelope.remove(Field.MESSAGE); diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/MessagePacketHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/MessagePacketHandler.java index 88b236f85..312bf8e11 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/MessagePacketHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/MessagePacketHandler.java @@ -37,7 +37,6 @@ public void handle(Envelope envelope) { try { packet.decode(session.getInitiatorKey()); - packet.verify(session.getAuthTag().get()); envelope.put(Field.MESSAGE, packet.getMessage()); } catch (AssertionError ex) { logger.error( diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/HandshakeHandlersTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/HandshakeHandlersTest.java index e57f4d58e..c79e64551 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/HandshakeHandlersTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/HandshakeHandlersTest.java @@ -2,6 +2,7 @@ import org.ethereum.beacon.db.Database; import org.ethereum.beacon.discovery.enr.NodeRecord; +import org.ethereum.beacon.discovery.packet.MessagePacket; import org.ethereum.beacon.discovery.packet.Packet; import org.ethereum.beacon.discovery.packet.WhoAreYouPacket; import org.ethereum.beacon.discovery.pipeline.Envelope; @@ -9,11 +10,14 @@ import org.ethereum.beacon.discovery.pipeline.Pipeline; import org.ethereum.beacon.discovery.pipeline.PipelineImpl; import org.ethereum.beacon.discovery.pipeline.handler.AuthHeaderMessagePacketHandler; +import org.ethereum.beacon.discovery.pipeline.handler.MessageHandler; +import org.ethereum.beacon.discovery.pipeline.handler.MessagePacketHandler; import org.ethereum.beacon.discovery.pipeline.handler.WhoAreYouPacketHandler; import org.ethereum.beacon.discovery.storage.AuthTagRepository; import org.ethereum.beacon.discovery.storage.NodeBucketStorage; import org.ethereum.beacon.discovery.storage.NodeTableStorage; import org.ethereum.beacon.discovery.storage.NodeTableStorageFactoryImpl; +import org.ethereum.beacon.discovery.task.TaskMessageFactory; import org.ethereum.beacon.discovery.task.TaskType; import org.ethereum.beacon.schedulers.Scheduler; import org.ethereum.beacon.schedulers.Schedulers; @@ -26,20 +30,25 @@ import java.util.ArrayList; import java.util.Random; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; +import static org.ethereum.beacon.discovery.pipeline.Field.BAD_PACKET; +import static org.ethereum.beacon.discovery.pipeline.Field.MESSAGE; import static org.ethereum.beacon.discovery.pipeline.Field.PACKET_AUTH_HEADER_MESSAGE; +import static org.ethereum.beacon.discovery.pipeline.Field.PACKET_MESSAGE; import static org.ethereum.beacon.discovery.pipeline.Field.SESSION; import static org.ethereum.beacon.discovery.storage.NodeTableStorage.DEFAULT_SERIALIZER; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; public class HandshakeHandlersTest { @Test - public void authHeaderHandlerTest() throws Exception { + public void authHandlerWithMessageRoundTripTest() throws Exception { // Node1 Pair nodePair1 = TestUtil.generateNode(30303); NodeRecord nodeRecord1 = nodePair1.getValue1(); @@ -78,12 +87,14 @@ public void authHeaderHandlerTest() throws Exception { nodeTableStorageFactory.createBucketStorage(database2, DEFAULT_SERIALIZER, nodeRecord2); // Node1 create AuthHeaderPacket - final Packet[] authHeaderPacket = new Packet[1]; - final CountDownLatch authHeaderPacketLatch = new CountDownLatch(1); + final Packet[] outgoing1Packets = new Packet[2]; + final Semaphore outgoing1PacketsSemaphore = new Semaphore(2); + outgoing1PacketsSemaphore.acquire(2); final Consumer outgoingMessages1to2 = packet -> { - authHeaderPacket[0] = packet; - authHeaderPacketLatch.countDown(); + System.out.println("Outgoing packet from 1 to 2: " + packet); + outgoing1Packets[outgoing1PacketsSemaphore.availablePermits()] = packet; + outgoing1PacketsSemaphore.release(1); }; AuthTagRepository authTagRepository1 = new AuthTagRepository(); NodeSession nodeSessionAt1For2 = @@ -112,7 +123,7 @@ public void authHeaderHandlerTest() throws Exception { rnd); Scheduler taskScheduler = Schedulers.createDefault().events(); - Pipeline outgoingPipeline = new PipelineImpl(); + Pipeline outgoingPipeline = new PipelineImpl().build(); WhoAreYouPacketHandler whoAreYouPacketHandlerNode1 = new WhoAreYouPacketHandler(outgoingPipeline, taskScheduler); Envelope envelopeAt1From2 = new Envelope(); @@ -129,16 +140,49 @@ public void authHeaderHandlerTest() throws Exception { CompletableFuture future = new CompletableFuture<>(); nodeSessionAt1For2.createNextRequest(TaskType.FINDNODE, future); whoAreYouPacketHandlerNode1.handle(envelopeAt1From2); - authHeaderPacketLatch.await(1, TimeUnit.SECONDS); + assert outgoing1PacketsSemaphore.tryAcquire(1, 1, TimeUnit.SECONDS); + outgoing1PacketsSemaphore.release(); // Node2 handle AuthHeaderPacket and finish handshake AuthHeaderMessagePacketHandler authHeaderMessagePacketHandlerNode2 = new AuthHeaderMessagePacketHandler(outgoingPipeline, taskScheduler); Envelope envelopeAt2From1 = new Envelope(); - envelopeAt2From1.put(PACKET_AUTH_HEADER_MESSAGE, authHeaderPacket[0]); + envelopeAt2From1.put(PACKET_AUTH_HEADER_MESSAGE, outgoing1Packets[0]); envelopeAt2From1.put(SESSION, nodeSessionAt2For1); assertFalse(nodeSessionAt2For1.isAuthenticated()); authHeaderMessagePacketHandlerNode2.handle(envelopeAt2From1); assertTrue(nodeSessionAt2For1.isAuthenticated()); + + // Node 1 handles message from Node 2 + MessagePacketHandler messagePacketHandler1 = new MessagePacketHandler(); + Envelope envelopeAt1From2WithMessage = new Envelope(); + BytesValue pingAuthTag = nodeSessionAt1For2.generateNonce(); + MessagePacket pingPacketFrom2To1 = + TaskMessageFactory.createPingPacket( + pingAuthTag, + nodeSessionAt2For1, + nodeSessionAt2For1 + .createNextRequest(TaskType.PING, new CompletableFuture<>()) + .getRequestId()); + envelopeAt1From2WithMessage.put(PACKET_MESSAGE, pingPacketFrom2To1); + envelopeAt1From2WithMessage.put(SESSION, nodeSessionAt1For2); + messagePacketHandler1.handle(envelopeAt1From2WithMessage); + assertNull(envelopeAt1From2WithMessage.get(BAD_PACKET)); + assertNotNull(envelopeAt1From2WithMessage.get(MESSAGE)); + + MessageHandler messageHandler = new MessageHandler(); + messageHandler.handle(envelopeAt1From2WithMessage); + assert outgoing1PacketsSemaphore.tryAcquire(2, 1, TimeUnit.SECONDS); + + // Node 2 handles message from Node 1 + MessagePacketHandler messagePacketHandler2 = new MessagePacketHandler(); + Envelope envelopeAt2From1WithMessage = new Envelope(); + Packet pongPacketFrom1To2 = outgoing1Packets[1]; + MessagePacket pongMessagePacketFrom1To2 = (MessagePacket) pongPacketFrom1To2; + envelopeAt2From1WithMessage.put(PACKET_MESSAGE, pongMessagePacketFrom1To2); + envelopeAt2From1WithMessage.put(SESSION, nodeSessionAt1For2); + messagePacketHandler2.handle(envelopeAt2From1WithMessage); + assertNull(envelopeAt2From1WithMessage.get(BAD_PACKET)); + assertNotNull(envelopeAt2From1WithMessage.get(MESSAGE)); } } From 5e64379a8f4241e1b211425ab961ce932fe630a6 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Thu, 24 Oct 2019 07:50:46 +0300 Subject: [PATCH 56/77] discovery: fix enrSeq behavior --- .../storage/NodeTableStorageFactory.java | 4 +++- .../storage/NodeTableStorageFactoryImpl.java | 21 ++++++++++++------- .../discovery/task/DiscoveryTaskManager.java | 18 +++++++++++++--- .../discovery/DiscoveryNetworkTest.java | 4 ++-- .../discovery/DiscoveryNoNetworkTest.java | 4 ++-- .../discovery/HandshakeHandlersTest.java | 4 ++-- .../discovery/storage/NodeTableTest.java | 9 ++++---- 7 files changed, 43 insertions(+), 21 deletions(-) diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactory.java b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactory.java index 6134b2a33..cddda722d 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactory.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactory.java @@ -3,15 +3,17 @@ import org.ethereum.beacon.chain.storage.impl.SerializerFactory; import org.ethereum.beacon.db.Database; import org.ethereum.beacon.discovery.enr.NodeRecord; +import tech.pegasys.artemis.util.uint.UInt64; import java.util.List; +import java.util.function.Function; import java.util.function.Supplier; public interface NodeTableStorageFactory { NodeTableStorage createTable( Database database, SerializerFactory serializerFactory, - Supplier homeNodeSupplier, + Function homeNodeSupplier, Supplier> bootNodes); NodeBucketStorage createBucketStorage( diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactoryImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactoryImpl.java index 7af567065..84df180ac 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactoryImpl.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactoryImpl.java @@ -2,11 +2,12 @@ import org.ethereum.beacon.chain.storage.impl.SerializerFactory; import org.ethereum.beacon.db.Database; -import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.NodeRecordInfo; -import tech.pegasys.artemis.util.bytes.Bytes32; +import org.ethereum.beacon.discovery.enr.NodeRecord; +import tech.pegasys.artemis.util.uint.UInt64; import java.util.List; +import java.util.function.Function; import java.util.function.Supplier; public class NodeTableStorageFactoryImpl implements NodeTableStorageFactory { @@ -24,15 +25,12 @@ private boolean isStorageEmpty(NodeTableStorage nodeTableStorage) { public NodeTableStorage createTable( Database database, SerializerFactory serializerFactory, - Supplier homeNodeSupplier, + Function homeNodeSupplier, Supplier> bootNodesSupplier) { NodeTableStorage nodeTableStorage = new NodeTableStorageImpl(database, serializerFactory); // Init storage with boot nodes if its empty if (isStorageEmpty(nodeTableStorage)) { - nodeTableStorage - .getHomeNodeSource() - .set(NodeRecordInfo.createDefault(homeNodeSupplier.get())); bootNodesSupplier .get() .forEach( @@ -44,7 +42,16 @@ public NodeTableStorage createTable( nodeTableStorage.get().save(nodeRecordInfo); }); } - ; + // Rewrite home node with updated sequence number on init + UInt64 oldSeq = + nodeTableStorage + .getHomeNodeSource() + .get() + .map(nr -> nr.getNode().getSeq()) + .orElse(UInt64.ZERO); + nodeTableStorage + .getHomeNodeSource() + .set(NodeRecordInfo.createDefault(homeNodeSupplier.apply(oldSeq))); return nodeTableStorage; } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/task/DiscoveryTaskManager.java b/discovery/src/main/java/org/ethereum/beacon/discovery/task/DiscoveryTaskManager.java index a08a20f9b..a845552a6 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/task/DiscoveryTaskManager.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/task/DiscoveryTaskManager.java @@ -181,6 +181,7 @@ private void liveCheckTask() { nodeRecord, () -> updateNode( + nodeRecord, new NodeRecordInfo( nodeRecord.getNode(), System.currentTimeMillis() / MS_IN_SECOND, @@ -188,6 +189,7 @@ private void liveCheckTask() { 0)), () -> updateNode( + nodeRecord, new NodeRecordInfo( nodeRecord.getNode(), System.currentTimeMillis() / MS_IN_SECOND, @@ -206,6 +208,7 @@ private void recursiveLookupTask() { () -> {}, () -> updateNode( + nodeRecord, new NodeRecordInfo( nodeRecord.getNode(), System.currentTimeMillis() / MS_IN_SECOND, @@ -213,8 +216,17 @@ private void recursiveLookupTask() { (nodeRecord.getRetry() + 1))))); } - private void updateNode(NodeRecordInfo nodeRecordInfo) { - nodeTable.save(nodeRecordInfo); - nodeBucketStorage.put(nodeRecordInfo); + private void updateNode(NodeRecordInfo oldNodeRecordInfo, NodeRecordInfo newNodeRecordInfo) { + // use node with latest seq known + if (newNodeRecordInfo.getNode().getSeq().compareTo(oldNodeRecordInfo.getNode().getSeq()) < 0) { + newNodeRecordInfo = + new NodeRecordInfo( + oldNodeRecordInfo.getNode(), + newNodeRecordInfo.getLastRetry(), + newNodeRecordInfo.getStatus(), + newNodeRecordInfo.getRetry()); + } + nodeTable.save(newNodeRecordInfo); + nodeBucketStorage.put(newNodeRecordInfo); } } diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java index 26f1050a7..fc69c0e88 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java @@ -43,7 +43,7 @@ public void test() throws Exception { nodeTableStorageFactory.createTable( database1, DEFAULT_SERIALIZER, - () -> nodeRecord1, + (oldSeq) -> nodeRecord1, () -> new ArrayList() { { @@ -56,7 +56,7 @@ public void test() throws Exception { nodeTableStorageFactory.createTable( database2, DEFAULT_SERIALIZER, - () -> nodeRecord2, + (oldSeq) -> nodeRecord2, () -> new ArrayList() { { diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java index 60b218cae..b63753264 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java @@ -49,7 +49,7 @@ public void test() throws Exception { nodeTableStorageFactory.createTable( database1, DEFAULT_SERIALIZER, - () -> nodeRecord1, + (oldSeq) -> nodeRecord1, () -> new ArrayList() { { @@ -62,7 +62,7 @@ public void test() throws Exception { nodeTableStorageFactory.createTable( database2, DEFAULT_SERIALIZER, - () -> nodeRecord2, + (oldSeq) -> nodeRecord2, () -> new ArrayList() { { diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/HandshakeHandlersTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/HandshakeHandlersTest.java index c79e64551..fa790a99c 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/HandshakeHandlersTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/HandshakeHandlersTest.java @@ -63,7 +63,7 @@ public void authHandlerWithMessageRoundTripTest() throws Exception { nodeTableStorageFactory.createTable( database1, DEFAULT_SERIALIZER, - () -> nodeRecord1, + (oldSeq) -> nodeRecord1, () -> new ArrayList() { { @@ -76,7 +76,7 @@ public void authHandlerWithMessageRoundTripTest() throws Exception { nodeTableStorageFactory.createTable( database2, DEFAULT_SERIALIZER, - () -> nodeRecord2, + (oldSeq) -> nodeRecord2, () -> new ArrayList() { { diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeTableTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeTableTest.java index a2046f54c..93c3fafc9 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeTableTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeTableTest.java @@ -21,7 +21,7 @@ import java.util.List; import java.util.Optional; import java.util.Set; -import java.util.function.Supplier; +import java.util.function.Function; import static org.ethereum.beacon.discovery.storage.NodeTableStorage.DEFAULT_SERIALIZER; import static org.junit.Assert.assertEquals; @@ -30,8 +30,8 @@ public class NodeTableTest { private static final NodeRecordFactory NODE_RECORD_FACTORY = NodeRecordFactory.DEFAULT; - private Supplier homeNodeSupplier = - () -> { + private Function homeNodeSupplier = + (oldSeq) -> { try { return NODE_RECORD_FACTORY.createFromValues( EnrScheme.V4, @@ -80,7 +80,8 @@ public void testCreate() throws Exception { nodeRecord.get(NodeRecord.FIELD_PKEY_SECP256K1), nodeRecord2.getNode().get(NodeRecord.FIELD_PKEY_SECP256K1)); assertEquals( - nodeTableStorage.get().getHomeNode().getNodeId(), homeNodeSupplier.get().getNodeId()); + nodeTableStorage.get().getHomeNode().getNodeId(), + homeNodeSupplier.apply(UInt64.ZERO).getNodeId()); } @Test From e0c22d80ae2a714d91186f691d6947bbac2fd471 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Thu, 24 Oct 2019 07:56:58 +0300 Subject: [PATCH 57/77] discovery: clarifies homeNode provider goal plus javadoc --- .../storage/NodeTableStorageFactory.java | 18 ++++++++++++++++-- .../storage/NodeTableStorageFactoryImpl.java | 13 +++++++++++-- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactory.java b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactory.java index cddda722d..523511e89 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactory.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactory.java @@ -10,11 +10,25 @@ import java.util.function.Supplier; public interface NodeTableStorageFactory { + /** + * Creates storage for nodes table + * + * @param database Database + * @param serializerFactory Serializer factory + * @param homeNodeProvider Home node provider, accepts old sequence number of home node, usually + * sequence number is increased by 1 on each restart and ENR is signed with new sequence + * number + * @param bootNodesSupplier boot nodes provider + * + * @return {@link NodeTableStorage} from `database` but if it doesn't exist, creates new one with + * home node provided by `homeNodeSupplier` and boot nodes provided with `bootNodesSupplier`. + * Uses `serializerFactory` for node records serialization. + */ NodeTableStorage createTable( Database database, SerializerFactory serializerFactory, - Function homeNodeSupplier, - Supplier> bootNodes); + Function homeNodeProvider, + Supplier> bootNodesSupplier); NodeBucketStorage createBucketStorage( Database database, SerializerFactory serializerFactory, NodeRecord homeNode); diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactoryImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactoryImpl.java index 84df180ac..b6155688f 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactoryImpl.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactoryImpl.java @@ -17,6 +17,15 @@ private boolean isStorageEmpty(NodeTableStorage nodeTableStorage) { } /** + * Creates storage for nodes table + * + * @param database Database + * @param serializerFactory Serializer factory + * @param homeNodeProvider Home node provider, accepts old sequence number of home node, usually + * sequence number is increased by 1 on each restart and ENR is signed with new sequence + * number + * @param bootNodesSupplier boot nodes provider + * * @return {@link NodeTableStorage} from `database` but if it doesn't exist, creates new one with * home node provided by `homeNodeSupplier` and boot nodes provided with `bootNodesSupplier`. * Uses `serializerFactory` for node records serialization. @@ -25,7 +34,7 @@ private boolean isStorageEmpty(NodeTableStorage nodeTableStorage) { public NodeTableStorage createTable( Database database, SerializerFactory serializerFactory, - Function homeNodeSupplier, + Function homeNodeProvider, Supplier> bootNodesSupplier) { NodeTableStorage nodeTableStorage = new NodeTableStorageImpl(database, serializerFactory); @@ -51,7 +60,7 @@ public NodeTableStorage createTable( .orElse(UInt64.ZERO); nodeTableStorage .getHomeNodeSource() - .set(NodeRecordInfo.createDefault(homeNodeSupplier.apply(oldSeq))); + .set(NodeRecordInfo.createDefault(homeNodeProvider.apply(oldSeq))); return nodeTableStorage; } From fcdf192dc5d765fc4c7ed0daccfb70d332a0d793 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Thu, 24 Oct 2019 18:02:22 +0300 Subject: [PATCH 58/77] discovery enr signature verification + tests --- .../discovery/DiscoveryManagerImpl.java | 9 ++- .../DiscoveryV5MessageProcessor.java | 7 ++- .../ethereum/beacon/discovery/Functions.java | 6 +- .../beacon/discovery/NodeRecordInfo.java | 3 +- .../discovery/enr/EnrSchemeInterpreter.java | 6 +- .../discovery/enr/EnrSchemeV4Interpreter.java | 32 +++++++++- .../beacon/discovery/enr/NodeRecord.java | 14 ++++- .../discovery/message/DiscoveryV5Message.java | 6 +- .../message/handler/NodesHandler.java | 1 + .../packet/AuthHeaderMessagePacket.java | 4 +- .../AuthHeaderMessagePacketHandler.java | 8 ++- .../pipeline/handler/MessageHandler.java | 10 ++- .../beacon/discovery/storage/NodeBucket.java | 5 +- .../storage/NodeSerializerFactory.java | 9 ++- .../discovery/storage/NodeTableStorage.java | 3 - .../storage/NodeTableStorageFactoryImpl.java | 8 +-- .../discovery/DiscoveryNetworkTest.java | 13 ++-- .../discovery/DiscoveryNoNetworkTest.java | 10 +-- .../discovery/HandshakeHandlersTest.java | 16 ++--- .../beacon/discovery/NodeRecordTest.java | 61 +++++++++++++++++++ .../ethereum/beacon/discovery/TestUtil.java | 14 +++-- .../mock/DiscoveryManagerNoNetwork.java | 10 ++- .../mock/EnrSchemeV4InterpreterMock.java | 11 ++++ .../discovery/storage/NodeBucketTest.java | 9 ++- .../discovery/storage/NodeTableTest.java | 20 +++--- 25 files changed, 217 insertions(+), 78 deletions(-) create mode 100644 discovery/src/test/java/org/ethereum/beacon/discovery/mock/EnrSchemeV4InterpreterMock.java diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java index 996be045b..4070fe83b 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java @@ -4,6 +4,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.ethereum.beacon.discovery.enr.NodeRecord; +import org.ethereum.beacon.discovery.enr.NodeRecordFactory; import org.ethereum.beacon.discovery.network.DiscoveryClient; import org.ethereum.beacon.discovery.network.DiscoveryClientImpl; import org.ethereum.beacon.discovery.network.DiscoveryServer; @@ -51,6 +52,7 @@ public class DiscoveryManagerImpl implements DiscoveryManager { private final Scheduler scheduler; private final Pipeline incomingPipeline = new PipelineImpl(); private final Pipeline outgoingPipeline = new PipelineImpl(); + private final NodeRecordFactory nodeRecordFactory; private DiscoveryClient discoveryClient; public DiscoveryManagerImpl( @@ -58,11 +60,13 @@ public DiscoveryManagerImpl( NodeBucketStorage nodeBucketStorage, NodeRecord homeNode, BytesValue homeNodePrivateKey, + NodeRecordFactory nodeRecordFactory, Scheduler serverScheduler, Scheduler clientScheduler, Scheduler taskScheduler) { AuthTagRepository authTagRepo = new AuthTagRepository(); this.scheduler = serverScheduler; + this.nodeRecordFactory = nodeRecordFactory; this.discoveryServer = new DiscoveryServerImpl( ((Bytes4) homeNode.get(NodeRecord.FIELD_IP_V4)), @@ -85,9 +89,10 @@ public DiscoveryManagerImpl( .addHandler(new UnknownPacketTypeByStatus()) .addHandler(new NotExpectedIncomingPacketHandler()) .addHandler(new WhoAreYouPacketHandler(outgoingPipeline, taskScheduler)) - .addHandler(new AuthHeaderMessagePacketHandler(outgoingPipeline, taskScheduler)) + .addHandler( + new AuthHeaderMessagePacketHandler(outgoingPipeline, taskScheduler, nodeRecordFactory)) .addHandler(new MessagePacketHandler()) - .addHandler(new MessageHandler()) + .addHandler(new MessageHandler(nodeRecordFactory)) .addHandler(new BadPacketHandler()); outgoingPipeline .addHandler(new OutgoingParcelHandler(outgoingSink)) diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryV5MessageProcessor.java b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryV5MessageProcessor.java index e2ba69348..24b8e0bab 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryV5MessageProcessor.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryV5MessageProcessor.java @@ -2,6 +2,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.ethereum.beacon.discovery.enr.NodeRecordFactory; import org.ethereum.beacon.discovery.message.DiscoveryV5Message; import org.ethereum.beacon.discovery.message.MessageCode; import org.ethereum.beacon.discovery.message.handler.FindNodeHandler; @@ -16,12 +17,14 @@ public class DiscoveryV5MessageProcessor implements DiscoveryMessageProcessor { private static final Logger logger = LogManager.getLogger(DiscoveryV5MessageProcessor.class); private final Map messageHandlers = new HashMap<>(); + private final NodeRecordFactory nodeRecordFactory; - public DiscoveryV5MessageProcessor() { + public DiscoveryV5MessageProcessor(NodeRecordFactory nodeRecordFactory) { messageHandlers.put(MessageCode.PING, new PingHandler()); messageHandlers.put(MessageCode.PONG, new PongHandler()); messageHandlers.put(MessageCode.FINDNODE, new FindNodeHandler()); messageHandlers.put(MessageCode.NODES, new NodesHandler()); + this.nodeRecordFactory = nodeRecordFactory; } @Override @@ -37,6 +40,6 @@ public void handleMessage(DiscoveryV5Message message, NodeSession session) { if (messageHandler == null) { throw new RuntimeException("Not implemented yet"); } - messageHandler.handle(message.create(), session); + messageHandler.handle(message.create(nodeRecordFactory), session); } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java b/discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java index 2402fd52b..868fc8e5e 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java @@ -10,7 +10,6 @@ import org.ethereum.beacon.crypto.Hashes; import org.web3j.crypto.ECKeyPair; import org.web3j.crypto.Sign; -import tech.pegasys.artemis.util.bytes.Bytes1; import tech.pegasys.artemis.util.bytes.Bytes32; import tech.pegasys.artemis.util.bytes.Bytes32s; import tech.pegasys.artemis.util.bytes.BytesValue; @@ -39,15 +38,14 @@ public static Bytes32 hash(BytesValue value) { * * @param key private key * @param x message - * @return ECDSA signature with properties merged together: v || r || s + * @return ECDSA signature with properties merged together: r || s */ public static BytesValue sign(BytesValue key, BytesValue x) { Sign.SignatureData signatureData = Sign.signMessage(x.extractArray(), ECKeyPair.create(key.extractArray())); - Bytes1 v = Bytes1.wrap(new byte[] {signatureData.getV()}); Bytes32 r = Bytes32.wrap(signatureData.getR()); Bytes32 s = Bytes32.wrap(signatureData.getS()); - return v.concat(r).concat(s); + return r.concat(s); } /** diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeRecordInfo.java b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeRecordInfo.java index 9df57717a..5ec4fd91a 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeRecordInfo.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeRecordInfo.java @@ -18,7 +18,6 @@ * last test of its availability */ public class NodeRecordInfo { - private static final NodeRecordFactory nodeRecordFactory = NodeRecordFactory.DEFAULT; private final NodeRecord node; private final Long lastRetry; private final NodeStatus status; @@ -35,7 +34,7 @@ public static NodeRecordInfo createDefault(NodeRecord nodeRecord) { return new NodeRecordInfo(nodeRecord, -1L, NodeStatus.ACTIVE, 0); } - public static NodeRecordInfo fromRlpBytes(BytesValue bytes) { + public static NodeRecordInfo fromRlpBytes(BytesValue bytes, NodeRecordFactory nodeRecordFactory) { RlpList internalList = (RlpList) RlpDecoder.decode(bytes.extractArray()).getValues().get(0); return new NodeRecordInfo( nodeRecordFactory.fromBytes(((RlpString) internalList.getValues().get(0)).getBytes()), diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrSchemeInterpreter.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrSchemeInterpreter.java index 860ffb73f..2b772e8fd 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrSchemeInterpreter.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrSchemeInterpreter.java @@ -8,8 +8,10 @@ public interface EnrSchemeInterpreter { EnrScheme getScheme(); /** Verifies that `nodeRecord` is of scheme implementation */ - default boolean verify(NodeRecord nodeRecord) { - return nodeRecord.getIdentityScheme().equals(getScheme()); + default void verify(NodeRecord nodeRecord) { + if (!nodeRecord.getIdentityScheme().equals(getScheme())) { + throw new RuntimeException("Interpreter and node record schemes do not match!"); + } } /** Delivers nodeId according to identity scheme scheme */ diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrSchemeV4Interpreter.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrSchemeV4Interpreter.java index 91be46811..e8126b694 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrSchemeV4Interpreter.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrSchemeV4Interpreter.java @@ -1,6 +1,10 @@ package org.ethereum.beacon.discovery.enr; +import org.bouncycastle.util.Arrays; import org.ethereum.beacon.crypto.Hashes; +import org.web3j.crypto.ECDSASignature; +import org.web3j.crypto.Hash; +import org.web3j.crypto.Sign; import org.web3j.rlp.RlpString; import tech.pegasys.artemis.util.bytes.Bytes16; import tech.pegasys.artemis.util.bytes.Bytes32; @@ -19,6 +23,7 @@ import static org.ethereum.beacon.discovery.enr.NodeRecord.FIELD_TCP_V6; import static org.ethereum.beacon.discovery.enr.NodeRecord.FIELD_UDP_V4; import static org.ethereum.beacon.discovery.enr.NodeRecord.FIELD_UDP_V6; +import static org.ethereum.beacon.util.Utils.extractBytesFromUnsignedBigInt; public class EnrSchemeV4Interpreter implements EnrSchemeInterpreter { @@ -37,9 +42,30 @@ public EnrSchemeV4Interpreter() { } @Override - public boolean verify(NodeRecord nodeRecord) { - return EnrSchemeInterpreter.super.verify(nodeRecord) - && nodeRecord.get(FIELD_PKEY_SECP256K1) != null; + public void verify(NodeRecord nodeRecord) { + EnrSchemeInterpreter.super.verify(nodeRecord); + if (nodeRecord.get(FIELD_PKEY_SECP256K1) == null) { + throw new RuntimeException( + String.format( + "Field %s not exists but required for scheme %s", FIELD_PKEY_SECP256K1, getScheme())); + } + BytesValue pubKey = (BytesValue) nodeRecord.get(FIELD_PKEY_SECP256K1); + ECDSASignature ecdsaSignature = + new ECDSASignature( + new BigInteger(1, nodeRecord.getSignature().slice(0, 32).extractArray()), + new BigInteger(1, nodeRecord.getSignature().slice(32).extractArray())); + byte[] msgHash = Hash.sha3(nodeRecord.serialize(false).extractArray()); + for (int recId = 0; recId < 4; ++recId) { + BigInteger calculatedPubKey = Sign.recoverFromSignature(1, ecdsaSignature, msgHash); + if (calculatedPubKey == null) { + continue; + } + if (Arrays.areEqual( + pubKey.extractArray(), extractBytesFromUnsignedBigInt(calculatedPubKey))) { + return; + } + } + assert false; } @Override diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecord.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecord.java index 658d6cc90..8881663fb 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecord.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecord.java @@ -133,15 +133,23 @@ public int hashCode() { return Objects.hashCode(seq, signature, fields); } + public void verify() { + enrSchemeInterpreter.verify(this); + } + public BytesValue serialize() { - assert getSignature() != null; - assert getSeq() != null; + return serialize(true); + } + public BytesValue serialize(boolean withSignature) { + assert getSeq() != null; // content = [seq, k, v, ...] // signature = sign(content) // record = [signature, seq, k, v, ...] List values = new ArrayList<>(); - values.add(RlpString.create(getSignature().extractArray())); + if (withSignature) { + values.add(RlpString.create(getSignature().extractArray())); + } values.add(RlpString.create(getSeq().toBI())); values.add(RlpString.create("id")); values.add(RlpString.create(getIdentityScheme().stringName())); diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/DiscoveryV5Message.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/DiscoveryV5Message.java index 790b3ef25..ec9df5fda 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/message/DiscoveryV5Message.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/DiscoveryV5Message.java @@ -14,7 +14,6 @@ import java.util.stream.Collectors; public class DiscoveryV5Message implements DiscoveryMessage { - private static final NodeRecordFactory nodeRecordFactory = NodeRecordFactory.DEFAULT; private final BytesValue bytes; private List payload = null; @@ -54,7 +53,7 @@ public BytesValue getRequestId() { return BytesValue.wrap(((RlpString) payload.get(0)).getBytes()); } - public V5Message create() { + public V5Message create(NodeRecordFactory nodeRecordFactory) { decode(); MessageCode code = MessageCode.fromNumber(getBytes().get(0)); switch (code) { @@ -62,7 +61,8 @@ public V5Message create() { { return new PingMessage( BytesValue.wrap(((RlpString) payload.get(0)).getBytes()), - UInt64.fromBytesBigEndian(Bytes8.leftPad(BytesValue.wrap(((RlpString) payload.get(1)).getBytes())))); + UInt64.fromBytesBigEndian( + Bytes8.leftPad(BytesValue.wrap(((RlpString) payload.get(1)).getBytes())))); } case PONG: { diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/NodesHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/NodesHandler.java index bd7bca3e5..fc3e58a56 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/NodesHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/NodesHandler.java @@ -63,6 +63,7 @@ public void handle(NodesMessage message, NodeSession session) { .getNodeRecords() .forEach( nodeRecordV5 -> { + nodeRecordV5.verify(); NodeRecordInfo nodeRecordInfo = NodeRecordInfo.createDefault(nodeRecordV5); if (!session.getNodeTable().getNode(nodeRecordV5.getNodeId()).isPresent()) { session.getNodeTable().save(nodeRecordInfo); diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/AuthHeaderMessagePacket.java b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/AuthHeaderMessagePacket.java index 0e31e73e4..94205ac3d 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/AuthHeaderMessagePacket.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/AuthHeaderMessagePacket.java @@ -42,7 +42,6 @@ */ public class AuthHeaderMessagePacket extends AbstractPacket { public static final String AUTH_SCHEME_NAME = "gcm"; - private static final NodeRecordFactory nodeRecordFactory = NodeRecordFactory.DEFAULT; private static final BytesValue DISCOVERY_ID_NONCE = BytesValue.wrap("discovery-id-nonce".getBytes()); private static final BytesValue ZERO_NONCE = BytesValue.wrap(new byte[12]); @@ -169,7 +168,8 @@ public void decodeEphemeralPubKey() { } /** Run {@link AuthHeaderMessagePacket#decodeEphemeralPubKey()} before second part */ - public void decodeMessage(BytesValue initiatorKey, BytesValue authResponseKey) { + public void decodeMessage( + BytesValue initiatorKey, BytesValue authResponseKey, NodeRecordFactory nodeRecordFactory) { if (decodedEphemeralPubKeyPt == null) { throw new RuntimeException("Run decodeEphemeralPubKey() before"); } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/AuthHeaderMessagePacketHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/AuthHeaderMessagePacketHandler.java index 9868aae59..e8341a897 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/AuthHeaderMessagePacketHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/AuthHeaderMessagePacketHandler.java @@ -4,6 +4,7 @@ import org.apache.logging.log4j.Logger; import org.ethereum.beacon.discovery.Functions; import org.ethereum.beacon.discovery.NodeSession; +import org.ethereum.beacon.discovery.enr.NodeRecordFactory; import org.ethereum.beacon.discovery.packet.AuthHeaderMessagePacket; import org.ethereum.beacon.discovery.pipeline.Envelope; import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; @@ -20,10 +21,13 @@ public class AuthHeaderMessagePacketHandler implements EnvelopeHandler { private static final Logger logger = LogManager.getLogger(AuthHeaderMessagePacketHandler.class); private final Pipeline outgoingPipeline; private final Scheduler scheduler; + private final NodeRecordFactory nodeRecordFactory; - public AuthHeaderMessagePacketHandler(Pipeline outgoingPipeline, Scheduler scheduler) { + public AuthHeaderMessagePacketHandler( + Pipeline outgoingPipeline, Scheduler scheduler, NodeRecordFactory nodeRecordFactory) { this.outgoingPipeline = outgoingPipeline; this.scheduler = scheduler; + this.nodeRecordFactory = nodeRecordFactory; } @Override @@ -60,7 +64,7 @@ public void handle(Envelope envelope) { session.getIdNonce()); session.setInitiatorKey(keys.getInitiatorKey()); session.setRecipientKey(keys.getRecipientKey()); - packet.decodeMessage(keys.getInitiatorKey(), keys.getAuthResponseKey()); + packet.decodeMessage(keys.getInitiatorKey(), keys.getAuthResponseKey(), nodeRecordFactory); packet.verify(session.getIdNonce()); envelope.put(Field.MESSAGE, packet.getMessage()); } catch (AssertionError ex) { diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/MessageHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/MessageHandler.java index 7c2622f0a..1d5071e7e 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/MessageHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/MessageHandler.java @@ -5,6 +5,7 @@ import org.ethereum.beacon.discovery.DiscoveryV5MessageProcessor; import org.ethereum.beacon.discovery.MessageProcessor; import org.ethereum.beacon.discovery.NodeSession; +import org.ethereum.beacon.discovery.enr.NodeRecordFactory; import org.ethereum.beacon.discovery.message.DiscoveryMessage; import org.ethereum.beacon.discovery.pipeline.Envelope; import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; @@ -13,8 +14,13 @@ public class MessageHandler implements EnvelopeHandler { private static final Logger logger = LogManager.getLogger(MessageHandler.class); - private static final MessageProcessor messageProcessor = - new MessageProcessor(new DiscoveryV5MessageProcessor());; + private final MessageProcessor messageProcessor; + + public MessageHandler(NodeRecordFactory nodeRecordFactory) { + this.messageProcessor = + new MessageProcessor(new DiscoveryV5MessageProcessor(nodeRecordFactory)); + ; + } @Override public void handle(Envelope envelope) { diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeBucket.java b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeBucket.java index 5eb431076..2854daae1 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeBucket.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeBucket.java @@ -2,6 +2,7 @@ import org.ethereum.beacon.discovery.NodeRecordInfo; import org.ethereum.beacon.discovery.NodeStatus; +import org.ethereum.beacon.discovery.enr.NodeRecordFactory; import org.web3j.rlp.RlpDecoder; import org.web3j.rlp.RlpEncoder; import org.web3j.rlp.RlpList; @@ -37,14 +38,14 @@ public class NodeBucket { }; private final TreeSet bucket = new TreeSet<>(COMPARATOR); - public static NodeBucket fromRlpBytes(BytesValue bytes) { + public static NodeBucket fromRlpBytes(BytesValue bytes, NodeRecordFactory nodeRecordFactory) { NodeBucket nodeBucket = new NodeBucket(); ((RlpList) RlpDecoder.decode(bytes.extractArray()).getValues().get(0)) .getValues().stream() .map(rt -> (RlpString) rt) .map(RlpString::getBytes) .map(BytesValue::wrap) - .map(NodeRecordInfo::fromRlpBytes) + .map((BytesValue bytes1) -> NodeRecordInfo.fromRlpBytes(bytes1, nodeRecordFactory)) .forEach(nodeBucket::put); return nodeBucket; } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeSerializerFactory.java b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeSerializerFactory.java index a78be3671..a3874811f 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeSerializerFactory.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeSerializerFactory.java @@ -2,6 +2,7 @@ import org.ethereum.beacon.chain.storage.impl.SerializerFactory; import org.ethereum.beacon.discovery.NodeRecordInfo; +import org.ethereum.beacon.discovery.enr.NodeRecordFactory; import tech.pegasys.artemis.util.bytes.BytesValue; import java.util.HashMap; @@ -12,12 +13,14 @@ public class NodeSerializerFactory implements SerializerFactory { private final Map> deserializerMap = new HashMap<>(); private final Map> serializerMap = new HashMap<>(); - public NodeSerializerFactory() { - deserializerMap.put(NodeRecordInfo.class, NodeRecordInfo::fromRlpBytes); + public NodeSerializerFactory(NodeRecordFactory nodeRecordFactory) { + deserializerMap.put( + NodeRecordInfo.class, bytes1 -> NodeRecordInfo.fromRlpBytes(bytes1, nodeRecordFactory)); serializerMap.put(NodeRecordInfo.class, o -> ((NodeRecordInfo) o).toRlpBytes()); deserializerMap.put(NodeIndex.class, NodeIndex::fromRlpBytes); serializerMap.put(NodeIndex.class, o -> ((NodeIndex) o).toRlpBytes()); - deserializerMap.put(NodeBucket.class, NodeBucket::fromRlpBytes); + deserializerMap.put( + NodeBucket.class, bytes -> NodeBucket.fromRlpBytes(bytes, nodeRecordFactory)); serializerMap.put(NodeBucket.class, o -> ((NodeBucket) o).toRlpBytes()); } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorage.java b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorage.java index 4cee79dc7..95a1078f8 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorage.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorage.java @@ -1,13 +1,10 @@ package org.ethereum.beacon.discovery.storage; -import org.ethereum.beacon.chain.storage.impl.SerializerFactory; import org.ethereum.beacon.db.source.SingleValueSource; import org.ethereum.beacon.discovery.NodeRecordInfo; /** Stores {@link NodeTable} and home node info */ public interface NodeTableStorage { - SerializerFactory DEFAULT_SERIALIZER = new NodeSerializerFactory(); - NodeTable get(); SingleValueSource getHomeNodeSource(); diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactoryImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactoryImpl.java index b6155688f..be8a4cc56 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactoryImpl.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactoryImpl.java @@ -25,7 +25,6 @@ private boolean isStorageEmpty(NodeTableStorage nodeTableStorage) { * sequence number is increased by 1 on each restart and ENR is signed with new sequence * number * @param bootNodesSupplier boot nodes provider - * * @return {@link NodeTableStorage} from `database` but if it doesn't exist, creates new one with * home node provided by `homeNodeSupplier` and boot nodes provided with `bootNodesSupplier`. * Uses `serializerFactory` for node records serialization. @@ -47,6 +46,7 @@ public NodeTableStorage createTable( if (!(nodeRecord instanceof NodeRecord)) { throw new RuntimeException("Only V4 node records are supported as boot nodes"); } + nodeRecord.verify(); NodeRecordInfo nodeRecordInfo = NodeRecordInfo.createDefault(nodeRecord); nodeTableStorage.get().save(nodeRecordInfo); }); @@ -58,9 +58,9 @@ public NodeTableStorage createTable( .get() .map(nr -> nr.getNode().getSeq()) .orElse(UInt64.ZERO); - nodeTableStorage - .getHomeNodeSource() - .set(NodeRecordInfo.createDefault(homeNodeProvider.apply(oldSeq))); + NodeRecord updatedHomeNodeRecord = homeNodeProvider.apply(oldSeq); + updatedHomeNodeRecord.verify(); + nodeTableStorage.getHomeNodeSource().set(NodeRecordInfo.createDefault(updatedHomeNodeRecord)); return nodeTableStorage; } diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java index fc69c0e88..e32b87a85 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java @@ -22,7 +22,8 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import static org.ethereum.beacon.discovery.storage.NodeTableStorage.DEFAULT_SERIALIZER; +import static org.ethereum.beacon.discovery.TestUtil.NODE_RECORD_FACTORY_NO_VERIFICATION; +import static org.ethereum.beacon.discovery.TestUtil.TEST_SERIALIZER; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -42,7 +43,7 @@ public void test() throws Exception { NodeTableStorage nodeTableStorage1 = nodeTableStorageFactory.createTable( database1, - DEFAULT_SERIALIZER, + TEST_SERIALIZER, (oldSeq) -> nodeRecord1, () -> new ArrayList() { @@ -51,11 +52,11 @@ public void test() throws Exception { } }); NodeBucketStorage nodeBucketStorage1 = - nodeTableStorageFactory.createBucketStorage(database1, DEFAULT_SERIALIZER, nodeRecord1); + nodeTableStorageFactory.createBucketStorage(database1, TEST_SERIALIZER, nodeRecord1); NodeTableStorage nodeTableStorage2 = nodeTableStorageFactory.createTable( database2, - DEFAULT_SERIALIZER, + TEST_SERIALIZER, (oldSeq) -> nodeRecord2, () -> new ArrayList() { @@ -64,13 +65,14 @@ public void test() throws Exception { } }); NodeBucketStorage nodeBucketStorage2 = - nodeTableStorageFactory.createBucketStorage(database2, DEFAULT_SERIALIZER, nodeRecord2); + nodeTableStorageFactory.createBucketStorage(database2, TEST_SERIALIZER, nodeRecord2); DiscoveryManagerImpl discoveryManager1 = new DiscoveryManagerImpl( nodeTableStorage1.get(), nodeBucketStorage1, nodeRecord1, nodePair1.getValue0(), + NODE_RECORD_FACTORY_NO_VERIFICATION, Schedulers.createDefault().newSingleThreadDaemon("server-1"), Schedulers.createDefault().newSingleThreadDaemon("client-1"), Schedulers.createDefault().newSingleThreadDaemon("tasks-1")); @@ -80,6 +82,7 @@ public void test() throws Exception { nodeBucketStorage2, nodeRecord2, nodePair2.getValue0(), + NODE_RECORD_FACTORY_NO_VERIFICATION, Schedulers.createDefault().newSingleThreadDaemon("server-2"), Schedulers.createDefault().newSingleThreadDaemon("client-2"), Schedulers.createDefault().newSingleThreadDaemon("tasks-2")); diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java index b63753264..8510bd87a 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java @@ -24,7 +24,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import static org.ethereum.beacon.discovery.storage.NodeTableStorage.DEFAULT_SERIALIZER; +import static org.ethereum.beacon.discovery.TestUtil.TEST_SERIALIZER; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -48,7 +48,7 @@ public void test() throws Exception { NodeTableStorage nodeTableStorage1 = nodeTableStorageFactory.createTable( database1, - DEFAULT_SERIALIZER, + TEST_SERIALIZER, (oldSeq) -> nodeRecord1, () -> new ArrayList() { @@ -57,11 +57,11 @@ public void test() throws Exception { } }); NodeBucketStorage nodeBucketStorage1 = - nodeTableStorageFactory.createBucketStorage(database1, DEFAULT_SERIALIZER, nodeRecord1); + nodeTableStorageFactory.createBucketStorage(database1, TEST_SERIALIZER, nodeRecord1); NodeTableStorage nodeTableStorage2 = nodeTableStorageFactory.createTable( database2, - DEFAULT_SERIALIZER, + TEST_SERIALIZER, (oldSeq) -> nodeRecord2, () -> new ArrayList() { @@ -71,7 +71,7 @@ public void test() throws Exception { } }); NodeBucketStorage nodeBucketStorage2 = - nodeTableStorageFactory.createBucketStorage(database2, DEFAULT_SERIALIZER, nodeRecord2); + nodeTableStorageFactory.createBucketStorage(database2, TEST_SERIALIZER, nodeRecord2); SimpleProcessor from1to2 = new SimpleProcessor<>( Schedulers.createDefault().newSingleThreadDaemon("from1to2-thread"), "from1to2"); diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/HandshakeHandlersTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/HandshakeHandlersTest.java index fa790a99c..edfeb07d4 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/HandshakeHandlersTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/HandshakeHandlersTest.java @@ -34,12 +34,13 @@ import java.util.concurrent.TimeUnit; import java.util.function.Consumer; +import static org.ethereum.beacon.discovery.TestUtil.NODE_RECORD_FACTORY_NO_VERIFICATION; +import static org.ethereum.beacon.discovery.TestUtil.TEST_SERIALIZER; import static org.ethereum.beacon.discovery.pipeline.Field.BAD_PACKET; import static org.ethereum.beacon.discovery.pipeline.Field.MESSAGE; import static org.ethereum.beacon.discovery.pipeline.Field.PACKET_AUTH_HEADER_MESSAGE; import static org.ethereum.beacon.discovery.pipeline.Field.PACKET_MESSAGE; import static org.ethereum.beacon.discovery.pipeline.Field.SESSION; -import static org.ethereum.beacon.discovery.storage.NodeTableStorage.DEFAULT_SERIALIZER; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; @@ -62,7 +63,7 @@ public void authHandlerWithMessageRoundTripTest() throws Exception { NodeTableStorage nodeTableStorage1 = nodeTableStorageFactory.createTable( database1, - DEFAULT_SERIALIZER, + TEST_SERIALIZER, (oldSeq) -> nodeRecord1, () -> new ArrayList() { @@ -71,11 +72,11 @@ public void authHandlerWithMessageRoundTripTest() throws Exception { } }); NodeBucketStorage nodeBucketStorage1 = - nodeTableStorageFactory.createBucketStorage(database1, DEFAULT_SERIALIZER, nodeRecord1); + nodeTableStorageFactory.createBucketStorage(database1, TEST_SERIALIZER, nodeRecord1); NodeTableStorage nodeTableStorage2 = nodeTableStorageFactory.createTable( database2, - DEFAULT_SERIALIZER, + TEST_SERIALIZER, (oldSeq) -> nodeRecord2, () -> new ArrayList() { @@ -84,7 +85,7 @@ public void authHandlerWithMessageRoundTripTest() throws Exception { } }); NodeBucketStorage nodeBucketStorage2 = - nodeTableStorageFactory.createBucketStorage(database2, DEFAULT_SERIALIZER, nodeRecord2); + nodeTableStorageFactory.createBucketStorage(database2, TEST_SERIALIZER, nodeRecord2); // Node1 create AuthHeaderPacket final Packet[] outgoing1Packets = new Packet[2]; @@ -145,7 +146,8 @@ public void authHandlerWithMessageRoundTripTest() throws Exception { // Node2 handle AuthHeaderPacket and finish handshake AuthHeaderMessagePacketHandler authHeaderMessagePacketHandlerNode2 = - new AuthHeaderMessagePacketHandler(outgoingPipeline, taskScheduler); + new AuthHeaderMessagePacketHandler( + outgoingPipeline, taskScheduler, NODE_RECORD_FACTORY_NO_VERIFICATION); Envelope envelopeAt2From1 = new Envelope(); envelopeAt2From1.put(PACKET_AUTH_HEADER_MESSAGE, outgoing1Packets[0]); envelopeAt2From1.put(SESSION, nodeSessionAt2For1); @@ -170,7 +172,7 @@ public void authHandlerWithMessageRoundTripTest() throws Exception { assertNull(envelopeAt1From2WithMessage.get(BAD_PACKET)); assertNotNull(envelopeAt1From2WithMessage.get(MESSAGE)); - MessageHandler messageHandler = new MessageHandler(); + MessageHandler messageHandler = new MessageHandler(NODE_RECORD_FACTORY_NO_VERIFICATION); messageHandler.handle(envelopeAt1From2WithMessage); assert outgoing1PacketsSemaphore.tryAcquire(2, 1, TimeUnit.SECONDS); diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/NodeRecordTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/NodeRecordTest.java index 540aa0cfd..863fcb5cd 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/NodeRecordTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/NodeRecordTest.java @@ -3,14 +3,28 @@ import org.ethereum.beacon.discovery.enr.EnrScheme; import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.enr.NodeRecordFactory; +import org.javatuples.Pair; import org.junit.Test; +import org.web3j.crypto.ECKeyPair; +import tech.pegasys.artemis.util.bytes.Bytes4; +import tech.pegasys.artemis.util.bytes.Bytes96; import tech.pegasys.artemis.util.bytes.BytesValue; import tech.pegasys.artemis.util.uint.UInt64; import java.net.InetAddress; +import java.util.Random; +import java.util.concurrent.atomic.AtomicBoolean; +import static org.ethereum.beacon.discovery.TestUtil.SEED; +import static org.ethereum.beacon.discovery.enr.NodeRecord.FIELD_IP_V4; +import static org.ethereum.beacon.discovery.enr.NodeRecord.FIELD_PKEY_SECP256K1; +import static org.ethereum.beacon.discovery.enr.NodeRecord.FIELD_TCP_V4; +import static org.ethereum.beacon.discovery.enr.NodeRecord.FIELD_UDP_V4; +import static org.ethereum.beacon.util.Utils.extractBytesFromUnsignedBigInt; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; /** * ENR serialization/deserialization test @@ -63,4 +77,51 @@ public void testLocalhostV4() throws Exception { assertEquals(expectedPublicKey, nodeRecordV4Restored.get(NodeRecord.FIELD_PKEY_SECP256K1)); assertEquals(expectedSignature, nodeRecordV4Restored.getSignature()); } + + @Test + public void testSignature() throws Exception { + Random rnd = new Random(SEED); + byte[] privKey = new byte[32]; + rnd.nextBytes(privKey); + ECKeyPair keyPair = ECKeyPair.create(privKey); + Bytes4 localIp = Bytes4.wrap(InetAddress.getByName("127.0.0.1").getAddress()); + NodeRecord nodeRecord0 = + NodeRecordFactory.DEFAULT.createFromValues( + EnrScheme.V4, + UInt64.ZERO, + Bytes96.ZERO, + Pair.with(FIELD_IP_V4, localIp), + Pair.with(FIELD_TCP_V4, 30303), + Pair.with(FIELD_UDP_V4, 30303), + Pair.with( + FIELD_PKEY_SECP256K1, + BytesValue.wrap(extractBytesFromUnsignedBigInt(keyPair.getPublicKey())))); + BytesValue signature0 = Functions.sign(BytesValue.wrap(privKey), nodeRecord0.serialize(false)); + nodeRecord0.setSignature(signature0); + nodeRecord0.verify(); + NodeRecord nodeRecord1 = + NodeRecordFactory.DEFAULT.createFromValues( + EnrScheme.V4, + UInt64.valueOf(1), + Bytes96.ZERO, + Pair.with(FIELD_IP_V4, localIp), + Pair.with(FIELD_TCP_V4, 30303), + Pair.with(FIELD_UDP_V4, 30303), + Pair.with( + FIELD_PKEY_SECP256K1, + BytesValue.wrap(extractBytesFromUnsignedBigInt(keyPair.getPublicKey())))); + BytesValue signature1 = Functions.sign(BytesValue.wrap(privKey), nodeRecord1.serialize(false)); + nodeRecord1.setSignature(signature1); + nodeRecord1.verify(); + assertNotEquals(nodeRecord0.serialize(), nodeRecord1.serialize()); + assertNotEquals(signature0, signature1); + nodeRecord1.setSignature(nodeRecord0.getSignature()); + AtomicBoolean exceptionThrown = new AtomicBoolean(false); + try { + nodeRecord1.verify(); + } catch (AssertionError ex) { + exceptionThrown.set(true); + } + assertTrue(exceptionThrown.get()); + } } diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/TestUtil.java b/discovery/src/test/java/org/ethereum/beacon/discovery/TestUtil.java index 7f2236610..e1318c0bc 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/TestUtil.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/TestUtil.java @@ -1,8 +1,11 @@ package org.ethereum.beacon.discovery; +import org.ethereum.beacon.chain.storage.impl.SerializerFactory; import org.ethereum.beacon.discovery.enr.EnrScheme; import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.enr.NodeRecordFactory; +import org.ethereum.beacon.discovery.mock.EnrSchemeV4InterpreterMock; +import org.ethereum.beacon.discovery.storage.NodeSerializerFactory; import org.ethereum.beacon.util.Utils; import org.javatuples.Pair; import org.web3j.crypto.ECKeyPair; @@ -17,12 +20,15 @@ import java.util.Random; public class TestUtil { - private static final NodeRecordFactory NODE_RECORD_FACTORY = NodeRecordFactory.DEFAULT; - private static final int SEED = 123456789; + public static final NodeRecordFactory NODE_RECORD_FACTORY_NO_VERIFICATION = + new NodeRecordFactory(new EnrSchemeV4InterpreterMock()); // doesn't verify ECDSA signature + public static final SerializerFactory TEST_SERIALIZER = + new NodeSerializerFactory(NODE_RECORD_FACTORY_NO_VERIFICATION); + static final int SEED = 123456789; /** * Generates node on 127.0.0.1 with provided port. Node key is random, but always the same for the - * same port + * same port. Signature is not valid if verified. * * @return */ @@ -44,7 +50,7 @@ public static Pair generateNode(int port) { final BytesValue pubKey = BytesValue.wrap(Utils.extractBytesFromUnsignedBigInt(ecKeyPair.getPublicKey())); NodeRecord nodeRecord = - NODE_RECORD_FACTORY.createFromValues( + NODE_RECORD_FACTORY_NO_VERIFICATION.createFromValues( EnrScheme.V4, UInt64.valueOf(1), Bytes96.EMPTY, diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java b/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java index 6e0471575..4549aad79 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java @@ -5,6 +5,7 @@ import org.ethereum.beacon.discovery.DiscoveryManager; import org.ethereum.beacon.discovery.DiscoveryManagerImpl; import org.ethereum.beacon.discovery.enr.NodeRecord; +import org.ethereum.beacon.discovery.enr.NodeRecordFactory; import org.ethereum.beacon.discovery.network.NetworkParcel; import org.ethereum.beacon.discovery.pipeline.Envelope; import org.ethereum.beacon.discovery.pipeline.Field; @@ -39,6 +40,8 @@ import java.util.concurrent.CompletableFuture; +import static org.ethereum.beacon.discovery.TestUtil.NODE_RECORD_FACTORY_NO_VERIFICATION; + /** * Implementation of {@link DiscoveryManager} without network as an opposite to Netty network * implementation {@link org.ethereum.beacon.discovery.DiscoveryManagerImpl} Outgoing packets could @@ -52,6 +55,8 @@ public class DiscoveryManagerNoNetwork implements DiscoveryManager { private final Publisher incomingPackets; private final Pipeline incomingPipeline = new PipelineImpl(); private final Pipeline outgoingPipeline = new PipelineImpl(); + private final NodeRecordFactory nodeRecordFactory = + NODE_RECORD_FACTORY_NO_VERIFICATION; // no signature verification public DiscoveryManagerNoNetwork( NodeTable nodeTable, @@ -79,9 +84,10 @@ public DiscoveryManagerNoNetwork( .addHandler(new UnknownPacketTypeByStatus()) .addHandler(new NotExpectedIncomingPacketHandler()) .addHandler(new WhoAreYouPacketHandler(outgoingPipeline, taskScheduler)) - .addHandler(new AuthHeaderMessagePacketHandler(outgoingPipeline, taskScheduler)) + .addHandler( + new AuthHeaderMessagePacketHandler(outgoingPipeline, taskScheduler, nodeRecordFactory)) .addHandler(new MessagePacketHandler()) - .addHandler(new MessageHandler()) + .addHandler(new MessageHandler(nodeRecordFactory)) .addHandler(new BadPacketHandler()); outgoingPipeline .addHandler(new OutgoingParcelHandler(outgoingSink)) diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/mock/EnrSchemeV4InterpreterMock.java b/discovery/src/test/java/org/ethereum/beacon/discovery/mock/EnrSchemeV4InterpreterMock.java new file mode 100644 index 000000000..42e4ce101 --- /dev/null +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/mock/EnrSchemeV4InterpreterMock.java @@ -0,0 +1,11 @@ +package org.ethereum.beacon.discovery.mock; + +import org.ethereum.beacon.discovery.enr.EnrSchemeV4Interpreter; +import org.ethereum.beacon.discovery.enr.NodeRecord; + +public class EnrSchemeV4InterpreterMock extends EnrSchemeV4Interpreter { + @Override + public void verify(NodeRecord nodeRecord) { + // Don't verify ECDSA + } +} diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeBucketTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeBucketTest.java index 5b9211f1d..0d7853350 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeBucketTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeBucketTest.java @@ -4,9 +4,9 @@ import org.ethereum.beacon.discovery.Functions; import org.ethereum.beacon.discovery.NodeRecordInfo; import org.ethereum.beacon.discovery.NodeStatus; +import org.ethereum.beacon.discovery.TestUtil; import org.ethereum.beacon.discovery.enr.EnrScheme; import org.ethereum.beacon.discovery.enr.NodeRecord; -import org.ethereum.beacon.discovery.enr.NodeRecordFactory; import org.javatuples.Pair; import org.junit.Test; import tech.pegasys.artemis.util.bytes.Bytes4; @@ -20,13 +20,12 @@ import java.util.Random; import java.util.stream.IntStream; -import static org.ethereum.beacon.discovery.storage.NodeTableStorage.DEFAULT_SERIALIZER; +import static org.ethereum.beacon.discovery.TestUtil.TEST_SERIALIZER; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; public class NodeBucketTest { - private static final NodeRecordFactory NODE_RECORD_FACTORY = NodeRecordFactory.DEFAULT; private final Random rnd = new Random(); private NodeRecordInfo generateUniqueRecord() { @@ -34,7 +33,7 @@ private NodeRecordInfo generateUniqueRecord() { byte[] pkey = new byte[33]; rnd.nextBytes(pkey); NodeRecord nodeRecord = - NODE_RECORD_FACTORY.createFromValues( + TestUtil.NODE_RECORD_FACTORY_NO_VERIFICATION.createFromValues( EnrScheme.V4, UInt64.valueOf(1), Bytes96.EMPTY, @@ -103,7 +102,7 @@ public void testStorage() { Database database = Database.inMemoryDB(); NodeTableStorageFactoryImpl nodeTableStorageFactory = new NodeTableStorageFactoryImpl(); NodeBucketStorage nodeBucketStorage = - nodeTableStorageFactory.createBucketStorage(database, DEFAULT_SERIALIZER, initial.getNode()); + nodeTableStorageFactory.createBucketStorage(database, TEST_SERIALIZER, initial.getNode()); for (int i = 0; i < 20; ) { NodeRecordInfo nodeRecordInfo = generateUniqueRecord(); diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeTableTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeTableTest.java index 93c3fafc9..1358813c8 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeTableTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeTableTest.java @@ -5,7 +5,6 @@ import org.ethereum.beacon.discovery.NodeStatus; import org.ethereum.beacon.discovery.enr.EnrScheme; import org.ethereum.beacon.discovery.enr.NodeRecord; -import org.ethereum.beacon.discovery.enr.NodeRecordFactory; import org.javatuples.Pair; import org.junit.Test; import tech.pegasys.artemis.util.bytes.Bytes32; @@ -23,17 +22,16 @@ import java.util.Set; import java.util.function.Function; -import static org.ethereum.beacon.discovery.storage.NodeTableStorage.DEFAULT_SERIALIZER; +import static org.ethereum.beacon.discovery.TestUtil.NODE_RECORD_FACTORY_NO_VERIFICATION; +import static org.ethereum.beacon.discovery.TestUtil.TEST_SERIALIZER; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; public class NodeTableTest { - private static final NodeRecordFactory NODE_RECORD_FACTORY = NodeRecordFactory.DEFAULT; - private Function homeNodeSupplier = (oldSeq) -> { try { - return NODE_RECORD_FACTORY.createFromValues( + return NODE_RECORD_FACTORY_NO_VERIFICATION.createFromValues( EnrScheme.V4, UInt64.valueOf(1), Bytes96.EMPTY, @@ -60,13 +58,13 @@ public class NodeTableTest { public void testCreate() throws Exception { final String localhostEnr = "-IS4QHCYrYZbAKWCBRlAy5zzaDZXJBGkcnh4MHcBFZntXNFrdvJjX04jRzjzCBOonrkTfj499SZuOh8R33Ls8RRcy5wBgmlkgnY0gmlwhH8AAAGJc2VjcDI1NmsxoQPKY0yuDUmstAHYpMa2_oxVtw0RW_QAdpzBQA8yWM0xOIN1ZHCCdl8"; - NodeRecord nodeRecord = NODE_RECORD_FACTORY.fromBase64(localhostEnr); + NodeRecord nodeRecord = NODE_RECORD_FACTORY_NO_VERIFICATION.fromBase64(localhostEnr); NodeTableStorageFactoryImpl nodeTableStorageFactory = new NodeTableStorageFactoryImpl(); Database database = Database.inMemoryDB(); NodeTableStorage nodeTableStorage = nodeTableStorageFactory.createTable( database, - DEFAULT_SERIALIZER, + TEST_SERIALIZER, homeNodeSupplier, () -> { List nodes = new ArrayList<>(); @@ -88,13 +86,13 @@ public void testCreate() throws Exception { public void testFind() throws Exception { final String localhostEnr = "-IS4QHCYrYZbAKWCBRlAy5zzaDZXJBGkcnh4MHcBFZntXNFrdvJjX04jRzjzCBOonrkTfj499SZuOh8R33Ls8RRcy5wBgmlkgnY0gmlwhH8AAAGJc2VjcDI1NmsxoQPKY0yuDUmstAHYpMa2_oxVtw0RW_QAdpzBQA8yWM0xOIN1ZHCCdl8"; - NodeRecord localHostNode = NODE_RECORD_FACTORY.fromBase64(localhostEnr); + NodeRecord localHostNode = NODE_RECORD_FACTORY_NO_VERIFICATION.fromBase64(localhostEnr); NodeTableStorageFactoryImpl nodeTableStorageFactory = new NodeTableStorageFactoryImpl(); Database database = Database.inMemoryDB(); NodeTableStorage nodeTableStorage = nodeTableStorageFactory.createTable( database, - DEFAULT_SERIALIZER, + TEST_SERIALIZER, homeNodeSupplier, () -> { List nodes = new ArrayList<>(); @@ -104,7 +102,7 @@ public void testFind() throws Exception { // node is adjusted to be close to localhostEnr NodeRecord closestNode = - NODE_RECORD_FACTORY.createFromValues( + NODE_RECORD_FACTORY_NO_VERIFICATION.createFromValues( EnrScheme.V4, UInt64.valueOf(1), Bytes96.EMPTY, @@ -132,7 +130,7 @@ public void testFind() throws Exception { .get(NodeRecord.FIELD_PKEY_SECP256K1), closestNode.get(NodeRecord.FIELD_PKEY_SECP256K1)); NodeRecord farNode = - NODE_RECORD_FACTORY.createFromValues( + NODE_RECORD_FACTORY_NO_VERIFICATION.createFromValues( EnrScheme.V4, UInt64.valueOf(1), Bytes96.EMPTY, From cc59e3b45e6efe82c46a236a4d00f5a93d706ae8 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Thu, 31 Oct 2019 18:27:28 +0300 Subject: [PATCH 59/77] discovery: add packet and message community encoding tests --- .../beacon/discovery/enr/NodeRecord.java | 17 +++- .../discovery/enr/NodeRecordFactory.java | 17 ++-- .../discovery/message/DiscoveryV5Message.java | 29 +------ .../discovery/message/FindNodeMessage.java | 9 +++ .../discovery/message/NodesMessage.java | 16 +++- .../beacon/discovery/message/PingMessage.java | 14 +++- .../beacon/discovery/message/PongMessage.java | 13 +++ .../packet/AuthHeaderMessagePacket.java | 22 ++++- .../discovery/packet/MessagePacket.java | 13 ++- .../beacon/discovery/packet/Packet.java | 2 +- .../beacon/discovery/packet/RandomPacket.java | 21 +++-- .../discovery/packet/WhoAreYouPacket.java | 5 ++ .../community/MessageEncodingTest.java | 70 ++++++++++++++++ .../community/PacketEncodingTest.java | 81 +++++++++++++++++++ 14 files changed, 278 insertions(+), 51 deletions(-) create mode 100644 discovery/src/test/java/org/ethereum/beacon/discovery/community/MessageEncodingTest.java create mode 100644 discovery/src/test/java/org/ethereum/beacon/discovery/community/PacketEncodingTest.java diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecord.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecord.java index 8881663fb..750f64b6f 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecord.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecord.java @@ -137,11 +137,11 @@ public void verify() { enrSchemeInterpreter.verify(this); } - public BytesValue serialize() { - return serialize(true); + public RlpList asRlp() { + return asRlp(true); } - public BytesValue serialize(boolean withSignature) { + public RlpList asRlp(boolean withSignature) { assert getSeq() != null; // content = [seq, k, v, ...] // signature = sign(content) @@ -160,7 +160,16 @@ public BytesValue serialize(boolean withSignature) { values.add(RlpString.create(keyPair.getKey())); values.add(enrSchemeInterpreter.encode(keyPair.getKey(), keyPair.getValue())); } - byte[] bytes = RlpEncoder.encode(new RlpList(values)); + + return new RlpList(values); + } + + public BytesValue serialize() { + return serialize(true); + } + + public BytesValue serialize(boolean withSignature) { + byte[] bytes = RlpEncoder.encode(asRlp(withSignature)); assert bytes.length <= 300; return BytesValue.wrap(bytes); } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordFactory.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordFactory.java index 491f03507..1817b3c71 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordFactory.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordFactory.java @@ -58,21 +58,16 @@ public NodeRecord fromBytes(BytesValue bytes) { return fromBytes(bytes.extractArray()); } - public NodeRecord fromBytes(byte[] bytes) { - // record = [signature, seq, k, v, ...] - RlpList rlpList = (RlpList) RlpDecoder.decode(bytes).getValues().get(0); + public NodeRecord fromRlpList(RlpList rlpList) { List values = rlpList.getValues(); if (values.size() < 4) { throw new RuntimeException( - String.format( - "Unable to deserialize ENR with less than 4 fields, [%s]", BytesValue.wrap(bytes))); + String.format("Unable to deserialize ENR with less than 4 fields, [%s]", values)); } RlpString id = (RlpString) values.get(2); if (!"id".equals(new String(id.getBytes()))) { throw new RuntimeException( - String.format( - "Unable to deserialize ENR with no id field at 2-3 records, [%s]", - BytesValue.wrap(bytes))); + String.format("Unable to deserialize ENR with no id field at 2-3 records, [%s]", values)); } RlpString idVersion = (RlpString) values.get(3); @@ -98,4 +93,10 @@ public NodeRecord fromBytes(byte[] bytes) { BytesValue.wrap(((RlpString) values.get(0)).getBytes()), values.subList(4, values.size())); } + + public NodeRecord fromBytes(byte[] bytes) { + // record = [signature, seq, k, v, ...] + RlpList rlpList = (RlpList) RlpDecoder.decode(bytes).getValues().get(0); + return fromRlpList(rlpList); + } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/DiscoveryV5Message.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/DiscoveryV5Message.java index ec9df5fda..65e09d933 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/message/DiscoveryV5Message.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/DiscoveryV5Message.java @@ -6,12 +6,9 @@ import org.web3j.rlp.RlpList; import org.web3j.rlp.RlpString; import org.web3j.rlp.RlpType; -import tech.pegasys.artemis.util.bytes.Bytes8; import tech.pegasys.artemis.util.bytes.BytesValue; -import tech.pegasys.artemis.util.uint.UInt64; import java.util.List; -import java.util.stream.Collectors; public class DiscoveryV5Message implements DiscoveryMessage { private final BytesValue bytes; @@ -59,37 +56,19 @@ public V5Message create(NodeRecordFactory nodeRecordFactory) { switch (code) { case PING: { - return new PingMessage( - BytesValue.wrap(((RlpString) payload.get(0)).getBytes()), - UInt64.fromBytesBigEndian( - Bytes8.leftPad(BytesValue.wrap(((RlpString) payload.get(1)).getBytes())))); + return PingMessage.fromRlp(payload); } case PONG: { - return new PongMessage( - BytesValue.wrap(((RlpString) payload.get(0)).getBytes()), - UInt64.fromBytesBigEndian( - Bytes8.leftPad(BytesValue.wrap(((RlpString) payload.get(1)).getBytes()))), - BytesValue.wrap(((RlpString) payload.get(2)).getBytes()), - ((RlpString) payload.get(3)).asPositiveBigInteger().intValueExact()); + return PongMessage.fromRlp(payload); } case FINDNODE: { - return new FindNodeMessage( - BytesValue.wrap(((RlpString) payload.get(0)).getBytes()), - ((RlpString) payload.get(1)).asPositiveBigInteger().intValueExact()); + return FindNodeMessage.fromRlp(payload); } case NODES: { - RlpList nodeRecords = (RlpList) payload.get(2); - return new NodesMessage( - BytesValue.wrap(((RlpString) payload.get(0)).getBytes()), - ((RlpString) payload.get(1)).asPositiveBigInteger().intValueExact(), - () -> - nodeRecords.getValues().stream() - .map(rs -> nodeRecordFactory.fromBytes(((RlpString) rs).getBytes())) - .collect(Collectors.toList()), - nodeRecords.getValues().size()); + return NodesMessage.fromRlp(payload, nodeRecordFactory); } default: { diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/FindNodeMessage.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/FindNodeMessage.java index 7f2b0f71b..7c36db08f 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/message/FindNodeMessage.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/FindNodeMessage.java @@ -4,9 +4,12 @@ import org.web3j.rlp.RlpEncoder; import org.web3j.rlp.RlpList; import org.web3j.rlp.RlpString; +import org.web3j.rlp.RlpType; import tech.pegasys.artemis.util.bytes.Bytes1; import tech.pegasys.artemis.util.bytes.BytesValue; +import java.util.List; + /** * FINDNODE queries for nodes at the given logarithmic distance from the recipient's node ID. The * node IDs of all nodes in the response must have a shared prefix length of distance with the @@ -24,6 +27,12 @@ public FindNodeMessage(BytesValue requestId, Integer distance) { this.distance = distance; } + public static FindNodeMessage fromRlp(List rlpList) { + return new FindNodeMessage( + BytesValue.wrap(((RlpString) rlpList.get(0)).getBytes()), + ((RlpString) rlpList.get(1)).asPositiveBigInteger().intValueExact()); + } + @Override public BytesValue getRequestId() { return requestId; diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/NodesMessage.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/NodesMessage.java index 953039daa..faa003767 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/message/NodesMessage.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/NodesMessage.java @@ -2,9 +2,11 @@ import com.google.common.base.Objects; import org.ethereum.beacon.discovery.enr.NodeRecord; +import org.ethereum.beacon.discovery.enr.NodeRecordFactory; import org.web3j.rlp.RlpEncoder; import org.web3j.rlp.RlpList; import org.web3j.rlp.RlpString; +import org.web3j.rlp.RlpType; import tech.pegasys.artemis.util.bytes.Bytes1; import tech.pegasys.artemis.util.bytes.BytesValue; @@ -38,6 +40,18 @@ public NodesMessage( this.nodeRecordsSize = nodeRecordsSize; } + public static NodesMessage fromRlp(List rlpList, NodeRecordFactory nodeRecordFactory) { + RlpList nodeRecords = (RlpList) rlpList.get(2); + return new NodesMessage( + BytesValue.wrap(((RlpString) rlpList.get(0)).getBytes()), + ((RlpString) rlpList.get(1)).asPositiveBigInteger().intValueExact(), + () -> + nodeRecords.getValues().stream() + .map(rl -> nodeRecordFactory.fromRlpList((RlpList) rl)) + .collect(Collectors.toList()), + nodeRecords.getValues().size()); + } + @Override public BytesValue getRequestId() { return requestId; @@ -69,7 +83,7 @@ public BytesValue getBytes() { RlpString.create(total), new RlpList( getNodeRecords().stream() - .map(n -> RlpString.create(n.serialize().extractArray())) + .map(NodeRecord::asRlp) .collect(Collectors.toList())))))); } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/PingMessage.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/PingMessage.java index 601e9582d..13638b3d0 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/message/PingMessage.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/PingMessage.java @@ -4,10 +4,14 @@ import org.web3j.rlp.RlpEncoder; import org.web3j.rlp.RlpList; import org.web3j.rlp.RlpString; +import org.web3j.rlp.RlpType; import tech.pegasys.artemis.util.bytes.Bytes1; +import tech.pegasys.artemis.util.bytes.Bytes8; import tech.pegasys.artemis.util.bytes.BytesValue; import tech.pegasys.artemis.util.uint.UInt64; +import java.util.List; + /** * PING checks whether the recipient is alive and informs it about the sender's ENR sequence number. */ @@ -22,6 +26,13 @@ public PingMessage(BytesValue requestId, UInt64 enrSeq) { this.enrSeq = enrSeq; } + public static PingMessage fromRlp(List rlpList) { + return new PingMessage( + BytesValue.wrap(((RlpString) rlpList.get(0)).getBytes()), + UInt64.fromBytesBigEndian( + Bytes8.leftPad(BytesValue.wrap(((RlpString) rlpList.get(1)).getBytes())))); + } + @Override public BytesValue getRequestId() { return requestId; @@ -38,7 +49,8 @@ public BytesValue getBytes() { BytesValue.wrap( RlpEncoder.encode( new RlpList( - RlpString.create(requestId.extractArray()), RlpString.create(enrSeq.toBI()))))); + RlpString.create(requestId.extractArray()), + RlpString.create(enrSeq.toBI()))))); } @Override diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/PongMessage.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/PongMessage.java index d42dbd7d7..33c3cf4a7 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/message/PongMessage.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/PongMessage.java @@ -4,10 +4,14 @@ import org.web3j.rlp.RlpEncoder; import org.web3j.rlp.RlpList; import org.web3j.rlp.RlpString; +import org.web3j.rlp.RlpType; import tech.pegasys.artemis.util.bytes.Bytes1; +import tech.pegasys.artemis.util.bytes.Bytes8; import tech.pegasys.artemis.util.bytes.BytesValue; import tech.pegasys.artemis.util.uint.UInt64; +import java.util.List; + /** PONG is the reply to PING {@link PingMessage} */ public class PongMessage implements V5Message { // Unique request id @@ -27,6 +31,15 @@ public PongMessage( this.recipientPort = recipientPort; } + public static PongMessage fromRlp(List rlpList) { + return new PongMessage( + BytesValue.wrap(((RlpString) rlpList.get(0)).getBytes()), + UInt64.fromBytesBigEndian( + Bytes8.leftPad(BytesValue.wrap(((RlpString) rlpList.get(1)).getBytes()))), + BytesValue.wrap(((RlpString) rlpList.get(2)).getBytes()), + ((RlpString) rlpList.get(3)).asPositiveBigInteger().intValueExact()); + } + @Override public BytesValue getRequestId() { return requestId; diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/AuthHeaderMessagePacket.java b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/AuthHeaderMessagePacket.java index 94205ac3d..ceabbdf99 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/AuthHeaderMessagePacket.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/AuthHeaderMessagePacket.java @@ -52,6 +52,24 @@ public AuthHeaderMessagePacket(BytesValue bytes) { super(bytes); } + public static AuthHeaderMessagePacket create( + Bytes32 tag, + BytesValue authResponseCipherText, + BytesValue idNonce, + BytesValue ephemeralPubkey, + BytesValue authTag, + BytesValue messageCipherText) { + RlpList authHeaderRlp = + new RlpList( + RlpString.create(authTag.extractArray()), + RlpString.create(idNonce.extractArray()), + RlpString.create(AUTH_SCHEME_NAME.getBytes()), + RlpString.create(ephemeralPubkey.extractArray()), + RlpString.create(authResponseCipherText.extractArray())); + BytesValue authHeader = BytesValue.wrap(RlpEncoder.encode(authHeaderRlp)); + return new AuthHeaderMessagePacket(tag.concat(authHeader).concat(messageCipherText)); + } + public static AuthHeaderMessagePacket create( Bytes32 homeNodeId, Bytes32 destNodeId, @@ -63,7 +81,7 @@ public static AuthHeaderMessagePacket create( BytesValue authTag, BytesValue initiatorKey, DiscoveryMessage message) { - BytesValue tag = Packet.createTag(homeNodeId, destNodeId); + Bytes32 tag = Packet.createTag(homeNodeId, destNodeId); BytesValue idNonceSig = Functions.sign( staticNodeKey, @@ -89,7 +107,7 @@ public static AuthHeaderMessagePacket create( BytesValue authHeader = BytesValue.wrap(RlpEncoder.encode(authHeaderRlp)); BytesValue encryptedData = Functions.aesgcm_encrypt(initiatorKey, authTag, message.getBytes(), tag.concat(authHeader)); - return new AuthHeaderMessagePacket(tag.concat(authHeader).concat(encryptedData)); + return create(tag, authResponse, idNonce, ephemeralPubkey, authTag, encryptedData); } public void verify(BytesValue expectedIdNonce) { diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/MessagePacket.java b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/MessagePacket.java index 3377a9960..cfbd8e7bf 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/MessagePacket.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/MessagePacket.java @@ -26,18 +26,23 @@ public MessagePacket(BytesValue bytes) { super(bytes); } + public static MessagePacket create( + Bytes32 tag, BytesValue authTag, BytesValue messageCipherText) { + byte[] authTagBytesRlp = RlpEncoder.encode(RlpString.create(authTag.extractArray())); + BytesValue authTagEncoded = BytesValue.wrap(authTagBytesRlp); + return new MessagePacket(tag.concat(authTagEncoded).concat(messageCipherText)); + } + public static MessagePacket create( Bytes32 homeNodeId, Bytes32 destNodeId, BytesValue authTag, BytesValue initiatorKey, DiscoveryMessage message) { - BytesValue tag = Packet.createTag(homeNodeId, destNodeId); - byte[] authTagBytesRlp = RlpEncoder.encode(RlpString.create(authTag.extractArray())); - BytesValue authTagEncoded = BytesValue.wrap(authTagBytesRlp); + Bytes32 tag = Packet.createTag(homeNodeId, destNodeId); BytesValue encryptedData = Functions.aesgcm_encrypt(initiatorKey, authTag, message.getBytes(), tag); - return new MessagePacket(tag.concat(authTagEncoded).concat(encryptedData)); + return create(tag, authTag, encryptedData); } public void verify(BytesValue expectedAuthTag) {} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/Packet.java b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/Packet.java index a6a93320f..7c6952ebe 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/Packet.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/Packet.java @@ -6,7 +6,7 @@ import tech.pegasys.artemis.util.bytes.BytesValue; public interface Packet { - static BytesValue createTag(Bytes32 homeNodeId, Bytes32 destNodeId) { + static Bytes32 createTag(Bytes32 homeNodeId, Bytes32 destNodeId) { return Bytes32s.xor(Functions.hash(destNodeId), homeNodeId); } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/RandomPacket.java b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/RandomPacket.java index 4b1be696d..fc8bca348 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/RandomPacket.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/RandomPacket.java @@ -19,6 +19,7 @@ * random-data = at least 44 bytes of random data */ public class RandomPacket extends AbstractPacket { + public static final int MIN_RANDOM_BYTES = 44; private RandomPacketDecoded decoded = null; public RandomPacket(BytesValue bytes) { @@ -26,13 +27,23 @@ public RandomPacket(BytesValue bytes) { } public static RandomPacket create( - Bytes32 homeNodeId, Bytes32 destNodeId, BytesValue authTag, Random rnd) { - BytesValue tag = Packet.createTag(homeNodeId, destNodeId); + Bytes32 homeNodeId, Bytes32 destNodeId, BytesValue authTag, BytesValue randomBytes) { + Bytes32 tag = Packet.createTag(homeNodeId, destNodeId); + return create(tag, authTag, randomBytes); + } + + public static RandomPacket create(Bytes32 tag, BytesValue authTag, BytesValue randomBytes) { + assert randomBytes.size() >= MIN_RANDOM_BYTES; // At least 44 bytes, spec defined byte[] authTagRlp = RlpEncoder.encode(RlpString.create(authTag.extractArray())); BytesValue authTagEncoded = BytesValue.wrap(authTagRlp); - byte[] randomBytes = new byte[44]; - rnd.nextBytes(randomBytes); // at least 44 bytes of random data - return new RandomPacket(tag.concat(authTagEncoded).concat(BytesValue.wrap(randomBytes))); + return new RandomPacket(tag.concat(authTagEncoded).concat(randomBytes)); + } + + public static RandomPacket create( + Bytes32 homeNodeId, Bytes32 destNodeId, BytesValue authTag, Random rnd) { + byte[] randomBytes = new byte[MIN_RANDOM_BYTES]; + rnd.nextBytes(randomBytes); // at least 44 bytes of random data, spec defined + return create(homeNodeId, destNodeId, authTag, BytesValue.wrap(randomBytes)); } public Bytes32 getHomeNodeId(Bytes32 destNodeId) { diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/WhoAreYouPacket.java b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/WhoAreYouPacket.java index fcba30e31..1f1803cb4 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/WhoAreYouPacket.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/WhoAreYouPacket.java @@ -32,6 +32,11 @@ public WhoAreYouPacket(BytesValue bytes) { public static WhoAreYouPacket create( Bytes32 destNodeId, BytesValue authTag, Bytes32 idNonce, UInt64 enrSeq) { BytesValue magic = getStartMagic(destNodeId); + return create(magic, authTag, idNonce, enrSeq); + } + + public static WhoAreYouPacket create( + BytesValue magic, BytesValue authTag, Bytes32 idNonce, UInt64 enrSeq) { byte[] rlpListEncoded = RlpEncoder.encode( new RlpList( diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/community/MessageEncodingTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/community/MessageEncodingTest.java new file mode 100644 index 000000000..729bc5b58 --- /dev/null +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/community/MessageEncodingTest.java @@ -0,0 +1,70 @@ +package org.ethereum.beacon.discovery.community; + +import org.ethereum.beacon.discovery.enr.NodeRecord; +import org.ethereum.beacon.discovery.enr.NodeRecordFactory; +import org.ethereum.beacon.discovery.message.FindNodeMessage; +import org.ethereum.beacon.discovery.message.NodesMessage; +import org.ethereum.beacon.discovery.message.PingMessage; +import org.ethereum.beacon.discovery.message.PongMessage; +import org.junit.Ignore; +import org.junit.Test; +import tech.pegasys.artemis.util.bytes.BytesValue; +import tech.pegasys.artemis.util.uint.UInt64; + +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +public class MessageEncodingTest { + @Test + public void encodePing() { + PingMessage pingMessage = + new PingMessage(BytesValue.wrap(UInt64.valueOf(1).toBI().toByteArray()), UInt64.valueOf(1)); + assertEquals(BytesValue.fromHexString("0x01c20101"), pingMessage.getBytes()); + } + + @Test + public void encodePong() throws Exception { + PongMessage pongMessage = + new PongMessage( + BytesValue.wrap(UInt64.valueOf(1).toBI().toByteArray()), + UInt64.valueOf(1), + BytesValue.wrap(InetAddress.getByName("127.0.0.1").getAddress()), + 5000); + assertEquals(BytesValue.fromHexString("0x02ca0101847f000001821388"), pongMessage.getBytes()); + } + + @Test + public void encodeFindNode() { + FindNodeMessage findNodeMessage = + new FindNodeMessage(BytesValue.wrap(UInt64.valueOf(1).toBI().toByteArray()), 256); + assertEquals(BytesValue.fromHexString("0x03c401820100"), findNodeMessage.getBytes()); + } + + @Test + @Ignore("Until fix resolution. Rlp encoding is not the same") + public void encodeNodes() { + NodeRecordFactory nodeRecordFactory = NodeRecordFactory.DEFAULT; + NodesMessage nodesMessage = + new NodesMessage( + BytesValue.wrap(UInt64.valueOf(1).toBI().toByteArray()), + 2, + () -> { + List nodeRecords = new ArrayList<>(); + nodeRecords.add( + nodeRecordFactory.fromBase64( + "-HW4QBzimRxkmT18hMKaAL3IcZF1UcfTMPyi3Q1pxwZZbcZVRI8DC5infUAB_UauARLOJtYTxaagKoGmIjzQxO2qUygBgmlkgnY0iXNlY3AyNTZrMaEDymNMrg1JrLQB2KTGtv6MVbcNEVv0AHacwUAPMljNMTg")); + nodeRecords.add( + nodeRecordFactory.fromBase64( + "-HW4QNfxw543Ypf4HXKXdYxkyzfcxcO-6p9X986WldfVpnVTQX1xlTnWrktEWUbeTZnmgOuAY_KUhbVV1Ft98WoYUBMBgmlkgnY0iXNlY3AyNTZrMaEDDiy3QkHAxPyOgWbxp5oF1bDdlYE6dLCUUp8xfVw50jU")); + return nodeRecords; + }, + 2); + assertEquals( + BytesValue.fromHexString( + "0x04f8f80102b8f4f8f2b877f875b8401ce2991c64993d7c84c29a00bdc871917551c7d330fca2dd0d69c706596dc655448f030b98a77d4001fd46ae0112ce26d613c5a6a02a81a6223cd0c4edaa53280182696482763489736563703235366b31a103ca634cae0d49acb401d8a4c6b6fe8c55b70d115bf400769cc1400f3258cd3138b877f875b840d7f1c39e376297f81d7297758c64cb37dcc5c3beea9f57f7ce9695d7d5a67553417d719539d6ae4b445946de4d99e680eb8063f29485b555d45b7df16a1850130182696482763489736563703235366b31a1030e2cb74241c0c4fc8e8166f1a79a05d5b0dd95813a74b094529f317d5c39d235"), + nodesMessage.getBytes()); + } +} diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/community/PacketEncodingTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/community/PacketEncodingTest.java new file mode 100644 index 000000000..d081487f9 --- /dev/null +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/community/PacketEncodingTest.java @@ -0,0 +1,81 @@ +package org.ethereum.beacon.discovery.community; + +import org.ethereum.beacon.discovery.packet.AuthHeaderMessagePacket; +import org.ethereum.beacon.discovery.packet.MessagePacket; +import org.ethereum.beacon.discovery.packet.RandomPacket; +import org.ethereum.beacon.discovery.packet.WhoAreYouPacket; +import org.junit.Test; +import tech.pegasys.artemis.util.bytes.Bytes32; +import tech.pegasys.artemis.util.bytes.BytesValue; +import tech.pegasys.artemis.util.uint.UInt64; + +import static org.junit.Assert.assertEquals; + +public class PacketEncodingTest { + @Test + public void encodeRandomPacketTest() { + RandomPacket randomPacket = + RandomPacket.create( + Bytes32.fromHexString( + "0x0101010101010101010101010101010101010101010101010101010101010101"), + BytesValue.fromHexString("0x020202020202020202020202"), + BytesValue.fromHexString( + "0x0404040404040404040404040404040404040404040404040404040404040404040404040404040404040404")); + assertEquals( + BytesValue.fromHexString( + "0x01010101010101010101010101010101010101010101010101010101010101018c0202020202020202020202020404040404040404040404040404040404040404040404040404040404040404040404040404040404040404"), + randomPacket.getBytes()); + } + + @Test + public void encodeWhoAreYouTest() { + WhoAreYouPacket whoAreYouPacket = + WhoAreYouPacket.create( + BytesValue.fromHexString( + "0x0101010101010101010101010101010101010101010101010101010101010101"), + BytesValue.fromHexString("0x020202020202020202020202"), + Bytes32.fromHexString( + "0x0303030303030303030303030303030303030303030303030303030303030303"), + UInt64.valueOf(1)); + assertEquals( + BytesValue.fromHexString( + "0101010101010101010101010101010101010101010101010101010101010101ef8c020202020202020202020202a0030303030303030303030303030303030303030303030303030303030303030301"), + whoAreYouPacket.getBytes()); + } + + @Test + public void encodeAuthPacketTest() { + AuthHeaderMessagePacket authHeaderMessagePacket = + AuthHeaderMessagePacket.create( + Bytes32.fromHexString( + "0x93a7400fa0d6a694ebc24d5cf570f65d04215b6ac00757875e3f3a5f42107903"), + BytesValue.fromHexString( + "0x570fbf23885c674867ab00320294a41732891457969a0f14d11c995668858b2ad731aa7836888020e2ccc6e0e5776d0d4bc4439161798565a4159aa8620992fb51dcb275c4f755c8b8030c82918898f1ac387f606852"), + BytesValue.fromHexString( + "0xe551b1c44264ab92bc0b3c9b26293e1ba4fed9128f3c3645301e8e119f179c65"), + BytesValue.fromHexString( + "0xb35608c01ee67edff2cffa424b219940a81cf2fb9b66068b1cf96862a17d353e22524fbdcdebc609f85cbd58ebe7a872b01e24a3829b97dd5875e8ffbc4eea81"), + BytesValue.fromHexString("0x27b5af763c446acd2749fe8e"), + BytesValue.fromHexString("0xa5d12a2d94b8ccb3ba55558229867dc13bfa3648")); + + assertEquals( + BytesValue.fromHexString( + "0x93a7400fa0d6a694ebc24d5cf570f65d04215b6ac00757875e3f3a5f42107903f8cc8c27b5af763c446acd2749fe8ea0e551b1c44264ab92bc0b3c9b26293e1ba4fed9128f3c3645301e8e119f179c658367636db840b35608c01ee67edff2cffa424b219940a81cf2fb9b66068b1cf96862a17d353e22524fbdcdebc609f85cbd58ebe7a872b01e24a3829b97dd5875e8ffbc4eea81b856570fbf23885c674867ab00320294a41732891457969a0f14d11c995668858b2ad731aa7836888020e2ccc6e0e5776d0d4bc4439161798565a4159aa8620992fb51dcb275c4f755c8b8030c82918898f1ac387f606852a5d12a2d94b8ccb3ba55558229867dc13bfa3648"), + authHeaderMessagePacket.getBytes()); + } + + @Test + public void encodeMessagePacketTest() { + MessagePacket messagePacket = + MessagePacket.create( + Bytes32.fromHexString( + "0x93a7400fa0d6a694ebc24d5cf570f65d04215b6ac00757875e3f3a5f42107903"), + BytesValue.fromHexString("0x27b5af763c446acd2749fe8e"), + BytesValue.fromHexString("0xa5d12a2d94b8ccb3ba55558229867dc13bfa3648")); + + assertEquals( + BytesValue.fromHexString( + "0x93a7400fa0d6a694ebc24d5cf570f65d04215b6ac00757875e3f3a5f421079038c27b5af763c446acd2749fe8ea5d12a2d94b8ccb3ba55558229867dc13bfa3648"), + messagePacket.getBytes()); + } +} From 134d754bf5e01a994c68b7b075d4cec37898ea19 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Sun, 3 Nov 2019 18:31:37 +0300 Subject: [PATCH 60/77] discovery: fix encodeNodes test as result is clarified to be correct --- .../beacon/discovery/community/MessageEncodingTest.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/community/MessageEncodingTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/community/MessageEncodingTest.java index 729bc5b58..1413bb6ae 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/community/MessageEncodingTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/community/MessageEncodingTest.java @@ -6,7 +6,6 @@ import org.ethereum.beacon.discovery.message.NodesMessage; import org.ethereum.beacon.discovery.message.PingMessage; import org.ethereum.beacon.discovery.message.PongMessage; -import org.junit.Ignore; import org.junit.Test; import tech.pegasys.artemis.util.bytes.BytesValue; import tech.pegasys.artemis.util.uint.UInt64; @@ -44,7 +43,6 @@ public void encodeFindNode() { } @Test - @Ignore("Until fix resolution. Rlp encoding is not the same") public void encodeNodes() { NodeRecordFactory nodeRecordFactory = NodeRecordFactory.DEFAULT; NodesMessage nodesMessage = @@ -64,7 +62,7 @@ public void encodeNodes() { 2); assertEquals( BytesValue.fromHexString( - "0x04f8f80102b8f4f8f2b877f875b8401ce2991c64993d7c84c29a00bdc871917551c7d330fca2dd0d69c706596dc655448f030b98a77d4001fd46ae0112ce26d613c5a6a02a81a6223cd0c4edaa53280182696482763489736563703235366b31a103ca634cae0d49acb401d8a4c6b6fe8c55b70d115bf400769cc1400f3258cd3138b877f875b840d7f1c39e376297f81d7297758c64cb37dcc5c3beea9f57f7ce9695d7d5a67553417d719539d6ae4b445946de4d99e680eb8063f29485b555d45b7df16a1850130182696482763489736563703235366b31a1030e2cb74241c0c4fc8e8166f1a79a05d5b0dd95813a74b094529f317d5c39d235"), + "0x04f8f20102f8eef875b8401ce2991c64993d7c84c29a00bdc871917551c7d330fca2dd0d69c706596dc655448f030b98a77d4001fd46ae0112ce26d613c5a6a02a81a6223cd0c4edaa53280182696482763489736563703235366b31a103ca634cae0d49acb401d8a4c6b6fe8c55b70d115bf400769cc1400f3258cd3138f875b840d7f1c39e376297f81d7297758c64cb37dcc5c3beea9f57f7ce9695d7d5a67553417d719539d6ae4b445946de4d99e680eb8063f29485b555d45b7df16a1850130182696482763489736563703235366b31a1030e2cb74241c0c4fc8e8166f1a79a05d5b0dd95813a74b094529f317d5c39d235"), nodesMessage.getBytes()); } } From 599108f17a04b74dd999296023cf4aae1656fbba Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Sun, 3 Nov 2019 23:05:29 +0300 Subject: [PATCH 61/77] discovery: finish community tests + fixes --- .../ethereum/beacon/discovery/Functions.java | 45 ++++--- .../packet/AuthHeaderMessagePacket.java | 79 ++++++------ .../beacon/discovery/NodeRecordTest.java | 11 +- .../AuthHeaderMessagePacketTest.java | 108 +++++++++++++++++ .../discovery/community/CryptoTest.java | 114 ++++++++++++++++++ .../community/PacketEncodingTest.java | 29 +++-- 6 files changed, 315 insertions(+), 71 deletions(-) create mode 100644 discovery/src/test/java/org/ethereum/beacon/discovery/community/AuthHeaderMessagePacketTest.java create mode 100644 discovery/src/test/java/org/ethereum/beacon/discovery/community/CryptoTest.java diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java b/discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java index 868fc8e5e..c7ad1c7ee 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java @@ -34,7 +34,7 @@ public static Bytes32 hash(BytesValue value) { } /** - * Creates a signature of message `x` using the given key + * Creates a signature of message `x` using the given key. * * @param key private key * @param x message @@ -42,7 +42,7 @@ public static Bytes32 hash(BytesValue value) { */ public static BytesValue sign(BytesValue key, BytesValue x) { Sign.SignatureData signatureData = - Sign.signMessage(x.extractArray(), ECKeyPair.create(key.extractArray())); + Sign.signMessage(x.extractArray(), ECKeyPair.create(key.extractArray()), false); Bytes32 r = Bytes32.wrap(signatureData.getR()); Bytes32 s = Bytes32.wrap(signatureData.getS()); return r.concat(s); @@ -102,6 +102,20 @@ public static BytesValue aesgcm_decrypt( } } + /** Derives key agreement ECDH by multiplying private key by public */ + public static BytesValue deriveECDHKeyAgreement(BytesValue srcPrivKey, BytesValue destPubKey) { + ECDomainParameters CURVE = + new ECDomainParameters( + CURVE_PARAMS.getCurve(), CURVE_PARAMS.getG(), CURVE_PARAMS.getN(), CURVE_PARAMS.getH()); + + byte[] destPubPointBytes = new byte[destPubKey.size() + 1]; + destPubPointBytes[0] = 0x04; // default prefix + System.arraycopy(destPubKey.extractArray(), 0, destPubPointBytes, 1, destPubKey.size()); + ECPoint pudDestPoint = CURVE.getCurve().decodePoint(destPubPointBytes); + ECPoint mult = pudDestPoint.multiply(new BigInteger(1, srcPrivKey.extractArray())); + return BytesValue.wrap(mult.getEncoded(true)); + } + /** * The ephemeral key is used to perform Diffie-Hellman key agreement with B's static public key * and the session keys are derived from it using the HKDF key derivation function. @@ -121,27 +135,24 @@ public static HKDFKeys hkdf_expand( BytesValue srcPrivKey, BytesValue destPubKey, BytesValue idNonce) { - try { - ECDomainParameters CURVE = - new ECDomainParameters( - CURVE_PARAMS.getCurve(), - CURVE_PARAMS.getG(), - CURVE_PARAMS.getN(), - CURVE_PARAMS.getH()); - - byte[] destPubPointBytes = new byte[destPubKey.size() + 1]; - destPubPointBytes[0] = 0x04; // default prefix - System.arraycopy(destPubKey.extractArray(), 0, destPubPointBytes, 1, destPubKey.size()); - ECPoint pudDestPoint = CURVE.getCurve().decodePoint(destPubPointBytes); - ECPoint mult = pudDestPoint.multiply(new BigInteger(1, srcPrivKey.extractArray())); - byte[] keyAgreement = mult.getEncoded(true); + BytesValue keyAgreement = deriveECDHKeyAgreement(srcPrivKey, destPubKey); + return hkdf_expand(srcNodeId, destNodeId, keyAgreement, idNonce); + } + /** + * {@link #hkdf_expand(BytesValue, BytesValue, BytesValue, BytesValue, BytesValue)} but with + * keyAgreement already derived by {@link #deriveECDHKeyAgreement(BytesValue, BytesValue)} + */ + public static HKDFKeys hkdf_expand( + BytesValue srcNodeId, BytesValue destNodeId, BytesValue keyAgreement, BytesValue idNonce) { + try { BytesValue info = BytesValue.wrap("discovery v5 key agreement".getBytes()) .concat(srcNodeId) .concat(destNodeId); HKDFParameters hkdfParameters = - new HKDFParameters(keyAgreement, idNonce.extractArray(), info.extractArray()); + new HKDFParameters( + keyAgreement.extractArray(), idNonce.extractArray(), info.extractArray()); Digest digest = new SHA256Digest(); HKDFBytesGenerator hkdfBytesGenerator = new HKDFBytesGenerator(digest); hkdfBytesGenerator.init(hkdfParameters); diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/AuthHeaderMessagePacket.java b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/AuthHeaderMessagePacket.java index ceabbdf99..d7c2d0b3a 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/AuthHeaderMessagePacket.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/AuthHeaderMessagePacket.java @@ -15,6 +15,7 @@ import tech.pegasys.artemis.util.bytes.Bytes32s; import tech.pegasys.artemis.util.bytes.BytesValue; +import javax.annotation.Nullable; import java.math.BigInteger; /** @@ -42,7 +43,7 @@ */ public class AuthHeaderMessagePacket extends AbstractPacket { public static final String AUTH_SCHEME_NAME = "gcm"; - private static final BytesValue DISCOVERY_ID_NONCE = + public static final BytesValue DISCOVERY_ID_NONCE = BytesValue.wrap("discovery-id-nonce".getBytes()); private static final BytesValue ZERO_NONCE = BytesValue.wrap(new byte[12]); private EphemeralPubKeyDecoded decodedEphemeralPubKeyPt = null; @@ -53,21 +54,40 @@ public AuthHeaderMessagePacket(BytesValue bytes) { } public static AuthHeaderMessagePacket create( - Bytes32 tag, - BytesValue authResponseCipherText, - BytesValue idNonce, - BytesValue ephemeralPubkey, - BytesValue authTag, - BytesValue messageCipherText) { + Bytes32 tag, BytesValue authHeader, BytesValue messageCipherText) { + return new AuthHeaderMessagePacket(tag.concat(authHeader).concat(messageCipherText)); + } + + public static BytesValue signIdNonce( + BytesValue idNonce, BytesValue staticNodeKey, BytesValue ephemeralPubkey) { + return Functions.sign( + staticNodeKey, Functions.hash(DISCOVERY_ID_NONCE.concat(idNonce).concat(ephemeralPubkey))); + } + + public static byte[] createAuthMessagePt(BytesValue idNonceSig, @Nullable NodeRecord nodeRecord) { + return RlpEncoder.encode( + new RlpList( + RlpString.create(5), + RlpString.create(idNonceSig.extractArray()), + RlpString.create( + nodeRecord == null ? new byte[0] : nodeRecord.serialize().extractArray()))); + } + + public static BytesValue encodeAuthResponse(byte[] authResponsePt, BytesValue authResponseKey) { + return Functions.aesgcm_encrypt( + authResponseKey, ZERO_NONCE, BytesValue.wrap(authResponsePt), BytesValue.EMPTY); + } + + public static BytesValue encodeAuthHeaderRlp( + BytesValue authTag, BytesValue idNonce, BytesValue ephemeralPubkey, BytesValue authResponse) { RlpList authHeaderRlp = new RlpList( RlpString.create(authTag.extractArray()), RlpString.create(idNonce.extractArray()), RlpString.create(AUTH_SCHEME_NAME.getBytes()), RlpString.create(ephemeralPubkey.extractArray()), - RlpString.create(authResponseCipherText.extractArray())); - BytesValue authHeader = BytesValue.wrap(RlpEncoder.encode(authHeaderRlp)); - return new AuthHeaderMessagePacket(tag.concat(authHeader).concat(messageCipherText)); + RlpString.create(authResponse.extractArray())); + return BytesValue.wrap(RlpEncoder.encode(authHeaderRlp)); } public static AuthHeaderMessagePacket create( @@ -76,38 +96,20 @@ public static AuthHeaderMessagePacket create( BytesValue authResponseKey, BytesValue idNonce, BytesValue staticNodeKey, - NodeRecord nodeRecord, + @Nullable NodeRecord nodeRecord, BytesValue ephemeralPubkey, BytesValue authTag, BytesValue initiatorKey, DiscoveryMessage message) { Bytes32 tag = Packet.createTag(homeNodeId, destNodeId); - BytesValue idNonceSig = - Functions.sign( - staticNodeKey, - Functions.hash(DISCOVERY_ID_NONCE.concat(idNonce).concat(ephemeralPubkey))); + BytesValue idNonceSig = signIdNonce(idNonce, staticNodeKey, ephemeralPubkey); idNonceSig = idNonceSig.slice(1); // Remove recovery id - byte[] authResponsePt = - RlpEncoder.encode( - new RlpList( - RlpString.create(5), - RlpString.create(idNonceSig.extractArray()), - RlpString.create( - nodeRecord == null ? new byte[0] : nodeRecord.serialize().extractArray()))); - BytesValue authResponse = - Functions.aesgcm_encrypt( - authResponseKey, ZERO_NONCE, BytesValue.wrap(authResponsePt), BytesValue.EMPTY); - RlpList authHeaderRlp = - new RlpList( - RlpString.create(authTag.extractArray()), - RlpString.create(idNonce.extractArray()), - RlpString.create(AUTH_SCHEME_NAME.getBytes()), - RlpString.create(ephemeralPubkey.extractArray()), - RlpString.create(authResponse.extractArray())); - BytesValue authHeader = BytesValue.wrap(RlpEncoder.encode(authHeaderRlp)); + byte[] authResponsePt = createAuthMessagePt(idNonceSig, nodeRecord); + BytesValue authResponse = encodeAuthResponse(authResponsePt, authResponseKey); + BytesValue authHeader = encodeAuthHeaderRlp(authTag, idNonce, ephemeralPubkey, authResponse); BytesValue encryptedData = - Functions.aesgcm_encrypt(initiatorKey, authTag, message.getBytes(), tag.concat(authHeader)); - return create(tag, authResponse, idNonce, ephemeralPubkey, authTag, encryptedData); + Functions.aesgcm_encrypt(initiatorKey, authTag, message.getBytes(), tag); + return create(tag, authHeader, encryptedData); } public void verify(BytesValue expectedIdNonce) { @@ -170,7 +172,6 @@ public void decodeEphemeralPubKey() { blank.tag = Bytes32.wrap(getBytes().slice(0, 32), 0); Pair decodeRes = RlpUtil.decodeFirstList(getBytes().slice(32)); blank.messageEncrypted = decodeRes.getValue1(); - int authHeaderLength = getBytes().size() - 32 - decodeRes.getValue1().size(); RlpList authHeaderParts = (RlpList) decodeRes.getValue0().getValues().get(0); // [auth-tag, id-nonce, auth-scheme-name, ephemeral-pubkey, auth-response] blank.authTag = BytesValue.wrap(((RlpString) authHeaderParts.getValues().get(0)).getBytes()); @@ -181,7 +182,6 @@ public void decodeEphemeralPubKey() { BytesValue.wrap(((RlpString) authHeaderParts.getValues().get(3)).getBytes()); blank.authResponse = BytesValue.wrap(((RlpString) authHeaderParts.getValues().get(4)).getBytes()); - blank.authHeaderRaw = getBytes().slice(32, authHeaderLength); this.decodedEphemeralPubKeyPt = blank; } @@ -207,15 +207,13 @@ public void decodeMessage( byte[] nodeRecordBytes = ((RlpString) authResponsePtParts.getValues().get(2)).getBytes(); blank.nodeRecord = nodeRecordBytes.length == 0 ? null : nodeRecordFactory.fromBytes(nodeRecordBytes); - BytesValue messageAad = - decodedEphemeralPubKeyPt.tag.concat(decodedEphemeralPubKeyPt.authHeaderRaw); blank.message = new DiscoveryV5Message( Functions.aesgcm_decrypt( initiatorKey, decodedEphemeralPubKeyPt.authTag, decodedEphemeralPubKeyPt.messageEncrypted, - messageAad)); + decodedEphemeralPubKeyPt.tag)); this.decodedMessagePt = blank; } @@ -250,7 +248,6 @@ private static class EphemeralPubKeyDecoded { private BytesValue idNonce; private BytesValue ephemeralPubkey; private BytesValue authResponse; - private BytesValue authHeaderRaw; private BytesValue messageEncrypted; } diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/NodeRecordTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/NodeRecordTest.java index 863fcb5cd..cae47193e 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/NodeRecordTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/NodeRecordTest.java @@ -6,6 +6,7 @@ import org.javatuples.Pair; import org.junit.Test; import org.web3j.crypto.ECKeyPair; +import org.web3j.crypto.Hash; import tech.pegasys.artemis.util.bytes.Bytes4; import tech.pegasys.artemis.util.bytes.Bytes96; import tech.pegasys.artemis.util.bytes.BytesValue; @@ -96,7 +97,10 @@ public void testSignature() throws Exception { Pair.with( FIELD_PKEY_SECP256K1, BytesValue.wrap(extractBytesFromUnsignedBigInt(keyPair.getPublicKey())))); - BytesValue signature0 = Functions.sign(BytesValue.wrap(privKey), nodeRecord0.serialize(false)); + BytesValue signature0 = + Functions.sign( + BytesValue.wrap(privKey), + BytesValue.wrap(Hash.sha3(nodeRecord0.serialize(false).extractArray()))); nodeRecord0.setSignature(signature0); nodeRecord0.verify(); NodeRecord nodeRecord1 = @@ -110,7 +114,10 @@ public void testSignature() throws Exception { Pair.with( FIELD_PKEY_SECP256K1, BytesValue.wrap(extractBytesFromUnsignedBigInt(keyPair.getPublicKey())))); - BytesValue signature1 = Functions.sign(BytesValue.wrap(privKey), nodeRecord1.serialize(false)); + BytesValue signature1 = + Functions.sign( + BytesValue.wrap(privKey), + BytesValue.wrap(Hash.sha3(nodeRecord1.serialize(false).extractArray()))); nodeRecord1.setSignature(signature1); nodeRecord1.verify(); assertNotEquals(nodeRecord0.serialize(), nodeRecord1.serialize()); diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/community/AuthHeaderMessagePacketTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/community/AuthHeaderMessagePacketTest.java new file mode 100644 index 000000000..d72688cbc --- /dev/null +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/community/AuthHeaderMessagePacketTest.java @@ -0,0 +1,108 @@ +package org.ethereum.beacon.discovery.community; + +import org.ethereum.beacon.discovery.Functions; +import org.ethereum.beacon.discovery.packet.AuthHeaderMessagePacket; +import org.junit.Test; +import tech.pegasys.artemis.util.bytes.Bytes32; +import tech.pegasys.artemis.util.bytes.BytesValue; + +import static org.junit.Assert.assertEquals; + +/** Tests {@link AuthHeaderMessagePacket} packet creation routines */ +public class AuthHeaderMessagePacketTest { + /** + * This first section entails signature generation, and adding any ENR into `auth-pt`. In this + * example, there is no ENR sent. This tests the signature generation and correct RLP encoding of + * the `auth-pt` before encryption. + */ + @Test + public void testAuthPtGeneration() { + BytesValue secretKey = + BytesValue.fromHexString( + "0x7e8107fe766b6d357205280acf65c24275129ca9e44c0fd00144ca50024a1ce7"); + BytesValue idNonce = + BytesValue.fromHexString( + "0xe551b1c44264ab92bc0b3c9b26293e1ba4fed9128f3c3645301e8e119f179c65"); + BytesValue ephemeralPubkey = + BytesValue.fromHexString( + "0xb35608c01ee67edff2cffa424b219940a81cf2fb9b66068b1cf96862a17d353e22524fbdcdebc609f85cbd58ebe7a872b01e24a3829b97dd5875e8ffbc4eea81"); + // enr: [] + BytesValue expectedAuthPt = + BytesValue.fromHexString( + "0xf84405b840f753ac31b017536bacd0d0238a1f849e741aef03b7ad5db1d4e64d7aa80689931f21e590edcf80ee32bb2f30707fec88fb62ea8fbcd65b9272e9a0175fea976b80"); + assertEquals( + expectedAuthPt, + BytesValue.wrap( + AuthHeaderMessagePacket.createAuthMessagePt( + AuthHeaderMessagePacket.signIdNonce(idNonce, secretKey, ephemeralPubkey), null))); + } + + /** + * The `auth-pt` must then be encrypted with AES-GCM. The auth-header uses a 12-byte 0 nonce with + * no authenticated data. + */ + @Test + public void testEncryptAuthMessagePt() { + BytesValue authRespKey = BytesValue.fromHexString("0x8c7caa563cebc5c06bb15fc1a2d426c3"); + BytesValue authPt = + BytesValue.fromHexString( + "0xf84405b840f753ac31b017536bacd0d0238a1f849e741aef03b7ad5db1d4e64d7aa80689931f21e590edcf80ee32bb2f30707fec88fb62ea8fbcd65b9272e9a0175fea976bc0"); + + BytesValue expectedAuthRespCiphertext = + BytesValue.fromHexString( + "0x570fbf23885c674867ab00320294a41732891457969a0f14d11c995668858b2ad731aa7836888020e2ccc6e0e5776d0d4bc4439161798565a4159aa8620992fb51dcb275c4f755c8b8030c82918898f1ac387f606852"); + assertEquals( + expectedAuthRespCiphertext, + AuthHeaderMessagePacket.encodeAuthResponse(authPt.extractArray(), authRespKey)); + } + + /** + * An authentication header is built. This test vector demonstrates the correct RLP-encoding of + * the authentication header with the above inputs. + */ + @Test + public void testAuthHeaderGeneration() { + BytesValue authTag = BytesValue.fromHexString("0x27b5af763c446acd2749fe8e"); + BytesValue idNonce = + BytesValue.fromHexString( + "0xe551b1c44264ab92bc0b3c9b26293e1ba4fed9128f3c3645301e8e119f179c65"); + BytesValue ephemeralPubkey = + BytesValue.fromHexString( + "0xb35608c01ee67edff2cffa424b219940a81cf2fb9b66068b1cf96862a17d353e22524fbdcdebc609f85cbd58ebe7a872b01e24a3829b97dd5875e8ffbc4eea81"); + BytesValue authRespCiphertext = + BytesValue.fromHexString( + "0x570fbf23885c674867ab00320294a41732891457969a0f14d11c995668858b2ad731aa7836888020e2ccc6e0e5776d0d4bc4439161798565a4159aa8620992fb51dcb275c4f755c8b8030c82918898f1ac387f606852"); + + BytesValue expectedAuthHeaderRlp = + BytesValue.fromHexString( + "0xf8cc8c27b5af763c446acd2749fe8ea0e551b1c44264ab92bc0b3c9b26293e1ba4fed9128f3c3645301e8e119f179c658367636db840b35608c01ee67edff2cffa424b219940a81cf2fb9b66068b1cf96862a17d353e22524fbdcdebc609f85cbd58ebe7a872b01e24a3829b97dd5875e8ffbc4eea81b856570fbf23885c674867ab00320294a41732891457969a0f14d11c995668858b2ad731aa7836888020e2ccc6e0e5776d0d4bc4439161798565a4159aa8620992fb51dcb275c4f755c8b8030c82918898f1ac387f606852"); + assertEquals( + expectedAuthHeaderRlp, + AuthHeaderMessagePacket.encodeAuthHeaderRlp( + authTag, idNonce, ephemeralPubkey, authRespCiphertext)); + } + + /** + * This combines the previously generated authentication header with encryption of the protocol + * message, providing the final rlp-encoded message with an authentication header. + */ + @Test + public void testEncodeMessage() { + Bytes32 tag = + Bytes32.fromHexString("0x93a7400fa0d6a694ebc24d5cf570f65d04215b6ac00757875e3f3a5f42107903"); + BytesValue authHeaderRlp = + BytesValue.fromHexString( + "0xf8cc8c27b5af763c446acd2749fe8ea0e551b1c44264ab92bc0b3c9b26293e1ba4fed9128f3c3645301e8e119f179c658367636db840b35608c01ee67edff2cffa424b219940a81cf2fb9b66068b1cf96862a17d353e22524fbdcdebc609f85cbd58ebe7a872b01e24a3829b97dd5875e8ffbc4eea81b856570fbf23885c674867ab00320294a41732891457969a0f14d11c995668858b2ad731aa7836888020e2ccc6e0e5776d0d4bc4439161798565a4159aa8620992fb51dcb275c4f755c8b8030c82918898f1ac387f606852"); + BytesValue encryptionKey = BytesValue.fromHexString("0x9f2d77db7004bf8a1a85107ac686990b"); + BytesValue messagePlaintext = BytesValue.fromHexString("0x01c20101"); + BytesValue authTag = BytesValue.fromHexString("0x27b5af763c446acd2749fe8e"); + + BytesValue expectedAuthMessageRlp = + BytesValue.fromHexString( + "0x93a7400fa0d6a694ebc24d5cf570f65d04215b6ac00757875e3f3a5f42107903f8cc8c27b5af763c446acd2749fe8ea0e551b1c44264ab92bc0b3c9b26293e1ba4fed9128f3c3645301e8e119f179c658367636db840b35608c01ee67edff2cffa424b219940a81cf2fb9b66068b1cf96862a17d353e22524fbdcdebc609f85cbd58ebe7a872b01e24a3829b97dd5875e8ffbc4eea81b856570fbf23885c674867ab00320294a41732891457969a0f14d11c995668858b2ad731aa7836888020e2ccc6e0e5776d0d4bc4439161798565a4159aa8620992fb51dcb275c4f755c8b8030c82918898f1ac387f606852a5d12a2d94b8ccb3ba55558229867dc13bfa3648"); + BytesValue encryptedData = + Functions.aesgcm_encrypt(encryptionKey, authTag, messagePlaintext, tag); + BytesValue authHeaderMessagePacket = tag.concat(authHeaderRlp).concat(encryptedData); + assertEquals(expectedAuthMessageRlp, authHeaderMessagePacket); + } +} diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/community/CryptoTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/community/CryptoTest.java new file mode 100644 index 000000000..1e85327cd --- /dev/null +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/community/CryptoTest.java @@ -0,0 +1,114 @@ +package org.ethereum.beacon.discovery.community; + +import org.ethereum.beacon.discovery.Functions; +import org.ethereum.beacon.discovery.packet.AuthHeaderMessagePacket; +import org.junit.Test; +import tech.pegasys.artemis.util.bytes.BytesValue; + +import static org.junit.Assert.assertEquals; + +/** Tests crypto functions */ +public class CryptoTest { + + /** + * The ECDH function takes the elliptic-curve scalar multiplication of a public key and a private + * key. The wire protocol describes this process. + * + *

The input public key is an uncompressed secp256k1 key (64 bytes) and the private key is a + * raw secp256k1 private key (32 bytes). + */ + @Test + public void testECDHFunction() { + BytesValue publicKey = + BytesValue.fromHexString( + "0x9961e4c2356d61bedb83052c115d311acb3a96f5777296dcf297351130266231503061ac4aaee666073d7e5bc2c80c3f5c5b500c1cb5fd0a76abbb6b675ad157"); + BytesValue secretKey = + BytesValue.fromHexString( + "0xfb757dc581730490a1d7a00deea65e9b1936924caaea8f44d476014856b68736"); + + BytesValue expectedSharedSecret = + BytesValue.fromHexString( + "0x033b11a2a1f214567e1537ce5e509ffd9b21373247f2a3ff6841f4976f53165e7e"); + assertEquals(expectedSharedSecret, Functions.deriveECDHKeyAgreement(secretKey, publicKey)); + } + + /** + * This test vector takes a secret key (as calculated from the previous test vector) along with + * two node id's and an `id-nonce`. This demonstrates the HKDF-EXPAND and HKDF-EXTRACT functions + * using the added key-agreement string as described in the wire specification. + * + *

Given a secret key (calculated from ECDH above) two `node-id`s (required to build the `info` + * as described in the specification) and the `id-nonce` (required for the HKDF-EXTRACT function), + * this should produce an `initiator-key`, `recipient-key` and an `auth-resp-key`. + */ + @Test + public void testHKDFExpand() { + BytesValue secretKey = + BytesValue.fromHexString( + "0x02a77e3aa0c144ae7c0a3af73692b7d6e5b7a2fdc0eda16e8d5e6cb0d08e88dd04"); + BytesValue nodeIdA = + BytesValue.fromHexString( + "0xa448f24c6d18e575453db13171562b71999873db5b286df957af199ec94617f7"); + BytesValue nodeIdB = + BytesValue.fromHexString( + "0x885bba8dfeddd49855459df852ad5b63d13a3fae593f3f9fa7e317fd43651409"); + BytesValue idNonce = + BytesValue.fromHexString( + "0x0101010101010101010101010101010101010101010101010101010101010101"); + + BytesValue expectedInitiatorKey = + BytesValue.fromHexString("0x238d8b50e4363cf603a48c6cc3542967"); + BytesValue expectedRecipientKey = + BytesValue.fromHexString("0xbebc0183484f7e7ca2ac32e3d72c8891"); + BytesValue expectedAuthResponseKey = + BytesValue.fromHexString("0xe987ad9e414d5b4f9bfe4ff1e52f2fae"); + Functions.HKDFKeys keys = Functions.hkdf_expand(nodeIdA, nodeIdB, secretKey, idNonce); + assertEquals(expectedInitiatorKey, keys.getInitiatorKey()); + assertEquals(expectedRecipientKey, keys.getRecipientKey()); + assertEquals(expectedAuthResponseKey, keys.getAuthResponseKey()); + } + + /** + * Nonce signatures should prefix the string `discovery-id-nonce` and post-fix the ephemeral key + * before taking the `sha256` hash of the `id-nonce`. + * + *

See {@link org.ethereum.beacon.discovery.packet.AuthHeaderMessagePacket}, idNonceSig is a + * part of this packet + */ + @Test + public void testIdNonceSigning() { + BytesValue idNonce = + BytesValue.fromHexString( + "0x02a77e3aa0c144ae7c0a3af73692b7d6e5b7a2fdc0eda16e8d5e6cb0d08e88dd04"); + BytesValue ephemeralKey = + BytesValue.fromHexString( + "0x9961e4c2356d61bedb83052c115d311acb3a96f5777296dcf297351130266231503061ac4aaee666073d7e5bc2c80c3f5c5b500c1cb5fd0a76abbb6b675ad157"); + BytesValue localSecretKey = + BytesValue.fromHexString( + "0xfb757dc581730490a1d7a00deea65e9b1936924caaea8f44d476014856b68736"); + + BytesValue expectedIdNonceSig = + BytesValue.fromHexString( + "0xcf2bf743fc2273709bbc5117fd72775b0661ce1b6e9dffa01f45e2307fb138b90da16364ee7ae1705b938f6648d7725d35fe7e3f200e0ea022c1360b9b2e7385"); + assertEquals( + expectedIdNonceSig, + AuthHeaderMessagePacket.signIdNonce(idNonce, localSecretKey, ephemeralKey)); + } + + /** + * This test vector demonstrates the `AES_GCM` encryption/decryption used in the wire protocol. + */ + @Test + public void testAESGCM() { + BytesValue encryptionKey = BytesValue.fromHexString("0x9f2d77db7004bf8a1a85107ac686990b"); + BytesValue nonce = BytesValue.fromHexString("0x27b5af763c446acd2749fe8e"); + BytesValue pt = BytesValue.fromHexString("0x01c20101"); + BytesValue ad = + BytesValue.fromHexString( + "0x93a7400fa0d6a694ebc24d5cf570f65d04215b6ac00757875e3f3a5f42107903"); + + BytesValue expectedMessageCiphertext = + BytesValue.fromHexString("a5d12a2d94b8ccb3ba55558229867dc13bfa3648"); + assertEquals(expectedMessageCiphertext, Functions.aesgcm_encrypt(encryptionKey, nonce, pt, ad)); + } +} diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/community/PacketEncodingTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/community/PacketEncodingTest.java index d081487f9..d16928298 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/community/PacketEncodingTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/community/PacketEncodingTest.java @@ -45,18 +45,25 @@ public void encodeWhoAreYouTest() { @Test public void encodeAuthPacketTest() { + Bytes32 tag = + Bytes32.fromHexString("0x93a7400fa0d6a694ebc24d5cf570f65d04215b6ac00757875e3f3a5f42107903"); + BytesValue authTag = BytesValue.fromHexString("0x27b5af763c446acd2749fe8e"); + BytesValue idNonce = + BytesValue.fromHexString( + "0xe551b1c44264ab92bc0b3c9b26293e1ba4fed9128f3c3645301e8e119f179c65"); + BytesValue ephemeralPubkey = + BytesValue.fromHexString( + "0xb35608c01ee67edff2cffa424b219940a81cf2fb9b66068b1cf96862a17d353e22524fbdcdebc609f85cbd58ebe7a872b01e24a3829b97dd5875e8ffbc4eea81"); + BytesValue authRespCiphertext = + BytesValue.fromHexString( + "0x570fbf23885c674867ab00320294a41732891457969a0f14d11c995668858b2ad731aa7836888020e2ccc6e0e5776d0d4bc4439161798565a4159aa8620992fb51dcb275c4f755c8b8030c82918898f1ac387f606852"); + BytesValue messageCiphertext = + BytesValue.fromHexString("0xa5d12a2d94b8ccb3ba55558229867dc13bfa3648"); + BytesValue authHeader = + AuthHeaderMessagePacket.encodeAuthHeaderRlp( + authTag, idNonce, ephemeralPubkey, authRespCiphertext); AuthHeaderMessagePacket authHeaderMessagePacket = - AuthHeaderMessagePacket.create( - Bytes32.fromHexString( - "0x93a7400fa0d6a694ebc24d5cf570f65d04215b6ac00757875e3f3a5f42107903"), - BytesValue.fromHexString( - "0x570fbf23885c674867ab00320294a41732891457969a0f14d11c995668858b2ad731aa7836888020e2ccc6e0e5776d0d4bc4439161798565a4159aa8620992fb51dcb275c4f755c8b8030c82918898f1ac387f606852"), - BytesValue.fromHexString( - "0xe551b1c44264ab92bc0b3c9b26293e1ba4fed9128f3c3645301e8e119f179c65"), - BytesValue.fromHexString( - "0xb35608c01ee67edff2cffa424b219940a81cf2fb9b66068b1cf96862a17d353e22524fbdcdebc609f85cbd58ebe7a872b01e24a3829b97dd5875e8ffbc4eea81"), - BytesValue.fromHexString("0x27b5af763c446acd2749fe8e"), - BytesValue.fromHexString("0xa5d12a2d94b8ccb3ba55558229867dc13bfa3648")); + AuthHeaderMessagePacket.create(tag, authHeader, messageCiphertext); assertEquals( BytesValue.fromHexString( From 66fcb7f9e0854d05967adfefaad92f42197c95ae Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Tue, 5 Nov 2019 11:44:18 +0300 Subject: [PATCH 62/77] discovery: update idNonce signing test with correct community vector --- .../org/ethereum/beacon/discovery/community/CryptoTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/community/CryptoTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/community/CryptoTest.java index 1e85327cd..2434ee9b4 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/community/CryptoTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/community/CryptoTest.java @@ -79,7 +79,7 @@ public void testHKDFExpand() { public void testIdNonceSigning() { BytesValue idNonce = BytesValue.fromHexString( - "0x02a77e3aa0c144ae7c0a3af73692b7d6e5b7a2fdc0eda16e8d5e6cb0d08e88dd04"); + "0xa77e3aa0c144ae7c0a3af73692b7d6e5b7a2fdc0eda16e8d5e6cb0d08e88dd04"); BytesValue ephemeralKey = BytesValue.fromHexString( "0x9961e4c2356d61bedb83052c115d311acb3a96f5777296dcf297351130266231503061ac4aaee666073d7e5bc2c80c3f5c5b500c1cb5fd0a76abbb6b675ad157"); @@ -89,7 +89,7 @@ public void testIdNonceSigning() { BytesValue expectedIdNonceSig = BytesValue.fromHexString( - "0xcf2bf743fc2273709bbc5117fd72775b0661ce1b6e9dffa01f45e2307fb138b90da16364ee7ae1705b938f6648d7725d35fe7e3f200e0ea022c1360b9b2e7385"); + "0xc5036e702a79902ad8aa147dabfe3958b523fd6fa36cc78e2889b912d682d8d35fdea142e141f690736d86f50b39746ba2d2fc510b46f82ee08f08fd55d133a4"); assertEquals( expectedIdNonceSig, AuthHeaderMessagePacket.signIdNonce(idNonce, localSecretKey, ephemeralKey)); From 2eb78e2f23708bfae6300283875eb06fbd92730f Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Tue, 5 Nov 2019 11:53:43 +0300 Subject: [PATCH 63/77] discovery: fix node encoding in AuthHeaderMessagePacket and tests --- .../beacon/discovery/packet/AuthHeaderMessagePacket.java | 9 +++++---- .../discovery/community/AuthHeaderMessagePacketTest.java | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/AuthHeaderMessagePacket.java b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/AuthHeaderMessagePacket.java index d7c2d0b3a..ca4901c0f 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/AuthHeaderMessagePacket.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/AuthHeaderMessagePacket.java @@ -69,8 +69,7 @@ public static byte[] createAuthMessagePt(BytesValue idNonceSig, @Nullable NodeRe new RlpList( RlpString.create(5), RlpString.create(idNonceSig.extractArray()), - RlpString.create( - nodeRecord == null ? new byte[0] : nodeRecord.serialize().extractArray()))); + nodeRecord == null ? new RlpList() : nodeRecord.asRlp())); } public static BytesValue encodeAuthResponse(byte[] authResponsePt, BytesValue authResponseKey) { @@ -204,9 +203,11 @@ public void decodeMessage( .equals(((RlpString) authResponsePtParts.getValues().get(0)).asPositiveBigInteger()); blank.idNonceSig = BytesValue.wrap(((RlpString) authResponsePtParts.getValues().get(1)).getBytes()); - byte[] nodeRecordBytes = ((RlpString) authResponsePtParts.getValues().get(2)).getBytes(); + RlpList nodeRecordDataList = ((RlpList) authResponsePtParts.getValues().get(2)); blank.nodeRecord = - nodeRecordBytes.length == 0 ? null : nodeRecordFactory.fromBytes(nodeRecordBytes); + nodeRecordDataList.getValues().isEmpty() + ? null + : nodeRecordFactory.fromRlpList(nodeRecordDataList); blank.message = new DiscoveryV5Message( Functions.aesgcm_decrypt( diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/community/AuthHeaderMessagePacketTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/community/AuthHeaderMessagePacketTest.java index d72688cbc..c93b86c03 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/community/AuthHeaderMessagePacketTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/community/AuthHeaderMessagePacketTest.java @@ -29,7 +29,7 @@ public void testAuthPtGeneration() { // enr: [] BytesValue expectedAuthPt = BytesValue.fromHexString( - "0xf84405b840f753ac31b017536bacd0d0238a1f849e741aef03b7ad5db1d4e64d7aa80689931f21e590edcf80ee32bb2f30707fec88fb62ea8fbcd65b9272e9a0175fea976b80"); + "0xf84405b840f753ac31b017536bacd0d0238a1f849e741aef03b7ad5db1d4e64d7aa80689931f21e590edcf80ee32bb2f30707fec88fb62ea8fbcd65b9272e9a0175fea976bc0"); assertEquals( expectedAuthPt, BytesValue.wrap( From 524833ed5255c3f10fe2032462541ae64d72b717 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Mon, 11 Nov 2019 14:44:56 +0300 Subject: [PATCH 64/77] discovery: add sign to the enr scheme interface as it's a part of scheme --- .../discovery/enr/EnrSchemeInterpreter.java | 3 ++ .../discovery/enr/EnrSchemeV4Interpreter.java | 31 ++++++++++++++++--- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrSchemeInterpreter.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrSchemeInterpreter.java index 2b772e8fd..cf9750f5f 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrSchemeInterpreter.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrSchemeInterpreter.java @@ -7,6 +7,9 @@ public interface EnrSchemeInterpreter { /** Returns supported scheme */ EnrScheme getScheme(); + /* Signs nodeRecord, modifying it */ + void sign(NodeRecord nodeRecord, Object signOptions); + /** Verifies that `nodeRecord` is of scheme implementation */ default void verify(NodeRecord nodeRecord) { if (!nodeRecord.getIdentityScheme().equals(getScheme())) { diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrSchemeV4Interpreter.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrSchemeV4Interpreter.java index e8126b694..ed57c4401 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrSchemeV4Interpreter.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrSchemeV4Interpreter.java @@ -1,7 +1,8 @@ package org.ethereum.beacon.discovery.enr; +import org.bouncycastle.math.ec.ECPoint; import org.bouncycastle.util.Arrays; -import org.ethereum.beacon.crypto.Hashes; +import org.ethereum.beacon.discovery.Functions; import org.web3j.crypto.ECDSASignature; import org.web3j.crypto.Hash; import org.web3j.crypto.Sign; @@ -16,6 +17,7 @@ import java.util.Map; import java.util.function.Function; +import static org.ethereum.beacon.discovery.enr.NodeRecord.FIELD_ID; import static org.ethereum.beacon.discovery.enr.NodeRecord.FIELD_IP_V4; import static org.ethereum.beacon.discovery.enr.NodeRecord.FIELD_IP_V6; import static org.ethereum.beacon.discovery.enr.NodeRecord.FIELD_PKEY_SECP256K1; @@ -31,6 +33,7 @@ public class EnrSchemeV4Interpreter implements EnrSchemeInterpreter { public EnrSchemeV4Interpreter() { fieldDecoders.put(FIELD_PKEY_SECP256K1, rlpString -> BytesValue.wrap(rlpString.getBytes())); + fieldDecoders.put(FIELD_ID, rlpString -> EnrScheme.fromString(new String(rlpString.getBytes()))); fieldDecoders.put( FIELD_IP_V4, rlpString -> Bytes4.wrap(BytesValue.wrap(rlpString.getBytes()), 0)); fieldDecoders.put(FIELD_TCP_V4, rlpString -> rlpString.asPositiveBigInteger().intValue()); @@ -49,19 +52,21 @@ public void verify(NodeRecord nodeRecord) { String.format( "Field %s not exists but required for scheme %s", FIELD_PKEY_SECP256K1, getScheme())); } - BytesValue pubKey = (BytesValue) nodeRecord.get(FIELD_PKEY_SECP256K1); + BytesValue pubKey = (BytesValue) nodeRecord.get(FIELD_PKEY_SECP256K1); // compressed + ECPoint ecPoint = Functions.publicKeyToPoint(pubKey); + BytesValue pubKeyUncompressed = BytesValue.wrap(ecPoint.getEncoded(false)).slice(1); ECDSASignature ecdsaSignature = new ECDSASignature( new BigInteger(1, nodeRecord.getSignature().slice(0, 32).extractArray()), new BigInteger(1, nodeRecord.getSignature().slice(32).extractArray())); byte[] msgHash = Hash.sha3(nodeRecord.serialize(false).extractArray()); for (int recId = 0; recId < 4; ++recId) { - BigInteger calculatedPubKey = Sign.recoverFromSignature(1, ecdsaSignature, msgHash); + BigInteger calculatedPubKey = Sign.recoverFromSignature(recId, ecdsaSignature, msgHash); if (calculatedPubKey == null) { continue; } if (Arrays.areEqual( - pubKey.extractArray(), extractBytesFromUnsignedBigInt(calculatedPubKey))) { + pubKeyUncompressed.extractArray(), extractBytesFromUnsignedBigInt(calculatedPubKey))) { return; } } @@ -76,7 +81,13 @@ public EnrScheme getScheme() { @Override public Bytes32 getNodeId(NodeRecord nodeRecord) { verify(nodeRecord); - return Hashes.sha256((BytesValue) nodeRecord.getKey(FIELD_PKEY_SECP256K1)); + BytesValue pkey = (BytesValue) nodeRecord.getKey(FIELD_PKEY_SECP256K1); + ECPoint pudDestPoint = Functions.publicKeyToPoint(pkey); + BytesValue xPart = + Bytes32.leftPad(BytesValue.wrap(pudDestPoint.getXCoord().toBigInteger().toByteArray())); + BytesValue yPart = + Bytes32.leftPad(BytesValue.wrap(pudDestPoint.getYCoord().toBigInteger().toByteArray())); + return Bytes32.wrap(Hash.sha3(xPart.concat(yPart).extractArray())); } @Override @@ -88,6 +99,14 @@ public Object decode(String key, RlpString rlpString) { return fieldDecoder.apply(rlpString); } + @Override + public void sign(NodeRecord nodeRecord, Object signOptions) { + BytesValue privateKey = (BytesValue) signOptions; + BytesValue signature = Functions.sign( + privateKey, + BytesValue.wrap(Hash.sha3(nodeRecord.serialize(false).extractArray()))); + nodeRecord.setSignature(signature); + } @Override public RlpString encode(String key, Object object) { if (object instanceof BytesValue) { @@ -96,6 +115,8 @@ public RlpString encode(String key, Object object) { return fromNumber((Number) object); } else if (object == null) { return RlpString.create(new byte[0]); + } else if (object instanceof EnrScheme) { + return RlpString.create(((EnrScheme) object).stringName()); } else { throw new RuntimeException( String.format( From 1c4d0b694dde9ab09750bcd033de0675d33e0229 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Tue, 19 Nov 2019 11:24:41 +0300 Subject: [PATCH 65/77] discovery: spec fixes, interop test with geth started but not finished --- .../beacon/discovery/DiscoveryManager.java | 7 ++ .../discovery/DiscoveryManagerImpl.java | 38 ++++-- .../ethereum/beacon/discovery/Functions.java | 90 ++++++++++---- .../beacon/discovery/NodeSession.java | 20 +++- .../discovery/enr/EnrSchemeV4Interpreter.java | 39 ++---- .../beacon/discovery/enr/NodeRecord.java | 18 ++- .../discovery/enr/NodeRecordFactory.java | 68 +++++++---- .../message/handler/FindNodeHandler.java | 41 +++++-- ...mpl.java => NettyDiscoveryClientImpl.java} | 70 ++--------- .../network/NettyDiscoveryServer.java | 13 ++ ...mpl.java => NettyDiscoveryServerImpl.java} | 36 +++++- .../packet/AuthHeaderMessagePacket.java | 17 ++- .../beacon/discovery/pipeline/Field.java | 1 + .../AuthHeaderMessagePacketHandler.java | 4 +- .../pipeline/handler/NewTaskHandler.java | 17 +++ .../handler/WhoAreYouPacketHandler.java | 3 +- .../beacon/discovery/storage/NodeBucket.java | 22 ++-- .../discovery/task/DiscoveryTaskManager.java | 31 +++-- .../beacon/discovery/task/LiveCheckTasks.java | 2 +- .../discovery/task/TaskMessageFactory.java | 2 +- .../beacon/discovery/task/TaskOptions.java | 13 ++ .../discovery/DiscoveryInteropTest.java | 111 ++++++++++++++++++ .../discovery/DiscoveryNetworkTest.java | 7 +- .../discovery/DiscoveryNoNetworkTest.java | 10 +- .../beacon/discovery/FunctionsTest.java | 53 +++++++++ .../discovery/HandshakeHandlersTest.java | 4 +- .../beacon/discovery/NodeRecordTest.java | 39 +++--- .../ethereum/beacon/discovery/SubTests.java | 5 +- .../ethereum/beacon/discovery/TestUtil.java | 41 +++++-- .../mock/DiscoveryManagerNoNetwork.java | 13 ++ .../mock/EnrSchemeV4InterpreterMock.java | 8 +- .../discovery/storage/NodeBucketTest.java | 58 +++------ .../discovery/storage/NodeTableTest.java | 93 +++------------ .../java/org/ethereum/beacon/util/Utils.java | 20 ++-- 34 files changed, 643 insertions(+), 371 deletions(-) rename discovery/src/main/java/org/ethereum/beacon/discovery/network/{DiscoveryClientImpl.java => NettyDiscoveryClientImpl.java} (53%) create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/network/NettyDiscoveryServer.java rename discovery/src/main/java/org/ethereum/beacon/discovery/network/{DiscoveryServerImpl.java => NettyDiscoveryServerImpl.java} (71%) create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/task/TaskOptions.java create mode 100644 discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryInteropTest.java diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManager.java b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManager.java index b1529aaee..5540c320e 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManager.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManager.java @@ -23,4 +23,11 @@ public interface DiscoveryManager { * handshake/bad message exchange. */ CompletableFuture executeTask(NodeRecord nodeRecord, TaskType taskType); + + /** + * Same as {@link #executeTask(NodeRecord, TaskType)} but doesn't update node liveness status in + * successful case + */ + CompletableFuture executeTaskWithoutLivenessUpdate( + NodeRecord nodeRecord, TaskType taskType); } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java index 4070fe83b..5ae97e9b8 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java @@ -6,9 +6,9 @@ import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.enr.NodeRecordFactory; import org.ethereum.beacon.discovery.network.DiscoveryClient; -import org.ethereum.beacon.discovery.network.DiscoveryClientImpl; -import org.ethereum.beacon.discovery.network.DiscoveryServer; -import org.ethereum.beacon.discovery.network.DiscoveryServerImpl; +import org.ethereum.beacon.discovery.network.NettyDiscoveryClientImpl; +import org.ethereum.beacon.discovery.network.NettyDiscoveryServer; +import org.ethereum.beacon.discovery.network.NettyDiscoveryServerImpl; import org.ethereum.beacon.discovery.network.NetworkParcel; import org.ethereum.beacon.discovery.pipeline.Envelope; import org.ethereum.beacon.discovery.pipeline.Field; @@ -33,6 +33,7 @@ import org.ethereum.beacon.discovery.storage.AuthTagRepository; import org.ethereum.beacon.discovery.storage.NodeBucketStorage; import org.ethereum.beacon.discovery.storage.NodeTable; +import org.ethereum.beacon.discovery.task.TaskOptions; import org.ethereum.beacon.discovery.task.TaskType; import org.ethereum.beacon.schedulers.Scheduler; import org.reactivestreams.Publisher; @@ -43,17 +44,20 @@ import tech.pegasys.artemis.util.bytes.BytesValue; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; public class DiscoveryManagerImpl implements DiscoveryManager { private static final Logger logger = LogManager.getLogger(DiscoveryManagerImpl.class); private final ReplayProcessor outgoingMessages = ReplayProcessor.cacheLast(); private final FluxSink outgoingSink = outgoingMessages.sink(); - private final DiscoveryServer discoveryServer; + private final NettyDiscoveryServer discoveryServer; private final Scheduler scheduler; private final Pipeline incomingPipeline = new PipelineImpl(); private final Pipeline outgoingPipeline = new PipelineImpl(); private final NodeRecordFactory nodeRecordFactory; private DiscoveryClient discoveryClient; + private CountDownLatch clientStarted = new CountDownLatch(1); public DiscoveryManagerImpl( NodeTable nodeTable, @@ -62,16 +66,19 @@ public DiscoveryManagerImpl( BytesValue homeNodePrivateKey, NodeRecordFactory nodeRecordFactory, Scheduler serverScheduler, - Scheduler clientScheduler, Scheduler taskScheduler) { AuthTagRepository authTagRepo = new AuthTagRepository(); this.scheduler = serverScheduler; this.nodeRecordFactory = nodeRecordFactory; this.discoveryServer = - new DiscoveryServerImpl( + new NettyDiscoveryServerImpl( ((Bytes4) homeNode.get(NodeRecord.FIELD_IP_V4)), (int) homeNode.get(NodeRecord.FIELD_UDP_V4)); - this.discoveryClient = new DiscoveryClientImpl(outgoingMessages, clientScheduler); + discoveryServer.useDatagramChannel( + channel -> { + discoveryClient = new NettyDiscoveryClientImpl(outgoingMessages, channel); + clientStarted.countDown(); + }); NodeIdToSession nodeIdToSession = new NodeIdToSession( homeNode, @@ -108,6 +115,11 @@ public void start() { outgoingPipeline.build(); Flux.from(discoveryServer.getIncomingPackets()).subscribe(incomingPipeline::push); discoveryServer.start(scheduler); + try { + clientStarted.await(2, TimeUnit.SECONDS); + } catch (InterruptedException e) { + throw new RuntimeException("Failed to start client", e); + } } @Override @@ -116,15 +128,27 @@ public void stop() { } public CompletableFuture executeTask(NodeRecord nodeRecord, TaskType taskType) { + return executeTaskImpl(nodeRecord, taskType, true); + } + + public CompletableFuture executeTaskImpl( + NodeRecord nodeRecord, TaskType taskType, boolean livenessUpdate) { Envelope envelope = new Envelope(); envelope.put(Field.NODE, nodeRecord); CompletableFuture future = new CompletableFuture<>(); envelope.put(Field.TASK, taskType); envelope.put(Field.FUTURE, future); + envelope.put(Field.TASK_OPTIONS, new TaskOptions(livenessUpdate)); outgoingPipeline.push(envelope); return future; } + @Override + public CompletableFuture executeTaskWithoutLivenessUpdate( + NodeRecord nodeRecord, TaskType taskType) { + return executeTaskImpl(nodeRecord, taskType, false); + } + @VisibleForTesting Publisher getOutgoingMessages() { return outgoingMessages; diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java b/discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java index c7ad1c7ee..885928a6b 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java @@ -7,7 +7,10 @@ import org.bouncycastle.crypto.params.ECDomainParameters; import org.bouncycastle.crypto.params.HKDFParameters; import org.bouncycastle.math.ec.ECPoint; +import org.bouncycastle.util.Arrays; import org.ethereum.beacon.crypto.Hashes; +import org.ethereum.beacon.util.Utils; +import org.web3j.crypto.ECDSASignature; import org.web3j.crypto.ECKeyPair; import org.web3j.crypto.Sign; import tech.pegasys.artemis.util.bytes.Bytes32; @@ -19,15 +22,20 @@ import javax.crypto.spec.SecretKeySpec; import java.math.BigInteger; import java.security.SecureRandom; -import java.security.SignatureException; import java.util.Random; +import static org.ethereum.beacon.util.Utils.extractBytesFromUnsignedBigInt; import static org.web3j.crypto.Sign.CURVE_PARAMS; public class Functions { + public static final ECDomainParameters SECP256K1_CURVE = + new ECDomainParameters( + CURVE_PARAMS.getCurve(), CURVE_PARAMS.getG(), CURVE_PARAMS.getN(), CURVE_PARAMS.getH()); + public static final int PUBKEY_SIZE = 64; private static final int RECIPIENT_KEY_LENGTH = 16; private static final int INITIATOR_KEY_LENGTH = 16; private static final int AUTH_RESP_KEY_LENGTH = 16; + private static final int MS_IN_SECOND = 1000; public static Bytes32 hash(BytesValue value) { return Hashes.sha256(value); @@ -37,35 +45,48 @@ public static Bytes32 hash(BytesValue value) { * Creates a signature of message `x` using the given key. * * @param key private key - * @param x message + * @param x message, not hashed * @return ECDSA signature with properties merged together: r || s */ public static BytesValue sign(BytesValue key, BytesValue x) { + BytesValue hash = Functions.hash(x); Sign.SignatureData signatureData = - Sign.signMessage(x.extractArray(), ECKeyPair.create(key.extractArray()), false); + Sign.signMessage(hash.extractArray(), ECKeyPair.create(key.extractArray()), false); Bytes32 r = Bytes32.wrap(signatureData.getR()); Bytes32 s = Bytes32.wrap(signatureData.getS()); return r.concat(s); } /** - * Recovers public key from message and signature + * Verifies that signature is made by signer * * @param signature Signature, ECDSA - * @param x message - * @return public key - * @throws SignatureException when recovery is not possible + * @param x message, not hashed + * @param pubKey Public key of supposed signer, compressed, 33 bytes + * @return whether `signature` reflects message `x` signed with `pubkey` */ - public static BytesValue recoverFromSignature(BytesValue signature, BytesValue x) - throws SignatureException { - BigInteger publicKey = - Sign.signedMessageToKey( - x.extractArray(), - new Sign.SignatureData( - signature.get(0), - signature.slice(1, 33).extractArray(), - signature.slice(33).extractArray())); - return BytesValue.wrap(publicKey.toByteArray()); + public static boolean verifyECDSASignature( + BytesValue signature, BytesValue x, BytesValue pubKey) { + assert pubKey.size() == 33; + ECPoint ecPoint = Functions.publicKeyToPoint(pubKey); + BytesValue pubKeyUncompressed = BytesValue.wrap(ecPoint.getEncoded(false)).slice(1); + ECDSASignature ecdsaSignature = + new ECDSASignature( + new BigInteger(1, signature.slice(0, 32).extractArray()), + new BigInteger(1, signature.slice(32).extractArray())); + byte[] msgHash = Functions.hash(x).extractArray(); + for (int recId = 0; recId < 4; ++recId) { + BigInteger calculatedPubKey = Sign.recoverFromSignature(recId, ecdsaSignature, msgHash); + if (calculatedPubKey == null) { + continue; + } + if (Arrays.areEqual( + pubKeyUncompressed.extractArray(), + extractBytesFromUnsignedBigInt(calculatedPubKey, PUBKEY_SIZE))) { + return true; + } + } + return false; } /** @@ -102,16 +123,31 @@ public static BytesValue aesgcm_decrypt( } } + public static ECPoint publicKeyToPoint(BytesValue pkey) { + byte[] destPubPointBytes; + if (pkey.size() == 64) { // uncompressed + destPubPointBytes = new byte[pkey.size() + 1]; + destPubPointBytes[0] = 0x04; // default prefix + System.arraycopy(pkey.extractArray(), 0, destPubPointBytes, 1, pkey.size()); + } else { + destPubPointBytes = pkey.extractArray(); + } + return SECP256K1_CURVE.getCurve().decodePoint(destPubPointBytes); + } + + /** Derives public key in SECP256K1, compressed */ + public static BytesValue derivePublicKeyFromPrivate(BytesValue privateKey) { + ECKeyPair ecKeyPair = ECKeyPair.create(privateKey.extractArray()); + final BytesValue pubKey = + BytesValue.wrap( + Utils.extractBytesFromUnsignedBigInt(ecKeyPair.getPublicKey(), PUBKEY_SIZE)); + ECPoint ecPoint = Functions.publicKeyToPoint(pubKey); + return BytesValue.wrap(ecPoint.getEncoded(true)); + } + /** Derives key agreement ECDH by multiplying private key by public */ public static BytesValue deriveECDHKeyAgreement(BytesValue srcPrivKey, BytesValue destPubKey) { - ECDomainParameters CURVE = - new ECDomainParameters( - CURVE_PARAMS.getCurve(), CURVE_PARAMS.getG(), CURVE_PARAMS.getN(), CURVE_PARAMS.getH()); - - byte[] destPubPointBytes = new byte[destPubKey.size() + 1]; - destPubPointBytes[0] = 0x04; // default prefix - System.arraycopy(destPubKey.extractArray(), 0, destPubPointBytes, 1, destPubKey.size()); - ECPoint pudDestPoint = CURVE.getCurve().decodePoint(destPubPointBytes); + ECPoint pudDestPoint = publicKeyToPoint(destPubKey); ECPoint mult = pudDestPoint.multiply(new BigInteger(1, srcPrivKey.extractArray())); return BytesValue.wrap(mult.getEncoded(true)); } @@ -171,6 +207,10 @@ public static HKDFKeys hkdf_expand( } } + public static long getTime() { + return System.currentTimeMillis() / MS_IN_SECOND; + } + public static Random getRandom() { return new SecureRandom(); } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeSession.java b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeSession.java index 33127e189..030cb9deb 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeSession.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeSession.java @@ -34,7 +34,6 @@ public class NodeSession { public static final int REQUEST_ID_SIZE = 8; private static final Logger logger = LogManager.getLogger(NodeSession.class); private static final int CLEANUP_DELAY_SECONDS = 60; - private final NodeRecord nodeRecord; private final NodeRecord homeNodeRecord; private final Bytes32 homeNodeId; private final AuthTagRepository authTagRepo; @@ -42,6 +41,7 @@ public class NodeSession { private final NodeBucketStorage nodeBucketStorage; private final Consumer outgoing; private final Random rnd; + private NodeRecord nodeRecord; private SessionStatus status = SessionStatus.INITIAL; private Bytes32 idNonce; private BytesValue initiatorKey; @@ -76,6 +76,15 @@ public NodeRecord getNodeRecord() { return nodeRecord; } + public synchronized void updateNodeRecord(NodeRecord nodeRecord) { + logger.trace( + () -> + String.format( + "NodeRecord updated from %s to %s in session %s", + this.nodeRecord, nodeRecord, this)); + this.nodeRecord = nodeRecord; + } + private void completeConnectFuture() { if (completableFuture != null) { completableFuture.complete(null); @@ -83,7 +92,7 @@ private void completeConnectFuture() { } } - public synchronized void sendOutgoing(Packet packet) { + public void sendOutgoing(Packet packet) { logger.trace(() -> String.format("Sending outgoing packet %s in session %s", packet, this)); outgoing.accept(packet); } @@ -215,6 +224,13 @@ public synchronized void clearRequestId(BytesValue requestId, TaskType taskType) assert taskType.equals(requestInfo.getTaskType()); } + public synchronized void updateLiveness() { + NodeRecordInfo nodeRecordInfo = + new NodeRecordInfo(getNodeRecord(), Functions.getTime(), NodeStatus.ACTIVE, 0); + nodeTable.save(nodeRecordInfo); + nodeBucketStorage.put(nodeRecordInfo); + } + private synchronized RequestInfo clearRequestId(BytesValue requestId) { RequestInfo requestInfo = requestIdStatuses.remove(requestId); requestExpirationScheduler.cancel(requestId); diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrSchemeV4Interpreter.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrSchemeV4Interpreter.java index ed57c4401..f0fe255c6 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrSchemeV4Interpreter.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrSchemeV4Interpreter.java @@ -1,11 +1,9 @@ package org.ethereum.beacon.discovery.enr; import org.bouncycastle.math.ec.ECPoint; -import org.bouncycastle.util.Arrays; import org.ethereum.beacon.discovery.Functions; -import org.web3j.crypto.ECDSASignature; +import org.ethereum.beacon.util.Utils; import org.web3j.crypto.Hash; -import org.web3j.crypto.Sign; import org.web3j.rlp.RlpString; import tech.pegasys.artemis.util.bytes.Bytes16; import tech.pegasys.artemis.util.bytes.Bytes32; @@ -25,7 +23,6 @@ import static org.ethereum.beacon.discovery.enr.NodeRecord.FIELD_TCP_V6; import static org.ethereum.beacon.discovery.enr.NodeRecord.FIELD_UDP_V4; import static org.ethereum.beacon.discovery.enr.NodeRecord.FIELD_UDP_V6; -import static org.ethereum.beacon.util.Utils.extractBytesFromUnsignedBigInt; public class EnrSchemeV4Interpreter implements EnrSchemeInterpreter { @@ -33,7 +30,8 @@ public class EnrSchemeV4Interpreter implements EnrSchemeInterpreter { public EnrSchemeV4Interpreter() { fieldDecoders.put(FIELD_PKEY_SECP256K1, rlpString -> BytesValue.wrap(rlpString.getBytes())); - fieldDecoders.put(FIELD_ID, rlpString -> EnrScheme.fromString(new String(rlpString.getBytes()))); + fieldDecoders.put( + FIELD_ID, rlpString -> EnrScheme.fromString(new String(rlpString.getBytes()))); fieldDecoders.put( FIELD_IP_V4, rlpString -> Bytes4.wrap(BytesValue.wrap(rlpString.getBytes()), 0)); fieldDecoders.put(FIELD_TCP_V4, rlpString -> rlpString.asPositiveBigInteger().intValue()); @@ -53,24 +51,8 @@ public void verify(NodeRecord nodeRecord) { "Field %s not exists but required for scheme %s", FIELD_PKEY_SECP256K1, getScheme())); } BytesValue pubKey = (BytesValue) nodeRecord.get(FIELD_PKEY_SECP256K1); // compressed - ECPoint ecPoint = Functions.publicKeyToPoint(pubKey); - BytesValue pubKeyUncompressed = BytesValue.wrap(ecPoint.getEncoded(false)).slice(1); - ECDSASignature ecdsaSignature = - new ECDSASignature( - new BigInteger(1, nodeRecord.getSignature().slice(0, 32).extractArray()), - new BigInteger(1, nodeRecord.getSignature().slice(32).extractArray())); - byte[] msgHash = Hash.sha3(nodeRecord.serialize(false).extractArray()); - for (int recId = 0; recId < 4; ++recId) { - BigInteger calculatedPubKey = Sign.recoverFromSignature(recId, ecdsaSignature, msgHash); - if (calculatedPubKey == null) { - continue; - } - if (Arrays.areEqual( - pubKeyUncompressed.extractArray(), extractBytesFromUnsignedBigInt(calculatedPubKey))) { - return; - } - } - assert false; + assert Functions.verifyECDSASignature( + nodeRecord.getSignature(), nodeRecord.serialize(false), pubKey); } @Override @@ -84,9 +66,11 @@ public Bytes32 getNodeId(NodeRecord nodeRecord) { BytesValue pkey = (BytesValue) nodeRecord.getKey(FIELD_PKEY_SECP256K1); ECPoint pudDestPoint = Functions.publicKeyToPoint(pkey); BytesValue xPart = - Bytes32.leftPad(BytesValue.wrap(pudDestPoint.getXCoord().toBigInteger().toByteArray())); + Bytes32.wrap( + Utils.extractBytesFromUnsignedBigInt(pudDestPoint.getXCoord().toBigInteger(), 32)); BytesValue yPart = - Bytes32.leftPad(BytesValue.wrap(pudDestPoint.getYCoord().toBigInteger().toByteArray())); + Bytes32.wrap( + Utils.extractBytesFromUnsignedBigInt(pudDestPoint.getYCoord().toBigInteger(), 32)); return Bytes32.wrap(Hash.sha3(xPart.concat(yPart).extractArray())); } @@ -102,11 +86,10 @@ public Object decode(String key, RlpString rlpString) { @Override public void sign(NodeRecord nodeRecord, Object signOptions) { BytesValue privateKey = (BytesValue) signOptions; - BytesValue signature = Functions.sign( - privateKey, - BytesValue.wrap(Hash.sha3(nodeRecord.serialize(false).extractArray()))); + BytesValue signature = Functions.sign(privateKey, nodeRecord.serialize(false)); nodeRecord.setSignature(signature); } + @Override public RlpString encode(String key, Object object) { if (object instanceof BytesValue) { diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecord.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecord.java index 750f64b6f..ff4e5ca00 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecord.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecord.java @@ -17,6 +17,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; /** * Ethereum Node Record @@ -26,6 +27,8 @@ public class NodeRecord { // Compressed secp256k1 public key, 33 bytes public static String FIELD_PKEY_SECP256K1 = "secp256k1"; + // Schema id + public static String FIELD_ID = "id"; // IPv4 address public static String FIELD_IP_V4 = "ip"; // TCP port, integer @@ -137,6 +140,10 @@ public void verify() { enrSchemeInterpreter.verify(this); } + public void sign(Object signOptions) { + enrSchemeInterpreter.sign(this, signOptions); + } + public RlpList asRlp() { return asRlp(true); } @@ -151,14 +158,13 @@ public RlpList asRlp(boolean withSignature) { values.add(RlpString.create(getSignature().extractArray())); } values.add(RlpString.create(getSeq().toBI())); - values.add(RlpString.create("id")); - values.add(RlpString.create(getIdentityScheme().stringName())); - for (Map.Entry keyPair : fields.entrySet()) { - if (keyPair.getValue() == null) { + List keySortedList = fields.keySet().stream().sorted().collect(Collectors.toList()); + for (String key : keySortedList) { + if (fields.get(key) == null) { continue; } - values.add(RlpString.create(keyPair.getKey())); - values.add(enrSchemeInterpreter.encode(keyPair.getKey(), keyPair.getValue())); + values.add(RlpString.create(key)); + values.add(enrSchemeInterpreter.encode(key, fields.get(key))); } return new RlpList(values); diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordFactory.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordFactory.java index 1817b3c71..fcee6d66e 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordFactory.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordFactory.java @@ -15,6 +15,8 @@ import java.util.List; import java.util.Map; +import static org.ethereum.beacon.discovery.enr.NodeRecord.FIELD_ID; + public class NodeRecordFactory { public static final NodeRecordFactory DEFAULT = new NodeRecordFactory(new EnrSchemeV4Interpreter()); @@ -28,23 +30,29 @@ public NodeRecordFactory(EnrSchemeInterpreter... enrSchemeInterpreters) { @SafeVarargs public final NodeRecord createFromValues( - EnrScheme enrScheme, - UInt64 seq, - BytesValue signature, - Pair... fieldKeyPairs) { - return createFromValues(enrScheme, seq, signature, Arrays.asList(fieldKeyPairs)); + UInt64 seq, BytesValue signature, Pair... fieldKeyPairs) { + return createFromValues(seq, signature, Arrays.asList(fieldKeyPairs)); } public NodeRecord createFromValues( - EnrScheme enrScheme, - UInt64 seq, - BytesValue signature, - List> fieldKeyPairs) { + UInt64 seq, BytesValue signature, List> fieldKeyPairs) { + Pair schemePair = null; + for (Pair pair : fieldKeyPairs) { + if (FIELD_ID.equals(pair.getValue0())) { + schemePair = pair; + break; + } + } + if (schemePair == null) { + throw new RuntimeException("ENR scheme is not defined in key-value pairs"); + } - EnrSchemeInterpreter enrSchemeInterpreter = interpreters.get(enrScheme); + EnrSchemeInterpreter enrSchemeInterpreter = interpreters.get(schemePair.getValue1()); if (enrSchemeInterpreter == null) { throw new RuntimeException( - String.format("No ethereum record interpreter found for identity scheme %s", enrScheme)); + String.format( + "No ethereum record interpreter found for identity scheme %s", + schemePair.getValue1())); } return NodeRecord.fromValues(enrSchemeInterpreter, seq, signature, fieldKeyPairs); @@ -64,26 +72,36 @@ public NodeRecord fromRlpList(RlpList rlpList) { throw new RuntimeException( String.format("Unable to deserialize ENR with less than 4 fields, [%s]", values)); } - RlpString id = (RlpString) values.get(2); - if (!"id".equals(new String(id.getBytes()))) { - throw new RuntimeException( - String.format("Unable to deserialize ENR with no id field at 2-3 records, [%s]", values)); - } - RlpString idVersion = (RlpString) values.get(3); - EnrScheme nodeIdentity = EnrScheme.fromString(new String(idVersion.getBytes())); - if (nodeIdentity == null) { - throw new RuntimeException( - String.format( - "Unknown node identity scheme '%s', couldn't create node record.", - idVersion.asString())); + // TODO: repair as id is not first now + EnrScheme nodeIdentity = null; + boolean idFound = false; + for (int i = 2; i < values.size(); i += 2) { + RlpString id = (RlpString) values.get(i); + if (!"id".equals(new String(id.getBytes()))) { + continue; + } + + RlpString idVersion = (RlpString) values.get(i + 1); + nodeIdentity = EnrScheme.fromString(new String(idVersion.getBytes())); + if (nodeIdentity == null) { // no interpreter for such id + throw new RuntimeException( + String.format( + "Unknown node identity scheme '%s', couldn't create node record.", + idVersion.asString())); + } + idFound = true; + break; + } + if (!idFound) { // no `id` key-values + throw new RuntimeException("Unknown node identity scheme, not defined in record "); } EnrSchemeInterpreter enrSchemeInterpreter = interpreters.get(nodeIdentity); if (enrSchemeInterpreter == null) { throw new RuntimeException( String.format( - "No ethereum record interpreter found for identity scheme %s", nodeIdentity)); + "No Ethereum record interpreter found for identity scheme %s", nodeIdentity)); } return NodeRecord.fromRawFields( @@ -91,7 +109,7 @@ public NodeRecord fromRlpList(RlpList rlpList) { UInt64.fromBytesBigEndian( Bytes8.leftPad(BytesValue.wrap(((RlpString) values.get(1)).getBytes()))), BytesValue.wrap(((RlpString) values.get(0)).getBytes()), - values.subList(4, values.size())); + values.subList(2, values.size())); } public NodeRecord fromBytes(byte[] bytes) { diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/FindNodeHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/FindNodeHandler.java index dabaa2dee..49b4b5c17 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/FindNodeHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/FindNodeHandler.java @@ -4,12 +4,15 @@ import org.apache.logging.log4j.Logger; import org.ethereum.beacon.discovery.NodeRecordInfo; import org.ethereum.beacon.discovery.NodeSession; +import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.message.DiscoveryV5Message; import org.ethereum.beacon.discovery.message.FindNodeMessage; import org.ethereum.beacon.discovery.message.NodesMessage; import org.ethereum.beacon.discovery.packet.MessagePacket; import org.ethereum.beacon.discovery.storage.NodeBucket; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; @@ -17,13 +20,16 @@ public class FindNodeHandler implements MessageHandler { private static final Logger logger = LogManager.getLogger(FindNodeHandler.class); + private static final int MAX_NODES_PER_MESSAGE = 12; public FindNodeHandler() {} @Override public void handle(FindNodeMessage message, NodeSession session) { + int start = + message.getDistance() == 0 ? 0 : 1; // home node from 0 should be returned only for 0 List nodeBuckets = - IntStream.range(0, message.getDistance()) + IntStream.range(start, message.getDistance()) .mapToObj(session::getBucket) .filter(Optional::isPresent) .map(Optional::get) @@ -33,8 +39,28 @@ public void handle(FindNodeMessage message, NodeSession session) { String.format( "Sending %s nodeBuckets in reply to request in session %s", nodeBuckets.size(), session)); - nodeBuckets.forEach( - bucket -> + List> nodeRecordsList = new ArrayList<>(); + int total = 0; + + // Repack to lists of MAX_NODES_PER_MESSAGE size + for (NodeBucket nodeBucket : nodeBuckets) { + for (NodeRecordInfo nodeRecordInfo : nodeBucket.getNodeRecords()) { + if (total % MAX_NODES_PER_MESSAGE == 0) { + nodeRecordsList.add(new ArrayList<>()); + } + List currentList = nodeRecordsList.get(nodeRecordsList.size() - 1); + currentList.add(nodeRecordInfo.getNode()); + ++total; + } + } + + // Send + if (nodeRecordsList.isEmpty()) { + nodeRecordsList.add(Collections.emptyList()); + } + int finalTotal = total; + nodeRecordsList.forEach( + recordsList -> session.sendOutgoing( MessagePacket.create( session.getHomeNodeId(), @@ -44,11 +70,8 @@ public void handle(FindNodeMessage message, NodeSession session) { DiscoveryV5Message.from( new NodesMessage( message.getRequestId(), - nodeBuckets.size(), - () -> - bucket.getNodeRecords().stream() - .map(NodeRecordInfo::getNode) - .collect(Collectors.toList()), - bucket.size()))))); + finalTotal, + () -> recordsList, + recordsList.size()))))); } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/network/DiscoveryClientImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/network/NettyDiscoveryClientImpl.java similarity index 53% rename from discovery/src/main/java/org/ethereum/beacon/discovery/network/DiscoveryClientImpl.java rename to discovery/src/main/java/org/ethereum/beacon/discovery/network/NettyDiscoveryClientImpl.java index 52b4c8bba..4f03f7a08 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/network/DiscoveryClientImpl.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/network/NettyDiscoveryClientImpl.java @@ -1,17 +1,12 @@ package org.ethereum.beacon.discovery.network; -import io.netty.bootstrap.Bootstrap; import io.netty.buffer.Unpooled; -import io.netty.channel.Channel; -import io.netty.channel.ChannelInitializer; -import io.netty.channel.nio.NioEventLoopGroup; import io.netty.channel.socket.DatagramPacket; import io.netty.channel.socket.nio.NioDatagramChannel; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.ethereum.beacon.discovery.enr.EnrScheme; import org.ethereum.beacon.discovery.enr.NodeRecord; -import org.ethereum.beacon.schedulers.Scheduler; import org.reactivestreams.Publisher; import reactor.core.publisher.Flux; import tech.pegasys.artemis.util.bytes.Bytes4; @@ -20,73 +15,30 @@ import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.UnknownHostException; -import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicBoolean; -/** Discovery UDP client */ -public class DiscoveryClientImpl implements DiscoveryClient { - private static final int RECREATION_TIMEOUT = 5000; +/** Netty discovery UDP client */ +public class NettyDiscoveryClientImpl implements DiscoveryClient { private static final int STOPPING_TIMEOUT = 10000; - private static final Logger logger = LogManager.getLogger(DiscoveryClientImpl.class); - private AtomicBoolean listen = new AtomicBoolean(true); - private CountDownLatch starting = new CountDownLatch(1); - private Channel channel; + private static final Logger logger = LogManager.getLogger(NettyDiscoveryClientImpl.class); + private AtomicBoolean listen = new AtomicBoolean(false); + private NioDatagramChannel channel; /** * Constructs UDP client using * * @param outgoingStream Stream of outgoing packets, client will forward them to the channel - * @param scheduler Scheduler to run client loop on + * @param channel Nio channel */ - public DiscoveryClientImpl(Publisher outgoingStream, Scheduler scheduler) { + public NettyDiscoveryClientImpl( + Publisher outgoingStream, NioDatagramChannel channel) { + this.channel = channel; Flux.from(outgoingStream) .subscribe( networkPacket -> send(networkPacket.getPacket().getBytes(), networkPacket.getNodeRecord())); - logger.info("Starting UDP discovery client"); - scheduler.execute(this::clientLoop); - try { - starting.await(); - logger.info("UDP discovery client started"); - } catch (InterruptedException e) { - throw new RuntimeException("Initialization of discovery client broke by interruption", e); - } - } - - private void clientLoop() { - NioEventLoopGroup group = new NioEventLoopGroup(1); - try { - while (listen.get()) { - Bootstrap b = new Bootstrap(); - b.group(group) - .channel(NioDatagramChannel.class) - .handler( - new ChannelInitializer() { - @Override - protected void initChannel(NioDatagramChannel ch) throws Exception { - starting.countDown(); - } - }); - - channel = b.bind(0).sync().channel(); - channel.closeFuture().sync(); - - if (!listen.get()) { - logger.info("Shutting down discovery client"); - break; - } - logger.error("Discovery client closed. Trying to restore after %s seconds delay"); - Thread.sleep(RECREATION_TIMEOUT); - } - } catch (Exception e) { - logger.error("Can't start discovery client", e); - } finally { - try { - group.shutdownGracefully().sync(); - } catch (Exception ex) { - logger.error("Failed to shutdown discovery client thread group", ex); - } - } + logger.info("UDP discovery client started"); + listen.set(true); } @Override diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/network/NettyDiscoveryServer.java b/discovery/src/main/java/org/ethereum/beacon/discovery/network/NettyDiscoveryServer.java new file mode 100644 index 000000000..eb1d39a88 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/network/NettyDiscoveryServer.java @@ -0,0 +1,13 @@ +package org.ethereum.beacon.discovery.network; + +import io.netty.channel.socket.nio.NioDatagramChannel; + +import java.util.concurrent.CompletableFuture; +import java.util.function.Consumer; + +/** Netty-specific extension of {@link DiscoveryServer}. Made to reuse server channel for client. */ +public interface NettyDiscoveryServer extends DiscoveryServer { + + /** Reuse Netty server channel with client, so you are able to send packets from the same port */ + CompletableFuture useDatagramChannel(Consumer consumer); +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/network/DiscoveryServerImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/network/NettyDiscoveryServerImpl.java similarity index 71% rename from discovery/src/main/java/org/ethereum/beacon/discovery/network/DiscoveryServerImpl.java rename to discovery/src/main/java/org/ethereum/beacon/discovery/network/NettyDiscoveryServerImpl.java index 415f933f9..a2baff38e 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/network/DiscoveryServerImpl.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/network/NettyDiscoveryServerImpl.java @@ -16,20 +16,26 @@ import java.net.InetAddress; import java.net.UnknownHostException; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Consumer; -public class DiscoveryServerImpl implements DiscoveryServer { +public class NettyDiscoveryServerImpl implements NettyDiscoveryServer { private static final int RECREATION_TIMEOUT = 5000; private static final int STOPPING_TIMEOUT = 10000; - private static final Logger logger = LogManager.getLogger(DiscoveryServerImpl.class); + private static final Logger logger = LogManager.getLogger(NettyDiscoveryServerImpl.class); private final ReplayProcessor incomingPackets = ReplayProcessor.cacheLast(); private final FluxSink incomingSink = incomingPackets.sink(); private final Integer udpListenPort; private final String udpListenHost; private AtomicBoolean listen = new AtomicBoolean(true); private Channel channel; + private NioDatagramChannel datagramChannel; + private Set> datagramChannelUsageQueue = new HashSet<>(); - public DiscoveryServerImpl(Bytes4 udpListenHost, Integer udpListenPort) { + public NettyDiscoveryServerImpl(Bytes4 udpListenHost, Integer udpListenPort) { try { this.udpListenHost = InetAddress.getByAddress(udpListenHost.extractArray()).getHostAddress(); } catch (UnknownHostException e) { @@ -58,6 +64,11 @@ public void initChannel(NioDatagramChannel ch) throws Exception { ch.pipeline() .addLast(new DatagramToBytesValue()) .addLast(new IncomingMessageSink(incomingSink)); + synchronized (NettyDiscoveryServerImpl.class) { + datagramChannel = ch; + datagramChannelUsageQueue.forEach( + nioDatagramChannelConsumer -> nioDatagramChannelConsumer.accept(ch)); + } } }); @@ -87,6 +98,25 @@ public Publisher getIncomingPackets() { return incomingPackets; } + /** Reuse Netty server channel with client, so you are able to send packets from the same port */ + @Override + public synchronized CompletableFuture useDatagramChannel( + Consumer consumer) { + CompletableFuture usage = new CompletableFuture<>(); + if (datagramChannel != null) { + consumer.accept(datagramChannel); + usage.complete(null); + } else { + datagramChannelUsageQueue.add( + nioDatagramChannel -> { + consumer.accept(nioDatagramChannel); + usage.complete(null); + }); + } + + return usage; + } + @Override public void stop() { if (listen.get()) { diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/AuthHeaderMessagePacket.java b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/AuthHeaderMessagePacket.java index ca4901c0f..0ec9399ca 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/AuthHeaderMessagePacket.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/AuthHeaderMessagePacket.java @@ -58,10 +58,16 @@ public static AuthHeaderMessagePacket create( return new AuthHeaderMessagePacket(tag.concat(authHeader).concat(messageCipherText)); } + public static BytesValue createIdNonceMessage(BytesValue idNonce, BytesValue ephemeralPubkey) { + BytesValue message = DISCOVERY_ID_NONCE.concat(idNonce).concat(ephemeralPubkey); + return message; + } + public static BytesValue signIdNonce( BytesValue idNonce, BytesValue staticNodeKey, BytesValue ephemeralPubkey) { - return Functions.sign( - staticNodeKey, Functions.hash(DISCOVERY_ID_NONCE.concat(idNonce).concat(ephemeralPubkey))); + BytesValue signed = + Functions.sign(staticNodeKey, createIdNonceMessage(idNonce, ephemeralPubkey)); + return signed; } public static byte[] createAuthMessagePt(BytesValue idNonceSig, @Nullable NodeRecord nodeRecord) { @@ -102,7 +108,6 @@ public static AuthHeaderMessagePacket create( DiscoveryMessage message) { Bytes32 tag = Packet.createTag(homeNodeId, destNodeId); BytesValue idNonceSig = signIdNonce(idNonce, staticNodeKey, ephemeralPubkey); - idNonceSig = idNonceSig.slice(1); // Remove recovery id byte[] authResponsePt = createAuthMessagePt(idNonceSig, nodeRecord); BytesValue authResponse = encodeAuthResponse(authResponsePt, authResponseKey); BytesValue authHeader = encodeAuthHeaderRlp(authTag, idNonce, ephemeralPubkey, authResponse); @@ -111,9 +116,13 @@ public static AuthHeaderMessagePacket create( return create(tag, authHeader, encryptedData); } - public void verify(BytesValue expectedIdNonce) { + public void verify(BytesValue expectedIdNonce, BytesValue remoteNodePubKey) { verifyDecode(); assert expectedIdNonce.equals(getIdNonce()); + assert Functions.verifyECDSASignature( + getIdNonceSig(), + createIdNonceMessage(getIdNonce(), getEphemeralPubkey()), + remoteNodePubKey); } public Bytes32 getHomeNodeId(Bytes32 destNodeId) { diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/Field.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/Field.java index e30276e28..52acbb5ab 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/Field.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/Field.java @@ -14,5 +14,6 @@ public enum Field { BAD_MESSAGE, // Bad, rejected message BAD_EXCEPTION, // Stores exception for bad packet or message TASK, // Task to perform + TASK_OPTIONS, // Task options FUTURE, // Completable future } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/AuthHeaderMessagePacketHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/AuthHeaderMessagePacketHandler.java index e8341a897..0d0204304 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/AuthHeaderMessagePacketHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/AuthHeaderMessagePacketHandler.java @@ -15,6 +15,7 @@ import tech.pegasys.artemis.util.bytes.BytesValue; import static org.ethereum.beacon.discovery.NodeSession.SessionStatus.AUTHENTICATED; +import static org.ethereum.beacon.discovery.enr.NodeRecord.FIELD_PKEY_SECP256K1; /** Handles {@link AuthHeaderMessagePacket} in {@link Field#PACKET_AUTH_HEADER_MESSAGE} field */ public class AuthHeaderMessagePacketHandler implements EnvelopeHandler { @@ -65,7 +66,8 @@ public void handle(Envelope envelope) { session.setInitiatorKey(keys.getInitiatorKey()); session.setRecipientKey(keys.getRecipientKey()); packet.decodeMessage(keys.getInitiatorKey(), keys.getAuthResponseKey(), nodeRecordFactory); - packet.verify(session.getIdNonce()); + packet.verify( + session.getIdNonce(), (BytesValue) session.getNodeRecord().get(FIELD_PKEY_SECP256K1)); envelope.put(Field.MESSAGE, packet.getMessage()); } catch (AssertionError ex) { logger.info( diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NewTaskHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NewTaskHandler.java index 9b87eac99..4af3201de 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NewTaskHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NewTaskHandler.java @@ -7,9 +7,11 @@ import org.ethereum.beacon.discovery.pipeline.EnvelopeHandler; import org.ethereum.beacon.discovery.pipeline.Field; import org.ethereum.beacon.discovery.pipeline.HandlerUtil; +import org.ethereum.beacon.discovery.task.TaskOptions; import org.ethereum.beacon.discovery.task.TaskType; import java.util.concurrent.CompletableFuture; +import java.util.function.BiConsumer; /** Enqueues task in session for any task found in {@link Field#TASK} */ public class NewTaskHandler implements EnvelopeHandler { @@ -25,6 +27,9 @@ public void handle(Envelope envelope) { if (!HandlerUtil.requireField(Field.TASK, envelope)) { return; } + if (!HandlerUtil.requireField(Field.TASK_OPTIONS, envelope)) { + return; + } if (!HandlerUtil.requireField(Field.SESSION, envelope)) { return; } @@ -40,8 +45,20 @@ public void handle(Envelope envelope) { NodeSession session = (NodeSession) envelope.get(Field.SESSION); CompletableFuture completableFuture = (CompletableFuture) envelope.get(Field.FUTURE); + TaskOptions taskOptions = (TaskOptions) envelope.get(Field.TASK_OPTIONS); + useTaskOptions(session, completableFuture, taskOptions); session.createNextRequest(task, completableFuture); envelope.remove(Field.TASK); envelope.remove(Field.FUTURE); } + + private void useTaskOptions(NodeSession nodeSession, CompletableFuture completableFuture, TaskOptions taskOptions) { + if (taskOptions.isLivenessUpdate()) { + completableFuture.whenComplete((aVoid, throwable) -> { + if (throwable == null) { + nodeSession.updateLiveness(); + } + }); + } + } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouPacketHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouPacketHandler.java index fb521541f..a9183bd86 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouPacketHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouPacketHandler.java @@ -23,6 +23,7 @@ import java.util.Optional; import java.util.function.Supplier; +import static org.ethereum.beacon.discovery.Functions.PUBKEY_SIZE; import static org.ethereum.beacon.discovery.enr.NodeRecord.FIELD_PKEY_SECP256K1; /** Handles {@link WhoAreYouPacket} in {@link Field#PACKET_WHOAREYOU} field */ @@ -90,7 +91,7 @@ public void handle(Envelope envelope) { envelope.getId(), session))); BytesValue ephemeralPubKey = - BytesValue.wrap(Utils.extractBytesFromUnsignedBigInt(ephemeralKey.getPublicKey())); + BytesValue.wrap(Utils.extractBytesFromUnsignedBigInt(ephemeralKey.getPublicKey(), PUBKEY_SIZE)); AuthHeaderMessagePacket response = AuthHeaderMessagePacket.create( session.getHomeNodeId(), diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeBucket.java b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeBucket.java index 2854daae1..1e5da8ea0 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeBucket.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeBucket.java @@ -11,7 +11,6 @@ import java.util.ArrayList; import java.util.Collection; -import java.util.Comparator; import java.util.List; import java.util.TreeSet; import java.util.function.Predicate; @@ -28,15 +27,8 @@ public class NodeBucket { private static final Predicate FILTER = nodeRecord -> nodeRecord.getStatus().equals(NodeStatus.ACTIVE); - private static final Comparator COMPARATOR = - (o1, o2) -> { - if (o1.getNode().getNodeId().equals(o2.getNode().getNodeId())) { - return 0; - } else { - return Long.signum(o1.getLastRetry() - o2.getLastRetry()); - } - }; - private final TreeSet bucket = new TreeSet<>(COMPARATOR); + private final TreeSet bucket = + new TreeSet<>((o1, o2) -> o2.getNode().hashCode() - o1.getNode().hashCode()); public static NodeBucket fromRlpBytes(BytesValue bytes, NodeRecordFactory nodeRecordFactory) { NodeBucket nodeBucket = new NodeBucket(); @@ -55,7 +47,15 @@ public synchronized boolean put(NodeRecordInfo nodeRecord) { if (!bucket.contains(nodeRecord)) { boolean modified = bucket.add(nodeRecord); if (bucket.size() > K) { - bucket.pollFirst(); + NodeRecordInfo worst = null; + for (NodeRecordInfo nodeRecordInfo : bucket) { + if (worst == null) { + worst = nodeRecordInfo; + } else if (worst.getLastRetry() > nodeRecordInfo.getLastRetry()) { + worst = nodeRecordInfo; + } + } + bucket.remove(worst); } return modified; } else { diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/task/DiscoveryTaskManager.java b/discovery/src/main/java/org/ethereum/beacon/discovery/task/DiscoveryTaskManager.java index a845552a6..6af6b7f3d 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/task/DiscoveryTaskManager.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/task/DiscoveryTaskManager.java @@ -1,6 +1,7 @@ package org.ethereum.beacon.discovery.task; import org.ethereum.beacon.discovery.DiscoveryManager; +import org.ethereum.beacon.discovery.Functions; import org.ethereum.beacon.discovery.NodeRecordInfo; import org.ethereum.beacon.discovery.NodeStatus; import org.ethereum.beacon.discovery.enr.NodeRecord; @@ -11,6 +12,7 @@ import java.time.Duration; import java.util.List; +import java.util.function.Consumer; import java.util.function.Predicate; import java.util.stream.Stream; @@ -21,7 +23,6 @@ public class DiscoveryTaskManager { private static final int LIVE_CHECK_DISTANCE = DEFAULT_DISTANCE; private static final int RECURSIVE_LOOKUP_DISTANCE = DEFAULT_DISTANCE; - private static final int MS_IN_SECOND = 1000; private static final int STATUS_EXPIRATION_SECONDS = 600; private static final int LIVE_CHECK_INTERVAL_SECONDS = 1; private static final int RECURSIVE_LOOKUP_INTERVAL_SECONDS = 10; @@ -50,7 +51,7 @@ public class DiscoveryTaskManager { */ private final Predicate LIVE_CHECK_NODE_RULE = nodeRecord -> { - long currentTime = System.currentTimeMillis() / MS_IN_SECOND; + long currentTime = Functions.getTime(); if (nodeRecord.getStatus() == NodeStatus.ACTIVE && nodeRecord.getLastRetry() > currentTime - STATUS_EXPIRATION_SECONDS) { return false; // no need to rediscover @@ -78,7 +79,7 @@ public class DiscoveryTaskManager { */ private final Predicate RECURSIVE_LOOKUP_NODE_RULE = nodeRecord -> { - long currentTime = System.currentTimeMillis() / MS_IN_SECOND; + long currentTime = Functions.getTime(); if (nodeRecord.getStatus() == NodeStatus.ACTIVE && nodeRecord.getLastRetry() > currentTime - STATUS_EXPIRATION_SECONDS) { return true; @@ -91,6 +92,7 @@ public class DiscoveryTaskManager { private final Predicate DEAD_RULE = nodeRecord -> nodeRecord.getRetry() >= MAX_RETRIES; + private final Consumer[] nodeRecordUpdatesConsumers; private boolean resetDead; private boolean removeDead; @@ -105,6 +107,8 @@ public class DiscoveryTaskManager { * status at startup and sets number of used retries to 0. Reset applies after remove, so if * remove is on, reset will be applied to 0 nodes * @param removeDead Whether to remove nodes that are found dead after several retries + * @param nodeRecordUpdatesConsumers consumers are executed when nodeRecord is updated with new + * sequence number, so it should be updated in nodeSession */ public DiscoveryTaskManager( DiscoveryManager discoveryManager, @@ -113,7 +117,8 @@ public DiscoveryTaskManager( NodeRecord homeNode, Scheduler scheduler, boolean resetDead, - boolean removeDead) { + boolean removeDead, + Consumer... nodeRecordUpdatesConsumers) { this.scheduler = scheduler; this.nodeTable = nodeTable; this.nodeBucketStorage = nodeBucketStorage; @@ -125,6 +130,7 @@ public DiscoveryTaskManager( discoveryManager, scheduler, Duration.ofSeconds(RETRY_TIMEOUT_SECONDS)); this.resetDead = resetDead; this.removeDead = removeDead; + this.nodeRecordUpdatesConsumers = nodeRecordUpdatesConsumers; } public void start() { @@ -183,16 +189,13 @@ private void liveCheckTask() { updateNode( nodeRecord, new NodeRecordInfo( - nodeRecord.getNode(), - System.currentTimeMillis() / MS_IN_SECOND, - NodeStatus.ACTIVE, - 0)), + nodeRecord.getNode(), Functions.getTime(), NodeStatus.ACTIVE, 0)), () -> updateNode( nodeRecord, new NodeRecordInfo( nodeRecord.getNode(), - System.currentTimeMillis() / MS_IN_SECOND, + Functions.getTime(), NodeStatus.SLEEP, (nodeRecord.getRetry() + 1))))); } @@ -211,11 +214,17 @@ private void recursiveLookupTask() { nodeRecord, new NodeRecordInfo( nodeRecord.getNode(), - System.currentTimeMillis() / MS_IN_SECOND, + Functions.getTime(), NodeStatus.SLEEP, (nodeRecord.getRetry() + 1))))); } + void onNodeRecordUpdate(NodeRecord nodeRecord) { + for (Consumer consumer : nodeRecordUpdatesConsumers) { + consumer.accept(nodeRecord); + } + } + private void updateNode(NodeRecordInfo oldNodeRecordInfo, NodeRecordInfo newNodeRecordInfo) { // use node with latest seq known if (newNodeRecordInfo.getNode().getSeq().compareTo(oldNodeRecordInfo.getNode().getSeq()) < 0) { @@ -225,6 +234,8 @@ private void updateNode(NodeRecordInfo oldNodeRecordInfo, NodeRecordInfo newNode newNodeRecordInfo.getLastRetry(), newNodeRecordInfo.getStatus(), newNodeRecordInfo.getRetry()); + } else { + onNodeRecordUpdate(newNodeRecordInfo.getNode()); } nodeTable.save(newNodeRecordInfo); nodeBucketStorage.put(newNodeRecordInfo); diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/task/LiveCheckTasks.java b/discovery/src/main/java/org/ethereum/beacon/discovery/task/LiveCheckTasks.java index f73a49aec..5476b3bb2 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/task/LiveCheckTasks.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/task/LiveCheckTasks.java @@ -43,7 +43,7 @@ public void add(NodeRecordInfo nodeRecordInfo, Runnable successCallback, Runnabl scheduler.execute( () -> { CompletableFuture retry = - discoveryManager.executeTask(nodeRecordInfo.getNode(), TaskType.PING); + discoveryManager.executeTaskWithoutLivenessUpdate(nodeRecordInfo.getNode(), TaskType.PING); taskTimeouts.put( nodeRecordInfo.getNode().getNodeId(), () -> diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/task/TaskMessageFactory.java b/discovery/src/main/java/org/ethereum/beacon/discovery/task/TaskMessageFactory.java index f9658198e..8638003f6 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/task/TaskMessageFactory.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/task/TaskMessageFactory.java @@ -9,7 +9,7 @@ import tech.pegasys.artemis.util.bytes.BytesValue; public class TaskMessageFactory { - public static final int DEFAULT_DISTANCE = 10; + public static final int DEFAULT_DISTANCE = 100; public static MessagePacket createPacketFromRequest( NodeSession.RequestInfo requestInfo, BytesValue authTag, NodeSession session) { diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/task/TaskOptions.java b/discovery/src/main/java/org/ethereum/beacon/discovery/task/TaskOptions.java new file mode 100644 index 000000000..a2d7f8afd --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/task/TaskOptions.java @@ -0,0 +1,13 @@ +package org.ethereum.beacon.discovery.task; + +public class TaskOptions { + private boolean livenessUpdate; + + public TaskOptions(boolean livenessUpdate) { + this.livenessUpdate = livenessUpdate; + } + + public boolean isLivenessUpdate() { + return livenessUpdate; + } +} diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryInteropTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryInteropTest.java new file mode 100644 index 000000000..1f885ce28 --- /dev/null +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryInteropTest.java @@ -0,0 +1,111 @@ +package org.ethereum.beacon.discovery; + +import org.ethereum.beacon.db.Database; +import org.ethereum.beacon.discovery.enr.NodeRecord; +import org.ethereum.beacon.discovery.packet.AuthHeaderMessagePacket; +import org.ethereum.beacon.discovery.packet.RandomPacket; +import org.ethereum.beacon.discovery.packet.UnknownPacket; +import org.ethereum.beacon.discovery.storage.NodeBucketStorage; +import org.ethereum.beacon.discovery.storage.NodeTableStorage; +import org.ethereum.beacon.discovery.storage.NodeTableStorageFactoryImpl; +import org.ethereum.beacon.discovery.task.TaskType; +import org.ethereum.beacon.schedulers.Schedulers; +import org.javatuples.Pair; +import org.junit.Ignore; +import org.junit.Test; +import reactor.core.publisher.Flux; +import tech.pegasys.artemis.util.bytes.BytesValue; + +import java.util.ArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import static org.ethereum.beacon.discovery.TestUtil.NODE_RECORD_FACTORY_NO_VERIFICATION; +import static org.ethereum.beacon.discovery.TestUtil.TEST_SERIALIZER; + +/** Same as {@link DiscoveryNoNetworkTest} but using real network */ +@Ignore("Finish me!!!, then ignore because it takes too long and requires geth docker") +public class DiscoveryInteropTest { + @Test + public void test() throws Exception { + // 1) start 2 nodes + Pair nodePair1 = TestUtil.generateNode(40412, true); + System.out.println(String.format("Node %s started", nodePair1.getValue1().getNodeId())); + NodeRecord nodeRecord1 = nodePair1.getValue1(); + NodeRecord nodeRecord2 = + NODE_RECORD_FACTORY_NO_VERIFICATION.fromBase64( + "-IS4QHa5-0-OmPRchyyBf9jHIWnQlZXthveUPp5_DoDnMMB0V9ChlzNq_fhFixvIr8xOQcKrYsWjjeIBoUIS8HSuWbgBgmlkgnY0gmlwhH8AAAGJc2VjcDI1NmsxoQMOLLdCQcDE_I6BZvGnmgXVsN2VgTp0sJRSnzF9XDnSNYN1ZHCCdl8"); // Geth node + NodeTableStorageFactoryImpl nodeTableStorageFactory = new NodeTableStorageFactoryImpl(); + Database database1 = Database.inMemoryDB(); + NodeTableStorage nodeTableStorage1 = + nodeTableStorageFactory.createTable( + database1, + TEST_SERIALIZER, + (oldSeq) -> nodeRecord1, + () -> + new ArrayList() { + { + add(nodeRecord2); + } + }); + NodeBucketStorage nodeBucketStorage1 = + nodeTableStorageFactory.createBucketStorage(database1, TEST_SERIALIZER, nodeRecord1); + DiscoveryManagerImpl discoveryManager1 = + new DiscoveryManagerImpl( + nodeTableStorage1.get(), + nodeBucketStorage1, + nodeRecord1, + nodePair1.getValue0(), + NODE_RECORD_FACTORY_NO_VERIFICATION, + Schedulers.createDefault().newSingleThreadDaemon("server-1"), + Schedulers.createDefault().newSingleThreadDaemon("tasks-1")); + + // 3) Expect standard 1 => 2 dialog + CountDownLatch randomSent1to2 = new CountDownLatch(1); + CountDownLatch whoareyouSent2to1 = new CountDownLatch(1); + CountDownLatch authPacketSent1to2 = new CountDownLatch(1); + CountDownLatch nodesSent2to1 = new CountDownLatch(1); + + Flux.from(discoveryManager1.getOutgoingMessages()) + .map(p -> new UnknownPacket(p.getPacket().getBytes())) + .subscribe( + networkPacket -> { + // 1 -> 2 random + if (randomSent1to2.getCount() != 0) { + RandomPacket randomPacket = networkPacket.getRandomPacket(); + System.out.println("1 => 2: " + randomPacket); + randomSent1to2.countDown(); + } else if (authPacketSent1to2.getCount() != 0) { + // 1 -> 2 auth packet with FINDNODES + AuthHeaderMessagePacket authHeaderMessagePacket = + networkPacket.getAuthHeaderMessagePacket(); + System.out.println("1 => 2: " + authHeaderMessagePacket); + authPacketSent1to2.countDown(); + } else { + throw new RuntimeException("Not expected!"); + } + }); + + // TODO: check that we receive correct nodes + + // 4) fire 1 to 2 dialog + discoveryManager1.start(); + discoveryManager1.executeTask(nodeRecord2, TaskType.FINDNODE); + + assert randomSent1to2.await(1, TimeUnit.SECONDS); + // assert whoareyouSent2to1.await(1, TimeUnit.SECONDS); + // int distance1To2 = Functions.logDistance(nodeRecord1.getNodeId(), + // nodeRecord2.getNodeId()); + // assertFalse(nodeBucketStorage1.get(distance1To2).isPresent()); + // assert authPacketSent1to2.await(1, TimeUnit.SECONDS); + // assert nodesSent2to1.await(1, TimeUnit.SECONDS); + Thread.sleep(1500); + // 1 sent findnodes to 2, received only (2) in answer, because 3 is not checked + // 1 added 2 to its nodeBuckets, because its now checked, but not before + // NodeBucket bucketAt1With2 = nodeBucketStorage1.get(distance1To2).get(); + // assertEquals(2, bucketAt1With2.size()); + // assertEquals( + // nodeRecord2.getNodeId(), + // bucketAt1With2.getNodeRecords().get(0).getNode().getNodeId()); + } +} diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java index e32b87a85..274137f0d 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java @@ -32,9 +32,8 @@ public class DiscoveryNetworkTest { @Test public void test() throws Exception { // 1) start 2 nodes - Pair nodePair1 = TestUtil.generateNode(30303); - Pair nodePair2 = TestUtil.generateNode(30304); - Pair nodePair3 = TestUtil.generateNode(40412); + Pair nodePair1 = TestUtil.generateNode(30303, true); + Pair nodePair2 = TestUtil.generateNode(30304, true); NodeRecord nodeRecord1 = nodePair1.getValue1(); NodeRecord nodeRecord2 = nodePair2.getValue1(); NodeTableStorageFactoryImpl nodeTableStorageFactory = new NodeTableStorageFactoryImpl(); @@ -74,7 +73,6 @@ public void test() throws Exception { nodePair1.getValue0(), NODE_RECORD_FACTORY_NO_VERIFICATION, Schedulers.createDefault().newSingleThreadDaemon("server-1"), - Schedulers.createDefault().newSingleThreadDaemon("client-1"), Schedulers.createDefault().newSingleThreadDaemon("tasks-1")); DiscoveryManagerImpl discoveryManager2 = new DiscoveryManagerImpl( @@ -84,7 +82,6 @@ public void test() throws Exception { nodePair2.getValue0(), NODE_RECORD_FACTORY_NO_VERIFICATION, Schedulers.createDefault().newSingleThreadDaemon("server-2"), - Schedulers.createDefault().newSingleThreadDaemon("client-2"), Schedulers.createDefault().newSingleThreadDaemon("tasks-2")); // 3) Expect standard 1 => 2 dialog diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java index 8510bd87a..14353cdab 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java @@ -37,9 +37,9 @@ public class DiscoveryNoNetworkTest { @Test public void test() throws Exception { // 1) start 2 nodes - Pair nodePair1 = TestUtil.generateNode(30303); - Pair nodePair2 = TestUtil.generateNode(30304); - Pair nodePair3 = TestUtil.generateNode(40412); + Pair nodePair1 = TestUtil.generateUnverifiedNode(30303); + Pair nodePair2 = TestUtil.generateUnverifiedNode(30304); + Pair nodePair3 = TestUtil.generateUnverifiedNode(40412); NodeRecord nodeRecord1 = nodePair1.getValue1(); NodeRecord nodeRecord2 = nodePair2.getValue1(); NodeTableStorageFactoryImpl nodeTableStorageFactory = new NodeTableStorageFactoryImpl(); @@ -121,8 +121,6 @@ public void test() throws Exception { networkPacket.getAuthHeaderMessagePacket(); System.out.println("1 => 2: " + authHeaderMessagePacket); authPacketSent1to2.countDown(); - } else { - throw new RuntimeException("Not expected!"); } }); Flux.from(from2to1) @@ -154,7 +152,7 @@ public void test() throws Exception { assert authPacketSent1to2.await(1, TimeUnit.SECONDS); assert nodesSent2to1.await(1, TimeUnit.SECONDS); Thread.sleep(50); - // 1 sent findnodes to 2, received only (2) in answer, because 3 is not checked + // 1 sent findnodes to 2, received 0 nodes in answer, because 3 is not checked // 1 added 2 to its nodeBuckets, because its now checked, but not before NodeBucket bucketAt1With2 = nodeBucketStorage1.get(distance1To2).get(); assertEquals(1, bucketAt1With2.size()); diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/FunctionsTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/FunctionsTest.java index 2ab709730..fd1e4b31c 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/FunctionsTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/FunctionsTest.java @@ -1,10 +1,14 @@ package org.ethereum.beacon.discovery; +import org.ethereum.beacon.discovery.packet.AuthHeaderMessagePacket; +import org.ethereum.beacon.util.Utils; import org.junit.Test; import org.web3j.crypto.ECKeyPair; import tech.pegasys.artemis.util.bytes.Bytes32; import tech.pegasys.artemis.util.bytes.BytesValue; +import static org.ethereum.beacon.discovery.Functions.PUBKEY_SIZE; +import static org.ethereum.beacon.discovery.packet.AuthHeaderMessagePacket.createIdNonceMessage; import static org.junit.Assert.assertEquals; public class FunctionsTest { @@ -84,4 +88,53 @@ public void testGcmSimple() { Functions.aesgcm_decrypt(authResponseKey, zeroNonce, authResponse, BytesValue.EMPTY); assertEquals(authResponsePt, authResponsePtDecrypted); } + + @Test + public void testRecoverFromSignature() throws Exception { + BytesValue idNonceSig = + BytesValue.fromHexString( + "0xcf2bf743fc2273709bbc5117fd72775b0661ce1b6e9dffa01f45e2307fb138b90da16364ee7ae1705b938f6648d7725d35fe7e3f200e0ea022c1360b9b2e7385"); + BytesValue ephemeralKey = + BytesValue.fromHexString( + "0x9961e4c2356d61bedb83052c115d311acb3a96f5777296dcf297351130266231503061ac4aaee666073d7e5bc2c80c3f5c5b500c1cb5fd0a76abbb6b675ad157"); + BytesValue nonce = + BytesValue.fromHexString( + "0x02a77e3aa0c144ae7c0a3af73692b7d6e5b7a2fdc0eda16e8d5e6cb0d08e88dd04"); + BytesValue privKey = + BytesValue.fromHexString( + "0xfb757dc581730490a1d7a00deea65e9b1936924caaea8f44d476014856b68736"); + BytesValue pubKeyUncompressed = + BytesValue.wrap( + Utils.extractBytesFromUnsignedBigInt( + ECKeyPair.create(privKey.extractArray()).getPublicKey(), PUBKEY_SIZE)); + BytesValue pubKey = + BytesValue.wrap(Functions.publicKeyToPoint(pubKeyUncompressed).getEncoded(true)); + + BytesValue message = + BytesValue.wrap("discovery-id-nonce".getBytes()).concat(nonce).concat(ephemeralKey); + assert Functions.verifyECDSASignature(idNonceSig, message, pubKey); + } + + @Test + public void testSignAndRecoverFromSignature() { + BytesValue idNonce = + BytesValue.fromHexString( + "0xd550ca9d62930c947efff75b58a4ea1b44716d841cc0d690879d4f3cab5a4e84"); + BytesValue ephemeralPubkey = + BytesValue.fromHexString( + "0xd9bc9158f0a0c40e75490de66ef44f865588d1c7110b29d0c479db19f7644ddad2d8e948cb933bd767437b173888409d73644a36ae1d068997217357a22d674f"); + BytesValue privKey = + BytesValue.fromHexString( + "0xb5a8efa45da6906663cf7a158cd506da71ae7d732a68220e6644468526bb098e"); + BytesValue pubKey = + BytesValue.wrap( + Functions.publicKeyToPoint( + BytesValue.wrap( + Utils.extractBytesFromUnsignedBigInt( + ECKeyPair.create(privKey.extractArray()).getPublicKey(), 64))) + .getEncoded(true)); + BytesValue idNonceSig = AuthHeaderMessagePacket.signIdNonce(idNonce, privKey, ephemeralPubkey); + assert Functions.verifyECDSASignature( + idNonceSig, createIdNonceMessage(idNonce, ephemeralPubkey), pubKey); + } } diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/HandshakeHandlersTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/HandshakeHandlersTest.java index edfeb07d4..33b01c869 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/HandshakeHandlersTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/HandshakeHandlersTest.java @@ -51,10 +51,10 @@ public class HandshakeHandlersTest { @Test public void authHandlerWithMessageRoundTripTest() throws Exception { // Node1 - Pair nodePair1 = TestUtil.generateNode(30303); + Pair nodePair1 = TestUtil.generateUnverifiedNode(30303); NodeRecord nodeRecord1 = nodePair1.getValue1(); // Node2 - Pair nodePair2 = TestUtil.generateNode(30304); + Pair nodePair2 = TestUtil.generateUnverifiedNode(30304); NodeRecord nodeRecord2 = nodePair2.getValue1(); Random rnd = new Random(); NodeTableStorageFactoryImpl nodeTableStorageFactory = new NodeTableStorageFactoryImpl(); diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/NodeRecordTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/NodeRecordTest.java index cae47193e..c10104f90 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/NodeRecordTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/NodeRecordTest.java @@ -6,7 +6,6 @@ import org.javatuples.Pair; import org.junit.Test; import org.web3j.crypto.ECKeyPair; -import org.web3j.crypto.Hash; import tech.pegasys.artemis.util.bytes.Bytes4; import tech.pegasys.artemis.util.bytes.Bytes96; import tech.pegasys.artemis.util.bytes.BytesValue; @@ -16,7 +15,9 @@ import java.util.Random; import java.util.concurrent.atomic.AtomicBoolean; +import static org.ethereum.beacon.discovery.Functions.PUBKEY_SIZE; import static org.ethereum.beacon.discovery.TestUtil.SEED; +import static org.ethereum.beacon.discovery.enr.NodeRecord.FIELD_ID; import static org.ethereum.beacon.discovery.enr.NodeRecord.FIELD_IP_V4; import static org.ethereum.beacon.discovery.enr.NodeRecord.FIELD_PKEY_SECP256K1; import static org.ethereum.beacon.discovery.enr.NodeRecord.FIELD_TCP_V4; @@ -68,15 +69,14 @@ public void testLocalhostV4() throws Exception { NodeRecord nodeRecordRestored = NODE_RECORD_FACTORY.fromBase64(localhostEnrRestored); assertEquals(EnrScheme.V4, nodeRecordRestored.getIdentityScheme()); - NodeRecord nodeRecordV4Restored = (NodeRecord) nodeRecordRestored; assertArrayEquals( InetAddress.getByName(expectedHost).getAddress(), - ((BytesValue) nodeRecordV4Restored.get(NodeRecord.FIELD_IP_V4)).extractArray()); - assertEquals(expectedUdpPort, nodeRecordV4Restored.get(NodeRecord.FIELD_UDP_V4)); - assertEquals(expectedTcpPort, nodeRecordV4Restored.get(NodeRecord.FIELD_TCP_V4)); - assertEquals(expectedSeqNumber, nodeRecordV4Restored.getSeq()); - assertEquals(expectedPublicKey, nodeRecordV4Restored.get(NodeRecord.FIELD_PKEY_SECP256K1)); - assertEquals(expectedSignature, nodeRecordV4Restored.getSignature()); + ((BytesValue) nodeRecordRestored.get(NodeRecord.FIELD_IP_V4)).extractArray()); + assertEquals(expectedUdpPort, nodeRecordRestored.get(NodeRecord.FIELD_UDP_V4)); + assertEquals(expectedTcpPort, nodeRecordRestored.get(NodeRecord.FIELD_TCP_V4)); + assertEquals(expectedSeqNumber, nodeRecordRestored.getSeq()); + assertEquals(expectedPublicKey, nodeRecordRestored.get(NodeRecord.FIELD_PKEY_SECP256K1)); + assertEquals(expectedSignature, nodeRecordRestored.getSignature()); } @Test @@ -84,44 +84,35 @@ public void testSignature() throws Exception { Random rnd = new Random(SEED); byte[] privKey = new byte[32]; rnd.nextBytes(privKey); - ECKeyPair keyPair = ECKeyPair.create(privKey); Bytes4 localIp = Bytes4.wrap(InetAddress.getByName("127.0.0.1").getAddress()); NodeRecord nodeRecord0 = NodeRecordFactory.DEFAULT.createFromValues( - EnrScheme.V4, UInt64.ZERO, Bytes96.ZERO, + Pair.with(FIELD_ID, EnrScheme.V4), Pair.with(FIELD_IP_V4, localIp), Pair.with(FIELD_TCP_V4, 30303), Pair.with(FIELD_UDP_V4, 30303), Pair.with( FIELD_PKEY_SECP256K1, - BytesValue.wrap(extractBytesFromUnsignedBigInt(keyPair.getPublicKey())))); - BytesValue signature0 = - Functions.sign( - BytesValue.wrap(privKey), - BytesValue.wrap(Hash.sha3(nodeRecord0.serialize(false).extractArray()))); - nodeRecord0.setSignature(signature0); + Functions.derivePublicKeyFromPrivate(BytesValue.wrap(privKey)))); + nodeRecord0.sign(BytesValue.wrap(privKey)); nodeRecord0.verify(); NodeRecord nodeRecord1 = NodeRecordFactory.DEFAULT.createFromValues( - EnrScheme.V4, UInt64.valueOf(1), Bytes96.ZERO, + Pair.with(FIELD_ID, EnrScheme.V4), Pair.with(FIELD_IP_V4, localIp), Pair.with(FIELD_TCP_V4, 30303), Pair.with(FIELD_UDP_V4, 30303), Pair.with( FIELD_PKEY_SECP256K1, - BytesValue.wrap(extractBytesFromUnsignedBigInt(keyPair.getPublicKey())))); - BytesValue signature1 = - Functions.sign( - BytesValue.wrap(privKey), - BytesValue.wrap(Hash.sha3(nodeRecord1.serialize(false).extractArray()))); - nodeRecord1.setSignature(signature1); + Functions.derivePublicKeyFromPrivate(BytesValue.wrap(privKey)))); + nodeRecord1.sign(BytesValue.wrap(privKey)); nodeRecord1.verify(); assertNotEquals(nodeRecord0.serialize(), nodeRecord1.serialize()); - assertNotEquals(signature0, signature1); + assertNotEquals(nodeRecord0.getSignature(), nodeRecord1.getSignature()); nodeRecord1.setSignature(nodeRecord0.getSignature()); AtomicBoolean exceptionThrown = new AtomicBoolean(false); try { diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/SubTests.java b/discovery/src/test/java/org/ethereum/beacon/discovery/SubTests.java index 191b21bb2..e58f10c07 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/SubTests.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/SubTests.java @@ -7,6 +7,7 @@ import java.math.BigInteger; +import static org.ethereum.beacon.discovery.Functions.PUBKEY_SIZE; import static org.junit.Assert.assertEquals; /** @@ -16,7 +17,7 @@ public class SubTests { /** * Tests BigInteger to byte[]. Take a look at {@link - * Utils#extractBytesFromUnsignedBigInt(BigInteger)} for understanding the issue. + * Utils#extractBytesFromUnsignedBigInt(BigInteger, int)} for understanding the issue. */ @Test public void testPubKeyBadPrefix() { @@ -24,7 +25,7 @@ public void testPubKeyBadPrefix() { BytesValue.fromHexString( "0xade78b68f25611ea57532f86bf01da909cc463465ed9efce9395403ff7fc99b5"); ECKeyPair badKey = ECKeyPair.create(privKey.extractArray()); - byte[] pubKey = Utils.extractBytesFromUnsignedBigInt(badKey.getPublicKey()); + byte[] pubKey = Utils.extractBytesFromUnsignedBigInt(badKey.getPublicKey(), PUBKEY_SIZE); assertEquals(64, pubKey.length); } } diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/TestUtil.java b/discovery/src/test/java/org/ethereum/beacon/discovery/TestUtil.java index e1318c0bc..b2b9581e0 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/TestUtil.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/TestUtil.java @@ -2,13 +2,12 @@ import org.ethereum.beacon.chain.storage.impl.SerializerFactory; import org.ethereum.beacon.discovery.enr.EnrScheme; +import org.ethereum.beacon.discovery.enr.EnrSchemeV4Interpreter; import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.enr.NodeRecordFactory; import org.ethereum.beacon.discovery.mock.EnrSchemeV4InterpreterMock; import org.ethereum.beacon.discovery.storage.NodeSerializerFactory; -import org.ethereum.beacon.util.Utils; import org.javatuples.Pair; -import org.web3j.crypto.ECKeyPair; import tech.pegasys.artemis.util.bytes.Bytes4; import tech.pegasys.artemis.util.bytes.Bytes96; import tech.pegasys.artemis.util.bytes.BytesValue; @@ -19,11 +18,16 @@ import java.util.ArrayList; import java.util.Random; +import static org.ethereum.beacon.discovery.enr.NodeRecord.FIELD_ID; + public class TestUtil { + public static final NodeRecordFactory NODE_RECORD_FACTORY = + new NodeRecordFactory(new EnrSchemeV4Interpreter()); public static final NodeRecordFactory NODE_RECORD_FACTORY_NO_VERIFICATION = new NodeRecordFactory(new EnrSchemeV4InterpreterMock()); // doesn't verify ECDSA signature public static final SerializerFactory TEST_SERIALIZER = new NodeSerializerFactory(NODE_RECORD_FACTORY_NO_VERIFICATION); + public static final String LOCALHOST = "127.0.0.1"; static final int SEED = 123456789; /** @@ -32,11 +36,23 @@ public class TestUtil { * * @return */ - public static Pair generateNode(int port) { + public static Pair generateUnverifiedNode(int port) { + return generateNode(port, false); + } + + /** + * Generates node on 127.0.0.1 with provided port. Node key is random, but always the same for the + * same port. + * + * @param port listen port + * @param verification whether to use valid signature + * @return + */ + public static Pair generateNode(int port, boolean verification) { final Random rnd = new Random(SEED); Bytes4 localIp = null; try { - localIp = Bytes4.wrap(InetAddress.getByName("127.0.0.1").getAddress()); + localIp = Bytes4.wrap(InetAddress.getByName(LOCALHOST).getAddress()); } catch (UnknownHostException e) { throw new RuntimeException(e); } @@ -46,21 +62,26 @@ public static Pair generateNode(int port) { } byte[] privateKey = new byte[32]; rnd.nextBytes(privateKey); - ECKeyPair ecKeyPair = ECKeyPair.create(privateKey); - final BytesValue pubKey = - BytesValue.wrap(Utils.extractBytesFromUnsignedBigInt(ecKeyPair.getPublicKey())); + + NodeRecordFactory nodeRecordFactory = + verification ? NODE_RECORD_FACTORY : NODE_RECORD_FACTORY_NO_VERIFICATION; NodeRecord nodeRecord = - NODE_RECORD_FACTORY_NO_VERIFICATION.createFromValues( - EnrScheme.V4, + nodeRecordFactory.createFromValues( UInt64.valueOf(1), Bytes96.EMPTY, new ArrayList>() { { + add(Pair.with(FIELD_ID, EnrScheme.V4)); add(Pair.with(NodeRecord.FIELD_IP_V4, finalLocalIp)); add(Pair.with(NodeRecord.FIELD_UDP_V4, port)); - add(Pair.with(NodeRecord.FIELD_PKEY_SECP256K1, pubKey)); + add( + Pair.with( + NodeRecord.FIELD_PKEY_SECP256K1, + Functions.derivePublicKeyFromPrivate(BytesValue.wrap(privateKey)))); } }); + + nodeRecord.sign(BytesValue.wrap(privateKey)); return Pair.with(BytesValue.wrap(privateKey), nodeRecord); } } diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java b/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java index 4549aad79..e6f57f9ba 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java @@ -30,6 +30,7 @@ import org.ethereum.beacon.discovery.storage.AuthTagRepository; import org.ethereum.beacon.discovery.storage.NodeBucketStorage; import org.ethereum.beacon.discovery.storage.NodeTable; +import org.ethereum.beacon.discovery.task.TaskOptions; import org.ethereum.beacon.discovery.task.TaskType; import org.ethereum.beacon.schedulers.Scheduler; import org.reactivestreams.Publisher; @@ -108,15 +109,27 @@ public void start() { public void stop() {} public CompletableFuture executeTask(NodeRecord nodeRecord, TaskType taskType) { + return executeTaskImpl(nodeRecord, taskType, true); + } + + public CompletableFuture executeTaskImpl( + NodeRecord nodeRecord, TaskType taskType, boolean livenessUpdate) { Envelope envelope = new Envelope(); envelope.put(Field.NODE, nodeRecord); CompletableFuture future = new CompletableFuture<>(); envelope.put(Field.TASK, taskType); envelope.put(Field.FUTURE, future); + envelope.put(Field.TASK_OPTIONS, new TaskOptions(livenessUpdate)); outgoingPipeline.push(envelope); return future; } + @Override + public CompletableFuture executeTaskWithoutLivenessUpdate( + NodeRecord nodeRecord, TaskType taskType) { + return executeTaskImpl(nodeRecord, taskType, false); + } + public Publisher getOutgoingMessages() { return outgoingMessages; } diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/mock/EnrSchemeV4InterpreterMock.java b/discovery/src/test/java/org/ethereum/beacon/discovery/mock/EnrSchemeV4InterpreterMock.java index 42e4ce101..2e2193324 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/mock/EnrSchemeV4InterpreterMock.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/mock/EnrSchemeV4InterpreterMock.java @@ -2,10 +2,16 @@ import org.ethereum.beacon.discovery.enr.EnrSchemeV4Interpreter; import org.ethereum.beacon.discovery.enr.NodeRecord; +import tech.pegasys.artemis.util.bytes.Bytes96; public class EnrSchemeV4InterpreterMock extends EnrSchemeV4Interpreter { @Override public void verify(NodeRecord nodeRecord) { - // Don't verify ECDSA + // Don't verify signature + } + + @Override + public void sign(NodeRecord nodeRecord, Object signOptions) { + nodeRecord.setSignature(Bytes96.ZERO); } } diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeBucketTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeBucketTest.java index 0d7853350..affda3dae 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeBucketTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeBucketTest.java @@ -5,18 +5,9 @@ import org.ethereum.beacon.discovery.NodeRecordInfo; import org.ethereum.beacon.discovery.NodeStatus; import org.ethereum.beacon.discovery.TestUtil; -import org.ethereum.beacon.discovery.enr.EnrScheme; import org.ethereum.beacon.discovery.enr.NodeRecord; -import org.javatuples.Pair; import org.junit.Test; -import tech.pegasys.artemis.util.bytes.Bytes4; -import tech.pegasys.artemis.util.bytes.Bytes96; -import tech.pegasys.artemis.util.bytes.BytesValue; -import tech.pegasys.artemis.util.uint.UInt64; -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.util.ArrayList; import java.util.Random; import java.util.stream.IntStream; @@ -28,35 +19,15 @@ public class NodeBucketTest { private final Random rnd = new Random(); - private NodeRecordInfo generateUniqueRecord() { - try { - byte[] pkey = new byte[33]; - rnd.nextBytes(pkey); - NodeRecord nodeRecord = - TestUtil.NODE_RECORD_FACTORY_NO_VERIFICATION.createFromValues( - EnrScheme.V4, - UInt64.valueOf(1), - Bytes96.EMPTY, - new ArrayList>() { - { - add( - Pair.with( - NodeRecord.FIELD_IP_V4, - Bytes4.wrap(InetAddress.getByName("127.0.0.1").getAddress()))); - add(Pair.with(NodeRecord.FIELD_UDP_V4, 30303)); - add(Pair.with(NodeRecord.FIELD_PKEY_SECP256K1, BytesValue.wrap(pkey))); - } - }); - return new NodeRecordInfo(nodeRecord, (long) rnd.nextInt(1000), NodeStatus.ACTIVE, 0); - } catch (UnknownHostException e) { - throw new RuntimeException(e); - } + private NodeRecordInfo generateUniqueRecord(int portInc) { + NodeRecord nodeRecord = TestUtil.generateUnverifiedNode(30303 + portInc).getValue1(); + return new NodeRecordInfo(nodeRecord, 0L, NodeStatus.ACTIVE, 0); } @Test public void testBucket() { NodeBucket nodeBucket = new NodeBucket(); - IntStream.range(0, 20).forEach(value -> nodeBucket.put(generateUniqueRecord())); + IntStream.range(0, 20).forEach(value -> nodeBucket.put(generateUniqueRecord(value))); assertEquals(NodeBucket.K, nodeBucket.size()); assertEquals(NodeBucket.K, nodeBucket.getNodeRecords().size()); @@ -67,11 +38,11 @@ public void testBucket() { lastRetrySaved = nodeRecordInfo.getLastRetry(); } NodeRecordInfo willNotInsertNode = - new NodeRecordInfo(generateUniqueRecord().getNode(), -2L, NodeStatus.ACTIVE, 0); + new NodeRecordInfo(generateUniqueRecord(25).getNode(), -2L, NodeStatus.ACTIVE, 0); nodeBucket.put(willNotInsertNode); assertFalse(nodeBucket.contains(willNotInsertNode)); NodeRecordInfo willInsertNode = - new NodeRecordInfo(generateUniqueRecord().getNode(), 1001L, NodeStatus.ACTIVE, 0); + new NodeRecordInfo(generateUniqueRecord(26).getNode(), 1001L, NodeStatus.ACTIVE, 0); NodeRecordInfo top = nodeBucket.getNodeRecords().get(NodeBucket.K - 1); // latest retry should be kept NodeRecordInfo bottom = nodeBucket.getNodeRecords().get(0); @@ -82,43 +53,46 @@ public void testBucket() { NodeRecordInfo willInsertNode2 = new NodeRecordInfo(willInsertNode.getNode(), 1002L, NodeStatus.ACTIVE, 0); nodeBucket.put(willInsertNode2); // replaces willInsertNode with better last retry - assertEquals(willInsertNode2, nodeBucket.getNodeRecords().get(NodeBucket.K - 1)); + assertTrue(nodeBucket.getNodeRecords().contains(willInsertNode2)); NodeRecordInfo willNotInsertNode3 = new NodeRecordInfo(willInsertNode.getNode(), 999L, NodeStatus.ACTIVE, 0); nodeBucket.put(willNotInsertNode3); // does not replace willInsertNode with worse last retry - assertEquals(willInsertNode2, nodeBucket.getNodeRecords().get(NodeBucket.K - 1)); // still 2nd - assertEquals(top, nodeBucket.getNodeRecords().get(NodeBucket.K - 2)); + assertTrue(nodeBucket.getNodeRecords().contains(willInsertNode2)); + assertTrue(nodeBucket.getNodeRecords().contains(top)); NodeRecordInfo willInsertNodeDead = new NodeRecordInfo(willInsertNode.getNode(), 1001L, NodeStatus.DEAD, 0); nodeBucket.put(willInsertNodeDead); // removes willInsertNode assertEquals(NodeBucket.K - 1, nodeBucket.size()); - assertFalse(nodeBucket.contains(willInsertNode)); + assertFalse(nodeBucket.contains(willInsertNode2)); } @Test public void testStorage() { - NodeRecordInfo initial = generateUniqueRecord(); + NodeRecordInfo initial = generateUniqueRecord(0); Database database = Database.inMemoryDB(); NodeTableStorageFactoryImpl nodeTableStorageFactory = new NodeTableStorageFactoryImpl(); NodeBucketStorage nodeBucketStorage = nodeTableStorageFactory.createBucketStorage(database, TEST_SERIALIZER, initial.getNode()); + int j = 1; for (int i = 0; i < 20; ) { - NodeRecordInfo nodeRecordInfo = generateUniqueRecord(); + NodeRecordInfo nodeRecordInfo = generateUniqueRecord(j); if (Functions.logDistance(initial.getNode().getNodeId(), nodeRecordInfo.getNode().getNodeId()) == 255) { nodeBucketStorage.put(nodeRecordInfo); ++i; } + ++j; } for (int i = 0; i < 3; ) { - NodeRecordInfo nodeRecordInfo = generateUniqueRecord(); + NodeRecordInfo nodeRecordInfo = generateUniqueRecord(j); if (Functions.logDistance(initial.getNode().getNodeId(), nodeRecordInfo.getNode().getNodeId()) == 254) { nodeBucketStorage.put(nodeRecordInfo); ++i; } + ++j; } assertEquals(16, nodeBucketStorage.get(255).get().size()); assertEquals(3, nodeBucketStorage.get(254).get().size()); diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeTableTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeTableTest.java index 1358813c8..2d252bde9 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeTableTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeTableTest.java @@ -3,18 +3,13 @@ import org.ethereum.beacon.db.Database; import org.ethereum.beacon.discovery.NodeRecordInfo; import org.ethereum.beacon.discovery.NodeStatus; -import org.ethereum.beacon.discovery.enr.EnrScheme; +import org.ethereum.beacon.discovery.TestUtil; import org.ethereum.beacon.discovery.enr.NodeRecord; -import org.javatuples.Pair; import org.junit.Test; import tech.pegasys.artemis.util.bytes.Bytes32; -import tech.pegasys.artemis.util.bytes.Bytes4; -import tech.pegasys.artemis.util.bytes.Bytes96; import tech.pegasys.artemis.util.bytes.BytesValue; import tech.pegasys.artemis.util.uint.UInt64; -import java.net.InetAddress; -import java.net.UnknownHostException; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -28,44 +23,21 @@ import static org.junit.Assert.assertTrue; public class NodeTableTest { - private Function homeNodeSupplier = - (oldSeq) -> { - try { - return NODE_RECORD_FACTORY_NO_VERIFICATION.createFromValues( - EnrScheme.V4, - UInt64.valueOf(1), - Bytes96.EMPTY, - new ArrayList>() { - { - add( - Pair.with( - NodeRecord.FIELD_IP_V4, - Bytes4.wrap(InetAddress.getByName("127.0.0.1").getAddress()))); - add(Pair.with(NodeRecord.FIELD_UDP_V4, 30303)); - add( - Pair.with( - NodeRecord.FIELD_PKEY_SECP256K1, - BytesValue.fromHexString( - "0bfb48004b1698f05872cf18b1f278998ad8f7d4c135aa41f83744e7b850ab6b98"))); - } - }); - } catch (UnknownHostException e) { - throw new RuntimeException(e); - } - }; + final String LOCALHOST_BASE64 = + "-IS4QHCYrYZbAKWCBRlAy5zzaDZXJBGkcnh4MHcBFZntXNFrdvJjX04jRzjzCBOonrkTfj499SZuOh8R33Ls8RRcy5wBgmlkgnY0gmlwhH8AAAGJc2VjcDI1NmsxoQMRo9bfkceoY0W04hSgYU5Q1R_mmq3Qp9pBPMAIduKrAYN1ZHCCdl8="; + private Function HOME_NODE_SUPPLIER = + (oldSeq) -> TestUtil.generateUnverifiedNode(30303).getValue1(); @Test public void testCreate() throws Exception { - final String localhostEnr = - "-IS4QHCYrYZbAKWCBRlAy5zzaDZXJBGkcnh4MHcBFZntXNFrdvJjX04jRzjzCBOonrkTfj499SZuOh8R33Ls8RRcy5wBgmlkgnY0gmlwhH8AAAGJc2VjcDI1NmsxoQPKY0yuDUmstAHYpMa2_oxVtw0RW_QAdpzBQA8yWM0xOIN1ZHCCdl8"; - NodeRecord nodeRecord = NODE_RECORD_FACTORY_NO_VERIFICATION.fromBase64(localhostEnr); + NodeRecord nodeRecord = NODE_RECORD_FACTORY_NO_VERIFICATION.fromBase64(LOCALHOST_BASE64); NodeTableStorageFactoryImpl nodeTableStorageFactory = new NodeTableStorageFactoryImpl(); Database database = Database.inMemoryDB(); NodeTableStorage nodeTableStorage = nodeTableStorageFactory.createTable( database, TEST_SERIALIZER, - homeNodeSupplier, + HOME_NODE_SUPPLIER, () -> { List nodes = new ArrayList<>(); nodes.add(nodeRecord); @@ -79,21 +51,19 @@ public void testCreate() throws Exception { nodeRecord2.getNode().get(NodeRecord.FIELD_PKEY_SECP256K1)); assertEquals( nodeTableStorage.get().getHomeNode().getNodeId(), - homeNodeSupplier.apply(UInt64.ZERO).getNodeId()); + HOME_NODE_SUPPLIER.apply(UInt64.ZERO).getNodeId()); } @Test public void testFind() throws Exception { - final String localhostEnr = - "-IS4QHCYrYZbAKWCBRlAy5zzaDZXJBGkcnh4MHcBFZntXNFrdvJjX04jRzjzCBOonrkTfj499SZuOh8R33Ls8RRcy5wBgmlkgnY0gmlwhH8AAAGJc2VjcDI1NmsxoQPKY0yuDUmstAHYpMa2_oxVtw0RW_QAdpzBQA8yWM0xOIN1ZHCCdl8"; - NodeRecord localHostNode = NODE_RECORD_FACTORY_NO_VERIFICATION.fromBase64(localhostEnr); + NodeRecord localHostNode = NODE_RECORD_FACTORY_NO_VERIFICATION.fromBase64(LOCALHOST_BASE64); NodeTableStorageFactoryImpl nodeTableStorageFactory = new NodeTableStorageFactoryImpl(); Database database = Database.inMemoryDB(); NodeTableStorage nodeTableStorage = nodeTableStorageFactory.createTable( database, TEST_SERIALIZER, - homeNodeSupplier, + HOME_NODE_SUPPLIER, () -> { List nodes = new ArrayList<>(); nodes.add(localHostNode); @@ -101,25 +71,7 @@ public void testFind() throws Exception { }); // node is adjusted to be close to localhostEnr - NodeRecord closestNode = - NODE_RECORD_FACTORY_NO_VERIFICATION.createFromValues( - EnrScheme.V4, - UInt64.valueOf(1), - Bytes96.EMPTY, - new ArrayList>() { - { - add( - Pair.with( - NodeRecord.FIELD_IP_V4, - Bytes4.wrap(InetAddress.getByName("127.0.0.2").getAddress()))); - add(Pair.with(NodeRecord.FIELD_UDP_V4, 30303)); - add( - Pair.with( - NodeRecord.FIELD_PKEY_SECP256K1, - BytesValue.fromHexString( - "aafb48004b1698f05872cf18b1f278998ad8f7d4c135aa41f83744e7b850ab6b98"))); - } - }); + NodeRecord closestNode = TestUtil.generateUnverifiedNode(30267).getValue1(); nodeTableStorage.get().save(new NodeRecordInfo(closestNode, -1L, NodeStatus.ACTIVE, 0)); assertEquals( nodeTableStorage @@ -129,28 +81,11 @@ public void testFind() throws Exception { .getNode() .get(NodeRecord.FIELD_PKEY_SECP256K1), closestNode.get(NodeRecord.FIELD_PKEY_SECP256K1)); - NodeRecord farNode = - NODE_RECORD_FACTORY_NO_VERIFICATION.createFromValues( - EnrScheme.V4, - UInt64.valueOf(1), - Bytes96.EMPTY, - new ArrayList>() { - { - add( - Pair.with( - NodeRecord.FIELD_IP_V4, - Bytes4.wrap(InetAddress.getByName("127.0.0.3").getAddress()))); - add(Pair.with(NodeRecord.FIELD_UDP_V4, 30303)); - add( - Pair.with( - NodeRecord.FIELD_PKEY_SECP256K1, - BytesValue.fromHexString( - "bafb48004b1698f05872cf18b1f278998ad8f7d4c135aa41f83744e7b850ab6b98"))); - } - }); + // node is adjusted to be far from localhostEnr + NodeRecord farNode = TestUtil.generateUnverifiedNode(30304).getValue1(); nodeTableStorage.get().save(new NodeRecordInfo(farNode, -1L, NodeStatus.ACTIVE, 0)); List closestNodes = - nodeTableStorage.get().findClosestNodes(closestNode.getNodeId(), 252); + nodeTableStorage.get().findClosestNodes(closestNode.getNodeId(), 254); assertEquals(2, closestNodes.size()); Set publicKeys = new HashSet<>(); closestNodes.forEach( diff --git a/util/src/main/java/org/ethereum/beacon/util/Utils.java b/util/src/main/java/org/ethereum/beacon/util/Utils.java index e2c565e41..9f3d4c46b 100644 --- a/util/src/main/java/org/ethereum/beacon/util/Utils.java +++ b/util/src/main/java/org/ethereum/beacon/util/Utils.java @@ -42,22 +42,28 @@ protected boolean removeEldestEntry(Map.Entry eldest) { } /** + * @param size required size, in bytes * @return byte array representation of BigInteger for unsigned numeric *

{@link BigInteger#toByteArray()} adds a bit for the sign. If you work with unsigned * numerics it's always a 0. But if an integer uses exactly 8-some bits, sign bit will add an * extra 0 byte to the result, which could broke some things. This method removes this * redundant prefix byte when extracting byte array from BigInteger */ - public static byte[] extractBytesFromUnsignedBigInt(BigInteger bigInteger) { + public static byte[] extractBytesFromUnsignedBigInt(BigInteger bigInteger, int size) { byte[] bigIntBytes = bigInteger.toByteArray(); - byte[] res; - if (bigIntBytes[0] == 0) { - res = new byte[bigIntBytes.length - 1]; + if (bigIntBytes.length == size) { + return bigIntBytes; + } else if (bigIntBytes.length == (size + 1)) { + byte[] res = new byte[size]; System.arraycopy(bigIntBytes, 1, res, 0, res.length); + return res; + } else if (bigIntBytes.length < size) { + byte[] res = new byte[size]; + System.arraycopy(bigIntBytes, 0, res, size - bigIntBytes.length, bigIntBytes.length); + return res; } else { - res = bigIntBytes; + throw new RuntimeException( + String.format("Cannot extract bytes of size %s from BigInteger [%s]", size, bigInteger)); } - - return res; } } From f29395fb202d6041e191cfb6ea81944c1eeb49ab Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Tue, 19 Nov 2019 13:57:19 +0300 Subject: [PATCH 66/77] discovery: fixed maximum packet size verification + nodes sent in findnodes reply --- .../message/handler/FindNodeHandler.java | 49 ++++++++++--------- .../discovery/packet/UnknownPacket.java | 7 +++ .../pipeline/handler/IncomingDataPacker.java | 17 +++++-- 3 files changed, 46 insertions(+), 27 deletions(-) diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/FindNodeHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/FindNodeHandler.java index 49b4b5c17..d5878e27f 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/FindNodeHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/FindNodeHandler.java @@ -15,44 +15,45 @@ import java.util.Collections; import java.util.List; import java.util.Optional; -import java.util.stream.Collectors; -import java.util.stream.IntStream; public class FindNodeHandler implements MessageHandler { private static final Logger logger = LogManager.getLogger(FindNodeHandler.class); - private static final int MAX_NODES_PER_MESSAGE = 12; + /** + * The maximum size of any packet is 1280 bytes. Implementations should not generate or process + * packets larger than this size. As per specification the maximum size of an ENR is 300 bytes. A + * NODES message containing all FINDNODE response records would be at least 4800 bytes, not + * including additional data such as the header. To stay below the size limit, NODES responses are + * sent as multiple messages and specify the total number of responses in the message. 4х300 = + * 1200 and we always have 80 bytes for everything else. + */ + private static final int MAX_NODES_PER_MESSAGE = 4; public FindNodeHandler() {} @Override public void handle(FindNodeMessage message, NodeSession session) { - int start = - message.getDistance() == 0 ? 0 : 1; // home node from 0 should be returned only for 0 - List nodeBuckets = - IntStream.range(start, message.getDistance()) - .mapToObj(session::getBucket) - .filter(Optional::isPresent) - .map(Optional::get) - .collect(Collectors.toList()); - logger.trace( - () -> - String.format( - "Sending %s nodeBuckets in reply to request in session %s", - nodeBuckets.size(), session)); + Optional nodeBucketOptional = session.getBucket(message.getDistance()); List> nodeRecordsList = new ArrayList<>(); int total = 0; // Repack to lists of MAX_NODES_PER_MESSAGE size - for (NodeBucket nodeBucket : nodeBuckets) { - for (NodeRecordInfo nodeRecordInfo : nodeBucket.getNodeRecords()) { - if (total % MAX_NODES_PER_MESSAGE == 0) { - nodeRecordsList.add(new ArrayList<>()); - } - List currentList = nodeRecordsList.get(nodeRecordsList.size() - 1); - currentList.add(nodeRecordInfo.getNode()); - ++total; + List bucketRecords = + nodeBucketOptional.isPresent() + ? nodeBucketOptional.get().getNodeRecords() + : Collections.emptyList(); + for (NodeRecordInfo nodeRecordInfo : bucketRecords) { + if (total % MAX_NODES_PER_MESSAGE == 0) { + nodeRecordsList.add(new ArrayList<>()); } + List currentList = nodeRecordsList.get(nodeRecordsList.size() - 1); + currentList.add(nodeRecordInfo.getNode()); + ++total; } + logger.trace( + () -> + String.format( + "Sending %s nodes in reply to request with distance %s in session %s", + bucketRecords.size(), message.getDistance(), session)); // Send if (nodeRecordsList.isEmpty()) { diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/UnknownPacket.java b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/UnknownPacket.java index 2a17c9248..84148408f 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/UnknownPacket.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/UnknownPacket.java @@ -7,6 +7,7 @@ /** Default packet form until its goal is known */ public class UnknownPacket extends AbstractPacket { + private static final int MAX_SIZE = 1280; public UnknownPacket(BytesValue bytes) { super(bytes); @@ -45,6 +46,12 @@ public Bytes32 getSourceNodeId(Bytes32 destNodeId) { return Bytes32s.xor(Hashes.sha256(destNodeId), Bytes32.wrap(xorTag, 0)); } + public void verify() { + if (getBytes().size() > MAX_SIZE) { + throw new RuntimeException(String.format("Packets should not exceed %s bytes", MAX_SIZE)); + } + } + @Override public String toString() { return "UnknownPacket{" diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/IncomingDataPacker.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/IncomingDataPacker.java index 5cfb7df9e..3b0ece51e 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/IncomingDataPacker.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/IncomingDataPacker.java @@ -30,9 +30,20 @@ public void handle(Envelope envelope) { envelope.getId())); UnknownPacket unknownPacket = new UnknownPacket((BytesValue) envelope.get(Field.INCOMING)); - envelope.put(Field.PACKET_UNKNOWN, unknownPacket); - logger.trace( - () -> String.format("Incoming packet %s in envelope #%s", unknownPacket, envelope.getId())); + try { + unknownPacket.verify(); + envelope.put(Field.PACKET_UNKNOWN, unknownPacket); + logger.trace( + () -> + String.format("Incoming packet %s in envelope #%s", unknownPacket, envelope.getId())); + } catch(Exception ex) { + envelope.put(Field.BAD_PACKET, unknownPacket); + envelope.put(Field.BAD_EXCEPTION, ex); + envelope.put(Field.BAD_MESSAGE, "Incoming packet verification not passed"); + logger.trace( + () -> + String.format("Bad incoming packet %s in envelope #%s", unknownPacket, envelope.getId())); + } envelope.remove(Field.INCOMING); } } From 2bf36a7976e8ef1a03e1b789787ed9482a8abf51 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Tue, 19 Nov 2019 18:59:12 +0300 Subject: [PATCH 67/77] discovery: fixes encoding flow, plus new task api --- .../beacon/discovery/DiscoveryManager.java | 17 ++-- .../discovery/DiscoveryManagerImpl.java | 20 ++--- .../ethereum/beacon/discovery/Functions.java | 16 ++-- .../beacon/discovery/NodeSession.java | 79 ++++--------------- .../discovery/enr/EnrSchemeV4Interpreter.java | 8 +- .../message/handler/NodesHandler.java | 68 +++++----------- .../packet/AuthHeaderMessagePacket.java | 9 ++- .../discovery/packet/MessagePacket.java | 4 +- .../AuthHeaderMessagePacketHandler.java | 7 +- .../handler/MessagePacketHandler.java | 2 +- .../pipeline/handler/NewTaskHandler.java | 14 +--- .../pipeline/handler/NextTaskHandler.java | 10 ++- .../handler/WhoAreYouPacketHandler.java | 6 +- .../pipeline/info/FindNodeRequestInfo.java | 42 ++++++++++ .../pipeline/info/GeneralRequestInfo.java | 57 +++++++++++++ .../discovery/pipeline/info/RequestInfo.java | 17 ++++ .../pipeline/info/RequestInfoFactory.java | 30 +++++++ .../discovery/task/DiscoveryTaskManager.java | 5 +- .../beacon/discovery/task/LiveCheckTasks.java | 6 +- .../discovery/task/RecursiveLookupTasks.java | 6 +- .../discovery/task/TaskMessageFactory.java | 24 +++--- .../beacon/discovery/task/TaskOptions.java | 10 +++ .../discovery/DiscoveryInteropTest.java | 12 ++- .../discovery/DiscoveryNetworkTest.java | 3 +- .../discovery/DiscoveryNoNetworkTest.java | 3 +- .../beacon/discovery/FunctionsTest.java | 4 +- .../discovery/HandshakeHandlersTest.java | 7 +- .../mock/DiscoveryManagerNoNetwork.java | 20 ++--- 28 files changed, 289 insertions(+), 217 deletions(-) create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/info/FindNodeRequestInfo.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/info/GeneralRequestInfo.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/info/RequestInfo.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/info/RequestInfoFactory.java diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManager.java b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManager.java index 5540c320e..c2a1d843f 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManager.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManager.java @@ -1,7 +1,6 @@ package org.ethereum.beacon.discovery; import org.ethereum.beacon.discovery.enr.NodeRecord; -import org.ethereum.beacon.discovery.task.TaskType; import java.util.concurrent.CompletableFuture; @@ -15,19 +14,21 @@ public interface DiscoveryManager { void stop(); /** - * Initiates task of task type with node `nodeRecord` + * Initiates FINDNODE with node `nodeRecord` * * @param nodeRecord Ethereum Node record - * @param taskType Task type + * @param distance Distance to search for * @return Future which is fired when reply is received or fails in timeout/not successful * handshake/bad message exchange. */ - CompletableFuture executeTask(NodeRecord nodeRecord, TaskType taskType); + CompletableFuture findNodes(NodeRecord nodeRecord, int distance); /** - * Same as {@link #executeTask(NodeRecord, TaskType)} but doesn't update node liveness status in - * successful case + * Initiates PING with node `nodeRecord` + * + * @param nodeRecord Ethereum Node record + * @return Future which is fired when reply is received or fails in timeout/not successful + * handshake/bad message exchange. */ - CompletableFuture executeTaskWithoutLivenessUpdate( - NodeRecord nodeRecord, TaskType taskType); + CompletableFuture ping(NodeRecord nodeRecord); } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java index 5ae97e9b8..3c1c1f15e 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java @@ -127,26 +127,26 @@ public void stop() { discoveryServer.stop(); } - public CompletableFuture executeTask(NodeRecord nodeRecord, TaskType taskType) { - return executeTaskImpl(nodeRecord, taskType, true); - } - - public CompletableFuture executeTaskImpl( - NodeRecord nodeRecord, TaskType taskType, boolean livenessUpdate) { + private CompletableFuture executeTaskImpl( + NodeRecord nodeRecord, TaskType taskType, TaskOptions taskOptions) { Envelope envelope = new Envelope(); envelope.put(Field.NODE, nodeRecord); CompletableFuture future = new CompletableFuture<>(); envelope.put(Field.TASK, taskType); envelope.put(Field.FUTURE, future); - envelope.put(Field.TASK_OPTIONS, new TaskOptions(livenessUpdate)); + envelope.put(Field.TASK_OPTIONS, taskOptions); outgoingPipeline.push(envelope); return future; } @Override - public CompletableFuture executeTaskWithoutLivenessUpdate( - NodeRecord nodeRecord, TaskType taskType) { - return executeTaskImpl(nodeRecord, taskType, false); + public CompletableFuture findNodes(NodeRecord nodeRecord, int distance) { + return executeTaskImpl(nodeRecord, TaskType.FINDNODE, new TaskOptions(true, distance)); + } + + @Override + public CompletableFuture ping(NodeRecord nodeRecord) { + return executeTaskImpl(nodeRecord, TaskType.PING, new TaskOptions(true)); } @VisibleForTesting diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java b/discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java index 885928a6b..3b6c3120a 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java @@ -12,6 +12,7 @@ import org.ethereum.beacon.util.Utils; import org.web3j.crypto.ECDSASignature; import org.web3j.crypto.ECKeyPair; +import org.web3j.crypto.Hash; import org.web3j.crypto.Sign; import tech.pegasys.artemis.util.bytes.Bytes32; import tech.pegasys.artemis.util.bytes.Bytes32s; @@ -41,17 +42,20 @@ public static Bytes32 hash(BytesValue value) { return Hashes.sha256(value); } + public static Bytes32 hashKeccak(BytesValue value) { + return Bytes32.wrap(Hash.sha3(value.extractArray())); + } + /** * Creates a signature of message `x` using the given key. * * @param key private key - * @param x message, not hashed + * @param x message, hashed * @return ECDSA signature with properties merged together: r || s */ public static BytesValue sign(BytesValue key, BytesValue x) { - BytesValue hash = Functions.hash(x); Sign.SignatureData signatureData = - Sign.signMessage(hash.extractArray(), ECKeyPair.create(key.extractArray()), false); + Sign.signMessage(x.extractArray(), ECKeyPair.create(key.extractArray()), false); Bytes32 r = Bytes32.wrap(signatureData.getR()); Bytes32 s = Bytes32.wrap(signatureData.getS()); return r.concat(s); @@ -61,7 +65,7 @@ public static BytesValue sign(BytesValue key, BytesValue x) { * Verifies that signature is made by signer * * @param signature Signature, ECDSA - * @param x message, not hashed + * @param x message, hashed * @param pubKey Public key of supposed signer, compressed, 33 bytes * @return whether `signature` reflects message `x` signed with `pubkey` */ @@ -74,9 +78,9 @@ public static boolean verifyECDSASignature( new ECDSASignature( new BigInteger(1, signature.slice(0, 32).extractArray()), new BigInteger(1, signature.slice(32).extractArray())); - byte[] msgHash = Functions.hash(x).extractArray(); for (int recId = 0; recId < 4; ++recId) { - BigInteger calculatedPubKey = Sign.recoverFromSignature(recId, ecdsaSignature, msgHash); + BigInteger calculatedPubKey = + Sign.recoverFromSignature(recId, ecdsaSignature, x.extractArray()); if (calculatedPubKey == null) { continue; } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeSession.java b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeSession.java index 030cb9deb..a5c4cfe43 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeSession.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeSession.java @@ -4,11 +4,13 @@ import org.apache.logging.log4j.Logger; import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.packet.Packet; +import org.ethereum.beacon.discovery.pipeline.info.RequestInfo; +import org.ethereum.beacon.discovery.pipeline.info.RequestInfoFactory; import org.ethereum.beacon.discovery.storage.AuthTagRepository; import org.ethereum.beacon.discovery.storage.NodeBucket; import org.ethereum.beacon.discovery.storage.NodeBucketStorage; import org.ethereum.beacon.discovery.storage.NodeTable; -import org.ethereum.beacon.discovery.task.TaskStatus; +import org.ethereum.beacon.discovery.task.TaskOptions; import org.ethereum.beacon.discovery.task.TaskType; import org.ethereum.beacon.util.ExpirationScheduler; import tech.pegasys.artemis.util.bytes.Bytes32; @@ -109,16 +111,25 @@ public void sendOutgoing(Packet packet) { * saved state in all circumstances. The easiest to implement is a random number. * * @param taskType Type of task, clarifies starting and reply message types - * @param future Future to be fired when task is succcessfully completed or exceptionally break + * @param taskOptions Task options + * @param future Future to be fired when task is successfully completed or exceptionally break * when its failed * @return info bundle. */ public synchronized RequestInfo createNextRequest( - TaskType taskType, CompletableFuture future) { + TaskType taskType, TaskOptions taskOptions, CompletableFuture future) { byte[] requestId = new byte[REQUEST_ID_SIZE]; rnd.nextBytes(requestId); BytesValue wrappedId = BytesValue.wrap(requestId); - RequestInfo requestInfo = new GeneralRequestInfo(taskType, AWAIT, wrappedId, future); + if (taskOptions.isLivenessUpdate()) { + future.whenComplete( + (aVoid, throwable) -> { + if (throwable == null) { + updateLiveness(); + } + }); + } + RequestInfo requestInfo = RequestInfoFactory.create(taskType, wrappedId, taskOptions, future); requestIdStatuses.put(wrappedId, requestInfo); requestExpirationScheduler.put( wrappedId, @@ -306,64 +317,4 @@ public enum SessionStatus { RANDOM_PACKET_SENT, // our node is initiator, we've sent random packet AUTHENTICATED } - - public interface RequestInfo { - TaskType getTaskType(); - - TaskStatus getTaskStatus(); - - BytesValue getRequestId(); - - CompletableFuture getFuture(); - } - - public static class GeneralRequestInfo implements RequestInfo { - private final TaskType taskType; - private final TaskStatus taskStatus; - private final BytesValue requestId; - private final CompletableFuture future; - - public GeneralRequestInfo( - TaskType taskType, - TaskStatus taskStatus, - BytesValue requestId, - CompletableFuture future) { - this.taskType = taskType; - this.taskStatus = taskStatus; - this.requestId = requestId; - this.future = future; - } - - @Override - public TaskType getTaskType() { - return taskType; - } - - @Override - public TaskStatus getTaskStatus() { - return taskStatus; - } - - @Override - public BytesValue getRequestId() { - return requestId; - } - - @Override - public CompletableFuture getFuture() { - return future; - } - - @Override - public String toString() { - return "GeneralRequestInfo{" - + "taskType=" - + taskType - + ", taskStatus=" - + taskStatus - + ", requestId=" - + requestId - + '}'; - } - } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrSchemeV4Interpreter.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrSchemeV4Interpreter.java index f0fe255c6..cea0d858d 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrSchemeV4Interpreter.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrSchemeV4Interpreter.java @@ -3,7 +3,6 @@ import org.bouncycastle.math.ec.ECPoint; import org.ethereum.beacon.discovery.Functions; import org.ethereum.beacon.util.Utils; -import org.web3j.crypto.Hash; import org.web3j.rlp.RlpString; import tech.pegasys.artemis.util.bytes.Bytes16; import tech.pegasys.artemis.util.bytes.Bytes32; @@ -52,7 +51,7 @@ public void verify(NodeRecord nodeRecord) { } BytesValue pubKey = (BytesValue) nodeRecord.get(FIELD_PKEY_SECP256K1); // compressed assert Functions.verifyECDSASignature( - nodeRecord.getSignature(), nodeRecord.serialize(false), pubKey); + nodeRecord.getSignature(), Functions.hashKeccak(nodeRecord.serialize(false)), pubKey); } @Override @@ -71,7 +70,7 @@ public Bytes32 getNodeId(NodeRecord nodeRecord) { BytesValue yPart = Bytes32.wrap( Utils.extractBytesFromUnsignedBigInt(pudDestPoint.getYCoord().toBigInteger(), 32)); - return Bytes32.wrap(Hash.sha3(xPart.concat(yPart).extractArray())); + return Functions.hashKeccak(xPart.concat(yPart)); } @Override @@ -86,7 +85,8 @@ public Object decode(String key, RlpString rlpString) { @Override public void sign(NodeRecord nodeRecord, Object signOptions) { BytesValue privateKey = (BytesValue) signOptions; - BytesValue signature = Functions.sign(privateKey, nodeRecord.serialize(false)); + BytesValue signature = + Functions.sign(privateKey, Functions.hashKeccak(nodeRecord.serialize(false))); nodeRecord.setSignature(signature); } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/NodesHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/NodesHandler.java index fc3e58a56..70af0f45c 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/NodesHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/NodesHandler.java @@ -5,12 +5,12 @@ import org.ethereum.beacon.discovery.NodeRecordInfo; import org.ethereum.beacon.discovery.NodeSession; import org.ethereum.beacon.discovery.message.NodesMessage; +import org.ethereum.beacon.discovery.pipeline.info.FindNodeRequestInfo; +import org.ethereum.beacon.discovery.pipeline.info.RequestInfo; import org.ethereum.beacon.discovery.task.TaskStatus; import org.ethereum.beacon.discovery.task.TaskType; -import tech.pegasys.artemis.util.bytes.BytesValue; import java.util.Optional; -import java.util.concurrent.CompletableFuture; public class NodesHandler implements MessageHandler { private static final Logger logger = LogManager.getLogger(FindNodeHandler.class); @@ -18,39 +18,29 @@ public class NodesHandler implements MessageHandler { @Override public void handle(NodesMessage message, NodeSession session) { // NODES total count handling - Optional requestInfoOpt = session.getRequestId(message.getRequestId()); + Optional requestInfoOpt = session.getRequestId(message.getRequestId()); if (!requestInfoOpt.isPresent()) { throw new RuntimeException( String.format( "Request #%s not found in session %s when handling message %s", message.getRequestId(), session, message)); } - NodeSession.RequestInfo requestInfo = requestInfoOpt.get(); - if (requestInfo instanceof FindNodeRequestInfo) { - int newNodesCount = ((FindNodeRequestInfo) requestInfo).getRemainingNodes() - 1; - if (newNodesCount == 0) { - session.clearRequestId(message.getRequestId(), TaskType.FINDNODE); - } else { - session.updateRequestInfo( - message.getRequestId(), - new FindNodeRequestInfo( - TaskStatus.IN_PROCESS, - message.getRequestId(), - requestInfo.getFuture(), - newNodesCount)); - } + FindNodeRequestInfo requestInfo = (FindNodeRequestInfo) requestInfoOpt.get(); + int newNodesCount = + requestInfo.getRemainingNodes() == null + ? message.getTotal() - 1 + : requestInfo.getRemainingNodes() - 1; + if (newNodesCount == 0) { + session.clearRequestId(message.getRequestId(), TaskType.FINDNODE); } else { - if (message.getTotal() > 1) { - session.updateRequestInfo( - message.getRequestId(), - new FindNodeRequestInfo( - TaskStatus.IN_PROCESS, - message.getRequestId(), - requestInfo.getFuture(), - message.getTotal() - 1)); - } else { - session.clearRequestId(message.getRequestId(), TaskType.FINDNODE); - } + session.updateRequestInfo( + message.getRequestId(), + new FindNodeRequestInfo( + TaskStatus.IN_PROCESS, + message.getRequestId(), + requestInfo.getFuture(), + requestInfo.getDistance(), + newNodesCount)); } // Parse node records @@ -71,26 +61,4 @@ public void handle(NodesMessage message, NodeSession session) { session.putRecordInBucket(nodeRecordInfo); }); } - - public static class FindNodeRequestInfo extends NodeSession.GeneralRequestInfo { - private final int remainingNodes; - - public FindNodeRequestInfo( - TaskStatus taskStatus, - BytesValue requestId, - CompletableFuture future, - int remainingNodes) { - super(TaskType.FINDNODE, taskStatus, requestId, future); - this.remainingNodes = remainingNodes; - } - - public int getRemainingNodes() { - return remainingNodes; - } - - @Override - public String toString() { - return "FindNodeRequestInfo{" + "remainingNodes=" + remainingNodes + '}'; - } - } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/AuthHeaderMessagePacket.java b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/AuthHeaderMessagePacket.java index 0ec9399ca..25ad3d32c 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/AuthHeaderMessagePacket.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/AuthHeaderMessagePacket.java @@ -66,7 +66,8 @@ public static BytesValue createIdNonceMessage(BytesValue idNonce, BytesValue eph public static BytesValue signIdNonce( BytesValue idNonce, BytesValue staticNodeKey, BytesValue ephemeralPubkey) { BytesValue signed = - Functions.sign(staticNodeKey, createIdNonceMessage(idNonce, ephemeralPubkey)); + Functions.sign( + staticNodeKey, Functions.hash(createIdNonceMessage(idNonce, ephemeralPubkey))); return signed; } @@ -121,7 +122,7 @@ public void verify(BytesValue expectedIdNonce, BytesValue remoteNodePubKey) { assert expectedIdNonce.equals(getIdNonce()); assert Functions.verifyECDSASignature( getIdNonceSig(), - createIdNonceMessage(getIdNonce(), getEphemeralPubkey()), + Functions.hash(createIdNonceMessage(getIdNonce(), getEphemeralPubkey())), remoteNodePubKey); } @@ -195,7 +196,7 @@ public void decodeEphemeralPubKey() { /** Run {@link AuthHeaderMessagePacket#decodeEphemeralPubKey()} before second part */ public void decodeMessage( - BytesValue initiatorKey, BytesValue authResponseKey, NodeRecordFactory nodeRecordFactory) { + BytesValue readKey, BytesValue authResponseKey, NodeRecordFactory nodeRecordFactory) { if (decodedEphemeralPubKeyPt == null) { throw new RuntimeException("Run decodeEphemeralPubKey() before"); } @@ -220,7 +221,7 @@ public void decodeMessage( blank.message = new DiscoveryV5Message( Functions.aesgcm_decrypt( - initiatorKey, + readKey, decodedEphemeralPubKeyPt.authTag, decodedEphemeralPubKeyPt.messageEncrypted, decodedEphemeralPubKeyPt.tag)); diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/MessagePacket.java b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/MessagePacket.java index cfbd8e7bf..2ea612a2b 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/MessagePacket.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/MessagePacket.java @@ -73,7 +73,7 @@ private void verifyDecode() { } } - public void decode(BytesValue initiatorKey) { + public void decode(BytesValue readKey) { if (decoded != null) { return; } @@ -86,7 +86,7 @@ public void decode(BytesValue initiatorKey) { .getBytes()); blank.message = new DiscoveryV5Message( - Functions.aesgcm_decrypt(initiatorKey, blank.authTag, getBytes().slice(45), blank.tag)); + Functions.aesgcm_decrypt(readKey, blank.authTag, getBytes().slice(45), blank.tag)); this.decoded = blank; } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/AuthHeaderMessagePacketHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/AuthHeaderMessagePacketHandler.java index 0d0204304..05a97c7b1 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/AuthHeaderMessagePacketHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/AuthHeaderMessagePacketHandler.java @@ -63,9 +63,10 @@ public void handle(Envelope envelope) { session.getStaticNodeKey(), ephemeralPubKey, session.getIdNonce()); - session.setInitiatorKey(keys.getInitiatorKey()); - session.setRecipientKey(keys.getRecipientKey()); - packet.decodeMessage(keys.getInitiatorKey(), keys.getAuthResponseKey(), nodeRecordFactory); + // Swap keys because we are not initiator, other side is + session.setInitiatorKey(keys.getRecipientKey()); + session.setRecipientKey(keys.getInitiatorKey()); + packet.decodeMessage(session.getRecipientKey(), keys.getAuthResponseKey(), nodeRecordFactory); packet.verify( session.getIdNonce(), (BytesValue) session.getNodeRecord().get(FIELD_PKEY_SECP256K1)); envelope.put(Field.MESSAGE, packet.getMessage()); diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/MessagePacketHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/MessagePacketHandler.java index 312bf8e11..e5809be7c 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/MessagePacketHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/MessagePacketHandler.java @@ -36,7 +36,7 @@ public void handle(Envelope envelope) { NodeSession session = (NodeSession) envelope.get(Field.SESSION); try { - packet.decode(session.getInitiatorKey()); + packet.decode(session.getRecipientKey()); envelope.put(Field.MESSAGE, packet.getMessage()); } catch (AssertionError ex) { logger.error( diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NewTaskHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NewTaskHandler.java index 4af3201de..20d818fc4 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NewTaskHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NewTaskHandler.java @@ -11,7 +11,6 @@ import org.ethereum.beacon.discovery.task.TaskType; import java.util.concurrent.CompletableFuture; -import java.util.function.BiConsumer; /** Enqueues task in session for any task found in {@link Field#TASK} */ public class NewTaskHandler implements EnvelopeHandler { @@ -46,19 +45,8 @@ public void handle(Envelope envelope) { CompletableFuture completableFuture = (CompletableFuture) envelope.get(Field.FUTURE); TaskOptions taskOptions = (TaskOptions) envelope.get(Field.TASK_OPTIONS); - useTaskOptions(session, completableFuture, taskOptions); - session.createNextRequest(task, completableFuture); + session.createNextRequest(task, taskOptions, completableFuture); envelope.remove(Field.TASK); envelope.remove(Field.FUTURE); } - - private void useTaskOptions(NodeSession nodeSession, CompletableFuture completableFuture, TaskOptions taskOptions) { - if (taskOptions.isLivenessUpdate()) { - completableFuture.whenComplete((aVoid, throwable) -> { - if (throwable == null) { - nodeSession.updateLiveness(); - } - }); - } - } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NextTaskHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NextTaskHandler.java index 90c4d9312..8d9467b24 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NextTaskHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NextTaskHandler.java @@ -10,6 +10,8 @@ import org.ethereum.beacon.discovery.pipeline.Field; import org.ethereum.beacon.discovery.pipeline.HandlerUtil; import org.ethereum.beacon.discovery.pipeline.Pipeline; +import org.ethereum.beacon.discovery.pipeline.info.GeneralRequestInfo; +import org.ethereum.beacon.discovery.pipeline.info.RequestInfo; import org.ethereum.beacon.discovery.task.TaskMessageFactory; import org.ethereum.beacon.discovery.task.TaskStatus; import org.ethereum.beacon.schedulers.Scheduler; @@ -57,13 +59,13 @@ public void handle(Envelope envelope) { "Envelope %s in NextTaskHandler, requirements are satisfied!", envelope.getId())); NodeSession session = (NodeSession) envelope.get(Field.SESSION); - Optional requestInfoOpt = session.getFirstAwaitRequestInfo(); + Optional requestInfoOpt = session.getFirstAwaitRequestInfo(); if (!requestInfoOpt.isPresent()) { logger.trace(() -> String.format("Envelope %s: no awaiting requests", envelope.getId())); return; } - NodeSession.RequestInfo requestInfo = requestInfoOpt.get(); + RequestInfo requestInfo = requestInfoOpt.get(); logger.trace( () -> String.format( @@ -84,8 +86,8 @@ public void handle(Envelope envelope) { MessagePacket messagePacket = TaskMessageFactory.createPacketFromRequest(requestInfo, authTag, session); session.sendOutgoing(messagePacket); - NodeSession.RequestInfo sentRequestInfo = - new NodeSession.GeneralRequestInfo( + RequestInfo sentRequestInfo = + new GeneralRequestInfo( requestInfo.getTaskType(), TaskStatus.SENT, requestInfo.getRequestId(), diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouPacketHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouPacketHandler.java index a9183bd86..fc95bc14d 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouPacketHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouPacketHandler.java @@ -14,6 +14,7 @@ import org.ethereum.beacon.discovery.pipeline.Field; import org.ethereum.beacon.discovery.pipeline.HandlerUtil; import org.ethereum.beacon.discovery.pipeline.Pipeline; +import org.ethereum.beacon.discovery.pipeline.info.RequestInfo; import org.ethereum.beacon.discovery.task.TaskMessageFactory; import org.ethereum.beacon.schedulers.Scheduler; import org.ethereum.beacon.util.Utils; @@ -78,7 +79,7 @@ public void handle(Envelope envelope) { session.setInitiatorKey(hkdfKeys.getInitiatorKey()); session.setRecipientKey(hkdfKeys.getRecipientKey()); BytesValue authResponseKey = hkdfKeys.getAuthResponseKey(); - Optional requestInfoOpt = session.getFirstAwaitRequestInfo(); + Optional requestInfoOpt = session.getFirstAwaitRequestInfo(); final V5Message message = requestInfoOpt .map(requestInfo -> TaskMessageFactory.createMessageFromRequest(requestInfo, session)) @@ -91,7 +92,8 @@ public void handle(Envelope envelope) { envelope.getId(), session))); BytesValue ephemeralPubKey = - BytesValue.wrap(Utils.extractBytesFromUnsignedBigInt(ephemeralKey.getPublicKey(), PUBKEY_SIZE)); + BytesValue.wrap( + Utils.extractBytesFromUnsignedBigInt(ephemeralKey.getPublicKey(), PUBKEY_SIZE)); AuthHeaderMessagePacket response = AuthHeaderMessagePacket.create( session.getHomeNodeId(), diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/info/FindNodeRequestInfo.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/info/FindNodeRequestInfo.java new file mode 100644 index 000000000..023bae1cf --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/info/FindNodeRequestInfo.java @@ -0,0 +1,42 @@ +package org.ethereum.beacon.discovery.pipeline.info; + +import org.ethereum.beacon.discovery.task.TaskStatus; +import org.ethereum.beacon.discovery.task.TaskType; +import tech.pegasys.artemis.util.bytes.BytesValue; + +import javax.annotation.Nullable; +import java.util.concurrent.CompletableFuture; + +public class FindNodeRequestInfo extends GeneralRequestInfo { + private final Integer remainingNodes; + private final int distance; + + public FindNodeRequestInfo( + TaskStatus taskStatus, + BytesValue requestId, + CompletableFuture future, + int distance, + @Nullable Integer remainingNodes) { + super(TaskType.FINDNODE, taskStatus, requestId, future); + this.distance = distance; + this.remainingNodes = remainingNodes; + } + + public int getDistance() { + return distance; + } + + public Integer getRemainingNodes() { + return remainingNodes; + } + + @Override + public String toString() { + return "FindNodeRequestInfo{" + + "remainingNodes=" + + remainingNodes + + ", distance=" + + distance + + '}'; + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/info/GeneralRequestInfo.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/info/GeneralRequestInfo.java new file mode 100644 index 000000000..fe5cffdf3 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/info/GeneralRequestInfo.java @@ -0,0 +1,57 @@ +package org.ethereum.beacon.discovery.pipeline.info; + +import org.ethereum.beacon.discovery.task.TaskStatus; +import org.ethereum.beacon.discovery.task.TaskType; +import tech.pegasys.artemis.util.bytes.BytesValue; + +import java.util.concurrent.CompletableFuture; + +public class GeneralRequestInfo implements RequestInfo { + private final TaskType taskType; + private final TaskStatus taskStatus; + private final BytesValue requestId; + private final CompletableFuture future; + + public GeneralRequestInfo( + TaskType taskType, + TaskStatus taskStatus, + BytesValue requestId, + CompletableFuture future) { + this.taskType = taskType; + this.taskStatus = taskStatus; + this.requestId = requestId; + this.future = future; + } + + @Override + public TaskType getTaskType() { + return taskType; + } + + @Override + public TaskStatus getTaskStatus() { + return taskStatus; + } + + @Override + public BytesValue getRequestId() { + return requestId; + } + + @Override + public CompletableFuture getFuture() { + return future; + } + + @Override + public String toString() { + return "GeneralRequestInfo{" + + "taskType=" + + taskType + + ", taskStatus=" + + taskStatus + + ", requestId=" + + requestId + + '}'; + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/info/RequestInfo.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/info/RequestInfo.java new file mode 100644 index 000000000..7b7cc2842 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/info/RequestInfo.java @@ -0,0 +1,17 @@ +package org.ethereum.beacon.discovery.pipeline.info; + +import org.ethereum.beacon.discovery.task.TaskStatus; +import org.ethereum.beacon.discovery.task.TaskType; +import tech.pegasys.artemis.util.bytes.BytesValue; + +import java.util.concurrent.CompletableFuture; + +public interface RequestInfo { + TaskType getTaskType(); + + TaskStatus getTaskStatus(); + + BytesValue getRequestId(); + + CompletableFuture getFuture(); +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/info/RequestInfoFactory.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/info/RequestInfoFactory.java new file mode 100644 index 000000000..05fe14672 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/info/RequestInfoFactory.java @@ -0,0 +1,30 @@ +package org.ethereum.beacon.discovery.pipeline.info; + +import org.ethereum.beacon.discovery.task.TaskOptions; +import org.ethereum.beacon.discovery.task.TaskType; +import tech.pegasys.artemis.util.bytes.BytesValue; + +import java.util.concurrent.CompletableFuture; + +import static org.ethereum.beacon.discovery.task.TaskStatus.AWAIT; + +public class RequestInfoFactory { + public static RequestInfo create( + TaskType taskType, BytesValue id, TaskOptions taskOptions, CompletableFuture future) { + switch (taskType) { + case FINDNODE: + { + return new FindNodeRequestInfo(AWAIT, id, future, taskOptions.getDistance(), null); + } + case PING: + { + return new GeneralRequestInfo(taskType, AWAIT, id, future); + } + default: + { + throw new RuntimeException( + String.format("Factory doesn't know how to create task with type %s", taskType)); + } + } + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/task/DiscoveryTaskManager.java b/discovery/src/main/java/org/ethereum/beacon/discovery/task/DiscoveryTaskManager.java index 6af6b7f3d..5f187f038 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/task/DiscoveryTaskManager.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/task/DiscoveryTaskManager.java @@ -17,12 +17,11 @@ import java.util.stream.Stream; import static org.ethereum.beacon.discovery.NodeStatus.DEAD; -import static org.ethereum.beacon.discovery.task.TaskMessageFactory.DEFAULT_DISTANCE; /** Manages recurrent node check task(s) */ public class DiscoveryTaskManager { - private static final int LIVE_CHECK_DISTANCE = DEFAULT_DISTANCE; - private static final int RECURSIVE_LOOKUP_DISTANCE = DEFAULT_DISTANCE; + private static final int LIVE_CHECK_DISTANCE = 100; + private static final int RECURSIVE_LOOKUP_DISTANCE = 100; private static final int STATUS_EXPIRATION_SECONDS = 600; private static final int LIVE_CHECK_INTERVAL_SECONDS = 1; private static final int RECURSIVE_LOOKUP_INTERVAL_SECONDS = 10; diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/task/LiveCheckTasks.java b/discovery/src/main/java/org/ethereum/beacon/discovery/task/LiveCheckTasks.java index 5476b3bb2..0b895dcce 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/task/LiveCheckTasks.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/task/LiveCheckTasks.java @@ -24,8 +24,7 @@ public class LiveCheckTasks { private final Set currentTasks = Sets.newConcurrentHashSet(); private final ExpirationScheduler taskTimeouts; - public LiveCheckTasks( - DiscoveryManager discoveryManager, Scheduler scheduler, Duration timeout) { + public LiveCheckTasks(DiscoveryManager discoveryManager, Scheduler scheduler, Duration timeout) { this.discoveryManager = discoveryManager; this.scheduler = scheduler; this.taskTimeouts = @@ -42,8 +41,7 @@ public void add(NodeRecordInfo nodeRecordInfo, Runnable successCallback, Runnabl scheduler.execute( () -> { - CompletableFuture retry = - discoveryManager.executeTaskWithoutLivenessUpdate(nodeRecordInfo.getNode(), TaskType.PING); + CompletableFuture retry = discoveryManager.ping(nodeRecordInfo.getNode()); taskTimeouts.put( nodeRecordInfo.getNode().getNodeId(), () -> diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/task/RecursiveLookupTasks.java b/discovery/src/main/java/org/ethereum/beacon/discovery/task/RecursiveLookupTasks.java index 5da805449..2c44e0fde 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/task/RecursiveLookupTasks.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/task/RecursiveLookupTasks.java @@ -19,6 +19,7 @@ * received. */ public class RecursiveLookupTasks { + private static final int DEFAULT_DISTANCE = 100; private final Scheduler scheduler; private final DiscoveryManager discoveryManager; private final Set currentTasks = Sets.newConcurrentHashSet(); @@ -43,11 +44,12 @@ public void add(NodeRecordInfo nodeRecordInfo, Runnable successCallback, Runnabl scheduler.execute( () -> { CompletableFuture retry = - discoveryManager.executeTask(nodeRecordInfo.getNode(), TaskType.FINDNODE); + discoveryManager.findNodes(nodeRecordInfo.getNode(), DEFAULT_DISTANCE); taskTimeouts.put( nodeRecordInfo.getNode().getNodeId(), () -> - retry.completeExceptionally(new RuntimeException("Timeout for node recursive lookup task"))); + retry.completeExceptionally( + new RuntimeException("Timeout for node recursive lookup task"))); retry.whenComplete( (aVoid, throwable) -> { if (throwable != null) { diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/task/TaskMessageFactory.java b/discovery/src/main/java/org/ethereum/beacon/discovery/task/TaskMessageFactory.java index 8638003f6..51261fe96 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/task/TaskMessageFactory.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/task/TaskMessageFactory.java @@ -6,13 +6,13 @@ import org.ethereum.beacon.discovery.message.PingMessage; import org.ethereum.beacon.discovery.message.V5Message; import org.ethereum.beacon.discovery.packet.MessagePacket; +import org.ethereum.beacon.discovery.pipeline.info.FindNodeRequestInfo; +import org.ethereum.beacon.discovery.pipeline.info.RequestInfo; import tech.pegasys.artemis.util.bytes.BytesValue; public class TaskMessageFactory { - public static final int DEFAULT_DISTANCE = 100; - public static MessagePacket createPacketFromRequest( - NodeSession.RequestInfo requestInfo, BytesValue authTag, NodeSession session) { + RequestInfo requestInfo, BytesValue authTag, NodeSession session) { switch (requestInfo.getTaskType()) { case PING: { @@ -20,7 +20,9 @@ public static MessagePacket createPacketFromRequest( } case FINDNODE: { - return createFindNodePacket(authTag, session, requestInfo.getRequestId()); + FindNodeRequestInfo nodeRequestInfo = (FindNodeRequestInfo) requestInfo; + return createFindNodePacket( + authTag, session, requestInfo.getRequestId(), nodeRequestInfo.getDistance()); } default: { @@ -30,8 +32,7 @@ public static MessagePacket createPacketFromRequest( } } - public static V5Message createMessageFromRequest( - NodeSession.RequestInfo requestInfo, NodeSession session) { + public static V5Message createMessageFromRequest(RequestInfo requestInfo, NodeSession session) { switch (requestInfo.getTaskType()) { case PING: { @@ -39,7 +40,8 @@ public static V5Message createMessageFromRequest( } case FINDNODE: { - return createFindNode(requestInfo.getRequestId()); + FindNodeRequestInfo nodeRequestInfo = (FindNodeRequestInfo) requestInfo; + return createFindNode(requestInfo.getRequestId(), nodeRequestInfo.getDistance()); } default: { @@ -65,8 +67,8 @@ public static PingMessage createPing(NodeSession session, BytesValue requestId) } public static MessagePacket createFindNodePacket( - BytesValue authTag, NodeSession session, BytesValue requestId) { - FindNodeMessage findNodeMessage = createFindNode(requestId); + BytesValue authTag, NodeSession session, BytesValue requestId, int distance) { + FindNodeMessage findNodeMessage = createFindNode(requestId, distance); return MessagePacket.create( session.getHomeNodeId(), session.getNodeRecord().getNodeId(), @@ -75,7 +77,7 @@ public static MessagePacket createFindNodePacket( DiscoveryV5Message.from(findNodeMessage)); } - public static FindNodeMessage createFindNode(BytesValue requestId) { - return new FindNodeMessage(requestId, DEFAULT_DISTANCE); + public static FindNodeMessage createFindNode(BytesValue requestId, int distance) { + return new FindNodeMessage(requestId, distance); } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/task/TaskOptions.java b/discovery/src/main/java/org/ethereum/beacon/discovery/task/TaskOptions.java index a2d7f8afd..80d91d763 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/task/TaskOptions.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/task/TaskOptions.java @@ -2,12 +2,22 @@ public class TaskOptions { private boolean livenessUpdate; + private int distance; public TaskOptions(boolean livenessUpdate) { this.livenessUpdate = livenessUpdate; } + public TaskOptions(boolean livenessUpdate, int distance) { + this.livenessUpdate = livenessUpdate; + this.distance = distance; + } + public boolean isLivenessUpdate() { return livenessUpdate; } + + public int getDistance() { + return distance; + } } diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryInteropTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryInteropTest.java index 1f885ce28..da2cc7bf2 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryInteropTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryInteropTest.java @@ -62,9 +62,8 @@ public void test() throws Exception { // 3) Expect standard 1 => 2 dialog CountDownLatch randomSent1to2 = new CountDownLatch(1); - CountDownLatch whoareyouSent2to1 = new CountDownLatch(1); CountDownLatch authPacketSent1to2 = new CountDownLatch(1); - CountDownLatch nodesSent2to1 = new CountDownLatch(1); + CountDownLatch nodesReceivedAt1 = new CountDownLatch(1); Flux.from(discoveryManager1.getOutgoingMessages()) .map(p -> new UnknownPacket(p.getPacket().getBytes())) @@ -81,8 +80,6 @@ public void test() throws Exception { networkPacket.getAuthHeaderMessagePacket(); System.out.println("1 => 2: " + authHeaderMessagePacket); authPacketSent1to2.countDown(); - } else { - throw new RuntimeException("Not expected!"); } }); @@ -90,16 +87,17 @@ public void test() throws Exception { // 4) fire 1 to 2 dialog discoveryManager1.start(); - discoveryManager1.executeTask(nodeRecord2, TaskType.FINDNODE); + int distance = Functions.logDistance(nodeRecord1.getNodeId(), nodeRecord2.getNodeId()); + discoveryManager1.findNodes(nodeRecord2, distance); assert randomSent1to2.await(1, TimeUnit.SECONDS); // assert whoareyouSent2to1.await(1, TimeUnit.SECONDS); // int distance1To2 = Functions.logDistance(nodeRecord1.getNodeId(), // nodeRecord2.getNodeId()); // assertFalse(nodeBucketStorage1.get(distance1To2).isPresent()); - // assert authPacketSent1to2.await(1, TimeUnit.SECONDS); + assert authPacketSent1to2.await(1, TimeUnit.SECONDS); // assert nodesSent2to1.await(1, TimeUnit.SECONDS); - Thread.sleep(1500); + Thread.sleep(5000); // 1 sent findnodes to 2, received only (2) in answer, because 3 is not checked // 1 added 2 to its nodeBuckets, because its now checked, but not before // NodeBucket bucketAt1With2 = nodeBucketStorage1.get(distance1To2).get(); diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java index 274137f0d..72328e82b 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java @@ -11,7 +11,6 @@ import org.ethereum.beacon.discovery.storage.NodeBucketStorage; import org.ethereum.beacon.discovery.storage.NodeTableStorage; import org.ethereum.beacon.discovery.storage.NodeTableStorageFactoryImpl; -import org.ethereum.beacon.discovery.task.TaskType; import org.ethereum.beacon.schedulers.Schedulers; import org.javatuples.Pair; import org.junit.Test; @@ -129,7 +128,7 @@ public void test() throws Exception { // 4) fire 1 to 2 dialog discoveryManager1.start(); discoveryManager2.start(); - discoveryManager1.executeTask(nodeRecord2, TaskType.FINDNODE); + discoveryManager1.findNodes(nodeRecord2, 0); assert randomSent1to2.await(1, TimeUnit.SECONDS); assert whoareyouSent2to1.await(1, TimeUnit.SECONDS); diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java index 14353cdab..d68ee0052 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java @@ -12,7 +12,6 @@ import org.ethereum.beacon.discovery.storage.NodeBucketStorage; import org.ethereum.beacon.discovery.storage.NodeTableStorage; import org.ethereum.beacon.discovery.storage.NodeTableStorageFactoryImpl; -import org.ethereum.beacon.discovery.task.TaskType; import org.ethereum.beacon.schedulers.Schedulers; import org.ethereum.beacon.stream.SimpleProcessor; import org.javatuples.Pair; @@ -143,7 +142,7 @@ public void test() throws Exception { // 4) fire 1 to 2 dialog discoveryManager1.start(); discoveryManager2.start(); - discoveryManager1.executeTask(nodeRecord2, TaskType.FINDNODE); + discoveryManager1.findNodes(nodeRecord2, 0); assert randomSent1to2.await(1, TimeUnit.SECONDS); assert whoareyouSent2to1.await(1, TimeUnit.SECONDS); diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/FunctionsTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/FunctionsTest.java index fd1e4b31c..b39f77182 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/FunctionsTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/FunctionsTest.java @@ -112,7 +112,7 @@ public void testRecoverFromSignature() throws Exception { BytesValue message = BytesValue.wrap("discovery-id-nonce".getBytes()).concat(nonce).concat(ephemeralKey); - assert Functions.verifyECDSASignature(idNonceSig, message, pubKey); + assert Functions.verifyECDSASignature(idNonceSig, Functions.hash(message), pubKey); } @Test @@ -135,6 +135,6 @@ public void testSignAndRecoverFromSignature() { .getEncoded(true)); BytesValue idNonceSig = AuthHeaderMessagePacket.signIdNonce(idNonce, privKey, ephemeralPubkey); assert Functions.verifyECDSASignature( - idNonceSig, createIdNonceMessage(idNonce, ephemeralPubkey), pubKey); + idNonceSig, Functions.hash(createIdNonceMessage(idNonce, ephemeralPubkey)), pubKey); } } diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/HandshakeHandlersTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/HandshakeHandlersTest.java index 33b01c869..28aa81774 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/HandshakeHandlersTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/HandshakeHandlersTest.java @@ -18,6 +18,7 @@ import org.ethereum.beacon.discovery.storage.NodeTableStorage; import org.ethereum.beacon.discovery.storage.NodeTableStorageFactoryImpl; import org.ethereum.beacon.discovery.task.TaskMessageFactory; +import org.ethereum.beacon.discovery.task.TaskOptions; import org.ethereum.beacon.discovery.task.TaskType; import org.ethereum.beacon.schedulers.Scheduler; import org.ethereum.beacon.schedulers.Schedulers; @@ -139,7 +140,7 @@ public void authHandlerWithMessageRoundTripTest() throws Exception { WhoAreYouPacket.create(nodePair1.getValue1().getNodeId(), authTag, idNonce, UInt64.ZERO)); envelopeAt1From2.put(Field.SESSION, nodeSessionAt1For2); CompletableFuture future = new CompletableFuture<>(); - nodeSessionAt1For2.createNextRequest(TaskType.FINDNODE, future); + nodeSessionAt1For2.createNextRequest(TaskType.FINDNODE, new TaskOptions(true), future); whoAreYouPacketHandlerNode1.handle(envelopeAt1From2); assert outgoing1PacketsSemaphore.tryAcquire(1, 1, TimeUnit.SECONDS); outgoing1PacketsSemaphore.release(); @@ -164,7 +165,7 @@ public void authHandlerWithMessageRoundTripTest() throws Exception { pingAuthTag, nodeSessionAt2For1, nodeSessionAt2For1 - .createNextRequest(TaskType.PING, new CompletableFuture<>()) + .createNextRequest(TaskType.PING, new TaskOptions(true), new CompletableFuture<>()) .getRequestId()); envelopeAt1From2WithMessage.put(PACKET_MESSAGE, pingPacketFrom2To1); envelopeAt1From2WithMessage.put(SESSION, nodeSessionAt1For2); @@ -182,7 +183,7 @@ public void authHandlerWithMessageRoundTripTest() throws Exception { Packet pongPacketFrom1To2 = outgoing1Packets[1]; MessagePacket pongMessagePacketFrom1To2 = (MessagePacket) pongPacketFrom1To2; envelopeAt2From1WithMessage.put(PACKET_MESSAGE, pongMessagePacketFrom1To2); - envelopeAt2From1WithMessage.put(SESSION, nodeSessionAt1For2); + envelopeAt2From1WithMessage.put(SESSION, nodeSessionAt2For1); messagePacketHandler2.handle(envelopeAt2From1WithMessage); assertNull(envelopeAt2From1WithMessage.get(BAD_PACKET)); assertNotNull(envelopeAt2From1WithMessage.get(MESSAGE)); diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java b/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java index e6f57f9ba..47551acae 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/mock/DiscoveryManagerNoNetwork.java @@ -108,26 +108,26 @@ public void start() { @Override public void stop() {} - public CompletableFuture executeTask(NodeRecord nodeRecord, TaskType taskType) { - return executeTaskImpl(nodeRecord, taskType, true); - } - - public CompletableFuture executeTaskImpl( - NodeRecord nodeRecord, TaskType taskType, boolean livenessUpdate) { + private CompletableFuture executeTaskImpl( + NodeRecord nodeRecord, TaskType taskType, TaskOptions taskOptions) { Envelope envelope = new Envelope(); envelope.put(Field.NODE, nodeRecord); CompletableFuture future = new CompletableFuture<>(); envelope.put(Field.TASK, taskType); envelope.put(Field.FUTURE, future); - envelope.put(Field.TASK_OPTIONS, new TaskOptions(livenessUpdate)); + envelope.put(Field.TASK_OPTIONS, taskOptions); outgoingPipeline.push(envelope); return future; } @Override - public CompletableFuture executeTaskWithoutLivenessUpdate( - NodeRecord nodeRecord, TaskType taskType) { - return executeTaskImpl(nodeRecord, taskType, false); + public CompletableFuture findNodes(NodeRecord nodeRecord, int distance) { + return executeTaskImpl(nodeRecord, TaskType.FINDNODE, new TaskOptions(true, distance)); + } + + @Override + public CompletableFuture ping(NodeRecord nodeRecord) { + return executeTaskImpl(nodeRecord, TaskType.PING, new TaskOptions(true)); } public Publisher getOutgoingMessages() { From f3412e3184f99e64631decb8f2bb4bde677f8946 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Tue, 19 Nov 2019 19:05:17 +0300 Subject: [PATCH 68/77] discovery: fix and add appropriate asserts in Geth interop test --- .../discovery/DiscoveryInteropTest.java | 29 +++++++++---------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryInteropTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryInteropTest.java index da2cc7bf2..215c439d5 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryInteropTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryInteropTest.java @@ -5,10 +5,10 @@ import org.ethereum.beacon.discovery.packet.AuthHeaderMessagePacket; import org.ethereum.beacon.discovery.packet.RandomPacket; import org.ethereum.beacon.discovery.packet.UnknownPacket; +import org.ethereum.beacon.discovery.storage.NodeBucket; import org.ethereum.beacon.discovery.storage.NodeBucketStorage; import org.ethereum.beacon.discovery.storage.NodeTableStorage; import org.ethereum.beacon.discovery.storage.NodeTableStorageFactoryImpl; -import org.ethereum.beacon.discovery.task.TaskType; import org.ethereum.beacon.schedulers.Schedulers; import org.javatuples.Pair; import org.junit.Ignore; @@ -22,9 +22,11 @@ import static org.ethereum.beacon.discovery.TestUtil.NODE_RECORD_FACTORY_NO_VERIFICATION; import static org.ethereum.beacon.discovery.TestUtil.TEST_SERIALIZER; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; -/** Same as {@link DiscoveryNoNetworkTest} but using real network */ -@Ignore("Finish me!!!, then ignore because it takes too long and requires geth docker") +/** Inter-operational test with Geth. Start it in docker separately */ +@Ignore("TODO: Add geth docker startup, then ignore because of long running") public class DiscoveryInteropTest { @Test public void test() throws Exception { @@ -92,18 +94,15 @@ public void test() throws Exception { assert randomSent1to2.await(1, TimeUnit.SECONDS); // assert whoareyouSent2to1.await(1, TimeUnit.SECONDS); - // int distance1To2 = Functions.logDistance(nodeRecord1.getNodeId(), - // nodeRecord2.getNodeId()); - // assertFalse(nodeBucketStorage1.get(distance1To2).isPresent()); - assert authPacketSent1to2.await(1, TimeUnit.SECONDS); - // assert nodesSent2to1.await(1, TimeUnit.SECONDS); - Thread.sleep(5000); - // 1 sent findnodes to 2, received only (2) in answer, because 3 is not checked + int distance1To2 = Functions.logDistance(nodeRecord1.getNodeId(), nodeRecord2.getNodeId()); + assertFalse(nodeBucketStorage1.get(distance1To2).isPresent()); + assert authPacketSent1to2.await(1, TimeUnit.SECONDS); + Thread.sleep(1000); + // 1 sent findnodes to 2, received only (2) in answer // 1 added 2 to its nodeBuckets, because its now checked, but not before - // NodeBucket bucketAt1With2 = nodeBucketStorage1.get(distance1To2).get(); - // assertEquals(2, bucketAt1With2.size()); - // assertEquals( - // nodeRecord2.getNodeId(), - // bucketAt1With2.getNodeRecords().get(0).getNode().getNodeId()); + NodeBucket bucketAt1With2 = nodeBucketStorage1.get(distance1To2).get(); + assertEquals(1, bucketAt1With2.size()); + assertEquals( + nodeRecord2.getNodeId(), bucketAt1With2.getNodeRecords().get(0).getNode().getNodeId()); } } From 310c825e6cb5bd085341b21cc2b72a9b13fe4a9f Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Fri, 22 Nov 2019 01:32:05 +0300 Subject: [PATCH 69/77] discovery: interop test with geth fixed --- .../discovery/DiscoveryInteropTest.java | 19 ++- discovery/src/test/resources/geth/Dockerfile | 21 ++++ discovery/src/test/resources/geth/test.sh | 5 + .../test/resources/geth/v5_interop_test.go | 111 ++++++++++++++++++ 4 files changed, 153 insertions(+), 3 deletions(-) create mode 100644 discovery/src/test/resources/geth/Dockerfile create mode 100644 discovery/src/test/resources/geth/test.sh create mode 100644 discovery/src/test/resources/geth/v5_interop_test.go diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryInteropTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryInteropTest.java index 215c439d5..68a73988c 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryInteropTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryInteropTest.java @@ -25,11 +25,24 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -/** Inter-operational test with Geth. Start it in docker separately */ -@Ignore("TODO: Add geth docker startup, then ignore because of long running") +/** + * Inter-operational test with Geth. Start it in docker separately + * + *

You need to build and run Geth discv5 test to interact with. Configure Geth running time in + * test.sh located in `resources/geth`, after that build docker and run it: + * cd discovery/src/test/resources/geth + * docker build -t gethv5:1.0 . && docker run --network host -d gethv5:1.0 + * + * + *

After container starts, fire this test to fall in Geth's side running time and it should pass! + * You could check Geth test logs by following command: + * docker logs + * + */ +@Ignore("Requires manual startup, takes a bit to start") public class DiscoveryInteropTest { @Test - public void test() throws Exception { + public void testInterop() throws Exception { // 1) start 2 nodes Pair nodePair1 = TestUtil.generateNode(40412, true); System.out.println(String.format("Node %s started", nodePair1.getValue1().getNodeId())); diff --git a/discovery/src/test/resources/geth/Dockerfile b/discovery/src/test/resources/geth/Dockerfile new file mode 100644 index 000000000..d82728d74 --- /dev/null +++ b/discovery/src/test/resources/geth/Dockerfile @@ -0,0 +1,21 @@ +# Build Geth branch with discv5 implementation +FROM golang:1.12-alpine as builder + +RUN apk add --no-cache make gcc musl-dev linux-headers git + +ARG branch=discover-v5-rebase +RUN git clone --depth 1 --branch $branch https://github.com/fjl/go-ethereum.git + +ADD . /go-ethereum +RUN cd /go/go-ethereum && go mod init github.com/ethereum/go-ethereum + +ADD v5_interop_test.go /go/go-ethereum/p2p/discover/v5_interop_test.go +ADD test.sh /test.sh +RUN chmod +x /test.sh + +# Idle run to download all dependencies for the test +ENV TEST_DURATION=1 +RUN cd /go/go-ethereum/p2p/discover && go test -run TestNodesServer + +EXPOSE 8545 8546 30303 30303/udp +ENTRYPOINT ["/test.sh"] \ No newline at end of file diff --git a/discovery/src/test/resources/geth/test.sh b/discovery/src/test/resources/geth/test.sh new file mode 100644 index 000000000..3f5ed130c --- /dev/null +++ b/discovery/src/test/resources/geth/test.sh @@ -0,0 +1,5 @@ +#!/bin/sh +#set -e +export TEST_DURATION=60 +echo "Run interop_test for ${TEST_DURATION} seconds" +cd /go/go-ethereum/p2p/discover && go test -v -run TestNodesServer diff --git a/discovery/src/test/resources/geth/v5_interop_test.go b/discovery/src/test/resources/geth/v5_interop_test.go new file mode 100644 index 000000000..21926ce9c --- /dev/null +++ b/discovery/src/test/resources/geth/v5_interop_test.go @@ -0,0 +1,111 @@ +package discover + +import ( + "crypto/ecdsa" + "fmt" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/internal/testlog" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/p2p/enr" + "net" + "os" + "strconv" + "testing" + "time" +) + +// Start two servers on 30302 and 30303 ports, run for $TEST_DURATION seconds, exit +func TestNodesServer(t *testing.T) { + var nodes []*UDPv5 + startPort := 30302 + for i := 0; i < 2; i++ { + var cfg Config + if len(nodes) > 0 { + bn := nodes[0].Self() + cfg.Bootnodes = []*enode.Node{bn} + } + var key *ecdsa.PrivateKey + if i == 1 { + // Predefined key for server on 30303 port, so we could easily connect to it outside the test + key, _ = crypto.HexToECDSA("fb757dc581730490a1d7a00deea65e9b1936924caaea8f44d476014856b68736") + } + curnode := startLocalhostOnPortV5(t, cfg, startPort+i, key) + fmt.Printf("port %v: %v\n", startPort+i, curnode.localNode.Node()) + fillTable(curnode.tab, Map(nodes, func(pv5 *UDPv5) *node { + return wrapNode(pv5.localNode.Node()) + })) // force fill buckets + <-curnode.tab.initDone + nodes = append(nodes, curnode) + defer curnode.Close() + } + + // Wait for interop call + fmt.Printf("Waiting for external nodes query\n") + c1 := make(chan string, 1) + + // Run your long running function in it's own goroutine and pass back it's + // response into our channel. + go func() { + text := LongRunningProcess() + c1 <- text + }() + + // Listen on our channel AND a timeout channel - which ever happens first. + duration, err := strconv.Atoi(os.Getenv("TEST_DURATION")) + if err != nil { + panic("TEST_DURATION env not set or failed to be parsed" + err.Error()) + } + select { + case res := <-c1: + fmt.Println(res) + case <-time.After(time.Duration(duration) * time.Second): + fmt.Println("out of time :(") + } +} + +func LongRunningProcess() string { + time.Sleep(9000 * time.Hour) + return "You will never see this :)" +} + +func Map(vs []*UDPv5, f func(*UDPv5) *node) []*node { + vsm := make([]*node, len(vs)) + for i, v := range vs { + vsm[i] = f(v) + } + return vsm +} + +func startLocalhostOnPortV5(t *testing.T, cfg Config, port int, key *ecdsa.PrivateKey) *UDPv5 { + if key == nil { + cfg.PrivateKey = newkey() + } else { + cfg.PrivateKey = key + } + db, _ := enode.OpenDB("") + ln := enode.NewLocalNode(db, cfg.PrivateKey) + + // Prefix logs with node ID. + lprefix := fmt.Sprintf("(%s)", ln.ID().TerminalString()) + lfmt := log.TerminalFormat(false) + cfg.Log = testlog.Logger(t, log.LvlTrace) + cfg.Log.SetHandler(log.FuncHandler(func(r *log.Record) error { + t.Logf("%s %s", lprefix, lfmt.Format(r)) + return nil + })) + + // Listen. + socket, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IP{127, 0, 0, 1}, Port: port}) + if err != nil { + t.Fatal(err) + } + realaddr := socket.LocalAddr().(*net.UDPAddr) + ln.SetStaticIP(realaddr.IP) + ln.Set(enr.UDP(realaddr.Port)) + udp, err := ListenV5(socket, ln, cfg) + if err != nil { + t.Fatal(err) + } + return udp +} From 000a3127752319924d8b055dbb5d7db5aa98c87a Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Fri, 22 Nov 2019 16:04:30 +0300 Subject: [PATCH 70/77] discovery: refactor ENR routines to have abstractions according to specification --- .../discovery/DiscoveryManagerImpl.java | 4 +- .../ethereum/beacon/discovery/RlpUtil.java | 36 +++++ .../beacon/discovery/enr/EnrField.java | 18 +++ .../discovery/enr/EnrFieldInterpreter.java | 10 ++ .../discovery/enr/EnrFieldInterpreterV4.java | 51 +++++++ .../beacon/discovery/enr/EnrFieldV4.java | 6 + .../beacon/discovery/enr/EnrScheme.java | 31 ----- .../discovery/enr/EnrSchemeV4Interpreter.java | 127 ------------------ .../beacon/discovery/enr/IdentitySchema.java | 31 +++++ ...er.java => IdentitySchemaInterpreter.java} | 9 +- .../enr/IdentitySchemaV4Interpreter.java | 50 +++++++ .../beacon/discovery/enr/NodeRecord.java | 64 +++------ .../discovery/enr/NodeRecordFactory.java | 32 +++-- .../message/handler/PingHandler.java | 6 +- .../network/NettyDiscoveryClientImpl.java | 10 +- .../AuthHeaderMessagePacketHandler.java | 5 +- .../handler/WhoAreYouPacketHandler.java | 5 +- .../beacon/discovery/NodeRecordTest.java | 52 ++++--- .../ethereum/beacon/discovery/TestUtil.java | 23 ++-- ...a => IdentitySchemaV4InterpreterMock.java} | 4 +- .../discovery/storage/NodeTableTest.java | 19 +-- 21 files changed, 303 insertions(+), 290 deletions(-) create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrField.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrFieldInterpreter.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrFieldInterpreterV4.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrFieldV4.java delete mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrScheme.java delete mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrSchemeV4Interpreter.java create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/enr/IdentitySchema.java rename discovery/src/main/java/org/ethereum/beacon/discovery/enr/{EnrSchemeInterpreter.java => IdentitySchemaInterpreter.java} (75%) create mode 100644 discovery/src/main/java/org/ethereum/beacon/discovery/enr/IdentitySchemaV4Interpreter.java rename discovery/src/test/java/org/ethereum/beacon/discovery/mock/{EnrSchemeV4InterpreterMock.java => IdentitySchemaV4InterpreterMock.java} (70%) diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java index 3c1c1f15e..293318833 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManagerImpl.java @@ -3,6 +3,7 @@ import com.google.common.annotations.VisibleForTesting; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.ethereum.beacon.discovery.enr.EnrField; import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.enr.NodeRecordFactory; import org.ethereum.beacon.discovery.network.DiscoveryClient; @@ -72,8 +73,7 @@ public DiscoveryManagerImpl( this.nodeRecordFactory = nodeRecordFactory; this.discoveryServer = new NettyDiscoveryServerImpl( - ((Bytes4) homeNode.get(NodeRecord.FIELD_IP_V4)), - (int) homeNode.get(NodeRecord.FIELD_UDP_V4)); + ((Bytes4) homeNode.get(EnrField.IP_V4)), (int) homeNode.get(EnrField.UDP_V4)); discoveryServer.useDatagramChannel( channel -> { discoveryClient = new NettyDiscoveryClientImpl(outgoingMessages, channel); diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/RlpUtil.java b/discovery/src/main/java/org/ethereum/beacon/discovery/RlpUtil.java index 91eebb0e6..5314ebf46 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/RlpUtil.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/RlpUtil.java @@ -1,12 +1,17 @@ package org.ethereum.beacon.discovery; +import org.ethereum.beacon.discovery.enr.IdentitySchema; import org.javatuples.Pair; import org.web3j.rlp.RlpDecoder; import org.web3j.rlp.RlpList; +import org.web3j.rlp.RlpString; import tech.pegasys.artemis.util.bytes.Bytes8; import tech.pegasys.artemis.util.bytes.BytesValue; import tech.pegasys.artemis.util.uint.UInt64; +import java.math.BigInteger; +import java.util.function.Function; + import static org.web3j.rlp.RlpDecoder.OFFSET_LONG_LIST; import static org.web3j.rlp.RlpDecoder.OFFSET_SHORT_LIST; @@ -49,4 +54,35 @@ public static Pair decodeFirstList(BytesValue data) { int len = RlpUtil.calcListLen(data); return Pair.with(RlpDecoder.decode(data.slice(0, len).extractArray()), data.slice(len)); } + + public static RlpString encode(Object object, Function errorMessageFunction) { + if (object instanceof BytesValue) { + return fromBytesValue((BytesValue) object); + } else if (object instanceof Number) { + return fromNumber((Number) object); + } else if (object == null) { + return RlpString.create(new byte[0]); + } else if (object instanceof IdentitySchema) { + return RlpString.create(((IdentitySchema) object).stringName()); + } else { + throw new RuntimeException(errorMessageFunction.apply(object)); + } + } + + private static RlpString fromNumber(Number number) { + if (number instanceof BigInteger) { + return RlpString.create((BigInteger) number); + } else if (number instanceof Long) { + return RlpString.create((Long) number); + } else if (number instanceof Integer) { + return RlpString.create((Integer) number); + } else { + throw new RuntimeException( + String.format("Couldn't serialize number %s : no serializer found.", number)); + } + } + + private static RlpString fromBytesValue(BytesValue bytes) { + return RlpString.create(bytes.extractArray()); + } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrField.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrField.java new file mode 100644 index 000000000..63fbb9870 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrField.java @@ -0,0 +1,18 @@ +package org.ethereum.beacon.discovery.enr; + +public interface EnrField { + // Schema id + String ID = "id"; + // IPv4 address + String IP_V4 = "ip"; + // TCP port, integer + String TCP_V4 = "tcp"; + // UDP port, integer + String UDP_V4 = "udp"; + // IPv6 address + String IP_V6 = "ip6"; + // IPv6-specific TCP port + String TCP_V6 = "tcp6"; + // IPv6-specific UDP port + String UDP_V6 = "udp6"; +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrFieldInterpreter.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrFieldInterpreter.java new file mode 100644 index 000000000..6a57eeef2 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrFieldInterpreter.java @@ -0,0 +1,10 @@ +package org.ethereum.beacon.discovery.enr; + +import org.web3j.rlp.RlpString; +import org.web3j.rlp.RlpType; + +public interface EnrFieldInterpreter { + Object decode(String key, RlpString rlpString); + + RlpType encode(String key, Object object); +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrFieldInterpreterV4.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrFieldInterpreterV4.java new file mode 100644 index 000000000..b6b78267f --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrFieldInterpreterV4.java @@ -0,0 +1,51 @@ +package org.ethereum.beacon.discovery.enr; + +import org.ethereum.beacon.discovery.RlpUtil; +import org.web3j.rlp.RlpString; +import org.web3j.rlp.RlpType; +import tech.pegasys.artemis.util.bytes.Bytes16; +import tech.pegasys.artemis.util.bytes.Bytes4; +import tech.pegasys.artemis.util.bytes.BytesValue; + +import java.util.HashMap; +import java.util.Map; +import java.util.function.Function; + +public class EnrFieldInterpreterV4 implements EnrFieldInterpreter { + public static EnrFieldInterpreterV4 DEFAULT = new EnrFieldInterpreterV4(); + + private Map> fieldDecoders = new HashMap<>(); + + public EnrFieldInterpreterV4() { + fieldDecoders.put( + EnrFieldV4.PKEY_SECP256K1, rlpString -> BytesValue.wrap(rlpString.getBytes())); + fieldDecoders.put( + EnrField.ID, rlpString -> IdentitySchema.fromString(new String(rlpString.getBytes()))); + fieldDecoders.put( + EnrField.IP_V4, rlpString -> Bytes4.wrap(BytesValue.wrap(rlpString.getBytes()), 0)); + fieldDecoders.put(EnrField.TCP_V4, rlpString -> rlpString.asPositiveBigInteger().intValue()); + fieldDecoders.put(EnrField.UDP_V4, rlpString -> rlpString.asPositiveBigInteger().intValue()); + fieldDecoders.put( + EnrField.IP_V6, rlpString -> Bytes16.wrap(BytesValue.wrap(rlpString.getBytes()), 0)); + fieldDecoders.put(EnrField.TCP_V6, rlpString -> rlpString.asPositiveBigInteger().intValue()); + fieldDecoders.put(EnrField.UDP_V6, rlpString -> rlpString.asPositiveBigInteger().intValue()); + } + + @Override + public Object decode(String key, RlpString rlpString) { + Function fieldDecoder = fieldDecoders.get(key); + if (fieldDecoder == null) { + throw new RuntimeException(String.format("No decoder found for field `%s`", key)); + } + return fieldDecoder.apply(rlpString); + } + + @Override + public RlpType encode(String key, Object object) { + return RlpUtil.encode( + object, + o -> + String.format( + "Couldn't encode field %s with value %s: no serializer found.", key, object)); + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrFieldV4.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrFieldV4.java new file mode 100644 index 000000000..52ff43882 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrFieldV4.java @@ -0,0 +1,6 @@ +package org.ethereum.beacon.discovery.enr; + +public interface EnrFieldV4 extends EnrField { + // Compressed secp256k1 public key, 33 bytes + String PKEY_SECP256K1 = "secp256k1"; +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrScheme.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrScheme.java deleted file mode 100644 index 9e8545543..000000000 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrScheme.java +++ /dev/null @@ -1,31 +0,0 @@ -package org.ethereum.beacon.discovery.enr; - -import java.util.HashMap; -import java.util.Map; - -/** Available identity schemes of {@link NodeRecord} */ -public enum EnrScheme { - V4("v4"); - - private static final Map nameMap = new HashMap<>(); - - static { - for (EnrScheme scheme : EnrScheme.values()) { - nameMap.put(scheme.name, scheme); - } - } - - private String name; - - private EnrScheme(String name) { - this.name = name; - } - - public static EnrScheme fromString(String name) { - return nameMap.get(name); - } - - public String stringName() { - return name; - } -} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrSchemeV4Interpreter.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrSchemeV4Interpreter.java deleted file mode 100644 index cea0d858d..000000000 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrSchemeV4Interpreter.java +++ /dev/null @@ -1,127 +0,0 @@ -package org.ethereum.beacon.discovery.enr; - -import org.bouncycastle.math.ec.ECPoint; -import org.ethereum.beacon.discovery.Functions; -import org.ethereum.beacon.util.Utils; -import org.web3j.rlp.RlpString; -import tech.pegasys.artemis.util.bytes.Bytes16; -import tech.pegasys.artemis.util.bytes.Bytes32; -import tech.pegasys.artemis.util.bytes.Bytes4; -import tech.pegasys.artemis.util.bytes.BytesValue; - -import java.math.BigInteger; -import java.util.HashMap; -import java.util.Map; -import java.util.function.Function; - -import static org.ethereum.beacon.discovery.enr.NodeRecord.FIELD_ID; -import static org.ethereum.beacon.discovery.enr.NodeRecord.FIELD_IP_V4; -import static org.ethereum.beacon.discovery.enr.NodeRecord.FIELD_IP_V6; -import static org.ethereum.beacon.discovery.enr.NodeRecord.FIELD_PKEY_SECP256K1; -import static org.ethereum.beacon.discovery.enr.NodeRecord.FIELD_TCP_V4; -import static org.ethereum.beacon.discovery.enr.NodeRecord.FIELD_TCP_V6; -import static org.ethereum.beacon.discovery.enr.NodeRecord.FIELD_UDP_V4; -import static org.ethereum.beacon.discovery.enr.NodeRecord.FIELD_UDP_V6; - -public class EnrSchemeV4Interpreter implements EnrSchemeInterpreter { - - private Map> fieldDecoders = new HashMap<>(); - - public EnrSchemeV4Interpreter() { - fieldDecoders.put(FIELD_PKEY_SECP256K1, rlpString -> BytesValue.wrap(rlpString.getBytes())); - fieldDecoders.put( - FIELD_ID, rlpString -> EnrScheme.fromString(new String(rlpString.getBytes()))); - fieldDecoders.put( - FIELD_IP_V4, rlpString -> Bytes4.wrap(BytesValue.wrap(rlpString.getBytes()), 0)); - fieldDecoders.put(FIELD_TCP_V4, rlpString -> rlpString.asPositiveBigInteger().intValue()); - fieldDecoders.put(FIELD_UDP_V4, rlpString -> rlpString.asPositiveBigInteger().intValue()); - fieldDecoders.put( - FIELD_IP_V6, rlpString -> Bytes16.wrap(BytesValue.wrap(rlpString.getBytes()), 0)); - fieldDecoders.put(FIELD_TCP_V6, rlpString -> rlpString.asPositiveBigInteger().intValue()); - fieldDecoders.put(FIELD_UDP_V6, rlpString -> rlpString.asPositiveBigInteger().intValue()); - } - - @Override - public void verify(NodeRecord nodeRecord) { - EnrSchemeInterpreter.super.verify(nodeRecord); - if (nodeRecord.get(FIELD_PKEY_SECP256K1) == null) { - throw new RuntimeException( - String.format( - "Field %s not exists but required for scheme %s", FIELD_PKEY_SECP256K1, getScheme())); - } - BytesValue pubKey = (BytesValue) nodeRecord.get(FIELD_PKEY_SECP256K1); // compressed - assert Functions.verifyECDSASignature( - nodeRecord.getSignature(), Functions.hashKeccak(nodeRecord.serialize(false)), pubKey); - } - - @Override - public EnrScheme getScheme() { - return EnrScheme.V4; - } - - @Override - public Bytes32 getNodeId(NodeRecord nodeRecord) { - verify(nodeRecord); - BytesValue pkey = (BytesValue) nodeRecord.getKey(FIELD_PKEY_SECP256K1); - ECPoint pudDestPoint = Functions.publicKeyToPoint(pkey); - BytesValue xPart = - Bytes32.wrap( - Utils.extractBytesFromUnsignedBigInt(pudDestPoint.getXCoord().toBigInteger(), 32)); - BytesValue yPart = - Bytes32.wrap( - Utils.extractBytesFromUnsignedBigInt(pudDestPoint.getYCoord().toBigInteger(), 32)); - return Functions.hashKeccak(xPart.concat(yPart)); - } - - @Override - public Object decode(String key, RlpString rlpString) { - Function fieldDecoder = fieldDecoders.get(key); - if (fieldDecoder == null) { - throw new RuntimeException(String.format("No decoder found for field `%s`", key)); - } - return fieldDecoder.apply(rlpString); - } - - @Override - public void sign(NodeRecord nodeRecord, Object signOptions) { - BytesValue privateKey = (BytesValue) signOptions; - BytesValue signature = - Functions.sign(privateKey, Functions.hashKeccak(nodeRecord.serialize(false))); - nodeRecord.setSignature(signature); - } - - @Override - public RlpString encode(String key, Object object) { - if (object instanceof BytesValue) { - return fromBytesValue((BytesValue) object); - } else if (object instanceof Number) { - return fromNumber((Number) object); - } else if (object == null) { - return RlpString.create(new byte[0]); - } else if (object instanceof EnrScheme) { - return RlpString.create(((EnrScheme) object).stringName()); - } else { - throw new RuntimeException( - String.format( - "Couldn't serialize node record field %s with value %s: no serializer found.", - key, object)); - } - } - - private RlpString fromNumber(Number number) { - if (number instanceof BigInteger) { - return RlpString.create((BigInteger) number); - } else if (number instanceof Long) { - return RlpString.create((Long) number); - } else if (number instanceof Integer) { - return RlpString.create((Integer) number); - } else { - throw new RuntimeException( - String.format("Couldn't serialize number %s : no serializer found.", number)); - } - } - - private RlpString fromBytesValue(BytesValue bytes) { - return RlpString.create(bytes.extractArray()); - } -} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/IdentitySchema.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/IdentitySchema.java new file mode 100644 index 000000000..5fce90f49 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/IdentitySchema.java @@ -0,0 +1,31 @@ +package org.ethereum.beacon.discovery.enr; + +import java.util.HashMap; +import java.util.Map; + +/** Available identity schemas of Ethereum {@link NodeRecord} signature */ +public enum IdentitySchema { + V4("v4"); + + private static final Map nameMap = new HashMap<>(); + + static { + for (IdentitySchema scheme : IdentitySchema.values()) { + nameMap.put(scheme.name, scheme); + } + } + + private String name; + + private IdentitySchema(String name) { + this.name = name; + } + + public static IdentitySchema fromString(String name) { + return nameMap.get(name); + } + + public String stringName() { + return name; + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrSchemeInterpreter.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/IdentitySchemaInterpreter.java similarity index 75% rename from discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrSchemeInterpreter.java rename to discovery/src/main/java/org/ethereum/beacon/discovery/enr/IdentitySchemaInterpreter.java index cf9750f5f..cb44d1458 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrSchemeInterpreter.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/IdentitySchemaInterpreter.java @@ -1,11 +1,10 @@ package org.ethereum.beacon.discovery.enr; -import org.web3j.rlp.RlpString; import tech.pegasys.artemis.util.bytes.Bytes32; -public interface EnrSchemeInterpreter { +public interface IdentitySchemaInterpreter { /** Returns supported scheme */ - EnrScheme getScheme(); + IdentitySchema getScheme(); /* Signs nodeRecord, modifying it */ void sign(NodeRecord nodeRecord, Object signOptions); @@ -19,8 +18,4 @@ default void verify(NodeRecord nodeRecord) { /** Delivers nodeId according to identity scheme scheme */ Bytes32 getNodeId(NodeRecord nodeRecord); - - Object decode(String key, RlpString rlpString); - - RlpString encode(String key, Object object); } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/IdentitySchemaV4Interpreter.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/IdentitySchemaV4Interpreter.java new file mode 100644 index 000000000..812f20134 --- /dev/null +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/IdentitySchemaV4Interpreter.java @@ -0,0 +1,50 @@ +package org.ethereum.beacon.discovery.enr; + +import org.bouncycastle.math.ec.ECPoint; +import org.ethereum.beacon.discovery.Functions; +import org.ethereum.beacon.util.Utils; +import tech.pegasys.artemis.util.bytes.Bytes32; +import tech.pegasys.artemis.util.bytes.BytesValue; + +public class IdentitySchemaV4Interpreter implements IdentitySchemaInterpreter { + @Override + public void verify(NodeRecord nodeRecord) { + IdentitySchemaInterpreter.super.verify(nodeRecord); + if (nodeRecord.get(EnrFieldV4.PKEY_SECP256K1) == null) { + throw new RuntimeException( + String.format( + "Field %s not exists but required for scheme %s", + EnrFieldV4.PKEY_SECP256K1, getScheme())); + } + BytesValue pubKey = (BytesValue) nodeRecord.get(EnrFieldV4.PKEY_SECP256K1); // compressed + assert Functions.verifyECDSASignature( + nodeRecord.getSignature(), Functions.hashKeccak(nodeRecord.serialize(false)), pubKey); + } + + @Override + public IdentitySchema getScheme() { + return IdentitySchema.V4; + } + + @Override + public Bytes32 getNodeId(NodeRecord nodeRecord) { + verify(nodeRecord); + BytesValue pkey = (BytesValue) nodeRecord.getKey(EnrFieldV4.PKEY_SECP256K1); + ECPoint pudDestPoint = Functions.publicKeyToPoint(pkey); + BytesValue xPart = + Bytes32.wrap( + Utils.extractBytesFromUnsignedBigInt(pudDestPoint.getXCoord().toBigInteger(), 32)); + BytesValue yPart = + Bytes32.wrap( + Utils.extractBytesFromUnsignedBigInt(pudDestPoint.getYCoord().toBigInteger(), 32)); + return Functions.hashKeccak(xPart.concat(yPart)); + } + + @Override + public void sign(NodeRecord nodeRecord, Object signOptions) { + BytesValue privateKey = (BytesValue) signOptions; + BytesValue signature = + Functions.sign(privateKey, Functions.hashKeccak(nodeRecord.serialize(false))); + nodeRecord.setSignature(signature); + } +} diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecord.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecord.java index ff4e5ca00..3d323cb6f 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecord.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecord.java @@ -20,63 +20,45 @@ import java.util.stream.Collectors; /** - * Ethereum Node Record + * Ethereum Node Record V4 * *

Node record as described in EIP-778 */ public class NodeRecord { - // Compressed secp256k1 public key, 33 bytes - public static String FIELD_PKEY_SECP256K1 = "secp256k1"; - // Schema id - public static String FIELD_ID = "id"; - // IPv4 address - public static String FIELD_IP_V4 = "ip"; - // TCP port, integer - public static String FIELD_TCP_V4 = "tcp"; - // UDP port, integer - public static String FIELD_UDP_V4 = "udp"; - // IPv6 address - public static String FIELD_IP_V6 = "ip6"; - // IPv6-specific TCP port - public static String FIELD_TCP_V6 = "tcp6"; - // IPv6-specific UDP port - public static String FIELD_UDP_V6 = "udp6"; - - private UInt64 seq; + private static final EnrFieldInterpreter enrFieldInterpreter = EnrFieldInterpreterV4.DEFAULT; + private final UInt64 seq; // Signature private BytesValue signature; // optional fields private Map fields = new HashMap<>(); + private IdentitySchemaInterpreter identitySchemaInterpreter; - private EnrSchemeInterpreter enrSchemeInterpreter; - - private NodeRecord(EnrSchemeInterpreter enrSchemeInterpreter, UInt64 seq, BytesValue signature) { + private NodeRecord( + IdentitySchemaInterpreter identitySchemaInterpreter, UInt64 seq, BytesValue signature) { this.seq = seq; this.signature = signature; - this.enrSchemeInterpreter = enrSchemeInterpreter; + this.identitySchemaInterpreter = identitySchemaInterpreter; } - private NodeRecord() {} - public static NodeRecord fromValues( - EnrSchemeInterpreter enrSchemeInterpreter, + IdentitySchemaInterpreter identitySchemaInterpreter, UInt64 seq, BytesValue signature, List> fieldKeyPairs) { - NodeRecord nodeRecord = new NodeRecord(enrSchemeInterpreter, seq, signature); + NodeRecord nodeRecord = new NodeRecord(identitySchemaInterpreter, seq, signature); fieldKeyPairs.forEach(objects -> nodeRecord.set(objects.getValue0(), objects.getValue1())); return nodeRecord; } public static NodeRecord fromRawFields( - EnrSchemeInterpreter enrSchemeInterpreter, + IdentitySchemaInterpreter identitySchemaInterpreter, UInt64 seq, BytesValue signature, List rawFields) { - NodeRecord nodeRecord = new NodeRecord(enrSchemeInterpreter, seq, signature); + NodeRecord nodeRecord = new NodeRecord(identitySchemaInterpreter, seq, signature); for (int i = 0; i < rawFields.size(); i += 2) { String key = new String(((RlpString) rawFields.get(i)).getBytes()); - nodeRecord.set(key, enrSchemeInterpreter.decode(key, (RlpString) rawFields.get(i + 1))); + nodeRecord.set(key, enrFieldInterpreter.decode(key, (RlpString) rawFields.get(i + 1))); } return nodeRecord; } @@ -85,8 +67,8 @@ public String asBase64() { return new String(Base64.getUrlEncoder().encode(serialize().extractArray())); } - public EnrScheme getIdentityScheme() { - return enrSchemeInterpreter.getScheme(); + public IdentitySchema getIdentityScheme() { + return identitySchemaInterpreter.getScheme(); } public void set(String key, Object value) { @@ -101,10 +83,6 @@ public UInt64 getSeq() { return seq; } - public void setSeq(UInt64 seq) { - this.seq = seq; - } - public BytesValue getSignature() { return signature; } @@ -137,11 +115,11 @@ public int hashCode() { } public void verify() { - enrSchemeInterpreter.verify(this); + identitySchemaInterpreter.verify(this); } public void sign(Object signOptions) { - enrSchemeInterpreter.sign(this, signOptions); + identitySchemaInterpreter.sign(this, signOptions); } public RlpList asRlp() { @@ -164,7 +142,7 @@ public RlpList asRlp(boolean withSignature) { continue; } values.add(RlpString.create(key)); - values.add(enrSchemeInterpreter.encode(key, fields.get(key))); + values.add(enrFieldInterpreter.encode(key, fields.get(key))); } return new RlpList(values); @@ -181,18 +159,18 @@ public BytesValue serialize(boolean withSignature) { } public Bytes32 getNodeId() { - return enrSchemeInterpreter.getNodeId(this); + return identitySchemaInterpreter.getNodeId(this); } @Override public String toString() { return "NodeRecordV4{" + "publicKey=" - + fields.get(FIELD_PKEY_SECP256K1) + + fields.get(EnrFieldV4.PKEY_SECP256K1) + ", ipV4address=" - + fields.get(FIELD_IP_V4) + + fields.get(EnrFieldV4.IP_V4) + ", udpPort=" - + fields.get(FIELD_UDP_V4) + + fields.get(EnrFieldV4.UDP_V4) + '}'; } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordFactory.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordFactory.java index fcee6d66e..534ff6ff0 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordFactory.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordFactory.java @@ -15,16 +15,14 @@ import java.util.List; import java.util.Map; -import static org.ethereum.beacon.discovery.enr.NodeRecord.FIELD_ID; - public class NodeRecordFactory { public static final NodeRecordFactory DEFAULT = - new NodeRecordFactory(new EnrSchemeV4Interpreter()); - Map interpreters = new HashMap<>(); + new NodeRecordFactory(new IdentitySchemaV4Interpreter()); + Map interpreters = new HashMap<>(); - public NodeRecordFactory(EnrSchemeInterpreter... enrSchemeInterpreters) { - for (EnrSchemeInterpreter enrSchemeInterpreter : enrSchemeInterpreters) { - interpreters.put(enrSchemeInterpreter.getScheme(), enrSchemeInterpreter); + public NodeRecordFactory(IdentitySchemaInterpreter... identitySchemaInterpreters) { + for (IdentitySchemaInterpreter identitySchemaInterpreter : identitySchemaInterpreters) { + interpreters.put(identitySchemaInterpreter.getScheme(), identitySchemaInterpreter); } } @@ -38,7 +36,7 @@ public NodeRecord createFromValues( UInt64 seq, BytesValue signature, List> fieldKeyPairs) { Pair schemePair = null; for (Pair pair : fieldKeyPairs) { - if (FIELD_ID.equals(pair.getValue0())) { + if (EnrField.ID.equals(pair.getValue0())) { schemePair = pair; break; } @@ -47,15 +45,15 @@ public NodeRecord createFromValues( throw new RuntimeException("ENR scheme is not defined in key-value pairs"); } - EnrSchemeInterpreter enrSchemeInterpreter = interpreters.get(schemePair.getValue1()); - if (enrSchemeInterpreter == null) { + IdentitySchemaInterpreter identitySchemaInterpreter = interpreters.get(schemePair.getValue1()); + if (identitySchemaInterpreter == null) { throw new RuntimeException( String.format( "No ethereum record interpreter found for identity scheme %s", schemePair.getValue1())); } - return NodeRecord.fromValues(enrSchemeInterpreter, seq, signature, fieldKeyPairs); + return NodeRecord.fromValues(identitySchemaInterpreter, seq, signature, fieldKeyPairs); } public NodeRecord fromBase64(String enrBase64) { @@ -74,7 +72,7 @@ public NodeRecord fromRlpList(RlpList rlpList) { } // TODO: repair as id is not first now - EnrScheme nodeIdentity = null; + IdentitySchema nodeIdentity = null; boolean idFound = false; for (int i = 2; i < values.size(); i += 2) { RlpString id = (RlpString) values.get(i); @@ -83,8 +81,8 @@ public NodeRecord fromRlpList(RlpList rlpList) { } RlpString idVersion = (RlpString) values.get(i + 1); - nodeIdentity = EnrScheme.fromString(new String(idVersion.getBytes())); - if (nodeIdentity == null) { // no interpreter for such id + nodeIdentity = IdentitySchema.fromString(new String(idVersion.getBytes())); + if (nodeIdentity == null) { // no interpreter for such id throw new RuntimeException( String.format( "Unknown node identity scheme '%s', couldn't create node record.", @@ -97,15 +95,15 @@ public NodeRecord fromRlpList(RlpList rlpList) { throw new RuntimeException("Unknown node identity scheme, not defined in record "); } - EnrSchemeInterpreter enrSchemeInterpreter = interpreters.get(nodeIdentity); - if (enrSchemeInterpreter == null) { + IdentitySchemaInterpreter identitySchemaInterpreter = interpreters.get(nodeIdentity); + if (identitySchemaInterpreter == null) { throw new RuntimeException( String.format( "No Ethereum record interpreter found for identity scheme %s", nodeIdentity)); } return NodeRecord.fromRawFields( - enrSchemeInterpreter, + identitySchemaInterpreter, UInt64.fromBytesBigEndian( Bytes8.leftPad(BytesValue.wrap(((RlpString) values.get(1)).getBytes()))), BytesValue.wrap(((RlpString) values.get(0)).getBytes()), diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/PingHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/PingHandler.java index b2da520fa..80e01e662 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/PingHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/handler/PingHandler.java @@ -1,7 +1,7 @@ package org.ethereum.beacon.discovery.message.handler; import org.ethereum.beacon.discovery.NodeSession; -import org.ethereum.beacon.discovery.enr.NodeRecord; +import org.ethereum.beacon.discovery.enr.EnrField; import org.ethereum.beacon.discovery.message.DiscoveryV5Message; import org.ethereum.beacon.discovery.message.PingMessage; import org.ethereum.beacon.discovery.message.PongMessage; @@ -15,8 +15,8 @@ public void handle(PingMessage message, NodeSession session) { new PongMessage( message.getRequestId(), session.getNodeRecord().getSeq(), - ((Bytes4) session.getNodeRecord().get(NodeRecord.FIELD_IP_V4)), - (int) session.getNodeRecord().get(NodeRecord.FIELD_UDP_V4)); + ((Bytes4) session.getNodeRecord().get(EnrField.IP_V4)), + (int) session.getNodeRecord().get(EnrField.UDP_V4)); session.sendOutgoing( MessagePacket.create( session.getHomeNodeId(), diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/network/NettyDiscoveryClientImpl.java b/discovery/src/main/java/org/ethereum/beacon/discovery/network/NettyDiscoveryClientImpl.java index 4f03f7a08..e8184cb68 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/network/NettyDiscoveryClientImpl.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/network/NettyDiscoveryClientImpl.java @@ -5,7 +5,8 @@ import io.netty.channel.socket.nio.NioDatagramChannel; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.ethereum.beacon.discovery.enr.EnrScheme; +import org.ethereum.beacon.discovery.enr.EnrField; +import org.ethereum.beacon.discovery.enr.IdentitySchema; import org.ethereum.beacon.discovery.enr.NodeRecord; import org.reactivestreams.Publisher; import reactor.core.publisher.Flux; @@ -60,7 +61,7 @@ public void stop() { @Override public void send(BytesValue data, NodeRecord recipient) { - if (!(recipient.getIdentityScheme().equals(EnrScheme.V4))) { + if (!(recipient.getIdentityScheme().equals(IdentitySchema.V4))) { String error = String.format( "Accepts only V4 version of recipient's node records. Got %s instead", recipient); @@ -71,9 +72,8 @@ public void send(BytesValue data, NodeRecord recipient) { try { address = new InetSocketAddress( - InetAddress.getByAddress( - ((Bytes4) recipient.get(NodeRecord.FIELD_IP_V4)).extractArray()), - (int) recipient.get(NodeRecord.FIELD_UDP_V4)); + InetAddress.getByAddress(((Bytes4) recipient.get(EnrField.IP_V4)).extractArray()), + (int) recipient.get(EnrField.UDP_V4)); } catch (UnknownHostException e) { String error = String.format("Failed to resolve host for node record: %s", recipient); logger.error(error); diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/AuthHeaderMessagePacketHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/AuthHeaderMessagePacketHandler.java index 05a97c7b1..1d235733b 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/AuthHeaderMessagePacketHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/AuthHeaderMessagePacketHandler.java @@ -4,6 +4,7 @@ import org.apache.logging.log4j.Logger; import org.ethereum.beacon.discovery.Functions; import org.ethereum.beacon.discovery.NodeSession; +import org.ethereum.beacon.discovery.enr.EnrFieldV4; import org.ethereum.beacon.discovery.enr.NodeRecordFactory; import org.ethereum.beacon.discovery.packet.AuthHeaderMessagePacket; import org.ethereum.beacon.discovery.pipeline.Envelope; @@ -15,7 +16,6 @@ import tech.pegasys.artemis.util.bytes.BytesValue; import static org.ethereum.beacon.discovery.NodeSession.SessionStatus.AUTHENTICATED; -import static org.ethereum.beacon.discovery.enr.NodeRecord.FIELD_PKEY_SECP256K1; /** Handles {@link AuthHeaderMessagePacket} in {@link Field#PACKET_AUTH_HEADER_MESSAGE} field */ public class AuthHeaderMessagePacketHandler implements EnvelopeHandler { @@ -68,7 +68,8 @@ public void handle(Envelope envelope) { session.setRecipientKey(keys.getInitiatorKey()); packet.decodeMessage(session.getRecipientKey(), keys.getAuthResponseKey(), nodeRecordFactory); packet.verify( - session.getIdNonce(), (BytesValue) session.getNodeRecord().get(FIELD_PKEY_SECP256K1)); + session.getIdNonce(), + (BytesValue) session.getNodeRecord().get(EnrFieldV4.PKEY_SECP256K1)); envelope.put(Field.MESSAGE, packet.getMessage()); } catch (AssertionError ex) { logger.info( diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouPacketHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouPacketHandler.java index fc95bc14d..990c8b119 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouPacketHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/WhoAreYouPacketHandler.java @@ -4,6 +4,7 @@ import org.apache.logging.log4j.Logger; import org.ethereum.beacon.discovery.Functions; import org.ethereum.beacon.discovery.NodeSession; +import org.ethereum.beacon.discovery.enr.EnrFieldV4; import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.message.DiscoveryV5Message; import org.ethereum.beacon.discovery.message.V5Message; @@ -25,7 +26,6 @@ import java.util.function.Supplier; import static org.ethereum.beacon.discovery.Functions.PUBKEY_SIZE; -import static org.ethereum.beacon.discovery.enr.NodeRecord.FIELD_PKEY_SECP256K1; /** Handles {@link WhoAreYouPacket} in {@link Field#PACKET_WHOAREYOU} field */ public class WhoAreYouPacketHandler implements EnvelopeHandler { @@ -64,7 +64,8 @@ public void handle(Envelope envelope) { if (packet.getEnrSeq().compareTo(session.getHomeNodeRecord().getSeq()) < 0) { respRecord = session.getHomeNodeRecord(); } - BytesValue remotePubKey = (BytesValue) session.getNodeRecord().getKey(FIELD_PKEY_SECP256K1); + BytesValue remotePubKey = + (BytesValue) session.getNodeRecord().getKey(EnrFieldV4.PKEY_SECP256K1); byte[] ephemeralKeyBytes = new byte[32]; Functions.getRandom().nextBytes(ephemeralKeyBytes); ECKeyPair ephemeralKey = ECKeyPair.create(ephemeralKeyBytes); diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/NodeRecordTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/NodeRecordTest.java index c10104f90..334956b50 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/NodeRecordTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/NodeRecordTest.java @@ -1,11 +1,12 @@ package org.ethereum.beacon.discovery; -import org.ethereum.beacon.discovery.enr.EnrScheme; +import org.ethereum.beacon.discovery.enr.EnrField; +import org.ethereum.beacon.discovery.enr.EnrFieldV4; +import org.ethereum.beacon.discovery.enr.IdentitySchema; import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.enr.NodeRecordFactory; import org.javatuples.Pair; import org.junit.Test; -import org.web3j.crypto.ECKeyPair; import tech.pegasys.artemis.util.bytes.Bytes4; import tech.pegasys.artemis.util.bytes.Bytes96; import tech.pegasys.artemis.util.bytes.BytesValue; @@ -15,14 +16,7 @@ import java.util.Random; import java.util.concurrent.atomic.AtomicBoolean; -import static org.ethereum.beacon.discovery.Functions.PUBKEY_SIZE; import static org.ethereum.beacon.discovery.TestUtil.SEED; -import static org.ethereum.beacon.discovery.enr.NodeRecord.FIELD_ID; -import static org.ethereum.beacon.discovery.enr.NodeRecord.FIELD_IP_V4; -import static org.ethereum.beacon.discovery.enr.NodeRecord.FIELD_PKEY_SECP256K1; -import static org.ethereum.beacon.discovery.enr.NodeRecord.FIELD_TCP_V4; -import static org.ethereum.beacon.discovery.enr.NodeRecord.FIELD_UDP_V4; -import static org.ethereum.beacon.util.Utils.extractBytesFromUnsignedBigInt; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; @@ -54,28 +48,28 @@ public void testLocalhostV4() throws Exception { "-IS4QHCYrYZbAKWCBRlAy5zzaDZXJBGkcnh4MHcBFZntXNFrdvJjX04jRzjzCBOonrkTfj499SZuOh8R33Ls8RRcy5wBgmlkgnY0gmlwhH8AAAGJc2VjcDI1NmsxoQPKY0yuDUmstAHYpMa2_oxVtw0RW_QAdpzBQA8yWM0xOIN1ZHCCdl8"; NodeRecord nodeRecord = NODE_RECORD_FACTORY.fromBase64(localhostEnr); - assertEquals(EnrScheme.V4, nodeRecord.getIdentityScheme()); + assertEquals(IdentitySchema.V4, nodeRecord.getIdentityScheme()); assertArrayEquals( InetAddress.getByName(expectedHost).getAddress(), - ((BytesValue) nodeRecord.get(NodeRecord.FIELD_IP_V4)).extractArray()); - assertEquals(expectedUdpPort, nodeRecord.get(NodeRecord.FIELD_UDP_V4)); - assertEquals(expectedTcpPort, nodeRecord.get(NodeRecord.FIELD_TCP_V4)); + ((BytesValue) nodeRecord.get(EnrField.IP_V4)).extractArray()); + assertEquals(expectedUdpPort, nodeRecord.get(EnrField.UDP_V4)); + assertEquals(expectedTcpPort, nodeRecord.get(EnrField.TCP_V4)); assertEquals(expectedSeqNumber, nodeRecord.getSeq()); - assertEquals(expectedPublicKey, nodeRecord.get(NodeRecord.FIELD_PKEY_SECP256K1)); + assertEquals(expectedPublicKey, nodeRecord.get(EnrFieldV4.PKEY_SECP256K1)); assertEquals(expectedSignature, nodeRecord.getSignature()); String localhostEnrRestored = nodeRecord.asBase64(); // The order of fields is not strict so we don't compare strings NodeRecord nodeRecordRestored = NODE_RECORD_FACTORY.fromBase64(localhostEnrRestored); - assertEquals(EnrScheme.V4, nodeRecordRestored.getIdentityScheme()); + assertEquals(IdentitySchema.V4, nodeRecordRestored.getIdentityScheme()); assertArrayEquals( InetAddress.getByName(expectedHost).getAddress(), - ((BytesValue) nodeRecordRestored.get(NodeRecord.FIELD_IP_V4)).extractArray()); - assertEquals(expectedUdpPort, nodeRecordRestored.get(NodeRecord.FIELD_UDP_V4)); - assertEquals(expectedTcpPort, nodeRecordRestored.get(NodeRecord.FIELD_TCP_V4)); + ((BytesValue) nodeRecordRestored.get(EnrField.IP_V4)).extractArray()); + assertEquals(expectedUdpPort, nodeRecordRestored.get(EnrField.UDP_V4)); + assertEquals(expectedTcpPort, nodeRecordRestored.get(EnrField.TCP_V4)); assertEquals(expectedSeqNumber, nodeRecordRestored.getSeq()); - assertEquals(expectedPublicKey, nodeRecordRestored.get(NodeRecord.FIELD_PKEY_SECP256K1)); + assertEquals(expectedPublicKey, nodeRecordRestored.get(EnrFieldV4.PKEY_SECP256K1)); assertEquals(expectedSignature, nodeRecordRestored.getSignature()); } @@ -89,12 +83,12 @@ public void testSignature() throws Exception { NodeRecordFactory.DEFAULT.createFromValues( UInt64.ZERO, Bytes96.ZERO, - Pair.with(FIELD_ID, EnrScheme.V4), - Pair.with(FIELD_IP_V4, localIp), - Pair.with(FIELD_TCP_V4, 30303), - Pair.with(FIELD_UDP_V4, 30303), + Pair.with(EnrField.ID, IdentitySchema.V4), + Pair.with(EnrField.IP_V4, localIp), + Pair.with(EnrField.TCP_V4, 30303), + Pair.with(EnrField.UDP_V4, 30303), Pair.with( - FIELD_PKEY_SECP256K1, + EnrFieldV4.PKEY_SECP256K1, Functions.derivePublicKeyFromPrivate(BytesValue.wrap(privKey)))); nodeRecord0.sign(BytesValue.wrap(privKey)); nodeRecord0.verify(); @@ -102,12 +96,12 @@ public void testSignature() throws Exception { NodeRecordFactory.DEFAULT.createFromValues( UInt64.valueOf(1), Bytes96.ZERO, - Pair.with(FIELD_ID, EnrScheme.V4), - Pair.with(FIELD_IP_V4, localIp), - Pair.with(FIELD_TCP_V4, 30303), - Pair.with(FIELD_UDP_V4, 30303), + Pair.with(EnrField.ID, IdentitySchema.V4), + Pair.with(EnrField.IP_V4, localIp), + Pair.with(EnrField.TCP_V4, 30303), + Pair.with(EnrField.UDP_V4, 30303), Pair.with( - FIELD_PKEY_SECP256K1, + EnrFieldV4.PKEY_SECP256K1, Functions.derivePublicKeyFromPrivate(BytesValue.wrap(privKey)))); nodeRecord1.sign(BytesValue.wrap(privKey)); nodeRecord1.verify(); diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/TestUtil.java b/discovery/src/test/java/org/ethereum/beacon/discovery/TestUtil.java index b2b9581e0..b5722e9e1 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/TestUtil.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/TestUtil.java @@ -1,11 +1,13 @@ package org.ethereum.beacon.discovery; import org.ethereum.beacon.chain.storage.impl.SerializerFactory; -import org.ethereum.beacon.discovery.enr.EnrScheme; -import org.ethereum.beacon.discovery.enr.EnrSchemeV4Interpreter; +import org.ethereum.beacon.discovery.enr.EnrField; +import org.ethereum.beacon.discovery.enr.EnrFieldV4; +import org.ethereum.beacon.discovery.enr.IdentitySchema; +import org.ethereum.beacon.discovery.enr.IdentitySchemaV4Interpreter; import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.enr.NodeRecordFactory; -import org.ethereum.beacon.discovery.mock.EnrSchemeV4InterpreterMock; +import org.ethereum.beacon.discovery.mock.IdentitySchemaV4InterpreterMock; import org.ethereum.beacon.discovery.storage.NodeSerializerFactory; import org.javatuples.Pair; import tech.pegasys.artemis.util.bytes.Bytes4; @@ -18,13 +20,12 @@ import java.util.ArrayList; import java.util.Random; -import static org.ethereum.beacon.discovery.enr.NodeRecord.FIELD_ID; - public class TestUtil { public static final NodeRecordFactory NODE_RECORD_FACTORY = - new NodeRecordFactory(new EnrSchemeV4Interpreter()); + new NodeRecordFactory(new IdentitySchemaV4Interpreter()); public static final NodeRecordFactory NODE_RECORD_FACTORY_NO_VERIFICATION = - new NodeRecordFactory(new EnrSchemeV4InterpreterMock()); // doesn't verify ECDSA signature + new NodeRecordFactory( + new IdentitySchemaV4InterpreterMock()); // doesn't verify ECDSA signature public static final SerializerFactory TEST_SERIALIZER = new NodeSerializerFactory(NODE_RECORD_FACTORY_NO_VERIFICATION); public static final String LOCALHOST = "127.0.0.1"; @@ -71,12 +72,12 @@ public static Pair generateNode(int port, boolean verifi Bytes96.EMPTY, new ArrayList>() { { - add(Pair.with(FIELD_ID, EnrScheme.V4)); - add(Pair.with(NodeRecord.FIELD_IP_V4, finalLocalIp)); - add(Pair.with(NodeRecord.FIELD_UDP_V4, port)); + add(Pair.with(EnrField.ID, IdentitySchema.V4)); + add(Pair.with(EnrField.IP_V4, finalLocalIp)); + add(Pair.with(EnrField.UDP_V4, port)); add( Pair.with( - NodeRecord.FIELD_PKEY_SECP256K1, + EnrFieldV4.PKEY_SECP256K1, Functions.derivePublicKeyFromPrivate(BytesValue.wrap(privateKey)))); } }); diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/mock/EnrSchemeV4InterpreterMock.java b/discovery/src/test/java/org/ethereum/beacon/discovery/mock/IdentitySchemaV4InterpreterMock.java similarity index 70% rename from discovery/src/test/java/org/ethereum/beacon/discovery/mock/EnrSchemeV4InterpreterMock.java rename to discovery/src/test/java/org/ethereum/beacon/discovery/mock/IdentitySchemaV4InterpreterMock.java index 2e2193324..631ff419b 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/mock/EnrSchemeV4InterpreterMock.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/mock/IdentitySchemaV4InterpreterMock.java @@ -1,10 +1,10 @@ package org.ethereum.beacon.discovery.mock; -import org.ethereum.beacon.discovery.enr.EnrSchemeV4Interpreter; +import org.ethereum.beacon.discovery.enr.IdentitySchemaV4Interpreter; import org.ethereum.beacon.discovery.enr.NodeRecord; import tech.pegasys.artemis.util.bytes.Bytes96; -public class EnrSchemeV4InterpreterMock extends EnrSchemeV4Interpreter { +public class IdentitySchemaV4InterpreterMock extends IdentitySchemaV4Interpreter { @Override public void verify(NodeRecord nodeRecord) { // Don't verify signature diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeTableTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeTableTest.java index 2d252bde9..49065548b 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeTableTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/storage/NodeTableTest.java @@ -4,6 +4,7 @@ import org.ethereum.beacon.discovery.NodeRecordInfo; import org.ethereum.beacon.discovery.NodeStatus; import org.ethereum.beacon.discovery.TestUtil; +import org.ethereum.beacon.discovery.enr.EnrFieldV4; import org.ethereum.beacon.discovery.enr.NodeRecord; import org.junit.Test; import tech.pegasys.artemis.util.bytes.Bytes32; @@ -47,8 +48,8 @@ public void testCreate() throws Exception { assertTrue(extendedEnr.isPresent()); NodeRecordInfo nodeRecord2 = extendedEnr.get(); assertEquals( - nodeRecord.get(NodeRecord.FIELD_PKEY_SECP256K1), - nodeRecord2.getNode().get(NodeRecord.FIELD_PKEY_SECP256K1)); + nodeRecord.get(EnrFieldV4.PKEY_SECP256K1), + nodeRecord2.getNode().get(EnrFieldV4.PKEY_SECP256K1)); assertEquals( nodeTableStorage.get().getHomeNode().getNodeId(), HOME_NODE_SUPPLIER.apply(UInt64.ZERO).getNodeId()); @@ -79,8 +80,8 @@ public void testFind() throws Exception { .getNode(closestNode.getNodeId()) .get() .getNode() - .get(NodeRecord.FIELD_PKEY_SECP256K1), - closestNode.get(NodeRecord.FIELD_PKEY_SECP256K1)); + .get(EnrFieldV4.PKEY_SECP256K1), + closestNode.get(EnrFieldV4.PKEY_SECP256K1)); // node is adjusted to be far from localhostEnr NodeRecord farNode = TestUtil.generateUnverifiedNode(30304).getValue1(); nodeTableStorage.get().save(new NodeRecordInfo(farNode, -1L, NodeStatus.ACTIVE, 0)); @@ -89,14 +90,14 @@ public void testFind() throws Exception { assertEquals(2, closestNodes.size()); Set publicKeys = new HashSet<>(); closestNodes.forEach( - n -> publicKeys.add((BytesValue) n.getNode().get(NodeRecord.FIELD_PKEY_SECP256K1))); - assertTrue(publicKeys.contains(localHostNode.get(NodeRecord.FIELD_PKEY_SECP256K1))); - assertTrue(publicKeys.contains(closestNode.get(NodeRecord.FIELD_PKEY_SECP256K1))); + n -> publicKeys.add((BytesValue) n.getNode().get(EnrFieldV4.PKEY_SECP256K1))); + assertTrue(publicKeys.contains(localHostNode.get(EnrFieldV4.PKEY_SECP256K1))); + assertTrue(publicKeys.contains(closestNode.get(EnrFieldV4.PKEY_SECP256K1))); List farNodes = nodeTableStorage.get().findClosestNodes(farNode.getNodeId(), 1); assertEquals(1, farNodes.size()); assertEquals( - farNodes.get(0).getNode().get(NodeRecord.FIELD_PKEY_SECP256K1), - farNode.get(NodeRecord.FIELD_PKEY_SECP256K1)); + farNodes.get(0).getNode().get(EnrFieldV4.PKEY_SECP256K1), + farNode.get(EnrFieldV4.PKEY_SECP256K1)); } /** From 2c61b551933b6ebab15c81ea2267a94f69d05406 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Fri, 22 Nov 2019 16:15:10 +0300 Subject: [PATCH 71/77] discovery: better interface for NodeRecord encoding routines --- .../enr/IdentitySchemaV4Interpreter.java | 4 +-- .../beacon/discovery/enr/NodeRecord.java | 27 ++++++++++++++----- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/IdentitySchemaV4Interpreter.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/IdentitySchemaV4Interpreter.java index 812f20134..b26f5e5c1 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/IdentitySchemaV4Interpreter.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/IdentitySchemaV4Interpreter.java @@ -18,7 +18,7 @@ public void verify(NodeRecord nodeRecord) { } BytesValue pubKey = (BytesValue) nodeRecord.get(EnrFieldV4.PKEY_SECP256K1); // compressed assert Functions.verifyECDSASignature( - nodeRecord.getSignature(), Functions.hashKeccak(nodeRecord.serialize(false)), pubKey); + nodeRecord.getSignature(), Functions.hashKeccak(nodeRecord.serializeNoSignature()), pubKey); } @Override @@ -44,7 +44,7 @@ public Bytes32 getNodeId(NodeRecord nodeRecord) { public void sign(NodeRecord nodeRecord, Object signOptions) { BytesValue privateKey = (BytesValue) signOptions; BytesValue signature = - Functions.sign(privateKey, Functions.hashKeccak(nodeRecord.serialize(false))); + Functions.sign(privateKey, Functions.hashKeccak(nodeRecord.serializeNoSignature())); nodeRecord.setSignature(signature); } } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecord.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecord.java index 3d323cb6f..f079cff97 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecord.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecord.java @@ -25,6 +25,12 @@ *

Node record as described in EIP-778 */ public class NodeRecord { + /** + * The canonical encoding of a node record is an RLP list of [signature, seq, k, v, ...]. The + * maximum encoded size of a node record is 300 bytes. Implementations should reject records + * larger than this size. + */ + public static final int MAX_ENCODED_SIZE = 300; private static final EnrFieldInterpreter enrFieldInterpreter = EnrFieldInterpreterV4.DEFAULT; private final UInt64 seq; // Signature @@ -123,10 +129,14 @@ public void sign(Object signOptions) { } public RlpList asRlp() { - return asRlp(true); + return asRlpImpl(true); } - public RlpList asRlp(boolean withSignature) { + public RlpList asRlpNoSignature() { + return asRlpImpl(false); + } + + private RlpList asRlpImpl(boolean withSignature) { assert getSeq() != null; // content = [seq, k, v, ...] // signature = sign(content) @@ -149,12 +159,17 @@ public RlpList asRlp(boolean withSignature) { } public BytesValue serialize() { - return serialize(true); + return serializeImpl(true); + } + + public BytesValue serializeNoSignature() { + return serializeImpl(false); } - public BytesValue serialize(boolean withSignature) { - byte[] bytes = RlpEncoder.encode(asRlp(withSignature)); - assert bytes.length <= 300; + private BytesValue serializeImpl(boolean withSignature) { + RlpType rlpRecord = withSignature ? asRlp() : asRlpNoSignature(); + byte[] bytes = RlpEncoder.encode(rlpRecord); + assert bytes.length <= MAX_ENCODED_SIZE; return BytesValue.wrap(bytes); } From ccc23a12560cd366f06ca2efd63c93675514252b Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Fri, 22 Nov 2019 17:08:12 +0300 Subject: [PATCH 72/77] discovery: javadocs and final cleanup --- .../beacon/discovery/DiscoveryManager.java | 1 + .../discovery/DiscoveryMessageProcessor.java | 3 ++- .../discovery/DiscoveryV5MessageProcessor.java | 8 ++++++-- .../ethereum/beacon/discovery/Functions.java | 14 ++++++++++++++ .../beacon/discovery/MessageProcessor.java | 18 +++++++++++------- .../ethereum/beacon/discovery/NodeSession.java | 14 +++++++++++++- .../ethereum/beacon/discovery/NodeStatus.java | 7 ++++--- .../{IdentityScheme.java => Protocol.java} | 14 ++++++-------- .../org/ethereum/beacon/discovery/RlpUtil.java | 16 +++++++++++++++- .../beacon/discovery/enr/EnrField.java | 3 +++ .../discovery/enr/EnrFieldInterpreter.java | 3 +++ .../beacon/discovery/enr/EnrFieldV4.java | 4 ++++ .../enr/IdentitySchemaInterpreter.java | 9 +++++++++ .../discovery/message/DiscoveryMessage.java | 5 +++-- .../discovery/message/DiscoveryV5Message.java | 6 +++--- .../beacon/discovery/message/V5Message.java | 2 ++ .../beacon/discovery/packet/Packet.java | 4 ++++ .../beacon/discovery/pipeline/Envelope.java | 4 ++-- .../beacon/discovery/pipeline/Pipeline.java | 8 ++++++++ .../discovery/pipeline/info/RequestInfo.java | 5 +++++ .../storage/NodeSerializerFactory.java | 1 + .../storage/NodeTableStorageFactory.java | 2 +- .../beacon/discovery/task/TaskOptions.java | 1 + 23 files changed, 121 insertions(+), 31 deletions(-) rename discovery/src/main/java/org/ethereum/beacon/discovery/{IdentityScheme.java => Protocol.java} (50%) diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManager.java b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManager.java index c2a1d843f..3e0c96823 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManager.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryManager.java @@ -9,6 +9,7 @@ * href="https://github.com/ethereum/devp2p/blob/master/discv5/discv5.md">https://github.com/ethereum/devp2p/blob/master/discv5/discv5.md */ public interface DiscoveryManager { + void start(); void stop(); diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryMessageProcessor.java b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryMessageProcessor.java index 0d51ec9fa..c23238b6a 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryMessageProcessor.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryMessageProcessor.java @@ -2,8 +2,9 @@ import org.ethereum.beacon.discovery.message.DiscoveryMessage; +/** Handles discovery messages of several types */ public interface DiscoveryMessageProcessor { - IdentityScheme getSupportedIdentity(); + Protocol getSupportedIdentity(); void handleMessage(M message, NodeSession session); } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryV5MessageProcessor.java b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryV5MessageProcessor.java index 24b8e0bab..f4d8c0c83 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryV5MessageProcessor.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/DiscoveryV5MessageProcessor.java @@ -14,6 +14,10 @@ import java.util.HashMap; import java.util.Map; +/** + * {@link DiscoveryV5Message} v5 messages processor. Uses several handlers, one fo each type of v5 + * message to handle appropriate message. + */ public class DiscoveryV5MessageProcessor implements DiscoveryMessageProcessor { private static final Logger logger = LogManager.getLogger(DiscoveryV5MessageProcessor.class); private final Map messageHandlers = new HashMap<>(); @@ -28,8 +32,8 @@ public DiscoveryV5MessageProcessor(NodeRecordFactory nodeRecordFactory) { } @Override - public IdentityScheme getSupportedIdentity() { - return IdentityScheme.V5; + public Protocol getSupportedIdentity() { + return Protocol.V5; } @Override diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java b/discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java index 3b6c3120a..f60ee7c46 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/Functions.java @@ -28,6 +28,7 @@ import static org.ethereum.beacon.util.Utils.extractBytesFromUnsignedBigInt; import static org.web3j.crypto.Sign.CURVE_PARAMS; +/** Set of cryptography and utilities functions used in discovery */ public class Functions { public static final ECDomainParameters SECP256K1_CURVE = new ECDomainParameters( @@ -38,10 +39,12 @@ public class Functions { private static final int AUTH_RESP_KEY_LENGTH = 16; private static final int MS_IN_SECOND = 1000; + /** SHA2 (SHA256) */ public static Bytes32 hash(BytesValue value) { return Hashes.sha256(value); } + /** SHA3 (Keccak256) */ public static Bytes32 hashKeccak(BytesValue value) { return Bytes32.wrap(Hash.sha3(value.extractArray())); } @@ -112,6 +115,10 @@ public static BytesValue aesgcm_encrypt( } } + /** + * AES-GCM decryption of `encoded` data with the given `key`, `nonce` and additional authenticated + * data `ad`. Size of `key` is 16 bytes (AES-128), size of `nonce` 12 bytes. + */ public static BytesValue aesgcm_decrypt( BytesValue privateKey, BytesValue nonce, BytesValue encoded, BytesValue aad) { try { @@ -127,6 +134,7 @@ public static BytesValue aesgcm_decrypt( } } + /** Maps public key to point on {@link #SECP256K1_CURVE} */ public static ECPoint publicKeyToPoint(BytesValue pkey) { byte[] destPubPointBytes; if (pkey.size() == 64) { // uncompressed @@ -211,10 +219,12 @@ public static HKDFKeys hkdf_expand( } } + /** Current time in seconds */ public static long getTime() { return System.currentTimeMillis() / MS_IN_SECOND; } + /** Random provider */ public static Random getRandom() { return new SecureRandom(); } @@ -241,6 +251,10 @@ public static int logDistance(Bytes32 nodeId1, Bytes32 nodeId2) { return logDistance; } + /** + * Stores set of keys derived by simple key derivation function (KDF) based on a hash-based + * message authentication code (HMAC) + */ public static class HKDFKeys { private final BytesValue initiatorKey; private final BytesValue recipientKey; diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/MessageProcessor.java b/discovery/src/main/java/org/ethereum/beacon/discovery/MessageProcessor.java index 945ab7caa..16a7bd3f7 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/MessageProcessor.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/MessageProcessor.java @@ -7,24 +7,28 @@ import java.util.HashMap; import java.util.Map; +/** + * Highest level processor which knows several processors for different versions of {@link + * DiscoveryMessage}'s. + */ public class MessageProcessor { private static final Logger logger = LogManager.getLogger(MessageProcessor.class); - private final Map messageHandlers = new HashMap<>(); + private final Map messageProcessors = new HashMap<>(); - public MessageProcessor(DiscoveryMessageProcessor... messageHandlers) { - for (int i = 0; i < messageHandlers.length; ++i) { - this.messageHandlers.put(messageHandlers[i].getSupportedIdentity(), messageHandlers[i]); + public MessageProcessor(DiscoveryMessageProcessor... messageProcessors) { + for (int i = 0; i < messageProcessors.length; ++i) { + this.messageProcessors.put(messageProcessors[i].getSupportedIdentity(), messageProcessors[i]); } } public void handleIncoming(DiscoveryMessage message, NodeSession session) { - IdentityScheme identityScheme = message.getIdentityScheme(); - DiscoveryMessageProcessor messageHandler = messageHandlers.get(identityScheme); + Protocol protocol = message.getProtocol(); + DiscoveryMessageProcessor messageHandler = messageProcessors.get(protocol); if (messageHandler == null) { String error = String.format( "Message %s with identity %s received in session %s is not supported", - message, identityScheme, session); + message, protocol, session); logger.error(error); throw new RuntimeException(error); } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeSession.java b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeSession.java index a5c4cfe43..b1328e165 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeSession.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeSession.java @@ -29,7 +29,8 @@ import static org.ethereum.beacon.discovery.task.TaskStatus.AWAIT; /** - * Stores session status and all keys for discovery session between us (homeNode) and the other node + * Stores session status and all keys for discovery message exchange between us, `homeNode` and the + * other `node` */ public class NodeSession { public static final int NONCE_SIZE = 12; @@ -147,6 +148,7 @@ public void run() { return requestInfo; } + /** Updates request info. Thread-safe. */ public synchronized void updateRequestInfo(BytesValue requestId, RequestInfo newRequestInfo) { RequestInfo oldRequestInfo = requestIdStatuses.remove(requestId); if (oldRequestInfo == null) { @@ -187,16 +189,19 @@ public synchronized void cancelAllRequests(String message) { }); } + /** Generates random nonce of {@link #NONCE_SIZE} size */ public synchronized BytesValue generateNonce() { byte[] nonce = new byte[NONCE_SIZE]; rnd.nextBytes(nonce); return BytesValue.wrap(nonce); } + /** If true indicates that handshake is complete */ public synchronized boolean isAuthenticated() { return SessionStatus.AUTHENTICATED.equals(status); } + /** Resets stored authTags for this session making them obsolete */ public void cleanup() { authTagRepo.expire(this); } @@ -213,6 +218,7 @@ public Bytes32 getHomeNodeId() { return homeNodeId; } + /** @return initiator key, also known as write key */ public BytesValue getInitiatorKey() { return initiatorKey; } @@ -221,6 +227,7 @@ public void setInitiatorKey(BytesValue initiatorKey) { this.initiatorKey = initiatorKey; } + /** @return recipient key, also known as read key */ public BytesValue getRecipientKey() { return recipientKey; } @@ -235,6 +242,7 @@ public synchronized void clearRequestId(BytesValue requestId, TaskType taskType) assert taskType.equals(requestInfo.getTaskType()); } + /** Updates nodeRecord {@link NodeStatus} to ACTIVE of the node associated with this session */ public synchronized void updateLiveness() { NodeRecordInfo nodeRecordInfo = new NodeRecordInfo(getNodeRecord(), Functions.getTime(), NodeStatus.ACTIVE, 0); @@ -253,6 +261,10 @@ public synchronized Optional getRequestId(BytesValue requestId) { return requestId == null ? Optional.empty() : Optional.of(requestInfo); } + /** + * Returns any queued {@link RequestInfo} which was not started because session is not + * authenticated + */ public synchronized Optional getFirstAwaitRequestInfo() { return requestIdStatuses.values().stream() .filter(requestInfo -> AWAIT.equals(requestInfo.getTaskStatus())) diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeStatus.java b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeStatus.java index 553214c85..c96dd707d 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/NodeStatus.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/NodeStatus.java @@ -3,10 +3,11 @@ import java.util.HashMap; import java.util.Map; +/** Status of {@link org.ethereum.beacon.discovery.enr.NodeRecord} */ public enum NodeStatus { - ACTIVE(0x01), - SLEEP(0x02), - DEAD(0x03); + ACTIVE(0x01), // Alive + SLEEP(0x02), // Didn't answer last time(s) + DEAD(0x03); // Didnt' answer for a long time private static final Map codeMap = new HashMap<>(); diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/IdentityScheme.java b/discovery/src/main/java/org/ethereum/beacon/discovery/Protocol.java similarity index 50% rename from discovery/src/main/java/org/ethereum/beacon/discovery/IdentityScheme.java rename to discovery/src/main/java/org/ethereum/beacon/discovery/Protocol.java index b57b28588..beaf30f9a 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/IdentityScheme.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/Protocol.java @@ -3,28 +3,26 @@ import java.util.HashMap; import java.util.Map; -/** - * Discovery protocol identity schemes - */ -public enum IdentityScheme { +/** Discovery protocol versions */ +public enum Protocol { V4("v4"), V5("v5"); - private static final Map nameMap = new HashMap<>(); + private static final Map nameMap = new HashMap<>(); static { - for (IdentityScheme scheme : IdentityScheme.values()) { + for (Protocol scheme : Protocol.values()) { nameMap.put(scheme.name, scheme); } } private String name; - private IdentityScheme(String name) { + private Protocol(String name) { this.name = name; } - public static IdentityScheme fromString(String name) { + public static Protocol fromString(String name) { return nameMap.get(name); } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/RlpUtil.java b/discovery/src/main/java/org/ethereum/beacon/discovery/RlpUtil.java index 5314ebf46..b56190ac0 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/RlpUtil.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/RlpUtil.java @@ -15,8 +15,16 @@ import static org.web3j.rlp.RlpDecoder.OFFSET_LONG_LIST; import static org.web3j.rlp.RlpDecoder.OFFSET_SHORT_LIST; +/** + * Handy utilities used for RLP encoding and decoding and not fulfilled by {@link + * org.web3j.rlp.RlpEncoder} and {@link RlpDecoder} + */ public class RlpUtil { - public static int calcListLen(BytesValue data) { + /** + * Calculates length of list beginning from the start of the data. So, there could everything else + * after first list in data, method helps to cut data in this case. + */ + private static int calcListLen(BytesValue data) { int prefix = data.get(0) & 0xFF; int prefixAddon = 1; if (prefix >= OFFSET_SHORT_LIST && prefix <= OFFSET_LONG_LIST) { @@ -55,6 +63,12 @@ public static Pair decodeFirstList(BytesValue data) { return Pair.with(RlpDecoder.decode(data.slice(0, len).extractArray()), data.slice(len)); } + /** + * Encodes object to {@link RlpString}. Supports numbers, {@link BytesValue} etc. + * + * @throws RuntimeException with errorMessageFunction applied with `object` when encoding is not + * possible + */ public static RlpString encode(Object object, Function errorMessageFunction) { if (object instanceof BytesValue) { return fromBytesValue((BytesValue) object); diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrField.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrField.java index 63fbb9870..291539445 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrField.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrField.java @@ -1,5 +1,8 @@ package org.ethereum.beacon.discovery.enr; +/** + * Fields of Ethereum Node Record + */ public interface EnrField { // Schema id String ID = "id"; diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrFieldInterpreter.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrFieldInterpreter.java index 6a57eeef2..507e271a6 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrFieldInterpreter.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrFieldInterpreter.java @@ -3,6 +3,9 @@ import org.web3j.rlp.RlpString; import org.web3j.rlp.RlpType; +/** + * Encoder/decoder for fields of ethereum node record + */ public interface EnrFieldInterpreter { Object decode(String key, RlpString rlpString); diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrFieldV4.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrFieldV4.java index 52ff43882..b216975b7 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrFieldV4.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/EnrFieldV4.java @@ -1,5 +1,9 @@ package org.ethereum.beacon.discovery.enr; +/** + * Fields of Ethereum Node Record V4 as defined by https://eips.ethereum.org/EIPS/eip-778 + */ public interface EnrFieldV4 extends EnrField { // Compressed secp256k1 public key, 33 bytes String PKEY_SECP256K1 = "secp256k1"; diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/IdentitySchemaInterpreter.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/IdentitySchemaInterpreter.java index cb44d1458..f1a685b20 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/IdentitySchemaInterpreter.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/IdentitySchemaInterpreter.java @@ -2,6 +2,15 @@ import tech.pegasys.artemis.util.bytes.Bytes32; +/** + * Interprets identity schema of ethereum node record: + * + *

    + *
  • derives node id from node record + *
  • >signs node record + *
  • verifies signature of node record + *
+ */ public interface IdentitySchemaInterpreter { /** Returns supported scheme */ IdentitySchema getScheme(); diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/DiscoveryMessage.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/DiscoveryMessage.java index fd05f81f4..b68e278ed 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/message/DiscoveryMessage.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/DiscoveryMessage.java @@ -1,10 +1,11 @@ package org.ethereum.beacon.discovery.message; -import org.ethereum.beacon.discovery.IdentityScheme; +import org.ethereum.beacon.discovery.Protocol; import tech.pegasys.artemis.util.bytes.BytesValue; +/** Discovery message */ public interface DiscoveryMessage { - IdentityScheme getIdentityScheme(); + Protocol getProtocol(); BytesValue getBytes(); } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/DiscoveryV5Message.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/DiscoveryV5Message.java index 65e09d933..3889bc37c 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/message/DiscoveryV5Message.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/DiscoveryV5Message.java @@ -1,6 +1,6 @@ package org.ethereum.beacon.discovery.message; -import org.ethereum.beacon.discovery.IdentityScheme; +import org.ethereum.beacon.discovery.Protocol; import org.ethereum.beacon.discovery.enr.NodeRecordFactory; import org.web3j.rlp.RlpDecoder; import org.web3j.rlp.RlpList; @@ -28,8 +28,8 @@ public BytesValue getBytes() { } @Override - public IdentityScheme getIdentityScheme() { - return IdentityScheme.V5; + public Protocol getProtocol() { + return Protocol.V5; } public MessageCode getCode() { diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/message/V5Message.java b/discovery/src/main/java/org/ethereum/beacon/discovery/message/V5Message.java index e96b14808..9fc7af86d 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/message/V5Message.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/message/V5Message.java @@ -2,7 +2,9 @@ import tech.pegasys.artemis.util.bytes.BytesValue; +/** Message of V5 discovery protocol version */ public interface V5Message { BytesValue getRequestId(); + BytesValue getBytes(); } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/Packet.java b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/Packet.java index 7c6952ebe..f39fdd650 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/Packet.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/Packet.java @@ -5,6 +5,10 @@ import tech.pegasys.artemis.util.bytes.Bytes32s; import tech.pegasys.artemis.util.bytes.BytesValue; +/** + * Network packet as defined by discovery v5 specification. See https://github.com/ethereum/devp2p/blob/master/discv5/discv5-wire.md#packet-encoding + */ public interface Packet { static Bytes32 createTag(Bytes32 homeNodeId, Bytes32 destNodeId) { return Bytes32s.xor(Functions.hash(destNodeId), homeNodeId); diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/Envelope.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/Envelope.java index 7280605ab..c3d703cbf 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/Envelope.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/Envelope.java @@ -4,15 +4,15 @@ import java.util.Map; import java.util.UUID; +/** Container for any kind of objects used in packet-messages-tasks flow */ public class Envelope { private UUID id; + private Map data = new HashMap<>(); public Envelope() { this.id = UUID.randomUUID(); } - private Map data = new HashMap<>(); - public synchronized void put(Field key, Object value) { data.put(key, value); } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/Pipeline.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/Pipeline.java index da58903f3..aa485126e 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/Pipeline.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/Pipeline.java @@ -2,12 +2,20 @@ import org.reactivestreams.Publisher; +/** + * Pipeline uses several {@link EnvelopeHandler} handlers to pass objects through the chain of + * linked handlers implementing pipeline (or chain of responsibility) pattern. + */ public interface Pipeline { + /** Builds configured pipeline making it active */ Pipeline build(); + /** Pushes object inside pipeline */ void push(Object object); + /** Adds handler at the end of current chain */ Pipeline addHandler(EnvelopeHandler envelopeHandler); + /** Stream from the exit of built pipeline */ Publisher getOutgoingEnvelopes(); } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/info/RequestInfo.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/info/RequestInfo.java index 7b7cc2842..4b7a0da2d 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/info/RequestInfo.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/info/RequestInfo.java @@ -6,12 +6,17 @@ import java.util.concurrent.CompletableFuture; +/** Stores info related to performed request */ public interface RequestInfo { + /** Task type, in execution of which request was created */ TaskType getTaskType(); + /** Status of corresponding task */ TaskStatus getTaskStatus(); + /** Id of request */ BytesValue getRequestId(); + /** Future that should be fired when request is fulfilled or cancelled due to errors */ CompletableFuture getFuture(); } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeSerializerFactory.java b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeSerializerFactory.java index a3874811f..c3d69f76d 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeSerializerFactory.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeSerializerFactory.java @@ -9,6 +9,7 @@ import java.util.Map; import java.util.function.Function; +/** Serializer for {@link NodeRecordInfo}, {@link NodeIndex} and {@link NodeBucket} */ public class NodeSerializerFactory implements SerializerFactory { private final Map> deserializerMap = new HashMap<>(); private final Map> serializerMap = new HashMap<>(); diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactory.java b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactory.java index 523511e89..cac4dc2d8 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactory.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/storage/NodeTableStorageFactory.java @@ -9,6 +9,7 @@ import java.util.function.Function; import java.util.function.Supplier; +/** Creates {@link NodeTableStorage} */ public interface NodeTableStorageFactory { /** * Creates storage for nodes table @@ -19,7 +20,6 @@ public interface NodeTableStorageFactory { * sequence number is increased by 1 on each restart and ENR is signed with new sequence * number * @param bootNodesSupplier boot nodes provider - * * @return {@link NodeTableStorage} from `database` but if it doesn't exist, creates new one with * home node provided by `homeNodeSupplier` and boot nodes provided with `bootNodesSupplier`. * Uses `serializerFactory` for node records serialization. diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/task/TaskOptions.java b/discovery/src/main/java/org/ethereum/beacon/discovery/task/TaskOptions.java index 80d91d763..e5fa6539f 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/task/TaskOptions.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/task/TaskOptions.java @@ -1,5 +1,6 @@ package org.ethereum.beacon.discovery.task; +/** Specific options to clarify task features */ public class TaskOptions { private boolean livenessUpdate; private int distance; From dc52d3900606843eaf11677cca87c3cdcb2cf38a Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Fri, 22 Nov 2019 21:25:35 +0300 Subject: [PATCH 73/77] discovery: fix tests when one node is not ready on init --- .../org/ethereum/beacon/discovery/DiscoveryNetworkTest.java | 2 +- .../org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java index 72328e82b..8850fefc8 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNetworkTest.java @@ -126,8 +126,8 @@ public void test() throws Exception { }); // 4) fire 1 to 2 dialog - discoveryManager1.start(); discoveryManager2.start(); + discoveryManager1.start(); discoveryManager1.findNodes(nodeRecord2, 0); assert randomSent1to2.await(1, TimeUnit.SECONDS); diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java index d68ee0052..b5eae2ea1 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryNoNetworkTest.java @@ -140,8 +140,8 @@ public void test() throws Exception { }); // 4) fire 1 to 2 dialog - discoveryManager1.start(); discoveryManager2.start(); + discoveryManager1.start(); discoveryManager1.findNodes(nodeRecord2, 0); assert randomSent1to2.await(1, TimeUnit.SECONDS); From 51d0185aede8c60458643532ebbaf2e88b7262cb Mon Sep 17 00:00:00 2001 From: Shahan Khatchadourian Date: Sat, 23 Nov 2019 17:21:13 -0500 Subject: [PATCH 74/77] Refactor method name to distinguish by parameter name --- .../ethereum/beacon/discovery/packet/WhoAreYouPacket.java | 6 +++--- .../pipeline/handler/NotExpectedIncomingPacketHandler.java | 2 +- .../ethereum/beacon/discovery/HandshakeHandlersTest.java | 2 +- .../beacon/discovery/community/PacketEncodingTest.java | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/WhoAreYouPacket.java b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/WhoAreYouPacket.java index 1f1803cb4..ddc3e229b 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/WhoAreYouPacket.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/WhoAreYouPacket.java @@ -29,13 +29,13 @@ public WhoAreYouPacket(BytesValue bytes) { super(bytes); } - public static WhoAreYouPacket create( + public static WhoAreYouPacket createFromNodeId( Bytes32 destNodeId, BytesValue authTag, Bytes32 idNonce, UInt64 enrSeq) { BytesValue magic = getStartMagic(destNodeId); - return create(magic, authTag, idNonce, enrSeq); + return createFromMagic(magic, authTag, idNonce, enrSeq); } - public static WhoAreYouPacket create( + public static WhoAreYouPacket createFromMagic( BytesValue magic, BytesValue authTag, Bytes32 idNonce, UInt64 enrSeq) { byte[] rlpListEncoded = RlpEncoder.encode( diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NotExpectedIncomingPacketHandler.java b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NotExpectedIncomingPacketHandler.java index d1d9ce9e2..af006a8c9 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NotExpectedIncomingPacketHandler.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/pipeline/handler/NotExpectedIncomingPacketHandler.java @@ -60,7 +60,7 @@ public void handle(Envelope envelope) { Bytes32 idNonce = Bytes32.wrap(idNonceBytes); session.setIdNonce(idNonce); WhoAreYouPacket whoAreYouPacket = - WhoAreYouPacket.create( + WhoAreYouPacket.createFromNodeId( session.getNodeRecord().getNodeId(), authTag, idNonce, diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/HandshakeHandlersTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/HandshakeHandlersTest.java index 28aa81774..875db6848 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/HandshakeHandlersTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/HandshakeHandlersTest.java @@ -137,7 +137,7 @@ public void authHandlerWithMessageRoundTripTest() throws Exception { authTagRepository1.put(authTag, nodeSessionAt1For2); envelopeAt1From2.put( Field.PACKET_WHOAREYOU, - WhoAreYouPacket.create(nodePair1.getValue1().getNodeId(), authTag, idNonce, UInt64.ZERO)); + WhoAreYouPacket.createFromNodeId(nodePair1.getValue1().getNodeId(), authTag, idNonce, UInt64.ZERO)); envelopeAt1From2.put(Field.SESSION, nodeSessionAt1For2); CompletableFuture future = new CompletableFuture<>(); nodeSessionAt1For2.createNextRequest(TaskType.FINDNODE, new TaskOptions(true), future); diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/community/PacketEncodingTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/community/PacketEncodingTest.java index d16928298..590d22e5b 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/community/PacketEncodingTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/community/PacketEncodingTest.java @@ -30,7 +30,7 @@ public void encodeRandomPacketTest() { @Test public void encodeWhoAreYouTest() { WhoAreYouPacket whoAreYouPacket = - WhoAreYouPacket.create( + WhoAreYouPacket.createFromMagic( BytesValue.fromHexString( "0x0101010101010101010101010101010101010101010101010101010101010101"), BytesValue.fromHexString("0x020202020202020202020202"), From 665e10dc0f0cc4e37e09b04714bbab2e59138349 Mon Sep 17 00:00:00 2001 From: Shahan Khatchadourian Date: Sat, 23 Nov 2019 18:06:41 -0500 Subject: [PATCH 75/77] Doc-ed changed method name --- .../ethereum/beacon/discovery/packet/WhoAreYouPacket.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/WhoAreYouPacket.java b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/WhoAreYouPacket.java index ddc3e229b..d8ec3beef 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/WhoAreYouPacket.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/WhoAreYouPacket.java @@ -29,6 +29,14 @@ public WhoAreYouPacket(BytesValue bytes) { super(bytes); } + /** + * Create a packet by converting {@code destNodeId} to a magic value + * @param destNodeId + * @param authTag + * @param idNonce + * @param enrSeq + * @return + */ public static WhoAreYouPacket createFromNodeId( Bytes32 destNodeId, BytesValue authTag, Bytes32 idNonce, UInt64 enrSeq) { BytesValue magic = getStartMagic(destNodeId); From 9b9f283e7f77559e4ebb0a56c3b17a3368f51632 Mon Sep 17 00:00:00 2001 From: Shahan Khatchadourian Date: Sat, 23 Nov 2019 18:07:43 -0500 Subject: [PATCH 76/77] Doc-ed changed method name --- .../ethereum/beacon/discovery/packet/WhoAreYouPacket.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/WhoAreYouPacket.java b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/WhoAreYouPacket.java index d8ec3beef..b1fec77e4 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/packet/WhoAreYouPacket.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/packet/WhoAreYouPacket.java @@ -31,11 +31,6 @@ public WhoAreYouPacket(BytesValue bytes) { /** * Create a packet by converting {@code destNodeId} to a magic value - * @param destNodeId - * @param authTag - * @param idNonce - * @param enrSeq - * @return */ public static WhoAreYouPacket createFromNodeId( Bytes32 destNodeId, BytesValue authTag, Bytes32 idNonce, UInt64 enrSeq) { From 4eaa680a58db4975242c075fd908cf402c454ccb Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Tue, 26 Nov 2019 18:56:39 +0300 Subject: [PATCH 77/77] discovery: simplify NodeRecord creation from fields. Use field setup for Geth node in interop test. --- .../beacon/discovery/enr/NodeRecord.java | 11 ++++++++-- .../discovery/enr/NodeRecordFactory.java | 12 +++++----- .../discovery/DiscoveryInteropTest.java | 22 +++++++++++++++---- .../beacon/discovery/NodeRecordTest.java | 3 --- .../ethereum/beacon/discovery/TestUtil.java | 18 +++++++-------- 5 files changed, 41 insertions(+), 25 deletions(-) diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecord.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecord.java index f079cff97..7ef5c7170 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecord.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecord.java @@ -7,6 +7,7 @@ import org.web3j.rlp.RlpString; import org.web3j.rlp.RlpType; import tech.pegasys.artemis.util.bytes.Bytes32; +import tech.pegasys.artemis.util.bytes.Bytes96; import tech.pegasys.artemis.util.bytes.BytesValue; import tech.pegasys.artemis.util.uint.UInt64; @@ -46,12 +47,18 @@ private NodeRecord( this.identitySchemaInterpreter = identitySchemaInterpreter; } + private NodeRecord( + IdentitySchemaInterpreter identitySchemaInterpreter, UInt64 seq) { + this.seq = seq; + this.signature = Bytes96.ZERO; + this.identitySchemaInterpreter = identitySchemaInterpreter; + } + public static NodeRecord fromValues( IdentitySchemaInterpreter identitySchemaInterpreter, UInt64 seq, - BytesValue signature, List> fieldKeyPairs) { - NodeRecord nodeRecord = new NodeRecord(identitySchemaInterpreter, seq, signature); + NodeRecord nodeRecord = new NodeRecord(identitySchemaInterpreter, seq); fieldKeyPairs.forEach(objects -> nodeRecord.set(objects.getValue0(), objects.getValue1())); return nodeRecord; } diff --git a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordFactory.java b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordFactory.java index 534ff6ff0..22481955f 100644 --- a/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordFactory.java +++ b/discovery/src/main/java/org/ethereum/beacon/discovery/enr/NodeRecordFactory.java @@ -27,13 +27,11 @@ public NodeRecordFactory(IdentitySchemaInterpreter... identitySchemaInterpreters } @SafeVarargs - public final NodeRecord createFromValues( - UInt64 seq, BytesValue signature, Pair... fieldKeyPairs) { - return createFromValues(seq, signature, Arrays.asList(fieldKeyPairs)); + public final NodeRecord createFromValues(UInt64 seq, Pair... fieldKeyPairs) { + return createFromValues(seq, Arrays.asList(fieldKeyPairs)); } - public NodeRecord createFromValues( - UInt64 seq, BytesValue signature, List> fieldKeyPairs) { + public NodeRecord createFromValues(UInt64 seq, List> fieldKeyPairs) { Pair schemePair = null; for (Pair pair : fieldKeyPairs) { if (EnrField.ID.equals(pair.getValue0())) { @@ -42,7 +40,7 @@ public NodeRecord createFromValues( } } if (schemePair == null) { - throw new RuntimeException("ENR scheme is not defined in key-value pairs"); + throw new RuntimeException("ENR scheme (ID) is not defined in key-value pairs"); } IdentitySchemaInterpreter identitySchemaInterpreter = interpreters.get(schemePair.getValue1()); @@ -53,7 +51,7 @@ public NodeRecord createFromValues( schemePair.getValue1())); } - return NodeRecord.fromValues(identitySchemaInterpreter, seq, signature, fieldKeyPairs); + return NodeRecord.fromValues(identitySchemaInterpreter, seq, fieldKeyPairs); } public NodeRecord fromBase64(String enrBase64) { diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryInteropTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryInteropTest.java index 68a73988c..921b14950 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryInteropTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/DiscoveryInteropTest.java @@ -1,6 +1,9 @@ package org.ethereum.beacon.discovery; import org.ethereum.beacon.db.Database; +import org.ethereum.beacon.discovery.enr.EnrField; +import org.ethereum.beacon.discovery.enr.EnrFieldV4; +import org.ethereum.beacon.discovery.enr.IdentitySchema; import org.ethereum.beacon.discovery.enr.NodeRecord; import org.ethereum.beacon.discovery.packet.AuthHeaderMessagePacket; import org.ethereum.beacon.discovery.packet.RandomPacket; @@ -15,13 +18,16 @@ import org.junit.Test; import reactor.core.publisher.Flux; import tech.pegasys.artemis.util.bytes.BytesValue; +import tech.pegasys.artemis.util.uint.UInt64; import java.util.ArrayList; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import static org.ethereum.beacon.discovery.TestUtil.NODE_RECORD_FACTORY; import static org.ethereum.beacon.discovery.TestUtil.NODE_RECORD_FACTORY_NO_VERIFICATION; import static org.ethereum.beacon.discovery.TestUtil.TEST_SERIALIZER; +import static org.ethereum.beacon.discovery.TestUtil.parseStringIP; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -41,15 +47,25 @@ */ @Ignore("Requires manual startup, takes a bit to start") public class DiscoveryInteropTest { + private static final BytesValue gethNodePrivKey = + BytesValue.fromHexString("fb757dc581730490a1d7a00deea65e9b1936924caaea8f44d476014856b68736"); + @Test public void testInterop() throws Exception { // 1) start 2 nodes Pair nodePair1 = TestUtil.generateNode(40412, true); System.out.println(String.format("Node %s started", nodePair1.getValue1().getNodeId())); NodeRecord nodeRecord1 = nodePair1.getValue1(); + // Geth node NodeRecord nodeRecord2 = - NODE_RECORD_FACTORY_NO_VERIFICATION.fromBase64( - "-IS4QHa5-0-OmPRchyyBf9jHIWnQlZXthveUPp5_DoDnMMB0V9ChlzNq_fhFixvIr8xOQcKrYsWjjeIBoUIS8HSuWbgBgmlkgnY0gmlwhH8AAAGJc2VjcDI1NmsxoQMOLLdCQcDE_I6BZvGnmgXVsN2VgTp0sJRSnzF9XDnSNYN1ZHCCdl8"); // Geth node + NODE_RECORD_FACTORY.createFromValues( + UInt64.valueOf(1L), + Pair.with(EnrField.ID, IdentitySchema.V4), + Pair.with(EnrField.IP_V4, parseStringIP("127.0.0.1")), + Pair.with(EnrField.UDP_V4, 30303), + Pair.with( + EnrFieldV4.PKEY_SECP256K1, Functions.derivePublicKeyFromPrivate(gethNodePrivKey))); + nodeRecord2.sign(gethNodePrivKey); NodeTableStorageFactoryImpl nodeTableStorageFactory = new NodeTableStorageFactoryImpl(); Database database1 = Database.inMemoryDB(); NodeTableStorage nodeTableStorage1 = @@ -98,8 +114,6 @@ public void testInterop() throws Exception { } }); - // TODO: check that we receive correct nodes - // 4) fire 1 to 2 dialog discoveryManager1.start(); int distance = Functions.logDistance(nodeRecord1.getNodeId(), nodeRecord2.getNodeId()); diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/NodeRecordTest.java b/discovery/src/test/java/org/ethereum/beacon/discovery/NodeRecordTest.java index 334956b50..74d904ac2 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/NodeRecordTest.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/NodeRecordTest.java @@ -8,7 +8,6 @@ import org.javatuples.Pair; import org.junit.Test; import tech.pegasys.artemis.util.bytes.Bytes4; -import tech.pegasys.artemis.util.bytes.Bytes96; import tech.pegasys.artemis.util.bytes.BytesValue; import tech.pegasys.artemis.util.uint.UInt64; @@ -82,7 +81,6 @@ public void testSignature() throws Exception { NodeRecord nodeRecord0 = NodeRecordFactory.DEFAULT.createFromValues( UInt64.ZERO, - Bytes96.ZERO, Pair.with(EnrField.ID, IdentitySchema.V4), Pair.with(EnrField.IP_V4, localIp), Pair.with(EnrField.TCP_V4, 30303), @@ -95,7 +93,6 @@ public void testSignature() throws Exception { NodeRecord nodeRecord1 = NodeRecordFactory.DEFAULT.createFromValues( UInt64.valueOf(1), - Bytes96.ZERO, Pair.with(EnrField.ID, IdentitySchema.V4), Pair.with(EnrField.IP_V4, localIp), Pair.with(EnrField.TCP_V4, 30303), diff --git a/discovery/src/test/java/org/ethereum/beacon/discovery/TestUtil.java b/discovery/src/test/java/org/ethereum/beacon/discovery/TestUtil.java index b5722e9e1..48071b4e8 100644 --- a/discovery/src/test/java/org/ethereum/beacon/discovery/TestUtil.java +++ b/discovery/src/test/java/org/ethereum/beacon/discovery/TestUtil.java @@ -11,7 +11,6 @@ import org.ethereum.beacon.discovery.storage.NodeSerializerFactory; import org.javatuples.Pair; import tech.pegasys.artemis.util.bytes.Bytes4; -import tech.pegasys.artemis.util.bytes.Bytes96; import tech.pegasys.artemis.util.bytes.BytesValue; import tech.pegasys.artemis.util.uint.UInt64; @@ -41,6 +40,14 @@ public static Pair generateUnverifiedNode(int port) { return generateNode(port, false); } + /** Parses string representation of IPv4. Say, `127.0.0.1` */ + public static Bytes4 parseStringIP(String ip) { + try { + return Bytes4.wrap(InetAddress.getByName(ip).getAddress()); + } catch (UnknownHostException e) { + throw new RuntimeException(e); + } + } /** * Generates node on 127.0.0.1 with provided port. Node key is random, but always the same for the * same port. @@ -51,13 +58,7 @@ public static Pair generateUnverifiedNode(int port) { */ public static Pair generateNode(int port, boolean verification) { final Random rnd = new Random(SEED); - Bytes4 localIp = null; - try { - localIp = Bytes4.wrap(InetAddress.getByName(LOCALHOST).getAddress()); - } catch (UnknownHostException e) { - throw new RuntimeException(e); - } - final Bytes4 finalLocalIp = localIp; + final Bytes4 finalLocalIp = parseStringIP(LOCALHOST); for (int i = 0; i < port; ++i) { rnd.nextBoolean(); // skip according to input } @@ -69,7 +70,6 @@ public static Pair generateNode(int port, boolean verifi NodeRecord nodeRecord = nodeRecordFactory.createFromValues( UInt64.valueOf(1), - Bytes96.EMPTY, new ArrayList>() { { add(Pair.with(EnrField.ID, IdentitySchema.V4));